2_pos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part-of-Speech (POS) Tagging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import statements\n",
    "\n",
    "from functions.basic import read_file\n",
    "from functions.processing import split_sents\n",
    "from functions.pos import get_pos_tags, count_tag_freqs, find_ngram_index, find_ngrams, ngram_count, get_n_ngrams, get_pos_diversity, list_individual_grams, grams_matrix_prep\n",
    "\n",
    "from functions.vis import write_vis, vis_subplots, calculate_weights, pos_heatmap\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#datasets\n",
    "\n",
    "j_all = read_file(\"data\\\\0_extraction_data\\\\j_all.pkl\")\n",
    "k_all = read_file(\"data\\\\0_extraction_data\\\\k_all.pkl\")\n",
    "\n",
    "#subsets\n",
    "j_cited = read_file(\"data\\\\1_text-stats_data\\\\j_cited_1.pkl\")\n",
    "j_not_cited = read_file(\"data\\\\1_text-stats_data\\\\j_not_cited_1.pkl\")\n",
    "k_cited = read_file(\"data\\\\1_text-stats_data\\\\k_cited_1.pkl\")\n",
    "k_not_cited = read_file(\"data\\\\1_text-stats_data\\\\k_not_cited_1.pkl\")\n",
    "\n",
    "#define indizes to merge dataframes later on\n",
    "j_cited_index = j_cited.index\n",
    "j_not_cited_index = j_not_cited.index\n",
    "k_cited_index = k_cited.index\n",
    "k_not_cited_index = k_not_cited.index\n",
    "\n",
    "#get overall length of all subsets\n",
    "j_c_length = j_cited.char_len.sum()\n",
    "k_c_length = k_cited.char_len.sum()\n",
    "j_nc_length = j_not_cited.char_len.sum()\n",
    "k_nc_length = k_not_cited.char_len.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text processing\n",
    "# this might take a while (should be less than 1 min.)\n",
    "\n",
    "#subsets \n",
    "\n",
    "#k cited\n",
    "k_c_text = [x for x in k_cited[\"text\"]]\n",
    "k_c_citnum = [x for x in k_cited[\"frequency\"]]\n",
    "k_c_sents = split_sents(k_c_text)\n",
    "\n",
    "#k not cited\n",
    "k_nc_text = [x for x in k_not_cited[\"text\"]]\n",
    "k_nc_citnum = [x for x in k_not_cited[\"frequency\"]]\n",
    "k_nc_sents = split_sents(k_nc_text)\n",
    "\n",
    "#j cited\n",
    "j_c_text = [x for x in j_cited[\"text\"]]\n",
    "j_c_citnum = [x for x in j_cited[\"frequency\"]]\n",
    "j_c_sents = split_sents(j_c_text)\n",
    "\n",
    "#j not cited\n",
    "j_nc_text = [x for x in j_not_cited[\"text\"]]\n",
    "j_nc_citnum = [x for x in j_not_cited[\"frequency\"]]\n",
    "j_nc_sents = split_sents(j_nc_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (Relative) Frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get pos tags (get_pos_tags) for each subset\n",
    "\n",
    "j_c_tagged_dict, j_c_tagged_list = get_pos_tags(j_c_sents)\n",
    "j_nc_tagged_dict, j_nc_tagged_list = get_pos_tags(j_nc_sents)\n",
    "k_c_tagged_dict, k_c_tagged_list = get_pos_tags(k_c_sents)\n",
    "k_nc_tagged_dict, k_nc_tagged_list = get_pos_tags(k_nc_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the relative tag frequencies (count_tag_freqs) for each subset\n",
    "j_c_tag_freqs, j_c_tags_used = count_tag_freqs(j_c_tagged_list)\n",
    "j_nc_tag_freqs, j_nc_tags_used = count_tag_freqs(j_nc_tagged_list)\n",
    "k_c_tag_freqs, k_c_tags_used = count_tag_freqs(k_c_tagged_list)\n",
    "k_nc_tag_freqs, k_nc_tags_used = count_tag_freqs(k_nc_tagged_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the relative tag frequencies (count_tag_freqs) and visualize them (heatmap)\n",
    "\n",
    "heatmap = pos_heatmap(k_nc_tag_freqs)\n",
    "#write_vis([\"pdf\", \"png\", \"html\"],\"k_nc_pos-heatmap_not-weighted\", heatmap)\n",
    "heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for weighted values, the function calculate_weights must be applied as well\n",
    "\n",
    "weights = calculate_weights(k_c_tag_freqs, k_c_citnum)\n",
    "\n",
    "# addition [26.02.]: the steps below could be simplified using the function apply_scaling() from summary.py\n",
    "\n",
    "#define min and max values for normalization\n",
    "min_norm = weights.min(axis = 1).sort_values(ascending=True)[0]\n",
    "max_norm = weights.max(axis = 1).sort_values(ascending=False)[0]\n",
    "\n",
    "heatmap = pos_heatmap((weights - min_norm) / (max_norm - min_norm))\n",
    "#write_vis([\"pdf\", \"png\", \"html\"],\"k_c_pos-heatmap_weighted\", heatmap)\n",
    "heatmap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example for counting individual ngram frequencies, e.g. 3-grams\n",
    "\n",
    "ngrams_found = find_ngrams(j_c_tagged_list, 3)\n",
    "ngram_counter = ngram_count(ngrams_found)\n",
    "ngram_counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get topn ngrams for a set number n of grams, e.g. get top 10 2- to 5-grams\n",
    "grams, gram_names = get_n_ngrams(n=list(range(6))[2:], topn=10, pos_tagged=j_c_tagged_list)\n",
    "\n",
    "# visualize the number of occurences\n",
    "subplots = vis_subplots(gram_names, grams, rowcount=2, colcount=2, showlabels = True, rel_yaxis = False)\n",
    "#write_vis([\"pdf\", \"png\",\"html\"],\"k_c_subplots_2-5-grams\", subplots)\n",
    "subplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find overlapping ngrams\n",
    "\n",
    "# first, define all ngram variables\n",
    "\n",
    "k_c_grams, k_c_gram_names = get_n_ngrams(n=list(range(6))[2:], topn=25, pos_tagged=k_c_tagged_list)\n",
    "k_nc_grams, k_nc_gram_names = get_n_ngrams(n=list(range(6))[2:], topn=25, pos_tagged=k_nc_tagged_list)\n",
    "j_c_grams, j_c_gram_names = get_n_ngrams(n=list(range(6))[2:], topn=25, pos_tagged=j_c_tagged_list)\n",
    "j_nc_grams, j_nc_gram_names = get_n_ngrams(n=list(range(6))[2:], topn=25, pos_tagged=j_nc_tagged_list)\n",
    "\n",
    "# then, create a list of all indivual ngrams that occur in any of our ngram lists\n",
    "all_grams = list_individual_grams([j_c_grams, j_nc_grams, k_c_grams, k_nc_grams])\n",
    "\n",
    "# use grams_matrix_prep to lookup the ngrams for each dataset in the all_grams list and return a \"binary\" or \"count\" values for their occurences\n",
    "\n",
    "j_c_grams_matrix = grams_matrix_prep(j_c_grams, all_grams, \"count\")\n",
    "j_nc_grams_matrix = grams_matrix_prep(j_nc_grams, all_grams, \"count\")\n",
    "k_c_grams_matrix = grams_matrix_prep(k_c_grams, all_grams, \"count\")\n",
    "k_nc_grams_matrix = grams_matrix_prep(k_nc_grams, all_grams, \"count\")\n",
    "\n",
    "matrix = [j_c_grams_matrix, j_nc_grams_matrix, k_c_grams_matrix, k_nc_grams_matrix]\n",
    "\n",
    "# create a dataframe for easier inspection, visualization\n",
    "df = pd.DataFrame.from_records(matrix)\n",
    "df = df.transpose()\n",
    "df.columns = [\"J(C)\", \"K(C)\", \"J(NC)\", \"K(NC)\"]\n",
    "df[\"sum\"] = df.sum(axis=1)\n",
    "df[\"ngram\"] = all_grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize the results\n",
    "# use option \"binary\" in grams_matrix_prep()\n",
    "\n",
    "fig = go.Figure()\n",
    "fig.add_trace(go.Heatmap(\n",
    "                    x=df.loc[:,\"J(C)\":\"K(NC)\"].columns, y=df.loc[:,\"J(C)\":\"K(NC)\"].index, z=df.loc[:,\"J(C)\":\"K(NC)\"], colorscale=[[0.0,\"#C16152\"], [0.5,\"#C16152\"], [0.5,\"#509F98\"], [1,\"#509F98\"]], colorbar=dict(tickvals=[0,1], ticktext=[0,1], lenmode=\"pixels\", len=100,)))\n",
    "fig.update_layout(template=\"plotly_dark\", font=dict(family = \"CMU Serif\", size=20),)\n",
    "fig.show()\n",
    "\n",
    "#write_vis([\"pdf\", \"png\",\"html\"],\"all_pos_2-5-grams_comparison\", fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# devide values by length of subsets to make them more comparable\n",
    "\n",
    "df = df.sort_values(by=\"sum\", ascending=False)\n",
    "\n",
    "def devide_byLength(df, cols, lengths):\n",
    "    for col, length in zip(cols, lengths):\n",
    "        df[col+\"_norm\"] = [round((val/length)*100, 2) for val in df[col]]\n",
    "    return df\n",
    "\n",
    "df = devide_byLength(df, [\"J(C)\", \"K(C)\", \"J(NC)\", \"K(NC)\"], [j_c_length, k_c_length, j_nc_length, k_nc_length])\n",
    "    \n",
    "#top 10 ngrams\n",
    "df[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out ngrams that only occur in one subset\n",
    "df.loc[(df[[\"K(C)\", \"J(NC)\", \"K(NC)\"]]==0).all(1)][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to find the index of passages that contain specific POS n-Grams, use find_ngram_index() \n",
    "\n",
    "idx_list = find_ngram_index(j_c_tagged_list, \"PUNCT, PRON, VERB, PRON\")\n",
    "print(idx_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diversity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate the diversity (Shannon entropy) of POS-Tags for each passage, using get_pos_diversity\n",
    "\n",
    "# Wertebereich liegt zwischen 0 und max_div\n",
    "#max_div = -numpy.log2(1/len(tag_freqs.index))\n",
    "\n",
    "j_c_div_scores = get_pos_diversity(j_c_tag_freqs)\n",
    "j_nc_div_scores = get_pos_diversity(j_nc_tag_freqs)\n",
    "k_c_div_scores = get_pos_diversity(k_c_tag_freqs)\n",
    "k_nc_div_scores = get_pos_diversity(k_nc_tag_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#append div_scores to previous dfs again\n",
    "\n",
    "# J cited\n",
    "j_c_div_scores.index = j_cited_index\n",
    "j_cited_2 = pd.concat([j_cited, j_c_div_scores], axis=1)\n",
    "\n",
    "# J not cited\n",
    "j_nc_div_scores.index = j_not_cited_index\n",
    "j_not_cited_2 = pd.concat([j_not_cited, j_nc_div_scores], axis=1)\n",
    "\n",
    "# K cited\n",
    "k_c_div_scores.index = k_cited_index\n",
    "k_cited_2 = pd.concat([k_cited, k_c_div_scores], axis=1)\n",
    "\n",
    "# K not cited\n",
    "k_nc_div_scores.index = k_not_cited_index\n",
    "k_not_cited_2 = pd.concat([k_not_cited, k_nc_div_scores], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save new dataframes again for the next step\n",
    "\n",
    "new_dfs = [j_cited_2, j_not_cited_2, k_cited_2, k_not_cited_2]\n",
    "paths = [\"data\\\\2_pos_data\\\\j_cited_2.pkl\", \"data\\\\2_pos_data\\\\j_not_cited_2.pkl\", \"data\\\\2_pos_data\\\\k_cited_2.pkl\", \"data\\\\2_pos_data\\\\k_not_cited_2.pkl\"]\n",
    "\n",
    "for df, path in zip(new_dfs, paths):\n",
    "    df.to_pickle(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scatter plot, to show pos_diversity by the passage position within a text\n",
    "\n",
    "fig = px.scatter(j_c_div_scores, x=j_c_div_scores[\"pos_diversity\"].index, y=\"pos_diversity\",color_discrete_sequence=[\"#509F98\"],opacity=.6)\n",
    "fig.update_layout(template=\"plotly_dark\", font=dict(family = \"CMU Serif\", size=20,), xaxis_title = \"position of passage (chronological order)\",)\n",
    "fig.show()\n",
    "\n",
    "#write_vis([\"pdf\", \"png\",\"html\"],\"j_c_pos_diversity\", fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# analyse relationship between pos_diversity and length of passage (token_count)\n",
    "\n",
    "# statsmodels needs to be installed to create trendline\n",
    "\n",
    "# for j_nc and k_nc subsets, drop 0 values to be able to use log_x=True as an option for the trendline below (like example below)\n",
    "# df = df.drop(df[df.diversity == 0].index)\n",
    "\n",
    "fig = px.scatter(j_cited_2, x=\"token_count\", y=\"pos_diversity\",color_discrete_sequence=[\"#509F98\"],opacity=.6, trendline=\"ols\",trendline_options=dict(log_x=True))\n",
    "\n",
    "#trendline_options=dict(log_x=True), log_x = True\n",
    "fig.update_layout(template=\"plotly_dark\", font=dict(family = \"CMU Serif\", size=20,), xaxis_title=\"token_count (log)\")\n",
    "fig.show()\n",
    "\n",
    "results = px.get_trendline_results(fig)\n",
    "print(results.px_fit_results.iloc[0].summary())\n",
    "\n",
    "#write_vis([\"pdf\", \"png\",\"html\"],\"j_c_diversity-tokencount_corr_logx\", fig)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7eef64f7e4c69d23018373627e30a15341c9b5e5118862fff2ef6eb02c852f48"
  },
  "kernelspec": {
   "display_name": "Python 3.7.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}