4_summary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import statements\n",
    "\n",
    "from functions.basic import read_file\n",
    "from functions.summary import apply_scaling\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import tree\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import sklearn.metrics as metrics\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#datasets\n",
    "\n",
    "j_all = read_file(\"data\\\\3_sentiment_data\\\\j_all_3.pkl\")\n",
    "k_all = read_file(\"data\\\\3_sentiment_data\\\\k_all_3.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Cleanup & Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop all columns that are no further needed\n",
    "\n",
    "j_all_prep = j_all.drop(columns=[\"cit_num\", \"sentiws\", \"germansentiment\"])\n",
    "j_all_prep = j_all_prep.rename(columns={\"rel_sentiws\": \"sentiws\", \"germansentiment_mapped\": \"germansentiment\"})\n",
    "\n",
    "k_all_prep = k_all.drop(columns=[\"cit_num\", \"sentiws\", \"germansentiment\"])\n",
    "k_all_prep = k_all_prep.rename(columns={\"rel_sentiws\": \"sentiws\", \"germansentiment_mapped\": \"germansentiment\"})\n",
    "\n",
    "#df cols that scaling should not be applied to:\n",
    "not_scaled = [\"startpos\", \"endpos\", \"text\", \"passage_type\",\"sentiws\", \"germansentiment\"]\n",
    "\n",
    "# J\n",
    "j_all_not_scaled_data = [j_all_prep[\"startpos\"], j_all_prep[\"endpos\"], j_all_prep[\"text\"], j_all_prep[\"passage_type\"], j_all_prep[\"sentiws\"], j_all_prep[\"germansentiment\"]]\n",
    "j_all_not_scaled = pd.concat(j_all_not_scaled_data, axis=1, keys=not_scaled)\n",
    "j_all_scaling = j_all_prep.drop(columns=not_scaled)\n",
    "\n",
    "# K\n",
    "k_all_not_scaled_data = [k_all_prep[\"startpos\"], k_all_prep[\"endpos\"], k_all_prep[\"text\"], k_all_prep[\"passage_type\"],k_all_prep[\"sentiws\"], k_all_prep[\"germansentiment\"]]\n",
    "k_all_not_scaled = pd.concat(k_all_not_scaled_data, axis=1, keys=not_scaled)\n",
    "k_all_scaling = k_all_prep.drop(columns=not_scaled)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scale all columns in []_all_scaling to [0,1]\n",
    "\n",
    "for df in [j_all_scaling, k_all_scaling]:\n",
    "    for col in df.columns:\n",
    "        df = apply_scaling(df, col, \"zero_pos\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine scaled and not scaled dataframes back together\n",
    "\n",
    "j_all_scaled = pd.concat([j_all_scaling, j_all_not_scaled],axis=1)\n",
    "k_all_scaled = pd.concat([k_all_scaling, k_all_not_scaled],axis=1)\n",
    "\n",
    "j_all_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data sets\n",
    "\n",
    "#j_all_scaled.to_json(\"j_all_4.json\", orient=\"index\")\n",
    "#k_all_scaled.to_json(\"k_all_4.json\", orient=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine J and K datasets\n",
    "\n",
    "all_prep = j_all_prep\n",
    "all_prep = pd.concat([all_prep, k_all_prep], ignore_index=True)\n",
    "\n",
    "all_pred = all_prep.drop(columns=[\"startpos\", \"endpos\", \"text\", \"passage_type\", \"token_count\",\"frequency\"])\n",
    "passage_type = all_prep.passage_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data\n",
    "\n",
    "X = all_pred.to_numpy()\n",
    "y = passage_type\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply decision tree\n",
    "\n",
    "clf = tree.DecisionTreeClassifier(max_depth = 3,random_state = 0)  # set max_depth to a small number like 3 to avoid overfitting\n",
    "clf.fit(X_train, y_train)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/visualizing-decision-trees-with-python-scikit-learn-graphviz-matplotlib-1c50b4aa68dc\n",
    "# visualize tree\n",
    "\n",
    "fn= all_pred.columns\n",
    "cn=[\"cited\", \"not_cited\"]\n",
    "\n",
    "fig, axes = plt.subplots(nrows = 1,ncols = 1,)\n",
    "\n",
    "vis = tree.plot_tree(clf,\n",
    "               feature_names = fn, \n",
    "               class_names=cn,\n",
    "               rounded=True, )\n",
    "\n",
    "#fig.savefig('tree.pdf', format='pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate results\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "cm = metrics.confusion_matrix(y_test, y_pred)\n",
    "#print(cm)\n",
    "print(metrics.classification_report(y_test, y_pred, labels=[\"cited\", \"not_cited\"]))#"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply kNN using parameter n_neighbors\n",
    "knn = KNeighborsClassifier(n_neighbors=12)\n",
    "\n",
    "# fit model and predict y for training data\n",
    "knn.fit(X_train, y_train)\n",
    "y_pred_knn = knn.predict(X_test)\n",
    "\n",
    "# evaluate results\n",
    "\n",
    "cm = metrics.confusion_matrix(y_test, y_pred_knn)\n",
    "#print(cm)\n",
    "print(metrics.classification_report(y_test, y_pred_knn, labels=[\"cited\", \"not_cited\"]))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7eef64f7e4c69d23018373627e30a15341c9b5e5118862fff2ef6eb02c852f48"
  },
  "kernelspec": {
   "display_name": "Python 3.7.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}