BERT_Step2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cdfb93b-5637-421a-9383-4f0c8dbee043",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import collections\n",
    "import tqdm\n",
    "import itertools\n",
    "import more_itertools\n",
    "import tabulate\n",
    "from tqdm.contrib.concurrent import process_map, thread_map\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import Counter\n",
    "from collections import Counter\n",
    "from nltk.util import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05b7a978-0741-430f-9765-3a0948a89b0b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"/global/u2/m/masare/ticket-text-analysis/bert_ticket_cleaned.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "072d88e5-8574-4158-a7a0-9aff490d9eff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0f03a86-6f70-4291-91a2-d47b049af637",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data[data.ID==197129].text_data.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38f7bdb5-5c45-432e-9a70-b53b9f4c4f2d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "patterns = {\n",
    "    \"python\": r\"(?:python|python3|\\S+\\.py)\", # python, analyze separately\n",
    "    \"GPD\": r\"(?:kaon_.part_smearing_opt|pion(?:\\S+)?|b_minus_Dx)\\.exe\", # m4240 Lattice QCD code\n",
    "    \"EXCESS\": r\"exess\", # Dr. Giuseppe Barca from The Australian National University (ANU) to port his existing EXtreme-scale Electronic Structure System (EXESS\n",
    "    \"WEST\": r\"wfreq\\.x\", \n",
    "    \"vasp\": r\"vasp\",\n",
    "    \"CP2k\": R\"cp2k(?:\\S+)?\", \n",
    "    \"QE\": r\"(?:pw|pp|pw2bgw|epw|neb|matdyn|dynmat|dos|pw2wannier90|q2r|ph|turbo_magnon|kgrid|bands|projwfc)\\.x\", # TODO: add all the other QE binaries\n",
    "    \"LAMMPS\": r\"lmp(?:-\\S+|_\\S+)?\",\n",
    "    \"NWCHEM\": r\"nwchem(?:\\S+)?\",\n",
    "    \"JDFTx\": r\"jdftx(?:_\\S+)?\",\n",
    "    \"IOR\": r\"(?:\\/|\\b|_)ior(?:\\b|_)\",\n",
    "    \"OpenMPI\": r\"(?:prted|orted)\",\n",
    "    \"chroma\": r\"(?:cf_)?chroma(?:\\S+)?\",\n",
    "    \"Wannier90\": r\"(?:postw90|wannier90)\\.x\",\n",
    "    \"E3SM\": r\"e3sm\\.exe\",\n",
    "    \"MILC\": r\"(?:ks_spectrum_hisq|su3_rhmc_hisq|gradientFlow|wilson_flow)(?:\\S+)?\", # gradientFlow, ks_spectrum_hisq\n",
    "    \"QLUA\": r\"qlua\", # https://usqcd-software.github.io/QLUA.html\n",
    "    \"Gromacs\": r\"gmx\", \n",
    "    \"BerkeleyGW\": r\"(?:hdf2wfn|kernel|absorption|epsilon|summarize_eigenvectors|siesta2bgw|sigma|eps0sym|epsinvomega|nonlinearoptics|inteqp|parabands|plotxct|siesta2bgw|wrapper_epsilon)\\.(?:cplx|real)?\\.x(?:\\S+)?\",\n",
    "    \"QMCPACK\": r\"qmcpack\",\n",
    "    \"OmniFold\": r\"omnifold\", # https://github.com/hep-lbdl/OmniFold\n",
    "    \"ONTEP\" : r\"onetep\",\n",
    "    \"WRF\": r\"wrf(?:\\.|_)(?:\\S+)?(?:exe|py|io|wy2017)(?:\\S+)?\",\n",
    "    \"COAWST\": r\"coawstM\", # https://www.usgs.gov/centers/whcmsc/science/coawst-a-coupled-ocean-atmosphere-wave-sediment-transport-modeling-system\n",
    "    \"M3GNet\": r\"M3GNet\", # \"https://github.com/materialsvirtuallab/m3gnet\n",
    "    \"LALIBE\": r\"lalibe\", # https://github.com/callat-qcd/lalibe\n",
    "    \"nvidia-smi\": r\"nvidia-smi\",\n",
    "    \"Octo-Tiger\": r\"octotiger\", # https://octotiger.stellar-group.org\n",
    "    \"GWU-QCD\": r\"(?:overlap_curseq|overlap_inverter|2pt_cfun_calc_milc2kentucky|wilsoninverter)\", # https://arxiv.org/abs/2001.05706, m102\n",
    "    \"CPS\": r\"(?:staggered_\\S+|wilsoninverter(?:\\S+)CPS)\", # Columbia Physics System Lattice QCD, intersection with GWU?\n",
    "    \"XGC\": r\"xgc(?:20230618|_|-es|-eem|-s|-gpu|-no_gpu|a-cpp)(?:\\S+)?\", # XGC (incl. xgca) https://epsi.pppl.gov/computing/xgc-1\n",
    "    \"XGCm\": r\"XGCm\", # https://xgcm.readthedocs.io/en/latest/index.html\n",
    "    \"halld_recon\": r\"halld_recon\", # JLab GLUEx workflow engine swif https://halldweb.jlab.org/DocDB/0037/003793/002/NERSC_AY2018_Hall_D.pdf\n",
    "    \"OSIRIS\": r\"osiris\", # https://picksc.physics.ucla.edu/osiris.html\n",
    "    #\"GENE\": r\"\\sgene(?:|_\\S+)\", #pattern matched as GENE instead of generates #added b and \\b for the white space but ended up getting errors\n",
    "    \"shifter\": r\"shifter\", # containers (shifter)\n",
    "    \"podman\": r\"podman-hpc\", # containers (podman-hpc)\n",
    "    \"flux\": r\"flux\", # flux workflow manager\n",
    "    \"Athena++\": r\"athena\", # maybe nocr_athhena doesn't belong? # https://www.athena-astro.app\n",
    "    \"WarpX\": r\"warpx\",\n",
    "    \"HPL\": r\"xhpl\",\n",
    "    \"AToM-2 SciDAC\": r\"(?:xolotl|run_parallel\\.exe)\", # https://github.com/ORNL-Fusion/xolotl, run_parallel also used in m3195, \n",
    "    \"EPOCH\": r\"epoch3d\", # (Extendable PIC Open Collaboration)\n",
    "    \"CGYRO\": r\"cgyro\", # https://gafusion.github.io/doc/cgyro.html, OpenACC Fortran\n",
    "    \"DFT-FE\": r\"dftfe\", # https://sites.google.com/umich.edu/dftfe\n",
    "    \"M3D-C1\": r\"m3dc1(?:\\S+)?\", # https://m3dc1.pppl.gov\n",
    "    \"Tristan\": r\"tristan\", # https://princetonuniversity.github.io/tristan-v2/\n",
    "    \"Fornax\": r\"fornax\", # https://www.astro.princeton.edu/~burrows/scidac4/fornax.html\n",
    "    \"ChronusQ\": r\"chronusq\", # https://arxiv.org/abs/1905.01381, https://github.com/xsligroup/chronusq_public MPI C++ OpenMP (gpu?)\n",
    "    \"VORPAL\": r\"vorpal\",  # 10.1016/j.jcp.2003.11.004 C++\n",
    "    \"ACE3P\": r\"track3p\", # https://abpcomputing.web.cern.ch/codes/codes_pages/ACE3P/\n",
    "    \"QDP++\": r\"(?:qdpxx|DStill)\", # /global/cfs/cdirs/m2986/software/perlmutter/code-qdpxx-6-2-23-gpu/install/wit/bin/DStill\"\n",
    "    \"FLASH\": r\"flash\", # https://flash.rochester.edu/site/flashcode/\n",
    "    \"MCFM\": r\"mcfm\", # \"https://mcfm.fnal.gov\"\n",
    "    \"TaskFarmer\": r\"tf_worker\", # TaskFarmer workflow manager\n",
    "    \"VLASIATOR\": r\"vlasiator\", # https://www.helsinki.fi/en/researchgroups/vlasiator\n",
    "    \"PROTEUS\": r\"proteus\", # TODO: but which one? https://github.com/nasa/PROTEUS or https://github.com/erdc/proteus or ???\n",
    "    \"SIESTA\": r\"siesta\", # mat sci code\n",
    "    \"SW4\": r\"sw4\", # Seismic\n",
    "    \"Gkeyll\": r\"gkyl\", #  https://github.com/ammarhakim/gkyl C++ CUDA NCCL\n",
    "    \"Trillinos\": r\"Tpetra\",\n",
    "    \"Petra-M\": r\"petram(?:s|p)\", # /global/common/software/petram/20230724/src/PetraM_Driver/README.md\n",
    "    \"HACC\": r\"hacc\",\n",
    "    \"CESM\": r\"cesm\",\n",
    "    \"Maestro\": r\"Maestro\",\n",
    "    \"ATS\": r\"(?:\\/|\\b)ats\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"NIMROD\": r\"nimrod\",\n",
    "    \"QBOX\": r\"qbox\",\n",
    "    \"ChomboCrunch\": r\"crunchDriver\",\n",
    "    \"FHI-aims\": r\"aims\\.x\",\n",
    "    \"phoenix\": r\"phoenix\",\n",
    "    \"MFDn\": r\"xmfdn\",  # xmfdn-vnn-lan , vnn = 2d body, lan = lanczos (lobpcg), xmfdn-(h2|vnn)-(lan|lobpcg)\n",
    "    \"Julia\": r\"(?:julia|\\S+\\.jl)\",\n",
    "    \"SOLPS\": r\"b2mn\\.exe\",\n",
    "    \"MetaHipMer\": r\"mhm2\",\n",
    "    \"CQL3d\": r\"xcql3d\",\n",
    "    \"NCCL Benchmark\": r\"all_reduce_perf\",\n",
    "    \"Fireworks\": r\"(?:continuous|repacking)_rlaunch(?:\\S+)?\\.sh\", # Fireworks workflow tool, Materials Genome among others, repacking_rlaunch_perlmutter.sh\n",
    "    \"DFTB+\": r\"dftb\\+\", \n",
    "    \"Hadrons\": r\"HadronsXmlRun(?:\\S+)?\", # https://aportelli.github.io/Hadrons-doc/#/ Grid based Lattice QCD workflows\n",
    "    \"AMReX\": r\"main3d\\.\", # name.compiler.arch.type.opt.gpu.ex ?\n",
    "    \"stochasticGW\": r\"sgw_perl\\.x\",\n",
    "    \"PeleC\": r\"PeleC\",\n",
    "    \"Octopus\": r\"octopus\",\n",
    "    \"GTC\": r\"gtc\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"GTS\": r\"gts\",  # TODO: review this pattern... as high potential for false positives\n",
    "    \"NVIDIA Tools\": r\"(?:nsys|ncu|nvprof)\", # Nsight systems, compute\n",
    "    \"CUDA-gdb\": r\"cuda-gdb\",\n",
    "    \"GAMESS\": r\"gamess\",\n",
    "    \"DASK\": \"dask-worker\", # fold into \"python\" ?\n",
    "    \"OSU Benchmarks\": r\"osu_\\S+\",\n",
    "    \"PARATEC\": r\"paratec\",\n",
    "    \"valgrind\": r\"valgrind\",\n",
    "    \"Berkeley-CS267\": r\"kmer_hash_(?:19|51)\", #https://github.com/Berkeley-CS267/hw3kmer_hash_51\n",
    "    \"gdb\": r\"gdb\",\n",
    "    \"Check MPI\": r\"check-(?:mpi|hybrid)\\.(?:\\S+)\\.pm\", # NERSC provided\n",
    "    \"ARM Forge\": r\"forge-backend\", # ARM Forge debugger\n",
    "    \"MATLAB\": r\"matlab\",\n",
    "    \"ABINIT\": r\"abinit\",\n",
    "    \"YAMBO\": r\"yambo\",\n",
    "    \"PolyFTS\": r\"PolyFTS\",\n",
    "    \"STREAM\": r\"stream\",\n",
    "    \"WEST\": r\"w(?:stat|freq|bse)\\.x\",\n",
    "    \"NCO\": r\"\\snc(?:remap|ks|diff|bo|es|rcat|add|substract|multiply|divide|pack|unpack)\", # https://nco.sourceforge.net/nco.html, look into this pattern well\n",
    "    \"SuperLU\": r\"superlu_dist\",\n",
    "    \"GEMM\": r\"gemm\", # DGEMM benchmark\n",
    "    \"ALICE\": r\"(?:process_LHC|process_embed|process_mc|alicepro|\\/alice\\/)\", # ALICE experiment data processing process_LHC18f5b.sh\n",
    "    \"MOLPRO\": r\"molpro\",\n",
    "    #\"DESI\": r\"desi\",\n",
    "    \"WRF\": r\"wrf\",\n",
    "    \"E3SM\": r\"E3SM\",\n",
    "    \"Alphafold\": \"af2c\", # /global/u1/m/mugao/projects/af2complex/git/src\n",
    "    \"Fungal/Algal Genome annotation\": r\"fnglanot\",\n",
    "    \"TOAST\": r\"toast\",\n",
    "    \"GX\": r\"gx\", # nmandell, https://gx.rtfd.io\n",
    "    \"KBASE\": r\"kbase\",\n",
    "    \"QTools\": r\"qtools\",\n",
    "    \"PyZFS\": r\"pyzfs\", #     https://github.com/hema-ted/pyzfs\n",
    "    \"FHI-aims\": r\"fhi(?:\\S+)aims\",\n",
    "    \"Chombo\": r\"(?:\\S+)\\.Linux\\.64\\.(?:g\\+\\+|CC)\\.ftn\\.(?:\\S+)\\.ex\", # e.g. driver2d.Linux.64.g++.ftn.OPT.MPI.PETSC.ex\n",
    "    \"PFLOTRAN\": r\"pflotran\",\n",
    "    \"NRQCD\": r\"NRQCD\", # /global/cfs/cdirs/m3760/Wei-Ping/measure/NRQCD\n",
    "}\n",
    "\n",
    "compiled_patterns = {\n",
    "    name: re.compile(pattern, re.IGNORECASE)\n",
    "    for name, pattern in patterns.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1279c746-d3ef-4099-ae7d-8d91ffc9ae58",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def match(string, patterns):\n",
    "    applications = []\n",
    "    for name, pattern in patterns.items():\n",
    "        #print(name)\n",
    "        #print(applications)\n",
    "        if pattern.search(string):\n",
    "            #print(\"found\", name)\n",
    "            applications.append(name) \n",
    "    return applications\n",
    "\n",
    "def f(x):\n",
    "    return match(x, compiled_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1be2380f-dc63-4b2e-b7e4-084484ca7d09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "names = process_map(f, data['text_data'], max_workers=32, chunksize=1000) #play with the numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fef6f117-2f34-440f-9c67-6cb0e62a0f56",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.insert(2,\"Applications\",names,True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "756fae7d-1881-4043-9fff-af924ae2f6cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data['text_data'][20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e837185a-a5fb-438f-9abe-f1641b5ca6ac",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "match(data['text_data'][0], {\"k\": re.compile('Lisa', re.IGNORECASE), \"p\": re.compile('thanks', re.IGNORECASE)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2621ed10-57fe-4efb-8931-6b339c6637d0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#counting occurrences of each match\n",
    "match_counts = data['Applications'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeb99c4e-2c56-4d98-900a-ec1ac70c0952",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print ('\\nCounts of each applications')\n",
    "print(match_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cb26350-ca37-4aaf-8286-5d350f9bae0b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#most common match\n",
    "most_common_match = match_counts.idxmax()\n",
    "most_common_match_count = match_counts.max()\n",
    "print(f\"\\nMost common match: {most_common_match} with {most_common_match_count} occurrences\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7caf29ca-5c5d-46c0-8523-1ceeb03e96e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#filter out empty matches\n",
    "filtered_data = data[data['Applications'] != '']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdc2d926-f5a2-43b3-ac03-39cc9b96d50b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count the occurrences of each match\n",
    "match_counts = filtered_data['Applications'].value_counts()\n",
    "\n",
    "# Display the counts of each match\n",
    "print(\"\\nCounts of each match:\")\n",
    "print(match_counts)\n",
    "\n",
    "# Find the most common match and its count\n",
    "most_common_match = match_counts.idxmax()\n",
    "most_common_match_count = match_counts.max()\n",
    "\n",
    "print(f\"\\nMost common match: {most_common_match} with {most_common_match_count} occurrences\")\n",
    "\n",
    "#select the top N most common matches\n",
    "N = 2\n",
    "top_n_matches = match_counts.head(N)\n",
    "\n",
    "print(f\"\\nTop {N} most common matches:\")\n",
    "print(top_n_matches)\n",
    "\n",
    "# Calculate the total number of occurrences for the top N matches\n",
    "total_top_n_matches = top_n_matches.sum()\n",
    "\n",
    "#print(f\"\\nTotal occurrences of the top {N} matches: {total_top_n_matches}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ca8baed-360c-4c8a-b304-3083de31bb15",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "953517aa-9884-4bb3-8bf6-f4a21a7131aa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Word Cloud Visuals \n",
    "\n",
    "# Combining all the matches into a single string\n",
    "all_matches = ' '.join([' '.join(match) if isinstance(match, list) else match for match in filtered_data['Applications']])\n",
    "\n",
    "# Generating the word cloud\n",
    "wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_matches)\n",
    "\n",
    "# Displaying the word cloud\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud, interpolation='bilinear')\n",
    "plt.axis('off')\n",
    "plt.title('Word Count of applications')\n",
    "plt.show()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5acb649e-9ee9-4aaf-9aa0-582ff74c9a8c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bar Plot of Most Common Applications\n",
    "\n",
    "# Flatten the list of matches and count the occurrences of each match\n",
    "flattened_matches = [match for sublist in filtered_data['Applications'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "match_counts = Counter(flattened_matches)\n",
    "\n",
    "# Converting to a DataFrame for plotting\n",
    "match_counts_df = pd.DataFrame(match_counts.items(), columns=['Match', 'Frequency']).sort_values(by='Frequency', ascending=False)\n",
    "\n",
    "# Plot the top N most common words\n",
    "N = 30\n",
    "top_n_words = match_counts_df.head(N)\n",
    "\n",
    "plt.figure(figsize=(12, 7))\n",
    "sns.barplot(x='Frequency', y='Match', data=top_n_words, palette='viridis', hue='Match',legend=True, dodge=False)\n",
    "plt.xlabel('Frequency per ticket', fontsize=14)\n",
    "plt.ylabel('Applications', fontsize=15)\n",
    "plt.title(f'Top {N} Most Common Applications in User Submitted Tickets',fontsize=14)\n",
    "plt.xticks(fontsize=12, fontweight='bold')\n",
    "plt.yticks(fontsize=12, fontweight='bold')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd55654-3eca-4467-9d89-5fc9d00257ce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 2D PREP\n",
    "patterns2 = {\n",
    "    \"login\": r\"(?i)\\blogin\\b\",\n",
    "    \"license\": r\"(?i)\\blicense\\b\",\n",
    "    \"password\": r\"(?i)\\bpassword\\b\",\n",
    "    \"error\": r\"(?i)\\berror\\b\",\n",
    "    \"crash\": r\"(?i)\\bcrash\\b\",\n",
    "    \"configuration\": r\"(?i)\\bconfiguration\\b\",\n",
    "    \"backup\": r\"(?i)\\bbackup\\b\",\n",
    "    \"restore\": r\"(?i)\\brestore\\b\",\n",
    "    \"network outage\": r\"(?i)\\bnetwork\\boutage\\b\",\n",
    "    \"log-in\": r\"(?i)\\blog-in\\b\",\n",
    "    \"activation\": r\"activation\",\n",
    "    \"blocked\": r\"(?i)\\bblocked\\b\",\n",
    "    \"locked\": r\"(?i)\\blocked\\b\",\n",
    "    \"approval\": r\"(?i)\\bapproval\\b\",\n",
    "    \"mfa\": r\"(?i)\\bmfa\\b\",\n",
    "    \"quota\": r\"(?i)\\bquota\\b\",  \n",
    "    \"connect\": r\"(?i)\\bconnect\\b\",\n",
    "    \"compile\": r\"(?i)\\bcompile\\b\",\n",
    "    \"link\": r\"(?i)\\blink\\b\",\n",
    "    \"build\": r\"(?i)\\bbuild\\b\",\n",
    "    \"make\": r\"(?i)\\bmake\\b\",\n",
    "    \"timeout\": r\"(?i)\\btimeout\\b\",\n",
    "    \"freeze\": r\"(?i)\\bfreeze\\b\",\n",
    "    \"unresponsive\": r\"(?i)\\bunresponsive\\b\",\n",
    "    \"lag\": r\"(?i)\\blag\\b\",\n",
    "    \"broken\": r\"(?i)\\bbroken\\b\",\n",
    "    \"request\": r\"(?i)\\brequest\\b\",\n",
    "    \"module\": r\"(?i)\\bmodule\\b\",\n",
    "    \"slow\": r\"(?i)\\bslow\\b\",\n",
    "    \"profile\": r\"(?i)\\bprofile\\b\",\n",
    "    \"performance\": r\"(?i)\\bperformance\\b\",\n",
    "    \"libraries\": r\"(?i)\\blibraries\\b\",\n",
    "    \"mpi\": r\"(?i)\\bmpi\\b\",\n",
    "    \"job\": r\"(?i)\\bjob\\b\",\n",
    "    \"parameter\": r\"(?i)\\bparameter\\b\",\n",
    "    \"hang\": r\"(?i)\\bhang\\b\",\n",
    "    \"certificate\": r\"(?i)\\bcertificate\\b\",\n",
    "    \"automated\": r\"(?i)\\bautomated\\b\",\n",
    "    \"documentation\": r\"(?i)\\bdocumentation\\b\",\n",
    "    \"docs\": r\"(?i)\\bdocs\\b\",\n",
    "    \"stop\": r\"(?i)\\bstop\\b\",\n",
    "    \"wrong\": r\"(?i)\\bwrong\\b\",\n",
    "    \"session\": r\"(?i)\\bsession\\b\",\n",
    "    \"node\": r\"(?i)\\bnode\\b\",\n",
    "    \"hpe\": r\"(?i)\\bhpe\\b\",\n",
    "    \"wait\": r\"(?i)\\bwait\\b\",\n",
    "    \"policy\": r\"(?i)\\bpolicy\\b\",\n",
    "    \"queue\": r\"(?i)\\bqueue\\b\",\n",
    "    \"permissions\": r\"(?i)\\bpermissions\\b\",\n",
    "    \"mpich\": r\"(?i)\\bmpich\\b\",\n",
    "    \"makefile\": r\"(?i)\\bmakefile\\b\",\n",
    "    \"crontab\": r\"(?i)\\bcrontab\\b\",\n",
    "    \"scrontab\": r\"(?i)\\bscrontab\\b\",\n",
    "    \"disconnect\": r\"(?i)\\bdisconnect\\b\",\n",
    "    \"user\": r\"(?i)\\buser\\b\",\n",
    "    \"account\": r\"(?i)\\baccount\\b\",\n",
    "    \"collab\": r\"(?i)\\bcollab\\b\",\n",
    "    \"ssh\": r\"(?i)\\bssh\\b\",\n",
    "    \"closed\": r\"(?i)\\bclosed\\b\",\n",
    "    \"save\": r\"(?i)\\bsave\\b\",\n",
    "    \"processes\": r\"(?i)\\bprocesses\\b\",\n",
    "    \"sharing\": r\"(?i)\\bsharing\\b\",\n",
    "    \"CFS\": r\"(?i)\\bCFS\\b\",\n",
    "    \"scratch\": r\"(?i)\\bscratch\\b\",\n",
    "    \"executable\": r\"(?i)\\bexecutable\\b\",\n",
    "    \"IO\": r\"(?i)\\bIO\\b\",\n",
    "    \"CPU\": r\"(?i)\\bCPU\\b\",\n",
    "    \"GPU\": r\"(?i)\\bGPU\\b\",\n",
    "    \"intel\": r\"(?i)\\bintel\\b\",\n",
    "    \"gnu\": r\"(?i)\\bgnu\\b\",\n",
    "    \"nvidia\": r\"(?i)\\bnvidia\\b\",\n",
    "    \"cray\": r\"(?i)\\bcray\\b\",\n",
    "    \"file\": r\"(?i)\\bfile\\b\",\n",
    "    \"qos\": r\"(?i)\\bqos\\b\",\n",
    "    \"OS\": r\"(?i)\\bOS\\b\",\n",
    "    \"shell\": r\"(?i)\\bshell\\b\",\n",
    "    \"bash\": r\"(?i)\\bbash\\b\",\n",
    "    \"csh\": r\"(?i)\\bcsh\\b\",\n",
    "    \"tsh\": r\"(?i)\\btsh\\b\",\n",
    "    \"ip\": r\"(?i)\\bip\\b\",\n",
    "    \"compute\": r\"(?i)\\bcompute\\b\",\n",
    "    \"workflow\": r\"(?i)\\bworkflow\\b\",\n",
    "    \"memory\": r\"(?i)\\bmemory\\b\",\n",
    "    \"utilization\": r\"(?i)\\butilization\\b\",\n",
    "    \"data\": r\"(?i)\\bdata\\b\",\n",
    "    \"application\": r\"(?i)\\bapplication\\b\",\n",
    "    \"framework\": r\"(?i)\\bframework\\b\",\n",
    "    \"realtime\": r\"(?i)\\brealtime\\b\",\n",
    "    \"allocation\": r\"(?i)\\ballocation\\b\",\n",
    "    \"reservation\": r\"(?i)\\breservation\\b\",\n",
    "    \"slurm\": r\"(?i)\\bslurm\\b\",\n",
    "    \"script\": r\"(?i)\\bscript\\b\",\n",
    "    \"group\": r\"(?i)\\bgroup\\b\",\n",
    "    \"membership\": r\"(?i)\\bmembership\\b\",\n",
    "    \"create\": r\"(?i)\\bcreate\\b\",\n",
    "    \"access\": r\"(?i)\\baccess\\b\",\n",
    "    \"issue\": r\"(?i)\\bissue\\b\",\n",
    "    \"source\": r\"(?i)\\bsource\\b\"\n",
    "\n",
    "    #\"alloacation_year\": r\"\\ballocation year\\b\"\n",
    "}\n",
    "\n",
    "compiled_patterns2 = {\n",
    "    name: re.compile(pattern, re.IGNORECASE)\n",
    "    for name, pattern in patterns2.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a46f1ca-bcc2-439a-a370-62f76cfabaf7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def match(string, patterns):\n",
    "    issues = []\n",
    "    for name, pattern in patterns.items():\n",
    "        #print(name)\n",
    "        #print(matches)\n",
    "        if pattern.search(string):\n",
    "            #print(\"found\", name)\n",
    "            issues.append(name) \n",
    "    return issues\n",
    "\n",
    "def f(x):\n",
    "    return match(x, compiled_patterns2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff322c99-0f06-483f-9c5c-8f33c394d6f1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "names2 = process_map(f, data['text_data'], max_workers=32, chunksize=1000) #play with the numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45b34076-b2a5-4ce2-b002-d3d53652f5ef",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb62612-131d-4cc9-85b9-92d15f552faf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.insert(2,\"Issues\",names2,True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65bc048a-3137-4dbf-9a13-5ded3b8e3d93",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Word Cloud Visuals \n",
    "\n",
    "# Combining all the matches into a single string\n",
    "all_matches = ' '.join([' '.join(match) if isinstance(match, list) else match for match in data['Issues']])\n",
    "\n",
    "# Generating the word cloud\n",
    "wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_matches)\n",
    "\n",
    "# Displaying the word cloud\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud, interpolation='bilinear')\n",
    "plt.axis('off')\n",
    "plt.title('Word Count of Issues')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0516392-6ce2-4f00-86cb-650b4d45afa0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bar Plot of Most Common Words\n",
    "\n",
    "# Flatten the list of matches and count the occurrences of each match\n",
    "flattened_matches = [match for sublist in data['Issues'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "match_counts = Counter(flattened_matches)\n",
    "\n",
    "# Converting to a DataFrame for plotting\n",
    "match_counts_df = pd.DataFrame(match_counts.items(), columns=['Match', 'Frequency']).sort_values(by='Frequency', ascending=False)\n",
    "\n",
    "# Plot the top N most common words\n",
    "N = 30\n",
    "top_n_words = match_counts_df.head(N)\n",
    "\n",
    "plt.figure(figsize=(12, 7))\n",
    "sns.barplot(x='Frequency', y='Match', data=top_n_words, palette='viridis', hue='Match',legend=True, dodge=False)\n",
    "plt.xlabel('Number of tickets containing the given term ',fontsize=14)\n",
    "plt.ylabel('Issues', fontsize=14)\n",
    "plt.title(f'Top {N} Most Common Issues in User Submitted Tickets', fontsize=14)\n",
    "plt.xticks(fontsize=12, fontweight='bold')\n",
    "plt.yticks(fontsize=12, fontweight='bold')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c422b24c-527b-4148-8d50-9781d8744d01",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "applications_keywords = [match for sublist in filtered_data['Applications'] for match in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "issues_keywords = flattened_matches = [match for sublist in data['Issues'] for match in (sublist if isinstance(sublist, list) else [sublist])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8e701e1-85dc-4f17-933b-0d89cb09893c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(issues_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66342b18-c722-475b-8f69-6f77e0f639f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ik = pd.Series(issues_keywords).value_counts()['closed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b360d27d-a559-4a9f-a080-cca5cbdb1ac6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ik"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b3c8b27-c3ba-4437-9771-600f0528377c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sacct",
   "language": "python",
   "name": "sacct"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}