csg_datafusion_ecg_db_generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ECG database generator\n",
    "By Stephen Larroque @ Coma Science Group, GIGA Research, University of Liege\n",
    "Creation date: 2018-10-25\n",
    "License: MIT\n",
    "v0.4.0\n",
    "\n",
    "DESCRIPTION:\n",
    "Generate a CSV file listing all the subjects from a folder containing all the ECG/PPG files with the subject's name in the filename.\n",
    "\n",
    "INSTALL NOTE:\n",
    "You need to pip install pandas before launching this script.\n",
    "Tested on Python 2.7.13\n",
    "\n",
    "USAGE:\n",
    "\n",
    "TODO:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forcefully autoreload all python modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUX FUNCTIONS\n",
    "\n",
    "import os, sys\n",
    "\n",
    "cur_path = os.path.realpath('.')\n",
    "sys.path.append(os.path.join(cur_path, 'csg_fileutil_libs'))  # for unidecode and cleanup_name, because it does not support relative paths (yet?)\n",
    "\n",
    "import re\n",
    "\n",
    "from csg_fileutil_libs.aux_funcs import save_dict_as_csv, _tqdm, recwalk, cleanup_name, cleanup_name_customregex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PARAMETERS\n",
    "\n",
    "# Path where all the ECG/PPG files are\n",
    "inputpath = r'G:\\dropbox\\Dropbox\\Carol_Francesco'\n",
    "# Output database (csv file)\n",
    "out_db = r'ecg_subjects.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filetypes_to_include = ['ecg', 'ecg 2', 'puls', 'puls 2', 'resp', 'resp 2']\n",
    "total = sum(1 for _ in recwalk(inputpath, sorting=True, folders=False, topdown=True, filetype=filetypes_to_include))\n",
    "\n",
    "res = {}\n",
    "for dirpath, filename in _tqdm(recwalk(inputpath, sorting=True, folders=False, topdown=True, filetype=filetypes_to_include), total=total):\n",
    "    # Separate filename and extension\n",
    "    filenameonly,extension = os.path.splitext(filename)\n",
    "    # Cleanup name (remove weird characters, accentuated characters and metadata like numbers and sequence infos like 'repos', 'spatial', etc)\n",
    "    (name, matches) = cleanup_name_customregex(cleanup_name(filenameonly), customregex = {'_': ' ',\n",
    "                       '(repos|rest)': '',\n",
    "                       'ecg': '',\n",
    "                       '[0-9]+': '',\n",
    "                       'pulse?': '',\n",
    "                       'respi?': '',\n",
    "                       'spa(t|c)ial': '',\n",
    "                       'tennis': '',\n",
    "                       'musique': '',\n",
    "                       '(compassion|meditation|neutre|opacite)': '',\n",
    "                       'sedation': '',\n",
    "                      }, returnmatches=True)\n",
    "    name = name.strip()\n",
    "    # Name is empty (eg, filename is '.thumbnail'), skip\n",
    "    if not name:\n",
    "        continue\n",
    "    # Add the name if not already in the dict\n",
    "    if not name in res:\n",
    "        res[name] = {}\n",
    "        # Create a unique set in the dict (so that we can add data types)\n",
    "        res[name]['ecg_types'] = set()\n",
    "        res[name]['ecg_infos'] = set()\n",
    "    # Add the data type (eg, ecg, puls, resp, etc)\n",
    "    res[name]['ecg_types'].add(extension.replace('.', ''))\n",
    "    res[name]['ecg_infos'].update(matches)\n",
    "\n",
    "print('Subjects count: %i' % len(res))\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the dict to a list of dict (to create a column for the key=subject name)\n",
    "res2 = [{'name': name, 'ecg_types': list(value['ecg_types']), 'ecg_infos': list(value['ecg_infos'])} for name,value in res.items()]\n",
    "res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if save_dict_as_csv(res2, out_db, fields_order=['name', 'ecg_types', 'ecg_infos'], csv_order_by='name'):\n",
    "    print('File successfully saved in: %s' % out_db)\n",
    "else:\n",
    "    print('Error when trying to save the database!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}