even-channels-delta-theta-alpha-bands.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-22T14:27:53.176046Z","iopub.execute_input":"2021-11-22T14:27:53.176653Z","iopub.status.idle":"2021-11-22T14:27:53.472525Z","shell.execute_reply.started":"2021-11-22T14:27:53.176568Z","shell.execute_reply":"2021-11-22T14:27:53.471796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy.io\nimport numpy as np\nfrom scipy import signal\nfrom sklearn.decomposition import PCA\nfrom collections import Counter\nfrom sklearn.datasets import make_classification\nfrom imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:53.47417Z","iopub.execute_input":"2021-11-22T14:27:53.474442Z","iopub.status.idle":"2021-11-22T14:27:53.954068Z","shell.execute_reply.started":"2021-11-22T14:27:53.474404Z","shell.execute_reply":"2021-11-22T14:27:53.953306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicDataProcess:\n\t@staticmethod\n\tdef TransposeElements(inputs):\n\t\toutputs = []\n\t\tfor i in inputs:\n\t\t\toutputs.append(i.transpose())\n\t\treturn np.array(outputs)\n\t@staticmethod\n\tdef LoadEEGFromFile(data_dir, is_train):\n\t\tif is_train:\n\t\t\ttrain_data = scipy.io.loadmat(data_dir + \"/Train/trainData.mat\")\n\t\telse:\n\t\t\ttest_data = scipy.io.loadmat(data_dir + \"/Test/testData.mat\")\n\t\tchannels = []\n\t\tif is_train:\n\t\t\tfor i in train_data['trainData']:\n\t\t\t\tchannels.append(np.array(i).transpose())\n\t\telse:\n\t\t\tfor i in test_data['testData']:\n\t\t\t\tchannels.append(np.array(i).transpose())\n\t\treturn channels\n\t@staticmethod\n\tdef LoadDataFromFile(filename):\n\t\tfile = open(filename)\n\t\traw_data = file.readlines()\n\t\tfile.close()\n\t\tdata = []\n\t\tfor i in raw_data:\n\t\t\tdata.append(int(i))\n\t\treturn data\n\t@staticmethod\n\tdef LowPassFilter(fc, data):\n\t\tfs = 250.0\n\t\tw = fc / (fs / 2.0) # Normalize the frequency\n\t\tb, a = signal.butter(9, w, 'low', analog=False)\n\t\toutput = signal.filtfilt(b, a, data)\n\t\treturn output\n\t@staticmethod\n\tdef GetFeatureByPCA(channels, data_size, with_filter, pca_threshold, reshaped, width):\n\t\ttrain_input = []\n\t\tfor i in range(data_size):\n\t\t\tp300_matrix = []\n\t\t\tfor channel in channels:\n\t\t\t\tleft = 125 - width / 2\n\t\t\t\tright = 125 + width / 2\n\t\t\t\traw_channel_data = channel[i][int(left):int(right)]\n\t\t\t\tif with_filter:\n\t\t\t\t\tfiltered_data = BasicDataProcess.LowPassFilter(15, raw_channel_data)\n\t\t\t\t\tp300_matrix.append(filtered_data.tolist())\n\t\t\t\telse:\n\t\t\t\t\tp300_matrix.append(raw_channel_data)\n\t\t\tpca = PCA()\n\t\t\tp300_pca = pca.fit_transform(p300_matrix)\n\t\t\tif reshaped:\n\t\t\t\ttrain_input.append(p300_pca[0:pca_threshold].reshape(-1).tolist())\n\t\t\telse:\n\t\t\t\ttrain_input.append(p300_pca[0:pca_threshold].tolist())\n\t\treturn train_input\n\t@staticmethod\n\tdef GetP300Inputs(channels, with_filter, data_size, reshaped, width):\n\t\tresult = []\n\t\tfor i in range(data_size):\n\t\t\tp300_matrix = []\n\t\t\tfor channel in channels:\n\t\t\t\tleft = 125 - width / 2\n\t\t\t\tright = 125 + width / 2\n\t\t\t\traw_channel_data = channel[i][int(left):int(right)]\n\t\t\t\tif(with_filter):\n\t\t\t\t\tfiltered_data = BasicDataProcess.LowPassFilter(20, raw_channel_data)\n\t\t\t\t\tp300_matrix.append(filtered_data.tolist())\n\t\t\t\telse:\n\t\t\t\t\tp300_matrix.append(raw_channel_data)\n\t\t\tif reshaped:\n\t\t\t\tresult.append(np.array(p300_matrix).reshape(-1).tolist())\n\t\t\telse:\n\t\t\t\tresult.append(np.array(p300_matrix))\n\t\treturn result\n\t@staticmethod\n\tdef GetTargetResults(proba_results):\n\t\tlabel_results = np.zeros(len(proba_results), dtype=np.int8)\n\t\tpos = 0\n\t\tmax_proba = 0\n\t\trecord = 0\n\t\tfor i in proba_results:\n\t\t\tif(i[1] > max_proba):\n\t\t\t\tmax_proba = i[1]\n\t\t\t\trecord = pos\n\t\t\tpos += 1\n\t\t\tif pos % 8 == 0:\n\t\t\t\tlabel_results[record] = 1\n\t\t\t\tmax_proba = 0\n\t\treturn label_results.tolist()\n\t@staticmethod\n\tdef GetLabelResults(targets, events):\n\t\tresult = []\n\t\tpos = 0\n\t\tfor i in targets:\n\t\t\tif i == 1:\n\t\t\t\tresult.append(events[pos])\n\t\t\tpos += 1\n\t\treturn result\n\t@staticmethod\n\tdef GetDifferences(a, b, size):\n\t\tresult = 0\n\t\tfor i in range(size):\n\t\t\tif a[i] != b[i]:\n\t\t\t\tresult += 1\n\t\treturn result\n\t@staticmethod\n\tdef GetMostFrequent(List): \n\t\tcounter = 0\n\t\tnum = List[0] \n\n\t\tfor i in List: \n\t\t\tcurr_frequency = List.count(i) \n\t\t\tif(curr_frequency > counter): \n\t\t\t\tcounter = curr_frequency \n\t\t\t\tnum = i \n\t\treturn num\n\t@staticmethod\n\tdef GetLabels(votes, block_size):\n\t\ttest_labels = []\n\t\tfor i in range(len(votes) // block_size):\n\t\t\tblock = np.array(votes[i * block_size : (i + 1) * block_size])\n\t\t\ttest_labels.append(BasicDataProcess.GetMostFrequent(block.tolist()))\n\t\treturn test_labels","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:53.955584Z","iopub.execute_input":"2021-11-22T14:27:53.95583Z","iopub.status.idle":"2021-11-22T14:27:53.984399Z","shell.execute_reply.started":"2021-11-22T14:27:53.955794Z","shell.execute_reply":"2021-11-22T14:27:53.983646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\nfrom sklearn.feature_selection import VarianceThreshold, SelectFromModel\n#from BasicDataProcess import BasicDataProcess\nfrom sklearn import svm\nfrom sklearn.svm import LinearSVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nimport time\nimport numpy as np\nimport os\nimport traceback\nfrom scipy.fftpack import rfft, irfft\n\n#timestr = time.strftime(\"%Y%m%d-%H%M%S\")\nresult_file_name = \"Output_Even_Channels\" + \".csv\"\nresult_file = open(result_file_name, \"w+\")\nresult_file.write(\"SUBJECT,SESSION,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,\\n\")\n\nall_results = []\nsuccessful = True","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:53.986669Z","iopub.execute_input":"2021-11-22T14:27:53.987067Z","iopub.status.idle":"2021-11-22T14:27:54.006466Z","shell.execute_reply.started":"2021-11-22T14:27:53.987017Z","shell.execute_reply":"2021-11-22T14:27:54.005687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GetNonTargetsAverage(train_inputs, train_targets):\n\tprint(\"--Get non-targets average matrix--\")\n\tnon_targets = None\n\tcount = 0\n\tfor i in range(len(train_targets)):\n\t\tif train_targets[i] == 0:\n\t\t\tcount += 1\n\t\t\tif non_targets is None:\n\t\t\t\tnon_targets = train_inputs[i]\n\t\t\telse:\n\t\t\t\tnon_targets = np.add(non_targets, train_inputs[i])\n\tnon_targets = non_targets / float(count)\n\tprint(\"--Done--\")\n\treturn non_targets","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:54.007855Z","iopub.execute_input":"2021-11-22T14:27:54.008386Z","iopub.status.idle":"2021-11-22T14:27:54.014495Z","shell.execute_reply.started":"2021-11-22T14:27:54.008349Z","shell.execute_reply":"2021-11-22T14:27:54.013736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ApplySpecialFilter(inputs, filter_feature, reshaped):\n\tprint(\"--Apply special filter to the inputs--\")\n\tresult = []\n\tfor single_input in inputs:\n\t\tepoch = []\n\t\tfor j in range(6):\n\t\t\tinput_fft = rfft(single_input[j])\n\t\t\tfilter_fft = rfft(filter_feature[j])\n\t\t\toutput = irfft(input_fft - filter_fft)\n\t\t\tepoch.append(output)   \n\t\tif reshaped:\n\t\t\tresult.append(np.array(epoch).reshape(-1))\n\t\telse:\n\t\t\tresult.append(np.array(epoch))\n\tprint(\"--Done--\")\n\treturn np.array(result)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:54.016062Z","iopub.execute_input":"2021-11-22T14:27:54.016662Z","iopub.status.idle":"2021-11-22T14:27:54.02456Z","shell.execute_reply.started":"2021-11-22T14:27:54.01661Z","shell.execute_reply":"2021-11-22T14:27:54.023827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def PreprocessData(data_dir, filter_applied, pca_applied, pca_threshold, reshaped, data_size, input_data, width):\n\tprint(\"--Load \" + data_dir + \" Successfully! Now start processing--\")\n\tprint(\"--Applying Low Pass Filter and reshape to input data(Train)...--\")\n\tif filter_applied:\n\t\tprint(\"Low pass filter: YES\")\n\telse:\n\t\tprint(\"Low pass filter: NO\")\n\tif pca_applied:\n\t\tresult = BasicDataProcess.GetFeatureByPCA(input_data, data_size, filter_applied, pca_threshold, reshaped, width)\n\t\tprint(\"PCA Applied with threshold: \" + str(pca_threshold))\n\telse:\n\t\tresult = BasicDataProcess.GetP300Inputs(input_data, filter_applied, data_size, reshaped, width)\n\t\tprint(\"PCA Applied: NO\")\n\tif reshaped:\n\t\tprint(\"Data set has been reshaped to 1D.\")\n\tprint(\"--Input data preprocessed!--\")\n\treturn result","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:54.026204Z","iopub.execute_input":"2021-11-22T14:27:54.026814Z","iopub.status.idle":"2021-11-22T14:27:54.034211Z","shell.execute_reply.started":"2021-11-22T14:27:54.026725Z","shell.execute_reply":"2021-11-22T14:27:54.033473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\nimport pandas\nfrom sklearn import model_selection\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nimport pywt\n","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:54.035681Z","iopub.execute_input":"2021-11-22T14:27:54.036205Z","iopub.status.idle":"2021-11-22T14:27:54.05229Z","shell.execute_reply.started":"2021-11-22T14:27:54.036171Z","shell.execute_reply":"2021-11-22T14:27:54.051669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import BaggingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfor sbj_no in range(1, 16):\n    print(\"===================SBJ%02d===================\" % sbj_no)\n    sbj_folder = \"../input/bciaut-p300/data\" + \"/SBJ%02d\" % sbj_no\n    for session_no in range(1, 8):\n        train_cwt = []\n        data_dir = sbj_folder + \"/S0\" + str(session_no)\n        train_data = BasicDataProcess.LoadEEGFromFile(data_dir, True)\n        train_data = np.array(train_data)\n        train_data = train_data[[1, 2,  3, 5, 6, 7], :, :]\n        train_targets = BasicDataProcess.LoadDataFromFile(data_dir + \"/Train/trainTargets.txt\")\n        \n        sample_width = 150\n        train_inputs = PreprocessData(data_dir, filter_applied = False, pca_applied = False, pca_threshold = 20, reshaped = False, data_size = len(train_targets), input_data = train_data, width = sample_width)\n        filter_feature = GetNonTargetsAverage(train_inputs, train_targets)\n        train_inputs = ApplySpecialFilter(train_inputs, filter_feature, reshaped = True)\n        train_targets = np.array(train_targets)\n        #train_inputs = []\n        lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(train_inputs, train_targets)\n        sel_model = SelectFromModel(lsvc, prefit=True)\n        train_inputs = sel_model.transform(train_inputs)\n        scale = [28, 32, 40, 45, 50, 60, 70, 80,90,110,150,200] \n        for event in range(0, len(train_inputs)):\n            cwtmatr, freqs = pywt.cwt(train_inputs[event, :], scale, 'mexh')\n            cwt_oneD = []\n            cwtmatr = np.array(cwtmatr)\n            cwt_oneD = cwtmatr.flatten()\n            #for i in range(0, len(cwtmatr)-1):\n             #   cwt_oneD = sum(cwtmatr[i+1, :], [])\n            train_cwt.append(cwt_oneD)\n        estimators = []\n        train_inp = np.concatenate((train_inputs, train_cwt), axis = 1)\n        sm = SMOTE(random_state=42)\n        train_fea, train_targets = sm.fit_resample(train_inp, train_targets)\n        model1 = LinearDiscriminantAnalysis()\n        model2 = MLPClassifier() \n        model3 = svm.SVC(gamma=\"scale\", probability=True, random_state=1, degree=12)\n        model4 = RandomForestClassifier();\n        estimators.append(('lda', model1))\n        estimators.append(('mlp', model2))\n        estimators.append(('svc', model3))\n        estimators.append(('rand', model4))\n        model = VotingClassifier(estimators, voting='soft', weights=[11, 3, 4, 7])\n        model.fit(train_fea, train_targets)\n        test_events = BasicDataProcess.LoadDataFromFile(data_dir + \"/Test/testEvents.txt\")\n        test_data = BasicDataProcess.LoadEEGFromFile(data_dir, False)\n        test_inputs = PreprocessData(data_dir, filter_applied = False, pca_applied = False, pca_threshold = 20, reshaped = False, data_size = len(test_events), input_data = test_data, width = sample_width)\n        test_inputs = ApplySpecialFilter(test_inputs, filter_feature, reshaped = True)\n        test_inputs = sel_model.transform(test_inputs)\n        #test_inputs = []\n        test_cwt = []\n        for event in range(0, len(test_inputs)):\n            cwt_test, freqs = pywt.cwt(test_inputs[event, :], scale, 'mexh')\n            cwt_one_test = []\n            cwt_test = np.array(cwt_test)\n            cwt_one_test = cwt_test.flatten()\n            #for i in range(0, len(cwtmatr)-1):\n             #   cwt_oneD = sum(cwtmatr[i+1, :], [])\n            test_cwt.append(cwt_one_test)\n        test_fea = np.concatenate((test_inputs, test_cwt), axis = 1)\n        raw_results = model.predict_proba(test_fea)\n        test_targets = BasicDataProcess.GetTargetResults(raw_results)\n        test_votes = BasicDataProcess.GetLabelResults(test_targets, test_events)\n        test_runs_per_block = BasicDataProcess.LoadDataFromFile(data_dir + \"/Test/runs_per_block.txt\")[0]\n        test_labels = BasicDataProcess.GetLabels(test_votes, test_runs_per_block)\n        all_results.append(test_labels)\n        result_file.write(str(sbj_no) + \",\" + str(session_no))\n        for i in test_labels:\n            result_file.write(\",\" + str(i))\n            result_file.write(\"\\n\")\n        \nif successful:\n    result_file.close()\n    os.system(\"python3 result_compare.py \" + result_file_name) ","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:27:54.053642Z","iopub.execute_input":"2021-11-22T14:27:54.054097Z","iopub.status.idle":"2021-11-22T15:21:14.98127Z","shell.execute_reply.started":"2021-11-22T14:27:54.05404Z","shell.execute_reply":"2021-11-22T15:21:14.979915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nnp.shape(train_fea)\n#len(cwtmatr)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T15:21:14.982257Z","iopub.status.idle":"2021-11-22T15:21:14.983091Z","shell.execute_reply.started":"2021-11-22T15:21:14.982821Z","shell.execute_reply":"2021-11-22T15:21:14.982848Z"},"trusted":true},"execution_count":null,"outputs":[]}]}