Skip to content

Instantly share code, notes, and snippets.

@Radi4
Last active February 4, 2018 09:13
Show Gist options
  • Save Radi4/1a463ca431027f18415abbb214b9103a to your computer and use it in GitHub Desktop.
Save Radi4/1a463ca431027f18415abbb214b9103a to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from keras.models import Model\n",
"from keras.layers import Dense, Input\n",
"import numpy as np\n",
"import mne\n",
"import os\n",
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.model_selection import train_test_split\n",
"import h5py\n",
"from sklearn.metrics import mean_squared_error"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"eeg_dir = \"./data/new_data/\"\n",
"eeg_names = [x for x in os.listdir(eeg_dir) \n",
" if x[-5:] == \".vhdr\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Extracting parameters from ./data/new_data/0311_pre_eeg_popova.vhdr...\n",
"Setting channel info structure...\n",
"Extracting parameters from ./data/new_data/2509_post_eeg_rinata.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 601549 = 0.000 ... 601.549 secs...\n",
"Extracting parameters from ./data/new_data/2510_post_eeg_osipova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 602949 = 0.000 ... 602.949 secs...\n",
"Extracting parameters from ./data/new_data/0311_post_eeg_mazurova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 603349 = 0.000 ... 603.349 secs...\n",
"Extracting parameters from ./data/new_data/3111_post_eeg_popova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 612499 = 0.000 ... 612.499 secs...\n",
"Extracting parameters from ./data/new_data/1909_pre_eeg_eliseev.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 608399 = 0.000 ... 608.399 secs...\n",
"Extracting parameters from ./data/new_data/14071200_limanskaya_pre_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 604549 = 0.000 ... 604.549 secs...\n",
"Extracting parameters from ./data/new_data/3010_pre_eeg_suhareva.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 609899 = 0.000 ... 609.899 secs...\n",
"Extracting parameters from ./data/new_data/2809_pre_eeg_lukyanov.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 615899 = 0.000 ... 615.899 secs...\n",
"Extracting parameters from ./data/new_data/1207_shevchenko_post_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 602849 = 0.000 ... 602.849 secs...\n",
"Extracting parameters from ./data/new_data/0708_pre_eeg_krasovskaya.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 604049 = 0.000 ... 604.049 secs...\n",
"Extracting parameters from ./data/new_data/1207_shevchenko.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2132749 = 0.000 ... 2132.749 secs...\n",
"Extracting parameters from ./data/new_data/0311_eeg_mazurova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2048799 = 0.000 ... 2048.799 secs...\n",
"Extracting parameters from ./data/new_data/3010_post_eeg_suhareva.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 605799 = 0.000 ... 605.799 secs...\n",
"Extracting parameters from ./data/new_data/2809_lukyanov.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2085449 = 0.000 ... 2085.449 secs...\n",
"Extracting parameters from ./data/new_data/1608_post_eeg_polshina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 632899 = 0.000 ... 632.899 secs...\n",
"Extracting parameters from ./data/new_data/17_11_pre_eeg_bachurina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 602149 = 0.000 ... 602.149 secs...\n",
"Extracting parameters from ./data/new_data/3110_maistrenko.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1914399 = 0.000 ... 1914.399 secs...\n",
"Extracting parameters from ./data/new_data/1509_pre_eeg_zvyagincev.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 606799 = 0.000 ... 606.799 secs...\n",
"Extracting parameters from ./data/new_data/2306_patrakeeva_post_eeg_rest.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 605199 = 0.000 ... 605.199 secs...\n",
"Extracting parameters from ./data/new_data/3010_eeg_suhareva.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1572349 = 0.000 ... 1572.349 secs...\n",
"Extracting parameters from ./data/new_data/1909_eliseev.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2093299 = 0.000 ... 2093.299 secs...\n",
"Extracting parameters from ./data/new_data/1909_post_eeg_eliseev.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 842249 = 0.000 ... 842.249 secs...\n",
"Extracting parameters from ./data/new_data/0307_STRELKOVA.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2103349 = 0.000 ... 2103.349 secs...\n",
"Extracting parameters from ./data/new_data/3110_post_eeg_maistrenko.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 620899 = 0.000 ... 620.899 secs...\n",
"Extracting parameters from ./data/new_data/2606_patrakeeva_main_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2106049 = 0.000 ... 2106.049 secs...\n",
"Extracting parameters from ./data/new_data/1608_rpre_eeg_polshina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 629449 = 0.000 ... 629.449 secs...\n",
"Extracting parameters from ./data/new_data/2510_pre_eeg_osipova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 604099 = 0.000 ... 604.099 secs...\n",
"Extracting parameters from ./data/new_data/0311_eeg_popova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1977699 = 0.000 ... 1977.699 secs...\n",
"Extracting parameters from ./data/new_data/0708_post_eeg_krasovskaya.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 614349 = 0.000 ... 614.349 secs...\n",
"Extracting parameters from ./data/new_data/14071200_limaskaya_post_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 604849 = 0.000 ... 604.849 secs...\n",
"Extracting parameters from ./data/new_data/2510_osipova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1958699 = 0.000 ... 1958.699 secs...\n",
"Extracting parameters from ./data/new_data/2809_post_eeg_lukyanov.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 602149 = 0.000 ... 602.149 secs...\n",
"Extracting parameters from ./data/new_data/1207_shevchenko_pre_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 614199 = 0.000 ... 614.199 secs...\n",
"Extracting parameters from ./data/new_data/0708_eeg_krasovskaya.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2023149 = 0.000 ... 2023.149 secs...\n",
"Extracting parameters from ./data/new_data/2306_patrakeeva_pre_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Extracting parameters from ./data/new_data/0307_POST_EEG_sTRELKOVA.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 602149 = 0.000 ... 602.149 secs...\n",
"Extracting parameters from ./data/new_data/2509_eeg_rinata.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2129299 = 0.000 ... 2129.299 secs...\n",
"Extracting parameters from ./data/new_data/2509-pre_eeg_rinata.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 724899 = 0.000 ... 724.899 secs...\n",
"Extracting parameters from ./data/new_data/17_11_post_eeg_bachurina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 610499 = 0.000 ... 610.499 secs...\n",
"Extracting parameters from ./data/new_data/1608_eeg_polshina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1961799 = 0.000 ... 1961.799 secs...\n",
"Extracting parameters from ./data/new_data/311_pre_eeg_mazurova.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 604349 = 0.000 ... 604.349 secs...\n",
"Extracting parameters from ./data/new_data/1509_zvyagincev.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1910849 = 0.000 ... 1910.849 secs...\n",
"Extracting parameters from ./data/new_data/1407_limanskaya_eeg.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 2406349 = 0.000 ... 2406.349 secs...\n",
"Extracting parameters from ./data/new_data/3110_pre_eeg_maistrenko.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 663199 = 0.000 ... 663.199 secs...\n",
"Extracting parameters from ./data/new_data/17_11_bachurina.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 1979549 = 0.000 ... 1979.549 secs...\n",
"Extracting parameters from ./data/new_data/0307_STRELKOVA_PRE_EEG.vhdr...\n",
"Setting channel info structure...\n",
"Reading 0 ... 601999 = 0.000 ... 601.999 secs...\n",
"2\n"
]
}
],
"source": [
"data = np.zeros((0,61))\n",
"\n",
"fail_count = 0\n",
"\n",
"for eeg_name in eeg_names:\n",
" try:\n",
" eeg_data = mne.io.read_raw_brainvision(eeg_dir + eeg_name, preload=True).get_data().T\n",
" data = np.append(data, eeg_data, axis = 0)\n",
" except:\n",
" fail_count += 1\n",
" continue\n",
"\n",
"print(fail_count)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"scaler = StandardScaler()\n",
"scaled_data = scaler.fit_transform(data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Петр и Девушки. Обе команды использовали StandartScaller"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Девушки"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import keras"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
" warnings.warn('No training configuration found in save file: '\n"
]
}
],
"source": [
"encoder = keras.models.load_model('./catherineglazkova_tasssshhhha/encoder.p')\n",
"decoder = keras.models.load_model('./catherineglazkova_tasssshhhha/decoder.p')"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.24630945127079723"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_encoded = encoder.predict(scaled_data)\n",
"alpha = data_encoded.shape[1] / 61\n",
"data_decoded = decoder.predict(data_encoded)\n",
"data_pred = scaler.inverse_transform(data_decoded)\n",
"mse = mean_squared_error(data, data_pred)\n",
"score = (1 + mse) * alpha\n",
"score"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.0016584351679086453"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mse"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(50462050, 15)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_encoded.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Петр"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import tempfile\n",
"def make_keras_picklable():\n",
" def __getstate__(self):\n",
" model_str = \"\"\n",
" with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
" keras.models.save_model(self, fd.name, overwrite=True)\n",
" model_str = fd.read()\n",
" d = { 'model_str': model_str }\n",
" return d\n",
"\n",
" def __setstate__(self, state):\n",
" with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
" fd.write(state['model_str'])\n",
" fd.flush()\n",
" model = keras.models.load_model(fd.name)\n",
" self.__dict__ = model.__dict__\n",
"\n",
"\n",
" cls = keras.models.Model\n",
" cls.__getstate__ = __getstate__\n",
" cls.__setstate__ = __setstate__\n",
"make_keras_picklable()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
" warnings.warn('No training configuration found in save file: '\n"
]
}
],
"source": [
"load_encoder = pickle.load(open(\"./Petr/encoder.pkl\", \"rb\"))\n",
"load_decoder = pickle.load(open(\"./Petr/decoder.pkl\", \"rb\"))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.9195437515270669"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_encoded = load_encoder.predict(scaled_data)\n",
"alpha = data_encoded.shape[1] / 61\n",
"data_decoded = load_decoder.predict(data_encoded)\n",
"data_pred = scaler.inverse_transform(data_decoded)\n",
"mse = mean_squared_error(data, data_pred)\n",
"score = (1 + mse) * alpha\n",
"score"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.0016458721991263316"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mse"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(50462050, 56)"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_encoded.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Комадна Валентин, Валерия, Костя"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from os import walk\n",
"\n",
"mne.set_log_level('WARNING')\n",
"data_path = u\"./data/new_data/\""
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def getBatches(arr, batch_size=50, axis=0):\n",
" n = arr.shape[axis] // batch_size\n",
" return np.array_split(arr, n, axis=axis)\n",
"\n",
"def prepareData(file_path):\n",
" raw = np.array(mne.io.read_raw_brainvision(file_path, preload=True).to_data_frame())\n",
" batches = np.array(getBatches(raw))\n",
" del raw\n",
" \n",
" return batches"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"files = []\n",
"data = []\n",
"for elem in walk(data_path):\n",
" for file in elem[-1]:\n",
" if file[-4:] == \"vhdr\":\n",
" files.append(file)\n",
"\n",
"data = np.ndarray(shape=(0, 50, 61))\n",
"flag = False\n",
"for file in files:\n",
" file_name = data_path + file\n",
" \n",
" try:\n",
" batches = prepareData(file_name)\n",
" data = np.concatenate((data, batches), axis=0)\n",
" except:\n",
" continue"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
" warnings.warn('No training configuration found in save file: '\n"
]
}
],
"source": [
"encoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/encoder_elu.h5')\n",
"decoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/decoder_elu.h5')"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1009241, 50, 61)"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.shape"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1009241, 1000)"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_encoded = encoder.predict(data)\n",
"data_encoded.shape"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1009241, 50, 61)"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_pred = decoder.predict(data_encoded)\n",
"data_pred.shape"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.32786885245901637"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"alpha = 1000 / (50 * 61)\n",
"alpha"
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"data_original = data.reshape((1009241 * 50, 61))\n",
"data_original_pred = data_pred.reshape((1009241 * 50, 61))\n",
"mse = mean_squared_error(data_original, data_original_pred)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"14357335228541.21"
]
},
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mse"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4707323025751.544"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"score = (1 + mse) * alpha\n",
"score"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Последняя команда делала так как сказано в начале статьи(делили еег по времени на группы размера 50). У них самый худший результат, но взоможно из-за того, что они не делала Scaller. У первой команды(девушки), в коде есть перебо размерности закодированных данных и там видно, что mse почти не меняется. Итоговый результат такой\n",
"1) Девушки\n",
"2) Петр и Марк\n",
"3) Валентин, Валерия и Константин"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<thead>\n",
"<tr><th>Team </th><th style=\"text-align: right;\"> Alpha</th><th style=\"text-align: right;\"> Mse</th></tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr><td>Девушки </td><td style=\"text-align: right;\"> 0.246 </td><td style=\"text-align: right;\">0.0016 </td></tr>\n",
"<tr><td>Петр </td><td style=\"text-align: right;\"> 0.91 </td><td style=\"text-align: right;\">0.00165 </td></tr>\n",
"<tr><td>Валентин, Валерия, Костя</td><td style=\"text-align: right;\"> 0.3279</td><td style=\"text-align: right;\">1.43573e+13</td></tr>\n",
"</tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from tabulate import tabulate\n",
"from IPython.display import HTML\n",
"HTML(tabulate([[\"Девушки\", 0.246, 0.0016], [\"Петр\", 0.91, 0.00165], [\"Валентин, Валерия, Костя\", 0.3279, 14357335228541.21]],\n",
" headers= ['Team', 'Alpha', 'Mse'], tablefmt='html'))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment