Leader_Board.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Dense, Input\n",
    "import numpy as np\n",
    "import mne\n",
    "import os\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import h5py\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_dir = \"./data/new_data/\"\n",
    "eeg_names = [x for x in os.listdir(eeg_dir) \n",
    "                 if x[-5:] == \".vhdr\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting parameters from ./data/new_data/0311_pre_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Extracting parameters from ./data/new_data/2509_post_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 601549  =      0.000 ...   601.549 secs...\n",
      "Extracting parameters from ./data/new_data/2510_post_eeg_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602949  =      0.000 ...   602.949 secs...\n",
      "Extracting parameters from ./data/new_data/0311_post_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 603349  =      0.000 ...   603.349 secs...\n",
      "Extracting parameters from ./data/new_data/3111_post_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 612499  =      0.000 ...   612.499 secs...\n",
      "Extracting parameters from ./data/new_data/1909_pre_eeg_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 608399  =      0.000 ...   608.399 secs...\n",
      "Extracting parameters from ./data/new_data/14071200_limanskaya_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604549  =      0.000 ...   604.549 secs...\n",
      "Extracting parameters from ./data/new_data/3010_pre_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 609899  =      0.000 ...   609.899 secs...\n",
      "Extracting parameters from ./data/new_data/2809_pre_eeg_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 615899  =      0.000 ...   615.899 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko_post_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602849  =      0.000 ...   602.849 secs...\n",
      "Extracting parameters from ./data/new_data/0708_pre_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604049  =      0.000 ...   604.049 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2132749  =      0.000 ...  2132.749 secs...\n",
      "Extracting parameters from ./data/new_data/0311_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2048799  =      0.000 ...  2048.799 secs...\n",
      "Extracting parameters from ./data/new_data/3010_post_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 605799  =      0.000 ...   605.799 secs...\n",
      "Extracting parameters from ./data/new_data/2809_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2085449  =      0.000 ...  2085.449 secs...\n",
      "Extracting parameters from ./data/new_data/1608_post_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 632899  =      0.000 ...   632.899 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_pre_eeg_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/3110_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1914399  =      0.000 ...  1914.399 secs...\n",
      "Extracting parameters from ./data/new_data/1509_pre_eeg_zvyagincev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 606799  =      0.000 ...   606.799 secs...\n",
      "Extracting parameters from ./data/new_data/2306_patrakeeva_post_eeg_rest.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 605199  =      0.000 ...   605.199 secs...\n",
      "Extracting parameters from ./data/new_data/3010_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1572349  =      0.000 ...  1572.349 secs...\n",
      "Extracting parameters from ./data/new_data/1909_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2093299  =      0.000 ...  2093.299 secs...\n",
      "Extracting parameters from ./data/new_data/1909_post_eeg_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 842249  =      0.000 ...   842.249 secs...\n",
      "Extracting parameters from ./data/new_data/0307_STRELKOVA.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2103349  =      0.000 ...  2103.349 secs...\n",
      "Extracting parameters from ./data/new_data/3110_post_eeg_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 620899  =      0.000 ...   620.899 secs...\n",
      "Extracting parameters from ./data/new_data/2606_patrakeeva_main_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2106049  =      0.000 ...  2106.049 secs...\n",
      "Extracting parameters from ./data/new_data/1608_rpre_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 629449  =      0.000 ...   629.449 secs...\n",
      "Extracting parameters from ./data/new_data/2510_pre_eeg_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604099  =      0.000 ...   604.099 secs...\n",
      "Extracting parameters from ./data/new_data/0311_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1977699  =      0.000 ...  1977.699 secs...\n",
      "Extracting parameters from ./data/new_data/0708_post_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 614349  =      0.000 ...   614.349 secs...\n",
      "Extracting parameters from ./data/new_data/14071200_limaskaya_post_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604849  =      0.000 ...   604.849 secs...\n",
      "Extracting parameters from ./data/new_data/2510_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1958699  =      0.000 ...  1958.699 secs...\n",
      "Extracting parameters from ./data/new_data/2809_post_eeg_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 614199  =      0.000 ...   614.199 secs...\n",
      "Extracting parameters from ./data/new_data/0708_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2023149  =      0.000 ...  2023.149 secs...\n",
      "Extracting parameters from ./data/new_data/2306_patrakeeva_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Extracting parameters from ./data/new_data/0307_POST_EEG_sTRELKOVA.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/2509_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2129299  =      0.000 ...  2129.299 secs...\n",
      "Extracting parameters from ./data/new_data/2509-pre_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 724899  =      0.000 ...   724.899 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_post_eeg_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 610499  =      0.000 ...   610.499 secs...\n",
      "Extracting parameters from ./data/new_data/1608_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1961799  =      0.000 ...  1961.799 secs...\n",
      "Extracting parameters from ./data/new_data/311_pre_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604349  =      0.000 ...   604.349 secs...\n",
      "Extracting parameters from ./data/new_data/1509_zvyagincev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1910849  =      0.000 ...  1910.849 secs...\n",
      "Extracting parameters from ./data/new_data/1407_limanskaya_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2406349  =      0.000 ...  2406.349 secs...\n",
      "Extracting parameters from ./data/new_data/3110_pre_eeg_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 663199  =      0.000 ...   663.199 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1979549  =      0.000 ...  1979.549 secs...\n",
      "Extracting parameters from ./data/new_data/0307_STRELKOVA_PRE_EEG.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 601999  =      0.000 ...   601.999 secs...\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "data = np.zeros((0,61))\n",
    "\n",
    "fail_count = 0\n",
    "\n",
    "for eeg_name in eeg_names:\n",
    "    try:\n",
    "        eeg_data = mne.io.read_raw_brainvision(eeg_dir + eeg_name, preload=True).get_data().T\n",
    "        data = np.append(data, eeg_data, axis = 0)\n",
    "    except:\n",
    "        fail_count += 1\n",
    "        continue\n",
    "\n",
    "print(fail_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "scaled_data = scaler.fit_transform(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Петр и Девушки. Обе команды использовали StandartScaller"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Девушки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "encoder = keras.models.load_model('./catherineglazkova_tasssshhhha/encoder.p')\n",
    "decoder = keras.models.load_model('./catherineglazkova_tasssshhhha/decoder.p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24630945127079723"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = encoder.predict(scaled_data)\n",
    "alpha = data_encoded.shape[1] / 61\n",
    "data_decoded = decoder.predict(data_encoded)\n",
    "data_pred = scaler.inverse_transform(data_decoded)\n",
    "mse = mean_squared_error(data, data_pred)\n",
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0016584351679086453"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50462050, 15)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Петр"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tempfile\n",
    "def make_keras_picklable():\n",
    "    def __getstate__(self):\n",
    "        model_str = \"\"\n",
    "        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
    "            keras.models.save_model(self, fd.name, overwrite=True)\n",
    "            model_str = fd.read()\n",
    "        d = { 'model_str': model_str }\n",
    "        return d\n",
    "\n",
    "    def __setstate__(self, state):\n",
    "        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
    "            fd.write(state['model_str'])\n",
    "            fd.flush()\n",
    "            model = keras.models.load_model(fd.name)\n",
    "        self.__dict__ = model.__dict__\n",
    "\n",
    "\n",
    "    cls = keras.models.Model\n",
    "    cls.__getstate__ = __getstate__\n",
    "    cls.__setstate__ = __setstate__\n",
    "make_keras_picklable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "load_encoder = pickle.load(open(\"./Petr/encoder.pkl\", \"rb\"))\n",
    "load_decoder = pickle.load(open(\"./Petr/decoder.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9195437515270669"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = load_encoder.predict(scaled_data)\n",
    "alpha = data_encoded.shape[1] / 61\n",
    "data_decoded = load_decoder.predict(data_encoded)\n",
    "data_pred = scaler.inverse_transform(data_decoded)\n",
    "mse = mean_squared_error(data, data_pred)\n",
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0016458721991263316"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50462050, 56)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Комадна Валентин, Валерия, Костя"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os import walk\n",
    "\n",
    "mne.set_log_level('WARNING')\n",
    "data_path = u\"./data/new_data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getBatches(arr, batch_size=50, axis=0):\n",
    "    n = arr.shape[axis] // batch_size\n",
    "    return np.array_split(arr, n, axis=axis)\n",
    "\n",
    "def prepareData(file_path):\n",
    "    raw = np.array(mne.io.read_raw_brainvision(file_path, preload=True).to_data_frame())\n",
    "    batches = np.array(getBatches(raw))\n",
    "    del raw\n",
    "    \n",
    "    return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = []\n",
    "data = []\n",
    "for elem in walk(data_path):\n",
    "    for file in elem[-1]:\n",
    "        if file[-4:] == \"vhdr\":\n",
    "            files.append(file)\n",
    "\n",
    "data = np.ndarray(shape=(0, 50, 61))\n",
    "flag = False\n",
    "for file in files:\n",
    "    file_name = data_path + file\n",
    "    \n",
    "    try:\n",
    "        batches =  prepareData(file_name)\n",
    "        data = np.concatenate((data, batches), axis=0)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "encoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/encoder_elu.h5')\n",
    "decoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/decoder_elu.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 50, 61)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 1000)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = encoder.predict(data)\n",
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 50, 61)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pred = decoder.predict(data_encoded)\n",
    "data_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32786885245901637"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha = 1000 / (50 * 61)\n",
    "alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_original = data.reshape((1009241 * 50, 61))\n",
    "data_original_pred = data_pred.reshape((1009241 * 50, 61))\n",
    "mse = mean_squared_error(data_original, data_original_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14357335228541.21"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4707323025751.544"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Последняя команда делала так как сказано в начале статьи(делили еег по времени на группы размера 50). У них самый худший результат, но взоможно из-за того, что они не делала Scaller. У первой команды(девушки), в коде есть перебо размерности закодированных данных и там видно, что mse почти не меняется. Итоговый результат такой\n",
    "1) Девушки\n",
    "2) Петр и Марк\n",
    "3) Валентин, Валерия и Константин"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Dense, Input\n",
    "import numpy as np\n",
    "import mne\n",
    "import os\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import h5py\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_dir = \"./data/new_data/\"\n",
    "eeg_names = [x for x in os.listdir(eeg_dir) \n",
    "                 if x[-5:] == \".vhdr\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting parameters from ./data/new_data/0311_pre_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Extracting parameters from ./data/new_data/2509_post_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 601549  =      0.000 ...   601.549 secs...\n",
      "Extracting parameters from ./data/new_data/2510_post_eeg_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602949  =      0.000 ...   602.949 secs...\n",
      "Extracting parameters from ./data/new_data/0311_post_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 603349  =      0.000 ...   603.349 secs...\n",
      "Extracting parameters from ./data/new_data/3111_post_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 612499  =      0.000 ...   612.499 secs...\n",
      "Extracting parameters from ./data/new_data/1909_pre_eeg_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 608399  =      0.000 ...   608.399 secs...\n",
      "Extracting parameters from ./data/new_data/14071200_limanskaya_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604549  =      0.000 ...   604.549 secs...\n",
      "Extracting parameters from ./data/new_data/3010_pre_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 609899  =      0.000 ...   609.899 secs...\n",
      "Extracting parameters from ./data/new_data/2809_pre_eeg_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 615899  =      0.000 ...   615.899 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko_post_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602849  =      0.000 ...   602.849 secs...\n",
      "Extracting parameters from ./data/new_data/0708_pre_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604049  =      0.000 ...   604.049 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2132749  =      0.000 ...  2132.749 secs...\n",
      "Extracting parameters from ./data/new_data/0311_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2048799  =      0.000 ...  2048.799 secs...\n",
      "Extracting parameters from ./data/new_data/3010_post_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 605799  =      0.000 ...   605.799 secs...\n",
      "Extracting parameters from ./data/new_data/2809_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2085449  =      0.000 ...  2085.449 secs...\n",
      "Extracting parameters from ./data/new_data/1608_post_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 632899  =      0.000 ...   632.899 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_pre_eeg_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/3110_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1914399  =      0.000 ...  1914.399 secs...\n",
      "Extracting parameters from ./data/new_data/1509_pre_eeg_zvyagincev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 606799  =      0.000 ...   606.799 secs...\n",
      "Extracting parameters from ./data/new_data/2306_patrakeeva_post_eeg_rest.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 605199  =      0.000 ...   605.199 secs...\n",
      "Extracting parameters from ./data/new_data/3010_eeg_suhareva.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1572349  =      0.000 ...  1572.349 secs...\n",
      "Extracting parameters from ./data/new_data/1909_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2093299  =      0.000 ...  2093.299 secs...\n",
      "Extracting parameters from ./data/new_data/1909_post_eeg_eliseev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 842249  =      0.000 ...   842.249 secs...\n",
      "Extracting parameters from ./data/new_data/0307_STRELKOVA.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2103349  =      0.000 ...  2103.349 secs...\n",
      "Extracting parameters from ./data/new_data/3110_post_eeg_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 620899  =      0.000 ...   620.899 secs...\n",
      "Extracting parameters from ./data/new_data/2606_patrakeeva_main_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2106049  =      0.000 ...  2106.049 secs...\n",
      "Extracting parameters from ./data/new_data/1608_rpre_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 629449  =      0.000 ...   629.449 secs...\n",
      "Extracting parameters from ./data/new_data/2510_pre_eeg_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604099  =      0.000 ...   604.099 secs...\n",
      "Extracting parameters from ./data/new_data/0311_eeg_popova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1977699  =      0.000 ...  1977.699 secs...\n",
      "Extracting parameters from ./data/new_data/0708_post_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 614349  =      0.000 ...   614.349 secs...\n",
      "Extracting parameters from ./data/new_data/14071200_limaskaya_post_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604849  =      0.000 ...   604.849 secs...\n",
      "Extracting parameters from ./data/new_data/2510_osipova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1958699  =      0.000 ...  1958.699 secs...\n",
      "Extracting parameters from ./data/new_data/2809_post_eeg_lukyanov.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/1207_shevchenko_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 614199  =      0.000 ...   614.199 secs...\n",
      "Extracting parameters from ./data/new_data/0708_eeg_krasovskaya.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2023149  =      0.000 ...  2023.149 secs...\n",
      "Extracting parameters from ./data/new_data/2306_patrakeeva_pre_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Extracting parameters from ./data/new_data/0307_POST_EEG_sTRELKOVA.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 602149  =      0.000 ...   602.149 secs...\n",
      "Extracting parameters from ./data/new_data/2509_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2129299  =      0.000 ...  2129.299 secs...\n",
      "Extracting parameters from ./data/new_data/2509-pre_eeg_rinata.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 724899  =      0.000 ...   724.899 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_post_eeg_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 610499  =      0.000 ...   610.499 secs...\n",
      "Extracting parameters from ./data/new_data/1608_eeg_polshina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1961799  =      0.000 ...  1961.799 secs...\n",
      "Extracting parameters from ./data/new_data/311_pre_eeg_mazurova.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 604349  =      0.000 ...   604.349 secs...\n",
      "Extracting parameters from ./data/new_data/1509_zvyagincev.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1910849  =      0.000 ...  1910.849 secs...\n",
      "Extracting parameters from ./data/new_data/1407_limanskaya_eeg.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 2406349  =      0.000 ...  2406.349 secs...\n",
      "Extracting parameters from ./data/new_data/3110_pre_eeg_maistrenko.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 663199  =      0.000 ...   663.199 secs...\n",
      "Extracting parameters from ./data/new_data/17_11_bachurina.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 1979549  =      0.000 ...  1979.549 secs...\n",
      "Extracting parameters from ./data/new_data/0307_STRELKOVA_PRE_EEG.vhdr...\n",
      "Setting channel info structure...\n",
      "Reading 0 ... 601999  =      0.000 ...   601.999 secs...\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "data = np.zeros((0,61))\n",
    "\n",
    "fail_count = 0\n",
    "\n",
    "for eeg_name in eeg_names:\n",
    "    try:\n",
    "        eeg_data = mne.io.read_raw_brainvision(eeg_dir + eeg_name, preload=True).get_data().T\n",
    "        data = np.append(data, eeg_data, axis = 0)\n",
    "    except:\n",
    "        fail_count += 1\n",
    "        continue\n",
    "\n",
    "print(fail_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "scaled_data = scaler.fit_transform(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Петр и Девушки. Обе команды использовали StandartScaller"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Девушки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "encoder = keras.models.load_model('./catherineglazkova_tasssshhhha/encoder.p')\n",
    "decoder = keras.models.load_model('./catherineglazkova_tasssshhhha/decoder.p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24630945127079723"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = encoder.predict(scaled_data)\n",
    "alpha = data_encoded.shape[1] / 61\n",
    "data_decoded = decoder.predict(data_encoded)\n",
    "data_pred = scaler.inverse_transform(data_decoded)\n",
    "mse = mean_squared_error(data, data_pred)\n",
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0016584351679086453"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50462050, 15)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Петр"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tempfile\n",
    "def make_keras_picklable():\n",
    "    def __getstate__(self):\n",
    "        model_str = \"\"\n",
    "        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
    "            keras.models.save_model(self, fd.name, overwrite=True)\n",
    "            model_str = fd.read()\n",
    "        d = { 'model_str': model_str }\n",
    "        return d\n",
    "\n",
    "    def __setstate__(self, state):\n",
    "        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:\n",
    "            fd.write(state['model_str'])\n",
    "            fd.flush()\n",
    "            model = keras.models.load_model(fd.name)\n",
    "        self.__dict__ = model.__dict__\n",
    "\n",
    "\n",
    "    cls = keras.models.Model\n",
    "    cls.__getstate__ = __getstate__\n",
    "    cls.__setstate__ = __setstate__\n",
    "make_keras_picklable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "load_encoder = pickle.load(open(\"./Petr/encoder.pkl\", \"rb\"))\n",
    "load_decoder = pickle.load(open(\"./Petr/decoder.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9195437515270669"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = load_encoder.predict(scaled_data)\n",
    "alpha = data_encoded.shape[1] / 61\n",
    "data_decoded = load_decoder.predict(data_encoded)\n",
    "data_pred = scaler.inverse_transform(data_decoded)\n",
    "mse = mean_squared_error(data, data_pred)\n",
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0016458721991263316"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50462050, 56)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Комадна Валентин, Валерия, Костя"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os import walk\n",
    "\n",
    "mne.set_log_level('WARNING')\n",
    "data_path = u\"./data/new_data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getBatches(arr, batch_size=50, axis=0):\n",
    "    n = arr.shape[axis] // batch_size\n",
    "    return np.array_split(arr, n, axis=axis)\n",
    "\n",
    "def prepareData(file_path):\n",
    "    raw = np.array(mne.io.read_raw_brainvision(file_path, preload=True).to_data_frame())\n",
    "    batches = np.array(getBatches(raw))\n",
    "    del raw\n",
    "    \n",
    "    return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = []\n",
    "data = []\n",
    "for elem in walk(data_path):\n",
    "    for file in elem[-1]:\n",
    "        if file[-4:] == \"vhdr\":\n",
    "            files.append(file)\n",
    "\n",
    "data = np.ndarray(shape=(0, 50, 61))\n",
    "flag = False\n",
    "for file in files:\n",
    "    file_name = data_path + file\n",
    "    \n",
    "    try:\n",
    "        batches =  prepareData(file_name)\n",
    "        data = np.concatenate((data, batches), axis=0)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/anaconda3/lib/python3.6/site-packages/keras/models.py:245: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    }
   ],
   "source": [
    "encoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/encoder_elu.h5')\n",
    "decoder = keras.models.load_model('./biryuk_v_nyaaak_erreth_akbe/decoder_elu.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 50, 61)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 1000)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoded = encoder.predict(data)\n",
    "data_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1009241, 50, 61)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pred = decoder.predict(data_encoded)\n",
    "data_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32786885245901637"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha = 1000 / (50 * 61)\n",
    "alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_original = data.reshape((1009241 * 50, 61))\n",
    "data_original_pred = data_pred.reshape((1009241 * 50, 61))\n",
    "mse = mean_squared_error(data_original, data_original_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14357335228541.21"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4707323025751.544"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = (1 + mse) * alpha\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Последняя команда делала так как сказано в начале статьи(делили еег по времени на группы размера 50). У них самый худший результат, но взоможно из-за того, что они не делала Scaller. У первой команды(девушки), в коде есть перебо размерности закодированных данных и там видно, что mse почти не меняется. Итоговый результат такой\n",
    "1) Девушки\n",
    "2) Петр и Марк\n",
    "3) Валентин, Валерия и Константин"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}