kohnakagawa · August 11, 2019 03:28
diff --git a/byte_histogram_classify.ipynb b/byte_histogram_classify.ipynb
 {
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import glob\n",
    "import sklearn\n",
    "import ember\n",
    "import numpy as np\n",
    "import yara\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_paths = glob.glob(os.path.join(\"data\", \"benignware\", \"*\"))\n",
    "m_paths = glob.glob(os.path.join(\"data\", \"malware\", \"*\"))\n",
    "labels = [0 for _ in b_paths] + [1 for _ in m_paths]\n",
    "data_paths = b_paths + m_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rule = yara.compile(source='rule IsPeFile {strings:$mz = \"MZ\"condition:$mz at 0 and uint32(uint32(0x3C)) == 0x4550}')\n",
    "path_labels = [(p, l) for p, l in zip(data_paths, labels) if rule.match(p)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fextractor = ember.PEFeatureExtractor()\n",
    "fvector = np.array([fextractor.feature_vector(bytez=open(p, \"rb\").read())[0:256] for p, _ in path_labels])\n",
    "labels = np.array([l for _, l in path_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"fvector.pickle\", \"wb\") as fb:\n",
    "    pickle.dump(fvector, fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = fvector\n",
    "y = labels\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "kf = KFold(n_splits=4)\n",
    "results = []\n",
    "for train_idx, test_idx in kf.split(X_train, y_train):\n",
    "    clf.fit(X_train[train_idx], y_train[train_idx])\n",
    "    y_pred = clf.predict(X_train[test_idx])\n",
    "    results.append(accuracy_score(y_train[test_idx], y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96415770609319\n"
     ]
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9974856321839081\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "y_pred_prob = clf.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhcVZnH8e8v+0ISIEHQBJJAIBAgGzGAG4oikQGRZSAMIjA4iIpCXMbgMgOMioKKOKAYcUM0AZElKggIAcUBQgcCZCEYEiAdIksIGCAdSPLOH+d2p7pT3V1dqeqq6vw+z1NPV9176tz31K2+b91z7z1XEYGZmVlHdat0AGZmVpucQMzMrChOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYgWTdLKk2ysdRzWR9Kqk3Suw3BGSQlKPzl52OUhaKOm9RbzP38kKcgKpUZKekrQu24D9Q9IvJG1XzmVGxK8j4oPlXEYuSe+QdJektZJekfR7SWM6a/l54rlb0sdzp0XEdhGxrEzL20vSbyW9mLX/UUmfk9S9HMsrVpbIRm1NHRGxb0Tc3c5ytkianf2dtOacQGrbURGxHTAemACcV+F4ipLvV7Skg4HbgZuBtwEjgUeAv5XjF3+1/ZKXtAfwALAC2D8iBgH/CkwCBpR4WRVre7V97tZBEeFHDT6Ap4AP5Ly+GPhjzuvewHeAZ4DngCuBvjnzjwbmA/8EngSmZNMHAT8FVgErga8D3bN5pwH3Zs9/BHynRUw3A5/Lnr8N+B3wArAc+GxOufOB64FrsuV/PE/7/gr8MM/0W4Grs+fvBeqBLwMvZp/JyYV8Bjnv/RLwD+BXwA7AH7KY12TPh2XlvwFsBBqAV4HLs+kBjMqe/wK4AvgjsJaUAPbIieeDwBLgFeCHwD352p6VvSZ3feaZPyJb9qlZ+14EvpIzfzJwH/Byti4vB3rlzA/g08DfgeXZtMtICeufwDzg3Tnlu2ef85NZ2+YBuwJ/yep6LftcTszKH0n6fr0M/B8wtsV390vAo8B6oAc53+cs9rosjueA72XTn8mW9Wr2OJic72RWZl/gDuCl7L1frvT/ald+VDwAP4pccc3/4YYBjwGX5cy/FJgN7Ej6xfp74KJs3uRsI3YYaS90KLB3Nu9G4MdAf+AtwFzgE9m8pn9W4D3ZxkbZ6x2AdaTE0S3bwPwX0AvYHVgGHJ6VPR94E/hIVrZvi7b1I22s35en3acDq7Ln7wU2AN8jJYtDsg3Z6AI+g8b3fjt7b19gMHBctvwBwG+Bm3KWfTctNvhsmUBWZ59vD+DXwKxs3pBsg3hsNu+c7DNoLYH8Azi9jfU/Ilv2T7LYx5E2xvtk8w8ADsqWNQJYDJzbIu47ss+mMal+NPsMegCfz2Lok837Iuk7NhpQtrzBLT+D7PUE4HngQFLiOZX0fe2d892dT0pAfXOmNX6f7wNOyZ5vBxzUos09cpZ1Gpu/kwNIyfLzQJ/s9YGV/l/tyo+KB+BHkSsu/cO9Svo1GMCdwPbZPJE2pLm/fg9m8y/NHwOX5qlz52wjlLunchIwJ3ue+88q0i/C92Sv/wO4K3t+IPBMi7rPA36ePT8f+EsbbRuWtWnvPPOmAG9mz99LSgL9c+ZfB3ytgM/gvcAbjRvIVuIYD6zJeX037SeQq3LmHQE8nj3/GHBfzjyREnBrCeRNsr3CVuY3bkyH5UybC0xtpfy5wI0t4j60ne/YGmBc9nwJcHQr5VomkB8B/9OizBLgkJzv7r/n+T43JpC/ABcAQ1ppc2sJ5CTg4XL+3/nR/OH+x9r2kYj4s6RDgN+QfuW+DOxE+hU9T1JjWZF+DUL65XdLnvqGAz2BVTnv60ba0DUTESFpFumf9i/Av5G6XRrreZukl3Pe0p3ULdVoizpzrAE2AW8FHm8x762k7pqmshHxWs7rp0l7Qe19BgAvRERD00ypH2mvZQppjwpggKTuEbGxjXhz/SPn+eukX9BkMTW1Ofv86tuoZzWprUUtT9JepD2zSaTPoQdprzBXs3Ug6QvAGVmsAQwkfacgfWeeLCAeSOv/VEmfyZnWK6s377JbOAO4EHhc0nLggoj4QwHL7UiMVgI+iN4FRMQ9pF+/38kmvUjqTto3IrbPHoMiHXCH9M+7R56qVpD2QIbkvG9gROzbyqJnAsdLGk7a6/hdTj3Lc+rYPiIGRMQRuWG30Z7XSN0Y/5pn9gmkva1GO0jqn/N6N+DZAj6DfDF8ntRFc2BEDCR100FKPG3GXIBVpD2rVGHKasNaL86fSd1pxfoRKfnumbXly2xuR6Om9kh6N/CfpM93h4jYntTN2fie1r4z+awAvtFi/feLiJn5lt1SRPw9Ik4idaF+G7g+W8ftff4rSN2l1kmcQLqO7wOHSRoXEZtIfeOXSnoLgKShkg7Pyv4UOF3S+yV1y+btHRGrSGc+fVfSwGzeHtkezhYi4mHShvoq4LaIaNzjmAuslfQlSX0ldZe0n6S3d6A900m/Yj8raYCkHSR9ndQNdUGLshdI6pVtBI8EflvAZ5DPAFLSeVnSjsB/t5j/HMVvoP4I7C/pI9mZR58Gdmmj/H8D75B0iaRdsvhHSbpG0vYFLG8A6ZjLq5L2Bj5ZQPkNpBMIekj6L9IeSKOrgP+RtKeSsZIGZ/Nafi4/Ac6SdGBWtr+kf5FU0Nljkj4qaadsHTZ+pzZlsW2i9XXwB+Ctks6V1Dv73hxYyDKtOE4gXUREvABcTTpwDeksl6XA/ZL+SfpFOzorO5d0MPpS0q/Me0jdDpD66nsBi0hdSdfTdlfKb4APZH8bY9lI2pCPJ52B1ZhkBnWgPfcCh5MOOq8idU1NAN4VEX/PKfqPLM5nSQetz4qIxm6vVj+DVnyfdED6ReB+4E8t5l9G2uNaI+kHhbYla8+LpD2qi0ndU2NIZxqtb6X8k6RkOQJYKOkV0h5eHem4V3u+QOpWXEvaoF/bTvnbSO19gvRZN9C8m+l7pONLt5MS009JnxWkY1q/lPSypBMioo50TOxy0rpZSjpWUagppDa/SvrMp0bEuoh4nXQ23N+yZR2U+6aIWEs6MeQo0vfi78D7OrBc66DGM2jMak525fI1EdFWV1BVktSNdBrxyRExp9LxmBXDeyBmnUTS4ZK2l9Sbzcck7q9wWGZFK1sCkfQzSc9LWtDKfEn6gaSl2RANE8sVi1mVOJh0ltCLpG6Wj0TEusqGZFa8snVhSXoP6TqFqyNivzzzjwA+QzpX/kDSRXA+4GVmViPKtgcSEX8hDSfQmqNJySUi4n5ge0mFnPduZmZVoJIXEg6l+Vke9dm0VS0LSjoTOBOgf//+B+y9996dEqCZWVcxb968FyNip1LWWRNXokfEDGAGwKRJk6Kurq7CEZmZ1RZJT5e6zkqehbWSNPRAo2HZNDMzqwGVTCCzgY9lZ2MdBLySXQltZmY1oGxdWJJmkkY8HZINGvffpIH6iIgrSYP5HUG6SvV10pXRZmZWI8qWQLLB0Nqa33hDGzMzq0G+Et3MzIpSE2dhmZlZ+954A157bcvH66+XZ3lOIGZmnSQCGhryb+RL8diwoXPb4wRiZpZj06b0i70cG/jXX0/1d0S/ftC//5aPHXbIP721xzvfWfrPygnEzGrOxo35N9Cvvrr1G/l1HRzeslu3/Bvs7baDnXfu2Ea+5aNv31R/tXICMbOyaK0/vhSP9Xlvw9W6nj3zb6B33BF23XXrNvK9e4Na3ix4G+EEYtbFbNqUNrANDelvZz5vaNjc/dPR/vg+ffJvoLf2V3z//imBWOk5gZiVwKZN6Rd3OTbIHX1fqQ6k9uiRfl336ZP+5ns+aNCW04vZwPfrB927lyZu6zxOIFazIjq+0S7Xr+833yxNm7p3b3uD3bs3DBgAO+3Udpmtfd67tzfo1j4nEOuQiLSxLPcGuZDnb7xRmjZ167Z549naBnW77WDw4PJtsBufe6NttcQJpAY0brQrtaFu+bwUpPY32v36pVMVy7Ghzp3Ww/8FZkXxv04VuvZaOPfcdDph48a7FHceltrfuDb2a5fzV3bjRntbPXPFrKtwAqlCc+fCiy/Cpz5V2o23N9pmVkpOIFWqTx+47LJKR2Fm1roqvsbRzMyqmROImZkVxV1YVSACnn4aHnggPW68sdIRmZm1zwmkAl55BR58cHPCmDsXnnsuzevTByZOhHPOqWyMZmbtcQIpsw0bYMECuP/+zQnj8cc3n5Y7ejQcfjgceGB6jB3rcXvMrDY4gZRQBNTXb04UDzwA8+ZtvhvY4MEpSUydCgcdBG9/e7pQzsysFjmBbIW1a6GurnnCWLUqzevVCyZMgI9/fPPexe67+zoMM+s6nEAKtHEjLFzYPFksWrT57mKjRsGhh25OFuPGpQv4zMy6KieQVqxc2TxZ1NWlexxA6nY68EA47rj0d/Lk1D1lZrYtcQIhJYZ581KiaDzYvXJlmtezJ4wfD6efvnnvYtQod0WZmW1zCWTjxnQWVO7exYIFaTrAyJHw7ndvThYTJqRTa83MrLkun0D+8Y/myeLBB9PBb0ijzk6eDOedtzlh7LRTZeM1M6sVXSqBvP46PPRQ84TxzDNpXo8e6RqLj350c7LYa690MyEzM+u4mk0gmzbBkiXNk8Wjj27uiho+PF1rcc45KVlMnAh9+1Y2ZjOzrqTmEsizz8Jhh6WuqFdeSdMGDEhdUV/60uazonbZpbJxmpl1dTWXQFatgre8BU48cXNX1N57+17SZmadreYSSJ8+MH9+paMwMzMfQjYzs6I4gZiZWVGcQMzMrChOIGZmVpSyJhBJUyQtkbRU0vQ883eTNEfSw5IelXREOeMxM7PSKVsCkdQduAL4EDAGOEnSmBbFvgpcFxETgKnAD8sVj5mZlVY590AmA0sjYllEvAHMAo5uUSaAgdnzQcCzZYzHzMxKqJwJZCiwIud1fTYt1/nARyXVA7cAn8lXkaQzJdVJqtu4cUM5YjUzsw6q9EH0k4BfRMQw4AjgV5K2iCkiZkTEpIiY1L17zV37aGbWJZUzgawEds15PSyblusM4DqAiLgP6AMMKWNMZmZWIuVMIA8Ce0oaKakX6SD57BZlngHeDyBpH1ICeaGMMZmZWYmULYFExAbgbOA2YDHpbKuFki6U9OGs2OeB/5D0CDATOC0iolwxmZlZ6ajWttd9+06KdevqKh2GmVlNkTQvIiaVss5KH0Q3M7Ma5QRiZmZFcQIxM7OiOIGYmVlRnEDMzKwoTiBmZlYUJxAzMyuKE4iZmRXFCcTMzIriBGJmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK0pBCURSL0mjyh2MmZnVjnYTiKR/AR4D7shej5d0Y7kDMzOz6lbIHsiFwIHAywARMR/w3oiZ2TaukATyZkS83GJabd0H18zMSq5HAWUWSzoB6CZpJPBZ4P7yhmVmZtWukD2Qs4EDgE3ADcB64JxyBmVmZtWvkD2QwyPiS8CXGidIOpaUTMzMbBtVyB7IV/NM+0qpAzEzs9rS6h6IpMOBKcBQSd/LmTWQ1J1lZmbbsLa6sJ4HFgANwMKc6WuB6eUMyszMql+rCSQiHgYelvTriGjoxJjMzKwGFHIQfaikbwBjgD6NEyNir7JFZWZmVa+Qg+i/AH4OCPgQcB1wbRljMjOzGlBIAukXEbcBRMSTEfFVUiIxM7NtWCFdWOsldQOelHQWsBIYUN6wzMys2hWSQKYB/UlDmHwDGAT8ezmDMjOz6tduAomIB7Kna4FTACQNLWdQZmZW/do8BiLp7ZI+ImlI9npfSVcDD7T1PjMz6/paTSCSLgJ+DZwM/EnS+cAc4BHAp/CamW3j2urCOhoYFxHrJO0IrAD2j4hlhVYuaQpwGdAduCoivpWnzAnA+aR7jDwSEf/WgfjNzKxC2kogDRGxDiAiXpL0RAeTR3fgCuAwoB54UNLsiFiUU2ZP4DzgnRGxRtJbimqFmZl1urYSyO6SGodsFzAy5zURcWw7dU8GljYmHUmzSHs1i3LK/AdwRUSsyep8voPxm5lZhbSVQI5r8fryDtY9lNTt1aiedG/1XHsBSPobqZvr/Ij4U8uKJJ0JnAnQs+e4DoZhZmbl0NZgind20vL3BN4LDAP+Imn/lvdgj4gZwAyAvn0n+X7sZmZVoJChTIq1Etg15/WwbFquemB2RLwZEcuBJ0gJxczMqlw5E8iDwJ6SRkrqBUwFZrcocxNp74PsWpO9gIIP1JuZWeUUnEAk9e5IxRGxATgbuA1YDFwXEQslXSjpw1mx24DVkhaRrjH5YkSs7shyzMysMhTR9iEFSZOBnwKDImI3SeOAj0fEZzojwJb69p0U69bVVWLRZmY1S9K8iJhUyjoL2QP5AXAksBogIh4B3lfKIMzMrPYUkkC6RcTTLaZtLEcwZmZWOwoZzn1F1o0V2dXlnyGdLWVmZtuwQvZAPgl8DtgNeA44KJtmZmbbsEL2QDZExNSyR2JmZjWlkD2QByXdIulUSb6VrZmZAQUkkIjYA/g6cADwmKSbJHmPxMxsG1fQhYQR8X8R8VlgIvBP0o2mzMxsG9ZuApG0naSTJf0emAu8ALyj7JGZmVlVK+Qg+gLg98DFEfHXMsdjZmY1opAEsntEbCp7JGZmVlNaTSCSvhsRnwd+J2mLAbMKuCOhmZl1YW3tgVyb/e3onQjNzGwb0NYdCedmT/eJiGZJRNLZQGfcsdDMzKpUIafx/nueaWeUOhAzM6stbR0DOZF0F8GRkm7ImTUAeDn/u8zMbFvR1jGQuaR7gAwDrsiZvhZ4uJxBmZlZ9WvrGMhyYDnw584Lx8zMakVbXVj3RMQhktYAuafxCoiI2LHs0ZmZWdVqqwur8ba1QzojEDMzqy2tnoWVc/X5rkD3iNgIHAx8AujfCbGZmVkVK+Q03ptIt7PdA/g5sCfwm7JGZWZmVa+QBLIpIt4EjgX+NyKmAUPLG5aZmVW7QhLIBkn/CpwC/CGb1rN8IZmZWS0o9Er095GGc18maSQws7xhmZlZtVPEFgPtbllI6gGMyl4ujYgNZY2qDX37Top16+oqtXgzs5okaV5ETCplne3eD0TSu4FfAStJ14DsIumUiPhbKQMxM7PaUsgNpS4FjoiIRQCS9iEllJJmMjMzqy2FHAPp1Zg8ACJiMdCrfCGZmVktKGQP5CFJVwLXZK9PxoMpmplt8wpJIGcBnwX+M3v9V+B/yxaRmZnVhDYTiKT9gT2AGyPi4s4JyczMakGrx0AkfZk0jMnJwB2S8t2Z0MzMtlFt7YGcDIyNiNck7QTcAvysc8IyM7Nq19ZZWOsj4jWAiHihnbJmZraNaSsp7C7phuxxI7BHzusb2nhfE0lTJC2RtFTS9DbKHScpJPnaEjOzGtFWF9ZxLV5f3pGKJXUn3Uv9MKAeeFDS7NxrSrJyA4BzgAc6Ur+ZmVVWW/dEv3Mr655MGjdrGYCkWcDRwKIW5f4H+Dbwxa1cnpmZdaJyHtcYCqzIeV1Pi/uISJoI7BoRf2yrIklnSqqTVLdxY8XGcTQzsxwVOzAuqRvwPeDz7ZWNiBkRMSkiJnXvXsi1j2ZmVm4FJxBJvTtY90rS/dQbDcumNRoA7AfcLekp4CBgtg+km5nVhnYTiKTJkh4D/p69HiepkKFMHgT2lDRSUi9gKjC7cWZEvBIRQyJiRESMAO4HPhwRvtmHmVkNKGQP5AfAkcBqgIh4hHSHwjZlN506G7gNWAxcFxELJV0o6cPFh2xmZtWgkAMK3SLiaUm50zYWUnlE3EK6gj132n+1Uva9hdRpZmbVoZAEskLSZCCyazs+AzxR3rDMzKzaFdKF9Ungc8BuwHOkg92fLGdQZmZW/drdA4mI50kHwM3MzJq0m0Ak/QSIltMj4syyRGRmZjWhkGMgf8553gc4huZXmJuZ2TaokC6sa3NfS/oVcG/ZIjIzs5pQzFAmI4GdSx2ImZnVlkKOgaxh8zGQbsBLQKv39jAzs21DmwlE6erBcWwew2pTRGxxQN3MzLY9bXZhZcnilojYmD2cPMzMDCjsGMh8SRPKHomZmdWUVruwJPXIBkScQLod7ZPAa4BIOycTOylGMzOrQm0dA5kLTAQ8cq6ZmW2hrQQigIh4spNiMTOzGtJWAtlJ0udamxkR3ytDPGZmViPaSiDdge3I9kTMzMxytZVAVkXEhZ0WiZmZ1ZS2TuP1noeZmbWqrQTy/k6LwszMak6rCSQiXurMQMzMrLYUMxqvmZmZE4iZmRXHCcTMzIriBGJmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYmZmRXECMTOzojiBmJlZUcqaQCRNkbRE0lJJ0/PM/5ykRZIelXSnpOHljMfMzEqnbAlEUnfgCuBDwBjgJEljWhR7GJgUEWOB64GLyxWPmZmVVjn3QCYDSyNiWUS8AcwCjs4tEBFzIuL17OX9wLAyxmNmZiVUzgQyFFiR87o+m9aaM4Bb882QdKakOkl1GzduKGGIZmZWrB6VDgBA0keBScAh+eZHxAxgBkDfvpOiE0MzM7NWlDOBrAR2zXk9LJvWjKQPAF8BDomI9WWMx8zMSqicXVgPAntKGimpFzAVmJ1bQNIE4MfAhyPi+TLGYmZmJVa2BBIRG4CzgduAxcB1EbFQ0oWSPpwVuwTYDvitpPmSZrdSnZmZVRlF1NYhhb59J8W6dXWVDsPMrKZImhcRk0pZp69ENzOzojiBmJlZUZxAzMysKE4gZmZWFCcQMzMrihOImZkVxQnEzMyK4gRiZmZFcQIxM7OiOIGYmVlRnEDMzKwoTiBmZlYUJxAzMyuKE4iZmRXFCcTMzIpSFfdEN7Pq8eabb1JfX09DQ0OlQ7Ei9OnTh2HDhtGzZ8+yL8sJxMyaqa+vZ8CAAYwYMQJJlQ7HOiAiWL16NfX19YwcObLsy3MXlpk109DQwODBg508apAkBg8e3Gl7j04gZrYFJ4/a1ZnrzgnEzMyK4gRiZlXppptuQhKPP/5407S7776bI488slm50047jeuvvx5IJwBMnz6dPffck4kTJ3LwwQdz6623bnUsF110EaNGjWL06NHcdtttecvcddddTJw4kf32249TTz2VDRs2ALBmzRqOOeYYxo4dy+TJk1mwYAEAS5YsYfz48U2PgQMH8v3vf79Znd/97neRxIsvvthmXZXiBGJmVWnmzJm8613vYubMmQW/52tf+xqrVq1iwYIFPPTQQ9x0002sXbt2q+JYtGgRs2bNYuHChfzpT3/iU5/6FBs3bmxWZtOmTZx66qnMmjWLBQsWMHz4cH75y18C8M1vfpPx48fz6KOPcvXVV3POOecAMHr0aObPn8/8+fOZN28e/fr145hjjmmqc8WKFdx+++3stttuTdNaq6tSfBaWmbXq3HNh/vzS1jl+PLT4ob2FV199lXvvvZc5c+Zw1FFHccEFF7Rb7+uvv85PfvITli9fTu/evQHYeeedOeGEE7Yq3ptvvpmpU6fSu3dvRo4cyahRo5g7dy4HH3xwU5nVq1fTq1cv9tprLwAOO+wwLrroIs444wwWLVrE9OnTAdh777156qmneO6559h5552b3n/nnXeyxx57MHz48KZp06ZN4+KLL+boo49umlZIXZ3JeyBmVnVuvvlmpkyZwl577cXgwYOZN29eu+9ZunQpu+22GwMHDmy37LRp05p1HzU+vvWtb21RduXKley6665Nr4cNG8bKlSublRkyZAgbNmygrq4OgOuvv54VK1YAMG7cOG644QYA5s6dy9NPP019fX2z98+aNYuTTjqpWfuHDh3KuHHjmpUrpK7O5D0QM2tVe3sK5TJz5sym7pmpU6cyc+ZMDjjggFbPMOromUeXXnrpVsfYcvmzZs1i2rRprF+/ng9+8IN0794dgOnTp3POOecwfvx49t9/fyZMmNA0D+CNN95g9uzZXHTRRUDak/rmN7/J7bffvsVy2qurszmBmFlVeemll7jrrrt47LHHkMTGjRuRxCWXXMLgwYNZs2bNFuWHDBnCqFGjeOaZZ/jnP//Z7l7ItGnTmDNnzhbTp06d2tRF1Gjo0KFNexOQLrQcOnToFu89+OCD+etf/wrA7bffzhNPPAHAwIED+fnPfw6kC/1GjhzJ7rvv3vS+W2+9lYkTJzZ1Qz355JMsX768ae+jvr6eiRMnMnfuXHbZZZc26+p0EVFTjz59DggzK59FixZVdPk//vGP48wzz2w27T3veU/cc8890dDQECNGjGiK8amnnorddtstXn755YiI+OIXvxinnXZarF+/PiIinn/++bjuuuu2Kp4FCxbE2LFjo6GhIZYtWxYjR46MDRs2bFHuueeei4iIhoaGOPTQQ+POO++MiIg1a9Y0xTNjxow45ZRTmr3vxBNPjJ/97GetLn/48OHxwgsvFFRXo3zrEKiLEm+PfQzEzKrKzJkzm52NBHDccccxc+ZMevfuzTXXXMPpp5/O+PHjOf7447nqqqsYNGgQAF//+tfZaaedGDNmDPvttx9HHnlkQcdE2rLvvvtywgknMGbMGKZMmcIVV1zR1G10xBFH8OyzzwJwySWXsM8++zB27FiOOuooDj30UAAWL17Mfvvtx+jRo7n11lu57LLLmup+7bXXuOOOOzj22GMLiqWtuipBKTHVjr59J8W6dXWVDsOsy1q8eDH77LNPpcOwrZBvHUqaFxGTSrkc74GYmVlRnEDMzKwoTiBmtoVa69q2zTpz3TmBmFkzffr0YfXq1U4iNSiy+4H06dOnU5bn60DMrJlhw4ZRX1/PCy+8UOlQrAiNdyTsDE4gZtZMz549O+Vudlb7ytqFJWmKpCWSlkqanmd+b0nXZvMfkDSinPGYmVnplC2BSOoOXAF8CBgDnCRpTItiZwBrImIUcCnw7XLFY2ZmpVXOPZDJwNKIWBYRbwCzgKNblDka+GX2/Hrg/fK9NM3MakI5j4EMBVbkvK4HDmytTERskPQKMBh4MbeQpDOBM7OX6yVV9jZc5TWEFu3vYrpy+7py28Dtq3WjS11hTRxEj4gZwAwASXWlvhy/mrh9tasrtw3cvlonqeRjQJWzC2slsGvO62HZtLxlJHJKw9sAAAdvSURBVPUABgGryxiTmZmVSDkTyIPAnpJGSuoFTAVmtygzGzg1e348cFf46iUzs5pQti6s7JjG2cBtQHfgZxGxUNKFpHHpZwM/BX4laSnwEinJtGdGuWKuEm5f7erKbQO3r9aVvH01N5y7mZlVB4+FZWZmRXECMTOzolQ8gWzNcCeSzsumL5F0eKF1dpYyte0pSY9Jml+O0/I6otj2SRosaY6kVyVd3uI9B2TtWyrpB5W8sLRM7bs7q3N+9nhL57RmS1vRvsMkzcvW0zxJh+a8pyrWX5na1hXW3eSc+B+RdEyhdeZV6pusd+RBOrj+JLA70At4BBjTosyngCuz51OBa7PnY7LyvYGRWT3dC6mzVtuWzXsKGFLJ9VaC9vUH3gWcBVze4j1zgYMAAbcCH+pi7bsbmFTj628C8Lbs+X7Aympaf2VsW1dYd/2AHtnztwLPk06mKmq7Wek9kK0Z7uRoYFZErI+I5cDSrL5C6uwM5WhbNSm6fRHxWkTcCzTkFpb0VmBgRNwf6Rt+NfCRsraidSVvX5XZmvY9HBHPZtMXAn2zX7zVsv5K3rZOibpwW9O+1yNiQza9D9B4FlVR281KJ5B8w50Mba1M1vDG4U5ae28hdXaGcrQN0gq/Pdu9PpPK2Zr2tVVnfTt1dpZytK/Rz7MuhK9VsIuuVO07DngoItZTPeuvHG1rVPPrTtKBkhYCjwFnZfOL2m5WOoFYx70rIiaSRjn+tKT3VDog65CTI2J/4N3Z45QKx1M0SfuSRtD+RKVjKbVW2tYl1l1EPBAR+wJvB86TVPTtCyudQLZmuJPW3ltInZ2hHG0jIhr/Pg/cSOW6tsoxVM3KrJ626uwsZRmKJ2f9rQV+Q42uP0nDSN+/j0XEkznlq2H9laNtXWbdNYqIxcCrZMd6CqhzSxU+GNQDWEY6UNx44GbfFmU+TfODQddlz/el+YHmZaQDQe3WWcNt6w8MyMr0B/4PmFJr6y5n/mm0fxD9iK7SvqzOIdnznqS+6bNqrX3A9ln5Y/PUW/H1V462daF1N5LNB9GHA8+SRiEuarvZ6Y3P82EcATxBOgPgK9m0C4EPZ8/7AL8lHUieC+ye896vZO9bQs7ZHvnq7AptI50h8Uj2WFjJtpWgfU+Rhq95ldTfOiabPglYkNV5OdloCV2hfaSkPw94NFt/l5GdXVdL7QO+CrwGzM95vKWa1l+p29aF1t0pWfzzgYeAj7RVZ3sPD2ViZmZFqfQxEDMzq1FOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYlVH0sacEUPnN44k2krZEZIWlGCZjSOtPiLpb5JGF1HHWZI+lj0/TdLbcuZdJWlMieN8UNL4At5zrqR+W7tss5acQKwarYuI8TmPpzppuSdHxDjSIHSXdPTNEXFlRFydvTwNeFvOvI9HxKKSRLk5zh9SWJznkkZhNSspJxCrCdmexl8lPZQ93pGnzL6S5mZ7LY9K2jOb/tGc6T+W1L2dxf0FGJW99/2SHs7uD/GzxpFZJX1L0qJsOd/Jpp0v6QuSjiddUPfrbJl9sz2HSdleStNGP9tTubzIOO8jZ8A7ST+SVCdpoaQLsmmfJSWyOZLmZNM+KOm+7HP8raTt2lmOWV5OIFaN+uZ0X92YTXseOCzSQJInAj/I876zgMsiYjxpA14vaZ+s/Duz6RuBk9tZ/lHAY9kgc78ATow0iF4P4JOSBgPHkIZ6GAt8PffNEXE9UEfaUxgfEetyZv8ue2+jE4FZRcY5Bbgp5/VXImISMBY4RNLYiPgBabiK90XE+yQNIV1t/YHss6wDPtfOcszy6lHpAMzyWJdtRHP1BC7P+vw3Anvled99wFeywfBuiIi/S3o/cADwYDb6dl9SMsrn15LWkYYh+QwwGlgeEU9k839JGmPoctK9Pn4q6Q/AHwptWES8IGmZpIOAvwN7A3/L6u1InL2A7YDcz+mEbIj/HqSbBY0hDb2R66Bs+t+y5fQifW5mHeYEYrViGvAcMI6057zFzZoi4jeSHgD+BbhF0idIg/r9MiLOK2AZJ0dE022CJe2Yr1BEbJA0GXg/cDxwNnBovrKtmAWcADwO3BgRkd1bouA4SeMyXQL8L3CspJHAF4C3R8QaSb8gjYfUkoA7IuKkDsRrlpe7sKxWDAJWRcQm0oBwWxwfkLQ7sCzrtrmZ1JVzJ3C8svtXS9pR0vACl7kEGCFpVPb6FOCe7JjBoIi4hZTYxuV571pgQCv13ki629tJpGRCR+OMNIjd14CDJO0NDCQNAviKpJ1J94vJF8v9wDsb2ySpv6R8e3Nm7XICsVrxQ+BUSY+Qun1ey1PmBGCBpPmkexxcnZ359FXSXRwfBe4gde+0KyIagNOB30p6DNgEXEnaGP8hq+9e8h9D+AVwZeNB9Bb1rgEWA8MjYm42rcNxZsdWvgt8MSIeAR4m7dX8htQt1mgG8CdJcyLiBdIZYjOz5dxH+jzNOsyj8ZqZWVG8B2JmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK4oTiJmZFeX/AQATTQNvM4rjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.5f' % roc_auc)\n",
    "plt.legend(loc = 'lower right')\n",
    "plt.xlim([0, 0.03])\n",
    "plt.ylim([0, 1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
 }
	{
	"cells": [
	{
	"cell_type": "code",
	"execution_count": 1,
	"metadata": {},
	"outputs": [],
	"source": [
	"import sys\n",
	"import os\n",
	"import glob\n",
	"import sklearn\n",
	"import ember\n",
	"import numpy as np\n",
	"import yara\n",
	"import pickle"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 2,
	"metadata": {},
	"outputs": [],
	"source": [
	"b_paths = glob.glob(os.path.join(\"data\", \"benignware\", \"*\"))\n",
	"m_paths = glob.glob(os.path.join(\"data\", \"malware\", \"*\"))\n",
	"labels = [0 for _ in b_paths] + [1 for _ in m_paths]\n",
	"data_paths = b_paths + m_paths"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 3,
	"metadata": {},
	"outputs": [],
	"source": [
	"rule = yara.compile(source='rule IsPeFile {strings:$mz = \"MZ\"condition:$mz at 0 and uint32(uint32(0x3C)) == 0x4550}')\n",
	"path_labels = [(p, l) for p, l in zip(data_paths, labels) if rule.match(p)]"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 4,
	"metadata": {},
	"outputs": [],
	"source": [
	"fextractor = ember.PEFeatureExtractor()\n",
	"fvector = np.array([fextractor.feature_vector(bytez=open(p, \"rb\").read())[0:256] for p, _ in path_labels])\n",
	"labels = np.array([l for _, l in path_labels])"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 5,
	"metadata": {},
	"outputs": [],
	"source": [
	"with open(\"fvector.pickle\", \"wb\") as fb:\n",
	" pickle.dump(fvector, fb)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 6,
	"metadata": {},
	"outputs": [],
	"source": [
	"from sklearn.model_selection import train_test_split\n",
	"X = fvector\n",
	"y = labels\n",
	"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 7,
	"metadata": {},
	"outputs": [
	{
	"name": "stderr",
	"output_type": "stream",
	"text": [
	"/usr/local/lib/python3.6/dist-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
	" \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
	]
	}
	],
	"source": [
	"from sklearn.ensemble import RandomForestClassifier\n",
	"from sklearn.metrics import accuracy_score\n",
	"from sklearn.model_selection import KFold\n",
	"\n",
	"clf = RandomForestClassifier()\n",
	"kf = KFold(n_splits=4)\n",
	"results = []\n",
	"for train_idx, test_idx in kf.split(X_train, y_train):\n",
	" clf.fit(X_train[train_idx], y_train[train_idx])\n",
	" y_pred = clf.predict(X_train[test_idx])\n",
	" results.append(accuracy_score(y_train[test_idx], y_pred))"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 8,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"0.96415770609319\n"
	]
	}
	],
	"source": [
	"y_pred = clf.predict(X_test)\n",
	"print(accuracy_score(y_test, y_pred))"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 9,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"0.9974856321839081\n"
	]
	}
	],
	"source": [
	"from sklearn.metrics import roc_curve, auc\n",
	"y_pred_prob = clf.predict_proba(X_test)[:,1]\n",
	"fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)\n",
	"roc_auc = auc(fpr, tpr)\n",
	"print(roc_auc)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 12,
	"metadata": {},
	"outputs": [
	{
	"data": {
	"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhcVZnH8e8v+0ISIEHQBJJAIBAgGzGAG4oikQGRZSAMIjA4iIpCXMbgMgOMioKKOKAYcUM0AZElKggIAcUBQgcCZCEYEiAdIksIGCAdSPLOH+d2p7pT3V1dqeqq6vw+z1NPV9176tz31K2+b91z7z1XEYGZmVlHdat0AGZmVpucQMzMrChOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYgWTdLKk2ysdRzWR9Kqk3Suw3BGSQlKPzl52OUhaKOm9RbzP38kKcgKpUZKekrQu24D9Q9IvJG1XzmVGxK8j4oPlXEYuSe+QdJektZJekfR7SWM6a/l54rlb0sdzp0XEdhGxrEzL20vSbyW9mLX/UUmfk9S9HMsrVpbIRm1NHRGxb0Tc3c5ytkianf2dtOacQGrbURGxHTAemACcV+F4ipLvV7Skg4HbgZuBtwEjgUeAv5XjF3+1/ZKXtAfwALAC2D8iBgH/CkwCBpR4WRVre7V97tZBEeFHDT6Ap4AP5Ly+GPhjzuvewHeAZ4DngCuBvjnzjwbmA/8EngSmZNMHAT8FVgErga8D3bN5pwH3Zs9/BHynRUw3A5/Lnr8N+B3wArAc+GxOufOB64FrsuV/PE/7/gr8MM/0W4Grs+fvBeqBLwMvZp/JyYV8Bjnv/RLwD+BXwA7AH7KY12TPh2XlvwFsBBqAV4HLs+kBjMqe/wK4AvgjsJaUAPbIieeDwBLgFeCHwD352p6VvSZ3feaZPyJb9qlZ+14EvpIzfzJwH/Byti4vB3rlzA/g08DfgeXZtMtICeufwDzg3Tnlu2ef85NZ2+YBuwJ/yep6LftcTszKH0n6fr0M/B8wtsV390vAo8B6oAc53+cs9rosjueA72XTn8mW9Wr2OJic72RWZl/gDuCl7L1frvT/ald+VDwAP4pccc3/4YYBjwGX5cy/FJgN7Ej6xfp74KJs3uRsI3YYaS90KLB3Nu9G4MdAf+AtwFzgE9m8pn9W4D3ZxkbZ6x2AdaTE0S3bwPwX0AvYHVgGHJ6VPR94E/hIVrZvi7b1I22s35en3acDq7Ln7wU2AN8jJYtDsg3Z6AI+g8b3fjt7b19gMHBctvwBwG+Bm3KWfTctNvhsmUBWZ59vD+DXwKxs3pBsg3hsNu+c7DNoLYH8Azi9jfU/Ilv2T7LYx5E2xvtk8w8ADsqWNQJYDJzbIu47ss+mMal+NPsMegCfz2Lok837Iuk7NhpQtrzBLT+D7PUE4HngQFLiOZX0fe2d892dT0pAfXOmNX6f7wNOyZ5vBxzUos09cpZ1Gpu/kwNIyfLzQJ/s9YGV/l/tyo+KB+BHkSsu/cO9Svo1GMCdwPbZPJE2pLm/fg9m8y/NHwOX5qlz52wjlLunchIwJ3ue+88q0i/C92Sv/wO4K3t+IPBMi7rPA36ePT8f+EsbbRuWtWnvPPOmAG9mz99LSgL9c+ZfB3ytgM/gvcAbjRvIVuIYD6zJeX037SeQq3LmHQE8nj3/GHBfzjyREnBrCeRNsr3CVuY3bkyH5UybC0xtpfy5wI0t4j60ne/YGmBc9nwJcHQr5VomkB8B/9OizBLgkJzv7r/n+T43JpC/ABcAQ1ppc2sJ5CTg4XL+3/nR/OH+x9r2kYj4s6RDgN+QfuW+DOxE+hU9T1JjWZF+DUL65XdLnvqGAz2BVTnv60ba0DUTESFpFumf9i/Av5G6XRrreZukl3Pe0p3ULdVoizpzrAE2AW8FHm8x762k7pqmshHxWs7rp0l7Qe19BgAvRERD00ypH2mvZQppjwpggKTuEbGxjXhz/SPn+eukX9BkMTW1Ofv86tuoZzWprUUtT9JepD2zSaTPoQdprzBXs3Ug6QvAGVmsAQwkfacgfWeeLCAeSOv/VEmfyZnWK6s377JbOAO4EHhc0nLggoj4QwHL7UiMVgI+iN4FRMQ9pF+/38kmvUjqTto3IrbPHoMiHXCH9M+7R56qVpD2QIbkvG9gROzbyqJnAsdLGk7a6/hdTj3Lc+rYPiIGRMQRuWG30Z7XSN0Y/5pn9gmkva1GO0jqn/N6N+DZAj6DfDF8ntRFc2BEDCR100FKPG3GXIBVpD2rVGHKasNaL86fSd1pxfoRKfnumbXly2xuR6Om9kh6N/CfpM93h4jYntTN2fie1r4z+awAvtFi/feLiJn5lt1SRPw9Ik4idaF+G7g+W8ftff4rSN2l1kmcQLqO7wOHSRoXEZtIfeOXSnoLgKShkg7Pyv4UOF3S+yV1y+btHRGrSGc+fVfSwGzeHtkezhYi4mHShvoq4LaIaNzjmAuslfQlSX0ldZe0n6S3d6A900m/Yj8raYCkHSR9ndQNdUGLshdI6pVtBI8EflvAZ5DPAFLSeVnSjsB/t5j/HMVvoP4I7C/pI9mZR58Gdmmj/H8D75B0iaRdsvhHSbpG0vYFLG8A6ZjLq5L2Bj5ZQPkNpBMIekj6L9IeSKOrgP+RtKeSsZIGZ/Nafi4/Ac6SdGBWtr+kf5FU0Nljkj4qaadsHTZ+pzZlsW2i9XXwB+Ctks6V1Dv73hxYyDKtOE4gXUREvABcTTpwDeksl6XA/ZL+SfpFOzorO5d0MPpS0q/Me0jdDpD66nsBi0hdSdfTdlfKb4APZH8bY9lI2pCPJ52B1ZhkBnWgPfcCh5MOOq8idU1NAN4VEX/PKfqPLM5nSQetz4qIxm6vVj+DVnyfdED6ReB+4E8t5l9G2uNaI+kHhbYla8+LpD2qi0ndU2NIZxqtb6X8k6RkOQJYKOkV0h5eHem4V3u+QOpWXEvaoF/bTvnbSO19gvRZN9C8m+l7pONLt5MS009JnxWkY1q/lPSypBMioo50TOxy0rpZSjpWUagppDa/SvrMp0bEuoh4nXQ23N+yZR2U+6aIWEs6MeQo0vfi78D7OrBc66DGM2jMak525fI1EdFWV1BVktSNdBrxyRExp9LxmBXDeyBmnUTS4ZK2l9Sbzcck7q9wWGZFK1sCkfQzSc9LWtDKfEn6gaSl2RANE8sVi1mVOJh0ltCLpG6Wj0TEusqGZFa8snVhSXoP6TqFqyNivzzzjwA+QzpX/kDSRXA+4GVmViPKtgcSEX8hDSfQmqNJySUi4n5ge0mFnPduZmZVoJIXEg6l+Vke9dm0VS0LSjoTOBOgf//+B+y9996dEqCZWVcxb968FyNip1LWWRNXokfEDGAGwKRJk6Kurq7CEZmZ1RZJT5e6zkqehbWSNPRAo2HZNDMzqwGVTCCzgY9lZ2MdBLySXQltZmY1oGxdWJJmkkY8HZINGvffpIH6iIgrSYP5HUG6SvV10pXRZmZWI8qWQLLB0Nqa33hDGzMzq0G+Et3MzIpSE2dhmZlZ+954A157bcvH66+XZ3lOIGZmnSQCGhryb+RL8diwoXPb4wRiZpZj06b0i70cG/jXX0/1d0S/ftC//5aPHXbIP721xzvfWfrPygnEzGrOxo35N9Cvvrr1G/l1HRzeslu3/Bvs7baDnXfu2Ea+5aNv31R/tXICMbOyaK0/vhSP9Xlvw9W6nj3zb6B33BF23XXrNvK9e4Na3ix4G+EEYtbFbNqUNrANDelvZz5vaNjc/dPR/vg+ffJvoLf2V3z//imBWOk5gZiVwKZN6Rd3OTbIHX1fqQ6k9uiRfl336ZP+5ns+aNCW04vZwPfrB927lyZu6zxOIFazIjq+0S7Xr+833yxNm7p3b3uD3bs3DBgAO+3Udpmtfd67tzfo1j4nEOuQiLSxLPcGuZDnb7xRmjZ167Z549naBnW77WDw4PJtsBufe6NttcQJpAY0brQrtaFu+bwUpPY32v36pVMVy7Ghzp3Ww/8FZkXxv04VuvZaOPfcdDph48a7FHceltrfuDb2a5fzV3bjRntbPXPFrKtwAqlCc+fCiy/Cpz5V2o23N9pmVkpOIFWqTx+47LJKR2Fm1roqvsbRzMyqmROImZkVxV1YVSACnn4aHnggPW68sdIRmZm1zwmkAl55BR58cHPCmDsXnnsuzevTByZOhHPOqWyMZmbtcQIpsw0bYMECuP/+zQnj8cc3n5Y7ejQcfjgceGB6jB3rcXvMrDY4gZRQBNTXb04UDzwA8+ZtvhvY4MEpSUydCgcdBG9/e7pQzsysFjmBbIW1a6GurnnCWLUqzevVCyZMgI9/fPPexe67+zoMM+s6nEAKtHEjLFzYPFksWrT57mKjRsGhh25OFuPGpQv4zMy6KieQVqxc2TxZ1NWlexxA6nY68EA47rj0d/Lk1D1lZrYtcQIhJYZ581KiaDzYvXJlmtezJ4wfD6efvnnvYtQod0WZmW1zCWTjxnQWVO7exYIFaTrAyJHw7ndvThYTJqRTa83MrLkun0D+8Y/myeLBB9PBb0ijzk6eDOedtzlh7LRTZeM1M6sVXSqBvP46PPRQ84TxzDNpXo8e6RqLj350c7LYa690MyEzM+u4mk0gmzbBkiXNk8Wjj27uiho+PF1rcc45KVlMnAh9+1Y2ZjOzrqTmEsizz8Jhh6WuqFdeSdMGDEhdUV/60uazonbZpbJxmpl1dTWXQFatgre8BU48cXNX1N57+17SZmadreYSSJ8+MH9+paMwMzMfQjYzs6I4gZiZWVGcQMzMrChOIGZmVpSyJhBJUyQtkbRU0vQ883eTNEfSw5IelXREOeMxM7PSKVsCkdQduAL4EDAGOEnSmBbFvgpcFxETgKnAD8sVj5mZlVY590AmA0sjYllEvAHMAo5uUSaAgdnzQcCzZYzHzMxKqJwJZCiwIud1fTYt1/nARyXVA7cAn8lXkaQzJdVJqtu4cUM5YjUzsw6q9EH0k4BfRMQw4AjgV5K2iCkiZkTEpIiY1L17zV37aGbWJZUzgawEds15PSyblusM4DqAiLgP6AMMKWNMZmZWIuVMIA8Ce0oaKakX6SD57BZlngHeDyBpH1ICeaGMMZmZWYmULYFExAbgbOA2YDHpbKuFki6U9OGs2OeB/5D0CDATOC0iolwxmZlZ6ajWttd9+06KdevqKh2GmVlNkTQvIiaVss5KH0Q3M7Ma5QRiZmZFcQIxM7OiOIGYmVlRnEDMzKwoTiBmZlYUJxAzMyuKE4iZmRXFCcTMzIriBGJmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK0pBCURSL0mjyh2MmZnVjnYTiKR/AR4D7shej5d0Y7kDMzOz6lbIHsiFwIHAywARMR/w3oiZ2TaukATyZkS83GJabd0H18zMSq5HAWUWSzoB6CZpJPBZ4P7yhmVmZtWukD2Qs4EDgE3ADcB64JxyBmVmZtWvkD2QwyPiS8CXGidIOpaUTMzMbBtVyB7IV/NM+0qpAzEzs9rS6h6IpMOBKcBQSd/LmTWQ1J1lZmbbsLa6sJ4HFgANwMKc6WuB6eUMyszMql+rCSQiHgYelvTriGjoxJjMzKwGFHIQfaikbwBjgD6NEyNir7JFZWZmVa+Qg+i/AH4OCPgQcB1wbRljMjOzGlBIAukXEbcBRMSTEfFVUiIxM7NtWCFdWOsldQOelHQWsBIYUN6wzMys2hWSQKYB/UlDmHwDGAT8ezmDMjOz6tduAomIB7Kna4FTACQNLWdQZmZW/do8BiLp7ZI+ImlI9npfSVcDD7T1PjMz6/paTSCSLgJ+DZwM/EnS+cAc4BHAp/CamW3j2urCOhoYFxHrJO0IrAD2j4hlhVYuaQpwGdAduCoivpWnzAnA+aR7jDwSEf/WgfjNzKxC2kogDRGxDiAiXpL0RAeTR3fgCuAwoB54UNLsiFiUU2ZP4DzgnRGxRtJbimqFmZl1urYSyO6SGodsFzAy5zURcWw7dU8GljYmHUmzSHs1i3LK/AdwRUSsyep8voPxm5lZhbSVQI5r8fryDtY9lNTt1aiedG/1XHsBSPobqZvr/Ij4U8uKJJ0JnAnQs+e4DoZhZmbl0NZgind20vL3BN4LDAP+Imn/lvdgj4gZwAyAvn0n+X7sZmZVoJChTIq1Etg15/WwbFquemB2RLwZEcuBJ0gJxczMqlw5E8iDwJ6SRkrqBUwFZrcocxNp74PsWpO9gIIP1JuZWeUUnEAk9e5IxRGxATgbuA1YDFwXEQslXSjpw1mx24DVkhaRrjH5YkSs7shyzMysMhTR9iEFSZOBnwKDImI3SeOAj0fEZzojwJb69p0U69bVVWLRZmY1S9K8iJhUyjoL2QP5AXAksBogIh4B3lfKIMzMrPYUkkC6RcTTLaZtLEcwZmZWOwoZzn1F1o0V2dXlnyGdLWVmZtuwQvZAPgl8DtgNeA44KJtmZmbbsEL2QDZExNSyR2JmZjWlkD2QByXdIulUSb6VrZmZAQUkkIjYA/g6cADwmKSbJHmPxMxsG1fQhYQR8X8R8VlgIvBP0o2mzMxsG9ZuApG0naSTJf0emAu8ALyj7JGZmVlVK+Qg+gLg98DFEfHXMsdjZmY1opAEsntEbCp7JGZmVlNaTSCSvhsRnwd+J2mLAbMKuCOhmZl1YW3tgVyb/e3onQjNzGwb0NYdCedmT/eJiGZJRNLZQGfcsdDMzKpUIafx/nueaWeUOhAzM6stbR0DOZF0F8GRkm7ImTUAeDn/u8zMbFvR1jGQuaR7gAwDrsiZvhZ4uJxBmZlZ9WvrGMhyYDnw584Lx8zMakVbXVj3RMQhktYAuafxCoiI2LHs0ZmZWdVqqwur8ba1QzojEDMzqy2tnoWVc/X5rkD3iNgIHAx8AujfCbGZmVkVK+Q03ptIt7PdA/g5sCfwm7JGZWZmVa+QBLIpIt4EjgX+NyKmAUPLG5aZmVW7QhLIBkn/CpwC/CGb1rN8IZmZWS0o9Er095GGc18maSQws7xhmZlZtVPEFgPtbllI6gGMyl4ujYgNZY2qDX37Top16+oqtXgzs5okaV5ETCplne3eD0TSu4FfAStJ14DsIumUiPhbKQMxM7PaUsgNpS4FjoiIRQCS9iEllJJmMjMzqy2FHAPp1Zg8ACJiMdCrfCGZmVktKGQP5CFJVwLXZK9PxoMpmplt8wpJIGcBnwX+M3v9V+B/yxaRmZnVhDYTiKT9gT2AGyPi4s4JyczMakGrx0AkfZk0jMnJwB2S8t2Z0MzMtlFt7YGcDIyNiNck7QTcAvysc8IyM7Nq19ZZWOsj4jWAiHihnbJmZraNaSsp7C7phuxxI7BHzusb2nhfE0lTJC2RtFTS9DbKHScpJPnaEjOzGtFWF9ZxLV5f3pGKJXUn3Uv9MKAeeFDS7NxrSrJyA4BzgAc6Ur+ZmVVWW/dEv3Mr655MGjdrGYCkWcDRwKIW5f4H+Dbwxa1cnpmZdaJyHtcYCqzIeV1Pi/uISJoI7BoRf2yrIklnSqqTVLdxY8XGcTQzsxwVOzAuqRvwPeDz7ZWNiBkRMSkiJnXvXsi1j2ZmVm4FJxBJvTtY90rS/dQbDcumNRoA7AfcLekp4CBgtg+km5nVhnYTiKTJkh4D/p69HiepkKFMHgT2lDRSUi9gKjC7cWZEvBIRQyJiRESMAO4HPhwRvtmHmVkNKGQP5AfAkcBqgIh4hHSHwjZlN506G7gNWAxcFxELJV0o6cPFh2xmZtWgkAMK3SLiaUm50zYWUnlE3EK6gj132n+1Uva9hdRpZmbVoZAEskLSZCCyazs+AzxR3rDMzKzaFdKF9Ungc8BuwHOkg92fLGdQZmZW/drdA4mI50kHwM3MzJq0m0Ak/QSIltMj4syyRGRmZjWhkGMgf8553gc4huZXmJuZ2TaokC6sa3NfS/oVcG/ZIjIzs5pQzFAmI4GdSx2ImZnVlkKOgaxh8zGQbsBLQKv39jAzs21DmwlE6erBcWwew2pTRGxxQN3MzLY9bXZhZcnilojYmD2cPMzMDCjsGMh8SRPKHomZmdWUVruwJPXIBkScQLod7ZPAa4BIOycTOylGMzOrQm0dA5kLTAQ8cq6ZmW2hrQQigIh4spNiMTOzGtJWAtlJ0udamxkR3ytDPGZmViPaSiDdge3I9kTMzMxytZVAVkXEhZ0WiZmZ1ZS2TuP1noeZmbWqrQTy/k6LwszMak6rCSQiXurMQMzMrLYUMxqvmZmZE4iZmRXHCcTMzIriBGJmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYmZmRXECMTOzojiBmJlZUcqaQCRNkbRE0lJJ0/PM/5ykRZIelXSnpOHljMfMzEqnbAlEUnfgCuBDwBjgJEljWhR7GJgUEWOB64GLyxWPmZmVVjn3QCYDSyNiWUS8AcwCjs4tEBFzIuL17OX9wLAyxmNmZiVUzgQyFFiR87o+m9aaM4Bb882QdKakOkl1GzduKGGIZmZWrB6VDgBA0keBScAh+eZHxAxgBkDfvpOiE0MzM7NWlDOBrAR2zXk9LJvWjKQPAF8BDomI9WWMx8zMSqicXVgPAntKGimpFzAVmJ1bQNIE4MfAhyPi+TLGYmZmJVa2BBIRG4CzgduAxcB1EbFQ0oWSPpwVuwTYDvitpPmSZrdSnZmZVRlF1NYhhb59J8W6dXWVDsPMrKZImhcRk0pZp69ENzOzojiBmJlZUZxAzMysKE4gZmZWFCcQMzMrihOImZkVxQnEzMyK4gRiZmZFcQIxM7OiOIGYmVlRnEDMzKwoTiBmZlYUJxAzMyuKE4iZmRXFCcTMzIpSFfdEN7Pq8eabb1JfX09DQ0OlQ7Ei9OnTh2HDhtGzZ8+yL8sJxMyaqa+vZ8CAAYwYMQJJlQ7HOiAiWL16NfX19YwcObLsy3MXlpk109DQwODBg508apAkBg8e3Gl7j04gZrYFJ4/a1ZnrzgnEzMyK4gRiZlXppptuQhKPP/5407S7776bI488slm50047jeuvvx5IJwBMnz6dPffck4kTJ3LwwQdz6623bnUsF110EaNGjWL06NHcdtttecvcddddTJw4kf32249TTz2VDRs2ALBmzRqOOeYYxo4dy+TJk1mwYAEAS5YsYfz48U2PgQMH8v3vf79Znd/97neRxIsvvthmXZXiBGJmVWnmzJm8613vYubMmQW/52tf+xqrVq1iwYIFPPTQQ9x0002sXbt2q+JYtGgRs2bNYuHChfzpT3/iU5/6FBs3bmxWZtOmTZx66qnMmjWLBQsWMHz4cH75y18C8M1vfpPx48fz6KOPcvXVV3POOecAMHr0aObPn8/8+fOZN28e/fr145hjjmmqc8WKFdx+++3stttuTdNaq6tSfBaWmbXq3HNh/vzS1jl+PLT4ob2FV199lXvvvZc5c+Zw1FFHccEFF7Rb7+uvv85PfvITli9fTu/evQHYeeedOeGEE7Yq3ptvvpmpU6fSu3dvRo4cyahRo5g7dy4HH3xwU5nVq1fTq1cv9tprLwAOO+wwLrroIs444wwWLVrE9OnTAdh777156qmneO6559h5552b3n/nnXeyxx57MHz48KZp06ZN4+KLL+boo49umlZIXZ3JeyBmVnVuvvlmpkyZwl577cXgwYOZN29eu+9ZunQpu+22GwMHDmy37LRp05p1HzU+vvWtb21RduXKley6665Nr4cNG8bKlSublRkyZAgbNmygrq4OgOuvv54VK1YAMG7cOG644QYA5s6dy9NPP019fX2z98+aNYuTTjqpWfuHDh3KuHHjmpUrpK7O5D0QM2tVe3sK5TJz5sym7pmpU6cyc+ZMDjjggFbPMOromUeXXnrpVsfYcvmzZs1i2rRprF+/ng9+8IN0794dgOnTp3POOecwfvx49t9/fyZMmNA0D+CNN95g9uzZXHTRRUDak/rmN7/J7bffvsVy2qurszmBmFlVeemll7jrrrt47LHHkMTGjRuRxCWXXMLgwYNZs2bNFuWHDBnCqFGjeOaZZ/jnP//Z7l7ItGnTmDNnzhbTp06d2tRF1Gjo0KFNexOQLrQcOnToFu89+OCD+etf/wrA7bffzhNPPAHAwIED+fnPfw6kC/1GjhzJ7rvv3vS+W2+9lYkTJzZ1Qz355JMsX768ae+jvr6eiRMnMnfuXHbZZZc26+p0EVFTjz59DggzK59FixZVdPk//vGP48wzz2w27T3veU/cc8890dDQECNGjGiK8amnnorddtstXn755YiI+OIXvxinnXZarF+/PiIinn/++bjuuuu2Kp4FCxbE2LFjo6GhIZYtWxYjR46MDRs2bFHuueeei4iIhoaGOPTQQ+POO++MiIg1a9Y0xTNjxow45ZRTmr3vxBNPjJ/97GetLn/48OHxwgsvFFRXo3zrEKiLEm+PfQzEzKrKzJkzm52NBHDccccxc+ZMevfuzTXXXMPpp5/O+PHjOf7447nqqqsYNGgQAF//+tfZaaedGDNmDPvttx9HHnlkQcdE2rLvvvtywgknMGbMGKZMmcIVV1zR1G10xBFH8OyzzwJwySWXsM8++zB27FiOOuooDj30UAAWL17Mfvvtx+jRo7n11lu57LLLmup+7bXXuOOOOzj22GMLiqWtuipBKTHVjr59J8W6dXWVDsOsy1q8eDH77LNPpcOwrZBvHUqaFxGTSrkc74GYmVlRnEDMzKwoTiBmtoVa69q2zTpz3TmBmFkzffr0YfXq1U4iNSiy+4H06dOnU5bn60DMrJlhw4ZRX1/PCy+8UOlQrAiNdyTsDE4gZtZMz549O+Vudlb7ytqFJWmKpCWSlkqanmd+b0nXZvMfkDSinPGYmVnplC2BSOoOXAF8CBgDnCRpTItiZwBrImIUcCnw7XLFY2ZmpVXOPZDJwNKIWBYRbwCzgKNblDka+GX2/Hrg/fK9NM3MakI5j4EMBVbkvK4HDmytTERskPQKMBh4MbeQpDOBM7OX6yVV9jZc5TWEFu3vYrpy+7py28Dtq3WjS11hTRxEj4gZwAwASXWlvhy/mrh9tasrtw3cvlonqeRjQJWzC2slsGvO62HZtLxlJHJKw9sAAAdvSURBVPUABgGryxiTmZmVSDkTyIPAnpJGSuoFTAVmtygzGzg1e348cFf46iUzs5pQti6s7JjG2cBtQHfgZxGxUNKFpHHpZwM/BX4laSnwEinJtGdGuWKuEm5f7erKbQO3r9aVvH01N5y7mZlVB4+FZWZmRXECMTOzolQ8gWzNcCeSzsumL5F0eKF1dpYyte0pSY9Jml+O0/I6otj2SRosaY6kVyVd3uI9B2TtWyrpB5W8sLRM7bs7q3N+9nhL57RmS1vRvsMkzcvW0zxJh+a8pyrWX5na1hXW3eSc+B+RdEyhdeZV6pusd+RBOrj+JLA70At4BBjTosyngCuz51OBa7PnY7LyvYGRWT3dC6mzVtuWzXsKGFLJ9VaC9vUH3gWcBVze4j1zgYMAAbcCH+pi7bsbmFTj628C8Lbs+X7Aympaf2VsW1dYd/2AHtnztwLPk06mKmq7Wek9kK0Z7uRoYFZErI+I5cDSrL5C6uwM5WhbNSm6fRHxWkTcCzTkFpb0VmBgRNwf6Rt+NfCRsraidSVvX5XZmvY9HBHPZtMXAn2zX7zVsv5K3rZOibpwW9O+1yNiQza9D9B4FlVR281KJ5B8w50Mba1M1vDG4U5ae28hdXaGcrQN0gq/Pdu9PpPK2Zr2tVVnfTt1dpZytK/Rz7MuhK9VsIuuVO07DngoItZTPeuvHG1rVPPrTtKBkhYCjwFnZfOL2m5WOoFYx70rIiaSRjn+tKT3VDog65CTI2J/4N3Z45QKx1M0SfuSRtD+RKVjKbVW2tYl1l1EPBAR+wJvB86TVPTtCyudQLZmuJPW3ltInZ2hHG0jIhr/Pg/cSOW6tsoxVM3KrJ626uwsZRmKJ2f9rQV+Q42uP0nDSN+/j0XEkznlq2H9laNtXWbdNYqIxcCrZMd6CqhzSxU+GNQDWEY6UNx44GbfFmU+TfODQddlz/el+YHmZaQDQe3WWcNt6w8MyMr0B/4PmFJr6y5n/mm0fxD9iK7SvqzOIdnznqS+6bNqrX3A9ln5Y/PUW/H1V462daF1N5LNB9GHA8+SRiEuarvZ6Y3P82EcATxBOgPgK9m0C4EPZ8/7AL8lHUieC+ye896vZO9bQs7ZHvnq7AptI50h8Uj2WFjJtpWgfU+Rhq95ldTfOiabPglYkNV5OdloCV2hfaSkPw94NFt/l5GdXVdL7QO+CrwGzM95vKWa1l+p29aF1t0pWfzzgYeAj7RVZ3sPD2ViZmZFqfQxEDMzq1FOIGZmVhQnEDMzK4oTiJmZFcUJxMzMiuIEYlVH0sacEUPnN44k2krZEZIWlGCZjSOtPiLpb5JGF1HHWZI+lj0/TdLbcuZdJWlMieN8UNL4At5zrqR+W7tss5acQKwarYuI8TmPpzppuSdHxDjSIHSXdPTNEXFlRFydvTwNeFvOvI9HxKKSRLk5zh9SWJznkkZhNSspJxCrCdmexl8lPZQ93pGnzL6S5mZ7LY9K2jOb/tGc6T+W1L2dxf0FGJW99/2SHs7uD/GzxpFZJX1L0qJsOd/Jpp0v6QuSjiddUPfrbJl9sz2HSdleStNGP9tTubzIOO8jZ8A7ST+SVCdpoaQLsmmfJSWyOZLmZNM+KOm+7HP8raTt2lmOWV5OIFaN+uZ0X92YTXseOCzSQJInAj/I876zgMsiYjxpA14vaZ+s/Duz6RuBk9tZ/lHAY9kgc78ATow0iF4P4JOSBgPHkIZ6GAt8PffNEXE9UEfaUxgfEetyZv8ue2+jE4FZRcY5Bbgp5/VXImISMBY4RNLYiPgBabiK90XE+yQNIV1t/YHss6wDPtfOcszy6lHpAMzyWJdtRHP1BC7P+vw3Anvled99wFeywfBuiIi/S3o/cADwYDb6dl9SMsrn15LWkYYh+QwwGlgeEU9k839JGmPoctK9Pn4q6Q/AHwptWES8IGmZpIOAvwN7A3/L6u1InL2A7YDcz+mEbIj/HqSbBY0hDb2R66Bs+t+y5fQifW5mHeYEYrViGvAcMI6057zFzZoi4jeSHgD+BbhF0idIg/r9MiLOK2AZJ0dE022CJe2Yr1BEbJA0GXg/cDxwNnBovrKtmAWcADwO3BgRkd1bouA4SeMyXQL8L3CspJHAF4C3R8QaSb8gjYfUkoA7IuKkDsRrlpe7sKxWDAJWRcQm0oBwWxwfkLQ7sCzrtrmZ1JVzJ3C8svtXS9pR0vACl7kEGCFpVPb6FOCe7JjBoIi4hZTYxuV571pgQCv13ki629tJpGRCR+OMNIjd14CDJO0NDCQNAviKpJ1J94vJF8v9wDsb2ySpv6R8e3Nm7XICsVrxQ+BUSY+Qun1ey1PmBGCBpPmkexxcnZ359FXSXRwfBe4gde+0KyIagNOB30p6DNgEXEnaGP8hq+9e8h9D+AVwZeNB9Bb1rgEWA8MjYm42rcNxZsdWvgt8MSIeAR4m7dX8htQt1mgG8CdJcyLiBdIZYjOz5dxH+jzNOsyj8ZqZWVG8B2JmZkVxAjEzs6I4gZiZWVGcQMzMrChOIGZmVhQnEDMzK4oTiJmZFeX/AQATTQNvM4rjAAAAAElFTkSuQmCC\n",
	"text/plain": [
	"<Figure size 432x288 with 1 Axes>"
	]
	},
	"metadata": {
	"needs_background": "light"
	},
	"output_type": "display_data"
	}
	],
	"source": [
	"import matplotlib.pyplot as plt\n",
	"plt.title('Receiver Operating Characteristic')\n",
	"plt.plot(fpr, tpr, 'b', label = 'AUC = %0.5f' % roc_auc)\n",
	"plt.legend(loc = 'lower right')\n",
	"plt.xlim([0, 0.03])\n",
	"plt.ylim([0, 1])\n",
	"plt.ylabel('True Positive Rate')\n",
	"plt.xlabel('False Positive Rate')\n",
	"plt.show()"
	]
	}
	],
	"metadata": {
	"kernelspec": {
	"display_name": "Python 3",
	"language": "python",
	"name": "python3"
	},
	"language_info": {
	"codemirror_mode": {
	"name": "ipython",
	"version": 3
	},
	"file_extension": ".py",
	"mimetype": "text/x-python",
	"name": "python",
	"nbconvert_exporter": "python",
	"pygments_lexer": "ipython3",
	"version": "3.6.8"
	}
	},
	"nbformat": 4,
	"nbformat_minor": 2
	}
No results found