pytorch benchmarks AVX512

pytorch avx benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "104148df-73a8-4b81-b6a4-7c77979951f0",
   "metadata": {},
   "source": [
    "## Installing pytorch benchamrks and requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b3b1dd40-ada3-4106-931a-a207dcdbe32e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 1)) (0.0.1)\n",
      "Requirement already satisfied: patch in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 2)) (1.16)\n",
      "Requirement already satisfied: py-cpuinfo in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 3)) (8.0.0)\n",
      "Requirement already satisfied: distro in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 4)) (1.6.0)\n",
      "Requirement already satisfied: iopath in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 5)) (0.1.9)\n",
      "Requirement already satisfied: pytest in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 6)) (6.2.5)\n",
      "Requirement already satisfied: pytest-benchmark in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 7)) (3.4.1)\n",
      "Requirement already satisfied: requests in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 8)) (2.26.0)\n",
      "Requirement already satisfied: tabulate in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 9)) (0.8.9)\n",
      "Requirement already satisfied: timm==0.4.5 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 10)) (0.4.5)\n",
      "Requirement already satisfied: MonkeyType in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 11)) (21.5.0)\n",
      "Requirement already satisfied: psutil in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 12)) (5.8.0)\n",
      "Requirement already satisfied: pyyaml in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from -r ./requirements.txt (line 13)) (6.0)\n",
      "Requirement already satisfied: torch>=1.4 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from timm==0.4.5->-r ./requirements.txt (line 10)) (1.10.0+cu113)\n",
      "Requirement already satisfied: torchvision in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from timm==0.4.5->-r ./requirements.txt (line 10)) (0.11.1+cu113)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from bs4->-r ./requirements.txt (line 1)) (4.10.0)\n",
      "Requirement already satisfied: tqdm in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from iopath->-r ./requirements.txt (line 5)) (4.62.3)\n",
      "Requirement already satisfied: portalocker in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from iopath->-r ./requirements.txt (line 5)) (2.3.2)\n",
      "Requirement already satisfied: packaging in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (21.0)\n",
      "Requirement already satisfied: attrs>=19.2.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (21.2.0)\n",
      "Requirement already satisfied: colorama in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (0.4.4)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (1.4.0)\n",
      "Requirement already satisfied: toml in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (0.10.2)\n",
      "Requirement already satisfied: pluggy<2.0,>=0.12 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (1.0.0)\n",
      "Requirement already satisfied: iniconfig in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (1.1.1)\n",
      "Requirement already satisfied: py>=1.8.2 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from pytest->-r ./requirements.txt (line 6)) (1.11.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from requests->-r ./requirements.txt (line 8)) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from requests->-r ./requirements.txt (line 8)) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from requests->-r ./requirements.txt (line 8)) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from requests->-r ./requirements.txt (line 8)) (2.0.7)\n",
      "Requirement already satisfied: mypy-extensions in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from MonkeyType->-r ./requirements.txt (line 11)) (0.4.3)\n",
      "Requirement already satisfied: libcst>=0.3.7 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from MonkeyType->-r ./requirements.txt (line 11)) (0.3.21)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from libcst>=0.3.7->MonkeyType->-r ./requirements.txt (line 11)) (0.7.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.2 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from libcst>=0.3.7->MonkeyType->-r ./requirements.txt (line 11)) (3.10.0.2)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from beautifulsoup4->bs4->-r ./requirements.txt (line 1)) (2.2.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from packaging->pytest->-r ./requirements.txt (line 6)) (2.4.7)\n",
      "Requirement already satisfied: pywin32>=226 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from portalocker->iopath->-r ./requirements.txt (line 5)) (228)\n",
      "Requirement already satisfied: numpy in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from torchvision->timm==0.4.5->-r ./requirements.txt (line 10)) (1.21.2)\n",
      "Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from torchvision->timm==0.4.5->-r ./requirements.txt (line 10)) (8.4.0)\n",
      "Requirement already satisfied: torch in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (1.10.0+cu113)\n",
      "Requirement already satisfied: torchvision in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (0.11.1+cu113)\n",
      "Requirement already satisfied: torchaudio in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (0.10.0+cu113)\n",
      "Requirement already satisfied: typing-extensions in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from torch) (3.10.0.2)\n",
      "Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from torchvision) (8.4.0)\n",
      "Requirement already satisfied: numpy in c:\\tools\\miniconda3\\envs\\py39\\lib\\site-packages (from torchvision) (1.21.2)\n",
      "Collecting natsort\n",
      "  Downloading natsort-8.0.0-py3-none-any.whl (37 kB)\n",
      "Installing collected packages: natsort\n",
      "Successfully installed natsort-8.0.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "PY = Path(sys.executable).as_posix()\n",
    "\n",
    "PROJECTS = Path.home() / \"Projects\"\n",
    "PROJECTS.mkdir(exist_ok=True, parents=True)\n",
    "os.chdir(PROJECTS)\n",
    "\n",
    "if not (PROJECTS / \"benchmark\").exists():\n",
    "    !git clone --depth 1 https://github.com/pytorch/benchmark.git\n",
    "\n",
    "os.chdir(\"benchmark\")\n",
    "\n",
    "!$PY -m pip install -r ./requirements.txt\n",
    "!$PY -m pip install torch torchvision torchaudio\n",
    "!$PY -m pip install natsort"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e00f11ad-482a-4da6-9ad3-08749c5e547c",
   "metadata": {},
   "source": [
    "## Running benchmarks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f59f0cb6-13c5-4d5d-b6e8-7bbacae2802e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tests:\n",
      "['test_resnet18_eval_cpu',\n",
      " 'test_resnet50_eval_cpu',\n",
      " 'test_Super_SloMo_eval_cpu',\n",
      " 'test_resnet18_train_cpu',\n",
      " 'test_resnet50_train_cpu',\n",
      " 'test_Super_SloMo_train_cpu',\n",
      " 'test_resnet18_eval_cuda',\n",
      " 'test_resnet50_eval_cuda',\n",
      " 'test_Super_SloMo_eval_cuda',\n",
      " 'test_resnet18_train_cuda',\n",
      " 'test_resnet50_train_cuda',\n",
      " 'test_Super_SloMo_train_cuda']\n",
      "['C:/tools/miniconda3/envs/py39/python.exe', '-Wignore', '-m', 'pytest', '--durations=0', 'test.py', '--ignore_machine_config', '-k', 'test_resnet18_eval_cpu or test_resnet50_eval_cpu or test_Super_SloMo_eval_cpu or test_resnet18_train_cpu or test_resnet50_train_cpu or test_Super_SloMo_train_cpu or test_resnet18_eval_cuda or test_resnet50_eval_cuda or test_Super_SloMo_eval_cuda or test_resnet18_train_cuda or test_resnet50_train_cuda or test_Super_SloMo_train_cuda']\n"
     ]
    }
   ],
   "source": [
    "from subprocess import run\n",
    "from shlex import split\n",
    "import re\n",
    "from pprint import pprint\n",
    "\n",
    "tests = \"test_resnet18_eval_cpu or test_resnet50_eval_cpu or test_Super_SloMo_eval_cpu\"\n",
    "tests += \" or \" + tests.replace(\"eval\", \"train\")\n",
    "tests += \" or \" + tests.replace(\"cpu\", \"cuda\")\n",
    "\n",
    "tests = tests.strip()\n",
    "\n",
    "print(\"tests:\")\n",
    "pprint(tests.replace(\"or\", \"\").split())\n",
    "\n",
    "cmd = split(f\"{PY} -Wignore -m pytest --durations=0 test.py --ignore_machine_config -k '{tests}'\")\n",
    "\n",
    "print(cmd)\n",
    "\n",
    "def with_avx512():\n",
    "    env = os.environ.copy()\n",
    "    env[\"ATEN_CPU_CAPABILITY\"] = \"AVX512\"\n",
    "    env[\"MKL_ENABLE_INSTRUCTIONS\"] = \"AVX512\"\n",
    "    return env\n",
    "    \n",
    "\n",
    "def unset_avx512():\n",
    "    env = os.environ.copy()\n",
    "    env[\"ATEN_CPU_CAPABILITY\"] = \"default\"\n",
    "    env[\"MKL_ENABLE_INSTRUCTIONS\"] = \"AVX\"\n",
    "    return env\n",
    "\n",
    "def show_torch_build_info(env):\n",
    "    print(run(split(f\"{PY} -c 'import torch; print(torch.__config__.show())'\"), env=env, capture_output=True, text=True).stdout)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "72c9811c-a163-4efa-b134-6cb56440c5cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch built with:\n",
      "  - C++ Version: 199711\n",
      "  - MSVC 192829337\n",
      "  - Intel(R) Math Kernel Library Version 2020.0.2 Product Build 20200624 for Intel(R) 64 architecture applications\n",
      "  - Intel(R) MKL-DNN v2.2.3 (Git Hash 7336ca9f055cf1bfa13efb658fe15dc9b41f0740)\n",
      "  - OpenMP 2019\n",
      "  - LAPACK is enabled (usually provided by MKL)\n",
      "  - CPU capability usage: AVX512\n",
      "  - CUDA Runtime 11.3\n",
      "  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_37,code=compute_37\n",
      "  - CuDNN 8.2\n",
      "  - Magma 2.5.4\n",
      "  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.3, CUDNN_VERSION=8.2.0, CXX_COMPILER=C:/w/b/windows/tmp_bin/sccache-cl.exe, CXX_FLAGS=/DWIN32 /D_WINDOWS /GR /EHsc /w /bigobj -DUSE_PTHREADPOOL -openmp:experimental -IC:/w/b/windows/mkl/include -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOCUPTI -DUSE_FBGEMM -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -DEDGE_PROFILER_USE_KINETO, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=1.10.0, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=OFF, USE_NNPACK=OFF, USE_OPENMP=ON, \n",
      "\n",
      "\n",
      "============================= test session starts =============================\n",
      "platform win32 -- Python 3.9.7, pytest-6.2.5, py-1.11.0, pluggy-1.0.0\n",
      "benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)\n",
      "rootdir: C:\\Users\\vadim\\Projects\\benchmark\n",
      "plugins: anyio-3.3.4, benchmark-3.4.1, html-3.1.1, metadata-1.11.0, sorter-0.3.1\n",
      "collected 425 items / 413 deselected / 12 selected\n",
      "\n",
      "test.py s.s.........                                                     [100%]\n",
      "\n",
      "============================== slowest durations ==============================\n",
      "17.33s call     test.py::TestBenchmark::test_resnet50_train_cpu\n",
      "7.42s call     test.py::TestBenchmark::test_resnet50_train_cuda\n",
      "7.24s call     test.py::TestBenchmark::test_Super_SloMo_train_cuda\n",
      "6.93s call     test.py::TestBenchmark::test_Super_SloMo_eval_cuda\n",
      "6.37s call     test.py::TestBenchmark::test_resnet18_train_cuda\n",
      "6.31s call     test.py::TestBenchmark::test_resnet18_train_cpu\n",
      "6.24s call     test.py::TestBenchmark::test_resnet50_eval_cuda\n",
      "5.98s call     test.py::TestBenchmark::test_resnet18_eval_cuda\n",
      "4.04s call     test.py::TestBenchmark::test_resnet50_eval_cpu\n",
      "3.33s call     test.py::TestBenchmark::test_Super_SloMo_eval_cpu\n",
      "2.97s call     test.py::TestBenchmark::test_Super_SloMo_train_cpu\n",
      "2.61s call     test.py::TestBenchmark::test_resnet18_eval_cpu\n",
      "\n",
      "(24 durations < 0.005s hidden.  Use -vv to show these durations.)\n",
      "========== 10 passed, 2 skipped, 413 deselected in 76.92s (0:01:16) ===========\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env = with_avx512()\n",
    "show_torch_build_info(env)\n",
    "with_avx512_out = run(cmd, capture_output=True, text=True, env=env).stdout\n",
    "print(with_avx512_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c6ea6072-7dbc-4343-8b87-606ff345a33b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch built with:\n",
      "  - C++ Version: 199711\n",
      "  - MSVC 192829337\n",
      "  - Intel(R) Math Kernel Library Version 2020.0.2 Product Build 20200624 for Intel(R) 64 architecture applications\n",
      "  - Intel(R) MKL-DNN v2.2.3 (Git Hash 7336ca9f055cf1bfa13efb658fe15dc9b41f0740)\n",
      "  - OpenMP 2019\n",
      "  - LAPACK is enabled (usually provided by MKL)\n",
      "  - CPU capability usage: NO AVX\n",
      "  - CUDA Runtime 11.3\n",
      "  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_37,code=compute_37\n",
      "  - CuDNN 8.2\n",
      "  - Magma 2.5.4\n",
      "  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.3, CUDNN_VERSION=8.2.0, CXX_COMPILER=C:/w/b/windows/tmp_bin/sccache-cl.exe, CXX_FLAGS=/DWIN32 /D_WINDOWS /GR /EHsc /w /bigobj -DUSE_PTHREADPOOL -openmp:experimental -IC:/w/b/windows/mkl/include -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOCUPTI -DUSE_FBGEMM -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -DEDGE_PROFILER_USE_KINETO, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=1.10.0, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=OFF, USE_NNPACK=OFF, USE_OPENMP=ON, \n",
      "\n",
      "\n",
      "============================= test session starts =============================\n",
      "platform win32 -- Python 3.9.7, pytest-6.2.5, py-1.11.0, pluggy-1.0.0\n",
      "benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)\n",
      "rootdir: C:\\Users\\vadim\\Projects\\benchmark\n",
      "plugins: anyio-3.3.4, benchmark-3.4.1, html-3.1.1, metadata-1.11.0, sorter-0.3.1\n",
      "collected 425 items / 413 deselected / 12 selected\n",
      "\n",
      "test.py s.s.........                                                     [100%]\n",
      "\n",
      "============================== slowest durations ==============================\n",
      "17.62s call     test.py::TestBenchmark::test_resnet50_train_cpu\n",
      "7.62s call     test.py::TestBenchmark::test_resnet50_train_cuda\n",
      "7.24s call     test.py::TestBenchmark::test_Super_SloMo_train_cuda\n",
      "7.03s call     test.py::TestBenchmark::test_resnet50_eval_cuda\n",
      "6.90s call     test.py::TestBenchmark::test_resnet18_train_cuda\n",
      "6.89s call     test.py::TestBenchmark::test_Super_SloMo_eval_cuda\n",
      "6.42s call     test.py::TestBenchmark::test_resnet18_train_cpu\n",
      "6.24s call     test.py::TestBenchmark::test_resnet18_eval_cuda\n",
      "4.87s call     test.py::TestBenchmark::test_resnet50_eval_cpu\n",
      "3.12s call     test.py::TestBenchmark::test_Super_SloMo_eval_cpu\n",
      "2.99s call     test.py::TestBenchmark::test_Super_SloMo_train_cpu\n",
      "2.83s call     test.py::TestBenchmark::test_resnet18_eval_cpu\n",
      "\n",
      "(24 durations < 0.005s hidden.  Use -vv to show these durations.)\n",
      "========== 10 passed, 2 skipped, 413 deselected in 79.88s (0:01:19) ===========\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env = unset_avx512()\n",
    "show_torch_build_info(env)\n",
    "without_avx512_out = run(cmd, capture_output=True, text=True, env=env).stdout\n",
    "print(without_avx512_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ff1f1da-f026-4e2c-a514-97e84ef61650",
   "metadata": {},
   "source": [
    "## Beautify results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "80ae7ac4-0383-41e3-bc27-a5cdd39ae226",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           name  with_avx512  without_avx512\n",
      "0     test_Super_SloMo_eval_cpu         3.33            3.12\n",
      "1    test_Super_SloMo_eval_cuda         6.93            6.89\n",
      "2    test_Super_SloMo_train_cpu         2.97            2.99\n",
      "3   test_Super_SloMo_train_cuda         7.24            7.24\n",
      "4        test_resnet18_eval_cpu         2.61            2.83\n",
      "5       test_resnet18_eval_cuda         5.98            6.24\n",
      "6       test_resnet18_train_cpu         6.31            6.42\n",
      "7      test_resnet18_train_cuda         6.37            6.90\n",
      "8        test_resnet50_eval_cpu         4.04            4.87\n",
      "9       test_resnet50_eval_cuda         6.24            7.03\n",
      "10      test_resnet50_train_cpu        17.33           17.62\n",
      "11     test_resnet50_train_cuda         7.42            7.62\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "from natsort import natsorted\n",
    "\n",
    "regexp = re.compile(pattern=r\"(^\\d+\\.\\d+)s.*::(test_.*)\", flags=re.MULTILINE)\n",
    "all_tests = set()\n",
    "with_avx512 = {}\n",
    "without_avx512 = {}\n",
    "\n",
    "for time, name in regexp.findall(with_avx512_out):\n",
    "    all_tests.add(name)\n",
    "    with_avx512[name] = float(time)\n",
    "\n",
    "    \n",
    "for time, name in regexp.findall(without_avx512_out):\n",
    "    all_tests.add(name)\n",
    "    without_avx512[name] = float(time)\n",
    "\n",
    "summary = dict(name=[], with_avx512=[], without_avx512=[])\n",
    "\n",
    "for name in natsorted(all_tests):\n",
    "    summary[\"name\"].append(name)\n",
    "    summary[\"with_avx512\"].append(with_avx512[name])\n",
    "    summary[\"without_avx512\"].append(without_avx512[name])\n",
    "\n",
    "summary_df = pd.DataFrame.from_dict(summary)\n",
    "\n",
    "print(summary_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d5a529c-1474-4bc1-94b7-0be1d264c992",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}