wip test

bart-perf-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tempfile\n",
    "import unittest\n",
    "from pathlib import Path\n",
    "import torch\n",
    "from transformers import AutoConfig, is_torch_available\n",
    "from transformers.testing_utils import require_torch, require_torch_and_cuda, torch_device, slow\n",
    "\n",
    "from transformers.benchmark.benchmark_utils import (\n",
    "    Benchmark,\n",
    "    Memory,\n",
    "    MemorySummary,\n",
    "    measure_peak_memory_cpu,\n",
    "    start_memory_tracing,\n",
    "    stop_memory_tracing,\n",
    ")\n",
    "\n",
    "if is_torch_available():\n",
    "    from transformers import (\n",
    "        PyTorchBenchmarkArguments,\n",
    "        PyTorchBenchmark,\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unittest\n",
    "self=unittest.TestCase()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hide_input": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: GeForce GTX TITAN X\n"
     ]
    }
   ],
   "source": [
    "import torch.cuda\n",
    "device_name = torch.cuda.get_device_name(0)\n",
    "print(f\"Device: {device_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "        # the intention of these tests is detection of future memory and speed regressions. As the library grows we want to detect if some code suddenly has a substantial impact on either the execution speed or the memory requirements. So at some point we measured the memory and speed results as the baseline and at the future time we check whether they are still about the same, giving 10% leeway for variations\n",
    "        #\n",
    "        # these snapshots were taken around transformers-3.0.2 (8edfaaa81b9995cedea2f8805e4c18c2b6cb5bfc)\n",
    "        mem_base = {\n",
    "            'init': 533004288,\n",
    "            'fwd': 563412992,  \n",
    "            'fwd_bwd': 573898752,\n",
    "        }\n",
    "        time_base = {\n",
    "            'init': 0.038,\n",
    "            'fwd': 0.03,  \n",
    "            'fwd_bwd': 0.06,\n",
    "        }\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "        #MODEL_ID = \"facebook/bart-base\"\n",
    "        MODEL_ID = \"sshleifer/tinier_bart\"\n",
    "        ss = 8\n",
    "        bs = 1\n",
    "\n",
    "        # benchmark __init__\n",
    "        benchmark_args = PyTorchBenchmarkArguments(\n",
    "            models=[MODEL_ID],\n",
    "            training=False,\n",
    "            no_inference=False,\n",
    "            sequence_lengths=[ss],\n",
    "            batch_sizes=[bs],\n",
    "            no_multi_process=False,\n",
    "            no_cuda=False,\n",
    "            no_speed=False,\n",
    "        )\n",
    "        benchmark = PyTorchBenchmark(benchmark_args)\n",
    "        config = benchmark.config_dict[MODEL_ID]\n",
    "        model_class = config.architectures[0]\n",
    "        transformers_module = __import__(\"transformers\", fromlist=[model_class])\n",
    "        model_cls = getattr(transformers_module, model_class)\n",
    "        device = benchmark.args.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "        # \n",
    "        def measure(func):\n",
    "            mem, _ = benchmark._measure_memory(func)\n",
    "            print(mem.bytes)\n",
    "            time = benchmark._measure_speed(func)\n",
    "            print(time)\n",
    "            return mem.bytes, time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "        def run_init():\n",
    "            model = model_cls(config)\n",
    "            return model\n",
    "            \n",
    "        # pre-run once in case a model gets downloaded for the first time\n",
    "        model = run_init()\n",
    "        \n",
    "        # prep for train\n",
    "        model.train();\n",
    "        model.to(device);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "        vocab_size = config.vocab_size if hasattr(config, \"vocab_size\") else config.encoder.vocab_size\n",
    "        input_ids = torch.randint(vocab_size, (bs, ss), dtype=torch.long, device=benchmark.args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "533004288\n",
      "0.037460965100035534\n"
     ]
    }
   ],
   "source": [
    "        # benchmark: __init__\n",
    "        mem, time = measure(run_init)\n",
    "        self.assertLess(mem, mem_base['init']*1.1)\n",
    "        self.assertLess(time, time_base['init']*1.1)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "542441472\n",
      "0.002792727100313641\n"
     ]
    }
   ],
   "source": [
    "        # a single forward pass. \n",
    "        def run_fwd():\n",
    "            loss = model(input_ids, labels=input_ids)[0]\n",
    "        mem, time = measure(run_fwd)\n",
    "        self.assertLess(mem, mem_base['fwd']*1.1)\n",
    "        self.assertLess(time, time_base['fwd']*1.1)                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550830080\n",
      "0.005676483298884705\n"
     ]
    }
   ],
   "source": [
    "        # a single forward and backward pass.\n",
    "        def run_fwd_bwd():\n",
    "            loss = model(input_ids, labels=input_ids)[0]        \n",
    "            loss.backward()\n",
    "        mem, time = measure(run_fwd_bwd)\n",
    "        self.assertLess(mem, mem_base['fwd_bwd']*1.1)\n",
    "        self.assertLess(time, time_base['fwd_bwd']*1.1)                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}