Last active
December 9, 2022 14:22
-
-
Save PiotrCzapla/90c446929d2d699a8fdce65bc8a95f38 to your computer and use it in GitHub Desktop.
Clean up a memory leak after unhandled exception in jupyter notebooks.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import sys, gc, traceback, collections\n", | |
| "import torch\n", | |
| "import objgraph" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Counter()\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "def print_tensor_count(message=\"\"):\n", | |
| " gc.collect()\n", | |
| " if message:\n", | |
| " print(message)\n", | |
| " print(collections.Counter(v.device if v.numel() else 'empty' for v in objgraph.by_type('torch.Tensor')))\n", | |
| "print_tensor_count()\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def allocate_tensor(should_raise=False): \n", | |
| " x = torch.rand(10000, 10000)\n", | |
| " print_tensor_count('Aafter allocation') \n", | |
| " x *= 2\n", | |
| " if should_raise:\n", | |
| " raise Exception('I am a bad function')\n", | |
| " return x\n", | |
| "\n", | |
| "def allocate_tensor_and_raise():\n", | |
| " try:\n", | |
| " allocate_tensor(True)\n", | |
| " finally:\n", | |
| " print_tensor_count('Finally block')\n", | |
| " " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def clean_up():\n", | |
| " import sys, gc, traceback\n", | |
| " if hasattr(sys, 'last_traceback'):\n", | |
| " traceback.clear_frames(sys.last_traceback)\n", | |
| " delattr(sys, 'last_traceback')\n", | |
| " if hasattr(sys, 'last_type'): delattr(sys, 'last_type')\n", | |
| " if hasattr(sys, 'last_value'): delattr(sys, 'last_value')\n", | |
| " gc.collect() \n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Aafter allocation\n", | |
| "Counter({device(type='cpu'): 1})\n", | |
| "Outside\n", | |
| "Counter()\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "allocate_tensor(should_raise=False)\n", | |
| "print_tensor_count('Outside')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Aafter allocation\n", | |
| "Counter({device(type='cpu'): 1})\n", | |
| "Finally block\n", | |
| "Counter({device(type='cpu'): 1})\n" | |
| ] | |
| }, | |
| { | |
| "ename": "Exception", | |
| "evalue": "I am a bad function", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 6\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m allocate_tensor_and_raise()\n", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 6\u001b[0m in \u001b[0;36mallocate_tensor_and_raise\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mallocate_tensor_and_raise\u001b[39m():\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m allocate_tensor(\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m print_tensor_count(\u001b[39m'\u001b[39m\u001b[39mFinally block\u001b[39m\u001b[39m'\u001b[39m)\n", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 6\u001b[0m in \u001b[0;36mallocate_tensor\u001b[0;34m(should_raise)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m x \u001b[39m*\u001b[39m\u001b[39m=\u001b[39m \u001b[39m2\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mif\u001b[39;00m should_raise:\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mI am a bad function\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#X22sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39mreturn\u001b[39;00m x\n", | |
| "\u001b[0;31mException\u001b[0m: I am a bad function" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "allocate_tensor_and_raise()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Is the tensor still there?\n", | |
| "Counter({device(type='cpu'): 1})\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "print_tensor_count('Is the tensor still there?')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Now let's see why the tensor is allocated using the objgraph module." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# !pip install objgraph\n", | |
| "# install dot" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 7.0.4 (20221203.1631)\n -->\n<!-- Title: ObjectGraph Pages: 1 -->\n<svg width=\"610pt\" height=\"791pt\"\n viewBox=\"0.00 0.00 610.00 791.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 787)\">\n<title>ObjectGraph</title>\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-787 606,-787 606,4 -4,4\"/>\n<!-- o5093649728 -->\n<g id=\"node1\" class=\"node\">\n<title>o5093649728</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"453,-38 226,-38 226,0 453,0 453,-38\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-22.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"red\">Tensor</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-7.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"red\">tensor([[0.6787, 1.9544, 1.4737,  ..., 0</text>\n</g>\n<!-- o4354156784 -->\n<g id=\"node2\" class=\"node\">\n<title>o4354156784</title>\n<polygon fill=\"#e8e8e8\" stroke=\"black\" points=\"587.5,-216 91.5,-216 91.5,-178 587.5,-178 587.5,-216\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-200.8\" font-family=\"Times,serif\" font-size=\"14.00\">frame</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-185.8\" font-family=\"Times,serif\" font-size=\"14.00\">/var/folders/q_/6ypbtk8976j5j9zhzn30df440000gn/T/ipykernel_83323/1095150491.py:6</text>\n</g>\n<!-- o4354156784->o5093649728 -->\n<g id=\"edge1\" class=\"edge\">\n<title>o4354156784->o5093649728</title>\n<path fill=\"none\" stroke=\"black\" d=\"M326.1,-177.76C317.19,-164.44 306.27,-145.54 301.5,-127 297.29,-110.64 297.29,-105.36 301.5,-89 305.23,-74.51 312.7,-59.81 320.05,-47.72\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"322.85,-49.83 325.29,-39.52 316.95,-46.06 322.85,-49.83\"/>\n</g>\n<!-- o5622562624 -->\n<g id=\"node3\" class=\"node\">\n<title>o5622562624</title>\n<polygon fill=\"#e8e8e8\" stroke=\"black\" points=\"368,-127 311,-127 311,-89 368,-89 368,-127\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">dict</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">2 items</text>\n</g>\n<!-- o4354156784->o5622562624 -->\n<g id=\"edge4\" class=\"edge\">\n<title>o4354156784->o5622562624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-177.55C339.5,-166.25 339.5,-151.52 339.5,-138.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-138.94 339.5,-128.94 336,-138.94 343,-138.94\"/>\n<text text-anchor=\"middle\" x=\"361\" y=\"-148.8\" font-family=\"Times,serif\" font-size=\"14.00\">f_locals</text>\n</g>\n<!-- o5622562624->o5093649728 -->\n<g id=\"edge2\" class=\"edge\">\n<title>o5622562624->o5093649728</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-88.55C339.5,-77.25 339.5,-62.52 339.5,-49.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-49.94 339.5,-39.94 336,-49.94 343,-49.94\"/>\n<text text-anchor=\"middle\" x=\"343\" y=\"-59.8\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n</g>\n<!-- o4353580288 -->\n<g id=\"node4\" class=\"node\">\n<title>o4353580288</title>\n<polygon fill=\"#d2d2d2\" stroke=\"black\" points=\"445.5,-291 233.5,-291 233.5,-253 445.5,-253 445.5,-291\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-275.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-260.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x1037e5d00></text>\n</g>\n<!-- o4353580288->o4354156784 -->\n<g id=\"edge3\" class=\"edge\">\n<title>o4353580288->o4354156784</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-252.58C339.5,-245.06 339.5,-236.17 339.5,-227.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-227.93 339.5,-217.93 336,-227.93 343,-227.93\"/>\n</g>\n<!-- o5088865280 -->\n<g id=\"node5\" class=\"node\">\n<title>o5088865280</title>\n<polygon fill=\"#bcbcbc\" stroke=\"black\" points=\"444,-366 235,-366 235,-328 444,-328 444,-366\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-350.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-335.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x12f51ec00></text>\n</g>\n<!-- o5088865280->o4353580288 -->\n<g id=\"edge5\" class=\"edge\">\n<title>o5088865280->o4353580288</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-327.58C339.5,-320.06 339.5,-311.17 339.5,-302.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-302.93 339.5,-292.93 336,-302.93 343,-302.93\"/>\n</g>\n<!-- o5240188288 -->\n<g id=\"node6\" class=\"node\">\n<title>o5240188288</title>\n<polygon fill=\"#a5a5a5\" stroke=\"black\" points=\"445.5,-441 233.5,-441 233.5,-403 445.5,-403 445.5,-441\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-425.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-410.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x13856ed80></text>\n</g>\n<!-- o5240188288->o5088865280 -->\n<g id=\"edge6\" class=\"edge\">\n<title>o5240188288->o5088865280</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-402.58C339.5,-395.06 339.5,-386.17 339.5,-377.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-377.93 339.5,-367.93 336,-377.93 343,-377.93\"/>\n</g>\n<!-- o5096634624 -->\n<g id=\"node7\" class=\"node\">\n<title>o5096634624</title>\n<polygon fill=\"#8f8f8f\" stroke=\"black\" points=\"444.5,-516 234.5,-516 234.5,-478 444.5,-478 444.5,-516\"/>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-500.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"339.5\" y=\"-485.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x12fc87900></text>\n</g>\n<!-- o5096634624->o5240188288 -->\n<g id=\"edge7\" class=\"edge\">\n<title>o5096634624->o5240188288</title>\n<path fill=\"none\" stroke=\"black\" d=\"M339.5,-477.58C339.5,-470.06 339.5,-461.17 339.5,-452.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"343,-452.93 339.5,-442.93 336,-452.93 343,-452.93\"/>\n</g>\n<!-- o4304662080 -->\n<g id=\"node8\" class=\"node\">\n<title>o4304662080</title>\n<polygon fill=\"#797979\" stroke=\"black\" points=\"328.5,-694 264.5,-694 264.5,-656 328.5,-656 328.5,-694\"/>\n<text text-anchor=\"middle\" x=\"296.5\" y=\"-678.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"296.5\" y=\"-663.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">99 items</text>\n</g>\n<!-- o4304662080->o5096634624 -->\n<g id=\"edge8\" class=\"edge\">\n<title>o4304662080->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M308.36,-655.51C311.42,-650.05 314.43,-643.94 316.5,-638 329.32,-601.16 335.08,-556.32 337.61,-527.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"341.09,-527.92 338.39,-517.68 334.11,-527.37 341.09,-527.92\"/>\n<text text-anchor=\"middle\" x=\"372.5\" y=\"-582.3\" font-family=\"Times,serif\" font-size=\"14.00\">last_traceback</text>\n</g>\n<!-- o5085571696 -->\n<g id=\"node10\" class=\"node\">\n<title>o5085571696</title>\n<polygon fill=\"#797979\" stroke=\"black\" points=\"303.5,-605 111.5,-605 111.5,-567 303.5,-567 303.5,-605\"/>\n<text text-anchor=\"middle\" x=\"207.5\" y=\"-589.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">Exception</text>\n<text text-anchor=\"middle\" x=\"207.5\" y=\"-574.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">Exception('I am a bad function')</text>\n</g>\n<!-- o4304662080->o5085571696 -->\n<g id=\"edge13\" class=\"edge\">\n<title>o4304662080->o5085571696</title>\n<path fill=\"none\" stroke=\"black\" d=\"M277.63,-655.55C265,-643.21 248.2,-626.78 234.21,-613.11\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"237.03,-610.97 227.43,-606.48 232.13,-615.98 237.03,-610.97\"/>\n<text text-anchor=\"middle\" x=\"285\" y=\"-626.8\" font-family=\"Times,serif\" font-size=\"14.00\">last_value</text>\n</g>\n<!-- o4353904832 -->\n<g id=\"node9\" class=\"node\">\n<title>o4353904832</title>\n<polygon fill=\"#797979\" stroke=\"black\" points=\"506.5,-605 442.5,-605 442.5,-567 506.5,-567 506.5,-605\"/>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-589.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-574.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">22 items</text>\n</g>\n<!-- o4353904832->o5096634624 -->\n<g id=\"edge9\" class=\"edge\">\n<title>o4353904832->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M445.87,-566.55C426,-553.75 399.32,-536.55 377.65,-522.59\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"379.66,-519.72 369.36,-517.24 375.87,-525.6 379.66,-519.72\"/>\n<text text-anchor=\"middle\" x=\"421\" y=\"-537.8\" font-family=\"Times,serif\" font-size=\"14.00\">tb</text>\n</g>\n<!-- o5085571696->o5096634624 -->\n<g id=\"edge10\" class=\"edge\">\n<title>o5085571696->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M235.49,-566.55C254.92,-553.75 281.01,-536.55 302.2,-522.59\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"303.86,-525.68 310.28,-517.26 300.01,-519.84 303.86,-525.68\"/>\n</g>\n<!-- o4304708176 -->\n<g id=\"node11\" class=\"node\">\n<title>o4304708176</title>\n<polygon fill=\"#136200\" stroke=\"black\" points=\"325.5,-783 267.5,-783 267.5,-745 325.5,-745 325.5,-783\"/>\n<text text-anchor=\"middle\" x=\"296.5\" y=\"-767.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">module</text>\n<text text-anchor=\"middle\" x=\"296.5\" y=\"-752.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">sys</text>\n</g>\n<!-- o4304708176->o4304662080 -->\n<g id=\"edge11\" class=\"edge\">\n<title>o4304708176->o4304662080</title>\n<path fill=\"none\" stroke=\"black\" d=\"M296.5,-744.55C296.5,-733.25 296.5,-718.52 296.5,-705.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"300,-705.94 296.5,-695.94 293,-705.94 300,-705.94\"/>\n<text text-anchor=\"middle\" x=\"320.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">__dict__</text>\n</g>\n<!-- o4353649744 -->\n<g id=\"node12\" class=\"node\">\n<title>o4353649744</title>\n<polygon fill=\"#626262\" stroke=\"black\" points=\"602,-694 347,-694 347,-656 602,-656 602,-694\"/>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-678.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">AutoFormattedTB</text>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-663.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\"><IPython.core.ultratb.AutoFormattedTB ob</text>\n</g>\n<!-- o4353649744->o4353904832 -->\n<g id=\"edge12\" class=\"edge\">\n<title>o4353649744->o4353904832</title>\n<path fill=\"none\" stroke=\"black\" d=\"M474.5,-655.55C474.5,-644.25 474.5,-629.52 474.5,-616.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"478,-616.94 474.5,-606.94 471,-616.94 478,-616.94\"/>\n<text text-anchor=\"middle\" x=\"498.5\" y=\"-626.8\" font-family=\"Times,serif\" font-size=\"14.00\">__dict__</text>\n</g>\n<!-- o5086678208 -->\n<g id=\"node13\" class=\"node\">\n<title>o5086678208</title>\n<polygon fill=\"#626262\" stroke=\"black\" points=\"153,-694 96,-694 96,-656 153,-656 153,-694\"/>\n<text text-anchor=\"middle\" x=\"124.5\" y=\"-678.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"124.5\" y=\"-663.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">3 items</text>\n</g>\n<!-- o5086678208->o5085571696 -->\n<g id=\"edge14\" class=\"edge\">\n<title>o5086678208->o5085571696</title>\n<path fill=\"none\" stroke=\"black\" d=\"M134.44,-655.89C140.59,-645.6 149.09,-632.85 158.5,-623 162,-619.34 165.91,-615.78 169.95,-612.43\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"171.99,-615.27 177.71,-606.35 167.68,-609.76 171.99,-615.27\"/>\n<text text-anchor=\"middle\" x=\"196.5\" y=\"-626.8\" font-family=\"Times,serif\" font-size=\"14.00\">error_in_exec</text>\n</g>\n<!-- o4353637376 -->\n<g id=\"node14\" class=\"node\">\n<title>o4353637376</title>\n<polygon fill=\"#4c4c4c\" stroke=\"black\" points=\"506.5,-783 442.5,-783 442.5,-745 506.5,-745 506.5,-783\"/>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-767.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"474.5\" y=\"-752.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">47 items</text>\n</g>\n<!-- o4353637376->o4353649744 -->\n<g id=\"edge15\" class=\"edge\">\n<title>o4353637376->o4353649744</title>\n<path fill=\"none\" stroke=\"black\" d=\"M474.5,-744.55C474.5,-733.25 474.5,-718.52 474.5,-705.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"478,-705.94 474.5,-695.94 471,-705.94 478,-705.94\"/>\n<text text-anchor=\"middle\" x=\"512.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">InteractiveTB</text>\n</g>\n<!-- o4359428880 -->\n<g id=\"node15\" class=\"node\">\n<title>o4359428880</title>\n<polygon fill=\"#4c4c4c\" stroke=\"black\" points=\"249,-783 0,-783 0,-745 249,-745 249,-783\"/>\n<text text-anchor=\"middle\" x=\"124.5\" y=\"-767.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">ExecutionResult</text>\n<text text-anchor=\"middle\" x=\"124.5\" y=\"-752.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\"><ExecutionResult object at 103d79b10, ex</text>\n</g>\n<!-- o4359428880->o5086678208 -->\n<g id=\"edge16\" class=\"edge\">\n<title>o4359428880->o5086678208</title>\n<path fill=\"none\" stroke=\"black\" d=\"M124.5,-744.55C124.5,-733.25 124.5,-718.52 124.5,-705.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"128,-705.94 124.5,-695.94 121,-705.94 128,-705.94\"/>\n<text text-anchor=\"middle\" x=\"148.5\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">__dict__</text>\n</g>\n</g>\n</svg>\n", | |
| "text/plain": [ | |
| "<graphviz.sources.Source at 0x103cf25c0>" | |
| ] | |
| }, | |
| "execution_count": 9, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "import objgraph\n", | |
| "objgraph.show_backrefs(objgraph.by_type('torch.Tensor'), 8)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "As expected the tensor is being hold by `sys.last_traceback` object but what is unexpected is that this trace is being hold in `ExecutionResult` as well as in `AutoFormattedTB` that is being part of global kernel object that can be accessed by `get_ipython()`. \n", | |
| "So simply clearing `sys` won't help. But let's try `traceback.clear_frames`. As per docs" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[0;31mSignature:\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | |
| "\u001b[0;31mDocstring:\u001b[0m Clear all references to local variables in the frames of a traceback.\n", | |
| "\u001b[0;31mFile:\u001b[0m /opt/homebrew/Caskroom/mambaforge/base/envs/sd/lib/python3.10/traceback.py\n", | |
| "\u001b[0;31mType:\u001b[0m function\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "traceback.clear_frames?" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 11, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "traceback.clear_frames(sys.last_traceback)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 12, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Is the tensor still there?\n", | |
| "Counter({device(type='cpu'): 1})\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "print_tensor_count('Is the tensor still there?')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "That is a bit unexpected isn't it? It seems that in this case the local frame of `allocate_tensor` isn't not removing it's variables for some reason, at least in python3.10" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 7.0.4 (20221203.1631)\n -->\n<!-- Title: ObjectGraph Pages: 1 -->\n<svg width=\"530pt\" height=\"791pt\"\n viewBox=\"0.00 0.00 529.50 791.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 787)\">\n<title>ObjectGraph</title>\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-787 525.5,-787 525.5,4 -4,4\"/>\n<!-- o5093649728 -->\n<g id=\"node1\" class=\"node\">\n<title>o5093649728</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"361.5,-38 134.5,-38 134.5,0 361.5,0 361.5,-38\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-22.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"red\">Tensor</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-7.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"red\">tensor([[0.6787, 1.9544, 1.4737,  ..., 0</text>\n</g>\n<!-- o5622562624 -->\n<g id=\"node2\" class=\"node\">\n<title>o5622562624</title>\n<polygon fill=\"#e8e8e8\" stroke=\"black\" points=\"276.5,-127 219.5,-127 219.5,-89 276.5,-89 276.5,-127\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-111.8\" font-family=\"Times,serif\" font-size=\"14.00\">dict</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">2 items</text>\n</g>\n<!-- o5622562624->o5093649728 -->\n<g id=\"edge1\" class=\"edge\">\n<title>o5622562624->o5093649728</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-88.55C248,-77.25 248,-62.52 248,-49.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-49.94 248,-39.94 244.5,-49.94 251.5,-49.94\"/>\n<text text-anchor=\"middle\" x=\"251.5\" y=\"-59.8\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n</g>\n<!-- o4354156784 -->\n<g id=\"node3\" class=\"node\">\n<title>o4354156784</title>\n<polygon fill=\"#d2d2d2\" stroke=\"black\" points=\"496,-216 0,-216 0,-178 496,-178 496,-216\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-200.8\" font-family=\"Times,serif\" font-size=\"14.00\">frame</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-185.8\" font-family=\"Times,serif\" font-size=\"14.00\">/var/folders/q_/6ypbtk8976j5j9zhzn30df440000gn/T/ipykernel_83323/1095150491.py:6</text>\n</g>\n<!-- o4354156784->o5622562624 -->\n<g id=\"edge2\" class=\"edge\">\n<title>o4354156784->o5622562624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-177.55C248,-166.25 248,-151.52 248,-138.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-138.94 248,-128.94 244.5,-138.94 251.5,-138.94\"/>\n<text text-anchor=\"middle\" x=\"269.5\" y=\"-148.8\" font-family=\"Times,serif\" font-size=\"14.00\">f_locals</text>\n</g>\n<!-- o4353580288 -->\n<g id=\"node4\" class=\"node\">\n<title>o4353580288</title>\n<polygon fill=\"#bcbcbc\" stroke=\"black\" points=\"354,-291 142,-291 142,-253 354,-253 354,-291\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-275.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-260.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x1037e5d00></text>\n</g>\n<!-- o4353580288->o4354156784 -->\n<g id=\"edge3\" class=\"edge\">\n<title>o4353580288->o4354156784</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-252.58C248,-245.06 248,-236.17 248,-227.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-227.93 248,-217.93 244.5,-227.93 251.5,-227.93\"/>\n</g>\n<!-- o5088865280 -->\n<g id=\"node5\" class=\"node\">\n<title>o5088865280</title>\n<polygon fill=\"#a5a5a5\" stroke=\"black\" points=\"352.5,-366 143.5,-366 143.5,-328 352.5,-328 352.5,-366\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-350.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-335.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x12f51ec00></text>\n</g>\n<!-- o5088865280->o4353580288 -->\n<g id=\"edge4\" class=\"edge\">\n<title>o5088865280->o4353580288</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-327.58C248,-320.06 248,-311.17 248,-302.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-302.93 248,-292.93 244.5,-302.93 251.5,-302.93\"/>\n</g>\n<!-- o5240188288 -->\n<g id=\"node6\" class=\"node\">\n<title>o5240188288</title>\n<polygon fill=\"#8f8f8f\" stroke=\"black\" points=\"354,-441 142,-441 142,-403 354,-403 354,-441\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-425.8\" font-family=\"Times,serif\" font-size=\"14.00\">traceback</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-410.8\" font-family=\"Times,serif\" font-size=\"14.00\"><traceback object at 0x13856ed80></text>\n</g>\n<!-- o5240188288->o5088865280 -->\n<g id=\"edge5\" class=\"edge\">\n<title>o5240188288->o5088865280</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-402.58C248,-395.06 248,-386.17 248,-377.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-377.93 248,-367.93 244.5,-377.93 251.5,-377.93\"/>\n</g>\n<!-- o5096634624 -->\n<g id=\"node7\" class=\"node\">\n<title>o5096634624</title>\n<polygon fill=\"#797979\" stroke=\"black\" points=\"353,-516 143,-516 143,-478 353,-478 353,-516\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-500.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">traceback</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-485.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\"><traceback object at 0x12fc87900></text>\n</g>\n<!-- o5096634624->o5240188288 -->\n<g id=\"edge6\" class=\"edge\">\n<title>o5096634624->o5240188288</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-477.58C248,-470.06 248,-461.17 248,-452.77\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-452.93 248,-442.93 244.5,-452.93 251.5,-452.93\"/>\n</g>\n<!-- o4304662080 -->\n<g id=\"node8\" class=\"node\">\n<title>o4304662080</title>\n<polygon fill=\"#626262\" stroke=\"black\" points=\"168,-694 104,-694 104,-656 168,-656 168,-694\"/>\n<text text-anchor=\"middle\" x=\"136\" y=\"-678.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"136\" y=\"-663.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">99 items</text>\n</g>\n<!-- o4304662080->o5096634624 -->\n<g id=\"edge7\" class=\"edge\">\n<title>o4304662080->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M109.05,-655.68C80.93,-634.26 43.69,-597.8 65,-567 81.03,-543.83 105.89,-528.43 132.18,-518.19\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"133.08,-521.59 141.29,-514.9 130.69,-515.01 133.08,-521.59\"/>\n<text text-anchor=\"middle\" x=\"104\" y=\"-582.3\" font-family=\"Times,serif\" font-size=\"14.00\">last_traceback</text>\n</g>\n<!-- o5085571696 -->\n<g id=\"node10\" class=\"node\">\n<title>o5085571696</title>\n<polygon fill=\"#626262\" stroke=\"black\" points=\"344,-605 152,-605 152,-567 344,-567 344,-605\"/>\n<text text-anchor=\"middle\" x=\"248\" y=\"-589.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">Exception</text>\n<text text-anchor=\"middle\" x=\"248\" y=\"-574.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">Exception('I am a bad function')</text>\n</g>\n<!-- o4304662080->o5085571696 -->\n<g id=\"edge12\" class=\"edge\">\n<title>o4304662080->o5085571696</title>\n<path fill=\"none\" stroke=\"black\" d=\"M159.75,-655.55C175.94,-642.98 197.58,-626.17 215.37,-612.34\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"217.34,-615.25 223.09,-606.35 213.04,-609.72 217.34,-615.25\"/>\n<text text-anchor=\"middle\" x=\"226.5\" y=\"-626.8\" font-family=\"Times,serif\" font-size=\"14.00\">last_value</text>\n</g>\n<!-- o4353904832 -->\n<g id=\"node9\" class=\"node\">\n<title>o4353904832</title>\n<polygon fill=\"#626262\" stroke=\"black\" points=\"426,-605 362,-605 362,-567 426,-567 426,-605\"/>\n<text text-anchor=\"middle\" x=\"394\" y=\"-589.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">dict</text>\n<text text-anchor=\"middle\" x=\"394\" y=\"-574.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">22 items</text>\n</g>\n<!-- o4353904832->o5096634624 -->\n<g id=\"edge8\" class=\"edge\">\n<title>o4353904832->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M363.04,-566.55C341.36,-553.63 312.17,-536.24 288.63,-522.21\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"290.6,-519.31 280.22,-517.2 287.02,-525.32 290.6,-519.31\"/>\n<text text-anchor=\"middle\" x=\"335.5\" y=\"-537.8\" font-family=\"Times,serif\" font-size=\"14.00\">tb</text>\n</g>\n<!-- o5085571696->o5096634624 -->\n<g id=\"edge9\" class=\"edge\">\n<title>o5085571696->o5096634624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M248,-566.55C248,-555.25 248,-540.52 248,-527.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"251.5,-527.94 248,-517.94 244.5,-527.94 251.5,-527.94\"/>\n</g>\n<!-- o4304708176 -->\n<g id=\"node11\" class=\"node\">\n<title>o4304708176</title>\n<polygon fill=\"#0f4c00\" stroke=\"black\" points=\"165,-783 107,-783 107,-745 165,-745 165,-783\"/>\n<text text-anchor=\"middle\" x=\"136\" y=\"-767.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">module</text>\n<text text-anchor=\"middle\" x=\"136\" y=\"-752.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">sys</text>\n</g>\n<!-- o4304708176->o4304662080 -->\n<g id=\"edge10\" class=\"edge\">\n<title>o4304708176->o4304662080</title>\n<path fill=\"none\" stroke=\"black\" d=\"M136,-744.55C136,-733.25 136,-718.52 136,-705.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"139.5,-705.94 136,-695.94 132.5,-705.94 139.5,-705.94\"/>\n<text text-anchor=\"middle\" x=\"160\" y=\"-715.8\" font-family=\"Times,serif\" font-size=\"14.00\">__dict__</text>\n</g>\n<!-- o4353649744 -->\n<g id=\"node12\" class=\"node\">\n<title>o4353649744</title>\n<polygon fill=\"#4c4c4c\" stroke=\"black\" points=\"521.5,-694 266.5,-694 266.5,-656 521.5,-656 521.5,-694\"/>\n<text text-anchor=\"middle\" x=\"394\" y=\"-678.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\">AutoFormattedTB</text>\n<text text-anchor=\"middle\" x=\"394\" y=\"-663.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"white\"><IPython.core.ultratb.AutoFormattedTB ob</text>\n</g>\n<!-- o4353649744->o4353904832 -->\n<g id=\"edge11\" class=\"edge\">\n<title>o4353649744->o4353904832</title>\n<path fill=\"none\" stroke=\"black\" d=\"M394,-655.55C394,-644.25 394,-629.52 394,-616.61\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"397.5,-616.94 394,-606.94 390.5,-616.94 397.5,-616.94\"/>\n<text text-anchor=\"middle\" x=\"418\" y=\"-626.8\" font-family=\"Times,serif\" font-size=\"14.00\">__dict__</text>\n</g>\n</g>\n</svg>\n", | |
| "text/plain": [ | |
| "<graphviz.sources.Source at 0x103a1f3a0>" | |
| ] | |
| }, | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "objgraph.show_backrefs(objgraph.by_type('torch.Tensor'), 8)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "What is good it seems that further execution of the cells removed the backreference from ExecutionResults so we are left with, following options:\n", | |
| "1. Replace the traceback by something that does not hold our tensors have you seen `1/0` - in code (that is what it do), let's try it. \n", | |
| "2. Cut the traceback chain as close to the head as possible. That would be at `sys.last_traceback.tb_next = None`\n", | |
| "3. Remove the traceback from all places it is being stored. " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 18, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "ename": "ZeroDivisionError", | |
| "evalue": "division by zero", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mZeroDivisionError\u001b[0m Traceback (most recent call last)", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 18\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y104sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mthis_should_disappear_from_frame_once_deleted\u001b[39;00m(): \u001b[39m.\u001b[39m\u001b[39m.\u001b[39m\u001b[39m.\u001b[39m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y104sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m1\u001b[39;49m\u001b[39m/\u001b[39;49m\u001b[39m0\u001b[39;49m\n", | |
| "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "class this_should_disappear_from_frame_once_deleted(): ...\n", | |
| "1/0 # to replace our last exception with something that does not hold a reference to the tensor. The frame captured here will hold only a reference to globals, not a copy so it isn't a problem." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 19, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Lets confirm that the tensor is really gone.\n", | |
| "Counter()\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "print_tensor_count('Lets confirm that the tensor is really gone.')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 20, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# now let's get rid of the test class and see if it still stays in memory after gc.collect()\n", | |
| "del this_should_disappear_from_frame_once_deleted" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 21, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "(False, False)" | |
| ] | |
| }, | |
| "execution_count": 21, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "gc.collect()\n", | |
| "'this_should_disappear_from_frame_once_deleted' in globals(), 'this_should_disappear_from_frame_once_deleted' in sys.last_traceback.tb_frame.f_globals" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "So 1/0 works pretty well, but unfortunately it stops the notebook execution, so run all won't work! But it is a good thing to remember if you don't have anything at hand.\n", | |
| "We need something that we can call from withing a fastai callback let's explore 2. and 3.\n", | |
| "\n", | |
| "\n", | |
| "## 2. Cut the traceback\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 22, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Aafter allocation\n", | |
| "Counter({device(type='cpu'): 1})\n", | |
| "Finally block\n", | |
| "Counter({device(type='cpu'): 1})\n" | |
| ] | |
| }, | |
| { | |
| "ename": "Exception", | |
| "evalue": "I am a bad function", | |
| "output_type": "error", | |
| "traceback": [ | |
| "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | |
| "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 23\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m allocate_tensor_and_raise()\n", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 23\u001b[0m in \u001b[0;36mallocate_tensor_and_raise\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mallocate_tensor_and_raise\u001b[39m():\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m allocate_tensor(\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m print_tensor_count(\u001b[39m'\u001b[39m\u001b[39mFinally block\u001b[39m\u001b[39m'\u001b[39m)\n", | |
| "\u001b[1;32m/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb Cell 23\u001b[0m in \u001b[0;36mallocate_tensor\u001b[0;34m(should_raise)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m x \u001b[39m*\u001b[39m\u001b[39m=\u001b[39m \u001b[39m2\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mif\u001b[39;00m should_raise:\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mI am a bad function\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39mreturn\u001b[39;00m x\n", | |
| "\u001b[0;31mException\u001b[0m: I am a bad function" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "allocate_tensor_and_raise()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 23, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Is the tensor still there?\n", | |
| "Counter({device(type='cpu'): 1})\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "print_tensor_count('Is the tensor still there?')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 24, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "It should be gone right?\n", | |
| "Counter()\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "sys.last_traceback.tb_next = None\n", | |
| "gc.collect()\n", | |
| "print_tensor_count('It should be gone right?')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Pretty good, the only remaining issue is that we can't get rid of the locals in the upper frame but let's check what they are" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 25, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "{'self': <ipykernel.zmqshell.ZMQInteractiveShell object at 0x1037f5690>, 'code_obj': <code object <cell line: 1> at 0x138593e10, file \"/var/folders/q_/6ypbtk8976j5j9zhzn30df440000gn/T/ipykernel_83323/2892648145.py\", line 1>, 'result': <ExecutionResult object at 1481168c0, execution_count=22 error_before_exec=None error_in_exec=I am a bad function info=<ExecutionInfo object at 148115600, raw_cell=\"allocate_tensor_and_raise()\" store_history=True silent=False shell_futures=True cell_id=vscode-notebook-cell:/Users/pczapla/Workspace/fastai/gpumem/clean_up_after_unhandled_exception.ipynb#Y114sZmlsZQ%3D%3D> result=None>, 'async_': False, '__tracebackhide__': '__ipython_bottom__', 'old_excepthook': <bound method IPKernelApp.excepthook of <ipykernel.kernelapp.IPKernelApp object at 0x100a2ac20>>, 'outflag': True}\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "print(sys.last_traceback.tb_frame.f_locals) " | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "> **This looks like content of jupyter's outer function that called the content of the cell. So there is no references to our variables! So even thought frame_clear() doesn't work too well, cutting exception seems to be doing pretty good job and is much safer than option 3.!**\n", | |
| "\n", | |
| "So we have our clean_up_function function. Next let's test this with real model training, and see if we can clean up all the cuda tensors afer Out Of Memory exception." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3.10.4 ('sd')", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.10.4" | |
| }, | |
| "vscode": { | |
| "interpreter": { | |
| "hash": "89a5cca1a28eb213eb14abdad89b9eabded0d66f27b6c5a9aad0c68445338715" | |
| } | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment