clear_last_traceback.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "25be71f4-4094-4f87-914a-974fd0fb54d5",
   "metadata": {},
   "source": [
    "# Trying to clear object references in the 'last traceback' in IPython\n",
    "\n",
    "IPython holds a reference (in a few places) on the last error and its traceback.\n",
    "This can cause problems, because any object references in any frame in the traceback will not get garbage collected naturally.\n",
    "\n",
    "If we can find the references, we can clean them up (and maybe suggest some improvements to IPython)\n",
    "\n",
    "Prompted by https://mastodon.social/@hannorein/113697884922216639\n",
    "\n",
    "Setup: an object that raises an exception in a method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "31e5952e-6b1a-4aea-90dd-1f7b1dd6ed62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "class Test:\n",
    "    def exception(self):\n",
    "        raise ValueError(\"test\")\n",
    "\n",
    "    def __del__(self):\n",
    "        print(\"freed!\")\n",
    "\n",
    "t = Test()\n",
    "print(sys.getrefcount(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "07959081-8d5c-43bf-b0f1-4a32ccbb1502",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "test",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexception\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[1], line 5\u001b[0m, in \u001b[0;36mTest.exception\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mexception\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m----> 5\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtest\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mValueError\u001b[0m: test"
     ]
    }
   ],
   "source": [
    "t.exception()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2a4237b2-493c-4ac2-87c5-30550fe69beb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(sys.getrefcount(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "06696c8e-9e37-47f8-888e-86342766c5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "del t"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a924a5d5-c806-4363-9a7e-32dafaac70b8",
   "metadata": {},
   "source": [
    "That should have printed 'freed', but it didn't.\n",
    "This is because the last error's traceback still holds a reference on `t`\n",
    "and IPython has references to the traceback in 3 places:\n",
    "\n",
    "1. `exception.__traceback__` on the exception raised (easier than clearing every reference to the Exception itself)\n",
    "2. `sys.last_traceback` for debugger support\n",
    "3. `get_ipython().InteractiveTB.tb` which is a cache value, and probably a bug that the value is still held\n",
    "\n",
    "If we clear these, the refcount goes back to normal and `t` is freed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1d6bfe23-8865-4486-a735-4ac440415762",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "freed!\n"
     ]
    }
   ],
   "source": [
    "# clear held references to the latest traceback\n",
    "import gc\n",
    "\n",
    "sys.last_value.__traceback__ = None\n",
    "sys.last_traceback = None\n",
    "get_ipython().InteractiveTB.tb = None\n",
    "# gc.collect seems to be needed to resolve some cycles somewhere\n",
    "gc.collect();\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "187c14c9-1c8f-410d-a37c-104a82110659",
   "metadata": {},
   "source": [
    "We can use IPython's `post_run_cell` hook to do this automatically on every cell execution,\n",
    "so you can put this in your `ipython_config.py`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c30ea9e6-5aab-4937-9733-33aacff1b7db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# do the above automatically when a cell fails\n",
    "import gc\n",
    "import sys\n",
    "import traceback\n",
    "\n",
    "def clear_last_tb(result):\n",
    "    if result.error_in_exec:\n",
    "        result.error_in_exec.__traceback__ = None\n",
    "        sys.last_traceback = None\n",
    "        get_ipython().InteractiveTB.tb = None\n",
    "        gc.collect()\n",
    "    \n",
    "get_ipython().events.register(\"post_run_cell\", clear_last_tb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3dfd36da-c1e3-4476-8b68-2ddbcaeb28df",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "test",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m t \u001b[38;5;241m=\u001b[39m Test()\n\u001b[0;32m----> 2\u001b[0m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexception\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[1], line 5\u001b[0m, in \u001b[0;36mTest.exception\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mexception\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m----> 5\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtest\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mValueError\u001b[0m: test"
     ]
    }
   ],
   "source": [
    "t = Test()\n",
    "t.exception()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "23ad0b26-c5ed-4cde-b431-c1ef27750a34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "freed!\n"
     ]
    }
   ],
   "source": [
    "print(sys.getrefcount(t))\n",
    "del t"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac56751-df74-4657-b5cb-3a99d237c8d1",
   "metadata": {},
   "source": [
    "Now we have enough information to propose an enhancement to IPython itself, perhaps."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}