Skip to content

Instantly share code, notes, and snippets.

@Andrei-Pozolotin

Andrei-Pozolotin/shiboken_discard.py

Last active January 15, 2025 15:06
Show Gist options
  • Save Andrei-Pozolotin/5d99c20cfd57b2199d4b1e6fe068fe9a to your computer and use it in GitHub Desktop.
Save Andrei-Pozolotin/5d99c20cfd57b2199d4b1e6fe068fe9a to your computer and use it in GitHub Desktop.
Download ZIP
PySide6: shiboken6 object memory leak https://github.com/FreeCAD/FreeCAD/issues/19066
Raw
shiboken_discard.py
#
#
#
from collections import defaultdict
import ctypes
from dataclasses import dataclass
import gc
import sys
from typing import Any
from typing import TextIO
from typing import Type
from typing import TypeAlias
import shiboken6
Shibo_Object:TypeAlias = shiboken6.Shiboken.Object # type:ignore
TrashObjectCount:TypeAlias = int
class PythonCore_API:
__Py_IncRef = ctypes.pythonapi.Py_IncRef
__Py_IncRef.argtypes = [ctypes.py_object]
__Py_IncRef.restype = None
__Py_DecRef = ctypes.pythonapi.Py_DecRef
__Py_DecRef.argtypes = [ctypes.py_object]
__Py_DecRef.restype = None
@classmethod
def python_extract_ref_count(self,
object_entry:Any,
) -> int:
return sys.getrefcount(object_entry)
@classmethod
def python_increase_ref_count(self,
object_entry:Any,
) -> None:
self.__Py_IncRef(object_entry)
@classmethod
def python_decrease_ref_count(self,
object_entry:Any,
) -> None:
self.__Py_DecRef(object_entry)
class TrashFilter_STD:
@classmethod
def trash_spread_keeper_with_refer_ents(self,
object_list:list[Any],
) -> list[Any]:
raise NotImplementedError
@classmethod
def trash_spread_keeper_with_refer_rers(self,
object_list:list[Any],
) -> list[Any]:
raise NotImplementedError
@classmethod
def trash_filter_leaker_with_refer_ents(self, # faster
object_list:list[Any],
) -> list[Any]:
leaker_ident_set = set(# start full then cull
id(object_entry) for object_entry in object_list
)
grbage_list = gc.get_objects()
object_list_ident = id(object_list)
for grbage_entry in grbage_list:
grbage_entry_ident = id(grbage_entry)
if grbage_entry_ident == object_list_ident: # object_list is also here
continue
refer_ents_list = gc.get_referents(grbage_entry) # minor cpu hog
if len(refer_ents_list) == 0:
continue
refer_ents_ident_set = set(
id(object_entry) for object_entry in refer_ents_list
)
leaker_ident_set.difference_update(refer_ents_ident_set)
leaker_list = list(
object_entry
for object_entry in object_list if id(object_entry) in leaker_ident_set
)
return leaker_list
@classmethod
def trash_filter_leaker_with_refer_rers(self, # slower
object_list:list[Any],
) -> list[Any]:
leaker_list = list()
for object_entry in object_list:
refer_rers_list = gc.get_referrers(object_entry) # major cpu hog
has_real_refer_rers = len(refer_rers_list) > 1 # refer_rers_list is "the one"
if has_real_refer_rers:
continue
leaker_list.append(object_entry)
return leaker_list
class TrashShiboFunc_QT:
@classmethod
def make_object_safe_repr(self,
object_entry:Any,
) -> str:
entry_typer = type(object_entry).__name__
entry_ident = hex(id(object_entry))
entry_repra = f"{entry_typer}() @ {entry_ident}"
return entry_repra
@classmethod
def extract_invalid_shibo_list(self,
base_type:Type[Shibo_Object]=Shibo_Object,
) -> list[Shibo_Object]:
result_list = []
object_list = gc.get_objects()
for object_entry in object_list:
object_type = type(object_entry)
if not issubclass(object_type, base_type):
continue
if shiboken6.isValid(object_entry):
continue
result_list.append(object_entry)
return result_list
__USE_FAST_FILTER = True
@classmethod
def produce_expired_shibo_list(self,
invalid_shibo_list:list[Shibo_Object],
) -> list[Shibo_Object]:
if self.__USE_FAST_FILTER:
expired_shibo_list = TrashFilter_STD.trash_filter_leaker_with_refer_ents(invalid_shibo_list)
else: # this is 3 times slower
expired_shibo_list = TrashFilter_STD.trash_filter_leaker_with_refer_rers(invalid_shibo_list)
return expired_shibo_list
@classmethod
def trash_report_object_list(self,
object_list:list[Any],
report_title:str="Trash Report",
report_file:TextIO=sys.stdout,
use_skip_empty_report:bool=False,
) -> None:
object_count = len(object_list)
if use_skip_empty_report and object_count == 0:
return
report_dict = defaultdict(int)
typer_bound = 0
for object_entry in object_list:
entry_typer = type(object_entry).__name__
typer_bound = max(typer_bound, len(entry_typer))
report_dict[entry_typer] += 1
report_dict = dict(sorted(report_dict.items()))
print("-"*50, file=report_file)
print(f"{report_title}: {object_count=}", file=report_file)
for entry_typer, entry_count in report_dict.items():
entry_typer = entry_typer.ljust(typer_bound)
print(f" {entry_typer} : {entry_count}", file=report_file)
print("-"*50, file=report_file)
@classmethod
def trash_discard_object_list(self,
object_list:list[Any],
report_title:str="Trash Discard",
use_discard_report:bool=False,
report_file:TextIO=sys.stdout,
use_skip_empty_discard:bool=True,
) -> None:
object_count = len(object_list)
if use_skip_empty_discard and object_count == 0:
return
if use_discard_report:
print("-"*50, file=report_file)
print(f"{report_title}: {object_count=}", file=report_file)
local_reference_count = 2 # object_entry, object_list
for object_entry in object_list:
if use_discard_report:
object_repra = self.make_object_safe_repr(object_entry)
print(f" {object_repra}", file=report_file)
while sys.getrefcount(object_entry) > local_reference_count + 1:
PythonCore_API.python_decrease_ref_count(object_entry)
if use_discard_report:
print("-"*50, file=report_file)
@dataclass
class TrashWorkResult_STD:
invalid_shibo_count:int = 0
expired_shibo_count:int = 0
@dataclass
class TrashShiboWork_STD:
use_report_summary:bool = True
use_collector_invocation:bool = False
use_report_object_discard:bool = False
def peform_trash_work(self) -> TrashWorkResult_STD:
gc.collect() # required
invalid_shibo_list = TrashShiboFunc_QT.extract_invalid_shibo_list()
expired_shibo_list = TrashShiboFunc_QT.produce_expired_shibo_list(invalid_shibo_list)
if self.use_report_summary:
TrashShiboFunc_QT.trash_report_object_list(
object_list=invalid_shibo_list,
report_title="PySide Invalid Object Report",
)
TrashShiboFunc_QT.trash_report_object_list(
object_list=expired_shibo_list,
report_title="PySide Expired Object Report",
)
if self.use_collector_invocation:
TrashShiboFunc_QT.trash_discard_object_list(
object_list=expired_shibo_list,
report_title="PySide Expired Discard Report",
use_discard_report=self.use_report_object_discard
)
return TrashWorkResult_STD(
invalid_shibo_count=len(invalid_shibo_list),
expired_shibo_count=len(expired_shibo_list),
)
print("SHIBOKEN DISCARD")
shibo_work = TrashShiboWork_STD(
use_report_summary=True,
use_collector_invocation=True,
use_report_object_discard=False,
)
trash_result = shibo_work.peform_trash_work()
print(f"{trash_result=}")
#
#
#
Raw
shiboken_report.log
09:16:34 --------------------------------------------------
09:16:34 PySide Invalid Object Report: object_count=275
09:16:34 Delegate : 15
09:16:34 QAbstractSpinBox : 4
09:16:34 QCheckBox : 9
09:16:34 QComboBox : 8
09:16:34 QCommonStyle : 1
09:16:34 QDialog : 1
09:16:34 QDialogButtonBox : 1
09:16:34 QDoubleSpinBox : 2
09:16:34 QFormLayout : 4
09:16:34 QFrame : 5
09:16:34 QGridLayout : 14
09:16:34 QGroupBox : 17
09:16:34 QHBoxLayout : 6
09:16:34 QHeaderView : 1
09:16:34 QLabel : 24
09:16:34 QLineEdit : 22
09:16:34 QListWidget : 5
09:16:34 QPlainTextEdit : 1
09:16:34 QPushButton : 29
09:16:34 QStandardItemModel : 15
09:16:34 QTabWidget : 1
09:16:34 QTableView : 15
09:16:34 QTableWidget : 1
09:16:34 QToolBox : 3
09:16:34 QToolButton : 5
09:16:34 QVBoxLayout : 35
09:16:34 QWidget : 31
09:16:34 --------------------------------------------------
09:16:34 --------------------------------------------------
09:16:34 PySide Expired Object Report: object_count=1
09:16:34 QTabWidget : 1
09:16:34 --------------------------------------------------
09:16:34 trash_result=TrashWorkResult_STD(invalid_shibo_count=275, expired_shibo_count=1)
Raw
shiboken_report.py
#
#
#
from collections import defaultdict
import ctypes
from dataclasses import dataclass
import gc
import sys
from typing import Any
from typing import TextIO
from typing import Type
from typing import TypeAlias
import shiboken6
Shibo_Object:TypeAlias = shiboken6.Shiboken.Object # type:ignore
TrashObjectCount:TypeAlias = int
class PythonCore_API:
__Py_IncRef = ctypes.pythonapi.Py_IncRef
__Py_IncRef.argtypes = [ctypes.py_object]
__Py_IncRef.restype = None
__Py_DecRef = ctypes.pythonapi.Py_DecRef
__Py_DecRef.argtypes = [ctypes.py_object]
__Py_DecRef.restype = None
@classmethod
def python_extract_ref_count(self,
object_entry:Any,
) -> int:
return sys.getrefcount(object_entry)
@classmethod
def python_increase_ref_count(self,
object_entry:Any,
) -> None:
self.__Py_IncRef(object_entry)
@classmethod
def python_decrease_ref_count(self,
object_entry:Any,
) -> None:
self.__Py_DecRef(object_entry)
class TrashFilter_STD:
@classmethod
def trash_spread_keeper_with_refer_ents(self,
object_list:list[Any],
) -> list[Any]:
raise NotImplementedError
@classmethod
def trash_spread_keeper_with_refer_rers(self,
object_list:list[Any],
) -> list[Any]:
raise NotImplementedError
@classmethod
def trash_filter_leaker_with_refer_ents(self, # faster
object_list:list[Any],
) -> list[Any]:
leaker_ident_set = set(# start full then cull
id(object_entry) for object_entry in object_list
)
grbage_list = gc.get_objects()
object_list_ident = id(object_list)
for grbage_entry in grbage_list:
grbage_entry_ident = id(grbage_entry)
if grbage_entry_ident == object_list_ident: # object_list is also here
continue
refer_ents_list = gc.get_referents(grbage_entry) # minor cpu hog
if len(refer_ents_list) == 0:
continue
refer_ents_ident_set = set(
id(object_entry) for object_entry in refer_ents_list
)
leaker_ident_set.difference_update(refer_ents_ident_set)
leaker_list = list(
object_entry
for object_entry in object_list if id(object_entry) in leaker_ident_set
)
return leaker_list
@classmethod
def trash_filter_leaker_with_refer_rers(self, # slower
object_list:list[Any],
) -> list[Any]:
leaker_list = list()
for object_entry in object_list:
refer_rers_list = gc.get_referrers(object_entry) # major cpu hog
has_real_refer_rers = len(refer_rers_list) > 1 # refer_rers_list is "the one"
if has_real_refer_rers:
continue
leaker_list.append(object_entry)
return leaker_list
class TrashShiboFunc_QT:
@classmethod
def make_object_safe_repr(self,
object_entry:Any,
) -> str:
entry_typer = type(object_entry).__name__
entry_ident = hex(id(object_entry))
entry_repra = f"{entry_typer}() @ {entry_ident}"
return entry_repra
@classmethod
def extract_invalid_shibo_list(self,
base_type:Type[Shibo_Object]=Shibo_Object,
) -> list[Shibo_Object]:
result_list = []
object_list = gc.get_objects()
for object_entry in object_list:
object_type = type(object_entry)
if not issubclass(object_type, base_type):
continue
if shiboken6.isValid(object_entry):
continue
result_list.append(object_entry)
return result_list
__USE_FAST_FILTER = True
@classmethod
def produce_expired_shibo_list(self,
invalid_shibo_list:list[Shibo_Object],
) -> list[Shibo_Object]:
if self.__USE_FAST_FILTER:
expired_shibo_list = TrashFilter_STD.trash_filter_leaker_with_refer_ents(invalid_shibo_list)
else: # this is 3 times slower
expired_shibo_list = TrashFilter_STD.trash_filter_leaker_with_refer_rers(invalid_shibo_list)
return expired_shibo_list
@classmethod
def trash_report_object_list(self,
object_list:list[Any],
report_title:str="Trash Report",
report_file:TextIO=sys.stdout,
use_skip_empty_report:bool=False,
) -> None:
object_count = len(object_list)
if use_skip_empty_report and object_count == 0:
return
report_dict = defaultdict(int)
typer_bound = 0
for object_entry in object_list:
entry_typer = type(object_entry).__name__
typer_bound = max(typer_bound, len(entry_typer))
report_dict[entry_typer] += 1
report_dict = dict(sorted(report_dict.items()))
print("-"*50, file=report_file)
print(f"{report_title}: {object_count=}", file=report_file)
for entry_typer, entry_count in report_dict.items():
entry_typer = entry_typer.ljust(typer_bound)
print(f" {entry_typer} : {entry_count}", file=report_file)
print("-"*50, file=report_file)
@classmethod
def trash_discard_object_list(self,
object_list:list[Any],
report_title:str="Trash Discard",
use_discard_report:bool=False,
report_file:TextIO=sys.stdout,
use_skip_empty_discard:bool=True,
) -> None:
object_count = len(object_list)
if use_skip_empty_discard and object_count == 0:
return
if use_discard_report:
print("-"*50, file=report_file)
print(f"{report_title}: {object_count=}", file=report_file)
local_reference_count = 2 # object_entry, object_list
for object_entry in object_list:
if use_discard_report:
object_repra = self.make_object_safe_repr(object_entry)
print(f" {object_repra}", file=report_file)
while sys.getrefcount(object_entry) > local_reference_count + 1:
PythonCore_API.python_decrease_ref_count(object_entry)
if use_discard_report:
print("-"*50, file=report_file)
@dataclass
class TrashWorkResult_STD:
invalid_shibo_count:int = 0
expired_shibo_count:int = 0
@dataclass
class TrashShiboWork_STD:
use_report_summary:bool = True
use_collector_invocation:bool = False
use_report_object_discard:bool = False
def peform_trash_work(self) -> TrashWorkResult_STD:
gc.collect() # required
invalid_shibo_list = TrashShiboFunc_QT.extract_invalid_shibo_list()
expired_shibo_list = TrashShiboFunc_QT.produce_expired_shibo_list(invalid_shibo_list)
if self.use_report_summary:
TrashShiboFunc_QT.trash_report_object_list(
object_list=invalid_shibo_list,
report_title="PySide Invalid Object Report",
)
TrashShiboFunc_QT.trash_report_object_list(
object_list=expired_shibo_list,
report_title="PySide Expired Object Report",
)
if self.use_collector_invocation:
TrashShiboFunc_QT.trash_discard_object_list(
object_list=expired_shibo_list,
report_title="PySide Expired Discard Report",
use_discard_report=self.use_report_object_discard
)
return TrashWorkResult_STD(
invalid_shibo_count=len(invalid_shibo_list),
expired_shibo_count=len(expired_shibo_list),
)
print("SHIBOKEN REPORT")
shibo_work = TrashShiboWork_STD(
use_report_summary=True,
use_collector_invocation=False,
use_report_object_discard=False,
)
trash_result = shibo_work.peform_trash_work()
print(f"{trash_result=}")
#
#
#
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment