brigxt0

How to get a stack trace for each thread in a running python script

Sometimes a python script will simply hang forever with no indication of where things went wrong. Perhaps it's polling a service that will never return a value that allows the program to move forward. Here's a way to see where the program is currently stuck.

Install gdb and pyrasite

Install gdb.

# Redhat, CentOS, etc

	from glob import glob

	import multiprocessing
	from concurrent.futures import ProcessPoolExecutor

	import cv2
	from PIL import Image
	import imagehash

	from tqdm import tqdm

	# client-task-pool.py

	from aiohttp import ClientSession, TCPConnector
	import asyncio
	import sys
	from pypeln.task import TaskPool

	limit = 1000

	# client-async-sem.py

	from aiohttp import ClientSession, TCPConnector
	import asyncio
	import sys

	limit = 1000

	async def fetch(url, session):
	async with session.get(url) as response:

	# timed.sh

	ulimit -n 3000 # increase to avoid "out of file descriptors" error

	python server.py &

	sleep 1 # Wait for server to start

	/usr/bin/time --format "Memory usage: %MKB\tTime: %e seconds\tCPU usage: %P" "$@"

	#!/usr/bin/env python

	"""
	process some data
	"""
	from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
	from joblib import Parallel, delayed

	import os
	import statistics