Skip to content

Instantly share code, notes, and snippets.

@mcchae
Last active August 12, 2020 08:10
Show Gist options
  • Save mcchae/a83c92b5173a262aab6f to your computer and use it in GitHub Desktop.
Save mcchae/a83c92b5173a262aab6f to your computer and use it in GitHub Desktop.
Python guppy memory profiling
#!/usr/bin/env python
#coding=utf8
##########################################################################################
class HeapMon:
"""
guppy 모듈을 이용하여 메모리 Heap의 소모량을 구하는 클래스
.. note:: guppy 모듈 설치 방법
sudo apt-get install python-dev
sudo pip install guppy
"""
#=====================================================================================
def __init__(self):
try:
from guppy import hpy
self.enabled = True
except:
self.enabled = False
if self.enabled:
self._h = hpy()
self.hsize = 0L
self.hdiff = 0L
#=====================================================================================
@staticmethod
def getReadableSize(lv):
"""
'1.23 GB' 처럼 TB(TerraByte), GB(GigaByte), MB(MegaByte), KB(KilloByte) 로 표시
:param lv: int 또는 long 형식의 값
:return: 사람이 읽기 쉬운 형태의 바이트 문자열 값
"""
if not isinstance(lv, (int, long)):
return '0'
if lv >= 1024*1024*1024*1024:
s = "%4.2f TB" % (float(lv)/(1024*1024*1024*1024))
elif lv >= 1024*1024*1024:
s = "%4.2f GB" % (float(lv)/(1024*1024*1024))
elif lv >= 1024*1024:
s = "%4.2f MB" % (float(lv)/(1024*1024))
elif lv >= 1024:
s = "%4.2f KB" % (float(lv)/1024)
else:
s = "%d B" % lv
return s
#=====================================================================================
def __repr__(self):
if not self.enabled:
return 'Not enabled. guppy module not found!'
if self.hdiff > 0:
s = 'Total %s, %s allocated' % \
(self.getReadableSize(self.hsize), self.getReadableSize(self.hdiff))
elif self.hdiff < 0:
s = 'Total %s, %s freed' % \
(self.getReadableSize(self.hsize), self.getReadableSize(-self.hdiff))
else:
s = 'Total %s, not changed' % self.getReadableSize(self.hsize)
return s
#=====================================================================================
def getHeap(self):
"""
현재 메모리 heap 상태 구하기
:return: 메모리 help 결과 (헤더정보 포함 Top 10 결과)
"""
if not self.enabled:
return None
return str(self._h.heap())
#=====================================================================================
def check(self, msg='', gtsize=1024*100):
"""
이 함수를 호출할 때마다 얼마큼의 메모리 크기 변화가 있었는데, 전체 얼마큼의 메모리를
잡고 있는지 나타냄 (함수 종료 직전에 이 함수를 호출하면 얼마나 메모리가 증가했는지
알수 있습니다)
:param msg: 출력 결과에 포함될 메시지 (앞에 포함)
:param gtsize: 메모리 증감분이 이 크기보다 작으면 결과는 빈 문자열
:return: gtsize 크기에 따라 보다 큰 증감분의 메모리 변화가 있으면 그 결과의 문자열 출력
예) "end do_main: Total 37.50 MB, 30.63 MB incresed"
"""
if not self.enabled:
return 'Not enabled. guppy module not found!'
hdr = self.getHeap().split('\n')[0]
chsize = long(hdr.split()[-2])
self.hdiff = chsize - self.hsize
self.hsize = chsize
if abs(self.hdiff) <= gtsize:
return ''
return '%s: %s'% (msg, str(self))
##########################################################################################
def do_main(hm):
print hm.check('start do_main') # 함수를 들어와서 메모리 가감을 찍어봅니다
biglist = []
for i in xrange(1000000):
biglist.append(i) # 백만개의 정수를 담고있는 list를 생성해봅니다
print hm.check('end do_main') # 함수를 종료하기 전에 메모리 가감을 찍어봅니다
##########################################################################################
if __name__ == '__main__':
hm = HeapMon() # 여기에 글로벌로 hm 이라는 메모리 힙 모니터링을 위한 인스턴스를 만들어 놓았습니다.
print hm.check('before do_main') # 메인에서 do_main() 함수 호출전에 최초 메모리 가감을 찍어봅니다
do_main(hm)
print hm.check('after do_main') # do_main() 함수 호출 후에 메모리 가감을 찍어봅니다
#!/usr/bin/env python
#coding=utf8
import gc
import os
##########################################################################################
class HeapMon:
#=====================================================================================
def __init__(self, with_guppy = False):
self.pid = os.getpid()
self.enabled = False
try:
if with_guppy:
from guppy import hpy
self.enabled = True
except:
pass
if self.enabled:
self._h = hpy()
self.hsize = 0L
self.hdiff = 0L
#=====================================================================================
@staticmethod
def getVmRSS(pid):
with open('/proc/%d/status' % pid) as ifp:
for line in ifp:
if 'VmRSS' in line:
eles = line.split()
if eles[-1].lower() == 'kb':
return long(eles[-2]) * 1024
if eles[-1].lower() == 'b':
return long(eles[-2])
if eles[-1].lower() == 'mb':
return long(eles[-2]) * 1024 * 1024
return 0
#=====================================================================================
@staticmethod
def getReadableSize(lv):
if not isinstance(lv, (int, long)):
return '0'
if lv >= 1024*1024*1024*1024:
s = "%4.2f TB" % (float(lv)/(1024*1024*1024*1024))
elif lv >= 1024*1024*1024:
s = "%4.2f GB" % (float(lv)/(1024*1024*1024))
elif lv >= 1024*1024:
s = "%4.2f MB" % (float(lv)/(1024*1024))
elif lv >= 1024:
s = "%4.2f KB" % (float(lv)/1024)
else:
s = "%d B" % lv
return s
#=====================================================================================
def __repr__(self):
# if not self.enabled:
# return 'Not enabled. guppy module not found!'
if self.hdiff > 0:
s = 'Total %s, %s incresed' % \
(self.getReadableSize(self.hsize), self.getReadableSize(self.hdiff))
elif self.hdiff < 0:
s = 'Total %s, %s decresed' % \
(self.getReadableSize(self.hsize), self.getReadableSize(-self.hdiff))
else:
s = 'Total %s, not changed' % self.getReadableSize(self.hsize)
return s
#=====================================================================================
def getHeap(self):
if not self.enabled:
return None
return str(self._h.heap())
#=====================================================================================
def check(self, msg=''):
# if not self.enabled:
# return 'Not enabled. guppy module not found!'
if not self.enabled:
chsize = self.getVmRSS(self.pid)
else:
hdr = self.getHeap().split('\n')[0]
chsize = long(hdr.split()[-2])
self.hdiff = chsize - self.hsize
self.hsize = chsize
return '%s: %s'% (msg, str(self))
##########################################################################################
def main(hm):
print hm.check('Before allocating')
# print 'Before allocating ', rss(),
iterations = 1000000
l = {}
for i in xrange(iterations):
l[i] = ({})
# print 'After allocating ', rss(),
print hm.check('After allocating')
# Ignore optimizations, just try to free whatever possible
# First kill
for i in xrange(iterations):
l[i] = None
print hm.check('After First kill')
# Second kill
l.clear()
print hm.check('After Second kill')
# Third kill
l = None
del l
print hm.check('After Third kill')
# Control shot
gc.collect()
# print 'After free ', rss(),
print hm.check('After free')
##########################################################################################
if __name__ == '__main__':
hm = HeapMon() # 여기에 글로벌로 hm 이라는 메모리 힙 모니터링을 위한 인스턴스를 만들어 놓았습니다.
for i in range(3):
print
main(hm)
@lyj911111
Copy link

윈도우는 안깔리나요?
pip install guppy 했는데 안되네요 ㅠㅠ
파이썬 3.6 사용하고있습니다.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment