Last active
August 12, 2020 08:10
-
-
Save mcchae/a83c92b5173a262aab6f to your computer and use it in GitHub Desktop.
Python guppy memory profiling
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
#coding=utf8 | |
########################################################################################## | |
class HeapMon: | |
""" | |
guppy 모듈을 이용하여 메모리 Heap의 소모량을 구하는 클래스 | |
.. note:: guppy 모듈 설치 방법 | |
sudo apt-get install python-dev | |
sudo pip install guppy | |
""" | |
#===================================================================================== | |
def __init__(self): | |
try: | |
from guppy import hpy | |
self.enabled = True | |
except: | |
self.enabled = False | |
if self.enabled: | |
self._h = hpy() | |
self.hsize = 0L | |
self.hdiff = 0L | |
#===================================================================================== | |
@staticmethod | |
def getReadableSize(lv): | |
""" | |
'1.23 GB' 처럼 TB(TerraByte), GB(GigaByte), MB(MegaByte), KB(KilloByte) 로 표시 | |
:param lv: int 또는 long 형식의 값 | |
:return: 사람이 읽기 쉬운 형태의 바이트 문자열 값 | |
""" | |
if not isinstance(lv, (int, long)): | |
return '0' | |
if lv >= 1024*1024*1024*1024: | |
s = "%4.2f TB" % (float(lv)/(1024*1024*1024*1024)) | |
elif lv >= 1024*1024*1024: | |
s = "%4.2f GB" % (float(lv)/(1024*1024*1024)) | |
elif lv >= 1024*1024: | |
s = "%4.2f MB" % (float(lv)/(1024*1024)) | |
elif lv >= 1024: | |
s = "%4.2f KB" % (float(lv)/1024) | |
else: | |
s = "%d B" % lv | |
return s | |
#===================================================================================== | |
def __repr__(self): | |
if not self.enabled: | |
return 'Not enabled. guppy module not found!' | |
if self.hdiff > 0: | |
s = 'Total %s, %s allocated' % \ | |
(self.getReadableSize(self.hsize), self.getReadableSize(self.hdiff)) | |
elif self.hdiff < 0: | |
s = 'Total %s, %s freed' % \ | |
(self.getReadableSize(self.hsize), self.getReadableSize(-self.hdiff)) | |
else: | |
s = 'Total %s, not changed' % self.getReadableSize(self.hsize) | |
return s | |
#===================================================================================== | |
def getHeap(self): | |
""" | |
현재 메모리 heap 상태 구하기 | |
:return: 메모리 help 결과 (헤더정보 포함 Top 10 결과) | |
""" | |
if not self.enabled: | |
return None | |
return str(self._h.heap()) | |
#===================================================================================== | |
def check(self, msg='', gtsize=1024*100): | |
""" | |
이 함수를 호출할 때마다 얼마큼의 메모리 크기 변화가 있었는데, 전체 얼마큼의 메모리를 | |
잡고 있는지 나타냄 (함수 종료 직전에 이 함수를 호출하면 얼마나 메모리가 증가했는지 | |
알수 있습니다) | |
:param msg: 출력 결과에 포함될 메시지 (앞에 포함) | |
:param gtsize: 메모리 증감분이 이 크기보다 작으면 결과는 빈 문자열 | |
:return: gtsize 크기에 따라 보다 큰 증감분의 메모리 변화가 있으면 그 결과의 문자열 출력 | |
예) "end do_main: Total 37.50 MB, 30.63 MB incresed" | |
""" | |
if not self.enabled: | |
return 'Not enabled. guppy module not found!' | |
hdr = self.getHeap().split('\n')[0] | |
chsize = long(hdr.split()[-2]) | |
self.hdiff = chsize - self.hsize | |
self.hsize = chsize | |
if abs(self.hdiff) <= gtsize: | |
return '' | |
return '%s: %s'% (msg, str(self)) | |
########################################################################################## | |
def do_main(hm): | |
print hm.check('start do_main') # 함수를 들어와서 메모리 가감을 찍어봅니다 | |
biglist = [] | |
for i in xrange(1000000): | |
biglist.append(i) # 백만개의 정수를 담고있는 list를 생성해봅니다 | |
print hm.check('end do_main') # 함수를 종료하기 전에 메모리 가감을 찍어봅니다 | |
########################################################################################## | |
if __name__ == '__main__': | |
hm = HeapMon() # 여기에 글로벌로 hm 이라는 메모리 힙 모니터링을 위한 인스턴스를 만들어 놓았습니다. | |
print hm.check('before do_main') # 메인에서 do_main() 함수 호출전에 최초 메모리 가감을 찍어봅니다 | |
do_main(hm) | |
print hm.check('after do_main') # do_main() 함수 호출 후에 메모리 가감을 찍어봅니다 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
#coding=utf8 | |
import gc | |
import os | |
########################################################################################## | |
class HeapMon: | |
#===================================================================================== | |
def __init__(self, with_guppy = False): | |
self.pid = os.getpid() | |
self.enabled = False | |
try: | |
if with_guppy: | |
from guppy import hpy | |
self.enabled = True | |
except: | |
pass | |
if self.enabled: | |
self._h = hpy() | |
self.hsize = 0L | |
self.hdiff = 0L | |
#===================================================================================== | |
@staticmethod | |
def getVmRSS(pid): | |
with open('/proc/%d/status' % pid) as ifp: | |
for line in ifp: | |
if 'VmRSS' in line: | |
eles = line.split() | |
if eles[-1].lower() == 'kb': | |
return long(eles[-2]) * 1024 | |
if eles[-1].lower() == 'b': | |
return long(eles[-2]) | |
if eles[-1].lower() == 'mb': | |
return long(eles[-2]) * 1024 * 1024 | |
return 0 | |
#===================================================================================== | |
@staticmethod | |
def getReadableSize(lv): | |
if not isinstance(lv, (int, long)): | |
return '0' | |
if lv >= 1024*1024*1024*1024: | |
s = "%4.2f TB" % (float(lv)/(1024*1024*1024*1024)) | |
elif lv >= 1024*1024*1024: | |
s = "%4.2f GB" % (float(lv)/(1024*1024*1024)) | |
elif lv >= 1024*1024: | |
s = "%4.2f MB" % (float(lv)/(1024*1024)) | |
elif lv >= 1024: | |
s = "%4.2f KB" % (float(lv)/1024) | |
else: | |
s = "%d B" % lv | |
return s | |
#===================================================================================== | |
def __repr__(self): | |
# if not self.enabled: | |
# return 'Not enabled. guppy module not found!' | |
if self.hdiff > 0: | |
s = 'Total %s, %s incresed' % \ | |
(self.getReadableSize(self.hsize), self.getReadableSize(self.hdiff)) | |
elif self.hdiff < 0: | |
s = 'Total %s, %s decresed' % \ | |
(self.getReadableSize(self.hsize), self.getReadableSize(-self.hdiff)) | |
else: | |
s = 'Total %s, not changed' % self.getReadableSize(self.hsize) | |
return s | |
#===================================================================================== | |
def getHeap(self): | |
if not self.enabled: | |
return None | |
return str(self._h.heap()) | |
#===================================================================================== | |
def check(self, msg=''): | |
# if not self.enabled: | |
# return 'Not enabled. guppy module not found!' | |
if not self.enabled: | |
chsize = self.getVmRSS(self.pid) | |
else: | |
hdr = self.getHeap().split('\n')[0] | |
chsize = long(hdr.split()[-2]) | |
self.hdiff = chsize - self.hsize | |
self.hsize = chsize | |
return '%s: %s'% (msg, str(self)) | |
########################################################################################## | |
def main(hm): | |
print hm.check('Before allocating') | |
# print 'Before allocating ', rss(), | |
iterations = 1000000 | |
l = {} | |
for i in xrange(iterations): | |
l[i] = ({}) | |
# print 'After allocating ', rss(), | |
print hm.check('After allocating') | |
# Ignore optimizations, just try to free whatever possible | |
# First kill | |
for i in xrange(iterations): | |
l[i] = None | |
print hm.check('After First kill') | |
# Second kill | |
l.clear() | |
print hm.check('After Second kill') | |
# Third kill | |
l = None | |
del l | |
print hm.check('After Third kill') | |
# Control shot | |
gc.collect() | |
# print 'After free ', rss(), | |
print hm.check('After free') | |
########################################################################################## | |
if __name__ == '__main__': | |
hm = HeapMon() # 여기에 글로벌로 hm 이라는 메모리 힙 모니터링을 위한 인스턴스를 만들어 놓았습니다. | |
for i in range(3): | |
main(hm) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
윈도우는 안깔리나요?
pip install guppy 했는데 안되네요 ㅠㅠ
파이썬 3.6 사용하고있습니다.