Skip to content

Instantly share code, notes, and snippets.

@richardkiss
Created April 5, 2014 06:34
Show Gist options
  • Save richardkiss/9988156 to your computer and use it in GitHub Desktop.
Save richardkiss/9988156 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python
"""
This is an example of what seems to be a garbage collection bug in Python 3.4.0
that does not exist in Python 3.3.3.
The example uses asyncio to create a producer that multiplexes to N consumers.
On my Mac, with N=100, 38 consumers are incorrectly garbage collected and only execute
once, dropping the surviving consumer count to 62.
Setting DO_GC to True fixes this for some unknown reason.
This problem does NOT seem to exist in Python 3.3.3 with asyncio-0.41.
Setting PYTHONASYNCIODEBUG=1 changes the number 62 to 78.
"""
import asyncio
import gc
import weakref
N = 100
Q_SET = weakref.WeakSet()
# setting this to True fixes the problem. But it's clearly a kludge.
DO_GC = False
CONSUMER_COUNT = 0
class Q_as_callable(object):
def __init__(self, n):
self.q = asyncio.Queue()
self.n = n
def __call__(self):
return (yield from self.q.get())
def new_get_next_msg_f(x):
q = Q_as_callable(x)
Q_SET.add(q)
return q
def send_to_qs(item):
for q in Q_SET:
q.q.put_nowait(item)
def producer():
global CONSUMER_COUNT
n = 0
while 1:
CONSUMER_COUNT = 0
send_to_qs(n)
yield from asyncio.sleep(1)
print("surviving consumers: %d" % CONSUMER_COUNT)
n += 1
def consumer(x):
global CONSUMER_COUNT
next_msg = new_get_next_msg_f(x)
while 1:
CONSUMER_COUNT += 1
v = yield from next_msg()
def main():
asyncio.async(producer())
for i in range(N):
asyncio.async(consumer(i))
## if you uncomment out the gc.collect, no consumers are lost.
## This does not make sense to me.
if DO_GC:
gc.collect()
asyncio.get_event_loop().run_forever()
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment