Created
April 5, 2014 06:34
-
-
Save richardkiss/9988156 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
""" | |
This is an example of what seems to be a garbage collection bug in Python 3.4.0 | |
that does not exist in Python 3.3.3. | |
The example uses asyncio to create a producer that multiplexes to N consumers. | |
On my Mac, with N=100, 38 consumers are incorrectly garbage collected and only execute | |
once, dropping the surviving consumer count to 62. | |
Setting DO_GC to True fixes this for some unknown reason. | |
This problem does NOT seem to exist in Python 3.3.3 with asyncio-0.41. | |
Setting PYTHONASYNCIODEBUG=1 changes the number 62 to 78. | |
""" | |
import asyncio | |
import gc | |
import weakref | |
N = 100 | |
Q_SET = weakref.WeakSet() | |
# setting this to True fixes the problem. But it's clearly a kludge. | |
DO_GC = False | |
CONSUMER_COUNT = 0 | |
class Q_as_callable(object): | |
def __init__(self, n): | |
self.q = asyncio.Queue() | |
self.n = n | |
def __call__(self): | |
return (yield from self.q.get()) | |
def new_get_next_msg_f(x): | |
q = Q_as_callable(x) | |
Q_SET.add(q) | |
return q | |
def send_to_qs(item): | |
for q in Q_SET: | |
q.q.put_nowait(item) | |
def producer(): | |
global CONSUMER_COUNT | |
n = 0 | |
while 1: | |
CONSUMER_COUNT = 0 | |
send_to_qs(n) | |
yield from asyncio.sleep(1) | |
print("surviving consumers: %d" % CONSUMER_COUNT) | |
n += 1 | |
def consumer(x): | |
global CONSUMER_COUNT | |
next_msg = new_get_next_msg_f(x) | |
while 1: | |
CONSUMER_COUNT += 1 | |
v = yield from next_msg() | |
def main(): | |
asyncio.async(producer()) | |
for i in range(N): | |
asyncio.async(consumer(i)) | |
## if you uncomment out the gc.collect, no consumers are lost. | |
## This does not make sense to me. | |
if DO_GC: | |
gc.collect() | |
asyncio.get_event_loop().run_forever() | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment