Skip to content

Instantly share code, notes, and snippets.

@adiroiban

adiroiban/example run

Last active January 31, 2019 11:51
Show Gist options
  • Save adiroiban/346dd455094d1762f0e69e9812309ad6 to your computer and use it in GitHub Desktop.
Save adiroiban/346dd455094d1762f0e69e9812309ad6 to your computer and use it in GitHub Desktop.
Download ZIP
Persistent pool not closed in Twisted
Raw
example run
Python 2.7.12 (default, Jul 1 2016, 15:12:24)
>>> import cryptography
>>> import OpenSSL
>>> cryptography.__version__
'1.7.2'
>>> OpenSSL.__version__
'16.2.0'
>>>
For google.com I got
$ py docs/web/examples/httpclient.py https://www.google.com
The response body will consist of 257 bytes.
Got some: <HTML><HEAD><meta http-equiv="content-type" conten
Response done
-----------------------
1488704564.24 after-done
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
-----------------------
1488704564.34 before-close
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
-----------------------
1488704564.38 after-close
-----------------------
getReaders: [<twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
For something like https://maranet.sharepoint.com
$ py docs/web/examples/httpclient.py https://maranet.sharepoint.com
The response body will consist of 13 bytes.
Got some: 403 FORBIDDEN
Response done
-----------------------
1488705165.79 after-done
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
-----------------------
1488705165.89 before-close
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
-----------------------
1488705283.68 after-close
-----------------------
getReaders: [<twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
Raw
httpsclient.py
#!/usr/bin/env python
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This example demonstrates how to make a simple http client.
Usage:
httpclient.py <url>
Don't forget the http:// when you type the web address!
"""
from __future__ import print_function
import sys
import time
from pprint import pprint
from twisted import version
from twisted.python import log
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.web.iweb import UNKNOWN_LENGTH
from twisted.web.http_headers import Headers
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone
class WriteToStdout(Protocol):
def connectionMade(self):
self.onConnLost = Deferred()
def dataReceived(self, data):
"""
Print out the html page received.
"""
print('Got some:', data[:50])
def connectionLost(self, reason):
if not reason.check(ResponseDone):
reason.printTraceback()
else:
print('Response done')
self.onConnLost.callback(None)
def cb_connection_done(result, pool):
"""
Called after the HTTP request was done.
"""
show_reactor('after-done')
reactor.callLater(0.1, cl_close, pool)
@inlineCallbacks
def cl_close(pool):
"""
Called to close the persistent pool.
"""
show_reactor('before-close')
yield pool.closeCachedConnections()
show_reactor('after-close')
reactor.stop()
def show_reactor(step):
print('-----------------------')
print('%s %s' % (time.time(), step))
print('-----------------------')
print('getReaders: %s' % (reactor.getReaders(),))
def main(reactor, url):
"""
We create a custom UserAgent and send a GET request to a web server.
"""
userAgent = 'Twisted/%s (httpclient.py)' % (version.short(),)
pool = HTTPConnectionPool(reactor, persistent=True)
agent = Agent(reactor, pool=pool)
d = agent.request(
'GET', url, Headers({'user-agent': [userAgent]}))
def cbResponse(response):
"""
Prints out the response returned by the web server.
"""
proto = WriteToStdout()
if response.length is not UNKNOWN_LENGTH:
print('The response body will consist of', response.length, 'bytes.')
else:
print('The response body length is unknown.')
response.deliverBody(proto)
return proto.onConnLost
d.addCallback(cbResponse)
d.addBoth(cb_connection_done, pool)
d.addErrback(log.err)
reactor.run()
if __name__ == '__main__':
main(reactor, *sys.argv[1:])
Raw
patch_tls.py
def patch_shutdownTLS(self):
"""
Initiate, or reply to, the shutdown handshake of the TLS layer.
"""
def try_shutdown():
try:
shutdownSuccess = self._tlsConnection.shutdown()
except SSL.Error:
# Mid-handshake, a call to shutdown() can result in a
# WantWantReadError, or rather an SSL_ERR_WANT_READ; but pyOpenSSL
# doesn't allow us to get at the error. See:
# https://github.com/pyca/pyopenssl/issues/91
#
# If the underlying BIO is non-blocking, SSL_shutdown() will also
# return when the underlying BIO could not satisfy the needs of
# SSL_shutdown() to continue the handshake.
# In this case a call to SSL_get_error() with the return value of
# SSL_shutdown() will yield SSL_ERROR_WANT_READ or
# SSL_ERROR_WANT_WRITE.
# The calling process then must repeat the call after taking
# appropriate action to satisfy the needs of SSL_shutdown()
shutdownSuccess = False
self._flushSendBIO()
if shutdownSuccess:
# Both sides have shutdown.
# This will also happen if we haven't started
# negotiation at all yet, in which case shutdown succeeds
# immediately.
if self._loseConnectionOnTLSShutdown:
# We will lose the connection on shutdown.
self.transport.loseConnection()
return
try:
low_transport = self.transport.socket
except AttributeError:
# The transport was already closed.
return
shutdown_start = getattr(self, '__shutdown_start', None)
if shutdown_start is None:
self.__shutdown_start = time()
if time() - self.__shutdown_start > 2:
self.transport.loseConnection()
return
# The shutdown was not successful.
# We assume that it was due to WantWantReadError, so we try to trigger
# the shutdown state by reading the low level socket and passing it
# to the BIO.
try:
data = low_transport.recv(2 ** 15)
if data:
# Only write if we got something on the wire.
self._tlsConnection.bio_write(data)
self._flushReceiveBIO()
except socket.error as error:
if error.errno in [errno.EAGAIN, tcp.EWOULDBLOCK]:
# Socket is not yet ready
reactor.callLater(0.01, try_shutdown)
elif error.errno == tcp.ECONNRESET:
# Remote was already closed.
self.transport.loseConnection()
else:
raise
try_shutdown()
TLSMemoryBIOProtocol._shutdownTLS = patch_shutdownTLS
@baby5
Copy link

baby5 commented Jan 31, 2019

thanks for your patch, and twisted has not fixed this problem :(

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment