$ twistd3 --nodaemon --pidfile= tftp --enable-reading --root-directory=$PWD
...
2017-04-05T14:08:49+0200 [-] RemoteOriginReadSession starting on 55337
2017-04-05T14:08:49+0200 [-] Starting protocol <tftp.bootstrap.RemoteOriginReadSession object at 0x7f0c16d9f2b0>
2017-04-05T14:08:49+0200 [RemoteOriginReadSession (UDP)] (UDP Port 55337 Closed)
2017-04-05T14:08:49+0200 [RemoteOriginReadSession (UDP)] Stopping protocol <tftp.bootstrap.RemoteOriginReadSession object at 0x7f0c16d9f2b0>
2017-04-05T14:08:50+0200 [-] Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 395, in startReactor
self.config, oldstdout, oldstderr, self.profiler, reactor)
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 310, in runReactorWithLogging
reactor.run()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1195, in run
self.mainLoop()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1204, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 825, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/usr/lib/python3/dist-packages/tftp/util.py", line 81, in _call_and_schedule
self.callable(*self.callable_args, **self.callable_kwargs)
File "/usr/lib/python3/dist-packages/twisted/internet/udp.py", line 268, in write
return self.socket.send(datagram)
builtins.AttributeError: 'Port' object has no attribute 'socket'To recreate:
import socket
from tftp.datagram import RRQDatagram
# Create datagram requesting read of an existing file. At least one
# option MUST be defined.
dg = RRQDatagram(b"something.txt", b"octet", {b"tsize": b"0"})
# Create a new DGRAM socket and send this packet to the running tftp
# daemon, then close the socket within ~4 seconds.
s = socket.socket(type=socket.SOCK_DGRAM)
s.sendto(dg.to_wire(), 0, ("127.0.0.1", 1069))
s.close()$ twistd3 --nodaemon --pidfile= tftp --enable-reading --root-directory=$PWD
...
2017-04-05T14:44:50+0200 [-] RemoteOriginReadSession starting on 37066
2017-04-05T14:44:50+0200 [-] Starting protocol <tftp.bootstrap.RemoteOriginReadSession object at 0x7f06256ee5c0>
2017-04-05T14:44:55+0200 [RemoteOriginReadSession (UDP)] (UDP Port 37066 Closed)
2017-04-05T14:44:55+0200 [RemoteOriginReadSession (UDP)] Stopping protocol <tftp.bootstrap.RemoteOriginReadSession object at 0x7f06256ee5c0>
2017-04-05T14:44:56+0200 [-] Timed during option negotiation process
2017-04-05T14:44:56+0200 [-] Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 395, in startReactor
self.config, oldstdout, oldstderr, self.profiler, reactor)
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 310, in runReactorWithLogging
reactor.run()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1195, in run
self.mainLoop()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1204, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 825, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/usr/lib/python3/dist-packages/tftp/bootstrap.py", line 207, in timedOut
self.cancel()
File "/usr/lib/python3/dist-packages/tftp/bootstrap.py", line 202, in cancel
self.transport.stopListening()
builtins.AttributeError: 'NoneType' object has no attribute 'stopListening'To recreate it's very similar to the previous example:
import socket, time
from tftp.datagram import RRQDatagram
# Create datagram requesting read of an existing file. At least one
# option MUST be defined.
dg = RRQDatagram(b"something.txt", b"octet", {b"tsize": b"0"})
# Create a new DGRAM socket and send this packet to the running tftp
# daemon, then immediately close the socket.
s = socket.socket(type=socket.SOCK_DGRAM)
s.sendto(dg.to_wire(), 0, ("127.0.0.1", 1069))
# Now sleep for a while. Less than ~4 and the previous (Port/socket)
# error will occur. More than ~5 and no error will occur.
time.sleep(4.5)
s.close()Client has disconnected, but Twisted and/or MAAS is blissfully ignorant.
2017-03-31 03:13:01 twisted.python.log: [info] ::ffff:127.0.0.1 - - [31/Mar/2017:03:13:00 +0000] "GET /MAAS/images-stream/ubuntu/ppc64el/ga-16.04/xenial/20170330.1/squashfs HTTP/1.1" 200 188739805 "-" "python-simplestreams/0.1"
2017-03-31 03:13:26 -: [critical] WSGI application error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 83, in callWithContext
self.contexts.pop()
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 857, in callInContext
return func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/twisted/web/wsgi.py", line 521, in run
self.started = True
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/web/wsgi.py", line 496, in run
appIterator = self.application(self.environ, self.startResponse)
File "/usr/lib/python3/dist-packages/django/core/handlers/wsgi.py", line 189, in __call__
response = self.get_response(request)
File "/usr/lib/python3/dist-packages/maasserver/utils/views.py", line 237, in get_response
request = reset_request(request)
File "/usr/lib/python3/dist-packages/maasserver/utils/views.py", line 88, in reset_request
wsgi_input._wrapped.seek(0)
builtins.ValueError: I/O operation on closed file.
2017-03-31 03:13:27 -: [critical] Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 364, in startReactor
self.config, oldstdout, oldstderr, self.profiler, reactor)
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 285, in runReactorWithLogging
reactor.run()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1194, in run
self.mainLoop()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1203, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 798, in runUntilCurrent
f(*a, **kw)
File "/usr/lib/python3/dist-packages/twisted/web/wsgi.py", line 512, in wsgiError
self.request.finish()
File "/usr/lib/python3/dist-packages/twisted/web/server.py", line 224, in finish
return http.Request.finish(self)
File "/usr/lib/python3/dist-packages/twisted/web/http.py", line 865, in finish
"Request.finish called on a request after its connection was lost; "
builtins.RuntimeError: Request.finish called on a request after its connection was lost; use Request.notifyFinish to keep track of this.
Appears to be due to an update in list_cluster_nodes_power_parameters
that touches many Node rows, thus causing conflicts.
Proper way to fix this: put power status into a separate table, perhaps even with historical info.
Way we'll probably fix this: reduce the number of Node records worked
on at a time.
2017-03-31 03:14:01 provisioningserver.rpc.common: [critical] Unhandled failure dispatching AMP command. This is probably a bug. Please ensure that this error is handled within application code or declared in the signature of the b'ListNodePowerParameters' command. [maas2-production-new-215:pid=22012:cmd=ListNodePowerParameters:ask=2dc8]
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1203, in mainLoop
self.runUntilCurrent()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 798, in runUntilCurrent
f(*a, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback
self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/protocols/amp.py", line 1149, in checkKnownErrors
key = error.trap(*command.allErrors)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 342, in trap
self.raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 368, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 246, in inContext
result = inContext.theWork()
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 262, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 857, in callInContext
return func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 225, in wrapper
result = func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/maasserver/utils/orm.py", line 603, in call_within_transaction
return func_outside_txn(*args, **kwargs)
File "/usr/lib/python3/dist-packages/maasserver/utils/orm.py", line 434, in retrier
return func(*args, **kwargs)
File "/usr/lib/python3.5/contextlib.py", line 30, in inner
return func(*args, **kwds)
File "/usr/lib/python3/dist-packages/maasserver/rpc/nodes.py", line 162, in list_cluster_nodes_power_parameters
system_id__in=system_ids).update(power_state_queried=now())
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 563, in update
rows = query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/lib/python3/dist-packages/django/db/models/sql/compiler.py", line 1062, in execute_sql
cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
File "/usr/lib/python3/dist-packages/django/db/models/sql/compiler.py", line 840, in execute_sql
cursor.execute(sql, params)
File "/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 64, in execute
return self.cursor.execute(sql, params)
File "/usr/lib/python3/dist-packages/django/db/utils.py", line 98, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/usr/lib/python3/dist-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 64, in execute
return self.cursor.execute(sql, params)
django.db.utils.OperationalError: could not serialize access due to concurrent update