Non-blocking latent slaves, potential fix for http://trac.buildbot.net/ticket/3169, taken from https://github.com/ClusterHQ/build.clusterhq.com/blob/master/flocker_bb/monkeypatch.py

non-blocking-latent.patch

diff --git a/master/buildbot/process/botmaster.py b/master/buildbot/process/botmaster.py
index 4f1f348..a5f28f3 100644
--- a/master/buildbot/process/botmaster.py
+++ b/master/buildbot/process/botmaster.py
@@ -317,11 +317,19 @@ class BotMaster(config.ReconfigurableServiceMixin, service.MultiService):
         """
         Call this when something suggests that a particular slave may now be
         available to start a build.
+        We delay this for 10 seconds, so that if multiple slaves start at the same
+        time, builds will be distributed between them.
 
         @param slave_name: the name of the slave
         """
-        builders = self.getBuildersForSlave(slave_name)
-        self.brd.maybeStartBuildsOn([b.name for b in builders])
+        def do_start():
+            log.msg(format="Really starting builds on %(slave_name)s",
+                    slave_name=slave_name)
+            builders = self.getBuildersForSlave(slave_name)
+            self.brd.maybeStartBuildsOn([b.name for b in builders])
+        log.msg(format="Waiting to start builds on %(slave_name)s",
+                slave_name=slave_name)
+        reactor.callLater(10, do_start)
 
     def maybeStartBuildsForAllBuilders(self):
         """
diff --git a/master/buildbot/process/buildrequestdistributor.py b/master/buildbot/process/buildrequestdistributor.py
index 2848510..194d5bd 100644
--- a/master/buildbot/process/buildrequestdistributor.py
+++ b/master/buildbot/process/buildrequestdistributor.py
@@ -496,12 +496,13 @@ class BuildRequestDistributor(service.Service):
 
             # get the actual builder object
             bldr = self.botmaster.builders.get(bldr_name)
-            try:
-                if bldr:
-                    yield self._maybeStartBuildsOnBuilder(bldr)
-            except Exception:
-                log.err(Failure(),
-                        "from maybeStartBuild for builder '%s'" % (bldr_name,))
+            if bldr:
+                d = self._maybeStartBuildsOnBuilder(bldr)
+                self._pendingMSBOCalls.append(d)
+                d.addErrback(
+                    log.err,
+                    "from maybeStartBuild for builder '%s'" % (bldr_name,))
+                d.addCallback(lambda _, d=d: self._pendingMSBOCalls.remove(d))
 
             self.activity_lock.release()