Hack - Recover from paramiko hangs with process timeouts

example_paramiko_hang.py

import paramiko
from pebble import concurrent
from concurrent.futures import TimeoutError

while not deviceTests_is_desktop_on_wifi_flag:
        print("Waiting for desk %s to complete challenge 1 - Connect to WiFi" % ourHackerDeskNumber)

        trySSHConnection = deviceTests_is_desktop_on_wifi(ourHackerDeskNumber)

        try:
            trySSHresults = trySSHConnection.result()
            if trySSHresults:
                deviceTests_is_desktop_on_wifi_flag = True
        except TimeoutError as error:
            print("unstable_function took longer than %d seconds" % error.args[1])
        except ProcessExpired as error:
            print("%s. Exit code: %d" % (error, error.exitcode))
        except Exception as error:
            print("unstable_function raised %s" % error)
            print(error.traceback)  # Python's traceback of remote process

        time.sleep(1)


@concurrent.process(timeout=10)
def deviceTests_is_desktop_on_wifi(ourHackerDeskNumber):
    # If we can connect, return true.
    print("C1Test> Connecting to SSH Client:", hackerDesks[ourHackerDeskNumber][0] )
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hackerDesks[ourHackerDeskNumber][0], username="root", timeout=2, key_filename='./keys/bhwh_watchdog')
        time.sleep(1)
        stdin, stdout, stderr = ssh.exec_command('uname -a')
        if stdout.channel.recv_exit_status() == 0:
            return True
        else:
            return False

    except (socket.timeout, TimeoutError, socket.error, paramiko.SSHException):
        print("SSH Connection Failed")
        return False

    finally:
        ssh.close()