Submitting a job to a remote Grid Engine cluster using Paramiko (Python SSH client)

paramiko_test.py

import tempfile                                                                                                                                                                                                   
import os
import time

import paramiko

# Hostname of a 'submission host' in the Grid Engine cluster we want to submit a job to
hostname = 'sharc.shef.ac.uk'
# Username for logging in to the cluster via SSH
username = "te1st"
# Path on the cluster where we will save our job submission script to
remote_qsub_script_path = os.path.join('/', 'home', username, "qsub_script_{}".format(int(time.time())))

# Job submission script
qsub_script = """#!/bin/bash
#$ -m bea
#$ -M [email protected]
#$ -P rse
echo "Hello from $HOSTNAME"
"""

# Save job submission script to a local temporary file
with tempfile.NamedTemporaryFile(mode='w', delete=False) as local_qsub_script_file:
    local_qsub_script_file.write(qsub_script)

with paramiko.SSHClient() as client:
    client.load_system_host_keys()
    client.set_missing_host_key_policy(paramiko.WarningPolicy)

    # Establish SSH connection
    client.connect(hostname, username=username)

    # Establish SFTP connection
    with client.open_sftp() as sftp:
        # Push job submission script to a particular path on the cluster
        sftp.put(local_qsub_script_file.name, remote_qsub_script_path)

    # Submit our Grid Engine job by running a remote 'qsub' command over SSH
    stdin, stdout, stderr = client.exec_command("qsub {}".format(remote_qsub_script_path))

    # Show the standard output and error of our job
    print("Standard output:")
    print(stdout.read())
    print("Standard error:")
    print(stderr.read())
    print("Exit status: {}".format(stdout.channel.recv_exit_status()))

Also just for other future folks, this gist was linked from https://learningpatterns.me/posts/2018-01-04-paramiko-example/.