Playing with RQ priorities

main.py

#!/usr/bin/env python3
"""
Experiment to show how rq priorities might work with jobs that will complete in different amounts of time...

IF you run a single worker (rqworker high low), this may do what you expect, but if you run multiple workers,
the low-priority tasks that take less time may complete before some of the high-priority tasks.


$ python main.py
Enqueued work for 0 in HIGH
Enqueued work for 1 in LOW
Enqueued work for 2 in HIGH
Enqueued work for 3 in LOW
Enqueued work for 4 in HIGH
Enqueued work for 5 in LOW
Enqueued work for 6 in HIGH
Enqueued work for 7 in LOW
Enqueued work for 8 in HIGH
Enqueued work for 9 in LOW

$ cat RESULTS.txt
[HIGH] Job 0 slept for 0 seconds. Completed in 0.0 seconds.
[LOW] Job 1 slept for 1 seconds. Completed in 1.0 seconds.
[HIGH] Job 2 slept for 2 seconds. Completed in 2.0 seconds.
[HIGH] Job 4 slept for 4 seconds. Completed in 4.0 seconds.
[HIGH] Job 6 slept for 6 seconds. Completed in 6.0 seconds.
[LOW] Job 5 slept for 5 seconds. Completed in 5.0 seconds.
[HIGH] Job 8 slept for 8 seconds. Completed in 8.0 seconds.
[LOW] Job 7 slept for 7 seconds. Completed in 7.0 seconds.
[LOW] Job 9 slept for 9 seconds. Completed in 9.0 seconds.

"""

from rq import Queue, Connection
from redis import Redis

import time


def job_handler(n, queue_name):
    t = time.time()

    msg = f"Job {n} will sleep for {n} seconds."
    time.sleep(n)
    elapsed = round(time.time() - t, 2)

    with open("RESULTS.txt", "a") as f:
        f.write(f"[{queue_name}] {msg} Completed in {elapsed} seconds.\n")


def make_work(n_jobs, conn):
    with Connection(Redis("localhost", 6379)):
        high_q = Queue("high")
        low_q = Queue("low")
        for i in range(n_jobs):

            func = "temp.temp.job_handler"
            if i % 2 == 0:
                high_q.enqueue(func, i, "HIGH")
                print(f"Enqueued work for {i} in HIGH")
            else:
                low_q.enqueue(func, i, "LOW")
                print(f"Enqueued work for {i} in LOW")


if __name__ == "__main__":
    make_work(10, r)