gc celery

pyproject.toml

[tool.poetry]
name = "evaluate-gc"
version = "0.1.0"
description = ""
authors = ["Stefan Verhoeven <stefan.verhoeven@gmail.com>"]

[tool.poetry.dependencies]
python = "^3.6"
celery = {extras = ["redis"], version = "^4.4.5"}

[tool.poetry.dev-dependencies]

[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

tasks.py

"""

Run worker with

    >>> pip install celery[redis]
    >>> docker run --rm -d -p 6379:6379 --name some-redis redis
    >>> celery worker -A tasks

In another shell

    >>> python tasks.py 
    process_submission
    main job id is 06238040-b505-4bd0-897b-19c318c194fa
    img2.json;img4.json;img1.json;img3.json;img6.json;img5.json

"""

import time
from celery import Celery, chord

app = Celery('tasks', broker='redis://localhost:6379', backend='redis://localhost:6379')
app.conf.result_backend = 'redis://localhost:6379/0'

@app.task
def run_algorithm(input_image, algorithm_id):
    print(f'Running algorithm {algorithm_id} on input {input_image}')
    return input_image.replace('.tiff', '.json')


@app.task
def run_evaluation(algorithm_results, algorithm_id):
    print(f'Running evaluation of algorithm {algorithm_id} with results {algorithm_results}')
    return ';'.join(algorithm_results)


def process_submission(challenge_id, algorithm_id):
    print('process_submission')
    images = [
        'img1.tiff',
        'img2.tiff',
        'img3.tiff',
        'img4.tiff',
        'img5.tiff',
        'img6.tiff',
    ]

    header = [run_algorithm.s(i, challenge_id) for i in images]
    callback = run_evaluation.s(algorithm_id)
    return chord(header)(callback)


def main():
    challenge_id = 123
    algorithm_id = 456

    result = process_submission(challenge_id, algorithm_id)
    print(f'main job id is {result.id}')
    print(result.get())


if __name__ == "__main__":
    main()

Outputs on client

[2020-06-18 16:01:31,247: WARNING/ForkPoolWorker-1] Running algorithm 123 on input img2.tiff
[2020-06-18 16:01:31,247: WARNING/ForkPoolWorker-8] Running algorithm 123 on input img1.tiff
[2020-06-18 16:01:31,252: WARNING/ForkPoolWorker-9] Running algorithm 123 on input img3.tiff
[2020-06-18 16:01:31,256: WARNING/ForkPoolWorker-2] Running algorithm 123 on input img4.tiff
[2020-06-18 16:01:31,263: WARNING/ForkPoolWorker-8] Running algorithm 123 on input img5.tiff
[2020-06-18 16:01:31,263: WARNING/ForkPoolWorker-1] Running algorithm 123 on input img6.tiff
[2020-06-18 16:01:31,286: WARNING/ForkPoolWorker-8] Running evaluation of algorithm 456 with results ['img2.json', 'img1.json', 'img3.json', 'img4.json', 'img6.json', 'img5.json']

Outputs on worker

process_submission
main job id is 8ef48b6c-fbf9-4076-9a2b-f13c1abd1133
img2.json;img1.json;img3.json;img4.json;img6.json;img5.json