python luigi example (https://github.com/spotify/luigi)

howto.bat

@echo off

luigid --port 8082

python luigi.py CountLetters --scheduler-host localhost


@rem Results at localhost:8082

luigi.py

import luigi
from faker import Faker


class GenerateWords(luigi.Task):
    def output(self):
        return luigi.LocalTarget("temp_words.txt")

    def run(self):
        fake = Faker()
        names = [fake.name() for _ in range(1000)]
        with self.output().open("w") as f:
            for name in names:
                f.write("{word}\n".format(word=name))


class CountLetters(luigi.Task):
    def requires(self):
        return GenerateWords()

    def output(self):
        return luigi.LocalTarget("temp_letter_counts.txt")

    def run(self):
        # read lines
        with self.input().open("r") as infile:
            words = infile.read().splitlines()'
            
        # write each word to output file with its corresponding letter count
        with self.output().open("w") as outfile:
            for word in words:
                outfile.write(
                    "{word} | {letter_count}\n".format(
                        word=word, letter_count=self.counter(word)
                    )
                )

    @staticmethod
    def counter(word):
        word = word.replace(" ", "")
        return len(word)


if __name__ == "__main__":
    luigi.run()