Blowing Minds

key_filter.py

class HiddenOperator(object):

    def __init__(bucket, prefix, key_filter=None):
        self.bucket = bucket
        self.prefix = prefix
        self.key_filter = key_filter

    def run():
        print("List keys")
        keys = [
            "prefix/",
            "prefix/1.txt",
            "prefix/2.txt",
            "prefix/3.txt",
            "prefix/5.csv"
        ]
        if key_filter:
            return 
        else:
        return filter(self.key_filter, keys)


## In the Dag File

import HiddenOperator

def is_csv(donut):
    if "csv" in donut:
        return True
    else:
        return False

x = HiddenOperator('bucket-name', 'prefix', key_filter=is_csv)