remove_metadata_for_sparkjob

import boto3
import logging


logger = logging.getLogger('dag')
logger.setLevel(logging.INFO)


def get_url(bucket_name, obj):
    return f"s3://{bucket}/{obj.key}"


def remove_conflict_metafiles(bucket_name, prefix):
    filters = ["_committed_", "_started_"]

    s3 = boto3.resource('s3')
    bucket = s3.Bucket(bucket_name)
    objs = []

    for obj in bucket.objects.filter(Prefix=prefix):
        for _filter in filters:
            prefix_filter = prefix + _filter
            if obj.key.startswith(prefix_filter):
                objs.append({'Key': obj.key})

    if len(objs) > 0:
        s3client = boto3.client("s3")
        resp = s3client.delete_objects(Bucket=bucket_name, Delete={'Objects': objs})
        logger.info(resp)