Script that removes `lineageTag` and `annotations` keys from Tabular Editor's `Save to folder` export

clean_tabular_editor_export.py

import argparse
import os
import json


def remove_keys_from_dict(d, keys):
    did_clean = False

    if isinstance(d, dict):
        for k, v in list(d.items()):
            if k in keys:
                d.pop(k, None)
                did_clean = True
            elif isinstance(v, dict):
                did_clean_sub = remove_keys_from_dict(d[k], keys)
                if did_clean_sub is True:
                    did_clean = True
            elif isinstance(v, list):
                for index, _ in enumerate(list(v)):
                    did_clean_sub = remove_keys_from_dict(v[index], keys)
                    if did_clean_sub is True:
                        did_clean = True

    elif isinstance(d, list):
        for index, _ in enumerate(list(d)):
            did_clean_sub = remove_keys_from_dict(d[index], keys)
            if did_clean_sub is True:
                did_clean = True
    return did_clean


def json_remove_keys(json_path, keys):
    with open(json_path, 'r') as json_fp:
        json_data = json.load(json_fp)

    did_clean = remove_keys_from_dict(json_data, keys)

    if did_clean:
        with open(json_path, 'w') as json_fp:
            json.dump(json_data, json_fp, indent=2, sort_keys=False)

        print(f'Cleaned {json_path}')


def get_json_files(path):
    json_files = []

    for subdir, _, files in os.walk(path):
        for filename in files:
            file_path = os.path.join(subdir, filename)

            if file_path.endswith('.json'):
                json_files.append(file_path)

    return json_files


def clean_tabular_editor_jsons(path):
    print(f'Searching for JSON files in {path}...')
    json_files = get_json_files(path)

    print(f'Starting cleaning of {len(json_files)} files...')
    for json_file in json_files:
        json_remove_keys(json_file, ['lineageTag', 'annotations'])

    print(f'Finished cleaning of {len(json_files)} files...')


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'path',
        help='The path to the directory with JSON files',
    )
    args = parser.parse_args()

    clean_tabular_editor_jsons(args.path)


if __name__ == '__main__':
    main()