Generate sources from catalog v1 schema

generate_sources.py

import json
import yaml

# Load catalog.json
with open('target/catalog.json', 'r') as f:
    catalog = json.load(f)

# Prepare the sources dictionary
sources = {
    'version': 2,
    'sources': []
}

# Extract relevant details from catalog.json
for unique_id, source_data in catalog['sources'].items():
    # Extract metadata
    metadata = source_data['metadata']
    source_name = metadata['name']
    schema = metadata['schema']
    database = metadata['database']
    
    # Find or create the source entry
    source_entry = next((source for source in sources['sources'] if source['name'] == source_name), None)
    if not source_entry:
        source_entry = {
            'name': source_name,
            'database': database,
            'schema': schema,
            'tables': []
        }
        sources['sources'].append(source_entry)

    # Add table and columns to the source entry
    table_entry = {
        'name': metadata['name'],
        'columns': []
    }
    for column_name, column_data in source_data['columns'].items():
        column_entry = {
            'name': column_name,
            'description': column_data.get('comment', '')
        }
        table_entry['columns'].append(column_entry)

    source_entry['tables'].append(table_entry)

# Write to sources.yml
with open('sources.yml', 'w') as f:
    yaml.dump(sources, f, default_flow_style=False, sort_keys=False)

print("sources.yml file has been generated successfully.")