Gather and merge configuration

demo.py

system_defaults = {}
config_paths = [
    Path.home()/'.myconfig',
]
env_config = config_filter(os.environ)
user_config = get_config(*config_paths)
config = d_merge(env_config, user_config, system_defaults)

merge_configuration.py

import configparser
from functools import partial
import os
from pathlib import Path
import yaml

def d_merge(a, b, *others):
    """Recursive dictionary merge."""
    c = a.copy()
    for k, v in b.items():
        if (k in c
            and isinstance(b.get(k), dict)
            and isinstance(c[k], collections.Mapping)):
            c[k] = d_merge(c[k], b[k])
        elif not k in c:
            c[k] = b[k]
    return (d_merge(c, *others) if bool(others) else c)


def lower_key(d):
    """Convert a dictionary to lower-case string keys."""
    return {str(k).lower():v for k, v in d.items()}

def white_list(keys, d):
    """Reduce a dictionary to items in a white list."""
    d = lower_key(d)
    output = OrderedDict()
    for key in keys:
        if str(key).lower() in d: output[key] = d[key]
    return dict(output)

# All the terms I'm willing to collect.
allowed_terms = [
    'access_key', 'secret_key', 'region_name', 'queue_url', 'db_host',
    'db_password', 'database', 'db_user',
]

# Creates a function, config_filter, that filters a dictionary for allowed terms.
config_filter = partial(white_list, allowed_terms)

def path_yield(*paths):
    """Convert strings to paths and yield them if they exist."""
    for path in paths:
        path = Path(path)
        if path.exists():
            yield path
            
def contents(path, mode='r'):
    """Read and return the path or return None."""
    try:
        return open(path, mode).read()
    except:
        return None
    
def path_contents(*paths, mode='r'):
    """For all paths, yield the contents."""
    for path in path_yield(*paths):
        yield contents(path)
        
def yaml_read(path):
    """Safely read a YAML path."""
    try:
        result = yaml.load(contents(path), Loader=yaml.FullLoader)
        return (config_filter(result) if bool(result) else {})
    except:
        return {}
    
def config_read(path):
    """Read an ini file, prepending the keys with the section name."""
    config = configparser.ConfigParser()
    config.read(path)
    d = {}
    for section in config.sections():
        d1 = {f'{section}_{k}':v for k, v in dict(config[section]).items()}
        d1 = config_filter(d1)
        d = d_merge(d, d1)
    return d
    
def is_ini(path): return '.ini' in Path(path).suffixes
def parse(path):
    """Read an ini or yaml file, returning a dictionary of key value pairs."""
    return (config_read(path) if is_ini(path) else yaml_read(path))
    
def get_config(*paths):
    """Read all configuration files, merging the results."""
    d = {}
    for path in path_yield(*paths):
        d = d_merge(d, parse(path))
    return d

This should work to gather any kind of configuration into one place. Features include:

• If the configuration appears more than once, the first time it occurs, it is used.
• It white lists the configuration, so I don't have rogue values from my environment in code or notebooks.
• It handles environment variables, configuration files (.ini format), and YAML files.
• It uses a consistent downcased keyspace.
• It builds a few stepwise utilities that I tend to need for other things (reading files safely, checking if files exist simply).

The demo.py file is an example usage of the functions.