Recursively merging dictionaries with boltons.iterutils.remap. Useful for @hynek's configs. https://twitter.com/hynek/status/696720593002041345

remerge.py

"""
This comes from Mahmoud Hashemi @mhashemi found at:
https://gist.github.com/mahmoud/db02d16ac89fa401b968

This is an extension of the technique first detailed here:

http://sedimental.org/remap.html#add_common_keys

In short, it calls remap on each container, back to front, using the accumulating
previous values as the default for the current iteration.

[NOTE: There is currently a bug/limitation where we cannot handle keys with
       arrays as values (c.f. 'zones' below).]
"""

from __future__ import print_function
from boltons.iterutils import remap, get_path, default_enter, default_visit, default_exit


defaults = {'host': '127.0.0.1',
            'port': 8000,
            'endpoints': {'persistence': {'host': '127.0.0.1',
                                          'port': 8888},
                          'cache': {'host': '127.0.0.1',
                                    'port': 8889}}}


overlay = {'host': '127.0.0.1',
           'port': 8080,
           'zones': [{'a': 1}],
           'endpoints': {'persistence': {'host': '10.2.2.2',
                                         'port': 5433}},
           'overlay_version': '5.0'}


cache_host_override = {'endpoints': {'cache': {'host': '127.0.0.2'}}}


INDENT = []

real_print = print

def print(*a, **kw):
    return real_print('  '.join(INDENT + ['']), *a, **kw)



def remerge(target_list, sourced=False):
    """Takes a list of containers (e.g., dicts) and merges them using
    boltons.iterutils.remap. Containers later in the list take
    precedence (last-wins).

    By default, returns a new, merged top-level container. With the
    *sourced* option, `remerge` expects a list of (*name*, container*)
    pairs, and will return a source map: a dictionary mapping between
    path and the name of the container it came from.

    """

    if not sourced:
        target_list = [(id(t), t) for t in target_list]

    ret = None
    source_map = {}

    def remerge_enter(path, key, value):
        print("- entering path={}, key={!r}, value={!r}".format(path, key, value))
        new_parent, new_items = default_enter(path, key, value)
        INDENT.append('')
        print("- got new_parent={}, new_items={}".format(new_parent, new_items))
        if ret and not path and key is None:
            print("- updating new_parent to ret={}".format(ret))
            new_parent = ret

        try:
            cur_val = get_path(ret, path + (key,))
            print("- cur_val={}".format(cur_val))
        except KeyError as ke:
            print(" X {!r} -- {!s}".format(ret, ke))
            pass
        else:
            # TODO: type check?
            print("- setting new_parent to {}".format(cur_val))
            new_parent = cur_val

        print("- returning new_parent={}, new_items={}".format(new_parent, new_items))
        return new_parent, new_items

    def remerge_exit(path, key, old_parent, new_parent, new_items):
        print("+ exiting path={}, key={}, old_parent={}, new_parent={}, new_items={}".format(path, key, old_parent, new_parent, new_items))
        INDENT.pop()
        return default_exit(path, key, old_parent, new_parent, new_items)

    for t_name, target in target_list:
        if sourced:
            def remerge_visit(path, key, value):
                source_map[path + (key,)] = t_name
                return True
        else:
            remerge_visit = default_visit

        ret = remap(target, enter=remerge_enter, visit=remerge_visit,
                    exit=remerge_exit)

    if not sourced:
        return ret
    return ret, source_map


def main():
    from pprint import pprint
    import json

    merged, source_map = remerge([('defaults', defaults),
                                  ('overlay', overlay),
                                  ('cache_host_override', cache_host_override),
                                  ],
                                 sourced=True)

    assert merged['host'] == '127.0.0.1'
    assert merged['port'] == 8080
    assert merged['endpoints']['persistence']['host'] == '10.2.2.2'
    assert merged['endpoints']['persistence']['port'] == 5433
    assert merged['endpoints']['cache']['host'] == '127.0.0.2'
    assert merged['endpoints']['cache']['port'] == 8889
    assert merged['overlay_version'] == '5.0'

    print(' ')
    print(' ')
    pprint(merged)
    print(' ')
    # pprint(source_map)
    # print(len(source_map), 'paths')

if __name__ == '__main__':
    main()

the_output.py

 - entering path=(), key=None, value={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}
   - got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000})
    X None -- could not access None from path (None,), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
   - returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000})
   - entering path=(), key='host', value='127.0.0.1'
     - got new_parent=127.0.0.1, new_items=False
      X None -- could not access 'host' from path ('host',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
     - returning new_parent=127.0.0.1, new_items=False
     - entering path=(), key='endpoints', value={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
       - got new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}})
        X None -- could not access 'endpoints' from path ('endpoints',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
       - returning new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}})
       - entering path=('endpoints',), key='cache', value={'host': '127.0.0.1', 'port': 8889}
         - got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
          X None -- could not access 'endpoints' from path ('endpoints', 'cache'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
         - returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8889})
         - entering path=('endpoints', 'cache'), key='host', value='127.0.0.1'
           - got new_parent=127.0.0.1, new_items=False
            X None -- could not access 'endpoints' from path ('endpoints', 'cache', 'host'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
           - returning new_parent=127.0.0.1, new_items=False
           - entering path=('endpoints', 'cache'), key='port', value=8889
             - got new_parent=8889, new_items=False
              X None -- could not access 'endpoints' from path ('endpoints', 'cache', 'port'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
             - returning new_parent=8889, new_items=False
             + exiting path=('endpoints',), key=cache, old_parent={'host': '127.0.0.1', 'port': 8889}, new_parent={}, new_items=[('host', '127.0.0.1'), ('port', 8889)]
           - entering path=('endpoints',), key='persistence', value={'host': '127.0.0.1', 'port': 8888}
             - got new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
              X None -- could not access 'endpoints' from path ('endpoints', 'persistence'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
             - returning new_parent={}, new_items=ItemsView({'host': '127.0.0.1', 'port': 8888})
             - entering path=('endpoints', 'persistence'), key='host', value='127.0.0.1'
               - got new_parent=127.0.0.1, new_items=False
                X None -- could not access 'endpoints' from path ('endpoints', 'persistence', 'host'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
               - returning new_parent=127.0.0.1, new_items=False
               - entering path=('endpoints', 'persistence'), key='port', value=8888
                 - got new_parent=8888, new_items=False
                  X None -- could not access 'endpoints' from path ('endpoints', 'persistence', 'port'), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
                 - returning new_parent=8888, new_items=False
                 + exiting path=('endpoints',), key=persistence, old_parent={'host': '127.0.0.1', 'port': 8888}, new_parent={}, new_items=[('host', '127.0.0.1'), ('port', 8888)]
               + exiting path=(), key=endpoints, old_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, new_parent={}, new_items=[('cache', {'host': '127.0.0.1', 'port': 8889}), ('persistence', {'host': '127.0.0.1', 'port': 8888})]
             - entering path=(), key='port', value=8000
               - got new_parent=8000, new_items=False
                X None -- could not access 'port' from path ('port',), got error: TypeError("'NoneType' object has no attribute '__getitem__'",)
               - returning new_parent=8000, new_items=False
               + exiting path=(), key=None, old_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}, new_parent={}, new_items=[('host', '127.0.0.1'), ('endpoints', {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}), ('port', 8000)]
             - entering path=(), key=None, value={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}
               - got new_parent={}, new_items=ItemsView({'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080})
               - updating new_parent to ret={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}
                X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access None from path (None,), got error: KeyError(None,)
               - returning new_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000}, new_items=ItemsView({'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080})
               - entering path=(), key='zones', value=[{'a': 1}]
                 - got new_parent=[], new_items=<enumerate object at 0x7f67f7a2f6e0>
                  X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones',), got error: KeyError('zones',)
                 - returning new_parent=[], new_items=<enumerate object at 0x7f67f7a2f6e0>
                 - entering path=('zones',), key=0, value={'a': 1}
                   - got new_parent={}, new_items=ItemsView({'a': 1})
                    X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones', 0), got error: KeyError('zones',)
                   - returning new_parent={}, new_items=ItemsView({'a': 1})
                   - entering path=('zones', 0), key='a', value=1
                     - got new_parent=1, new_items=False
                      X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, 'port': 8000} -- could not access 'zones' from path ('zones', 0, 'a'), got error: KeyError('zones',)
                     - returning new_parent=1, new_items=False
                     + exiting path=('zones',), key=0, old_parent={'a': 1}, new_parent={}, new_items=[('a', 1)]
                   + exiting path=(), key=zones, old_parent=[{'a': 1}], new_parent=[], new_items=[(0, {'a': 1})]
                 - entering path=(), key='host', value='127.0.0.1'
                   - got new_parent=127.0.0.1, new_items=False
                   - cur_val=127.0.0.1
                   - setting new_parent to 127.0.0.1
                   - returning new_parent=127.0.0.1, new_items=False
                   - entering path=(), key='endpoints', value={'persistence': {'host': '10.2.2.2', 'port': 5433}}
                     - got new_parent={}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
                     - cur_val={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
                     - setting new_parent to {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}
                     - returning new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '127.0.0.1', 'port': 8888}}, new_items=ItemsView({'persistence': {'host': '10.2.2.2', 'port': 5433}})
                     - entering path=('endpoints',), key='persistence', value={'host': '10.2.2.2', 'port': 5433}
                       - got new_parent={}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
                       - cur_val={'host': '127.0.0.1', 'port': 8888}
                       - setting new_parent to {'host': '127.0.0.1', 'port': 8888}
                       - returning new_parent={'host': '127.0.0.1', 'port': 8888}, new_items=ItemsView({'host': '10.2.2.2', 'port': 5433})
                       - entering path=('endpoints', 'persistence'), key='host', value='10.2.2.2'
                         - got new_parent=10.2.2.2, new_items=False
                         - cur_val=127.0.0.1
                         - setting new_parent to 127.0.0.1
                         - returning new_parent=127.0.0.1, new_items=False
                         - entering path=('endpoints', 'persistence'), key='port', value=5433
                           - got new_parent=5433, new_items=False
                           - cur_val=8888
                           - setting new_parent to 8888
                           - returning new_parent=8888, new_items=False
                           + exiting path=('endpoints',), key=persistence, old_parent={'host': '10.2.2.2', 'port': 5433}, new_parent={'host': '127.0.0.1', 'port': 8888}, new_items=[('host', '10.2.2.2'), ('port', 5433)]
                         + exiting path=(), key=endpoints, old_parent={'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=[('persistence', {'host': '10.2.2.2', 'port': 5433})]
                       - entering path=(), key='overlay_version', value='5.0'
                         - got new_parent=5.0, new_items=False
                          X {'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'port': 8000} -- could not access 'overlay_version' from path ('overlay_version',), got error: KeyError('overlay_version',)
                         - returning new_parent=5.0, new_items=False
                         - entering path=(), key='port', value=8080
                           - got new_parent=8080, new_items=False
                           - cur_val=8000
                           - setting new_parent to 8000
                           - returning new_parent=8000, new_items=False
                           + exiting path=(), key=None, old_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_parent={'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'port': 8000}, new_items=[('zones', [{'a': 1}]), ('host', '127.0.0.1'), ('endpoints', {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}), ('overlay_version', '5.0'), ('port', 8080)]
                         - entering path=(), key=None, value={'endpoints': {'cache': {'host': '127.0.0.2'}}}
                           - got new_parent={}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
                           - updating new_parent to ret={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}
                            X {'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080} -- could not access None from path (None,), got error: KeyError(None,)
                           - returning new_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_items=ItemsView({'endpoints': {'cache': {'host': '127.0.0.2'}}})
                           - entering path=(), key='endpoints', value={'cache': {'host': '127.0.0.2'}}
                             - got new_parent={}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
                             - cur_val={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}
                             - setting new_parent to {'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}
                             - returning new_parent={'cache': {'host': '127.0.0.1', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=ItemsView({'cache': {'host': '127.0.0.2'}})
                             - entering path=('endpoints',), key='cache', value={'host': '127.0.0.2'}
                               - got new_parent={}, new_items=ItemsView({'host': '127.0.0.2'})
                               - cur_val={'host': '127.0.0.1', 'port': 8889}
                               - setting new_parent to {'host': '127.0.0.1', 'port': 8889}
                               - returning new_parent={'host': '127.0.0.1', 'port': 8889}, new_items=ItemsView({'host': '127.0.0.2'})
                               - entering path=('endpoints', 'cache'), key='host', value='127.0.0.2'
                                 - got new_parent=127.0.0.2, new_items=False
                                 - cur_val=127.0.0.1
                                 - setting new_parent to 127.0.0.1
                                 - returning new_parent=127.0.0.1, new_items=False
                                 + exiting path=('endpoints',), key=cache, old_parent={'host': '127.0.0.2'}, new_parent={'host': '127.0.0.1', 'port': 8889}, new_items=[('host', '127.0.0.2')]
                               + exiting path=(), key=endpoints, old_parent={'cache': {'host': '127.0.0.2'}}, new_parent={'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, new_items=[('cache', {'host': '127.0.0.2', 'port': 8889})]
                             + exiting path=(), key=None, old_parent={'endpoints': {'cache': {'host': '127.0.0.2'}}}, new_parent={'zones': [{'a': 1}], 'host': '127.0.0.1', 'endpoints': {'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}}, 'overlay_version': '5.0', 'port': 8080}, new_items=[('endpoints', {'cache': {'host': '127.0.0.2', 'port': 8889}, 'persistence': {'host': '10.2.2.2', 'port': 5433}})]
                            
                            
{'endpoints': {'cache': {'host': '127.0.0.2', 'port': 8889},
               'persistence': {'host': '10.2.2.2', 'port': 5433}},
 'host': '127.0.0.1',
 'overlay_version': '5.0',
 'port': 8080,
 'zones': [{'a': 1}]}