Consolidate lists of lists of dictionaries based on a key

go.py

import itertools

import yaml


# - name: Build the data structure
#   set_fact:
#     data_sources:
#     - data: interfaces.results
#       match_key: name
#       nested_dict: interfaces
#     - data: l2_interfaces.results
#       match_key: name
#       nested_dict: l2_interfaces
#     - data: l3_interfaces.results
#       match_key: name
#       nested_dict: interfaces
#     - data: ospf_interfaces.results
#       match_key: name
#       nested_dict: ospf_interfaces
#     - data: net_neighbors
#       match_key: port
#       nested_dict: facts_net

# - name: Combine all the dictionaries based on a key
#   set_fact:
#     combined: data_source|ansible.utils.consolidate(fail_missing=True, fail_duplicate=True)


data_sources = [
    {
        "data": [
            {"name": "Loopback888"},
            {"name": "Loopback999"},
            {"name": "GigabitEthernet0/0"},
            {"name": "GigabitEthernet0/1"},
            {"name": "GigabitEthernet0/2"},
        ],
        "match_key": "name",
        "source": "acl_interfaces",
    },
    {  # inconsistency
        "data": [
            {
                "description": "Configured by Ansible Team",
                "enabled": False,
                "name": "Loopback888",
            },
            {"enabled": True, "name": "Loopback999"},
            {
                "description": "Configured and Managed By Ansible Team",
                "enabled": True,
                "name": "GigabitEthernet0/0",
            },
            {
                "description": "This is a user template",
                "enabled": True,
                "name": "GigabitEthernet0/1",
            },
            {
                "description": "This is a user template",
                "enabled": True,
                "name": "GigabitEthernet0/2",
            },
        ],
        "match_key": "name",
        "source": "interfaces",
    },
    {
        "data": [
            {"name": "GigabitEthernet0/0"},
            {
                "mode": "access",
                "name": "GigabitEthernet0/1",
                "trunk": {
                    "allowed_vlans": [
                        "11",
                        "12",
                        "59",
                        "67",
                        "75",
                        "77",
                        "81",
                        "100",
                        "400-408",
                        "411-413",
                        "415",
                        "418",
                        "982",
                        "986",
                        "988",
                        "993",
                    ]
                },
            },
            {
                "mode": "trunk",
                "name": "GigabitEthernet0/2",
                "trunk": {
                    "allowed_vlans": [
                        "11",
                        "12",
                        "59",
                        "67",
                        "75",
                        "77",
                        "81",
                        "100",
                        "400-408",
                        "411-413",
                        "415",
                        "418",
                        "982",
                        "986",
                        "988",
                        "993",
                    ],
                    "encapsulation": "dot1q",
                },
            },
        ],
        "match_key": "name",
        "source": "l2_interfaces",
    },
    {
        "data": [
            {"ipv4": [{"address": "10.8.38.76/24"}], "name": "GigabitEthernet0/0"},
            {"name": "GigabitEthernet0/1"},
            {"name": "GigabitEthernet0/2"},
            {"name": "Loopback888"},
            {"name": "Loopback999"},
        ],
        "match_key": "name",
        "source": "l3_interfaces",
    },
    {
        "data": [
            {"no_name": True},
            {"name": "GigabitEthernet0/1"},
            {"name": "GigabitEthernet100/100"},
        ],
        "match_key": "name",
        "source": "some_interfaces",
    },
]

op = {}
results = []

# Check for missing and duplicate match key
errors = []
for ds_idx, data_source in enumerate(data_sources):
    match_key = data_source["match_key"]
    ds_values = []

    for dd_idx, data_dict in enumerate(data_source["data"]):
        try:
            ds_values.append(data_dict[match_key])
        except KeyError:
            errors.append(f"Missing '{match_key}' in data source {ds_idx} in list entry {dd_idx}")
            continue

    if sorted(set(ds_values)) != sorted(ds_values):
        errors.append(f"Duplicate values in data source {ds_idx}")
    results.append(set(ds_values))
    print("\n".join(errors))


# Check for differences between data sources
errors = []
all_values = set(itertools.chain.from_iterable(results))

for ds_idx, ds_values in enumerate(results):
    missing_match = all_values - ds_values
    if missing_match:
        errors.append(f"Missing match value(s) {missing_match} in data source {ds_idx}")
print("\n".join(errors))


# Assemble the whole data structure
final_structure = {}
for data_source in data_sources:
    match_key = data_source["match_key"]
    source = data_source["source"]
    data_dict = {d[match_key]: d for d in data_source["data"] if match_key in d}
    for value in sorted(all_values):
        if value not in final_structure:
            final_structure[value] = {}
        final_structure[value][source] = data_dict.get(value, {})

        
print(yaml.dump(final_structure, default_flow_style=False))