Understanding Aggregations on MongoBD

aggregate_lookup_events_for_server.js

// 1000 servers
// 1000000 events, randomly distributed across servers

db.getCollection("server").aggregate([
    { $match: { created_by: "sivy"}},
    { $lookup: {
        from: "event",
        let: { server_uuid: "$instance_uuid" },
        pipeline: [
            { $match: {
                $expr: {
                    $and: [
                        { $eq: ["$instance_uuid", "$$server_uuid"]},
                    ],
                }
            }},
            // {$project: { message: 1 }},
            {$count: "count"},
        ],
        as: "events",
    }},
    {$match: { events: { $ne: [] }}},
])

event.js

// example event document
// there maybe 100-1000 events per server, and it should be possible to 
// run queries against just event data.
// event_data just for example, can be 20x this size
{
    "_id" : ObjectId("5ab543355ecc0926d6f75697"),
    "instance_uuid" : "bb9d9e74-7122-4b11-92a2-a29216500475",
    "message" : "Event 4",
    "event_type" : "info",
    "label" : "puppet",
    "event_data" : {
        "command" : "C:\"\\Program Files\\Puppet Labs\\Puppet\"\\bin\\puppet config set server puppet.devint.pce.wellsfargo.net --section main",
        "puppet_master" : "puppet.devint.pce.wellsfargo.net"
    }
}

results.js

// Machine: Macbook Pro i7 2.2, 16G RAM
// time: 533 sec
[
  {
    "_id" : ObjectId("5ab5353b5ecc091c40246eef"),
    "instance_uuid" : "bb9d9e74-7122-4b11-92a2-a29216500003",
    "name" : "Server 3",
    "created_by" : "sivy",
    "events" : [ 
        {
            "count" : 899
        }
    ]
  }
  // ... 1000 servers
]

server.js

// example server document
// realitically they're 10x this size
{
    "_id" : ObjectId("5ab5353b5ecc091c40246eec"),
    "instance_uuid" : "bb9d9e74-7122-4b11-92a2-a29216500000",
    "name" : "Server 0",
    "created_by" : "sivy"
}