Working with MongoDB TTL (Time-To-Live) Index

README.md

Working With MongoDB TTL Index

https://hassansin.github.io/working-with-mongodb-ttl-index

commands.js

> db.adminCommand({getParameter:1, ttlMonitorSleepSecs: 1});
{ "ttlMonitorSleepSecs" : 60, "ok" : 1 }

> db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 3600}); // 1 minute
{ "was" : 60, "ok" : 1 }

db.setLogLevel(1, "index");

db.adminCommand({setParameter:1, ttlMonitorEnabled:false});

/*Expire documents after a specified number of seconds*/

//find the expireAfterSeconds value from collection indexes
var expireAfterSeconds = db.ttldemo.getIndexes().filter(function(i){
    return i.hasOwnProperty('expireAfterSeconds');
})[0].expireAfterSeconds;

// epoch time
var startTime = new Date(1970,0,1);
// end time
var endTime = new Date(Date.now() - expireAfterSeconds*1000);
db.collection.remove({created_at: { $gt: startTime, $lte: endTime }});


//drop db
db.ttldemo.drop();

//create ttl index in foreground with expireAfterSeconds
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});

// create some records that are already expired
// so they are eligible for removal when TTLMonitor runs next time

for(var i=0; i<10000; i++){

    // With integer timestamp
    if(i%2===0) {
        db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000).getTime()});
    }
    // With BSON Date
    else{
        db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000)});
    }

}

db.ttldemo.count();  //10000

sleep(60000); // wait one minute for TTL thread to kick in

db.ttldemo.count();   //5000, only documents with BSON Date indexes field are deleted

/*## Expire Documents at a Specific Clock Time or Dynamic TTL*/

db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 5}); // run TTL monitor in every 5 seconds
sleep(60000); // wait a minute for new sleep interval to take effect

db.ttldemo.drop();
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 0});

// Create document with Random expiry time ranging from 0 to 60 seconds in future
for(var i=0; i< 10000; i++){
    db.ttldemo.insert({
        created_at: new Date(Date.now() + Math.floor(Math.random()*60000))
    })
}
for(var i=0; i<13;i++){
    print(db.ttldemo.count());
    sleep(5*1000); //wait 5 seconds
}
db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 60});

/*## TTL Thread Stats*/

db.serverStatus().metrics.ttl
{
    "deletedDocuments": NumberLong(918478),
    "passes": NumberLong(24818)
}

/*## TTL index on Capped collection*/


db.createCollection('ttlCappedDemo', {capped: true, size: 4096});
db.ttlCappedDemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});
db.ttlCappedDemo.insert({created_at: new Date(Date.now() - 60*1000)});
db.ttlCappedDemo.count(); // 1
sleep(60000); // wait one minute for TTL thread to kick in
db.ttlCappedDemo.count();  // still 1
db.ttlCappedDemo.drop();


/*## Update TTL Index options*/

db.ttldemo.drop();
// create TTL index with 60 seconds  expiration threshold
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});
db.ttldemo.getIndexes();

// Update the index with 120 seconds expiration threshold
db.runCommand({
    collMod: 'ttldemo',
    index: {keyPattern: {created_at: 1}, expireAfterSeconds: 120}
});

[
  {
    "v" : 1,
    "key" : {
            "_id" : 1
    },
    "name" : "_id_",
    "ns" : "test.ttldemo"
  },
  {
    "v" : 1,
    "key" : {
            "created_at" : 1
    },
    "name" : "created_at_1",
    "ns" : "test.ttldemo",
    "expireAfterSeconds" : 60
  }
]

{ "expireAfterSeconds_old" : 60, "expireAfterSeconds_new" : 120, "ok" : 1 }

/*## TTL with Partial Indexes*/

db.ttldemo.drop();
db.ttldemo.createIndex( {created_at: 1}, {
    expireAfterSeconds: 60,
    partialFilterExpression: {
        z: { $exists: true }
    }
});
db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000)});
db.ttldemo.insert({
    created_at: new Date(Date.now() - 60*1000),
    z: 1
});
db.ttldemo.count(); // 2
sleep(60*1000);
db.ttldemo.count(); // 1
db.ttldemo.count({z: {$exists: true}}); // 0

/*## TTL Monitor on Replica Sets*/

//Primary
"repl": {
    "ismaster": true,
    "secondary": false,
},
"metrics": {
    "ttl": {
        "deletedDocuments": 921860,
        "passes": 25434
    }
}

//Secondary
"repl": {
    "ismaster": false,
    "secondary": true,
},
"metrics": {
    "ttl": {
        "deletedDocuments": 0,
        "passes": 25375
    }
}


/*## Find TTL indexes*/

var ttlIndexes = db.getCollectionNames().reduce((prev, collection) => {
    return [...prev, ...db[collection].getIndexes().filter( (i) => { 
        return i.hasOwnProperty('expireAfterSeconds') && Number.isInteger(i.expireAfterSeconds);
    })]
}, [])
printjson(ttlIndexes);

[
    {
        "v" : 1,
        "key" : {
                "created_at" : 1
        },
        "name" : "created_at_1",
        "ns" : "test.ttldemo",
        "expireAfterSeconds" : 0
    }
]

output.txt

## TTLMonitor Thread Log

2016-04-07T23:57:42.908+0000 D INDEX    [TTLMonitor] TTLMonitor thread awake
2016-04-07T23:57:42.908+0000 D INDEX    [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-07T23:57:42.908+0000 D INDEX    [TTLMonitor]    TTL deleted: 0

2016-04-07T23:58:42.908+0000 D INDEX    [TTLMonitor] TTLMonitor thread awake
2016-04-07T23:58:42.908+0000 D INDEX    [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-07T23:58:43.057+0000 D INDEX    [TTLMonitor]    TTL deleted: 10000


## Expire Documents at a Specific Clock Time or Dynamic TTL

9833
9077
8299
7459
6653
5785
4936
4123
3263
2400
1536
659
42

## TTL index on Capped collection

2016-04-08T00:25:43.244+0000 D INDEX    [TTLMonitor] TTLMonitor thread awake
2016-04-08T00:25:43.244+0000 D INDEX    [TTLMonitor] TTL -- ns: test.ttlCappedDemo key: { created_at: 1.0 }
2016-04-08T00:25:43.244+0000 I STORAGE  [TTLMonitor] failing remove on a capped ns test.ttlCappedDemo
2016-04-08T00:25:43.267+0000 E INDEX    [TTLMonitor] Error processing ttl index: { v: 1, key: { created_at: 1.0 }, name: "created_at_1", ns: "test.ttlCappedDemo", expireAfterSeconds: 60.0 } -- 10089 cannot remove from a capped collection
2016-04-08T00:25:43.267+0000 D INDEX    [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-08T00:25:43.267+0000 D INDEX    [TTLMonitor]    TTL deleted: 0