Skip to content

Instantly share code, notes, and snippets.

@hassansin
Last active September 16, 2023 13:32
Show Gist options
  • Save hassansin/bcb7cdfdcdb74f7c2427b7a0a23686b6 to your computer and use it in GitHub Desktop.
Save hassansin/bcb7cdfdcdb74f7c2427b7a0a23686b6 to your computer and use it in GitHub Desktop.
Working with MongoDB TTL (Time-To-Live) Index
> db.adminCommand({getParameter:1, ttlMonitorSleepSecs: 1});
{ "ttlMonitorSleepSecs" : 60, "ok" : 1 }
> db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 3600}); // 1 minute
{ "was" : 60, "ok" : 1 }
db.setLogLevel(1, "index");
db.adminCommand({setParameter:1, ttlMonitorEnabled:false});
/*Expire documents after a specified number of seconds*/
//find the expireAfterSeconds value from collection indexes
var expireAfterSeconds = db.ttldemo.getIndexes().filter(function(i){
return i.hasOwnProperty('expireAfterSeconds');
})[0].expireAfterSeconds;
// epoch time
var startTime = new Date(1970,0,1);
// end time
var endTime = new Date(Date.now() - expireAfterSeconds*1000);
db.collection.remove({created_at: { $gt: startTime, $lte: endTime }});
//drop db
db.ttldemo.drop();
//create ttl index in foreground with expireAfterSeconds
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});
// create some records that are already expired
// so they are eligible for removal when TTLMonitor runs next time
for(var i=0; i<10000; i++){
// With integer timestamp
if(i%2===0) {
db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000).getTime()});
}
// With BSON Date
else{
db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000)});
}
}
db.ttldemo.count(); //10000
sleep(60000); // wait one minute for TTL thread to kick in
db.ttldemo.count(); //5000, only documents with BSON Date indexes field are deleted
/*## Expire Documents at a Specific Clock Time or Dynamic TTL*/
db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 5}); // run TTL monitor in every 5 seconds
sleep(60000); // wait a minute for new sleep interval to take effect
db.ttldemo.drop();
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 0});
// Create document with Random expiry time ranging from 0 to 60 seconds in future
for(var i=0; i< 10000; i++){
db.ttldemo.insert({
created_at: new Date(Date.now() + Math.floor(Math.random()*60000))
})
}
for(var i=0; i<13;i++){
print(db.ttldemo.count());
sleep(5*1000); //wait 5 seconds
}
db.adminCommand({setParameter:1, ttlMonitorSleepSecs: 60});
/*## TTL Thread Stats*/
db.serverStatus().metrics.ttl
{
"deletedDocuments": NumberLong(918478),
"passes": NumberLong(24818)
}
/*## TTL index on Capped collection*/
db.createCollection('ttlCappedDemo', {capped: true, size: 4096});
db.ttlCappedDemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});
db.ttlCappedDemo.insert({created_at: new Date(Date.now() - 60*1000)});
db.ttlCappedDemo.count(); // 1
sleep(60000); // wait one minute for TTL thread to kick in
db.ttlCappedDemo.count(); // still 1
db.ttlCappedDemo.drop();
/*## Update TTL Index options*/
db.ttldemo.drop();
// create TTL index with 60 seconds expiration threshold
db.ttldemo.createIndex({created_at: 1}, {expireAfterSeconds: 60});
db.ttldemo.getIndexes();
// Update the index with 120 seconds expiration threshold
db.runCommand({
collMod: 'ttldemo',
index: {keyPattern: {created_at: 1}, expireAfterSeconds: 120}
});
[
{
"v" : 1,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "test.ttldemo"
},
{
"v" : 1,
"key" : {
"created_at" : 1
},
"name" : "created_at_1",
"ns" : "test.ttldemo",
"expireAfterSeconds" : 60
}
]
{ "expireAfterSeconds_old" : 60, "expireAfterSeconds_new" : 120, "ok" : 1 }
/*## TTL with Partial Indexes*/
db.ttldemo.drop();
db.ttldemo.createIndex( {created_at: 1}, {
expireAfterSeconds: 60,
partialFilterExpression: {
z: { $exists: true }
}
});
db.ttldemo.insert({created_at: new Date(Date.now() - 60*1000)});
db.ttldemo.insert({
created_at: new Date(Date.now() - 60*1000),
z: 1
});
db.ttldemo.count(); // 2
sleep(60*1000);
db.ttldemo.count(); // 1
db.ttldemo.count({z: {$exists: true}}); // 0
/*## TTL Monitor on Replica Sets*/
//Primary
"repl": {
"ismaster": true,
"secondary": false,
},
"metrics": {
"ttl": {
"deletedDocuments": 921860,
"passes": 25434
}
}
//Secondary
"repl": {
"ismaster": false,
"secondary": true,
},
"metrics": {
"ttl": {
"deletedDocuments": 0,
"passes": 25375
}
}
/*## Find TTL indexes*/
var ttlIndexes = db.getCollectionNames().reduce((prev, collection) => {
return [...prev, ...db[collection].getIndexes().filter( (i) => {
return i.hasOwnProperty('expireAfterSeconds') && Number.isInteger(i.expireAfterSeconds);
})]
}, [])
printjson(ttlIndexes);
[
{
"v" : 1,
"key" : {
"created_at" : 1
},
"name" : "created_at_1",
"ns" : "test.ttldemo",
"expireAfterSeconds" : 0
}
]
## TTLMonitor Thread Log
2016-04-07T23:57:42.908+0000 D INDEX [TTLMonitor] TTLMonitor thread awake
2016-04-07T23:57:42.908+0000 D INDEX [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-07T23:57:42.908+0000 D INDEX [TTLMonitor] TTL deleted: 0
2016-04-07T23:58:42.908+0000 D INDEX [TTLMonitor] TTLMonitor thread awake
2016-04-07T23:58:42.908+0000 D INDEX [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-07T23:58:43.057+0000 D INDEX [TTLMonitor] TTL deleted: 10000
## Expire Documents at a Specific Clock Time or Dynamic TTL
9833
9077
8299
7459
6653
5785
4936
4123
3263
2400
1536
659
42
## TTL index on Capped collection
2016-04-08T00:25:43.244+0000 D INDEX [TTLMonitor] TTLMonitor thread awake
2016-04-08T00:25:43.244+0000 D INDEX [TTLMonitor] TTL -- ns: test.ttlCappedDemo key: { created_at: 1.0 }
2016-04-08T00:25:43.244+0000 I STORAGE [TTLMonitor] failing remove on a capped ns test.ttlCappedDemo
2016-04-08T00:25:43.267+0000 E INDEX [TTLMonitor] Error processing ttl index: { v: 1, key: { created_at: 1.0 }, name: "created_at_1", ns: "test.ttlCappedDemo", expireAfterSeconds: 60.0 } -- 10089 cannot remove from a capped collection
2016-04-08T00:25:43.267+0000 D INDEX [TTLMonitor] TTL -- ns: test.ttldemo key: { created_at: 1.0 }
2016-04-08T00:25:43.267+0000 D INDEX [TTLMonitor] TTL deleted: 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment