SQLite3 Read replica using LitestreamIO

restore.py

'''
Native read replicas might come soon to Litestream: 
- https://github.com/benbjohnson/litestream/issues/15
While waiting for that, the following script does a good enough job.
Our use case is restoring the DB on a different node for use with Grafana.
'''
import boto3
import subprocess
import os
from datetime import datetime, timezone
from time import sleep

INTERVAL=60
DB_NAME="my_db.db"
TEMP_NAME="my_db_temp.db"
S3_KEY="path/to/my_db.db"
BUCKET_NAME="my_bucket"

s3 = boto3.client('s3', region_name='ap-south-1')

# 1-1-1970 00:00:00
last_sync = datetime.fromtimestamp(0).replace(tzinfo=timezone.utc)

start_after = ''

while True:
    # Small optimisation to ensure we sync only if S3 has an updated object
    objects = s3.list_objects_v2(Bucket=BUCKET_NAME, Prefix=S3_KEY, StartAfter=start_after)
    contents = objects.get('Contents')
    last_modified = last_sync
    if contents:
        for o in contents:
            modified = o['LastModified']
            if modified > last_modified:
                start_after = o['Key']
                last_modified = modified
        if last_modified > last_sync:
            last_sync = last_modified
            print("Sync started with last_modified:", last_sync)
            subprocess.run(["litestream", "restore", "-o", TEMP_NAME, "s3://{}/{}".format(BUCKET_NAME, S3_KEY)])
            # we need to mv the file as litestream will not restore over an existing DB.
            subprocess.call(["mv", TEMP_NAME, DB_NAME])
            print("Sync completed")
    sleep(INTERVAL)