run various reader update commands in parallel, like cron would; https://github.com/lemon24/reader/issues/175

run-reader-update-like-cron-would.py

"""
Run `reader update` and `reader update --new-only && reader search update``
in parallel, like cron would, but at a "scaled down" timeframe.

We use vcrpy to avoid making network calls.

https://github.com/lemon24/reader/issues/175

"""
import os
import time
import random
from collections import Counter
from subprocess import run
from threading import Thread
from queue import Queue, Empty
from reader import make_reader, ReaderError
import logging
import vcr
import sqlite3

def reset_db():
    run("rm db.sqlite*", shell=True)
    run("cp reader.sqlite.2020-06-23 db.sqlite", shell=True)


MINUTELY_UPDATES_PER_MINUTE = 8


def my_make_reader():
    reader = make_reader('db.sqlite')
    reader._storage.db.execute(
        f"PRAGMA busy_timeout = {int(5000/MINUTELY_UPDATES_PER_MINUTE)};"
    )
    reader._pagination_chunk_size = int(256/MINUTELY_UPDATES_PER_MINUTE)
    return reader


def do_hourly():
    reader = my_make_reader()
    reader.update_feeds()

def do_minutely():
    my_make_reader().update_feeds(new_only=True)
    my_make_reader().update_search()


def worker(rv_q, fn, tag):
    try:
        fn()
        rv_q.put((tag, True))
    except ReaderError as e:
        assert "database is locked" in str(e)
        rv_q.put((tag, False))


def main():
    logging.basicConfig()
    logging.getLogger('reader').setLevel(logging.INFO)

    mode = 'wal'

    reset_db()        
    (mode,), = my_make_reader()._storage.db.execute(f"PRAGMA journal_mode = {mode};")

    rv_q = Queue()

    start = time.perf_counter()

    main_thread = Thread(target=worker, args=(rv_q, do_hourly, 'hourly'))
    main_thread.start()

    threads = []
    for i in range(MINUTELY_UPDATES_PER_MINUTE):
        Thread(target=worker, args=(rv_q, do_minutely, 'minutely'), daemon=True).start()

        main_thread.join(60 / MINUTELY_UPDATES_PER_MINUTE)
        if not main_thread.is_alive():
            break
    else:
        main_thread.join()

    end = time.perf_counter()
    
    counter = Counter()
    try:
        while True:
            counter.update([rv_q.get(False)])
    except Empty:
        pass

    print(mode, round(end - start, 2), dict(counter))

if False:
    # run this once to load the responses
    with vcr.use_cassette('cassette.yaml'):
        do_hourly()
else:
    with vcr.use_cassette('cassette.yaml'):
        main()