reader config file prototype for https://github.com/lemon24/reader/issues/177

config.py

from reader import make_reader
from reader._plugins import import_string, Loader, LoaderError
import json
import click
from dataclasses import dataclass

IMPORT_KWARGS = ('storage_cls', 'search_cls')
MERGE_KWARGS = ('plugins', )


def merge_config(*configs):
    rv = {}
    to_merge = {}
    for config in configs:
        config = config.copy()
        for name in MERGE_KWARGS:
            if name in config:
                to_merge.setdefault(name, []).append(config.pop(name))
        rv.update(config)
    for name, dicts in to_merge.items():
        rv[name] = merge_config(*dicts)
    return rv


def make_reader_from_config(config):
    config = config.copy()

    plugins = config.pop('plugins', {})

    for name in IMPORT_KWARGS:
        thing = config.get(name)
        if thing and isinstance(thing, str):
            config[name] = import_string(thing)

    reader = make_reader(args)

    try:
        loader = Loader(plugins)
        loader.load_plugins(reader)
    except LoaderError as e:
        reader.close()
        raise

    return reader


def read_config(*args):
    return {
        'reader': {
            'url': 'config-reader-url',
            'plugins': {'reader-config-plugin': None},
        },
        'cli': {
            #'url': 'config-cli-url',
            'plugins': {'cli-config-plugin': None},
            'defaults': {
                #'db': 'cli-default-db',
                'plugin': ('cli-reader-default-plugin',),
                'update': {'new_only': True},
                'serve': {'plugin': ('cli-serve-default-plugin',)}
            },
        },
        'app': {
            'url': 'config-app-url',
            'plugins': {'app-config-plugin': None},
        }
    }


def get_default_values(cli, defaults):
    for param in cli.params:
        if param.default is not None:
            yield param.default

        default = defaults.get(param.name)
        if default is not None:
            yield default

    for command_name, command in getattr(cli, 'commands', {}).items():
        yield from get_default_values(command, defaults.get(command_name))


def is_default(value):
    return any(
        # this may fail for interned strings or ints or whatever
        value is v
        for v in click.get_current_context().find_root()._known_default_values
    )


def split_defaults(dict):
    defaults = {}
    options = {}
    for k, v in dict.items():
        if not v:
            continue
        (defaults if is_default(v) else options)[k] = v
    return defaults, options


def load_config_callback(ctx, _, config):
    ctx.default_map = config['cli'].pop('defaults', {})
    ctx._known_default_values = list(get_default_values(ctx.command, ctx.default_map))
    ctx.obj = config
    return config


# NOTE: the defaults from default_map won't show up in --help;
# "show_default and default_map don't play nice together"
# https://github.com/pallets/click/issues/1548


@click.group()
@click.option('--db', default='~/.config/reader/db.sqlite', show_default=True)
@click.option(
    '--config',
    type=read_config, default='~/.config/reader/config.file',
    callback=load_config_callback, is_eager=True, expose_value=False,
)
@click.option('--plugin', multiple=True)
@click.pass_obj
def cli(config, db, plugin):
    # cli default < config[cli][defaults] < config[reader] < config[cli,app] < cli envvar,option

    options = {
        'url': db,
        'plugins': {p: None for p in plugin},
    }

    default_options, user_options = split_defaults(options)

    # wrap reader section with options; 
    # will be used by app to spawn non-app readers
    config['reader_final'] = merge_config(
        default_options, 
        config['reader'], 
        user_options,
    )

    # merge reader and cli, wrapping with options
    config['cli_final'] = merge_config(
        default_options, 
        config['reader'], 
        config['cli'], 
        user_options,
    )

    # save defaults and options so other commands can use them
    config['options'] = options


@cli.command()
@click.option('--new-only/--no-new-only')
@click.pass_obj
def update(config, new_only):
    # the 1.5 reader._cli.pass_reader would do the merge for all of the commands
    print('new_only:', new_only)
    print('make_reader_from_config(cli_config) would get called with:')
    print(json.dumps(config['cli_final'], indent=4))


@cli.command()
@click.option('--plugin', multiple=True)
@click.pass_obj
def serve(config, plugin):
    # serve uses the app config section, not the cli one
    # (to mirror what WSGI would do)

    options = merge_config(config['options'], {
        'plugins': {p: None for p in plugin},
    })

    default_options, user_options = split_defaults(options)

    # merge reader and app, wrapping in our options and the cli-wide options
    config['app_final'] = merge_config(
        default_options,
        config['reader'],
        config['app'], 
        user_options,
    )

    print('make_app_from_config(reader_config, app_config) would get called with:')
    print(json.dumps(config['reader_final'], indent=4))
    print(json.dumps(config['app_final'], indent=4))


def wsgi():
    # reader._app.wsgi
    import os
    import reader._app, readed._config
    config = reader._config.read_config(os.environ['READER_CONFIG'])
    app = reader._app.create_app(app)


if __name__ == '__main__':
    cli()

config.yaml

# using YAML because it's easy to write,
# may end up using a different format

# this is the "default" section
reader:
    # make_reader args,
    # per https://github.com/lemon24/reader/issues/168#issuecomment-642002049

    feed_root: /path/to/feeds

    url: db.sqlite
    # alternatively (a wrapper needs to import it from a string),
    storage_cls: reader._storage:Storage
    storage_arg: ...

    # optional
    search_url: 'sqlite-parasitic:'
    # alternatively,
    search_cls: package.module:Class
    search_arg: ...

    # an extension handled by the extension
    plugins:
        package.module:
        module2:
            key: value

cli:
    # more make_reader args; they replace the reader.* stuff
    feed_root: ''

    # ... with the exception of plugins; these get merged
    plugins:
        cli.module:

    # another special case, CLI defaults (become the Click default_map);
    # the CLI needs to pop this before merging with 'reader'
    defaults:
        verbose: 0  # --verbose, global verbosity

        # this provides the default to --plugin; using --plugin *replaces*
        # the default (merge happens only for config[$section][plugins]).
        # the command group extends config[cli][plugins] with the --plugin values
        plugin: [default_cli_plugin]

        add:  # add --update -v
            update: yes
            verbose: 1

        update:  # update --new-only --workers 10 -vv
            new_only: yes
            workers: 10
            verbose: 2

        serve:  # serve --port 8888 --plugin whatever
            port: 8888
            # similar to the global --plugin, except
            # serve extends config[app][plugins] with the --plugin values
            plugin: [default_app_plugi]

app:
    # more make_reader args, same as for cli

    # app plugins, merged into reader.plugins
    plugins:
        webapp.module:
        'webapp.module2:init_app':