perfparams - convert a simple .conf file to couchdb .ini and vm.args files

perfparams.py

#!/usr/bin/env python3

import optparse
from dataclasses import dataclass
from collections import defaultdict


@dataclass
class Param:
    name: str
    val: str

@dataclass
class IniParam(Param):
    def get(self):
        sec, key = self.name.split('.', 1)
        sec, key, val = (sec.strip(), key.strip(), self.val.strip())
        keysplit = key.split('.')
        if len(keysplit) == 1:
            return (sec, key, val)
        elif len(keysplit) == 2:
            key, prop = keysplit
            return (sec, key, prop, val)

@dataclass
class VmParam(Param):
    SCHED_PARAMS = dict(
        SP     = lambda v : '+SP %s:%s' % (v,v),
        SDPCpu = lambda v : '+SDPCpu %s:%s' % (v,v),
    )

    def get(self):
        if self.name in VmParam.SCHED_PARAMS:
            convfun = VmParam.SCHED_PARAMS[self.name]
            return convfun(self.val)
        return f'+{self.name} {self.val}'

def convert(conf):
    ini_params = []
    vm_params = []
    for line in conf.splitlines():
        line = line.split("#")[0]
        line = line.strip()
        if not line:
            continue
        (k, v) = line.split("=")
        (k, v) = (k.strip(), v.strip())
        if k.startswith("vm."):
            k = k[len("vm."):]
            vm_params.append(VmParam(k, v))
        else:
            ini_params.append(IniParam(k, v))
    ini_result = render_ini(ini_params)
    vm_result = render_vm(vm_params)
    return (ini_result, vm_result)

def render_ini(ini_params):
    props = defaultdict(dict)
    params = []
    for param in ini_params:
        paramval = param.get()
        if len(paramval) == 3:
            params.append(param)
        if len(paramval) == 4:
            section, key, prop, val = paramval
            props[(section, key)][prop] = val
    for (section, key), propdict in props.items():
        propkvs = ["{%s,%s}" % (pk, pv) for (pk, pv) in propdict.items()]
        propstr = "[" + ",".join(propkvs) + "]"
        params.append(IniParam("%s.%s" % (section, key), propstr))
    ini = defaultdict(dict)
    for param in params:
        (section, key, val) = param.get()
        ini[section][key] = val
    output_lines = []
    for section in ini:
        keys = ini[section]
        output_lines.append(f'\n[{section}]')
        for key in keys:
            val = keys[key]
            output_lines.append(f'{key} = {val}')
    return '\n'.join(output_lines) + '\n'

def render_vm(vm_params):
    output_lines = ['']
    for param in vm_params:
        output_lines.append(param.get())
    return '\n'.join(output_lines) + '\n'

def main():
    opts, args = get_args()
    with open(opts.conf) as confh:
        conf = confh.read()
    print(f"Read input file: {opts.conf}")
    (ini, vmargs) = convert(conf)
    with open(opts.ini, "w") as inih:
        inih.write(ini)
    print(f"Wrote ini settings to: {opts.ini}")
    with open(opts.args, "w") as vmargsh:
        vmargsh.write(vmargs)
    print(f"Wrote vm.args settings to: {opts.args}")

def get_args():
    parser = optparse.OptionParser(description="Renders an .ini and vm.args file from conf file")
    parser.add_option(
        "-c",
        "--conf",
        default="params.conf",
        help="Input file",
    )
    parser.add_option(
        "-i",
        "--ini",
        default="local.ini",
        help="INI file to write to",
    )
    parser.add_option(
        "-a",
        "--args",
        default="vm.args",
        help="vm.args file to write to",
    )
    return parser.parse_args()


if __name__ == "__main__":
    main()

Example input file:

#
# CouchDB parameters
#

cluster.q = 2 # 2..64

couchdb.btree_chunk_size = 1279 # 64..65536
couchdb.update_lru_on_read = false
couchdb.file_compression = deflate_6 # none | snappy | deflate_1..9
couchdb.max_dbs_open = 500 # 500..5000

mem3.shard_cache_size = 25000 # 5000..100000

mem3.rev_chunk_size = 5000 # 100..15000
mem3.sync_concurrency = 10 # 1..100
mem3.sync_delay = 5000 # 1000..30000
mem3.sync_frequency = 500 # 100..10000

rexi.buffer_count = 2000 # 200..20000
rexi.use_kill_all = false
rexi.stream_limit = 5 # 1..100

database_compaction.doc_buffer_size = 524288 # 16384..67108864
database_compaction.checkpoint_after = 5242880 # 65536..67108864
view_compaction.keyvalue_buffer_size = 2097152 # 16384..67108864

ddoc_cache.max_size = 104857600 # 1048576..268435456
ddoc_cache.refresh_timeout = 67000 # 11000..310000

view_updater.queue_memory_cap = 100000 # 25000...1000000
view_updater.queue_item_cap = 500 # 100..10000
view_updater.min_writer_items = 100 # 10..1000
view_updater.min_writer_size = 16777216 # 4194304..67108864
query_server_config.commit_freq = 5 # 1..30

fabric.all_docs_concurrency = 10 # 1..1000
couchdb.changes_doc_ids_optimization_threshold = 100 # 10..1000

couchdb.attachment_stream_buffer_size = 4096 # 1024..1048576
attachments.compression_level = 8 # 1..9

query_server_config.os_process_limit = 100 # 50..1000
query_server_config.os_process_soft_limit = 100 # 20..1000
query_server_config.os_process_idle_limit = 300 # 50..1000
query_server_config.revs_limit = 20 # 5..100

chttpd.backlog = 512 # 16..2048
chttpd.server_options.acceptor_pool_size = 16 # 4..128
chttpd.server_options.sndbuf = undefined # undefined | 65536..1048576
chttpd.server_options.recbuf = undefined # undefined | 65536..1048576
chttpd.server_options.buffer = undefined # undefined | 65536..1048576
chttpd.server_options.nodelay = false

chttpd.socket_options.sndbuf = 262144    # undefined | 65536..1048576
chttpd.socket_options.recbuf = undefined # undefined | 65536..1048576
chttpd.socket_options.buffer = undefined # undefined | 65536..1048576
chttpd.socket_options.nodelay = true

off_heap_mqd.couch_log_server = true
off_heap_mqd.couch_server = true
off_heap_mqd.ddoc_cache_lru = true
off_heap_mqd.mem3_shards = true
off_heap_mqd.rexi_server = true

#
# Erlang VM params. These go into the vm.args file.
#

vm.SP = 100 # 10..100
vm.SDPcpu = 100 # 10..100
vm.SDio = 10 # 4..100
vm.stbt = u # u|ns|ts|ps|s|nnts|nnps|tnnps|db
vm.sbwt = medium # none|very_short|short|medium|long|very_long
vm.sbwtdio = short # none|very_short|short|medium|long|very_long
vm.scl = true
vm.spp = false
vm.sss = 128 # 40..8192
vm.sssdcpu = 40 # 40..8192
vm.sssdio = 40 # 40..8192
vm.sub = true
vm.swt = medium # very_low|low|medium|high|very_high
vm.swtdcpu = medium # very_low|low|medium|high|very_high
vm.swtdio = medium # very_low|low|medium|high|very_high
vm.hms = 233 # 233..4096
vm.hmbs = 46422 # 10000..100000
vm.zdbbl = 32768 # 8192..524288
vm.IOp = 1 # 1..10
vm.IOt = 1 # 1..10

Example rendered .ini:

[cluster]
q = 2

[couchdb]
btree_chunk_size = 1279
update_lru_on_read = false
file_compression = deflate_6
max_dbs_open = 500
changes_doc_ids_optimization_threshold = 100
attachment_stream_buffer_size = 4096

[mem3]
shard_cache_size = 25000
rev_chunk_size = 5000
sync_concurrency = 10
sync_delay = 5000
sync_frequency = 500

[rexi]
buffer_count = 2000
use_kill_all = false
stream_limit = 5

[database_compaction]
doc_buffer_size = 524288
checkpoint_after = 5242880

[view_compaction]
keyvalue_buffer_size = 2097152

[ddoc_cache]
max_size = 104857600
refresh_timeout = 67000

[view_updater]
queue_memory_cap = 100000
queue_item_cap = 500
min_writer_items = 100
min_writer_size = 16777216

[query_server_config]
commit_freq = 5
os_process_limit = 100
os_process_soft_limit = 100
os_process_idle_limit = 300
revs_limit = 20

[fabric]
all_docs_concurrency = 10

[attachments]
compression_level = 8

[chttpd]
backlog = 512
server_options = [{acceptor_pool_size,16},{sndbuf,undefined},{recbuf,undefined},{buffer,undefined},{nodelay,false}]
socket_options = [{sndbuf,262144},{recbuf,undefined},{buffer,undefined},{nodelay,true}]

[off_heap_mqd]
couch_log_server = true
couch_server = true
ddoc_cache_lru = true
mem3_shards = true
rexi_server = true

vm.args:

+SP 100:100
+SDPcpu 100
+SDio 10
+stbt u
+sbwt medium
+sbwtdio short
+scl true
+spp false
+sss 128
+sssdcpu 40
+sssdio 40
+sub true
+swt medium
+swtdcpu medium
+swtdio medium
+hms 233
+hmbs 46422
+zdbbl 32768
+IOp 1
+IOt 1