minio-stats.py

import redis
import time
import json

"""
nodes = {
    'minio': [
        {
            'host': '10.241.0.232',
            'port': 6379,
            'channel': 'minio-stats',
            'client': None,
            'previous': None,
        }
    ],
    'storage': [
        {
            'host': '2a02:2788:54:1096:e459:8eff:fe52:1859',
            'port': 9900,
            'namespace': '91-1',
            'client': None,
        },
        {
            'host': '2a02:2788:54:1096:84c5:fcff:fe3c:5dd4',
            'port': 9900,
            'namespace': '91-0',
            'client': None,
        }
    ]
}
"""

class MinioStats:
    def __init__(self, nodes):
        self.nodes = nodes
        self.connect()

    def connect(self):
        for minio in self.nodes['minio']:
            print("[+] connecting minio [%s, %d]" % (minio['host'], minio['port']))
            r = redis.Redis(minio['host'], minio['port'])

            minio['client'] = r.pubsub()
            minio['client'].subscribe(minio['channel'])

            print("[+] waiting for minio statistics")

            while True:
                message = minio['client'].get_message()
                if message == None:
                    continue

                if message['type'] == 'subscribe':
                    break

                time.sleep(0.1)

        for zdb in self.nodes['storage']:
            print("[+] connecting zdb [%s, %d]" % (zdb['host'], zdb['port']))
            r = redis.Redis(zdb['host'], zdb['port'])
            zdb['client'] = r

        if len(self.nodes['minio']) == 0:
            print("[-] WARNING: no minio container found on config")
            time.sleep(3)

    def monitor(self):
        print("")
        print("  == Monitoring ==")
        print("")

        while True:
            self.monitor_once()
            time.sleep(1)

    def zdb_info(self, client, namespace):
        info = client.execute_command("NSINFO %s" % namespace)
        info = info.decode('utf-8')

        data = info.split("\n")
        values = {}

        for entry in data:
            if entry == '' or entry.startswith('#'):
                continue

            keys = entry.split(': ')
            values[keys[0]] = keys[1]

        return values

    def minio_info(self, client):
        while True:
            info = client.get_message()
            if info == None:
                time.sleep(0.1)
                continue

            if info['type'] != 'message':
                continue

            data = json.loads(info['data'].decode('utf-8'))
            return data

    def monitor_once(self):
        for minio in self.nodes['minio']:
            info = self.minio_info(minio['client'])

            if minio['previous'] == None:
                minio['previous'] = info
                continue

            usage = info['cpu_usage'] - minio['previous']['cpu_usage']
            difftime = info['timestamp'] - minio['previous']['timestamp']
            computed = usage / (difftime * 10000000)

            minio['previous'] = info

            print("CPU Usage: %0.2f%% / RAM: %.2f MB" % (computed, info['memory_usage'] / 1000000))

        index = 1
        total = 0

        for zdb in self.nodes['storage']:
            info = self.zdb_info(zdb['client'], zdb['namespace'])
            print("Storage #%d: %s MB" % (index, info['data_size_mb']))

            index += 1
            total += float(info['data_size_mb'])

        print("Storage (total): %.2f MB" % total)

        print("")

if __name__ == '__main__':
    filename = "/tmp/minio-nodes.json"
    nodes = {}

    with open(filename) as f:
        nodes = json.loads(f.read())

    stats = MinioStats(nodes)
    stats.monitor()

minio.sh

#!/bin/bash
set -e

tfuser="/opt/go/src/github.com/threefoldtech/tfexplorer/cmds/tfuser/tfuser"
bcdb="http://127.0.0.1:8080/explorer"
seed="/tmp/zos/converted.seed"

zdbs() {
    args=""

    for node in $(cat nodes); do
        echo "[+] building zdb: $node"
        $tfuser --bcdb $bcdb --seed $seed generate storage zdb --node $node --size 4 --type ssd --mode seq --password supersecret > zdb-$node.json
        args="$args --zdb zdb-$node.json --duration 2h"
    done

    resource="http://127.0.0.1:8080/explorer/reservations/73"

    winfo=$($tfuser --seed $seed --bcdb $bcdb provision $args | grep ^Resou | awk '{ print $2 }')
    resource="${bcdb}${winfo}"

    echo "[+] provision sent, waiting deployment [$resource]"
    sleep 10

    echo "[+] fetching: $resource"
    curl $resource | jq
}

minio() {
    cfg="91-1:supersecret@[2a02:2788:54:1096:e459:8eff:fe52:1859]:9900,91-0:supersecret@[2a02:2788:54:1096:84c5:fcff:fe3c:5dd4]:9900"

    $tfuser --bcdb $bcdb --seed $seed generate container \
        --node $(cat minihost) \
        --flist https://hub.grid.tf/tf-official-apps/minio:latest.flist \
        --network mx_core_minio \
        --ip 172.25.1.12 \
        --memory 1024 \
        --envs "SHARDS=$cfg" \
        --envs DATA=1 \
        --envs PARITY=1 \
        --envs ACCESS_KEY=minio \
        --envs SECRET_KEY=passwordpassword \
        --envs "SSH_KEY=ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxTTKdZ6equQKnjlBW4Tty52Rq2nsKl424r5mf75Qa/24Vwjh4Mw14XgV0OAnBvsT9PBAGUIiLFdHr3iXJHizBTrr3oExaM9G0ixH/bOTvXxeeomBxoPRv1pZRgYqS8wGtP1FB3550aPXeYh6dxmT8Vk3x4TATrASjBCJ4QSMzmQUusiNbqJgrGDV2RHWESet1tCtxpJo2/bEU3BDQutELiKYmD0ugqCBmMb3lVzDTovh26OBYf8+0oWy55vfuK/Pcl+2lwvZuLkDrSZBsY9yfAEIvQCRZIwkCOHoUOoUJQviIku54DGpVl9T+SNHeI4ZYYyeN3gsz+O3o9aUgXv5Z" \
        --public6 \
        --stats redis://10.241.0.232:6379/minio-stats \
        --stdout redis://10.241.0.232:6379/minio-stdout \
        --stderr redis://10.241.0.232:6379/minio-stderr > container.json

    $tfuser --bcdb $bcdb --seed $seed provision --container container.json --duration 2h
}

# zdbs
# minio

mksetup.py

from urllib.parse import urlparse
import requests
import time
import json
import pprint
import os

class MinioSetup:
    def __init__(self, output=None):
        self.output = output
        self.nodes = {
            'minio': [],
            'storage': [],
        }

        if output is not None:
            if os.path.exists(output):
                self.preload()

    def preload(self):
        pass

    def fetch(self, url):
        print("[+] fetching: %s" % url)

        r = requests.get(url)
        data = r.json()

        for container in data['data_reservation']['containers']:
            # skipping non minio container
            if container['flist'] != 'https://hub.grid.tf/tf-official-apps/minio:latest.flist':
                continue

            if len(container['stats_aggregator']) == 0:
                print("[-] WARNING: minio container doesn't have stats aggregator")
                continue

            url = urlparse(container['stats_aggregator'][0]['data']['endpoint'])

            self.nodes['minio'].append({
                'host': url.netloc.split(':')[0],
                'port': url.netloc.split(':')[1],
                'channel': url.path[1:],
                'client': None,
                'previous': None,
            })


        for result in data['results']:
            # skipping non zdb result
            if 'IPs' not in result['data_json'] or  'Namespace' not in result['data_json']:
                continue

            info = result['data_json']
            self.nodes['storage'].append({
                'host': info['IPs'][0],
                'port': info['Port'],
                'namespace': info['Namespace'],
                'client': None
            })

    def save(self):
        payload = json.dumps(self.nodes)

        if self.output is None:
            print(payload)
            return

        with open(self.output, 'w') as f:
            print("[+] writing configuration: %s" % self.output)
            f.write(payload + "\n")

if __name__ == '__main__':
    setup = MinioSetup("/tmp/nodes.json")
    setup.fetch("https://explorer.grid.tf/explorer/reservations/11047")
    setup.fetch("https://explorer.grid.tf/explorer/reservations/11048")
    setup.save()