ollama python utility script

olu.py

#!/usr/bin/env python3
import json
import os
import shutil
import sys

OLLAMA_PATH = '/home/ollama'

def loadJSON(filename):
    if os.path.exists(filename):
        with open(filename) as f:
            return json.load(f)
    return []

def saveJSON(filename, data):
    folder = os.path.dirname(filename)
    os.makedirs(folder, exist_ok=True)
    f = open(filename, 'w')
    f.write(json.dumps(data, indent=2))

class MediaBlob:
    def __init__(self, type = '', digest = '', size = 0):
        self.type = type
        self.digest = digest
        self.size = size
        
    def path(self):
        return os.path.join("/home/ollama/models/blobs/", self.digest)
    
    def getSize(self):
        return os.path.exists(self.path()) and os.path.getsize(self.path())
    
    def ok(self):
        return os.path.exists(self.path()) and self.size == self.getSize()
    
    @classmethod
    def ParseType(self, type):
        if 'container.image' in type:
            return 'config'
        return type.split('.')[-1]

    @classmethod
    def Parse(self, jdata: dict):
        rv = MediaBlob()
        rv.type = MediaBlob.ParseType(jdata['mediaType'])
        rv.digest = jdata['digest']
        rv.size = jdata['size']
        return rv
    
    def to_dict(self):
        return {
            'type': self.type,
            'digest': self.digest,
            'size': self.size,
            'path': self.path(),
            'actual_size': self.getSize(),
            'ok': self.ok(),
            'content': self.content(),
        }
        
    def content(self):
        if self.type != 'model' and self.getSize() < 1024:
            if self.type in ['config', 'params']:
                return loadJSON(self.path())
            return open(self.path(), 'r').read()

class Manifest:
    def __init__(self):
        self.name = ''
        self.config: MediaBlob = MediaBlob()
        self.layers: dict[str, MediaBlob] = {}
    
    @classmethod
    def LibPath(self):
        return os.path.join(OLLAMA_PATH, 'models', 'manifests', 'registry.ollama.ai', 'library')
    
    @classmethod
    def List(self, opt = None):
        path = Manifest.LibPath()
        dirs = os.listdir(path)
        rv = []
        for dir in dirs:
            filenames = os.listdir(os.path.join(path, dir))
            for filename in filenames:
                name = f"{dir}:{filename}"
                line = name
                show = True
                if opt:
                    show = False
                    mf = Manifest.Load(name)
                    if opt == 'sha':
                        line += ',' + mf.layers['model'].digest
                        show = True
                    elif opt == 'ctx':
                        params = mf.layers['params'].content()
                        if 'num_ctx' in params:
                            line += ' ' + str(params['num_ctx'])
                            show = True
                if show:
                    rv.append(line)
        rv.sort()
        return rv
    
    @classmethod
    def Load(self, name):
        filename = self.path(name)
        jdata = loadJSON(filename)
        rv = Manifest()
        if len(jdata) == 0:
            return rv
        rv.name = name
        rv.config = MediaBlob.Parse(jdata['config'])
        for layer in jdata['layers']:
            layer_name = MediaBlob.ParseType(layer['mediaType'])
            if layer_name != 'config':
                rv.layers[layer_name] = MediaBlob.Parse(layer)
        return rv
            
    @classmethod
    def path(self, name):
        name = str.replace(name, ':', os.path.sep)
        return os.path.join(Manifest.LibPath(), name)
        
    def to_dict(self):
        layers: dict[str, MediaBlob] = {}
        for layer in self.layers.values():
            layers[layer.type] = layer.to_dict()
        return {
            'name': self.name,
            'path': self.path(self.name),
            'config': self.config.to_dict(),
            'layers': layers,
        }
        
    def SaveParams(self, params):
        filename = self.layers['params'].path()
        backup = f"{filename}~"
        # backup original params first time
        if not os.path.exists(backup):
            shutil.copy2(filename, backup)
        saveJSON(filename, params)

    def print(self):
        print(json.dumps(self.to_dict(), indent=2))
    
if __name__ == '__main__':
    if len(sys.argv) == 1:
        print(f"usage: {sys.argv[0]} command")
        print("list [ctx|sha]")
        print("read model_name")
        print("setctx model_name value")
        print("restore_params model_name")
        sys.exit()
    
    if len(sys.argv) >= 2 and sys.argv[1] == 'list':
        if len(sys.argv) == 3:
            opt = sys.argv[2]
        else:
            opt = None
        rv = Manifest.List(opt)
        print('\n'.join(rv))

    elif len(sys.argv) == 3 and sys.argv[1] == 'read':
        name = sys.argv[2]
        manifest = Manifest.Load(name)
        manifest.print()

    elif len(sys.argv) == 4 and sys.argv[1] == 'setctx':
        name = sys.argv[2]
        value = int(sys.argv[3])
        manifest = Manifest.Load(name)
        params = manifest.layers['params'].content()
        params['num_ctx'] = value
        manifest.SaveParams(params)

    elif len(sys.argv) == 3 and sys.argv[1] == 'restore_params':
        name = sys.argv[2]
        manifest = Manifest.Load(name)
        path = manifest.layers['params'].path()
        backup = f"{path}~"
        if not os.path.exists(backup):
            print("no backup to restore from, running setctx will automatically create a backup")
        else:
            print(f"restoring params from {backup}")
            shutil.copy2(backup, path)

Ollama python utility script

This is a script I wrote to view and edit the default model context size setting (num_ctx)

It can be used to fix models that have the wrong context size set, which display the following error:
Error: llama runner process has terminated

It assumes your ollama user folder is located in /home/ollama

Save olu.py to any folder and open a terminal in that folder.

Commands:

List all installed models
./olu.py list

Only list installed models with a context size (num_ctx) set.
If a model does not appear, then ollama will use the setting thats hard coded into the model (usually 2048 for llama based models).
./olu.py list ctx

Example

$ ./olu.py list ctx
dl:orangetin-openhermes-mixtral-8x7b.Q4_K_M 32768
dl:orca2-13b-16k-q8_0 16384
dl:starling-lm-7b-alpha.Q8_0 4096
dolphin-mixtral:8x7b-v2.6-q6_K 16384
llama2:70b 4096
mixtral:8x7b-instruct-v0.1-q4_K_M 32768
openchat:7b-v3.5-q6_K 4096
orca2:13b-q8_0 8192
wizardcoder:34b-python 16384
yarn-mistral:7b-128k-q6_K 131072

List installed models with their sha256 hash
./olu.py list sha

View model configuration metadata
./olu.py read dolphin-mixtral

Change the num_ctx setting for dolphin-mixtral.
Parameters are backed up automatically the first time this is run on a model.
sudo ./olu.py setctx dolphin-mixtral 16384

Restore the original parameters (only if setctx has been used previously)
sudo ./olu.py restore_params dolphin-mixtral

Limitations
sudo is required to change or restore settings.

Due to the way Ollama stores it's configuration, some models may share the same parameters file if they have the same base model.
If you change num_ctx for one, it'll apply to all models using the same parameters file.

eg.
codellama:34b-instruct and codellama:34b-instruct-q6_K