Local inference - llama command line and OMP config - 26 Sep 2026

config.yml

modelRoles:
  default: qwen/unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_M:xhigh
modelTags:
  qwen:
    name: Qwen 3.8 27B
cycleOrder:
  - default
symbolPreset: unicode
theme:
  dark: titanium
setupVersion: 2
defaultThinkingLevel: xhigh
composer:
  shape: box
spelling:
  autocomplete: false
hideThinkingBlock: false
compaction:
  thresholdPercent: 85
  thresholdTokens: -1
  asyncEnabled: false
  idleEnabled: false
web_search:
  enabled: true
exa:
  enabled: true
searxng:
  endpoint: http://SEARXNG_IP_ADDRESS_GOES_HERE:12001
recap:
  enabled: false
completion:
  notify: "off"

models.yml

providers:
  qwen:
    baseUrl: http://LLAMA_IP_ADDRESS_GOES_HERE:12000/v1
    api: openai-completions
    auth: none
    authHeader: false
    models:
      - id: unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_M
        name: Qwen 3.8 27B
        reasoning: true
        thinking:
          mode: effort
          efforts: [low, medium, xhigh]
          defaultLevel: xhigh
        input: [text]
        supportsTools: true
        cost:
          input: 0
          output: 0
          cacheRead: 0
          cacheWrite: 0
        contextWindow: 131072
        maxTokens: 65535
        compat:
          supportsReasoningParams: true
          replayReasoningContent: true
          thinkingFormat: qwen-chat-template
          reasoningDisableMode: qwen-template-false
          qwenPreserveThinking: true
          qwenTemplateReasoningEffort: true

run-llama.bat

.\llama-server.exe -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_M -np 1 -c 131072 -fa on --cache-type-k q8_0 --cache-type-v q8_0 -ub 512 -b 2048 -fit off -ngl all --spec-type none --cache-idle-slots  --spec-type draft-mtp --spec-draft-model .\models\mtp-Qwen3.8-27B-Q4_0.gguf --spec-draft-ngl all --spec-draft-n-max 2 --spec-draft-type-k q4_0 --spec-draft-type-v q4_0 --sleep-idle-seconds 900 --cache-ram 0 --port 12000 --host LISTEN_IP_ADDRESS_GOES_HERE