This guide walks through setting up Kimodo from scratch, downloading a custom text encoder (KIMODO-Meta3_llm2vec_NF4), and forcing the application to use your local offline model instead of making API calls to Hugging Face.
First, create a fresh Python virtual environment to keep dependencies clean, and activate it.
python -m venv venv
source venv/bin/activateUpgrade the Hugging Face Hub package and download the specific model snapshot directly to your local directory.
pip install --upgrade huggingface_hub
# This will download the model to a folder named './KIMODO-Meta3_llm2vec_NF4'
python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='Aero-Ex/KIMODO-Meta3_llm2vec_NF4', local_dir='./KIMODO-Meta3_llm2vec_NF4')"Next, grab the core Kimodo repository and its Viser dependency, then install them in editable mode.
git clone https://github.com/Aero-Ex/kimodo.git
cd kimodo
git clone https://github.com/nv-tlabs/kimodo-viser.git
pip install -e kimodo-viser
set SKIP_MOTION_CORRECTION_IN_SETUP=1 && pip install -e .since we are skipping the building process for motion_correction package you have to use following to install on your system based on the python version.
Grab the appropriate wheel for your OS and Python version from the Aero-Ex/kimodo releases page.
For example, to install the Python 3.12 wheel for Windows, run:
pip install https://github.com/Aero-Ex/kimodo/releases/download/v1.0.0/motion_correction-1.0.0-cp312-cp312-win_amd64.whlNow, install the required quantization and transformer libraries.
pip install bitsandbytes
pip install -U transformers==5.1.0To force Kimodo to use your newly downloaded local model, you need to provide the path of downloaded text_encoder its default wrapper file.
Open the following file in your text editor:
kimodo/kimodo/model/llm2vec/llm2vec_wrapper.py
Important: Make sure to update the
custom_dirvariable on line 27 with the actual absolute path to the folder you downloaded in Step 2.
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""LLM2Vec encoder wrapper for Kimodo text conditioning."""
import os
import gc
import numpy as np
import torch
from torch import nn
from .llm2vec import LLM2Vec
class LLM2VecEncoder(nn.Module):
"""LLM2Vec text embeddings."""
def __init__(
self,
base_model_name_or_path: str,
peft_model_name_or_path: str,
dtype: str,
llm_dim: int,
) -> None:
super().__init__()
self.torch_dtype = getattr(torch, dtype)
self.llm_dim = llm_dim
# Update this path to where your model is actually located!
self.custom_dir = "/home/aero/kimodo/KIMODO-Meta3_llm2vec_NF4"
print(f"[LLM2VecEncoder] Initialized (Waiting for first use to load weights)...")
self.model = None
def unload(self):
"""Offload the model weights to System RAM (CPU) if currently on GPU."""
if self.model is not None:
if self.get_device().type == "cuda":
print(f"[LLM2VecEncoder] Offloading 5.4GB model to System RAM...")
self.model.model.to("cpu")
gc.collect()
import platform
if platform.system() == "Linux":
try:
import ctypes
ctypes.CDLL("libc.so.6").malloc_trim(0)
except Exception:
pass
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
def reload(self):
"""Move from System RAM to VRAM."""
if self.model is None:
print(f"[LLM2VecEncoder] Model was None. Reloading from disk (15s delay)...")
self.model = LLM2Vec.from_pretrained(
base_model_name_or_path=self.custom_dir,
peft_model_name_or_path=None,
torch_dtype=self.torch_dtype,
device_map="cpu"
)
from kimodo.demo.memory_manager import manager
manager.ensure_vram_capacity(5400 * 1024 * 1024, device="cuda:0", exclude_name="text_encoder")
curr_device = self.get_device()
if curr_device.type != "cuda":
print(f"[LLM2VecEncoder] Moving weights to GPU (cuda:0)...")
self.model.model.to("cuda:0")
gc.collect()
import platform
if platform.system() == "Linux":
try:
import ctypes
ctypes.CDLL("libc.so.6").malloc_trim(0)
except Exception:
pass
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
manager.log_memory_usage("Encoder Transfer Complete (RAM Reclaimed)")
else:
print(f"[LLM2VecEncoder] Model already on GPU ({curr_device})")
def get_device(self):
if self.model is None:
return torch.device("cpu")
for p in self.model.model.parameters():
if p.device.type != "meta":
return p.device
return torch.device("cpu")
def delete(self):
"""Reclaim RAM without deleting from disk unless absolutely necessary."""
# We no longer delete the model by default to avoid slow reloads.
# Just unload to CPU instead.
self.unload()
def __call__(self, text: list[str] | str):
self.reload() # Auto-reload if called
is_string = False
if isinstance(text, str):
text = [text]
is_string = True
results = []
for t in text:
with torch.no_grad():
emb = self.model.encode([t])
results.append(emb)
encoded_text = np.concatenate(results, axis=0)
assert len(encoded_text.shape)
assert self.llm_dim == encoded_text.shape[-1]
encoded_text = encoded_text[:, None]
lengths = np.ones(len(encoded_text), dtype=int).tolist()
if is_string:
encoded_text = encoded_text[0]
lengths = lengths[0]
encoded_text = torch.tensor(encoded_text).to(self.get_device())
return encoded_text, lengthsBefore launching the UI, Windows users must ensure Node.js is installed.
For the latest stable (LTS) version:
winget install -e --id OpenJS.NodeJS.LTSFor the current version (latest features):
winget install -e --id OpenJS.NodeJSVerification: After installation, restart your terminal and verify the versions:
node -v
npm -v
npx -vFinally, launch the application.
python -m kimodo.demo # or run with --offload for less than 8gb vram hardware
Well Done !