qmd remove llama.cpp

qmd-remote-llama-cpp.patch

--- package/src/llm.ts	2026-02-25 01:15:53.728175466 +0200
+++ package/src/llm.ts	2026-02-25 01:45:46.035923784 +0200
@@ -1173,6 +1173,198 @@
   return defaultSessionManager.canUnload();
 }
 
+
+// =============================================================================
+// Remote LLM Implementation (for GPU servers via llama-server HTTP API)
+// =============================================================================
+
+/**
+ * Remote LLM that calls a llama-server instance via HTTP.
+ * Enable by setting QMD_REMOTE_URL environment variable.
+ *
+ * Supports all methods needed for `qmd embed`: embed, embedBatch, tokenize,
+ * detokenize, countTokens. Other methods are stubbed.
+ *
+ * Usage:
+ *   # Start llama-server on GPU machine:
+ *   llama-server -m embeddinggemma-300M-Q8_0.gguf --embedding --port 8080
+ *
+ *   # SSH tunnel + run:
+ *   ssh -L 8080:localhost:8080 ubuntu@gpu-host -N &
+ *   QMD_REMOTE_URL=http://localhost:8080 QMD_EMBED_MODEL=embeddinggemma qmd embed
+ */
+class RemoteLLM implements LLM {
+  private baseUrl: string;
+  /** Cache last tokenized content for local detokenize (avoids remote round-trips during chunking) */
+  private lastTokenizedContent: string | null = null;
+  private lastTokenizedCharsPerToken: number = 2;
+  private embedModel: string;
+  private rerankModel: string;
+  private genModel: string;
+
+  constructor(baseUrl: string) {
+    this.baseUrl = baseUrl.replace(/\/$/, '');
+    this.embedModel = process.env.QMD_EMBED_MODEL || 'embeddinggemma';
+    this.rerankModel = process.env.QMD_RERANK_MODEL || 'qwen3-reranker';
+    this.genModel = process.env.QMD_GEN_MODEL || 'qwen3-0.6b';
+  }
+
+  /** Fetch with timeout and retry for transient failures */
+  private async fetchWithRetry(url: string, init: RequestInit, retries = 5, timeoutMs = 30000): Promise<Response> {
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        const res = await fetch(url, { ...init, signal: controller.signal });
+        clearTimeout(timer);
+        
+        // Retry on server errors (5xx)
+        if (!res.ok && res.status >= 500 && attempt < retries) {
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          console.error(`Remote fetch got ${res.status}, retry ${attempt}/${retries}, waiting ${delay}ms...`);
+          await new Promise(r => setTimeout(r, delay));
+          continue;
+        }
+        return res;
+      } catch (err: any) {
+        const isLastAttempt = attempt === retries;
+        const errMsg = err?.message || err?.code || String(err);
+        if (!isLastAttempt) {
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          console.error(`Remote fetch retry ${attempt}/${retries} (${errMsg}), waiting ${delay}ms...`);
+          await new Promise(r => setTimeout(r, delay));
+        } else {
+          throw err;
+        }
+      }
+    }
+    throw new Error('unreachable');
+  }
+
+  async embed(text: string, options: EmbedOptions = {}): Promise<EmbeddingResult | null> {
+    try {
+      const sanitized = text.replace(/[\uD800-\uDFFF]/g, '\uFFFD');
+      const res = await this.fetchWithRetry(`${this.baseUrl}/v1/embeddings`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ input: sanitized, model: this.embedModel }),
+      }, 3, 60000);
+      if (!res.ok) throw new Error(`Remote embed failed: ${res.status} ${res.statusText}`);
+      const data = await res.json() as any;
+      return { embedding: data.data[0].embedding, model: this.embedModel };
+    } catch (err) {
+      console.error('Remote embed error:', err);
+      return null;
+    }
+  }
+
+  async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
+    if (texts.length === 0) return [];
+    try {
+      const sanitized = texts.map(t => t.replace(/[\uD800-\uDFFF]/g, '\uFFFD'));
+      const res = await this.fetchWithRetry(`${this.baseUrl}/v1/embeddings`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ input: sanitized, model: this.embedModel }),
+      }, 3, 120000);
+      if (!res.ok) throw new Error(`Remote embedBatch failed: ${res.status} ${res.statusText}`);
+      const data = await res.json() as any;
+      return data.data.map((d: any) => ({ embedding: d.embedding, model: this.embedModel }));
+    } catch (err) {
+      console.error('Remote embedBatch error, retrying individually:', err);
+      return Promise.all(texts.map(t => this.embed(t)));
+    }
+  }
+
+  async tokenize(text: string): Promise<readonly LlamaToken[]> {
+    this.lastTokenizedContent = text;
+    const charsPerToken = 2;
+    this.lastTokenizedCharsPerToken = charsPerToken;
+    const approxTokenCount = Math.ceil(text.length / charsPerToken);
+    const tokens = Array.from({ length: approxTokenCount }, (_, i) => i) as unknown as LlamaToken[];
+    return tokens;
+  }
+
+  async detokenize(tokens: readonly LlamaToken[]): Promise<string> {
+    if (!this.lastTokenizedContent || tokens.length === 0) return '';
+    const ids = tokens as unknown as number[];
+    const start = ids[0] * this.lastTokenizedCharsPerToken;
+    const end = (ids[ids.length - 1] + 1) * this.lastTokenizedCharsPerToken;
+    return this.lastTokenizedContent.slice(start, Math.min(end, this.lastTokenizedContent.length));
+  }
+
+  async countTokens(text: string): Promise<number> {
+    return Math.ceil(text.length / 2);
+  }
+
+  async rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult> {
+    try {
+      const res = await this.fetchWithRetry(`${this.baseUrl}/v1/rerank`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: this.rerankModel,
+          query: query,
+          documents: documents.map(d => d.text || d.content || '')
+        }),
+      }, 3, 60000);
+      if (!res.ok) throw new Error(`Remote rerank failed: ${res.status} ${res.statusText}`);
+      const data = await res.json() as any;
+      return {
+        results: data.results.map((r: any, i: number) => ({
+          file: documents[r.index]?.file || '',
+          score: r.relevance_score,
+          index: r.index
+        })),
+        model: this.rerankModel
+      };
+    } catch (err) {
+      console.error('Remote rerank error, using passthrough:', err);
+      return {
+        results: documents.map((d, i) => ({ file: d.file, score: 1 - i * 0.01, index: i })),
+        model: 'remote-passthrough',
+      };
+    }
+  }
+
+  async generate(prompt: string, options?: GenerateOptions): Promise<GenerateResult | null> {
+    try {
+      const res = await this.fetchWithRetry(`${this.baseUrl}/v1/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: this.genModel,
+          prompt: prompt,
+          max_tokens: options?.maxTokens || 256,
+          temperature: options?.temperature || 0.7,
+          stream: false
+        }),
+      }, 3, 60000);
+      if (!res.ok) throw new Error(`Remote generate failed: ${res.status} ${res.statusText}`);
+      const data = await res.json() as any;
+      return {
+        text: data.choices[0].text,
+        model: this.genModel,
+        done: true
+      };
+    } catch (err) {
+      console.error('Remote generate error:', err);
+      return null;
+    }
+  }
+
+  async modelExists(model: string): Promise<ModelInfo> {
+    return { name: model, exists: true };
+  }
+
+  async expandQuery(query: string): Promise<Queryable[]> {
+    return [{ type: 'lex', text: query }];
+  }
+
+  async dispose(): Promise<void> { /* no-op */ }
+}
+
+
 // =============================================================================
 // Singleton for default LlamaCpp instance
 // =============================================================================
@@ -1180,15 +1372,24 @@
 let defaultLlamaCpp: LlamaCpp | null = null;
 
 /**
- * Get the default LlamaCpp instance (creates one if needed)
+ * Get the default LlamaCpp instance (creates one if needed).
+ *
+ * If QMD_REMOTE_URL is set, returns a RemoteLLM that calls a remote llama-server
+ * instead of loading models locally. This enables GPU-accelerated embedding via
+ * SSH tunnel to a GPU cloud instance.
  */
 export function getDefaultLlamaCpp(): LlamaCpp {
   if (!defaultLlamaCpp) {
-    defaultLlamaCpp = new LlamaCpp();
+    const remoteUrl = process.env.QMD_REMOTE_URL;
+    if (remoteUrl) {
+      console.log(`Using remote LLM server: ${remoteUrl}`);
+      defaultLlamaCpp = new RemoteLLM(remoteUrl) as any;
+    } else {
+      defaultLlamaCpp = new LlamaCpp();
+    }
   }
   return defaultLlamaCpp;
 }
-
 /**
  * Set a custom default LlamaCpp instance (useful for testing)
  */
--- package/src/qmd.ts	2026-02-25 01:15:53.728175466 +0200
+++ package/src/qmd.ts	2026-02-25 01:17:07.605349581 +0200
@@ -1476,7 +1476,8 @@
 }
 
 function renderProgressBar(percent: number, width: number = 30): string {
-  const filled = Math.round((percent / 100) * width);
+  const clamped = Math.max(0, Math.min(100, percent));
+  const filled = Math.round((clamped / 100) * width);
   const empty = width - filled;
   const bar = "█".repeat(filled) + "░".repeat(empty);
   return bar;
--- package/src/store.ts	2026-02-25 01:15:53.729134967 +0200
+++ package/src/store.ts	2026-02-25 01:17:36.190149651 +0200
@@ -2217,9 +2217,12 @@
   embeddedAt: string
 ): void {
   const hashSeq = `${hash}_${seq}`;
-  const insertVecStmt = db.prepare(`INSERT OR REPLACE INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`);
+  // vec0 virtual tables don't support INSERT OR REPLACE, so delete first if exists
+  const deleteVecStmt = db.prepare(`DELETE FROM vectors_vec WHERE hash_seq = ?`);
+  const insertVecStmt = db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`);
   const insertContentVectorStmt = db.prepare(`INSERT OR REPLACE INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, ?, ?, ?, ?)`);
 
+  deleteVecStmt.run(hashSeq);
   insertVecStmt.run(hashSeq, embedding);
   insertContentVectorStmt.run(hash, seq, pos, model, embeddedAt);
 }