From 7e8a9cd55b02820c7bb423d4b9d40cfb720b8d2e Mon Sep 17 00:00:00 2001 From: cinnaboot Date: Sat, 14 Mar 2026 23:37:47 +0000 Subject: [PATCH] Initial commit --- README.md | 48 +++++++++ curl_output.json | 1 + llama-cpp-provider.ts | 221 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 README.md create mode 100644 curl_output.json create mode 100644 llama-cpp-provider.ts diff --git a/README.md b/README.md new file mode 100644 index 0000000..2aa669d --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +# Llama.cpp Provider Extension for Pi + +This extension dynamically registers models from a llama.cpp `llama-server` instance with their context window sizes. + +## Installation + +### Option 1: Quick Test +```bash +pi -e /path/to/pi_agent_llama_provider/llama-cpp-provider.ts +``` + +### Option 2: Auto-Discovery +Copy the extension to your global extensions directory: +```bash +cp llama-cpp-provider.ts ~/.pi/agent/extensions/ +``` + +Then run `/reload` in pi to load it automatically. + +### Option 3: Custom Server URL +Set the `LLAMA_SERVER_URL` environment variable before starting pi: +```bash +export LLAMA_SERVER_URL="http://your-server:port/v1" +pi +``` + +Or modify the `LLAMA_SERVER_URL` constant in the extension file. + +## Usage + +Once loaded, the extension will: + +1. Fetch all models from the llama.cpp server +2. Parse the context window size from each model's preset +3. Register them as a `llama-cpp` provider +4. Make them available via `/model` or `Ctrl+L` + +### Commands + +- `/llama-reload` - Reload and re-load models from the server +- `/llama-list` - List all available models with their context sizes + +### Known Issues + +If you run this extension as your only provider for pi agent, you'll get a +warning about no loaded models. My fix was to use an empty model list in the +default models.ini file: +https://github.com/badlogic/pi-mono/blob/main/packages/coding-agent/docs/models.md diff --git a/curl_output.json b/curl_output.json new file mode 100644 index 0000000..30fca3d --- /dev/null +++ b/curl_output.json @@ -0,0 +1 @@ +{"data":[{"id":"GLM-4.7-Flash","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","GLM-4.7-Flash","--ctx-size","32000","--model","./models/unsloth_GLM-4.7-Flash-GGUF_GLM-4.7-Flash-UD-Q3_K_XL.gguf","--parallel","1"],"preset":"[GLM-4.7-Flash]\njinja = 1\nctx-size = 32000\nmodel = ./models/unsloth_GLM-4.7-Flash-GGUF_GLM-4.7-Flash-UD-Q3_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3-4B-Instruct","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","Qwen3-4B-Instruct","--ctx-size","32000","--model","./models/unsloth_Qwen3-4B-Instruct-2507-GGUF_Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3-4B-Instruct]\njinja = 1\nctx-size = 32000\nmodel = ./models/unsloth_Qwen3-4B-Instruct-2507-GGUF_Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3-Coder-30B","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","Qwen3-Coder-30B","--ctx-size","32000","--model","./models/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3-Coder-30B]\njinja = 1\nctx-size = 32000\nmodel = ./models/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3.5-35B","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"loaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--chat-template-kwargs","{\"enable_thinking\":false}","--host","127.0.0.1","--jinja","--min-p","0.0","--port","40051","--temperature","0.6","--top-k","20","--top-p","0.95","--alias","Qwen3.5-35B","--ctx-size","64000","--model","./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3.5-35B]\nchat-template-kwargs = {\"enable_thinking\":false}\njinja = 1\nmin-p = 0.0\ntemperature = 0.6\ntop-k = 20\ntop-p = 0.95\nctx-size = 64000\nmodel = ./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"default","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","default","--ctx-size","32000","--parallel","1"],"preset":"[default]\njinja = 1\nctx-size = 32000\nparallel = 1\n\n"}}],"object":"list"} \ No newline at end of file diff --git a/llama-cpp-provider.ts b/llama-cpp-provider.ts new file mode 100644 index 0000000..5bfa916 --- /dev/null +++ b/llama-cpp-provider.ts @@ -0,0 +1,221 @@ +/** + * Llama.cpp Provider Extension + * + * Dynamically registers models from a llama.cpp llama-server instance + * by fetching /v1/models, loading them, and extracting context window sizes + * + * Usage: + * pi -e ./llama-cpp-provider.ts + * + * Or add to ~/.pi/agent/extensions/ for auto-discovery + */ + +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import type { Model } from "@mariozechner/pi-ai"; + +// Configuration - can be overridden via environment variable +const LLAMA_SERVER_URL = + process.env.LLAMA_SERVER_URL || "http://example.com:8080/v1"; + +interface LlamaModel { + id: string; + aliases: string[]; + tags: string[]; + object: string; + owned_by: string; + created: number; + status: { + value: string; + args: string[]; + preset: string; + }; +} + +interface LlamaModelsResponse { + data: LlamaModel[]; + object: string; +} + +/** + * Parse context window size from llama.cpp preset string + * + * Example preset: + * ``` + * [Qwen3.5-35B] + * chat-template-kwargs = {"enable_thinking":false} + * jinja = 1 + * min-p = 0.0 + * temperature = 0.6 + * top-k = 20 + * top-p = 0.95 + * ctx-size = 64000 + * model = ./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf + * parallel = 1 + * ``` + */ +function parseContextWindow(preset: string): number { + const match = preset.match(/ctx-size\s*=\s*(\d+)/); + return match ? parseInt(match[1], 10) : 32000; // Default to 32k if not found +} + +/** + * Fetch models from llama.cpp server + */ +async function fetchLlamaModels(): Promise { + try { + const response = await fetch(`${LLAMA_SERVER_URL}/models`); + if (!response.ok) { + throw new Error(`Failed to fetch models: ${response.status} ${response.statusText}`); + } + const data = (await response.json()) as LlamaModelsResponse; + return data.data || []; + } catch (error) { + console.error("[llama-cpp-provider] Error fetching models:", error); + return []; + } +} + +/** + * Load a model on the llama.cpp server + * POST /models/load with { "model": "model_id" } + */ +async function loadLlamaModel(modelId: string): Promise { + try { + const response = await fetch(`${LLAMA_SERVER_URL}/models/load`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ model: modelId }), + }); + + if (!response.ok) { + throw new Error(`Failed to load model ${modelId}: ${response.status} ${response.statusText}`); + } + + const result = await response.json(); + return result.success || false; + } catch (error) { + console.error(`[llama-cpp-provider] Error loading model ${modelId}:`, error); + return false; + } +} + +/** + * Convert llama.cpp model to pi-ai Model configuration + */ +function llamaModelToPiModel(llamaModel: LlamaModel): Model<"openai-completions"> { + const contextWindow = parseContextWindow(llamaModel.status.preset); + + return { + id: llamaModel.id, + name: llamaModel.id, + api: "openai-completions", + provider: "llama-cpp", + baseUrl: LLAMA_SERVER_URL.replace("/v1", ""), + reasoning: false, // llama.cpp doesn't support reasoning in the pi-ai sense + input: ["text"], // Check if model supports images based on ID or preset + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow, + maxTokens: Math.floor(contextWindow / 2), // Conservative estimate + }; +} + +/** + * Extension entry point + */ +export default function (pi: ExtensionAPI) { + let registered = false; + let models: Model<"openai-completions">[] = []; + + /** + * Fetch and register models from llama.cpp server + */ + async function registerModels(ctx?: import("@mariozechner/pi-coding-agent").ExtensionContext) { + const llamaModels = await fetchLlamaModels(); + + if (llamaModels.length === 0) { + ctx?.ui.notify( + "No models found from llama.cpp server. Check URL and server status.", + "warning" + ); + return; + } + + models = llamaModels.map(llamaModelToPiModel); + + pi.registerProvider("llama-cpp", { + baseUrl: LLAMA_SERVER_URL.replace("/v1", ""), + apiKey: "llama", // llama.cpp doesn't require auth, any value works + api: "openai-completions", + models, + }); + + ctx?.ui.notify( + `Registered ${models.length} models from llama.cpp server`, + "info" + ); + + registered = true; + } + + /** + * Reload models (useful if models are added/removed from server) + */ + async function reloadModels(ctx?: import("@mariozechner/pi-coding-agent").ExtensionContext) { + if (!registered) { + await registerModels(ctx); + return; + } + + // Unregister and re-register to update models + pi.unregisterProvider("llama-cpp"); + await registerModels(ctx); + } + + // Register models on load (no ctx available at extension load time) + registerModels().then(() => { + // Set runtime API key override after models are registered + // This prevents the "No models available" warning in the TUI + // Note: ctx is not available here, so we'll set it via session_start or commands + }); + + // Register a command to reload models + pi.registerCommand("llama-reload", { + description: "Reload models from llama.cpp server", + handler: async (_args: string, ctx) => { + ctx.ui.setWorkingMessage("Fetching and loading models from llama.cpp server..."); + await reloadModels(ctx); + ctx.ui.setWorkingMessage(); + }, + }); + + // Also register a command to list models + pi.registerCommand("llama-list", { + description: "List available models from llama.cpp server", + handler: async (_args: string, ctx) => { + const llamaModels = await fetchLlamaModels(); + + if (llamaModels.length === 0) { + ctx.ui.notify("No models found from llama.cpp server", "warning"); + return; + } + + const lines = llamaModels.map((m) => { + const ctxSize = parseContextWindow(m.status.preset); + const status = m.status.value; + return ` ${m.id} - ${ctxSize} tokens - ${status}`; + }); + + ctx.ui.notify( + `Available models (${llamaModels.length}):\n${lines.join("\n")}`, + "info" + ); + }, + }); +}