Initial commit

README.md

@@ -0,0 +1,48 @@
+# Llama.cpp Provider Extension for Pi
+
+This extension dynamically registers models from a llama.cpp `llama-server` instance with their context window sizes.
+
+## Installation
+
+### Option 1: Quick Test
+```bash
+pi -e /path/to/pi_agent_llama_provider/llama-cpp-provider.ts
+```
+
+### Option 2: Auto-Discovery
+Copy the extension to your global extensions directory:
+```bash
+cp llama-cpp-provider.ts ~/.pi/agent/extensions/
+```
+
+Then run `/reload` in pi to load it automatically.
+
+### Option 3: Custom Server URL
+Set the `LLAMA_SERVER_URL` environment variable before starting pi:
+```bash
+export LLAMA_SERVER_URL="http://your-server:port/v1"
+pi
+```
+
+Or modify the `LLAMA_SERVER_URL` constant in the extension file.
+
+## Usage
+
+Once loaded, the extension will:
+
+1. Fetch all models from the llama.cpp server
+2. Parse the context window size from each model's preset
+3. Register them as a `llama-cpp` provider
+4. Make them available via `/model` or `Ctrl+L`
+
+### Commands
+
+- `/llama-reload` - Reload and re-load models from the server
+- `/llama-list` - List all available models with their context sizes
+
+### Known Issues
+
+If you run this extension as your only provider for pi agent, you'll get a
+warning about no loaded models. My fix was to use an empty model list in the
+default models.ini file:
+https://github.com/badlogic/pi-mono/blob/main/packages/coding-agent/docs/models.md

curl_output.json

@@ -0,0 +1 @@
+{"data":[{"id":"GLM-4.7-Flash","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","GLM-4.7-Flash","--ctx-size","32000","--model","./models/unsloth_GLM-4.7-Flash-GGUF_GLM-4.7-Flash-UD-Q3_K_XL.gguf","--parallel","1"],"preset":"[GLM-4.7-Flash]\njinja = 1\nctx-size = 32000\nmodel = ./models/unsloth_GLM-4.7-Flash-GGUF_GLM-4.7-Flash-UD-Q3_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3-4B-Instruct","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","Qwen3-4B-Instruct","--ctx-size","32000","--model","./models/unsloth_Qwen3-4B-Instruct-2507-GGUF_Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3-4B-Instruct]\njinja = 1\nctx-size = 32000\nmodel = ./models/unsloth_Qwen3-4B-Instruct-2507-GGUF_Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3-Coder-30B","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","Qwen3-Coder-30B","--ctx-size","32000","--model","./models/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3-Coder-30B]\njinja = 1\nctx-size = 32000\nmodel = ./models/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"Qwen3.5-35B","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"loaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--chat-template-kwargs","{\"enable_thinking\":false}","--host","127.0.0.1","--jinja","--min-p","0.0","--port","40051","--temperature","0.6","--top-k","20","--top-p","0.95","--alias","Qwen3.5-35B","--ctx-size","64000","--model","./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf","--parallel","1"],"preset":"[Qwen3.5-35B]\nchat-template-kwargs = {\"enable_thinking\":false}\njinja = 1\nmin-p = 0.0\ntemperature = 0.6\ntop-k = 20\ntop-p = 0.95\nctx-size = 64000\nmodel = ./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\nparallel = 1\n\n"}},{"id":"default","aliases":[],"tags":[],"object":"model","owned_by":"llamacpp","created":1773431100,"status":{"value":"unloaded","args":["/home/llama/llama.cpp/build/bin/llama-server","--host","127.0.0.1","--jinja","--port","0","--alias","default","--ctx-size","32000","--parallel","1"],"preset":"[default]\njinja = 1\nctx-size = 32000\nparallel = 1\n\n"}}],"object":"list"}

llama-cpp-provider.ts

@@ -0,0 +1,221 @@
+/**
+ * Llama.cpp Provider Extension
+ *
+ * Dynamically registers models from a llama.cpp llama-server instance
+ * by fetching /v1/models, loading them, and extracting context window sizes
+ *
+ * Usage:
+ *   pi -e ./llama-cpp-provider.ts
+ *
+ * Or add to ~/.pi/agent/extensions/ for auto-discovery
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import type { Model } from "@mariozechner/pi-ai";
+
+// Configuration - can be overridden via environment variable
+const LLAMA_SERVER_URL =
+  process.env.LLAMA_SERVER_URL || "http://example.com:8080/v1";
+
+interface LlamaModel {
+  id: string;
+  aliases: string[];
+  tags: string[];
+  object: string;
+  owned_by: string;
+  created: number;
+  status: {
+    value: string;
+    args: string[];
+    preset: string;
+  };
+}
+
+interface LlamaModelsResponse {
+  data: LlamaModel[];
+  object: string;
+}
+
+/**
+ * Parse context window size from llama.cpp preset string
+ *
+ * Example preset:
+ * ```
+ * [Qwen3.5-35B]
+ * chat-template-kwargs = {"enable_thinking":false}
+ * jinja = 1
+ * min-p = 0.0
+ * temperature = 0.6
+ * top-k = 20
+ * top-p = 0.95
+ * ctx-size = 64000
+ * model = ./models/unsloth/unsloth_Qwen3.5-35B-A3B-GGUF_Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf
+ * parallel = 1
+ * ```
+ */
+function parseContextWindow(preset: string): number {
+  const match = preset.match(/ctx-size\s*=\s*(\d+)/);
+  return match ? parseInt(match[1], 10) : 32000; // Default to 32k if not found
+}
+
+/**
+ * Fetch models from llama.cpp server
+ */
+async function fetchLlamaModels(): Promise<LlamaModel[]> {
+  try {
+    const response = await fetch(`${LLAMA_SERVER_URL}/models`);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch models: ${response.status} ${response.statusText}`);
+    }
+    const data = (await response.json()) as LlamaModelsResponse;
+    return data.data || [];
+  } catch (error) {
+    console.error("[llama-cpp-provider] Error fetching models:", error);
+    return [];
+  }
+}
+
+/**
+ * Load a model on the llama.cpp server
+ * POST /models/load with { "model": "model_id" }
+ */
+async function loadLlamaModel(modelId: string): Promise<boolean> {
+  try {
+    const response = await fetch(`${LLAMA_SERVER_URL}/models/load`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ model: modelId }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to load model ${modelId}: ${response.status} ${response.statusText}`);
+    }
+
+    const result = await response.json();
+    return result.success || false;
+  } catch (error) {
+    console.error(`[llama-cpp-provider] Error loading model ${modelId}:`, error);
+    return false;
+  }
+}
+
+/**
+ * Convert llama.cpp model to pi-ai Model configuration
+ */
+function llamaModelToPiModel(llamaModel: LlamaModel): Model<"openai-completions"> {
+  const contextWindow = parseContextWindow(llamaModel.status.preset);
+
+  return {
+    id: llamaModel.id,
+    name: llamaModel.id,
+    api: "openai-completions",
+    provider: "llama-cpp",
+    baseUrl: LLAMA_SERVER_URL.replace("/v1", ""),
+    reasoning: false, // llama.cpp doesn't support reasoning in the pi-ai sense
+    input: ["text"], // Check if model supports images based on ID or preset
+    cost: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+    },
+    contextWindow,
+    maxTokens: Math.floor(contextWindow / 2), // Conservative estimate
+  };
+}
+
+/**
+ * Extension entry point
+ */
+export default function (pi: ExtensionAPI) {
+  let registered = false;
+  let models: Model<"openai-completions">[] = [];
+
+  /**
+   * Fetch and register models from llama.cpp server
+   */
+  async function registerModels(ctx?: import("@mariozechner/pi-coding-agent").ExtensionContext) {
+    const llamaModels = await fetchLlamaModels();
+
+    if (llamaModels.length === 0) {
+      ctx?.ui.notify(
+        "No models found from llama.cpp server. Check URL and server status.",
+        "warning"
+      );
+      return;
+    }
+
+    models = llamaModels.map(llamaModelToPiModel);
+
+    pi.registerProvider("llama-cpp", {
+      baseUrl: LLAMA_SERVER_URL.replace("/v1", ""),
+      apiKey: "llama", // llama.cpp doesn't require auth, any value works
+      api: "openai-completions",
+      models,
+    });
+
+    ctx?.ui.notify(
+      `Registered ${models.length} models from llama.cpp server`,
+      "info"
+    );
+
+    registered = true;
+  }
+
+  /**
+   * Reload models (useful if models are added/removed from server)
+   */
+  async function reloadModels(ctx?: import("@mariozechner/pi-coding-agent").ExtensionContext) {
+    if (!registered) {
+      await registerModels(ctx);
+      return;
+    }
+
+    // Unregister and re-register to update models
+    pi.unregisterProvider("llama-cpp");
+    await registerModels(ctx);
+  }
+
+  // Register models on load (no ctx available at extension load time)
+  registerModels().then(() => {
+    // Set runtime API key override after models are registered
+    // This prevents the "No models available" warning in the TUI
+    // Note: ctx is not available here, so we'll set it via session_start or commands
+  });
+
+  // Register a command to reload models
+  pi.registerCommand("llama-reload", {
+    description: "Reload models from llama.cpp server",
+    handler: async (_args: string, ctx) => {
+      ctx.ui.setWorkingMessage("Fetching and loading models from llama.cpp server...");
+      await reloadModels(ctx);
+      ctx.ui.setWorkingMessage();
+    },
+  });
+
+  // Also register a command to list models
+  pi.registerCommand("llama-list", {
+    description: "List available models from llama.cpp server",
+    handler: async (_args: string, ctx) => {
+      const llamaModels = await fetchLlamaModels();
+
+      if (llamaModels.length === 0) {
+        ctx.ui.notify("No models found from llama.cpp server", "warning");
+        return;
+      }
+
+      const lines = llamaModels.map((m) => {
+        const ctxSize = parseContextWindow(m.status.preset);
+        const status = m.status.value;
+        return `  ${m.id} - ${ctxSize} tokens - ${status}`;
+      });
+
+      ctx.ui.notify(
+        `Available models (${llamaModels.length}):\n${lines.join("\n")}`,
+        "info"
+      );
+    },
+  });
+}