llama-cpp: re-add w/ turboquant

2026-04-02 13:42:39 -04:00
parent 7e779ca0f7
commit f342521d46
5 changed files with 104 additions and 0 deletions
--- a/services/llama-cpp.nix
+++ b/services/llama-cpp.nix
@@ -0,0 +1,40 @@
+{
+  pkgs,
+  service_configs,
+  config,
+  inputs,
+  lib,
+  ...
+}:
+{
+  services.llama-cpp = {
+    enable = true;
+    model = toString (
+      pkgs.fetchurl {
+        url = "https://huggingface.co/Jackrong/Qwen3.5-9B-Claude-4.6-Opus-Reasoning-Distilled-v2-GGUF/resolve/main/Qwen3.5-9B.Q4_K_M.gguf";
+        sha256 = "8fbbc7b04a7d4b052d14b7aa97c8bf2014d39ceca8c2baaa043711712ba71ccc";
+      }
+    );
+    port = service_configs.ports.private.llama_cpp.port;
+    host = "0.0.0.0";
+    package = (lib.optimizePackage inputs.llamacpp.packages.${pkgs.system}.vulkan);
+    extraFlags = [
+      "-ngl"
+      "12"
+      "-c"
+      "16384"
+      "-ctk"
+      "q8_0"
+      "-ctv"
+      "turbo3"
+    ];
+  };
+
+  # have to do this in order to get vulkan to work
+  systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false;
+
+  services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${toString config.services.llama-cpp.port}
+  '';
+}