llama-cpp: set batch size to 4096

2026-04-06 02:29:37 -04:00
parent 9addb1569a
commit 6d47f02a0f
1 changed files with 4 additions and 0 deletions
--- a/services/llama-cpp/llama-cpp.nix
+++ b/services/llama-cpp/llama-cpp.nix
@@ -40,6 +40,10 @@ in
      "--metrics"
      "--alias"
      modelAlias
+      "-b"
+      "4096"
+      "-ub"
+      "4096"
    ];
  };