Simon Gardling
0aeb6c5523
Generate and encrypt a Bearer token for llama-cpp's built-in auth. Remove caddy_auth from the vhost since basic auth blocks Bearer-only clients. Internal sidecars (xmrig-pause, annotations) connect directly to localhost and are unaffected (/slots is public).
47 lines
1.2 KiB
Nix
47 lines
1.2 KiB
Nix
{
|
|
pkgs,
|
|
service_configs,
|
|
config,
|
|
inputs,
|
|
lib,
|
|
...
|
|
}:
|
|
{
|
|
services.llama-cpp = {
|
|
enable = true;
|
|
model = toString (
|
|
pkgs.fetchurl {
|
|
url = "https://huggingface.co/Jackrong/Qwen3.5-9B-Claude-4.6-Opus-Reasoning-Distilled-v2-GGUF/resolve/main/Qwen3.5-9B.Q4_K_M.gguf";
|
|
sha256 = "8fbbc7b04a7d4b052d14b7aa97c8bf2014d39ceca8c2baaa043711712ba71ccc";
|
|
}
|
|
);
|
|
port = service_configs.ports.private.llama_cpp.port;
|
|
host = "0.0.0.0";
|
|
package = (lib.optimizePackage inputs.llamacpp.packages.${pkgs.system}.default);
|
|
extraFlags = [
|
|
# "-ngl"
|
|
# "12"
|
|
"-c"
|
|
"32768"
|
|
"-ctk"
|
|
"q8_0"
|
|
"-ctv"
|
|
"turbo4"
|
|
"-fa"
|
|
"on"
|
|
"--api-key-file"
|
|
config.age.secrets.llama-cpp-api-key.path
|
|
];
|
|
};
|
|
|
|
# have to do this in order to get vulkan to work
|
|
systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false;
|
|
|
|
# Auth handled by llama-cpp --api-key-file (Bearer token).
|
|
# No caddy_auth — the API key is the auth layer, and caddy_auth's basic
|
|
# auth would block Bearer-only clients like oh-my-pi.
|
|
services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
|
|
reverse_proxy :${toString config.services.llama-cpp.port}
|
|
'';
|
|
}
|