titaniumtown/nixos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "re-add llama.cpp (test?)"

Browse Source 
This reverts commit 943fa2f531.

Maybe will un-revert once turboquant becomes a thing?
This commit is contained in:
Simon Gardling
2026-03-30 02:41:39 -04:00
parent 82a383482e
commit 65c13babac
5 changed files with 0 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
services/llama-cpp.nix
View File

@@ -1,36 +0,0 @@
{
pkgs,
service_configs,
config,
inputs,
lib,
...
}:
{
services.llama-cpp = {
enable = true;
model = builtins.toString (
pkgs.fetchurl {
url = "https://huggingface.co/mradermacher/Qwen3.5-27B-Claude-4.6-Opus-Reasoning-Distilled-GGUF/resolve/main/Qwen3.5-27B-Claude-4.6-Opus-Reasoning-Distilled.Q5_K_M.gguf";
sha256 = "1b08df702dc729104de8894d3f6c6f52505e9f07c5d9b236b3efda3ae187bda2";
}
);
port = service_configs.ports.private.llama_cpp.port;
host = "0.0.0.0";
package = (lib.optimizePackage inputs.llamacpp.packages.${pkgs.system}.vulkan);
extraFlags = [
"-ngl"
"12"
"-c"
"16384"
];
};
# have to do this in order to get vulkan to work
systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false;
services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
import ${config.age.secrets.caddy_auth.path}
reverse_proxy :${builtins.toString config.services.llama-cpp.port}
'';
}