titaniumtown/server-config
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama-cpp: maybe use vulkan?
All checks were successful
Build and Deploy / deploy (push) Successful in 8m30s

Browse Source
This commit is contained in:
Simon Gardling
2026-04-06 02:12:46 -04:00
parent 3e46c5bfa5
commit 0a927ea893
3 changed files with 11 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
flake.lock generated
View File

@@ -325,16 +325,16 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1775236905, "lastModified": 1774922513,
"narHash": "sha256-tHshzR/k6D/r5UhJCfJ9b/mJgsbn7ODtnZrDlimhOOI=", "narHash": "sha256-TKk1i8AZzxy4/z0MkqKxoGf/CQDvoL+jo8JDtZeCRy8=",
"owner": "TheTom", "owner": "apollosenvy",
"repo": "llama-cpp-turboquant", "repo": "llama-cpp-turboquant",
"rev": "bc05a6803e48f17e0f2c7a99fce9b50d03882de7", "rev": "9e80e93ceb115bc5055997c373d8c09bfa47a565",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "TheTom", "owner": "apollosenvy",
"ref": "feature/turboquant-kv-cache", "ref": "pr/vulkan-turbo3",
"repo": "llama-cpp-turboquant", "repo": "llama-cpp-turboquant",
"type": "github" "type": "github"
} }

3
flake.nix
View File

@@ -29,7 +29,8 @@
}; };
llamacpp = { llamacpp = {
url = "github:TheTom/llama-cpp-turboquant/feature/turboquant-kv-cache"; # url = "github:TheTom/llama-cpp-turboquant/feature/turboquant-kv-cache";
url = "github:apollosenvy/llama-cpp-turboquant/pr/vulkan-turbo3";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };

6
services/llama-cpp/llama-cpp.nix
View File

@@ -23,10 +23,10 @@ in
); );
port = service_configs.ports.private.llama_cpp.port; port = service_configs.ports.private.llama_cpp.port;
host = "0.0.0.0"; host = "0.0.0.0";
package = (lib.optimizePackage inputs.llamacpp.packages.${pkgs.system}.default); package = (lib.optimizePackage inputs.llamacpp.packages.${pkgs.system}.vulkan);
extraFlags = [ extraFlags = [
# "-ngl" "-ngl"
# "12" "999"
"-c" "-c"
"65536" "65536"
"-ctk" "-ctk"