Compare commits
2 Commits
dad3867144
...
bef4ac7ddc
| Author | SHA1 | Date | |
|---|---|---|---|
bef4ac7ddc
|
|||
|
12469de580
|
30
flake.lock
generated
30
flake.lock
generated
@@ -304,11 +304,11 @@
|
||||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1775754862,
|
||||
"narHash": "sha256-8y9cz8+cyeA7KtA7+Q3bXjyFJV5nM38Fc0E4qPw7WDk=",
|
||||
"lastModified": 1775866084,
|
||||
"narHash": "sha256-mWn8D/oXXAaqeFFFRorKHvTLw5V9M8eYzAWRr4iffag=",
|
||||
"owner": "nix-community",
|
||||
"repo": "lanzaboote",
|
||||
"rev": "bea51aaee00688794a877f308007590a6cc8e378",
|
||||
"rev": "29d2cca7fc3841708c1d48e2d1272f79db1538b6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -368,11 +368,11 @@
|
||||
"systems": "systems_3"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1775752089,
|
||||
"narHash": "sha256-+psXqZ1SvQw7L8HgCQINmob9zLnvK433b2k080lBPH0=",
|
||||
"lastModified": 1775791757,
|
||||
"narHash": "sha256-3BS1Hw+3A3uf4G/8zwts3ZgxSnYq0y+QntbwO+b6KEw=",
|
||||
"owner": "Infinidoge",
|
||||
"repo": "nix-minecraft",
|
||||
"rev": "1beacd3bdadabfac884dedd56176966c141214d8",
|
||||
"rev": "c4c6a33affcc15cde3df06083e96cda87f9a7627",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -399,11 +399,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1775595990,
|
||||
"narHash": "sha256-OEf7YqhF9IjJFYZJyuhAypgU+VsRB5lD4DuiMws5Ltc=",
|
||||
"lastModified": 1775811116,
|
||||
"narHash": "sha256-t+HZK42pB6N+i5RGbuy7Xluez/VvWbembBdvzsc23Ss=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "4e92bbcdb030f3b4782be4751dc08e6b6cb6ccf2",
|
||||
"rev": "54170c54449ea4d6725efd30d719c5e505f1c10e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -624,11 +624,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1775701952,
|
||||
"narHash": "sha256-xj9u8fz2hTTTELMorqox0hPWrmAvGRnQUEnlj+vCjFo=",
|
||||
"lastModified": 1775896579,
|
||||
"narHash": "sha256-uU9t4oqG7MbJHYjYnEPfnOcSMBPxK4wACeaOXvE0Ezg=",
|
||||
"owner": "nix-community",
|
||||
"repo": "srvos",
|
||||
"rev": "f56f1053ae9f878501d3a8ae1961c73d1d7abce3",
|
||||
"rev": "7983ea7a44f40fcc1c35b0ca8e54e794a26b09e2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -715,11 +715,11 @@
|
||||
"trackerlist": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1775686189,
|
||||
"narHash": "sha256-kzEDJKptaVToSg/wpub0bLjAVRmkYOorjPsNqlpxWdU=",
|
||||
"lastModified": 1775858976,
|
||||
"narHash": "sha256-LJ+A/x7g3jhS7R9Jkyt3E4Be2jo/bJVStDDJLG5AL7c=",
|
||||
"owner": "ngosang",
|
||||
"repo": "trackerslist",
|
||||
"rev": "ce9c0afc3885d0592caa91f0d4359f315ef7428c",
|
||||
"rev": "b4dac394ef1eff0ab51c0cb7004c05826988d846",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
From 320c29c2dbe3c8df56374a9ec19a7fe5c124d4f8 Mon Sep 17 00:00:00 2001
|
||||
From: Piotr Wilkin <piotr.wilkin@syndatis.com>
|
||||
Date: Tue, 7 Apr 2026 00:54:00 +0200
|
||||
Subject: [PATCH 1/2] YATF (Yet Another Tokenizer Fix) for Gemma 4. With tests!
|
||||
|
||||
---
|
||||
convert_hf_to_gguf_update.py | 1 +
|
||||
models/ggml-vocab-gemma-4.gguf | Bin 0 -> 15776467 bytes
|
||||
models/ggml-vocab-gemma-4.gguf.inp | 111 +++++++++++++++++++++++++++++
|
||||
models/ggml-vocab-gemma-4.gguf.out | 46 ++++++++++++
|
||||
src/llama-vocab.cpp | 13 +++-
|
||||
tests/CMakeLists.txt | 1 +
|
||||
6 files changed, 170 insertions(+), 2 deletions(-)
|
||||
create mode 100644 models/ggml-vocab-gemma-4.gguf
|
||||
create mode 100644 models/ggml-vocab-gemma-4.gguf.inp
|
||||
create mode 100644 models/ggml-vocab-gemma-4.gguf.out
|
||||
|
||||
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
|
||||
index 086f1c22863..f1d70d62e73 100755
|
||||
--- a/convert_hf_to_gguf_update.py
|
||||
+++ b/convert_hf_to_gguf_update.py
|
||||
@@ -114,6 +114,7 @@ class TOKENIZER_TYPE(IntEnum):
|
||||
{"name": "viking", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B", }, # Also used for Viking 13B and 33B
|
||||
{"name": "gemma", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2b", },
|
||||
{"name": "gemma-2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", },
|
||||
+ {"name": "gemma-4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/google/gemma-4-E2B-it", },
|
||||
{"name": "jais", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", },
|
||||
{"name": "jais-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inceptionai/Jais-2-8B-Chat", },
|
||||
{"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },
|
||||
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
|
||||
index de9a9466bc7..e9e276ab999 100644
|
||||
--- a/src/llama-vocab.cpp
|
||||
+++ b/src/llama-vocab.cpp
|
||||
@@ -658,9 +658,18 @@ struct llm_tokenizer_bpe_session {
|
||||
const auto token = vocab.text_to_token(str);
|
||||
|
||||
if (token == LLAMA_TOKEN_NULL) {
|
||||
+ static const char * hex = "0123456789ABCDEF";
|
||||
for (auto j = str.begin(); j != str.end(); ++j) {
|
||||
- std::string byte_str(1, *j);
|
||||
- auto token_multibyte = vocab.text_to_token(byte_str);
|
||||
+ llama_token token_multibyte = LLAMA_TOKEN_NULL;
|
||||
+ if (tokenizer.byte_encode) {
|
||||
+ std::string byte_str(1, *j);
|
||||
+ token_multibyte = vocab.text_to_token(byte_str);
|
||||
+ } else {
|
||||
+ // For non-byte-encoded BPE (e.g. gemma-4), byte tokens use <0xXX> format
|
||||
+ const uint8_t ch = (uint8_t)*j;
|
||||
+ const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };
|
||||
+ token_multibyte = vocab.text_to_token(buf);
|
||||
+ }
|
||||
if (token_multibyte != LLAMA_TOKEN_NULL) {
|
||||
output.push_back(token_multibyte);
|
||||
}
|
||||
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
|
||||
index 5e87c8b34e1..cd4bc5ef1d3 100644
|
||||
--- a/tests/CMakeLists.txt
|
||||
+++ b/tests/CMakeLists.txt
|
||||
@@ -124,6 +124,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJE
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
|
||||
+llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)
|
||||
|
||||
From 0e98596dec124c6968132ef042c21ccdb20d1304 Mon Sep 17 00:00:00 2001
|
||||
From: Piotr Wilkin <piotr.wilkin@syndatis.com>
|
||||
Date: Tue, 7 Apr 2026 00:58:08 +0200
|
||||
Subject: [PATCH 2/2] Remove unnecessary hash from update script.
|
||||
|
||||
---
|
||||
convert_hf_to_gguf_update.py | 1 -
|
||||
1 file changed, 1 deletion(-)
|
||||
|
||||
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
|
||||
index f1d70d62e73..086f1c22863 100755
|
||||
--- a/convert_hf_to_gguf_update.py
|
||||
+++ b/convert_hf_to_gguf_update.py
|
||||
@@ -114,7 +114,6 @@ class TOKENIZER_TYPE(IntEnum):
|
||||
{"name": "viking", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B", }, # Also used for Viking 13B and 33B
|
||||
{"name": "gemma", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2b", },
|
||||
{"name": "gemma-2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", },
|
||||
- {"name": "gemma-4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/google/gemma-4-E2B-it", },
|
||||
{"name": "jais", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", },
|
||||
{"name": "jais-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inceptionai/Jais-2-8B-Chat", },
|
||||
{"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },
|
||||
@@ -9,7 +9,7 @@
|
||||
}:
|
||||
let
|
||||
cfg = config.services.llama-cpp;
|
||||
modelUrl = "https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF/resolve/main/google_gemma-4-E2B-it-Q4_K_M.gguf";
|
||||
modelUrl = "https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF/resolve/main/google_gemma-4-E2B-it-IQ2_M.gguf";
|
||||
modelAlias = lib.removeSuffix ".gguf" (baseNameOf modelUrl);
|
||||
in
|
||||
{
|
||||
@@ -25,7 +25,7 @@ in
|
||||
model = toString (
|
||||
pkgs.fetchurl {
|
||||
url = modelUrl;
|
||||
sha256 = "5efe645db4e1909c7a1f4a9608df18e6c14383f5e86777fc49f769f9ba7d5fdf";
|
||||
sha256 = "17e869ac54d0e59faa884d5319fc55ad84cd866f50f0b3073fbb25accc875a23";
|
||||
}
|
||||
);
|
||||
port = service_configs.ports.private.llama_cpp.port;
|
||||
@@ -33,7 +33,6 @@ in
|
||||
package = lib.optimizePackage (
|
||||
inputs.llamacpp.packages.${pkgs.system}.vulkan.overrideAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [
|
||||
../patches/llamacpp/0003-gemma4-tokenizer-fix.patch
|
||||
];
|
||||
})
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user