fix(gguf): 移除不必要的 tokenizer.pre 要求

Signed-off-by: YdrMaster <[email protected]>

Author: YdrMaster

Cargo.toml

@@ -34,7 +34,7 @@ itertools = "0.13"
 env_logger = "0.11"
 build-script-cfg = "0.0"
 
-operators = { git = "https://github.com/YdrMaster/operators-rs", rev = "c81600f", default-features = false }
+operators = { git = "https://github.com/YdrMaster/operators-rs", rev = "d88d716", default-features = false }
 
 search-cl-tools = { git = "https://github.com/InfiniTensor/clrt", rev = "9b6289d" }
 search-infini-tools = { git = "https://github.com/InfiniTensor/infini-rt", rev = "f40bcb5" }

gguf/src/tokenizer.rs

@@ -61,7 +61,6 @@ impl Tokenizer {
     }
 
     fn bpe_from_gguf(gguf: &GGufModel) -> Self {
-        let _pre = gguf.get_str("tokenizer.ggml.pre").unwrap();
         let tokens = gguf.tokenizer_ggml_tokens().unwrap();
         let scores = gguf.tokenizer_ggml_scores().unwrap();
         let token_type = gguf.tokenizer_ggml_token_type().unwrap();