[tokenizer] Updates tokenizer to 0.20.3 (#3514)

deepjavalibrary · Nov 11, 2024 · 5f0e5e5 · 5f0e5e5
1 parent f6068aa
commit 5f0e5e5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml
@@ -13,7 +13,7 @@ candle-flash-attn = { version = "*", optional = true }
 candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "cf789b7dd6d4abb19b03b9556442f94f0588b4a0", optional = true }
 candle-layer-norm = { git = "https://github.com/xyang16/candle-layer-norm", rev = "e574de6a7f88bafbede8edf9ee43170c6a8ce51a", optional = true }
 candle-rotary = { git = "https://github.com/huggingface/candle-rotary", rev = "0a718a0856569a92f3112e64f10d07e4447822e8", optional = true }
-tokenizers = { version = "0.20.0", features = ["http"] }
+tokenizers = { version = "0.20.3", features = ["http"] }
 half = "2.4.0"
 tracing = "0.1.40"
 safetensors = "0.4.3"

diff --git a/extensions/tokenizers/rust/src/lib.rs b/extensions/tokenizers/rust/src/lib.rs
@@ -59,7 +59,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
     let mut parameters = FromPretrainedParameters::default();
     if !hf_token.is_null() {
         let hf_token: String = env.get_string(&hf_token).unwrap().into();
-        parameters.auth_token = Some(hf_token);
+        parameters.token = Some(hf_token);
     }
     let tokenizer = Tokenizer::from_pretrained(identifier, Some(parameters));