[tokenizer] add optional tokenizerPath Prior to modelPath (#3120)

* [tokenizer] add optional tokenizerPath Prior to modelPath --------- Co-authored-by: Frank Liu <[email protected]>
deepjavalibrary · Apr 25, 2024 · 6efe660 · 6efe660
1 parent c03669f
commit 6efe660
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -30,6 +30,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -686,7 +687,6 @@ static PaddingStrategy fromValue(String value) {
     /** The builder for creating huggingface tokenizer. */
     public static final class Builder {
 
-        private Path tokenizerPath;
         private NDManager manager;
         private Map<String, String> options;
 
@@ -724,7 +724,7 @@ public Builder optTokenizerName(String tokenizerName) {
          * @return this builder
          */
         public Builder optTokenizerPath(Path tokenizerPath) {
-            this.tokenizerPath = tokenizerPath;
+            options.putIfAbsent("tokenizerPath", tokenizerPath.toString());
             return this;
         }
 
@@ -894,9 +894,11 @@ public HuggingFaceTokenizer build() throws IOException {
             if (tokenizerName != null) {
                 return managed(HuggingFaceTokenizer.newInstance(tokenizerName, options));
             }
-            if (tokenizerPath == null) {
+            String path = options.get("tokenizerPath");
+            if (path == null) {
                 throw new IllegalArgumentException("Missing tokenizer path.");
             }
+            Path tokenizerPath = Paths.get(path);
             if (Files.isDirectory(tokenizerPath)) {
                 Path tokenizerFile = tokenizerPath.resolve("tokenizer.json");
                 if (Files.exists(tokenizerFile)) {

diff --git a/...ns/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java b/...ns/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java
@@ -64,6 +64,7 @@ public void testCrossEncoderTranslator()
                         .optBlock(block)
                         .optEngine("PyTorch")
                         .optArgument("tokenizer", "bert-base-cased")
+                        .optArgument("tokenizerPath", modelDir)
                         .optOption("hasParameter", "false")
                         .optTranslatorFactory(new CrossEncoderTranslatorFactory())
                         .build();