ProteinTokenizerTransform

Genentech · Jul 19, 2024 · 6896b71 · 6896b71
1 parent 785fc18
commit 6896b71
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 10 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,7 +22,7 @@ requires-python = ">=3.10"
 
 [project.optional-dependencies]
 all = [
-    "beignet[datasets,docs,mdtraj,test]",
+    "beignet[datasets,docs,mdtraj,test,tokenizers]",
 ]
 datasets = [
     "biopython",
@@ -44,6 +44,9 @@ test = [
     "pytest-mock",
     "scipy",
 ]
+tokenizers = [
+    "transformers",
+]
 
 [tool.ruff]
 lint.select = [

diff --git a/src/beignet/data/tokenizers/protein_tokenizer/special_tokens_map.json b/src/beignet/data/tokenizers/protein_tokenizer/special_tokens_map.json
@@ -1,7 +1,7 @@
 {
-  "cls_token": "<cls>",
-  "eos_token": "<eos>",
-  "mask_token": "<mask>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
+    "cls_token": "<cls>",
+    "eos_token": "<eos>",
+    "mask_token": "<mask>",
+    "pad_token": "<pad>",
+    "unk_token": "<unk>"
 }
diff --git a/src/beignet/data/tokenizers/protein_tokenizer/tokenizer_config.json b/src/beignet/data/tokenizers/protein_tokenizer/tokenizer_config.json
@@ -1,6 +1,6 @@
 {
-  "clean_up_tokenization_spaces": true,
-  "do_lower_case": false,
-  "model_max_length": 1024,
-  "tokenizer_class": "ProteinMLMTokenizer"
+    "clean_up_tokenization_spaces": true,
+    "do_lower_case": false,
+    "model_max_length": 1024,
+    "tokenizer_class": "ProteinTokenizer"
 }