Release 1.2.0

deeppavlov · Jun 6, 2023 · d39944b · d39944b
2 parents 0448c73 + 852f5be
commit d39944b
Show file tree

Hide file tree

Showing 85 changed files with 3,601 additions and 2,327 deletions.
diff --git a/deeppavlov/_meta.py b/deeppavlov/_meta.py
@@ -1,4 +1,4 @@
-__version__ = '1.1.1'
+__version__ = '1.2.0'
 __author__ = 'Neural Networks and Deep Learning lab, MIPT'
 __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
 __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']

diff --git a/...iers/glue/glue_cola_cased_bert_torch.json → ...s/classifiers/glue/glue_cola_roberta.json b/...iers/glue/glue_cola_cased_bert_torch.json → ...s/classifiers/glue/glue_cola_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "cola",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -120,11 +120,19 @@
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_cola_torch_cased_bert",
-      "BASE_MODEL": "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "cola",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_cola_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/...iers/glue/glue_mrpc_cased_bert_torch.json → ...s/classifiers/glue/glue_mrpc_roberta.json b/...iers/glue/glue_mrpc_cased_bert_torch.json → ...s/classifiers/glue/glue_mrpc_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "mrpc",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -11,38 +11,46 @@
     "class_name": "huggingface_dataset_iterator",
     "features": ["sentence1", "sentence2"],
     "label": "label",
-    "use_label_name": false,
     "seed": 42
   },
   "chainer": {
     "in": ["sentence1", "sentence2"],
-    "in_y": ["y_ids"],
+    "in_y": ["y"],
     "pipe": [
       {
         "class_name": "torch_transformers_preprocessor",
         "vocab_file": "{BASE_MODEL}",
         "do_lower_case": false,
-        "max_seq_length": 100,
+        "max_seq_length": 256,
         "in": ["sentence1", "sentence2"],
         "out": ["bert_features"]
       },
+      {
+        "id": "classes_vocab",
+        "class_name": "simple_vocab",
+        "fit_on": ["y"],
+        "save_path": "{MODEL_PATH}/classes.dict",
+        "load_path": "{MODEL_PATH}/classes.dict",
+        "in": ["y"],
+        "out": ["y_ids"]
+      },
       {
         "in": ["y_ids"],
         "out": ["y_onehot"],
         "class_name": "one_hotter",
-        "depth": 2,
+        "depth": "#classes_vocab.len",
         "single_vector": true
       },
       {
         "class_name": "torch_transformers_classifier",
-        "n_classes": 2,
+        "n_classes": "#classes_vocab.len",
         "return_probas": true,
         "pretrained_bert": "{BASE_MODEL}",
         "save_path": "{MODEL_PATH}/model",
         "load_path": "{MODEL_PATH}/model",
         "optimizer": "AdamW",
         "optimizer_parameters": {
-          "lr": 2e-05
+          "lr": 1e-06
         },
         "learning_rate_drop_patience": 3,
         "learning_rate_drop_div": 2.0,
@@ -55,32 +63,42 @@
         "out": ["y_pred_ids"],
         "class_name": "proba2labels",
         "max_proba": true
+      },
+      {
+        "in": ["y_pred_ids"],
+        "out": ["y_pred_labels"],
+        "ref": "classes_vocab"
       }
     ],
-    "out": ["y_pred_ids"]
+    "out": ["y_pred_labels"]
   },
   "train": {
-    "batch_size": 100,
-    "metrics": [
-      "f1",
-      "accuracy"
-    ],
-    "validation_patience": 10,
+    "batch_size": 4,
+    "metrics": ["accuracy"],
+    "epochs": 2,
     "val_every_n_epochs": 1,
     "log_every_n_epochs": 1,
     "show_examples": false,
-    "evaluation_targets": ["train", "valid"],
+    "evaluation_targets": ["valid"],
     "class_name": "torch_trainer",
     "tensorboard_log_dir": "{MODEL_PATH}/",
     "pytest_max_batches": 2
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_mrpc_torch_cased_bert",
-      "BASE_MODEL": "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "mrpc",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_mrpc_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/...iers/glue/glue_qnli_cased_bert_torch.json → ...s/classifiers/glue/glue_qnli_roberta.json b/...iers/glue/glue_qnli_cased_bert_torch.json → ...s/classifiers/glue/glue_qnli_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "qnli",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -86,11 +86,19 @@
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_qnli_torch_cased_bert",
-      "BASE_MODEL": "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "qnli",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_qnli_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/...fiers/glue/glue_qqp_cased_bert_torch.json → ...gs/classifiers/glue/glue_qqp_roberta.json b/...fiers/glue/glue_qqp_cased_bert_torch.json → ...gs/classifiers/glue/glue_qqp_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "qqp",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -76,11 +76,19 @@
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_qqp_torch_cased_bert",
-      "BASE_MODEL" : "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "qqp",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_qqp_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/...iers/glue/glue_sst2_cased_bert_torch.json → ...s/classifiers/glue/glue_sst2_roberta.json b/...iers/glue/glue_sst2_cased_bert_torch.json → ...s/classifiers/glue/glue_sst2_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "sst2",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -120,11 +120,19 @@
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_sst2_torch_cased_bert",
-      "BASE_MODEL": "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "sst2",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_sst2_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/...iers/glue/glue_stsb_cased_bert_torch.json → ...s/classifiers/glue/glue_stsb_roberta.json b/...iers/glue/glue_stsb_cased_bert_torch.json → ...s/classifiers/glue/glue_stsb_roberta.json
@@ -1,8 +1,8 @@
 {
   "dataset_reader": {
     "class_name": "huggingface_dataset_reader",
-    "path": "glue",
-    "name": "stsb",
+    "path": "{COMPETITION}",
+    "name": "{TASK}",
     "train": "train",
     "valid": "validation",
     "test": "test"
@@ -47,7 +47,7 @@
     "out": ["y_pred"]
   },
   "train": {
-    "batch_size": 128,
+    "batch_size": 32,
     "metrics": [
       "pearson_correlation",
       "spearman_correlation"
@@ -63,11 +63,19 @@
   },
   "metadata": {
     "variables": {
+      "BASE_MODEL": "roberta-large",
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
       "MODELS_PATH": "{ROOT_PATH}/models",
-      "MODEL_PATH": "{MODELS_PATH}/classifiers/glue_stsb_torch_cased_bert",
-      "BASE_MODEL": "bert-base-cased"
-    }
+      "COMPETITION": "glue",
+      "TASK": "stsb",
+      "MODEL_PATH": "{MODELS_PATH}/{COMPETITION}/{TASK}/{BASE_MODEL}"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/glue/glue_stsb_roberta.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
   }
 }
diff --git a/deeppavlov/configs/entity_extraction/entity_detection_en.json b/deeppavlov/configs/entity_extraction/entity_detection_en.json
@@ -5,17 +5,15 @@
       {
         "class_name": "ner_chunker",
         "batch_size": 16,
-        "max_chunk_len" : 180,
         "max_seq_len" : 300,
         "vocab_file": "{TRANSFORMER}",
         "in": ["x"],
         "out": ["x_chunk", "chunk_nums", "chunk_sentences_offsets", "chunk_sentences"]
       },
       {
-        "thres_proba": 0.05,
+        "thres_proba": 0.6,
         "o_tag": "O",
         "tags_file": "{NER_PATH}/tag.dict",
-        "return_entities_with_tags": true,
         "class_name": "entity_detection_parser",
         "id": "edp"
       },

diff --git a/deeppavlov/configs/entity_extraction/entity_detection_ru.json b/deeppavlov/configs/entity_extraction/entity_detection_ru.json
@@ -5,7 +5,6 @@
       {
         "class_name": "ner_chunker",
         "batch_size": 16,
-        "max_chunk_len" : 180,
         "max_seq_len" : 300,
         "vocab_file": "{TRANSFORMER}",
         "in": ["x"],
@@ -15,7 +14,6 @@
         "thres_proba": 0.05,
         "o_tag": "O",
         "tags_file": "{NER_PATH}/tag.dict",
-        "return_entities_with_tags": true,
         "class_name": "entity_detection_parser",
         "id": "edp"
       },

diff --git a/deeppavlov/configs/entity_extraction/entity_extraction_en.json b/deeppavlov/configs/entity_extraction/entity_extraction_en.json
@@ -9,11 +9,11 @@
       },
       {
         "config_path": "{CONFIGS_PATH}/entity_extraction/entity_linking_en.json",
-        "in": ["entity_substr", "tags", "sentences", "entity_offsets", "sentences_offsets"],
-        "out": ["entity_ids", "entity_conf", "entity_pages"]
+        "in": ["entity_substr", "tags", "probas", "sentences", "entity_offsets", "sentences_offsets"],
+        "out": ["entity_ids", "entity_conf", "entity_pages", "entity_labels"]
       }
     ],
-    "out": ["entity_substr", "tags", "entity_offsets", "entity_ids", "entity_conf", "entity_pages"]
+    "out": ["entity_substr", "tags", "entity_offsets", "entity_ids", "entity_conf", "entity_pages", "entity_labels"]
   },
   "metadata": {
     "variables": {

diff --git a/deeppavlov/configs/entity_extraction/entity_extraction_ru.json b/deeppavlov/configs/entity_extraction/entity_extraction_ru.json
@@ -9,11 +9,11 @@
       },
       {
         "config_path": "{CONFIGS_PATH}/entity_extraction/entity_linking_ru.json",
-        "in": ["entity_substr", "tags", "sentences", "entity_offsets", "sentences_offsets"],
-        "out": ["entity_ids", "entity_conf", "entity_pages"]
+        "in": ["entity_substr", "tags", "probas", "sentences", "entity_offsets", "sentences_offsets"],
+        "out": ["entity_ids", "entity_conf", "entity_pages", "entity_labels"]
       }
     ],
-    "out": ["entity_substr", "tags", "entity_offsets", "entity_ids", "entity_conf", "entity_pages"]
+    "out": ["entity_substr", "tags", "entity_offsets", "entity_ids", "entity_conf", "entity_pages", "entity_labels"]
   },
   "metadata": {
     "variables": {