add finance train dataset

qbc2016 · Apr 11, 2024 · 997e2df · 997e2df
1 parent 5a048f1
commit 997e2df
Showing 1 changed file with 14 additions and 1 deletion.
diff --git a/federatedscope/llm/dataloader/dataloader.py b/federatedscope/llm/dataloader/dataloader.py
@@ -287,11 +287,24 @@ def load_llm_dataset(config=None, **kwargs):
                 '/medical_tc_train.jsonl', config.data.root)
             os.rename(os.path.join(config.data.root, 'train.jsonl'), fp)
         list_data_dict = load_jsonl(fp,
-                                    instruction='question',
+                                    instruction='instruction',
                                     input='input',
                                     output='output',
                                     category='output')
         dataset = LLMDataset(list_data_dict, tokenizer)
+    elif dataset_name.lower() == "finance":
+        fp = os.path.join(config.data.root, 'finance_train_data.jsonl')
+        if not os.path.exists(fp):
+            download_url(
+                'https://federatedscope.oss-cn-beijing.aliyuncs.com/FS-LLM'
+                '/finance_train_data.jsonl', config.data.root)
+            os.rename(os.path.join(config.data.root, 'train.jsonl'), fp)
+        list_data_dict = load_jsonl(fp,
+                                    instruction='instruction',
+                                    input='input',
+                                    output='output',
+                                    category='category')
+        dataset = LLMDataset(list_data_dict, tokenizer)
     elif dataset_name.lower() == 'code_search_net':
         from tqdm import tqdm
         from federatedscope.llm.dataset.code_search_net import \