add qwen pretrain test

llm/run_pretrain.py

@@ -115,6 +115,7 @@ class ModelArguments:
     Arguments pertaining to which model/config/tokenizer we are going to pre-train from.
     """
 
+    model_type: Optional[str] = field(default="llama", metadata={"help": "Use for CI test."})
     model_name_or_path: str = field(
         default="__internal_testing__/tiny-random-llama",
         metadata={

paddlenlp/trainer/trainer.py

@@ -2510,6 +2510,7 @@ def predict(
             test_dataloader,
             description="Prediction",
             ignore_keys=ignore_keys,
+            prediction_loss_only=True if self.compute_metrics is None else None,
             metric_key_prefix=metric_key_prefix,
             max_eval_iters=self.args.max_evaluate_steps,
         )

tests/fixtures/llm/pretrain.yaml

@@ -25,6 +25,9 @@ pretrain:
     chatglm:
       model_type: chatglm
       model_name_or_path: __internal_testing__/tiny-fused-chatglm
+    qwen:
+      model_type: qwen
+      model_name_or_path: __internal_testing__/tiny-fused-qwen
 
 inference-predict:
   default:
@@ -44,4 +47,4 @@ inference-infer:
     dtype: float16
     batch_size: 2
     decode_strategy: greedy_search
-    max_length: 20
+    max_length: 20

tests/llm/test_pretrain.py

@@ -31,6 +31,7 @@
     ["model_dir"],
     [
         ["llama"],
+        ["qwen"],
     ],
 )
 class PretrainTest(LLMTest, unittest.TestCase):
@@ -41,16 +42,30 @@ def setUp(self) -> None:
         LLMTest.setUp(self)
 
         self.dataset_dir = tempfile.mkdtemp()
-        self.model_codes_dir = os.path.join(self.root_path, self.model_dir)
-        sys.path.insert(0, self.model_codes_dir)
+        if self.model_dir != "qwen":
+            self.model_codes_dir = os.path.join(self.root_path, self.model_dir)
+            sys.path.insert(0, self.model_codes_dir)
+        else:
+            self.model_codes_dir = self.root_path
 
     def tearDown(self) -> None:
         LLMTest.tearDown(self)
 
-        sys.path.remove(self.model_codes_dir)
+        if self.model_dir != "qwen":
+            sys.path.remove(self.model_codes_dir)
+
         shutil.rmtree(self.dataset_dir)
 
     def test_pretrain(self):
+
+        pretrain_flag = False
+        for key, value in sys.modules.items():
+            if "run_pretrain" in key:
+                pretrain_flag = True
+                break
+        if pretrain_flag:
+            del sys.modules["run_pretrain"]
+
         # Run pretrain
         URL = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy"
         URL2 = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz"
@@ -67,5 +82,6 @@ def test_pretrain(self):
 
             main()
 
-        self.run_predictor({"inference_model": True})
+        if self.model_dir != "qwen":
+            self.run_predictor({"inference_model": True})
         self.run_predictor({"inference_model": False})

tests/requirements.txt

@@ -7,3 +7,4 @@ tool_helpers
 fast_tokenizer_python
 sacremoses
 pydantic==1.10.9
+tiktoken