diff --git a/examples/onnxruntime/training/language-modeling/run_clm.py b/examples/onnxruntime/training/language-modeling/run_clm.py
index cfe72186bc7..d4a473993ab 100644
--- a/examples/onnxruntime/training/language-modeling/run_clm.py
+++ b/examples/onnxruntime/training/language-modeling/run_clm.py
@@ -495,6 +495,8 @@ def tokenize_function(examples):
 
     if hasattr(config, "max_position_embeddings"):
         max_pos_embeddings = config.max_position_embeddings
+        if max_pos_embeddings < 0:
+            max_pos_embeddings = 1024
     else:
         # Define a default value if the attribute is missing in the config.
         max_pos_embeddings = 1024