remove max_num_batched_tokens

apecloud · Nov 9, 2023 · 8d62843 · 8d62843
1 parent 79809b6
commit 8d62843
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/addons/llm/templates/scripts.yaml b/addons/llm/templates/scripts.yaml
@@ -54,7 +54,7 @@ data:
         sleep 1
         continue
       fi
-      python -m vllm.entrypoints.api_server --host 0.0.0.0 --port 8000 --model ${MODEL_NAME} --gpu-memory-utilization 0.95 --max-num-seqs 512 --max-num-batched-tokens 8192 --tensor-parallel-size ${KB_VLLM_N} ${EXTRA_ARGS} 2>&1 > log 
+      python -m vllm.entrypoints.api_server --host 0.0.0.0 --port 8000 --model ${MODEL_NAME} --gpu-memory-utilization 0.95 --max-num-seqs 512 --tensor-parallel-size ${KB_VLLM_N} ${EXTRA_ARGS} 2>&1 > log 
       code=$?
       if [ $code -eq 0 ]; then
         break