chore(trtllm): ensure max throughput scheduling policy is selected

huggingface · Oct 21, 2024 · 4a0f05e · 4a0f05e
1 parent 1b56a33
commit 4a0f05e
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/backends/trtllm/lib/backend.cpp b/backends/trtllm/lib/backend.cpp
@@ -53,6 +53,7 @@ tle::ExecutorConfig huggingface::tgi::backends::GetExecutorConfig(const json &co
     // Define some configuration variables
     execConfig.setKvCacheConfig(tle::KvCacheConfig(true));
     execConfig.setEnableChunkedContext(computeCapabilities.isPostAmpere());
+    execConfig.setSchedulerConfig(tle::SchedulerConfig(tle::CapacitySchedulerPolicy::kMAX_UTILIZATION));
     return execConfig;
 }