diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000..5cfe0dcb065dc
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+vllm/*.so
diff --git a/Dockerfile b/Dockerfile
index 6ef03b843f457..bd66afe79c7e1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -81,4 +81,3 @@ COPY --from=build /workspace/vllm/*.so /workspace/vllm/
 COPY vllm vllm
 
 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
-