diff --git a/.gitignore b/.gitignore
index 06e8923ced311..ef8b4845a9176 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,8 @@
 # This file has been modified by Neural Magic
 
+# nm-vllm commit id, generated by setup.py
+venv/commit_id.py
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
diff --git a/setup.py b/setup.py
index 714769b43935a..fc5f713b02396 100644
--- a/setup.py
+++ b/setup.py
@@ -33,16 +33,12 @@ def embed_commit_hash():
     try:
         commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"],
                                             encoding="utf-8").strip()
+        commit_contents = f'__commit__ = "{commit_id}"\n'
 
-        version_file = os.path.join(ROOT_DIR, "vllm", "version.py")
-        with open(version_file, encoding="utf-8") as f:
-            version_contents = f.read()
-
-        version_contents = version_contents.replace("COMMIT_HASH_PLACEHOLDER",
-                                                    f"{commit_id}")
-
+        version_file = os.path.join(ROOT_DIR, "vllm", "commit_id.py")
         with open(version_file, "w", encoding="utf-8") as f:
-            f.write(version_contents)
+            f.write(commit_contents)
+
     except subprocess.CalledProcessError as e:
         warnings.warn(f"failed to get commit hash:\n{e}",
                       RuntimeWarning,
diff --git a/vllm/version.py b/vllm/version.py
index fc25433285af7..3f3f49f83edf9 100644
--- a/vllm/version.py
+++ b/vllm/version.py
@@ -1,3 +1,14 @@
 # UPSTREAM SYNC: take downstream
+import warnings
+
+try:
+    import vllm.commit_id
+    __commit__ = vllm.commit_id.__commit__
+except Exception as e:
+    warnings.warn(f"failed to read commit hash:\n{e}",
+                  RuntimeWarning,
+                  stacklevel=2)
+    __commit__ = "COMMIT_HASH_PLACEHOLDER"
+
 __version__ = "0.5.1"
 __commit__ = "07abe05bea44b7f4cc8c567527063c3992a8f301"