Write a compile_commands.json from build_ext

Produce a JSON database of the compiler commands executed while building
extension modules as `build/compile_commands.json`. This is usable by
various C and C++ language servers, linters, and IDEs, like `clangd`,
`clang-tidy`, and CLion. These tools need to understand the header
search path and macros passed on the compiler command line in order to
correctly interpret source files. In the case of Python extension
modules, the developer might not even know all of the compiler flags
that are being used, since some are inherited from the interpreter via
`sysconfig`.

newsfragments/4358.feature.rst

@@ -0,0 +1 @@
+A ``compile_commands.json`` is now written to the ``build/`` directory when building extension modules.

setuptools/_distutils/unixccompiler.py

@@ -145,6 +145,10 @@ class UnixCCompiler(CCompiler):
     if sys.platform == "cygwin":
         exe_extension = ".exe"
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.compile_commands = []
+
     def preprocess(
         self,
         source,
@@ -185,7 +189,11 @@ def preprocess(
     def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
         compiler_so = compiler_fixup(self.compiler_so, cc_args + extra_postargs)
         try:
-            self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs)
+            cmd = compiler_so + cc_args + [src, '-o', obj] + extra_postargs
+            self.spawn(cmd)
+            self.compile_commands.append(
+                {"directory": os.getcwd(), "arguments": cmd, "file": src}
+            )
         except DistutilsExecError as msg:
             raise CompileError(msg)
 

setuptools/command/build_ext.py

@@ -1,9 +1,10 @@
+import json
 import os
 import sys
 import itertools
 from importlib.machinery import EXTENSION_SUFFIXES
 from importlib.util import cache_from_source as _compiled_file_name
-from typing import Dict, Iterator, List, Tuple
+from typing import Any, Dict, Iterator, List, Tuple
 from pathlib import Path
 
 from distutils.command.build_ext import build_ext as _du_build_ext
@@ -89,10 +90,31 @@ def run(self):
         """Build extensions in build directory, then copy if --inplace"""
         old_inplace, self.inplace = self.inplace, 0
         _build_ext.run(self)
+        self._update_compilation_database(
+            getattr(self.compiler, "compile_commands", [])
+        )
         self.inplace = old_inplace
         if old_inplace:
             self.copy_extensions_to_source()
 
+    def _update_compilation_database(self, commands: List[Dict[str, Any]]) -> None:
+        build_base = Path(self.get_finalized_command('build').build_base)
+        build_base.mkdir(exist_ok=True)
+
+        db_file = build_base / "compile_commands.json"
+        try:
+            database = json.loads(db_file.read_text(encoding="utf-8"))
+        except OSError:
+            database = []
+
+        # Drop existing entries for newly built files
+        built_files = {command["file"] for command in commands}
+        database = [obj for obj in database if obj.get("file") not in built_files]
+
+        database.extend(commands)
+        output = json.dumps(database, allow_nan=False, indent=4, sort_keys=True)
+        db_file.write_text(output, encoding="utf-8")
+
     def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]:
         fullname = self.get_ext_fullname(ext.name)
         filename = self.get_ext_filename(fullname)
@@ -342,8 +364,9 @@ def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):
         if compile and os.path.exists(stub_file):
             raise BaseError(stub_file + " already exists! Please delete.")
         if not self.dry_run:
-            with open(stub_file, 'w', encoding="utf-8") as f:
-                content = '\n'.join([
+            f = open(stub_file, 'w')
+            f.write(
+                '\n'.join([
                     "def __bootstrap__():",
                     "   global __bootstrap__, __file__, __loader__",
                     "   import sys, os, pkg_resources, importlib.util" + if_dl(", dl"),
@@ -367,7 +390,8 @@ def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):
                     "__bootstrap__()",
                     "",  # terminal \n
                 ])
-                f.write(content)
+            )
+            f.close()
         if compile:
             self._compile_and_remove_stub(stub_file)