Avoid relinking python modules on every rebuild (#1334)

Avoid rerunning `conduit_python_py_setup`'s `pip install` command on every invocation of `make` or `ninja` by recording its last execution time with an output time stamp file. Whenever it does rerun, such as when an input file changes, relink dependent python modules since the `pip install` command wipes out its `--target` directory. This can be achieved by making `conduit_python_py_setup` an INTERFACE library that propagates the time stamp file as a link dependency of dependent python modules through `INTERFACE_LINK_DEPENDS`.
LLNL · Nov 18, 2024 · 289f76b · 289f76b
1 parent 0d8db6d
commit 289f76b
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 35 deletions.
diff --git a/src/cmake/thirdparty/SetupPython.cmake b/src/cmake/thirdparty/SetupPython.cmake
@@ -238,18 +238,26 @@ FUNCTION(PYTHON_ADD_PIP_SETUP)
         string(REGEX REPLACE "/" "\\\\" abs_dest_path  ${abs_dest_path})
     endif()
 
+    # Use a timestamp file to track when the following pip
+    # command was last executed w.r.t. its dependencies.
+    set(stamp ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}.stamp)
+
     # NOTE: With pip, you can't directly control build dir with an arg
     # like we were able to do with distutils, you have to use TMPDIR
     # TODO: we might want to  explore this in the future
-    add_custom_command(OUTPUT  ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build
+    add_custom_command(OUTPUT ${stamp}
             COMMAND ${PYTHON_EXECUTABLE} -m pip install . -V --upgrade
             --disable-pip-version-check --no-warn-script-location
             --target "${abs_dest_path}"
+            COMMAND ${CMAKE_COMMAND} -E touch ${stamp}
             DEPENDS  ${args_PY_SETUP_FILE} ${args_PY_SOURCES}
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-    add_custom_target(${args_NAME} ALL DEPENDS
-                      ${CMAKE_CURRENT_BINARY_DIR}/${args_NAME}_build)
+    # The above pip command wipes the --target directory,
+    # so any dependent modules need to be linked afterwards.
+    # Propagate this this dependency as a usage requirement.
+    add_library(${args_NAME} INTERFACE ${stamp})
+    set_property(TARGET ${args_NAME} APPEND PROPERTY INTERFACE_LINK_DEPENDS ${stamp})
 
     # also use pip for the install ...
     # if PYTHON_MODULE_INSTALL_PREFIX is set, install there
@@ -359,7 +367,7 @@ FUNCTION(PYTHON_ADD_COMPILED_MODULE)
 
     # win32, link to python
     if(WIN32)
-        target_link_libraries(${args_NAME} ${PYTHON_LIBRARIES})
+        target_link_libraries(${args_NAME} PRIVATE ${PYTHON_LIBRARIES})
     endif()
 
     # support installing the python module components to an
@@ -435,8 +443,8 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE)
                                SOURCES       ${args_SOURCES}
                                FOLDER        ${args_FOLDER})
 
-    # args_NAME depends on "${args_NAME}_py_setup"
-    add_dependencies( ${args_NAME} "${args_NAME}_py_setup")
+    # "${args_NAME}" depends on "${args_NAME}_py_setup"
+    target_link_libraries("${args_NAME}" PRIVATE "${args_NAME}_py_setup")
 
 ENDFUNCTION(PYTHON_ADD_HYBRID_MODULE)
 

diff --git a/src/libs/blueprint/python/CMakeLists.txt b/src/libs/blueprint/python/CMakeLists.txt
@@ -14,10 +14,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 #############################################################
 # blueprint.mcarray
@@ -32,10 +32,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_mcarray_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_mcarray_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_mcarray_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_mcarray_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_mcarray_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 # add mcarray examples submodule
 PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_mcarray_examples_python
@@ -46,10 +46,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_mcarray_examples_pyth
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_mcarray_examples_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_mcarray_examples_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_mcarray_examples_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_mcarray_examples_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 #############################################################
 # blueprint.mesh
@@ -64,10 +64,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_mesh_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_mesh_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_mesh_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_mesh_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_mesh_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 
 # add mesh examples submodule
@@ -79,10 +79,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_mesh_examples_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_mesh_examples_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_mesh_examples_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_mesh_examples_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_mesh_examples_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 #############################################################
 # blueprint.table
@@ -97,10 +97,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_table_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_table_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_table_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_table_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_table_python PRIVATE conduit conduit_blueprint conduit_python_build)
 
 
 # add table examples submodule
@@ -112,7 +112,7 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_blueprint_table_examples_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_blueprint_table_examples_python conduit_python_py_setup)
+target_link_libraries(conduit_blueprint_table_examples_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_blueprint_table_examples_python conduit conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_blueprint_table_examples_python PRIVATE conduit conduit_blueprint conduit_python_build)
diff --git a/src/libs/conduit/python/CMakeLists.txt b/src/libs/conduit/python/CMakeLists.txt
@@ -44,7 +44,7 @@ PYTHON_ADD_HYBRID_MODULE(NAME          conduit_python
                          FOLDER        libs/python)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_python conduit conduit_python_build)
+target_link_libraries(conduit_python PRIVATE conduit conduit_python_build)
 
 
 #############################################################
@@ -60,10 +60,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_utils_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_utils_python conduit_python_py_setup)
+target_link_libraries(conduit_utils_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_utils_python conduit conduit_python_build)
+target_link_libraries(conduit_utils_python PRIVATE conduit conduit_python_build)
 
 # install the capi header so other python modules can use it
 # support alt install dir for python module via PYTHON_MODULE_INSTALL_PREFIX

diff --git a/src/libs/relay/python/CMakeLists.txt b/src/libs/relay/python/CMakeLists.txt
@@ -14,10 +14,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_relay_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_relay_python conduit_python_py_setup)
+target_link_libraries(conduit_relay_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs
-target_link_libraries(conduit_relay_python conduit conduit_relay conduit_python_build)
+target_link_libraries(conduit_relay_python PRIVATE conduit conduit_relay conduit_python_build)
 
 # add relay io submodule
 PYTHON_ADD_COMPILED_MODULE(NAME          conduit_relay_io_python
@@ -28,10 +28,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_relay_io_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_relay_io_python conduit_python_py_setup)
+target_link_libraries(conduit_relay_io_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_relay_io_python conduit conduit_relay conduit_python_build)
+target_link_libraries(conduit_relay_io_python PRIVATE conduit conduit_relay conduit_python_build)
 
 # add relay io blueprint submodule
 PYTHON_ADD_COMPILED_MODULE(NAME          conduit_relay_io_blueprint_python
@@ -42,10 +42,10 @@ PYTHON_ADD_COMPILED_MODULE(NAME          conduit_relay_io_blueprint_python
                            FOLDER        libs/python)
 
 # compiled modules depend on output dir structure created by main module setup
-add_dependencies( conduit_relay_io_blueprint_python conduit_python_py_setup)
+target_link_libraries(conduit_relay_io_blueprint_python PRIVATE conduit_python_py_setup)
 
 # link with the proper libs (beyond python)
-target_link_libraries(conduit_relay_io_blueprint_python conduit conduit_relay conduit_blueprint conduit_python_build)
+target_link_libraries(conduit_relay_io_blueprint_python PRIVATE conduit conduit_relay conduit_blueprint conduit_python_build)
 
 if(SILO_FOUND)
 
@@ -58,10 +58,10 @@ if(SILO_FOUND)
                                FOLDER        libs/python)
 
     # compiled modules depend on output dir structure created by main module setup
-    add_dependencies( conduit_relay_io_silo_python conduit_python_py_setup)
+    target_link_libraries(conduit_relay_io_silo_python PRIVATE conduit_python_py_setup)
 
     # link with the proper libs (beyond python)
-    target_link_libraries(conduit_relay_io_silo_python conduit conduit_relay conduit_blueprint conduit_python_build)
+    target_link_libraries(conduit_relay_io_silo_python PRIVATE conduit conduit_relay conduit_blueprint conduit_python_build)
 
 endif()
 
@@ -75,10 +75,10 @@ if(ENABLE_RELAY_WEBSERVER)
                                FOLDER        libs/python)
 
     # compiled modules depend on output dir structure created by main module setup
-    add_dependencies( conduit_relay_web_python conduit_python_py_setup)
+    target_link_libraries(conduit_relay_web_python PRIVATE conduit_python_py_setup)
 
     # link with the proper libs (beyond python)
-    target_link_libraries(conduit_relay_web_python conduit conduit_relay conduit_python_build)
+    target_link_libraries(conduit_relay_web_python PRIVATE conduit conduit_relay conduit_python_build)
 endif()
 
 ################################################################
@@ -96,10 +96,10 @@ if(MPI_FOUND)
                                FOLDER        libs/python)
 
     # compiled modules depend on output dir structure created by main module setup
-    add_dependencies( conduit_relay_mpi_python conduit_python_py_setup)
+    target_link_libraries(conduit_relay_mpi_python PRIVATE conduit_python_py_setup)
 
     # link with the proper libs (beyond python)
-    target_link_libraries(conduit_relay_mpi_python conduit conduit_relay_mpi conduit_python_build)
+    target_link_libraries(conduit_relay_mpi_python PRIVATE conduit conduit_relay_mpi conduit_python_build)
 
 endif()