Skip to content

Commit

Permalink
Produce a fatal error if cudf is unable to find pyarrow include direc…
Browse files Browse the repository at this point in the history
…tory (#13976)

Produce a fatal error if cudf python is unable to find pyarrow include directory. Previously the failure only presented itself while trying to compile cython files which failed to include headers from pyarrow.

_Previously:_

```
    FAILED: cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o
    /usr/bin/sccache /usr/bin/g++ -DFMT_HEADER_ONLY=1 -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO -DSPDLOG_FMT_EXTERNAL -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_CPP -Davro_EXPORTS -I/usr/include/python3.10 -I/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/numpy/core/include -I/home/coder/cudf/python/cudf/cudf/_lib -I/home/coder/cudf/cpp/build/release/_deps/libcudacxx-src/lib/cmake/libcudacxx/../../../include -I/home/coder/cudf/cpp/build/release/_deps/thrust-src -I/home/coder/cudf/cpp/build/release/_deps/thrust-src/dependencies/cub -isystem /home/coder/cudf/cpp/build/release/_deps/dlpack-src/include -isystem /home/coder/cudf/cpp/build/release/_deps/jitify-src -isystem /home/coder/cudf/cpp/include -isystem /home/coder/cudf/cpp/build/release/include -isystem /home/coder/rmm/include -isystem /usr/local/cuda/include -isystem /home/coder/fmt/include -isystem /home/coder/rmm/build/release/_deps/spdlog-src/include -O3 -DNDEBUG -fPIC -MD -MT cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o -MF cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o.d -o cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o -c /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/avro.cxx
    /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/avro.cxx:1291:10: fatal error: arrow/python/platform.h: No such file or directory
     1291 | #include "arrow/python/platform.h"
          |          ^~~~~~~~~~~~~~~~~~~~~~~~~
    compilation terminated.
    [6/23] Building CXX object cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o
    FAILED: cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o
    /usr/bin/sccache /usr/bin/g++ -DFMT_HEADER_ONLY=1 -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO -DSPDLOG_FMT_EXTERNAL -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_CPP -Dcsv_EXPORTS -I/usr/include/python3.10 -I/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/numpy/core/include -I/home/coder/cudf/python/cudf/cudf/_lib -I/home/coder/cudf/cpp/build/release/_deps/libcudacxx-src/lib/cmake/libcudacxx/../../../include -I/home/coder/cudf/cpp/build/release/_deps/thrust-src -I/home/coder/cudf/cpp/build/release/_deps/thrust-src/dependencies/cub -isystem /home/coder/cudf/cpp/build/release/_deps/dlpack-src/include -isystem /home/coder/cudf/cpp/build/release/_deps/jitify-src -isystem /home/coder/cudf/cpp/include -isystem /home/coder/cudf/cpp/build/release/include -isystem /home/coder/rmm/include -isystem /usr/local/cuda/include -isystem /home/coder/fmt/include -isystem /home/coder/rmm/build/release/_deps/spdlog-src/include -O3 -DNDEBUG -fPIC -MD -MT cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o -MF cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o.d -o cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o -c /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/csv.cxx
    /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/csv.cxx:1292:10: fatal error: arrow/python/platform.h: No such file or directory
     1292 | #include "arrow/python/platform.h"
          |          ^~~~~~~~~~~~~~~~~~~~~~~~~
    compilation terminated.
...
```
_With these changes:_
```
    CMake Error at cudf/_lib/CMakeLists.txt:107 (message):
      Error while trying to obtain pyarrow include dir:
    
      Traceback (most recent call last):
    
        File "<string>", line 1, in <module>
        File "/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/pyarrow/__init__.py", line 65, in <module>
          import pyarrow.lib as _lib
    
      ImportError: libarrow.so.1200: cannot open shared object file: No such file
      or directory
```

Authors:
  - Christopher Harris (https://github.com/cwharris)
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Vyas Ramasubramani (https://github.com/vyasr)

URL: #13976
  • Loading branch information
cwharris authored Sep 11, 2023
1 parent 0bcad6c commit bc304a2
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions python/cudf/cudf/_lib/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,15 @@ find_package(Python 3.9 REQUIRED COMPONENTS Interpreter)
execute_process(
COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())"
OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
ERROR_VARIABLE PYARROW_ERROR
RESULT_VARIABLE PYARROW_RESULT
OUTPUT_STRIP_TRAILING_WHITESPACE
)

if(${PYARROW_RESULT})
message(FATAL_ERROR "Error while trying to obtain pyarrow include directory:\n${PYARROW_ERROR}")
endif()

set(targets_using_arrow_headers interop avro csv orc json parquet)
foreach(target IN LISTS targets_using_arrow_headers)
target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")
Expand Down

0 comments on commit bc304a2

Please sign in to comment.