From 30428fd9ae870d48b318cb6a1220c7c8619c0724 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 28 Sep 2023 12:58:14 -0400 Subject: [PATCH] Building faiss statically each time. Will slow down CI but alleviate some of the conda issues. --- .../bench_ann_cuda-118_arch-x86_64.yaml | 3 +-- conda/recipes/raft-ann-bench-cpu/meta.yaml | 4 ++-- conda/recipes/raft-ann-bench/meta.yaml | 16 ++++++++-------- cpp/bench/ann/CMakeLists.txt | 1 + cpp/cmake/thirdparty/get_faiss.cmake | 4 ++-- dependencies.yaml | 3 +-- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 6ceeb568ef..8652c6f076 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -17,7 +17,6 @@ dependencies: - cudatoolkit - cxx-compiler - cython>=3.0.0 -- faiss-proc=*=cuda - gcc_linux-64=11.* - glog>=0.6.0 - h5py>=3.8.0 @@ -31,10 +30,10 @@ dependencies: - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 - libfaiss-avx2=1.7.4 -- libfaiss=1.7.4 - nccl>=2.9.9 - ninja - nlohmann_json>=3.11.2 +- rmm=23.10.* - scikit-build>=0.13.1 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/recipes/raft-ann-bench-cpu/meta.yaml b/conda/recipes/raft-ann-bench-cpu/meta.yaml index f7a34a4bb3..7df2c60f7e 100644 --- a/conda/recipes/raft-ann-bench-cpu/meta.yaml +++ b/conda/recipes/raft-ann-bench-cpu/meta.yaml @@ -47,7 +47,7 @@ requirements: host: - glog {{ glog_version }} - matplotlib - - faiss-cpu {{ faiss_version }} +# - faiss-cpu {{ faiss_version }} - nlohmann_json {{ nlohmann_json_version }} - python - pyyaml @@ -56,7 +56,7 @@ requirements: run: - glog {{ glog_version }} - h5py {{ h5py_version }} - - faiss-cpu {{ faiss_version }} +# - faiss-cpu {{ faiss_version }} - matplotlib - python - pyyaml diff --git a/conda/recipes/raft-ann-bench/meta.yaml b/conda/recipes/raft-ann-bench/meta.yaml index a2ab0af643..2270460df6 100644 --- a/conda/recipes/raft-ann-bench/meta.yaml +++ b/conda/recipes/raft-ann-bench/meta.yaml @@ -71,10 +71,10 @@ requirements: - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet - {% if cuda_major == "11" %} - - faiss-proc=*=cuda - - libfaiss {{ faiss_version }} - {% endif %} +# {% if cuda_major == "11" %} +# - faiss-proc=*=cuda +# - libfaiss {{ faiss_version }} +# {% endif %} - h5py {{ h5py_version }} - benchmark - matplotlib @@ -93,10 +93,10 @@ requirements: {% endif %} - glog {{ glog_version }} # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet - {% if cuda_major == "11" %} - - faiss-proc=*=cuda - - libfaiss {{ faiss_version }} - {% endif %} +# {% if cuda_major == "11" %} +# - faiss-proc=*=cuda +# - libfaiss {{ faiss_version }} +# {% endif %} - h5py {{ h5py_version }} - benchmark - glog {{ glog_version }} diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index e307de5b1d..f95cf67e5c 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -77,6 +77,7 @@ if(RAFT_ANN_BENCH_USE_FAISS_GPU_FLAT OR RAFT_ANN_BENCH_USE_FAISS_CPU_IFFLAT ) set(RAFT_ANN_BENCH_USE_FAISS ON) + set(RAFT_USE_FAISS_STATIC ON) endif() set(RAFT_ANN_BENCH_USE_RAFT OFF) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index bb7dc1eb5c..b2d0cf20cf 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -90,8 +90,8 @@ function(find_and_configure_faiss) endfunction() find_and_configure_faiss(VERSION 1.7.4 - REPOSITORY ${RAFT_FAISS_GIT_REPOSITORY} - PINNED_TAG ${RAFT_FAISS_GIT_TAG} + REPOSITORY git@github.com:facebookresearch/faiss.git + PINNED_TAG v1.7.4 BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} EXCLUDE_FROM_ALL ${RAFT_EXCLUDE_FAISS_FROM_ALL} ENABLE_GPU ${RAFT_FAISS_ENABLE_GPU}) diff --git a/dependencies.yaml b/dependencies.yaml index 9e208aac5f..1baf741a60 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -181,9 +181,8 @@ dependencies: - nlohmann_json>=3.11.2 - glog>=0.6.0 - h5py>=3.8.0 - - libfaiss=1.7.4 - benchmark>=1.8.2 - - faiss-proc=*=cuda + - rmm=23.10.* specific: - output_types: [conda, pyproject, requirements] matrices: