fix nvidia-cuda/llama.cpp issue

error message for reference:

RuntimeError: Unexpected error from cudaGetDeviceCount(). Did you run some
cuda functions before calling NumCudaDevices() that might have already set an
error? Error 803: system has unsupported display driver / cuda driver
combination

Signed-off-by: Anupam Kumar <[email protected]>

Dockerfile

@@ -11,10 +11,6 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
 RUN apt-get -y clean
 RUN rm -rf /var/lib/apt/lists/*
 
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute
-ENV DEBIAN_FRONTEND=dialog
-
 # Set working directory
 WORKDIR /app
 
@@ -27,9 +23,10 @@ RUN python3 -m pip install --no-cache-dir https://github.com/abetlen/llama-cpp-p
 RUN sed -i '/llama_cpp_python/d' requirements.txt
 RUN python3 -m pip install --no-cache-dir --no-deps -r requirements.txt
 
-# CUDA 12.1 compat lib
-ENV LD_LIBRARY_PATH=/usr/local/cuda/compat:$LD_LIBRARY_PATH
-ENV LIBRARY_PATH=/usr/local/cuda/compat:$LIBRARY_PATH
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute
+ENV DEBIAN_FRONTEND dialog
+ENV AA_DOCKER_ENV 1
 
 # Copy application files
 COPY context_chat_backend context_chat_backend

hwdetect.sh

@@ -21,6 +21,11 @@ if [ -z "$accel" ]; then
 	echo "Detected hardware: $accel"
 fi
 
+# llama.cpp fix for cpu in docker
+if [ "${AA_DOCKER_ENV:-0}" = "1" ] & [ "$accel" = "cpu" ]; then
+	ln -sf /usr/local/cuda/compat/libcuda.so.1 /lib/x86_64-linux-gnu/
+fi
+
 # if argument is "config", copy the detected hw config to the persistent storage and exit
 if [ "$1" = "config" ]; then
 	if [ ! -d "$APP_PERSISTENT_STORAGE" ]; then