set n_gpu_layers to -1 for Apple Silicon

defog-ai · Feb 11, 2024 · cda14d1 · cda14d1
1 parent 0bff58f
commit cda14d1
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/sqlcoder/query_routes.py b/sqlcoder/query_routes.py
@@ -60,7 +60,7 @@
         hf_hub_download(repo_id="defog/sqlcoder-7b-2", filename="sqlcoder-7b-q5_k_m.gguf", local_dir=defog_path)
 
     if device_type == "apple_silicon":
-        llm = Llama(model_path=filepath, n_gpu_layers=1, n_ctx=4096)
+        llm = Llama(model_path=filepath, n_gpu_layers=-1, n_ctx=4096)
     else:
         llm = Llama(model_path=filepath, n_ctx=4096)
 
@@ -124,4 +124,4 @@ async def query(request: Request):
         "data": data,
         "columns": columns,
         "ran_successfully": True
-    }
+    }