update globals

06_gpu_and_ml/vllm_inference.py

@@ -23,6 +23,7 @@
 from modal import Image, Secret, Stub, method
 
 MODEL_DIR = "/model"
+BASE_MODEL = "mistralai/Mistral-7B-Instruct-v0.1"
 
 
 # ## Define a container image
@@ -46,15 +47,11 @@ def download_model_to_folder():
 
     snapshot_download(
         BASE_MODEL,
-        local_dir="/model",
+        local_dir=MODEL_DIR,
         token=os.environ["HUGGINGFACE_TOKEN"],
     )
 
 
-MODEL_DIR = "/model"
-BASE_MODEL = "mistralai/Mistral-7B-Instruct-v0.1"
-
-
 # ### Image definition
 # We’ll start from a Dockerhub image recommended by `vLLM`, upgrade the older
 # version of `torch` (from 1.14) to a new one specifically built for CUDA 11.8.