From bc896fa99ea2be3b91ff7aa7829eb0dbb87701a1 Mon Sep 17 00:00:00 2001 From: Salar Hosseini Date: Tue, 29 Oct 2024 17:17:47 +0000 Subject: [PATCH] Import tt-metal model via pythonpath instead of symlink Signed-off-by: Salar Hosseini --- examples/offline_inference_tt.py | 4 ++-- examples/server_example_tt.py | 4 ++-- tt_metal/README.md | 6 +----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/offline_inference_tt.py b/examples/offline_inference_tt.py index 98879f0fe28b5..67e0c733f20d9 100644 --- a/examples/offline_inference_tt.py +++ b/examples/offline_inference_tt.py @@ -15,8 +15,8 @@ from vllm.inputs.data import TokensPrompt from vllm.engine.multiprocessing.client import MQLLMEngineClient -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from tt_metal.models.demos.t3000.llama2_70b.tt.llama_generation import TtLlamaModelForGeneration +# Import and register model from tt-metal +from models.demos.t3000.llama2_70b.tt.llama_generation import TtLlamaModelForGeneration ModelRegistry.register_model("TTLlamaForCausalLM", TtLlamaModelForGeneration) diff --git a/examples/server_example_tt.py b/examples/server_example_tt.py index ee4e03a5e2f30..8116a54584593 100644 --- a/examples/server_example_tt.py +++ b/examples/server_example_tt.py @@ -4,8 +4,8 @@ from vllm import ModelRegistry -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from tt_metal.models.demos.t3000.llama2_70b.tt.llama_generation import TtLlamaModelForGeneration +# Import and register model from tt-metal +from models.demos.t3000.llama2_70b.tt.llama_generation import TtLlamaModelForGeneration ModelRegistry.register_model("TTLlamaForCausalLM", TtLlamaModelForGeneration) diff --git a/tt_metal/README.md b/tt_metal/README.md index 0c9900d83500a..2878b4963eb7e 100644 --- a/tt_metal/README.md +++ b/tt_metal/README.md @@ -43,11 +43,7 @@ To run Meta-Llama-3.1, it is required to have access to the model on Hugging Fac ## Preparing the tt-metal models -1. Create a symbolic link to the tt-metal models folder inside vLLM: - ```sh - cd tt_metal - ln -s /models ./models - ``` +1. Ensure that `$PYTHONPATH` contains the path to tt-metal (should already have been done when installing tt-metal) 2. For the desired model, follow the setup instructions (if any) for the corresponding tt-metal demo. E.g. For Llama-3.1-70B, follow the [demo instructions](https://github.com/tenstorrent/tt-metal/tree/main/models/demos/t3000/llama3_70b) for preparing the weights and environment variables. ## Running the offline inference example