From 682fb986c8880105fa913c2461f69965e1c0ed4f Mon Sep 17 00:00:00 2001 From: Eric Zhang Date: Wed, 3 Jan 2024 20:01:35 +0000 Subject: [PATCH] Fix TEI and TGI-Mixtral - TEI: Issue with HuggingFace secrets again - TGI-Mixtral: https://github.com/huggingface/text-generation-inference/issues/1342 --- 06_gpu_and_ml/embeddings/text_embeddings_inference.py | 8 +++++++- 06_gpu_and_ml/tgi_mixtral.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/06_gpu_and_ml/embeddings/text_embeddings_inference.py b/06_gpu_and_ml/embeddings/text_embeddings_inference.py index c740f842f..a88fc78a2 100644 --- a/06_gpu_and_ml/embeddings/text_embeddings_inference.py +++ b/06_gpu_and_ml/embeddings/text_embeddings_inference.py @@ -29,7 +29,13 @@ def spawn_server() -> subprocess.Popen: - process = subprocess.Popen(["text-embeddings-router"] + LAUNCH_FLAGS) + process = subprocess.Popen( + ["text-embeddings-router"] + LAUNCH_FLAGS, + env={ + **os.environ, + "HUGGING_FACE_HUB_TOKEN": os.environ["HUGGINGFACE_TOKEN"], + }, + ) # Poll until webserver at 127.0.0.1:8000 accepts connections before running inputs. while True: diff --git a/06_gpu_and_ml/tgi_mixtral.py b/06_gpu_and_ml/tgi_mixtral.py index 7d61767e3..dbd612b5e 100644 --- a/06_gpu_and_ml/tgi_mixtral.py +++ b/06_gpu_and_ml/tgi_mixtral.py @@ -70,7 +70,7 @@ def download_model(): # Finally, we install the `text-generation` client to interface with TGI's Rust webserver over `localhost`. tgi_image = ( - Image.from_registry("ghcr.io/huggingface/text-generation-inference:1.3.1") + Image.from_registry("ghcr.io/huggingface/text-generation-inference:1.3.3") .dockerfile_commands("ENTRYPOINT []") .run_function(download_model, timeout=60 * 20) .pip_install("text-generation")