Skip to content

Commit

Permalink
Fix TEI and TGI-Mixtral
Browse files Browse the repository at this point in the history
- TEI: Issue with HuggingFace secrets again
- TGI-Mixtral: huggingface/text-generation-inference#1342
  • Loading branch information
ekzhang committed Jan 3, 2024
1 parent 37d3552 commit 682fb98
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 2 deletions.
8 changes: 7 additions & 1 deletion 06_gpu_and_ml/embeddings/text_embeddings_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,13 @@


def spawn_server() -> subprocess.Popen:
process = subprocess.Popen(["text-embeddings-router"] + LAUNCH_FLAGS)
process = subprocess.Popen(
["text-embeddings-router"] + LAUNCH_FLAGS,
env={
**os.environ,
"HUGGING_FACE_HUB_TOKEN": os.environ["HUGGINGFACE_TOKEN"],
},
)

# Poll until webserver at 127.0.0.1:8000 accepts connections before running inputs.
while True:
Expand Down
2 changes: 1 addition & 1 deletion 06_gpu_and_ml/tgi_mixtral.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def download_model():
# Finally, we install the `text-generation` client to interface with TGI's Rust webserver over `localhost`.

tgi_image = (
Image.from_registry("ghcr.io/huggingface/text-generation-inference:1.3.1")
Image.from_registry("ghcr.io/huggingface/text-generation-inference:1.3.3")
.dockerfile_commands("ENTRYPOINT []")
.run_function(download_model, timeout=60 * 20)
.pip_install("text-generation")
Expand Down

0 comments on commit 682fb98

Please sign in to comment.