From f4d3aaf2d4d7e708e8008822b7c367975d2ce8c3 Mon Sep 17 00:00:00 2001
From: Funtowicz Morgan <mfuntowicz@users.noreply.github.com>
Date: Mon, 8 Apr 2024 15:46:12 +0200
Subject: [PATCH] Fix rule and instructions for TGI (#18)

* Fix rule and instructions for TGI

* Add remaining of sentence

* Address comments

* Again
---
 Makefile                      |  4 +--
 docs/source/howto/serving.mdx | 51 ++++++++++++++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 82a4acea..c06e50d6 100644
--- a/Makefile
+++ b/Makefile
@@ -46,8 +46,8 @@ tpu-tgi:
 	docker build --rm -f text-generation-inference/Dockerfile \
 	             --build-arg VERSION=$(VERSION) \
 	             --build-arg TGI_VERSION=$(TGI_VERSION) \
-				 -t tpu-tgi:$(VERSION) .
-	docker tag tpu-tgi:$(VERSION) tpu-tgi:latest
+				 -t huggingface/optimum-tpu:$(VERSION)-tgi .
+	docker tag huggingface/optimum-tpu:$(VERSION)-tgi tpu-tgi:latest
 
 # Run code quality checks
 style_check:
diff --git a/docs/source/howto/serving.mdx b/docs/source/howto/serving.mdx
index 8a406f06..77b894b9 100644
--- a/docs/source/howto/serving.mdx
+++ b/docs/source/howto/serving.mdx
@@ -1,3 +1,52 @@
 # Deploying a Text-Generation Inference server on a Google Cloud TPU instance
 
-Stay tuned!
+## Context
+
+Text-Generation-Inference (TGI) is a highly optimized serving engine enabling serving Large Language Models (LLMs) in a way
+that better leverages the underlying hardware, Cloud TPU in this case.
+
+## Deploy TGI on Cloud TPU instance
+
+We assume the reader already has a Cloud TPU instance up and running.
+If this is not the case, please see our guide to deploy one [here](./deploy.mdx)
+
+### Docker Container Build
+
+Optimum-TPU provides a `make tpu-tgi` command at the root level to help you create local docker image.
+
+### Docker Container Run
+
+```
+OPTIMUM_TPU_VERSION=0.1.0b1
+docker run -p 8080:80 \
+       --net=host --privileged \
+       -v $(pwd)/data:/data \
+       -e HF_TOKEN=${HF_TOKEN} \
+       -e HF_BATCH_SIZE=1 \
+       -e HF_SEQUENCE_LENGTH=1024 \
+       huggingface/optimum-tpu:${OPTIMUM_TPU_VERSION}-tgi \
+       --model-id google/gemma-2b \
+       --max-concurrent-requests 4 \
+       --max-input-length 512 \
+       --max-total-tokens 1024 \
+       --max-batch-prefill-tokens 512 \
+       --max-batch-total-tokens 1024
+```
+
+### Executing requests against the service
+
+You can query the model using either the `/generate` or `/generate_stream` routes:
+
+```
+curl 127.0.0.1:8080/generate \
+    -X POST \
+    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -H 'Content-Type: application/json'
+```
+
+```
+curl 127.0.0.1:8080/generate_stream \
+    -X POST \
+    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -H 'Content-Type: application/json'
+```