fix: update all vlm forward args, pass shared libraries to final laye…

…r in docker and doc bump

Dockerfile_amd

@@ -317,6 +317,10 @@ COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/loca
 COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib/"
 
+# Copy the ffmpeg libraries
+COPY --from=builder /usr/lib/x86_64-linux-gnu/* /usr/lib/x86_64-linux-gnu-copy/
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu-copy"
+
 # AWS Sagemaker compatible image
 FROM base AS sagemaker
 

Dockerfile_intel

@@ -232,6 +232,10 @@ COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/loca
 # Install launcher
 COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
 
+# Copy the ffmpeg libraries
+COPY --from=builder /usr/lib/x86_64-linux-gnu/* /usr/lib/x86_64-linux-gnu-copy/
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu-copy"
+
 FROM ${PLATFORM} AS final
 ENV ATTENTION=paged
 ENV PREFIX_CACHING=0

docs/openapi.json

@@ -10,7 +10,7 @@
       "name": "Apache 2.0",
       "url": "https://www.apache.org/licenses/LICENSE-2.0"
     },
-    "version": "3.0.1-dev0"
+    "version": "3.0.2-dev0"
   },
   "paths": {
     "/": {

server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py

@@ -81,6 +81,8 @@ def forward(
         image_sizes: Optional[torch.Tensor] = None,
         adapter_data: Optional[torch.Tensor] = None,
         image_grid_thw: Optional[torch.LongTensor] = None,
+        video_pixel_values: Optional[torch.FloatTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         inputs_embeds = self.text_model.embed_tokens(input_ids)
         # TODO This is odd but apparently pali gemma position ids start at 1.

server/text_generation_server/models/custom_modeling/idefics2.py

@@ -751,6 +751,8 @@ def forward(
         image_sizes: Optional[torch.Tensor] = None,
         adapter_data: Optional[torch.Tensor] = None,
         image_grid_thw: Optional[torch.LongTensor] = None,
+        video_pixel_values: Optional[torch.FloatTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
     ):
         inputs_embeds = self.text_model.embed_tokens(input_ids)
         if pixel_values is not None:

server/text_generation_server/models/custom_modeling/llava_next.py

@@ -181,6 +181,8 @@ def forward(
         image_sizes: Optional[torch.LongTensor] = None,
         adapter_data: Optional[torch.Tensor] = None,
         image_grid_thw: Optional[torch.LongTensor] = None,
+        video_pixel_values: Optional[torch.FloatTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
     ):
         inputs_embeds = self.text_model.embed_tokens(input_ids)
         if pixel_values is not None and len(pixel_values) > 0:

server/text_generation_server/models/mllama_causal_lm.py

@@ -148,7 +148,8 @@ def batch_tokenized_inputs(
         if image_inputs is not None:
             assert len(image_indices) == image_inputs["pixel_values"].shape[0]
 
-        return batch_tokenized_inputs, image_inputs
+        video_inputs = None
+        return batch_tokenized_inputs, image_inputs, video_inputs
 
     @classmethod
     def from_pb_processor(

server/text_generation_server/models/pali_gemma.py

@@ -68,4 +68,6 @@ def batch_tokenized_inputs(
             image_inputs = new_image_inputs
         else:
             image_inputs = None
-        return batch_tokenized_inputs, image_inputs
+
+        video_inputs = None
+        return batch_tokenized_inputs, image_inputs, video_inputs