Make liveportrait image work!

runner/dl_checkpoints.sh

@@ -79,15 +79,21 @@ function download_all_models() {
 
     # Download live-video-to-video models.
     huggingface-cli download KBlueLeaf/kohaku-v2.1 --include "*.safetensors" "*.json" "*.txt" --exclude ".onnx" ".onnx_data" --cache-dir models
-    huggingface-cli download warmshao/FasterLivePortrait --local-dir models/FasterLivePortrait
+    huggingface-cli download stabilityai/sd-turbo --include "*.safetensors" "*.json" "*.txt" --exclude ".onnx" ".onnx_data" --cache-dir models
+    huggingface-cli download warmshao/FasterLivePortrait --local-dir models/FasterLivePortrait--checkpoints
 }
 
 function build_tensorrt_models() {
     printf "\nBuilding TensorRT models...\n"
 
+    mkdir -p models/StreamDiffusion--engines
+    # TODO: Build StreamDiffusion TensorRT engines
+
     docker run --rm -v ./models:/models --gpus all \
         livepeer/ai-runner:live-base-liveportrait \
-        bash -c "cd /app/app/live/FasterLivePortrait && sh scripts/all_onnx2trt.sh && sh scripts/all_onnx2trt_animal.sh"
+        bash -c "cd /app/app/live/FasterLivePortrait && \
+                    sh scripts/all_onnx2trt.sh && \
+                    sh scripts/all_onnx2trt_animal.sh"
 }
 
 # Download models with a restrictive license.

runner/docker/Dockerfile.live-base-liveportrait

@@ -41,12 +41,27 @@ RUN pip install --no-cache-dir --upgrade pip==${PIP_VERSION} setuptools==69.5.1
 RUN pip install --no-cache-dir tensorrt==8.6.1 numpy==1.26.4
 
 # Clone the FasterLivePortrait repository
-RUN mkdir -p /app/app/live && git clone https://github.com/warmshao/FasterLivePortrait.git /app/app/live/FasterLivePortrait
+RUN mkdir -p /app/app/live && \
+    git clone https://github.com/warmshao/FasterLivePortrait.git /app/app/live/FasterLivePortrait && \
+    cd /app/app/live/FasterLivePortrait && \
+    git checkout 6aa8104b03499ebe2a881c8fbaf55ff628235f4f
+
 WORKDIR /app/app/live/FasterLivePortrait
+
+COPY images/flame-serious.jpg \
+    images/flame-smile.jpg \
+    images/heart.jpg \
+    images/pirate.jpg \
+    ./assets/examples/source/
+
+# FasterLivePortrait doesn't pin versions so we use a custom requirements.txt
+COPY requirements-liveportrait.txt requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Create a symlink on ./checkpoints to the directory that is mounted with the models
-# TODO: Figure out if we can have only 1 of these symlinks
-RUN ln -s /models/FasterLivePortrait ./checkpoints
-RUN ln -s /models/FasterLivePortrait /app/app/checkpoints
-RUN ln -s /models/FasterLivePortrait /app/app/live/checkpoints
+# TODO: Setup dependencies for animal models
+
+WORKDIR /app
+
+# Create symlinks for checkpoints as FasterLivePortrait relies heavily on relative paths
+RUN ln -s /models/FasterLivePortrait--checkpoints /app/app/live/FasterLivePortrait/checkpoints
+RUN ln -s /models/FasterLivePortrait--checkpoints ./checkpoints

runner/docker/Dockerfile.live-base-streamdiffusion

@@ -32,3 +32,8 @@ RUN python -m streamdiffusion.tools.install-tensorrt
 # Set environment variables for NVIDIA drivers
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility,video
+
+WORKDIR /app
+
+# Create symlink for where StreamDiffusion builds TensorRT engines at runtime
+RUN ln -s /models/StreamDiffusion--engines ./engines

runner/requirements-liveportrait.txt

@@ -0,0 +1,12 @@
+ffmpeg-python==0.2.0
+gradio==5.6.0
+insightface==0.7.3
+mediapipe==0.10.15
+numpy==1.26.4
+omegaconf==2.3.0
+onnx==1.17.0
+opencv-python==4.10.0.84
+pycuda==2024.1.2
+scikit-image==0.24.0
+torchgeometry==0.1.2
+torchvision==0.19.1