Compile modules needed for WHIP support in gstreamer image (#571)

.github/workflows/publish-gstreamer-amd64.yaml

@@ -0,0 +1,13 @@
+name: GStreamer build AMD64
+
+on:
+  workflow_dispatch:
+
+jobs:
+  gstreamer-build-amd64:
+    uses: ./.github/workflows/gstreamer-base.yaml
+    with:
+      buildjet-runs-on: buildjet-8vcpu-ubuntu-2204 
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/publish-gstreamer-arm64.yaml

@@ -0,0 +1,13 @@
+name: GStreamer build ARM64
+
+on:
+  workflow_dispatch:
+
+jobs:
+  gstreamer-build-arm64:
+    uses: ./.github/workflows/gstreamer-base.yaml
+    with:
+      buildjet-runs-on: namespace-profile-arm-16
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/publish-gstreamer-base.yaml

@@ -0,0 +1,81 @@
+on:
+  workflow_call:
+    inputs:
+      buildjet-runs-on:
+        required: true
+        type: string
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+env:
+  GST_VERSION: "1.22.8"
+  LIBNICE_VERSION: "0.1.21"
+
+jobs:
+  base-gstreamer-build:
+    runs-on: ${{ inputs.buildjet-runs-on }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push base
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/gstreamer
+          push: true
+          load: true
+          build-args: |
+            GSTREAMER_VERSION=${{ env.GST_VERSION }}
+            LIBNICE_VERSION=${{ env.LIBNICE_VERSION }}
+          file: ./build/gstreamer/Dockerfile-base
+          tags: livekit/gstreamer:base-${{ env.GST_VERSION }}
+
+      - name: Build and push dev
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/gstreamer
+          push: true
+          load: true
+          build-args: |
+            GSTREAMER_VERSION=${{ env.GST_VERSION }}
+            LIBNICE_VERSION=${{ env.LIBNICE_VERSION }}
+          file: ./build/gstreamer/Dockerfile-dev
+          tags: livekit/gstreamer:dev-${{ env.GST_VERSION }}
+
+      - name: Build and push prod
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/gstreamer
+          push: true
+          load: true
+          build-args: |
+            GSTREAMER_VERSION=${{ env.GST_VERSION }}
+            LIBNICE_VERSION=${{ env.LIBNICE_VERSION }}
+          file: ./build/gstreamer/Dockerfile-prod
+          tags: livekit/gstreamer:prod-${{ env.GST_VERSION }}
+
+      - name: Prune docker state
+        run: docker rmi livekit/gstreamer:base-${{ env.GST_VERSION }}
+
+      - name: Build and push prod RS
+        uses: docker/build-push-action@v5
+        with:
+          context: ./build/gstreamer
+          push: true
+          build-args: |
+            GSTREAMER_VERSION=${{ env.GST_VERSION }}
+            LIBNICE_VERSION=${{ env.LIBNICE_VERSION }}
+          file: ./build/gstreamer/Dockerfile-prod-rs
+          tags: livekit/gstreamer:prod-rs-${{ env.GST_VERSION }}

build/gstreamer/Dockerfile-base

@@ -1,15 +1,31 @@
-FROM ubuntu:22.04
+FROM ubuntu:23.10
 
 ARG GSTREAMER_VERSION
 
+ARG LIBNICE_VERSION
+
 COPY install-dependencies /
 
 RUN /install-dependencies
 
+ENV PATH=/root/.cargo/bin:$PATH
+
 RUN for lib in gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-libav; \
         do \
             wget https://gstreamer.freedesktop.org/src/$lib/$lib-$GSTREAMER_VERSION.tar.xz && \
             tar -xf $lib-$GSTREAMER_VERSION.tar.xz && \
             rm $lib-$GSTREAMER_VERSION.tar.xz && \
             mv $lib-$GSTREAMER_VERSION $lib; \
         done
+
+# rust plugins are apparently only realeased on gitlab
+
+RUN wget https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/archive/gstreamer-$GSTREAMER_VERSION/gst-plugins-rs-gstreamer-$GSTREAMER_VERSION.tar.gz && \
+tar xfz gst-plugins-rs-gstreamer-$GSTREAMER_VERSION.tar.gz && \
+rm gst-plugins-rs-gstreamer-$GSTREAMER_VERSION.tar.gz && \
+mv gst-plugins-rs-gstreamer-$GSTREAMER_VERSION gst-plugins-rs
+
+RUN wget https://libnice.freedesktop.org/releases/libnice-$LIBNICE_VERSION.tar.gz && \
+tar xfz libnice-$LIBNICE_VERSION.tar.gz && \
+rm libnice-$LIBNICE_VERSION.tar.gz && \
+mv libnice-$LIBNICE_VERSION libnice

build/gstreamer/Dockerfile-dev

@@ -6,10 +6,12 @@ ENV DEBUG=true
 ENV OPTIMIZATIONS=false
 
 COPY compile /
+COPY compile-rs /
 
 RUN /compile
+RUN /compile-rs
 
-FROM ubuntu:22.04
+FROM ubuntu:23.10
 
 COPY install-dependencies /
 

build/gstreamer/Dockerfile-prod

@@ -9,7 +9,7 @@ COPY compile /
 
 RUN /compile
 
-FROM ubuntu:22.04
+FROM ubuntu:23.10
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
@@ -22,10 +22,10 @@ RUN apt-get update && \
         libaa1 \
         libaom3 \
         libass9 \
-        libavcodec58 \
-        libavfilter7 \
-        libavformat58 \
-        libavutil56 \
+        libavcodec60 \
+        libavfilter9 \
+        libavformat60 \
+        libavutil58 \
         libbs2b0 \
         libbz2-1.0 \
         libcaca0 \
@@ -70,7 +70,7 @@ RUN apt-get update && \
         libogg0 \
         libopencore-amrnb0 \
         libopencore-amrwb0 \
-        libopenexr25 \
+        libopenexr-3-1-30 \
         libopenjp2-7 \
         libopus0 \
         liborc-0.4-0 \
@@ -86,7 +86,7 @@ RUN apt-get update && \
         libsoup2.4-1 \
         libspandsp2 \
         libspeex1 \
-        libsrt1.4-openssl \
+        libsrt1.5-openssl \
         libsrtp2-1 \
         libssl3 \
         libtag1v5 \
@@ -106,7 +106,7 @@ RUN apt-get update && \
         libwebrtc-audio-processing1 \
         libwildmidi2 \
         libwoff1 \
-        libx264-163 \
+        libx264-164 \
         libx265-199 \
         libxkbcommon0 \
         libxslt1.1 \

build/gstreamer/Dockerfile-prod-rs

@@ -0,0 +1,19 @@
+ARG GSTREAMER_VERSION
+
+FROM livekit/gstreamer:${GSTREAMER_VERSION}-base
+
+FROM livekit/gstreamer:${GSTREAMER_VERSION}-dev
+
+COPY --from=0 /gst-plugins-rs /gst-plugins-rs
+
+ENV DEBUG=false
+ENV OPTIMIZATIONS=true
+ENV PATH=/root/.cargo/bin:$PATH
+
+COPY compile-rs /
+
+RUN /compile-rs 
+
+FROM livekit/gstreamer:${GSTREAMER_VERSION}-prod
+
+COPY --from=1 /compiled-binaries /

build/gstreamer/compile

@@ -1,10 +1,14 @@
 #!/bin/bash
 set -euxo pipefail
 
-for repo in gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-libav; do
+for repo in gstreamer libnice gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-libav; do
   pushd $repo
 
-  opts="-D prefix=/usr -D tests=disabled -D doc=disabled"
+  opts="-D prefix=/usr"
+
+  if [[ $repo != "libnice" ]]; then
+    opts="$opts -D tests=disabled -D doc=disabled"
+  fi
 
   if [[ $repo == "gstreamer" ]]; then
     opts="$opts -D examples=disabled -D introspection=disabled"

build/gstreamer/compile-rs

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euxo pipefail
+
+for repo in gst-plugins-rs; do
+  pushd $repo
+
+  # strip binaries in debug mode
+  mv Cargo.toml Cargo.toml.old
+  sed s,'\[profile.release\]','[profile.release]\nstrip="debuginfo"', Cargo.toml.old > Cargo.toml 
+
+  opts="-D prefix=/usr -D tests=disabled -D doc=disabled"
+
+  if [[ $DEBUG == 'true' ]]; then
+    if [[ $OPTIMIZATIONS == 'true' ]]; then
+      opts="$opts -D buildtype=debugoptimized"
+    else
+      opts="$opts -D buildtype=debug"
+    fi
+  else
+    opts="$opts -D buildtype=release -D b_lto=true"
+  fi
+
+  meson build $opts
+
+  # This is needed for other plugins to be built properly
+  ninja -C build install
+  # This is where we'll grab build artifacts from
+  DESTDIR=/compiled-binaries ninja -C build install
+  popd
+done
+
+gst-inspect-1.0

build/gstreamer/install-dependencies

@@ -11,6 +11,7 @@ apt-get install -y --no-install-recommends \
   bubblewrap \
   ca-certificates \
   cmake \
+  curl \
   flex \
   flite1-dev \
   gcc \
@@ -110,11 +111,18 @@ apt-get install -y --no-install-recommends \
   libxslt1-dev \
   libzbar-dev \
   libzvbi-dev \
+  meson \
+  ninja-build \
   python3 \
-  python3-pip \
   ruby \
   wget \
   xdg-dbus-proxy
-pip3 install meson ninja
 apt-get clean
 rm -rf /var/lib/apt/lists/*
+
+# install rust
+curl -o install-rustup.sh --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs 
+sh install-rustup.sh -y
+source "$HOME/.cargo/env"
+cargo install cargo-c
+rm -rf install-rustup.sh

magefile.go

@@ -29,6 +29,7 @@ import (
 
 const (
 	gstVersion      = "1.22.8"
+	libniceVersion  = "0.1.21"
 	chromiumVersion = "117.0.5874.0"
 	dockerBuild     = "docker build"
 	dockerBuildX    = "docker buildx build --push --platform linux/amd64,linux/arm64"
@@ -157,19 +158,16 @@ func BuildGStreamer() error {
 	return buildGstreamer(dockerBuild)
 }
 
-func PublishGStreamer() error {
-	return buildGstreamer(dockerBuildX)
-}
-
 func buildGstreamer(cmd string) error {
-	commands := []string{"docker pull ubuntu:22.04"}
-	for _, build := range []string{"base", "dev", "prod"} {
+	commands := []string{"docker pull ubuntu:23.10"}
+	for _, build := range []string{"base", "dev", "prod", "prod-rs"} {
 		commands = append(commands, fmt.Sprintf("%s"+
 			" --build-arg GSTREAMER_VERSION=%s"+
+			" --build-arg LIBNICE_VERSION=%s"+
 			" -t livekit/gstreamer:%s-%s"+
 			" -f build/gstreamer/Dockerfile-%s"+
 			" ./build/gstreamer",
-			cmd, gstVersion, gstVersion, build, build,
+			cmd, gstVersion, libniceVersion, gstVersion, build, build,
 		))
 	}