Static build with extra options for backend linux

.gitignore

@@ -3,3 +3,4 @@ workspace
 .idea
 linux
 .artifacts
+build/

build-ffmpeg

@@ -15,7 +15,7 @@ LDEXEFLAGS=""
 EXTRALIBS="-ldl -lpthread -lm -lz"
 MACOS_M1=false
 CONFIGURE_OPTIONS=()
-NONFREE_AND_GPL=false
+NONFREE_AND_GPL=true
 LATEST=false
 
 # Check for Apple Silicon
@@ -26,20 +26,34 @@ if [[ ("$(uname -m)" == "arm64") && ("$OSTYPE" == "darwin"*) ]]; then
   MACOS_M1=true
 fi
 
+is_mac () {
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    return 0
+  fi
+
+  return 1
+}
+
+# read by gcc and cmake
+# https://cmake.org/cmake/help/latest/variable/CMAKE_OSX_DEPLOYMENT_TARGET.html
+export MACOSX_DEPLOYMENT_TARGET=10.11
+
 # Speed up the process
 # Env Var NUMJOBS overrides automatic detection
 if [[ -n "$NUMJOBS" ]]; then
   MJOBS="$NUMJOBS"
 elif [[ -f /proc/cpuinfo ]]; then
   MJOBS=$(grep -c processor /proc/cpuinfo)
-elif [[ "$OSTYPE" == "darwin"* ]]; then
+elif is_mac; then
   MJOBS=$(sysctl -n machdep.cpu.thread_count)
   CONFIGURE_OPTIONS=("--enable-videotoolbox")
   MACOS_LIBTOOL="$(which libtool)" # gnu libtool is installed in this script and need to avoid name conflict
 else
   MJOBS=4
 fi
 
+CONFIGURE_OPTIONS+=("--extra-version=beamcube")
+
 make_dir() {
   remove_dir "$1"
   if ! mkdir "$1"; then
@@ -168,6 +182,13 @@ library_exists() {
   return 0
 }
 
+set_mac_install_name () {
+  local name="${2:-$1}"
+  if is_mac; then
+    execute install_name_tool -id "${WORKSPACE}/lib/$name" $1
+  fi
+}
+
 build_done() {
   echo "$2" > "$PACKAGES/$1.done"
 }
@@ -238,7 +259,7 @@ while (($# > 0)); do
       cleanup
     fi
     if [[ "$1" == "--full-static" ]]; then
-      if [[ "$OSTYPE" == "darwin"* ]]; then
+      if is_mac; then
         echo "Error: A full static binary can only be build on Linux."
         exit 1
       fi
@@ -530,7 +551,7 @@ fi
 if build "libvpx" "1.10.0"; then
   download "https://github.com/webmproject/libvpx/archive/refs/tags/v1.10.0.tar.gz" "libvpx-1.10.0.tar.gz"
 
-  if [[ "$OSTYPE" == "darwin"* ]]; then
+  if is_mac; then
     echo "Applying Darwin patch"
     sed "s/,--version-script//g" build/make/Makefile >build/make/Makefile.patched
     sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched >build/make/Makefile
@@ -583,6 +604,16 @@ if $NONFREE_AND_GPL; then
   CONFIGURE_OPTIONS+=("--enable-libvidstab")
 fi
 
+if build "twolame" "0.4.0"; then
+  download "https://downloads.sourceforge.net/twolame/twolame-0.4.0.tar.gz"
+  execute ./configure --prefix="${WORKSPACE}" --enable-static --disable-shared
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "twolame" "0.4.0"
+fi
+CONFIGURE_OPTIONS+=("--enable-libtwolame")
+
 if build "av1" "ae2be80"; then
   # libaom ae2be80 == v3.1.2
   download "https://aomedia.googlesource.com/aom/+archive/ae2be8030200925895fa6e98bd274ffdb595cbf6.tar.gz" "av1.tar.gz" "av1"
@@ -600,6 +631,18 @@ if build "av1" "ae2be80"; then
 fi
 CONFIGURE_OPTIONS+=("--enable-libaom")
 
+if build "dav1d" "0.7.1"; then
+  download "https://code.videolan.org/videolan/dav1d/-/archive/0.7.1/dav1d-0.7.1.tar.gz"
+  make_dir "$PACKAGES"/dav1d_build
+  cd "$PACKAGES"/dav1d_build || exit
+  execute meson --prefix="${WORKSPACE}" --libdir="${WORKSPACE}/lib" --default-library=static ../dav1d-0.7.1
+  execute ninja
+  execute ninja install
+
+  build_done "dav1d"
+fi
+CONFIGURE_OPTIONS+=("--enable-libdav1d")
+
 if build "zimg" "3.0.3"; then
   download "https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.3.tar.gz" "zimg-3.0.3.tar.gz" "zimg"
   cd zimg-release-3.0.3 || exit
@@ -751,6 +794,39 @@ if $NONFREE_AND_GPL; then
   CONFIGURE_OPTIONS+=("--enable-libfdk-aac")
 fi
 
+if build "shine" "3.1.1"; then
+  download "https://github.com/toots/shine/archive/3.1.1.tar.gz" "shine-3.1.1.tar.gz"
+  execute autoreconf -i
+  execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "shine" "3.1.1"
+fi
+CONFIGURE_OPTIONS+=("--enable-libshine")
+
+if build "soxr" "0.1.3"; then
+  download "http://ftp.debian.org/debian/pool/main/libs/libsoxr/libsoxr_0.1.3.orig.tar.xz" "soxr-0.1.3.tar.xz"
+  make_dir build
+  cd build || exit
+  execute cmake .. -DWITH_OPENMP:BOOL=OFF -Wno-dev -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=OFF
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "soxr" "0.1.3"
+fi
+CONFIGURE_OPTIONS+=("--enable-libsoxr")
+
+if build "speex" "1.2.0"; then
+  download "http://downloads.us.xiph.org/releases/speex/speex-1.2.0.tar.gz"
+  execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "speex" "1.2.0"
+fi
+CONFIGURE_OPTIONS+=("--enable-libspeex")
+
 ##
 ## image library
 ##
@@ -784,11 +860,23 @@ if [[ "$OSTYPE" != "darwin"* ]]; then
     execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../
     execute make -j $MJOBS
     execute make install
-
     build_done "libwebp" "1.2.1"
   fi
   CONFIGURE_OPTIONS+=("--enable-libwebp")
 fi
+
+if build "openjpeg" "2.3.1"; then
+  download "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz" "openjpeg-v2.3.1.tar.gz"
+  make_dir build
+  cd build || exit
+  execute cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DBUILD_SHARED_LIBS:bool=OFF
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "openjpeg" "2.3.1"
+fi
+CONFIGURE_OPTIONS+=("--enable-libopenjpeg")
+
 ##
 ## other library
 ##
@@ -802,6 +890,30 @@ if build "libsdl" "2.0.14"; then
   build_done "libsdl" "2.0.14"
 fi
 
+if build "snappy" "1.1.8"; then
+  download "https://github.com/google/snappy/archive/1.1.8.tar.gz"
+  make_dir build
+  cd build || exit
+  execute cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DBUILD_SHARED_LIBS:bool=OFF -DSNAPPY_BUILD_TESTS:bool=OFF -DSNAPPY_BUILD_BENCHMARKS:bool=OFF
+  execute make -j $MJOBS
+
+  set_mac_install_name "libsnappy.1.1.8.dylib" "libsnappy.1.dylib"
+
+  execute make install
+
+  build_done "snappy" "1.1.8"
+fi
+CONFIGURE_OPTIONS+=("--enable-libsnappy")
+
+if build "xz" "5.2.5"; then
+  download "https://tukaani.org/xz/xz-5.2.5.tar.gz"
+  execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static
+  execute make -j $MJOBS
+  execute make install
+
+  build_done "xz" "5.2.5"
+fi
+
 if $NONFREE_AND_GPL; then
   if build "srt" "1.4.3"; then
     download "https://github.com/Haivision/srt/archive/v1.4.3.tar.gz" "srt-1.4.3.tar.gz"
@@ -883,7 +995,6 @@ download "https://github.com/FFmpeg/FFmpeg/archive/refs/heads/release/$FFMPEG_VE
   --disable-shared \
   --enable-pthreads \
   --enable-static \
-  --enable-small \
   --enable-version3 \
   --extra-cflags="${CFLAGS}" \
   --extra-ldexeflags="${LDEXEFLAGS}" \
@@ -894,11 +1005,46 @@ download "https://github.com/FFmpeg/FFmpeg/archive/refs/heads/release/$FFMPEG_VE
   --prefix="${WORKSPACE}" \
   --extra-version="${EXTRA_VERSION}"
 
+if is_mac; then
+  execute dsymutil libavcodec/libavcodec.dylib -o libavcodec.dSYM
+  execute dsymutil libavdevice/libavdevice.dylib -o libavdevice.dSYM
+  execute dsymutil libavfilter/libavfilter.dylib -o libavfilter.dSYM
+  execute dsymutil libavformat/libavformat.dylib -o libavformat.dSYM
+  # we're not building this one (libswresample replaces this)
+  # execute dsymutil libavresample/libavresample.dylib -o libavresample.dSYM
+  execute dsymutil libavutil/libavutil.dylib -o libavutil.dSYM
+  execute dsymutil libpostproc/libpostproc.dylib -o libpostproc.dSYM
+  execute dsymutil libswresample/libswresample.dylib -o libswresample.dSYM
+  execute dsymutil libswscale/libswscale.dylib -o libswscale.dSYM
+fi
+
+if [[ -n "$SENTRY_AUTH_TOKEN" ]] && [[ -n "$SENTRY_ORG" ]] && [[ -n "$SENTRY_PROJECT" ]]; then
+  # TODO: install sentry-cli when running on CI
+  # reads SENTRY_AUTH_TOKEN, SENTRY_ORG and SENTRY_PROJECT from env
+  # https://docs.sentry.io/product/cli/configuration/#configuration-values
+  sentry-cli upload-dif .
+fi
+
+# strip symbols from binaries (i.e., make them smaller)
+if is_mac; then
+    execute strip -x libavcodec/libavcodec.dylib
+    execute strip -x libavdevice/libavdevice.dylib
+    execute strip -x libavfilter/libavfilter.dylib
+    execute strip -x libavformat/libavformat.dylib
+    execute strip -x libavutil/libavutil.dylib
+    execute strip -x libpostproc/libpostproc.dylib
+    execute strip -x libswresample/libswresample.dylib
+    execute strip -x libswscale/libswscale.dylib
+    execute strip -x ffmpeg
+    execute strip -x ffprobe
+    execute strip -x ffplay
+fi
+
 execute make -j $MJOBS
 execute make install
 
 INSTALL_FOLDER="/usr/bin"
-if [[ "$OSTYPE" == "darwin"* ]]; then
+if is_mac; then
   INSTALL_FOLDER="/usr/local/bin"
 fi
 
@@ -941,4 +1087,11 @@ elif [[ ! "$SKIPINSTALL" == "yes" ]]; then
   esac
 fi
 
+cd $CWD
+if is_mac; then
+  node copy_mac_libs.js
+  cd "$WORKSPACE/mac"
+  execute zip --symlinks -r "ffmpeg-ffprobe-shared-darwin-x86_64.${VERSION}.zip" *
+fi
+
 exit 0

full-static.dockerfile

@@ -5,14 +5,14 @@ ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility,video
 
 RUN apt-get update \
-    && apt-get -y --no-install-recommends install build-essential curl ca-certificates python3 python-is-python3 ninja-build meson \
+    && apt-get -y --no-install-recommends install build-essential curl ca-certificates python3 python3-pip python-is-python3 ninja-build meson \
     && apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* \
     && update-ca-certificates
 
 WORKDIR /app
 COPY ./build-ffmpeg /app/build-ffmpeg
 
-RUN AUTOINSTALL=yes /app/build-ffmpeg --build --full-static
+RUN AUTOINSTALL=yes /app/build-ffmpeg --build --full-static --enable-gpl-and-non-free
 
 # Check shared library
 RUN ! ldd /app/workspace/bin/ffmpeg

publish-to-s3.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ $# -ne 3 ]; then
+	echo "Required arguments: BUCKET PLATFORM ARCH"
+  exit 1
+fi
+
+BUCKET=$1
+PLATFORM=$2
+ARCH=$3
+
+# rename the ffmpeg/ffprobe files
+cp build/ffmpeg "build/ffmpeg-$PLATFORM-$ARCH"
+cp build/ffprobe "build/ffprobe-$PLATFORM-$ARCH"
+
+# upload to s3
+s3cmd put "build/ffmpeg-$PLATFORM-$ARCH" "build/ffprobe-$PLATFORM-$ARCH" "s3://$BUCKET/"