Merge pull request #1383 from palonso/support-apple-silicon-cross-build

Add changes to support building arm64 macos wheels

.github/workflows/build-wheels-cibuildwheel.yml

@@ -8,21 +8,24 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-11]
-        config: [pyproject-tensorflow.toml, pyproject.toml]
+        os: [ubuntu-20.04, macos-12, macos-14]
+        config: [pyproject, pyproject-tensorflow]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - name: Fetch release tags from GitHub
         # Workaround for https://github.com/actions/checkout/issues/290
         run: git fetch --tags --force
+
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.11.4
+        uses: pypa/cibuildwheel@v2.16.5
         with:
-          config-file: ${{ matrix.config }}
+          config-file: ${{ matrix.config }}.toml
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
+          name: artifact-${{ matrix.os }}-${{ matrix.config }}

pyproject-tensorflow.toml

@@ -4,11 +4,12 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64"
 
 # Only support x86_64 for essentia-tensorflow
 build = "cp**-manylinux_x86_64"
-skip = ["pp*", "*-musllinux*", "*i686"]
+# TODO: skipping Python 3.12 for now until we create manylinux images supporting this version.
+skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"]
 
 environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 }
 
-before-all = [
+before-build = [
   "PYBIN=/opt/python/cp36-cp36m/bin/",
   "\"${PYBIN}/python\" waf configure --with-gaia --with-tensorflow --build-static --static-dependencies --pkg-config-path=\"${PKG_CONFIG_PATH}\"",
   "\"${PYBIN}/python\" waf",
@@ -24,11 +25,13 @@ skip = ["pp*"]
 
 environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 }
 
-before-all = [
+before-build = [
   "brew install pkg-config gcc readline sqlite gdbm freetype libpng",
-  "brew install eigen libyaml fftw [email protected] libsamplerate libtag chromaprint",
-  "brew install tensorflow",
+  "brew install eigen libyaml fftw [email protected] libsamplerate libtag",
   "brew link --force [email protected]",
+  "brew install chromaprint",
+  "brew link --overwrite [email protected]",
+  "brew install tensorflow",
   #"brew tap MTG/essentia",
   #"brew install gaia --HEAD",
   "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\"",
@@ -39,6 +42,40 @@ before-all = [
 test-command = "python -c 'import essentia; import essentia.standard; import essentia.streaming; from essentia.standard import MonoLoader, MetadataReader, YamlInput, Chromaprinter, TensorflowPredict'"
 
 
+[[tool.cibuildwheel.overrides]]
+select = "*macosx_arm64*"
+
+skip = ["pp*"]
+
+environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 }
+
+before-build = [
+  "brew install pkg-config gcc readline sqlite gdbm libpng",
+  "brew install eigen libyaml fftw [email protected] libsamplerate libtag",
+  "brew link --force [email protected]",
+  "brew install chromaprint",
+  "brew link --overwrite [email protected]",
+  "brew install tensorflow",
+  "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse",
+  "python waf",
+  "sudo python waf install",
+]
+
+# On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and
+# libavutil.54.31.100, depend on libSDL1.2-compat, which is a compatibility
+# layer for SDL2. libSDL1.2-compat expects SDL2 to be installed in the default
+# brew location (i.e., /opt/homebrew/opt/sdl2/lib), so the user would need to
+# install it via brew manually. Alternativelly, we can manualy copy the SDL2
+# libs into the wheel. This is a temporary solution, and in the long term we
+# should move to FFmpeg > 2.X.
+repair-wheel-command = [
+  "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}",
+  "mkdir -p {dest_dir}/essentia/.dylibs",
+  "cp /opt/homebrew/opt/sdl2/lib/libSDL2*.dylib {dest_dir}/essentia/.dylibs",
+  "wheel_rel=$(echo {wheel} | grep -o '[^/]*$')",
+  "cd {dest_dir} && zip -u {dest_dir}/$wheel_rel essentia/.dylibs/*"
+]
+
 [build-system]
 
 requires = ["wheel", "setuptools", "oldest-supported-numpy"]

pyproject.toml

@@ -4,11 +4,12 @@
 manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64"
 manylinux-i686-image = "mtgupf/essentia-builds:manylinux2014_i686"
 
-skip = ["pp*", "*-musllinux*", "*i686"]
+# TODO: skipping Python 3.12 for now until we create manylinux images supporting this version.
+skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"]
 
 environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 }
 
-before-all = [
+before-build = [
   "PYBIN=/opt/python/cp36-cp36m/bin/",
   "\"${PYBIN}/python\" waf configure --with-gaia --build-static --static-dependencies --pkg-config-path=\"${PKG_CONFIG_PATH}\"",
   "\"${PYBIN}/python\" waf",
@@ -24,10 +25,12 @@ skip = ["pp*"]
 
 environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 }
 
-before-all = [
+before-build = [
   "brew install pkg-config gcc readline sqlite gdbm freetype libpng",
-  "brew install eigen libyaml fftw [email protected] libsamplerate libtag chromaprint",
+  "brew install eigen libyaml fftw [email protected] libsamplerate libtag",
   "brew link --force [email protected]",
+  "brew install chromaprint",
+  "brew link --overwrite [email protected]",
   #"brew tap MTG/essentia",
   #"brew install gaia --HEAD",
   "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\"",
@@ -37,6 +40,38 @@ before-all = [
 
 test-command = "python -c 'import essentia; import essentia.standard; import essentia.streaming; from essentia.standard import MonoLoader, MetadataReader, YamlInput, Chromaprinter'"
 
+[[tool.cibuildwheel.overrides]]
+select = "*macosx_arm64*"
+
+skip = ["pp*"]
+
+environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 }
+
+before-build = [
+  "brew install pkg-config gcc readline sqlite gdbm libpng",
+  "brew install eigen libyaml fftw [email protected] libsamplerate libtag",
+  "brew link --force [email protected]",
+  "brew install chromaprint",
+  "brew link --overwrite [email protected]",
+  "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse",
+  "python waf",
+  "sudo python waf install",
+]
+
+# On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and
+# libavutil.54.31.100, depend on libSDL1.2-compat, which is a compatibility
+# layer for SDL2. libSDL1.2-compat expects SDL2 to be installed in the default
+# brew location (i.e., /opt/homebrew/opt/sdl2/lib), so the user would need to
+# install it via brew manually. Alternativelly, we can manualy copy the SDL2
+# libs into the wheel. This is a temporary solution, and in the long term we
+# should move to FFmpeg > 2.X.
+repair-wheel-command = [
+  "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}",
+  "mkdir -p {dest_dir}/essentia/.dylibs",
+  "cp /opt/homebrew/opt/sdl2/lib/libSDL2*.dylib {dest_dir}/essentia/.dylibs",
+  "wheel_rel=$(echo {wheel} | grep -o '[^/]*$')",
+  "cd {dest_dir} && zip -u {dest_dir}/$wheel_rel essentia/.dylibs/*"
+]
 
 [build-system]
 

setup.py

@@ -1,6 +1,7 @@
 import shutil
 import os
 import glob
+import subprocess
 import sys
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
@@ -36,20 +37,25 @@ def run(self):
         var_skip_3rdparty = 'ESSENTIA_WHEEL_SKIP_3RDPARTY'
         var_only_python = 'ESSENTIA_WHEEL_ONLY_PYTHON'
 
+        var_macos_arm64 = os.getenv('ESSENTIA_MACOSX_ARM64')
+        macos_arm64_flags = []
+        if var_macos_arm64 == '1':
+            macos_arm64_flags = ['--arch=arm64', '--no-msse']
+
         if var_skip_3rdparty in os.environ and os.environ[var_skip_3rdparty]=='1':
             print('Skipping building static 3rdparty dependencies (%s=1)' %  var_skip_3rdparty)
         else:
-            os.system('./packaging/build_3rdparty_static_debian.sh')
+            subprocess.run('./packaging/build_3rdparty_static_debian.sh', check=True)
 
         if var_only_python in os.environ and os.environ[var_only_python]=='1':
             print('Skipping building the core libessentia library (%s=1)' %  var_only_python)
-            os.system('%s waf configure --only-python --static-dependencies '
-                      '--prefix=tmp' % PYTHON)
+            subprocess.run([PYTHON,  'waf', 'configure', '--only-python', '--static-dependencies',
+                      '--prefix=tmp'] + macos_arm64_flags, check=True)
         else:
-            os.system('%s waf configure --build-static --static-dependencies '
-                      '--with-python --prefix=tmp' % PYTHON)
-        os.system('%s waf' % PYTHON)
-        os.system('%s waf install' % PYTHON)
+            subprocess.run([PYTHON, 'waf', 'configure', '--build-static', '--static-dependencies'
+                      '--with-python --prefix=tmp'] + macos_arm64_flags, check=True)
+        subprocess.run([PYTHON, 'waf'], check=True)
+        subprocess.run([PYTHON, 'waf', 'install'], check=True)
 
         library = glob.glob('tmp/lib/python*/*-packages/essentia')[0]
 

wscript

@@ -194,6 +194,12 @@ def configure(ctx):
             ctx.env.LINKFLAGS += ['-arch', 'i386', '-arch', 'x86_64']
             ctx.env.LDFLAGS = ['-arch', 'i386', '-arch', 'x86_64']
 
+        if ctx.options.ARCH == 'arm64':
+            ctx.env.CFLAGS += ['-arch', 'arm64']
+            ctx.env.CXXFLAGS += ['-arch', 'arm64']
+            ctx.env.LINKFLAGS += ['-arch', 'arm64']
+            ctx.env.LDFLAGS += ['-arch', 'arm64']
+
     elif sys.platform.startswith('linux'):
         # include -pthread flag because not all versions of gcc provide it automatically
         ctx.env.CXXFLAGS += ['-pthread']