From 7da1ef2ebd26253108ef1e8407ac3710d08b27a8 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:59:22 +0100 Subject: [PATCH] Add build CI (#34) --- .github/workflows/build.yml | 147 +++++++++++++++++++++++++++++++++++ _pyceres/helpers.h | 7 ++ _pyceres/logging.h | 5 +- ci/enter_vs_dev_shell.ps1 | 25 ++++++ ci/install-ceres-centos.sh | 16 ++++ ci/install-ceres-macos.sh | 19 +++++ ci/install-ceres-windows.ps1 | 28 +++++++ ci/vcpkg-dependencies.txt | 3 + pyproject.toml | 18 +++++ 9 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.yml create mode 100644 ci/enter_vs_dev_shell.ps1 create mode 100755 ci/install-ceres-centos.sh create mode 100755 ci/install-ceres-macos.sh create mode 100755 ci/install-ceres-windows.ps1 create mode 100644 ci/vcpkg-dependencies.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..10fb834 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,147 @@ +name: Build and publish wheels + +on: + push: + branches: + - main + pull_request: + types: [ assigned, opened, synchronize, reopened ] + release: + types: [ published, edited ] + workflow_dispatch: + +jobs: + build: + name: Build on ${{ matrix.config.os }} ${{ matrix.config.arch }} + runs-on: ${{ matrix.config.os }} + strategy: + matrix: + config: [ + {os: ubuntu-latest}, + {os: macos-13, arch: x86_64}, + {os: macos-13, arch: arm64}, + {os: windows-latest}, + ] + env: + COMPILER_CACHE_VERSION: 1 + COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + id: cache-builds + with: + key: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }} + restore-keys: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }} + path: ${{ env.COMPILER_CACHE_DIR }} + - name: Set env (macOS) + if: runner.os == 'macOS' + run: | + if [[ ${{ matrix.config.arch }} == "x86_64" ]]; then + VCPKG_TARGET_TRIPLET="x64-osx" + elif [[ ${{ matrix.config.arch }} == "arm64" ]]; then + VCPKG_TARGET_TRIPLET="arm64-osx-release" + else + exit 1 + fi + echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "$GITHUB_ENV" + + VCPKG_INSTALLATION_ROOT="/Users/runner/work/vcpkg" + CMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" + CMAKE_OSX_ARCHITECTURES=${{ matrix.config.arch }} + echo "VCPKG_INSTALLATION_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "$GITHUB_ENV" + echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "$GITHUB_ENV" + echo "CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" >> "$GITHUB_ENV" + + # Fix: cibuildhweel cannot interpolate env variables. + CONFIG_SETTINGS="cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" + CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" + CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" + echo "CIBW_CONFIG_SETTINGS_MACOS=${CONFIG_SETTINGS}" >> "$GITHUB_ENV" + + # vcpkg binary caching + VCPKG_CACHE_DIR="${COMPILER_CACHE_DIR}/vcpkg" + VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite" + echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV" + - name: Set env (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $VCPKG_INSTALLATION_ROOT="${{ github.workspace }}/vcpkg" + echo "VCPKG_INSTALLATION_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "${env:GITHUB_ENV}" + $CMAKE_TOOLCHAIN_FILE = "${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" + echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "${env:GITHUB_ENV}" + $VCPKG_TARGET_TRIPLET = "x64-windows" + echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "${env:GITHUB_ENV}" + + # Fix: cibuildhweel cannot interpolate env variables. + $CMAKE_TOOLCHAIN_FILE = $CMAKE_TOOLCHAIN_FILE.replace('\', '/') + $CONFIG_SETTINGS = "cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" + $CONFIG_SETTINGS = "${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" + echo "CIBW_CONFIG_SETTINGS_WINDOWS=${CONFIG_SETTINGS}" >> "${env:GITHUB_ENV}" + $CIBW_REPAIR_WHEEL_COMMAND = "delvewheel repair -v --add-path ${VCPKG_INSTALLATION_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/bin -w {dest_dir} {wheel}" + echo "CIBW_REPAIR_WHEEL_COMMAND_WINDOWS=${CIBW_REPAIR_WHEEL_COMMAND}" >> "${env:GITHUB_ENV}" + + # vcpkg binary caching + $VCPKG_CACHE_DIR = "${env:COMPILER_CACHE_DIR}/vcpkg" + $VCPKG_BINARY_SOURCES = "clear;files,${VCPKG_CACHE_DIR},readwrite" + echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "${env:GITHUB_ENV}" + - name: Set env (Ubuntu) + if: runner.os == 'Linux' + run: | + VCPKG_TARGET_TRIPLET="x64-linux-release" + echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "$GITHUB_ENV" + + VCPKG_INSTALLATION_ROOT="${{ github.workspace }}/vcpkg" + CMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" + echo "VCPKG_INSTALLATION_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "$GITHUB_ENV" + echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "$GITHUB_ENV" + + # Fix: cibuildhweel cannot interpolate env variables. + CONFIG_SETTINGS="cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" + CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" + echo "CIBW_CONFIG_SETTINGS_LINUX=${CONFIG_SETTINGS}" >> "$GITHUB_ENV" + + # Remap caching paths to the container + CONTAINER_COMPILER_CACHE_DIR="/compiler-cache" + CIBW_CONTAINER_ENGINE="docker; create_args: -v ${COMPILER_CACHE_DIR}:${CONTAINER_COMPILER_CACHE_DIR}" + echo "CIBW_CONTAINER_ENGINE=${CIBW_CONTAINER_ENGINE}" >> "$GITHUB_ENV" + echo "CONTAINER_COMPILER_CACHE_DIR=${CONTAINER_COMPILER_CACHE_DIR}" >> "$GITHUB_ENV" + + # vcpkg binary caching + VCPKG_CACHE_DIR="${CONTAINER_COMPILER_CACHE_DIR}/vcpkg" + VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite" + echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV" + + CIBW_ENVIRONMENT_PASS_LINUX="VCPKG_TARGET_TRIPLET VCPKG_INSTALLATION_ROOT CMAKE_TOOLCHAIN_FILE VCPKG_BINARY_SOURCES CONTAINER_COMPILER_CACHE_DIR" + echo "CIBW_ENVIRONMENT_PASS_LINUX=${CIBW_ENVIRONMENT_PASS_LINUX}" >> "$GITHUB_ENV" + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + CIBW_ARCHS_MACOS: ${{ matrix.config.arch }} + - name: Archive wheels + uses: actions/upload-artifact@v4 + with: + name: pyceres-${{ matrix.config.os }}-${{ matrix.config.arch }} + path: wheelhouse/pyceres-*.whl + + pypi-publish: + name: Publish wheels to PyPI + needs: build + runs-on: ubuntu-latest + # We publish the wheel to pypi when a new tag is pushed, + # either by creating a new GitHub release or explictly with `git tag` + if: ${{ github.event_name == 'release' || startsWith(github.ref, 'refs/tags') }} + steps: + - name: Download wheels + uses: actions/download-artifact@v4 + with: + path: ./artifacts/ + - name: Move wheels + run: mkdir ./wheelhouse && mv ./artifacts/**/*.whl ./wheelhouse/ + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + packages_dir: ./wheelhouse/ diff --git a/_pyceres/helpers.h b/_pyceres/helpers.h index 0022a0b..8fea333 100644 --- a/_pyceres/helpers.h +++ b/_pyceres/helpers.h @@ -13,6 +13,13 @@ #include #include +#ifdef _MSC_VER +#include +typedef SSIZE_T ssize_t; +#else +#include +#endif + using namespace pybind11::literals; namespace py = pybind11; diff --git a/_pyceres/logging.h b/_pyceres/logging.h index 22f2a13..05c78fd 100644 --- a/_pyceres/logging.h +++ b/_pyceres/logging.h @@ -29,7 +29,10 @@ void PyBindLogStack(const char* data, T size) { std::cerr << std::endl << std::endl << std::endl; } -__attribute__((noreturn)) void PyBindLogTermination() { +#ifdef __GNUC__ +__attribute__((noreturn)) +#endif +void PyBindLogTermination() { std::chrono::milliseconds timespan(2000); // or whatever py::scoped_estream_redirect stream( std::cerr, // std::ostream& diff --git a/ci/enter_vs_dev_shell.ps1 b/ci/enter_vs_dev_shell.ps1 new file mode 100644 index 0000000..9396c5c --- /dev/null +++ b/ci/enter_vs_dev_shell.ps1 @@ -0,0 +1,25 @@ +if (!$env:VisualStudioDevShell) { + $vswhere = "${Env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + if (!(Test-Path $vswhere)) { + throw "Failed to find vswhere.exe" + } + + & $vswhere -latest -format json + $vsInstance = & $vswhere -latest -format json | ConvertFrom-Json + if ($LASTEXITCODE) { + throw "vswhere.exe returned exit code $LASTEXITCODE" + } + + Import-Module "$($vsInstance.installationPath)/Common7/Tools/Microsoft.VisualStudio.DevShell.dll" + $prevCwd = Get-Location + try { + Enter-VsDevShell $vsInstance.instanceId -DevCmdArguments "-no_logo -host_arch=amd64 -arch=amd64" + } catch { + Write-Host $_ + Write-Error "Failed to enter Visual Studio Dev Shell" + exit 1 + } + Set-Location $prevCwd + + $env:VisualStudioDevShell = $true +} diff --git a/ci/install-ceres-centos.sh b/ci/install-ceres-centos.sh new file mode 100755 index 0000000..d6f5b97 --- /dev/null +++ b/ci/install-ceres-centos.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e -x +uname -a +CURRDIR=$(pwd) + +yum install -y gcc gcc-c++ ninja-build curl zip unzip tar + +DEPENDENCIES=$(cat ${CURRDIR}/ci/vcpkg-dependencies.txt) +git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} +cd ${VCPKG_INSTALLATION_ROOT} +git checkout ${VCPKG_COMMIT_ID} +./bootstrap-vcpkg.sh +./vcpkg install --recurse --clean-after-build \ + --triplet=${VCPKG_TARGET_TRIPLET} \ + ${DEPENDENCIES} +./vcpkg integrate install diff --git a/ci/install-ceres-macos.sh b/ci/install-ceres-macos.sh new file mode 100755 index 0000000..12c4840 --- /dev/null +++ b/ci/install-ceres-macos.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -x -e +CURRDIR=$(pwd) + +brew update +brew install git cmake ninja llvm + +# When building lapack-reference, vcpkg/cmake looks for gfortran. +ln -s $(which gfortran-13) "$(dirname $(which gfortran-13))/gfortran" + +DEPENDENCIES=$(cat ${CURRDIR}/ci/vcpkg-dependencies.txt) +git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} +cd ${VCPKG_INSTALLATION_ROOT} +git checkout ${VCPKG_COMMIT_ID} +./bootstrap-vcpkg.sh +./vcpkg install --recurse --clean-after-build \ + --triplet=${VCPKG_TARGET_TRIPLET} \ + ${DEPENDENCIES} +./vcpkg integrate install diff --git a/ci/install-ceres-windows.ps1 b/ci/install-ceres-windows.ps1 new file mode 100755 index 0000000..074c061 --- /dev/null +++ b/ci/install-ceres-windows.ps1 @@ -0,0 +1,28 @@ +$CURRDIR = $PWD + +$COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin" +New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR} +$env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path + +$NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe" +If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { + $zip_path = "${env:TEMP}/ninja.zip" + $url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip" + curl.exe -L -o ${zip_path} ${url} + Expand-Archive -LiteralPath ${zip_path} -DestinationPath ${COMPILER_TOOLS_DIR} + Remove-Item ${zip_path} +} + +cd ${CURRDIR} +git clone https://github.com/microsoft/vcpkg ${env:VCPKG_INSTALLATION_ROOT} +cd ${env:VCPKG_INSTALLATION_ROOT} +git checkout "${env:VCPKG_COMMIT_ID}" +./bootstrap-vcpkg.bat + +cd ${CURRDIR} +& "./ci/enter_vs_dev_shell.ps1" + +[System.Collections.ArrayList]$DEPS = Get-Content -Path "./ci/vcpkg-dependencies.txt" +& "${env:VCPKG_INSTALLATION_ROOT}/vcpkg.exe" install --recurse --clean-after-build ` + --triplet="${env:VCPKG_TARGET_TRIPLET}" @DEPS +& "${env:VCPKG_INSTALLATION_ROOT}/vcpkg.exe" integrate install diff --git a/ci/vcpkg-dependencies.txt b/ci/vcpkg-dependencies.txt new file mode 100644 index 0000000..c23473c --- /dev/null +++ b/ci/vcpkg-dependencies.txt @@ -0,0 +1,3 @@ +ceres[lapack,suitesparse] +gflags +glog diff --git a/pyproject.toml b/pyproject.toml index 8a0684a..b600390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,21 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", ] + +[tool.cibuildwheel] +build = "cp3{8,9,10,11}-{macosx,manylinux,win}*" +archs = ["auto64"] +test-command = "python -c \"import pyceres; print(pyceres.__version__)\"" + +[tool.cibuildwheel.environment] +VCPKG_COMMIT_ID = "fa6e6a6ec3224f1d3697d544edef6272a59cd834" + +[tool.cibuildwheel.linux] +before-all = "{package}/ci/install-ceres-centos.sh" + +[tool.cibuildwheel.macos] +before-all = "{package}/ci/install-ceres-macos.sh" + +[tool.cibuildwheel.windows] +before-all = "powershell -File {package}/ci/install-ceres-windows.ps1" +before-build = "pip install delvewheel"