From 448bc760782855c421146664bde76a3b47387fa8 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:56:22 +0000 Subject: [PATCH 01/14] Update actions versions (except actions-rs/toolchain) --- .github/workflows/ci-ffi-python.yml | 20 ++++++++--------- .github/workflows/ci-ffi-ruby.yml | 8 +++---- .github/workflows/publish.yml | 18 +++++++-------- .github/workflows/release.yml | 34 ++++++++++++++--------------- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 458ed5d..6510460 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest # TODO try using grafana runners steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build in Docker run: make wheel/linux/amd64 @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - PYTHON_VERSION: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + PYTHON_VERSION: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] needs: [ 'linux-build' ] name: Linux Test runs-on: ubuntu-latest @@ -36,18 +36,18 @@ jobs: PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} steps: - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.PYTHON_VERSION }} architecture: x64 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: "linux.whl" path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip install *.whl" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: docker run -d -p4040:4040 grafana/pyroscope - run: python pyroscope_ffi/python/scripts/tests/test.py @@ -56,7 +56,7 @@ jobs: runs-on: github-hosted-ubuntu-arm64 steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build in Docker run: make wheel/linux/arm64 @@ -70,8 +70,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - name: Upgrade pip @@ -104,7 +104,7 @@ jobs: runs-on: macos-${{ matrix.macos-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: 1.76.0 @@ -112,7 +112,7 @@ jobs: profile: minimal override: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index c1db0e7..ed6658e 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: make gem/linux/amd64 - uses: actions/upload-artifact@v4 with: @@ -37,7 +37,7 @@ jobs: RUST_TARGET: ${{ matrix.target }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: '3.1' @@ -67,13 +67,13 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.RUBY_VERSION }} - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: "linux.gem" path: "${{github.workspace}}/ruby" - run: "cd ${{ github.workspace }}/ruby && ls -l" - run: "cd ${{ github.workspace }}/ruby && gem install *.gem" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run Ruby Script run: pyroscope_ffi/ruby/scripts/tests/test.rb env: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b0ba7b2..aad4067 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.event.release.tag_name, 'lib-')" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -26,7 +26,7 @@ jobs: # runs-on: ubuntu-latest # if: "startsWith(github.event.release.tag_name, 'cli-')" # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # with: # submodules: recursive # - uses: actions-rs/toolchain@v1 @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.event.release.tag_name, 'pprofrs-')" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.event.release.tag_name, 'rbspy-')" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.event.release.tag_name, 'pyspy-')" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -90,7 +90,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.event.release.tag_name, 'python-')" steps: - - uses: robinraju/release-downloader@v1.4 + - uses: robinraju/release-downloader@v1 with: tag: ${{ github.event.release.tag_name }} fileName: "*" @@ -99,7 +99,7 @@ jobs: out-file-path: "dist" token: ${{ secrets.GITHUB_TOKEN }} - name: Publish a Python distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@v1.12.2 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} @@ -110,7 +110,7 @@ jobs: outputs: files_json: ${{ steps.list-files.outputs.files_json }} steps: - - uses: robinraju/release-downloader@v1.4 + - uses: robinraju/release-downloader@v1 with: tag: ${{ github.event.release.tag_name }} fileName: "*" @@ -138,7 +138,7 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '3.1' - - uses: robinraju/release-downloader@v1.4 + - uses: robinraju/release-downloader@v1 with: tag: ${{ github.event.release.tag_name }} fileName: "*" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 671431c..0f1186e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,8 +21,8 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/clibuilder-')" steps: - - uses: actions/checkout@v3 - - uses: docker/login-action@v2 + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 name: Login to Docker Hub with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -34,8 +34,8 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/manylinux-')" steps: - - uses: actions/checkout@v3 - - uses: docker/login-action@v2 + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 name: Login to Docker Hub with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -63,10 +63,10 @@ jobs: needs: cli-release runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - - uses: docker/login-action@v2 + - uses: docker/login-action@v3 name: Login to Docker Hub with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -75,7 +75,7 @@ jobs: - run: DOCKER_EXTRA="--output=." make cli/docker-image - name: Upload release archive - uses: actions/upload-release-asset@v1.0.1 + uses: actions/upload-release-asset@v1.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -128,7 +128,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: make wheel/linux/amd64 - uses: actions/upload-artifact@v4 with: @@ -148,7 +148,7 @@ jobs: runs-on: github-hosted-ubuntu-arm64 steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: make wheel/linux/arm64 - uses: actions/upload-artifact@v4 with: @@ -181,7 +181,7 @@ jobs: runs-on: macos-${{ matrix.macos-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: @@ -190,7 +190,7 @@ jobs: profile: minimal override: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.11 @@ -215,9 +215,9 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - name: Upgrade pip @@ -261,7 +261,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: make gem/linux/amd64 - uses: actions/upload-artifact@v4 with: @@ -281,7 +281,7 @@ jobs: name: Release Linux gem arm64 runs-on: github-hosted-ubuntu-arm64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: make gem/linux/arm64 - uses: actions/upload-artifact@v4 with: @@ -313,7 +313,7 @@ jobs: runs-on: macos-${{ matrix.macos-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: '3.1' @@ -346,7 +346,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: From aaefd72d79d11c46c98902a70211e1fae234c2c0 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 02:03:51 +0000 Subject: [PATCH 02/14] Replace actions-rs/toolchain with dtolnay/rust-toolchain --- .github/workflows/ci-ffi-python.yml | 6 ++---- .github/workflows/ci-ffi-ruby.yml | 6 ++---- .github/workflows/publish.yml | 15 +++++---------- .github/workflows/release.yml | 8 ++------ 4 files changed, 11 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 6510460..62e74d4 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -105,12 +105,10 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: 1.76.0 - target: ${{ matrix.target }} - profile: minimal - override: true + targets: ${{ matrix.target }} - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index ed6658e..693ca14 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -41,12 +41,10 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '3.1' - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: 1.76.0 - target: ${{ matrix.target }} - profile: minimal - override: true + targets: ${{ matrix.target }} - run: make gem/mac/${{ matrix.mk-arch }} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aad4067..217c677 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,10 +12,9 @@ jobs: if: "startsWith(github.event.release.tag_name, 'lib-')" steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable - override: true - name: publish pyroscope crate continue-on-error: true run: | @@ -29,10 +28,9 @@ jobs: # - uses: actions/checkout@v4 # with: # submodules: recursive -# - uses: actions-rs/toolchain@v1 +# - uses: dtolnay/rust-toolchain@v1 # with: # toolchain: stable -# override: true # - name: install libunwind (for pprof) # run: sudo apt install libunwind8-dev # - name: publish pyroscope crate @@ -46,10 +44,9 @@ jobs: if: "startsWith(github.event.release.tag_name, 'pprofrs-')" steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable - override: true - name: publish pprofrs crate continue-on-error: true run: | @@ -61,10 +58,9 @@ jobs: if: "startsWith(github.event.release.tag_name, 'rbspy-')" steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable - override: true - name: publish rbspy crate continue-on-error: true run: | @@ -76,10 +72,9 @@ jobs: if: "startsWith(github.event.release.tag_name, 'pyspy-')" steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable - override: true - name: publish pyspy crate continue-on-error: true run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f1186e..ce23724 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -183,12 +183,10 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: 1.76.0 target: ${{ matrix.target }} - profile: minimal - override: true - uses: actions/setup-python@v5 with: @@ -317,12 +315,10 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '3.1' - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: 1.76.0 target: ${{ matrix.target }} - profile: minimal - override: true - run: make pyroscope_ffi/clean gem/mac/${{ matrix.mk-arch }} From 7edd9025b199f191f3a16ea9f88bb5d41af5b4c8 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:01:45 +0000 Subject: [PATCH 03/14] Remove EOL Python & Ruby --- .github/workflows/ci-ffi-python.yml | 2 +- .github/workflows/ci-ffi-ruby.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 62e74d4..822abd6 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - PYTHON_VERSION: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + PYTHON_VERSION: [ '3.9', '3.10', '3.11', '3.12', '3.13'] needs: [ 'linux-build' ] name: Linux Test runs-on: ubuntu-latest diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 693ca14..7e2c129 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -57,7 +57,7 @@ jobs: matrix: PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] PYROSCOPE_ONCPU: [1, 0] - RUBY_VERSION: ['2.6', '2.7', '3.0', '3.1', '3.2'] + RUBY_VERSION: ['3.1', '3.2', '3.3'] needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest From 3941cb097fbbe4a678310100707de0e23c0a0817 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:13:42 +0000 Subject: [PATCH 04/14] Use stable rust toolchain --- .github/workflows/ci-ffi-python.yml | 2 +- .github/workflows/ci-ffi-ruby.yml | 2 +- .github/workflows/release.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 822abd6..05909e2 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -107,7 +107,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: 1.76.0 + toolchain: stable targets: ${{ matrix.target }} - uses: actions/setup-python@v5 diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 7e2c129..c51b86f 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -43,7 +43,7 @@ jobs: ruby-version: '3.1' - uses: dtolnay/rust-toolchain@v1 with: - toolchain: 1.76.0 + toolchain: stable targets: ${{ matrix.target }} - run: make gem/mac/${{ matrix.mk-arch }} - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce23724..97ecb94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -185,7 +185,7 @@ jobs: - uses: dtolnay/rust-toolchain@v1 with: - toolchain: 1.76.0 + toolchain: stable target: ${{ matrix.target }} - uses: actions/setup-python@v5 @@ -317,7 +317,7 @@ jobs: ruby-version: '3.1' - uses: dtolnay/rust-toolchain@v1 with: - toolchain: 1.76.0 + toolchain: stable target: ${{ matrix.target }} - run: make pyroscope_ffi/clean gem/mac/${{ matrix.mk-arch }} From 17c44198f529768d6d713a1beb51d556a1ad36ba Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:22:37 +0000 Subject: [PATCH 05/14] Update builders --- docker/Dockerfile.cli_builder | 10 +++++----- docker/gem.Dockerfile | 2 +- docker/wheel.Dockerfile | 18 ++++++++++++++++-- ffi.mk | 4 ++-- pyroscope_ffi/python/manylinux.sh | 2 +- pyroscope_ffi/python/setup.cfg | 3 ++- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile.cli_builder b/docker/Dockerfile.cli_builder index 590626b..799f021 100644 --- a/docker/Dockerfile.cli_builder +++ b/docker/Dockerfile.cli_builder @@ -1,7 +1,7 @@ -FROM alpine:3.18 as rust-builder +FROM alpine:3.20 AS rust-builder RUN apk add wget ARG TARGETPLATFORM @@ -20,15 +20,15 @@ RUN case "${TARGETPLATFORM}" in \ RUN source ./.env && \ wget https://static.rust-lang.org/rustup/dist/${RUST_TARGET}/rustup-init && \ chmod +x rustup-init && \ - ./rustup-init -y --default-toolchain=1.76.0 --default-host=${RUST_TARGET} + ./rustup-init -y --default-toolchain=stable --default-host=${RUST_TARGET} ENV PATH=/root/.cargo/bin:$PATH RUN apk add make gcc musl-dev RUN source ./.env && \ - wget https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz && \ - tar -zxvf libunwind-1.6.2.tar.gz && \ - cd libunwind-1.6.2 && \ + wget https://github.com/libunwind/libunwind/releases/download/v1.8.1/libunwind-1.8.1.tar.gz && \ + tar -zxvf libunwind-1.8.1.tar.gz && \ + cd libunwind-1.8.1 && \ ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation --prefix=/usr/local/musl/${RUST_TARGET} && \ make -j16 && \ make install diff --git a/docker/gem.Dockerfile b/docker/gem.Dockerfile index 06bb887..603dded 100644 --- a/docker/gem.Dockerfile +++ b/docker/gem.Dockerfile @@ -20,7 +20,7 @@ RUN cargo build -p thread_id --release -FROM ruby:3.1 as builder-gem +FROM ruby:3.3 as builder-gem WORKDIR /gem ADD pyroscope_ffi/ruby /gem/ diff --git a/docker/wheel.Dockerfile b/docker/wheel.Dockerfile index 40c2274..28194c6 100644 --- a/docker/wheel.Dockerfile +++ b/docker/wheel.Dockerfile @@ -1,6 +1,20 @@ -ARG BASE +ARG PLATFORM=x86_64 +FROM quay.io/pypa/manylinux2014_${PLATFORM} AS builder -FROM ${BASE} as builder +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init \ + && chmod +x ./rustup-init \ + && ./rustup-init -y --default-toolchain=stable --default-host=$(arch)-unknown-linux-gnu +ENV PATH=/root/.cargo/bin:$PATH +RUN yum -y install gcc libffi-devel openssl-devel wget gcc-c++ glibc-devel make + +# for python +ENV LIBUNWIND_VERSION=1.8.1 +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && cd libunwind-${LIBUNWIND_VERSION} \ + && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ + && make \ + && make install WORKDIR /pyroscope-rs diff --git a/ffi.mk b/ffi.mk index 9ff25cd..95f3d36 100644 --- a/ffi.mk +++ b/ffi.mk @@ -14,7 +14,7 @@ pyroscope_ffi/clean: .phony: wheel/linux/amd64 wheel/linux/amd64: pyroscope_ffi/clean docker buildx build \ - --build-arg=BASE=$(MANYLINUX_PREFIX)_$(BUILD_ARCH_AMD):$(MANYLINUX_VERSION) \ + --build-arg=PLATFORM=x86_64 \ --platform=linux/amd64 \ --output=pyroscope_ffi/python \ -f docker/wheel.Dockerfile \ @@ -23,7 +23,7 @@ wheel/linux/amd64: pyroscope_ffi/clean .phony: wheel/linux/arm64 wheel/linux/arm64: pyroscope_ffi/clean docker buildx build \ - --build-arg=BASE=$(MANYLINUX_PREFIX)_$(BUILD_ARCH_ARM):$(MANYLINUX_VERSION) \ + --build-arg=PLATFORM=aarch64 \ --platform=linux/arm64 \ --output=pyroscope_ffi/python \ -f docker/wheel.Dockerfile \ diff --git a/pyroscope_ffi/python/manylinux.sh b/pyroscope_ffi/python/manylinux.sh index 9a17cb3..ea338dd 100755 --- a/pyroscope_ffi/python/manylinux.sh +++ b/pyroscope_ffi/python/manylinux.sh @@ -5,7 +5,7 @@ cargo --version # Build wheels # todo this one is deprecated, use "build" package -/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel +/opt/python/cp39-cp39/bin/python setup.py bdist_wheel # Audit wheels for wheel in dist/*-linux_*.whl; do diff --git a/pyroscope_ffi/python/setup.cfg b/pyroscope_ffi/python/setup.cfg index 43c5679..cd0aee6 100644 --- a/pyroscope_ffi/python/setup.cfg +++ b/pyroscope_ffi/python/setup.cfg @@ -8,7 +8,8 @@ url = https://pyroscope.io maintainer = Abid Omar maintainer_email = contact@pyroscope.io license = Apache 2.0 -license_file = LICENSE +license_files = + LICENSE classifiers = Intended Audience :: Developers License :: OSI Approved :: Apache Software License From b1f7d63e232912b4e899e43a09649dc0e9a0c41b Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:24:48 +0000 Subject: [PATCH 06/14] Run CI on all branch pushes --- .github/workflows/ci-ffi-python.yml | 2 +- .github/workflows/ci-ffi-ruby.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 05909e2..413b06a 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -2,7 +2,7 @@ name: Python Package on: push: - branches: [ main ] + branches: pull_request: branches: [ main ] diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index c51b86f..87c5944 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -2,7 +2,7 @@ name: Ruby Package on: push: - branches: [main] + branches: pull_request: branches: [main] From f1151bd3136e5fb0c2977ca9468d8a12947a1def Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:07:51 +0000 Subject: [PATCH 07/14] Use mac14, streamline job names --- .github/workflows/ci-ffi-python.yml | 8 +++----- .github/workflows/ci-ffi-ruby.yml | 5 +++-- .github/workflows/release.yml | 12 +++++------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 413b06a..4831831 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -91,16 +91,14 @@ jobs: fail-fast: false matrix: include: - - macos-version: "11.0" + - macos-version: "14" target: x86_64-apple-darwin - py-platform: macosx-11_0_x86_64 mk-arch: amd64 - - macos-version: "11.0" + - macos-version: "14" target: aarch64-apple-darwin - py-platform: macosx-11_0_arm64 mk-arch: arm64 - name: macOS - ${{ matrix.py-platform }} + name: macOS - ${{ matrix.target }} runs-on: macos-${{ matrix.macos-version }} steps: diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 87c5944..d9ad8c3 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -24,12 +24,13 @@ jobs: fail-fast: false matrix: include: - - macos-version: "11.0" + - macos-version: "14" target: x86_64-apple-darwin mk-arch: amd64 - - macos-version: "11.0" + - macos-version: "14" target: aarch64-apple-darwin mk-arch: arm64 + name: macOS - ${{ matrix.target }} runs-on: macos-${{ matrix.macos-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 97ecb94..c26d32e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -168,16 +168,14 @@ jobs: fail-fast: false matrix: include: - - macos-version: "11.0" + - macos-version: "14" target: x86_64-apple-darwin - py-platform: macosx-11_0_x86_64 mk-arch: amd64 - - macos-version: "11.0" + - macos-version: "14" target: aarch64-apple-darwin - py-platform: macosx-11_0_arm64 mk-arch: arm64 - name: macOS - ${{ matrix.py-platform }} + name: macOS - ${{ matrix.target }} runs-on: macos-${{ matrix.macos-version }} steps: @@ -300,10 +298,10 @@ jobs: fail-fast: false matrix: include: - - macos-version: "11.0" + - macos-version: "14" target: x86_64-apple-darwin mk-arch: amd64 - - macos-version: "11.0" + - macos-version: "14" target: aarch64-apple-darwin mk-arch: arm64 From f7cdaa4d2a84512672e8cf26288752e84e43475e Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:04:33 +0000 Subject: [PATCH 08/14] Fix upload artifact names --- .github/workflows/ci-ffi-python.yml | 2 +- .github/workflows/ci-ffi-ruby.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 4831831..4772c0b 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -115,5 +115,5 @@ jobs: - run: make wheel/mac/${{ matrix.mk-arch }} - uses: actions/upload-artifact@v4 with: - name: ${{ github.sha }} + name: ${{ github.sha }}-python-${{ matrix.target }} path: pyroscope_ffi/python/dist/* diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index d9ad8c3..ea7fb9b 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -49,7 +49,7 @@ jobs: - run: make gem/mac/${{ matrix.mk-arch }} - uses: actions/upload-artifact@v4 with: - name: ${{ github.sha }} + name: ${{ github.sha }}-ruby-${{ matrix.target }} path: pyroscope_ffi/ruby/pkg/*.gem linux-test: From 7089abcb491acc3292dbe4c4edd5619c09b1e0b9 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:28:51 +0000 Subject: [PATCH 09/14] Format dockerfiles & lock rust version & add vars for libunwind --- docker/Dockerfile.cli | 4 +--- docker/Dockerfile.cli_builder | 15 ++++++--------- docker/Dockerfile.manylinux2014_aarch64 | 21 +++++++++++---------- docker/Dockerfile.manylinux2014_x86_64 | 20 ++++++++++---------- docker/gem.Dockerfile | 5 +---- docker/wheel.Dockerfile | 21 +++++++++++---------- 6 files changed, 40 insertions(+), 46 deletions(-) diff --git a/docker/Dockerfile.cli b/docker/Dockerfile.cli index a587c95..8443092 100644 --- a/docker/Dockerfile.cli +++ b/docker/Dockerfile.cli @@ -8,11 +8,9 @@ ADD src ./src ADD Cargo.toml ./Cargo.toml RUN --mount=type=cache,target=/root/.cargo/registry \ - cd pyroscope_cli && \ + cd pyroscope_cli && cargo build --release --bin pyroscope-cli - - FROM scratch as final COPY --from=builder /app/pyroscope_cli/target/release/pyroscope-cli /pyroscope-cli diff --git a/docker/Dockerfile.cli_builder b/docker/Dockerfile.cli_builder index 799f021..7bdabb6 100644 --- a/docker/Dockerfile.cli_builder +++ b/docker/Dockerfile.cli_builder @@ -1,13 +1,9 @@ - - - FROM alpine:3.20 AS rust-builder RUN apk add wget ARG TARGETPLATFORM - - +ENV RUST_VERSION=1.82 RUN case "${TARGETPLATFORM}" in \ "linux/amd64") \ echo 'export RUST_TARGET="x86_64-unknown-linux-musl"' > ./.env ;; \ @@ -20,15 +16,16 @@ RUN case "${TARGETPLATFORM}" in \ RUN source ./.env && \ wget https://static.rust-lang.org/rustup/dist/${RUST_TARGET}/rustup-init && \ chmod +x rustup-init && \ - ./rustup-init -y --default-toolchain=stable --default-host=${RUST_TARGET} + ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=${RUST_TARGET} ENV PATH=/root/.cargo/bin:$PATH +ENV LIBUNWIND_VERSION=1.8.1 RUN apk add make gcc musl-dev RUN source ./.env && \ - wget https://github.com/libunwind/libunwind/releases/download/v1.8.1/libunwind-1.8.1.tar.gz && \ - tar -zxvf libunwind-1.8.1.tar.gz && \ - cd libunwind-1.8.1 && \ + wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && \ + tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && \ + cd libunwind-${LIBUNWIND_VERSION} && \ ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation --prefix=/usr/local/musl/${RUST_TARGET} && \ make -j16 && \ make install diff --git a/docker/Dockerfile.manylinux2014_aarch64 b/docker/Dockerfile.manylinux2014_aarch64 index ddd5cf9..57226bb 100644 --- a/docker/Dockerfile.manylinux2014_aarch64 +++ b/docker/Dockerfile.manylinux2014_aarch64 @@ -1,16 +1,17 @@ FROM --platform=linux/aarch64 quay.io/pypa/manylinux2014_aarch64 - -RUN curl https://static.rust-lang.org/rustup/dist/aarch64-unknown-linux-musl/rustup-init -o ./rustup-init \ - && chmod +x ./rustup-init \ - && ./rustup-init -y --default-toolchain=1.76.0 --default-host=aarch64-unknown-linux-gnu +ENV RUST_VERSION=1.82 +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && + chmod +x ./rustup-init && + ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget # for python -RUN wget https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz \ - && tar -zxvf libunwind-1.6.2.tar.gz \ - && cd libunwind-1.6.2 \ - && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ - && make \ - && make install +ENV LIBUNWIND_VERSION=1.8.1 +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && + tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && + cd libunwind-${LIBUNWIND_VERSION} && + ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && + make && + make install diff --git a/docker/Dockerfile.manylinux2014_x86_64 b/docker/Dockerfile.manylinux2014_x86_64 index da07ba2..74ace0c 100644 --- a/docker/Dockerfile.manylinux2014_x86_64 +++ b/docker/Dockerfile.manylinux2014_x86_64 @@ -1,16 +1,16 @@ FROM --platform=linux/amd64 quay.io/pypa/manylinux2014_x86_64 - -RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-musl/rustup-init -o ./rustup-init \ - && chmod +x ./rustup-init \ - && ./rustup-init -y --default-toolchain=1.76.0 --default-host=x86_64-unknown-linux-gnu +ENV RUST_VERSION=1.82 +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && + chmod +x ./rustup-init && + ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget # for python -RUN wget https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz \ - && tar -zxvf libunwind-1.6.2.tar.gz \ - && cd libunwind-1.6.2 \ - && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ - && make \ - && make install +RUN wget https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz && + tar -zxvf libunwind-1.6.2.tar.gz && + cd libunwind-1.6.2 && + ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && + make && + make install diff --git a/docker/gem.Dockerfile b/docker/gem.Dockerfile index 603dded..44f36b2 100644 --- a/docker/gem.Dockerfile +++ b/docker/gem.Dockerfile @@ -1,6 +1,5 @@ ARG BASE - FROM ${BASE} as builder-native WORKDIR /pyroscope-rs @@ -18,8 +17,6 @@ ADD pyroscope_ffi/ pyroscope_ffi/ RUN cargo build -p rbspy --release RUN cargo build -p thread_id --release - - FROM ruby:3.3 as builder-gem WORKDIR /gem ADD pyroscope_ffi/ruby /gem/ @@ -32,4 +29,4 @@ ARG TARGET_TASK RUN rake ${TARGET_TASK} FROM scratch -COPY --from=builder-gem /gem/pkg/ /pkg/ \ No newline at end of file +COPY --from=builder-gem /gem/pkg/ /pkg/ diff --git a/docker/wheel.Dockerfile b/docker/wheel.Dockerfile index 28194c6..0297d66 100644 --- a/docker/wheel.Dockerfile +++ b/docker/wheel.Dockerfile @@ -1,20 +1,21 @@ ARG PLATFORM=x86_64 FROM quay.io/pypa/manylinux2014_${PLATFORM} AS builder -RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init \ - && chmod +x ./rustup-init \ - && ./rustup-init -y --default-toolchain=stable --default-host=$(arch)-unknown-linux-gnu +ENV RUST_VERSION=1.82 +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && + chmod +x ./rustup-init && + ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget gcc-c++ glibc-devel make # for python ENV LIBUNWIND_VERSION=1.8.1 -RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz \ - && tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz \ - && cd libunwind-${LIBUNWIND_VERSION} \ - && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ - && make \ - && make install +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && + tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && + cd libunwind-${LIBUNWIND_VERSION} && + ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && + make && + make install WORKDIR /pyroscope-rs @@ -32,4 +33,4 @@ ADD pyroscope_ffi/ pyroscope_ffi/ RUN cd /pyroscope-rs/pyroscope_ffi/python && ./manylinux.sh FROM scratch -COPY --from=builder /pyroscope-rs/pyroscope_ffi/python/dist dist/ \ No newline at end of file +COPY --from=builder /pyroscope-rs/pyroscope_ffi/python/dist dist/ From 8a3bc5dcf5516d149bee473bb7a5f65a5e609c64 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:35:11 +0000 Subject: [PATCH 10/14] Fix dockerfile errors on linebreaks --- docker/Dockerfile.manylinux2014_aarch64 | 18 +++++++++--------- docker/Dockerfile.manylinux2014_x86_64 | 19 ++++++++++--------- docker/wheel.Dockerfile | 18 +++++++++--------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docker/Dockerfile.manylinux2014_aarch64 b/docker/Dockerfile.manylinux2014_aarch64 index 57226bb..2c1fab4 100644 --- a/docker/Dockerfile.manylinux2014_aarch64 +++ b/docker/Dockerfile.manylinux2014_aarch64 @@ -1,17 +1,17 @@ FROM --platform=linux/aarch64 quay.io/pypa/manylinux2014_aarch64 ENV RUST_VERSION=1.82 -RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && - chmod +x ./rustup-init && - ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init \ + && chmod +x ./rustup-init \ + && ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget # for python ENV LIBUNWIND_VERSION=1.8.1 -RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && - tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && - cd libunwind-${LIBUNWIND_VERSION} && - ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && - make && - make install +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && cd libunwind-${LIBUNWIND_VERSION} \ + && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ + && make \ + && make install diff --git a/docker/Dockerfile.manylinux2014_x86_64 b/docker/Dockerfile.manylinux2014_x86_64 index 74ace0c..97de0e3 100644 --- a/docker/Dockerfile.manylinux2014_x86_64 +++ b/docker/Dockerfile.manylinux2014_x86_64 @@ -1,16 +1,17 @@ FROM --platform=linux/amd64 quay.io/pypa/manylinux2014_x86_64 ENV RUST_VERSION=1.82 -RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && - chmod +x ./rustup-init && - ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init \ + && chmod +x ./rustup-init \ + && ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget # for python -RUN wget https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz && - tar -zxvf libunwind-1.6.2.tar.gz && - cd libunwind-1.6.2 && - ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && - make && - make install +ENV LIBUNWIND_VERSION=1.8.1 +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && cd libunwind-${LIBUNWIND_VERSION} \ + && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ + && make \ + && make install diff --git a/docker/wheel.Dockerfile b/docker/wheel.Dockerfile index 0297d66..4e4a867 100644 --- a/docker/wheel.Dockerfile +++ b/docker/wheel.Dockerfile @@ -2,20 +2,20 @@ ARG PLATFORM=x86_64 FROM quay.io/pypa/manylinux2014_${PLATFORM} AS builder ENV RUST_VERSION=1.82 -RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init && - chmod +x ./rustup-init && - ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu +RUN curl https://static.rust-lang.org/rustup/dist/$(arch)-unknown-linux-musl/rustup-init -o ./rustup-init \ + && chmod +x ./rustup-init \ + && ./rustup-init -y --default-toolchain=${RUST_VERSION} --default-host=$(arch)-unknown-linux-gnu ENV PATH=/root/.cargo/bin:$PATH RUN yum -y install gcc libffi-devel openssl-devel wget gcc-c++ glibc-devel make # for python ENV LIBUNWIND_VERSION=1.8.1 -RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && - tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && - cd libunwind-${LIBUNWIND_VERSION} && - ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation && - make && - make install +RUN wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz \ + && cd libunwind-${LIBUNWIND_VERSION} \ + && ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation \ + && make \ + && make install WORKDIR /pyroscope-rs From dea3db20a0b784dc483c05c4ff5b3419aaff76af Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:29:41 +0000 Subject: [PATCH 11/14] Optimize build processes --- .github/workflows/ci-ffi-python.yml | 39 +++++++++++++---------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 4772c0b..dd7a195 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -4,29 +4,38 @@ on: push: branches: pull_request: - branches: [ main ] + branches: + - main jobs: linux-build: - name: Linux - amd64 - runs-on: ubuntu-latest # TODO try using grafana runners + strategy: + matrix: + ARCH: + - "amd64" + - "arm64" + name: Linux Build - ${{ matrix.ARCH }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Build in Docker - run: make wheel/linux/amd64 + run: make wheel/linux/${{ matrix.ARCH }} - uses: actions/upload-artifact@v4 with: - name: "linux.whl" + name: "linux_${{ matrix.ARCH }}.whl" path: pyroscope_ffi/python/dist/* linux-test: strategy: fail-fast: false matrix: - PYTHON_VERSION: [ '3.9', '3.10', '3.11', '3.12', '3.13'] - needs: [ 'linux-build' ] + PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"] + needs: ["linux-build"] name: Linux Test runs-on: ubuntu-latest env: @@ -42,7 +51,7 @@ jobs: architecture: x64 - uses: actions/download-artifact@v4 with: - name: "linux.whl" + name: "linux_amd64.whl" path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" @@ -51,20 +60,6 @@ jobs: - run: docker run -d -p4040:4040 grafana/pyroscope - run: python pyroscope_ffi/python/scripts/tests/test.py - linux-arm-build: - name: Linux - arm64 - runs-on: github-hosted-ubuntu-arm64 - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v4 - - name: Build in Docker - run: make wheel/linux/arm64 - - - uses: actions/upload-artifact@v4 - with: - name: "linux-arm.whl" - path: pyroscope_ffi/python/dist/* - sdist-build: name: sdist runs-on: ubuntu-latest From a29824aacd09d4a640b935ead622bdaa78b37af3 Mon Sep 17 00:00:00 2001 From: Kwong Tung Nan Date: Fri, 22 Nov 2024 00:31:33 +0800 Subject: [PATCH 12/14] Revert "Optimize build processes" This reverts commit dea3db20a0b784dc483c05c4ff5b3419aaff76af. --- .github/workflows/ci-ffi-python.yml | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index dd7a195..4772c0b 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -4,38 +4,29 @@ on: push: branches: pull_request: - branches: - - main + branches: [ main ] jobs: linux-build: - strategy: - matrix: - ARCH: - - "amd64" - - "arm64" - name: Linux Build - ${{ matrix.ARCH }} - runs-on: ubuntu-latest + name: Linux - amd64 + runs-on: ubuntu-latest # TODO try using grafana runners steps: - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: Build in Docker - run: make wheel/linux/${{ matrix.ARCH }} + run: make wheel/linux/amd64 - uses: actions/upload-artifact@v4 with: - name: "linux_${{ matrix.ARCH }}.whl" + name: "linux.whl" path: pyroscope_ffi/python/dist/* linux-test: strategy: fail-fast: false matrix: - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"] - needs: ["linux-build"] + PYTHON_VERSION: [ '3.9', '3.10', '3.11', '3.12', '3.13'] + needs: [ 'linux-build' ] name: Linux Test runs-on: ubuntu-latest env: @@ -51,7 +42,7 @@ jobs: architecture: x64 - uses: actions/download-artifact@v4 with: - name: "linux_amd64.whl" + name: "linux.whl" path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" @@ -60,6 +51,20 @@ jobs: - run: docker run -d -p4040:4040 grafana/pyroscope - run: python pyroscope_ffi/python/scripts/tests/test.py + linux-arm-build: + name: Linux - arm64 + runs-on: github-hosted-ubuntu-arm64 + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v4 + - name: Build in Docker + run: make wheel/linux/arm64 + + - uses: actions/upload-artifact@v4 + with: + name: "linux-arm.whl" + path: pyroscope_ffi/python/dist/* + sdist-build: name: sdist runs-on: ubuntu-latest From 1f3f80be5058345be9f06f64ecfb6479efc9aec1 Mon Sep 17 00:00:00 2001 From: Kwong Tung Nan Date: Fri, 22 Nov 2024 00:34:59 +0800 Subject: [PATCH 13/14] Set rust toolchain to use 1.82.0 Signed-off-by: Kwong Tung Nan --- .github/workflows/ci-ffi-python.yml | 2 +- .github/workflows/ci-ffi-ruby.yml | 2 +- .github/workflows/publish.yml | 10 +++++----- .github/workflows/release.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 4772c0b..233e021 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -105,7 +105,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 targets: ${{ matrix.target }} - uses: actions/setup-python@v5 diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index ea7fb9b..5b26027 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -44,7 +44,7 @@ jobs: ruby-version: '3.1' - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 targets: ${{ matrix.target }} - run: make gem/mac/${{ matrix.mk-arch }} - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 217c677..b370c15 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 - name: publish pyroscope crate continue-on-error: true run: | @@ -30,7 +30,7 @@ jobs: # submodules: recursive # - uses: dtolnay/rust-toolchain@v1 # with: -# toolchain: stable +# toolchain: 1.82.0 # - name: install libunwind (for pprof) # run: sudo apt install libunwind8-dev # - name: publish pyroscope crate @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 - name: publish pprofrs crate continue-on-error: true run: | @@ -60,7 +60,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 - name: publish rbspy crate continue-on-error: true run: | @@ -74,7 +74,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 - name: publish pyspy crate continue-on-error: true run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c26d32e..d61e504 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -183,7 +183,7 @@ jobs: - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 target: ${{ matrix.target }} - uses: actions/setup-python@v5 @@ -315,7 +315,7 @@ jobs: ruby-version: '3.1' - uses: dtolnay/rust-toolchain@v1 with: - toolchain: stable + toolchain: 1.82.0 target: ${{ matrix.target }} - run: make pyroscope_ffi/clean gem/mac/${{ matrix.mk-arch }} From 01de317d875d4f95a2f438279b7fdff4d2bdac95 Mon Sep 17 00:00:00 2001 From: Kwong Tung Nan Date: Fri, 22 Nov 2024 00:36:56 +0800 Subject: [PATCH 14/14] Format dockerfile Signed-off-by: Kwong Tung Nan --- docker/Dockerfile.cli | 2 +- docker/Dockerfile.cli_builder | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile.cli b/docker/Dockerfile.cli index 8443092..ef1cb21 100644 --- a/docker/Dockerfile.cli +++ b/docker/Dockerfile.cli @@ -8,7 +8,7 @@ ADD src ./src ADD Cargo.toml ./Cargo.toml RUN --mount=type=cache,target=/root/.cargo/registry \ - cd pyroscope_cli && + cd pyroscope_cli && \ cargo build --release --bin pyroscope-cli FROM scratch as final diff --git a/docker/Dockerfile.cli_builder b/docker/Dockerfile.cli_builder index 7bdabb6..c690c8f 100644 --- a/docker/Dockerfile.cli_builder +++ b/docker/Dockerfile.cli_builder @@ -23,12 +23,12 @@ ENV PATH=/root/.cargo/bin:$PATH ENV LIBUNWIND_VERSION=1.8.1 RUN apk add make gcc musl-dev RUN source ./.env && \ - wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && \ - tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && \ - cd libunwind-${LIBUNWIND_VERSION} && \ - ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation --prefix=/usr/local/musl/${RUST_TARGET} && \ - make -j16 && \ - make install + wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz && \ + tar -zxvf libunwind-${LIBUNWIND_VERSION}.tar.gz && \ + cd libunwind-${LIBUNWIND_VERSION} && \ + ./configure --disable-minidebuginfo --enable-ptrace --disable-tests --disable-documentation --prefix=/usr/local/musl/${RUST_TARGET} && \ + make -j16 && \ + make install RUN source ./.env && \ wget https://zlib.net/zlib-1.3.1.tar.gz && \