fix: Use cross to compile rocksdb

getdozer · Sep 26, 2023 · 9f01a22 · 9f01a22
1 parent 2532585
commit 9f01a22
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 69 deletions.
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -3,7 +3,6 @@ name: Dozer Coverage
 on:
   workflow_dispatch:
   push:
-    branches: [release, release-dev, release-test]
     tags:
       - "v*.*.*"
 

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -1,5 +1,6 @@
 name: Release
 on:
+  workflow_dispatch:
   push:
     branches: [release, release-dev, release-test, main]
     tags:
@@ -28,7 +29,7 @@ jobs:
       prerelease: ${{ steps.version.outputs.prerelease }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Generate a changelog
@@ -66,99 +67,62 @@ jobs:
 
   release-linux-aarch64:
     name: Release Linux binary for aarch64
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20-16-cores
     needs: prepare
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - build: aarch64
-            os: ubuntu-20-16-cores
-            target: aarch64-unknown-linux-gnu
-            linker: gcc-aarch64-linux-gnu
-            use-cross: true
-            file_name: dozer
-            asset_name: dozer-linux-aarch64
+    env:
+      CARGO_TARGET: aarch64-unknown-linux-gnu
+      DEB_NAME: dozer-linux-aarch64
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Install minimal stable with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          target: ${{ matrix.target }}
-          components: rustfmt, clippy
-      - name: APT update
-        run: |
-          sudo apt update
-      - name: Install target specific tools
-        if: matrix.use-cross
-        run: |
-          sudo apt-get install -y ${{ matrix.linker }}
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Rust cache
+        uses: swatinem/rust-cache@v2
 
-      - name: ⚡ Cache
-        uses: actions/cache@v3
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@master
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/.crates.toml
-            ~/.cargo/.crates2.json
-            ~/.cargo/.package-cache
-            ~/.cargo/registry/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-release-${{ hashFiles('Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-release-${{ hashFiles('Cargo.lock') }}
-            ${{ runner.os }}-cargo-release-
+          toolchain: stable
+          target: ${{ env.CARGO_TARGET }}
 
-      - name: Configure target aarch64 GNU
-        if: matrix.target == 'aarch64-unknown-linux-gnu'
-        run: |
-          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
-          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
-          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
-      - name: Cargo build
-        uses: actions-rs/cargo@v1
+      - name: Install cross
+        uses: baptiste0928/cargo-install@v1
         with:
-          command: build
-          use-cross: false
-          args: --release --target ${{ matrix.target }} --bin ${{ matrix.file_name }}
-      - name: List target output files
-        run: ls -lR ./target
+          crate: cross
+          cache-key: '${{ env.CARGO_TARGET }}'
+
+      - name: Build dozer
+        run: cross build --profile=release --target ${{ env.CARGO_TARGET }} --bin dozer
 
       - name: Install cargo-deb
-        run: cargo install cargo-deb
+        uses: baptiste0928/cargo-install@v1
+        with:
+          crate: cargo-deb
+          cache-key: '${{ env.CARGO_TARGET }}'
 
       - name: Compile deb file
-        run: cargo-deb -p dozer-cli --target ${{ matrix.target }} --output ./deb/${{matrix.asset_name}}.deb
+        run: cargo deb -p dozer-cli --target ${{ env.CARGO_TARGET }} --no-build --no-strip --output ./deb/${{ env.DEB_NAME }}.deb
 
       - name: Prepare release assets
         shell: bash
         run: |
           mkdir -p release
           cp {LICENSE,README.md,CHANGELOG.md} release/ 2> /dev/null || echo "Copy Failed...Ignoring.."
-          cp target/${{ matrix.target }}/release/${{matrix.file_name}} release/
+          cp target/${{ env.CARGO_TARGET }}/release/dozer release/
 
-          mv release/ ${{matrix.asset_name}}/
+          mv release/ ${{ env.DEB_NAME }}/
 
-          tar -czvf ${{matrix.asset_name}}.tar.gz ${{matrix.asset_name}}/
+          tar -czvf ${{ env.DEB_NAME }}.tar.gz ${{ env.DEB_NAME }}/
 
-          cp deb/${{matrix.asset_name}}.deb ${{matrix.asset_name}}/
+          cp deb/${{ env.DEB_NAME }}.deb ${{ env.DEB_NAME }}/
 
-          ls -l ${{matrix.asset_name}}
+          ls -l ${{ env.DEB_NAME }}
 
       - name: Upload the release
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{matrix.asset_name}}*
+          file: ${{ env.DEB_NAME }}*
           file_glob: true
           overwrite: true
           tag: ${{ needs.prepare.outputs.version }}
@@ -169,10 +133,10 @@ jobs:
       - name: Set env variables
         env:
           VERSION: ${{ needs.prepare.outputs.version }}
-          RELEASE_NAME: ${{matrix.asset_name}}.tar.gz
+          RELEASE_NAME: ${{ env.DEB_NAME }}.tar.gz
         run: |
           echo "RELEASE_NAME=${{env.RELEASE_NAME}}" >> $GITHUB_ENV
-          echo "DEB_NAME=${{matrix.asset_name}}.deb" >> $GITHUB_ENV
+          echo "DEB_NAME=${{ env.DEB_NAME }}.deb" >> $GITHUB_ENV
           echo "VERSION=${{env.VERSION}}" >> $GITHUB_ENV
           echo "ARTIFACT_URL=https://${{ env.BUCKET_NAME }}.s3.ap-southeast-1.amazonaws.com/${{ env.VERSION }}/${{ env.RELEASE_NAME }}" >> $GITHUB_ENV
 

diff --git a/Cross.toml b/Cross.toml
@@ -0,0 +1,5 @@
+[target.aarch64-unknown-linux-gnu]
+dockerfile = "./ci/Dockerfile.aarch64-unknown-linux-gnu"
+
+[target.x86_64-unknown-linux-gnu]
+dockerfile = "./ci/Dockerfile.x86_64-unknown-linux-gnu"
diff --git a/ci/Dockerfile.aarch64-unknown-linux-gnu b/ci/Dockerfile.aarch64-unknown-linux-gnu
@@ -0,0 +1,16 @@
+FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main@sha256:b4f5bf74812f9bb6516140d4b83d1f173c2d5ce0523f3e1c2253d99d851c734f
+
+ENV PKG_CONFIG_ALLOW_CROSS="true"
+
+RUN dpkg --add-architecture arm64 && \
+    apt-get update && \
+    apt-get install --assume-yes clang-8 libclang-8-dev binutils-aarch64-linux-gnu zlib1g-dev:arm64 unzip
+
+# INSTALL PROTOBUF
+ENV PROTOBUF_FILE_NAME=protoc-3.18.2-linux-x86_64.zip
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.18.2/${PROTOBUF_FILE_NAME}
+ENV PROTOC_DIR=/usr/local/protoc
+RUN unzip ${PROTOBUF_FILE_NAME} -d ${PROTOC_DIR}
+RUN chmod -R a+xr ${PROTOC_DIR}
+ENV PROTOC=${PROTOC_DIR}/bin/protoc
+RUN rm ${PROTOBUF_FILE_NAME}
diff --git a/ci/Dockerfile.x86_64-unknown-linux-gnu b/ci/Dockerfile.x86_64-unknown-linux-gnu
@@ -0,0 +1,14 @@
+FROM ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main@sha256:bf0cd3027befe882feb5a2b4040dc6dbdcb799b25c5338342a03163cea43da1b
+
+RUN apt-get update && \
+    apt-get install --assume-yes clang libclang-dev binutils-aarch64-linux-gnu unzip
+
+# INSTALL PROTOBUF
+ENV PROTOBUF_FILE_NAME=protoc-3.18.2-linux-x86_64.zip
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.18.2/${PROTOBUF_FILE_NAME}
+RUN unzip ${PROTOBUF_FILE_NAME} -d /usr/local/protoc
+ENV PROTOC_DIR=/usr/local/protoc
+RUN unzip ${PROTOBUF_FILE_NAME} -d ${PROTOC_DIR}
+RUN chmod -R a+xr ${PROTOC_DIR}
+ENV PROTOC=${PROTOC_DIR}/bin/protoc
+RUN rm ${PROTOBUF_FILE_NAME}
diff --git a/ci/README.md b/ci/README.md
@@ -0,0 +1,10 @@
+# Cross compilation
+
+We use [cross](https://github.com/cross-rs/cross) to work around [bug](https://github.com/rust-lang/rust-bindgen/issues/1229) in `bind-gen`.
+
+To test cross compilation locally:
+
+```bash
+cargo install cross
+cross build --target ${target} --bin dozer
+```
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,7 +3,6 @@ name: Dozer Coverage @@
     on:
       workflow_dispatch:
       push:
-        branches: [release, release-dev, release-test]
         tags:
           - "v*.*.*"
@@ Expand Down @@