chore(ci): Add integration test CI docker image with all the arrows (#…

…557) This PR adds an integration test run for every commit by caching a recent docker image build (weekly) containing the built versions of integration tests from the Arrow monorepo and arrow-rs. This should result in a CI job with a reasonable run time that provides high-level testing and type coverage. This is based on the arrow-rs version of the integration test job ( https://github.com/apache/arrow-rs/blob/6d4e2f2ceaf423031b0bc72f54c547dd77a0ddbb/.github/workflows/integration.yml ). --------- Co-authored-by: Jacob Wujciak-Jens <[email protected]>
apache · Jul 17, 2024 · bfa1bee · bfa1bee
1 parent 812428e
commit bfa1bee
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 15 deletions.
diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-name: Build Docker Images
+name: docker-build
 
 on:
   workflow_dispatch:
@@ -30,24 +30,29 @@ permissions:
   contents: read
   packages: write
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   build-docker:
-    name: "docker-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
+    name: "${{ matrix.config.service }}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
     runs-on: ${{ matrix.config.runs_on }}
     strategy:
       fail-fast: false
       matrix:
         config:
-          - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64" }
-          - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64" }
-          - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64" }
-          - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64" }
+          - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64", service: "verify" }
+          - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64", service: "verify" }
+          - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64", service: "verify" }
+          - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64", service: "verify" }
+          - { runs_on: "ubuntu-latest", arch: "amd64", platform: "integration", service: "integration" }
 
-          - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64" }
-          - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64" }
-          - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch: "arm64" }
+          - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64", service: "verify" }
+          - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64", service: "verify" }
+          - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch: "arm64", service: "verify" }
 
-          - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x" }
+          - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x", service: "verify" }
 
     env:
       DOCKER_CLIENT_TIMEOUT: 360
@@ -64,20 +69,20 @@ jobs:
           docker context create builders
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           endpoint: builders
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
         if: matrix.config.arch == 's390x'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Build
         env:
@@ -94,15 +99,15 @@ jobs:
           NANOARROW_PLATFORM: ${{ matrix.config.platform }}
           DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
         run: |
-          docker-compose push verify
+          docker-compose push ${{ matrix.config.service }}
 
   # Build some arch-agnostic images for non-verify docker bits
   build-docker-manifest:
     runs-on: ubuntu-latest
     needs: build-docker
     steps:
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.actor }}

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: integration
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - .github/workflows/integration.yaml
+      - src/nanoarrow/**
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run integration tests
+        run: |
+          echo "::group::Docker Pull"
+          docker compose run --rm -e GITHUB_ACTIONS integration
diff --git a/ci/docker/integration.dockerfile b/ci/docker/integration.dockerfile
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apache/arrow-dev:amd64-conda-integration
+
+ENV ARROW_USE_CCACHE=OFF \
+    ARROW_CPP_EXE_PATH=/build/cpp/debug \
+    ARROW_NANOARROW_PATH=/build/nanoarrow \
+    ARROW_RUST_EXE_PATH=/build/rust/debug \
+    BUILD_DOCS_CPP=OFF \
+    ARROW_INTEGRATION_CPP=ON \
+    ARROW_INTEGRATION_CSHARP=ON \
+    ARROW_INTEGRATION_GO=ON \
+    ARROW_INTEGRATION_JAVA=ON \
+    ARROW_INTEGRATION_JS=ON \
+    ARCHERY_INTEGRATION_WITH_NANOARROW="1" \
+    ARCHERY_INTEGRATION_WITH_RUST="1"
+
+# These are necessary because the github runner overrides $HOME
+# https://github.com/actions/runner/issues/863
+ENV RUSTUP_HOME=/root/.rustup
+ENV CARGO_HOME=/root/.cargo
+
+ENV ARROW_USE_CCACHE=OFF
+ENV ARROW_CPP_EXE_PATH=/build/cpp/debug
+ENV ARROW_NANOARROW_PATH=/build/nanoarrow
+ENV ARROW_RUST_EXE_PATH=/build/rust/debug
+ENV BUILD_DOCS_CPP=OFF
+
+# Clone the arrow monorepo
+RUN git clone https://github.com/apache/arrow.git /arrow-integration --recurse-submodules
+
+# Clone the arrow-rs repo
+RUN git clone https://github.com/apache/arrow-rs /arrow-integration/rust
+
+# Workaround: stable rust is not compatible with glibc provided by the
+# provided arrow docker image https://github.com/apache/arrow/issues/41637
+RUN cd /arrow-integration/rust && rustup override set 1.77
+
+# Build all the integrations except nanoarrow (since we'll do that ourselves on each run)
+RUN ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
+    conda run --no-capture-output \
+    /arrow-integration/ci/scripts/integration_arrow_build.sh \
+    /arrow-integration \
+    /build
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -46,3 +46,21 @@ services:
       - ./ci/scripts/build-docs.sh:/build-docs.sh
       - ${NANOARROW_DOCKER_SOURCE_DIR}:/nanoarrow
     command: "/bin/bash /build-docs.sh /nanoarrow"
+
+  integration:
+    # Based on an arrow-dev repo image that is amd64 only
+    image: ${REPO}:integration-amd64
+    build:
+      context: .
+      cache_from:
+        - ${REPO}:integration-amd64
+      dockerfile: ci/docker/integration.dockerfile
+    volumes:
+      - ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow
+    command:
+      ["echo '::group::Build nanoarrow' &&
+        conda run --no-capture-output /arrow-integration/ci/scripts/nanoarrow_build.sh /arrow-integration /build &&
+        echo '::endgroup::' &&
+        echo '::group::Run integration tests' &&
+        conda run --no-capture-output /arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build &&
+        echo '::endgroup::'"]