From 48aaf07c912ce1e073698de876c48acd23e5680d Mon Sep 17 00:00:00 2001 From: Ryohsuke Mitsudome Date: Wed, 6 Nov 2024 12:58:31 +0900 Subject: [PATCH 1/2] ci: add build-and-test-differential action Signed-off-by: Ryohsuke Mitsudome --- .../build-and-test-differential/action.yaml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/actions/build-and-test-differential/action.yaml diff --git a/.github/actions/build-and-test-differential/action.yaml b/.github/actions/build-and-test-differential/action.yaml new file mode 100644 index 00000000..89893e9f --- /dev/null +++ b/.github/actions/build-and-test-differential/action.yaml @@ -0,0 +1,111 @@ +name: build-and-test-differential +description: "" + +inputs: + rosdistro: + description: "" + required: true + container: + description: "" + required: true + container-suffix: + description: "" + required: true + runner: + description: "" + required: true + build-depends-repos: + description: "" + required: true + build-pre-command: + description: "" + required: true + codecov-token: + description: "" + required: true + +runs: + using: composite + steps: + - name: Show disk space before the tasks + run: df -h + shell: bash + + - name: Show machine specs + run: lscpu && free -h + shell: bash + + - name: Remove exec_depend + uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + + - name: Create ccache directory + run: | + mkdir -p ${CCACHE_DIR} + du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + - name: Attempt to restore ccache + uses: actions/cache/restore@v4 + with: + path: | + /root/.ccache + key: ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}-${{ github.event.pull_request.base.sha }} + restore-keys: | + ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}- + + - name: Show ccache stats before build and reset stats + run: | + du -sh ${CCACHE_DIR} && ccache -s + ccache --zero-stats + shell: bash + + - name: Export CUDA state as a variable for adding to cache key + run: | + build_type_cuda_state=nocuda + if [[ "${{ inputs.container-suffix }}" == "-cuda" ]]; then + build_type_cuda_state=cuda + fi + echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" + echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" + shell: bash + + - name: Build + if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + with: + rosdistro: ${{ inputs.rosdistro }} + target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + build-depends-repos: ${{ inputs.build-depends-repos }} + cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + build-pre-command: ${{ inputs.build-pre-command }} + + - name: Show ccache stats after build + run: du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + - name: Test + id: test + if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-test@v1 + with: + rosdistro: ${{ inputs.rosdistro }} + target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + build-depends-repos: ${{ inputs.build-depends-repos }} + + - name: Upload coverage to CodeCov + if: ${{ steps.test.outputs.coverage-report-files != '' }} + uses: codecov/codecov-action@v4 + with: + files: ${{ steps.test.outputs.coverage-report-files }} + fail_ci_if_error: false + verbose: true + flags: differential + token: ${{ inputs.codecov-token }} + + - name: Show disk space after the tasks + run: df -h + shell: bash From d2be15e98479b8fa02d048f2cf4c015c8c01a561 Mon Sep 17 00:00:00 2001 From: Ryohsuke Mitsudome Date: Wed, 6 Nov 2024 12:59:02 +0900 Subject: [PATCH 2/2] ci: update build-and-test-differential workflow Signed-off-by: Ryohsuke Mitsudome --- .../build-and-test-differential.yaml | 123 +++++++++++------- 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index 5ce94179..cda16666 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -2,68 +2,93 @@ name: build-and-test-differential on: pull_request: + types: + - opened + - synchronize + - reopened + - labeled + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +env: + CC: /usr/lib/ccache/gcc + CXX: /usr/lib/ccache/g++ jobs: + make-sure-label-is-present: + uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1 + with: + label: tag:run-build-and-test-differential + + make-sure-require-cuda-label-is-present: + uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1 + with: + label: tag:require-cuda-build-and-test + build-and-test-differential: - runs-on: ubuntu-latest - container: ${{ matrix.container }} + needs: [make-sure-label-is-present, make-sure-require-cuda-label-is-present] + if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false matrix: rosdistro: - humble + container-suffix: + - "" + - -cuda include: - rosdistro: humble - container: ros:humble + container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe build-depends-repos: build_depends.repos + - container-suffix: -cuda + runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large + build-pre-command: taskset --cpu-list 0-6 + - container-suffix: "" + runner: ubuntu-latest + build-pre-command: "" steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Remove exec_depend - uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 - - - name: Get modified packages - id: get-modified-packages - uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + - name: Set PR fetch depth + run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" + shell: bash - - name: Build - if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} - uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + - name: Checkout PR branch and all PR commits + uses: actions/checkout@v4 with: - rosdistro: ${{ matrix.rosdistro }} - target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} - build-depends-repos: ${{ matrix.build-depends-repos }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.PR_FETCH_DEPTH }} - - name: Test - id: test - if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} - uses: autowarefoundation/autoware-github-actions/colcon-test@v1 + - name: Run build-and-test-differential action + if: ${{ !(matrix.container-suffix == '-cuda') || needs.make-sure-require-cuda-label-is-present.outputs.result == 'true' }} + uses: ./.github/actions/build-and-test-differential with: rosdistro: ${{ matrix.rosdistro }} - target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + container: ${{ matrix.container }} + container-suffix: ${{ matrix.container-suffix }} + runner: ${{ matrix.runner }} build-depends-repos: ${{ matrix.build-depends-repos }} - - - name: Upload coverage to CodeCov - if: ${{ steps.test.outputs.coverage-report-files != '' }} - uses: codecov/codecov-action@v4 - with: - files: ${{ steps.test.outputs.coverage-report-files }} - fail_ci_if_error: false - verbose: true - flags: differential + build-pre-command: ${{ matrix.build-pre-command }} + codecov-token: ${{ secrets.CODECOV_TOKEN }} clang-tidy-differential: - runs-on: ubuntu-latest - container: ros:humble needs: build-and-test-differential + runs-on: ubuntu-22.04 + container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe-cuda steps: - - name: Check out repository + - name: Set PR fetch depth + run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" + + - name: Checkout PR branch and all PR commits uses: actions/checkout@v4 with: - fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.PR_FETCH_DEPTH }} + + - name: Show disk space before the tasks + run: df -h - name: Remove exec_depend uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 @@ -72,20 +97,22 @@ jobs: id: get-modified-packages uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 - - name: Get modified files - id: get-modified-files - uses: tj-actions/changed-files@v35 - with: - files: | - **/*.cpp - **/*.hpp + - name: Get changed files (existing files only) + id: get-changed-files + run: | + echo "changed-files=$(git diff --name-only "origin/${{ github.base_ref }}"...HEAD | grep -E '\.(cpp|hpp)$' | while read -r file; do [ -e "$file" ] && echo -n "$file "; done)" >> $GITHUB_OUTPUT + shell: bash - name: Run clang-tidy - if: ${{ steps.get-modified-files.outputs.all_changed_files != '' }} + if: ${{ steps.get-changed-files.outputs.changed-files != '' }} uses: autowarefoundation/autoware-github-actions/clang-tidy@v1 with: rosdistro: humble target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} - target-files: ${{ steps.get-modified-files.outputs.all_changed_files }} - clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy + target-files: ${{ steps.get-changed-files.outputs.changed-files }} + clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy-ci build-depends-repos: build_depends.repos + cache-key-element: cuda + + - name: Show disk space after the tasks + run: df -h