Merge branch 'main' into update-json-schema-check

autowarefoundation · Nov 13, 2024 · 212eebb · 212eebb
2 parents 8084e14 + b0c6686
commit 212eebb
Show file tree

Hide file tree

Showing 3 changed files with 188 additions and 50 deletions.
diff --git a/.github/actions/build-and-test-differential/action.yaml b/.github/actions/build-and-test-differential/action.yaml
@@ -0,0 +1,111 @@
+name: build-and-test-differential
+description: ""
+
+inputs:
+  rosdistro:
+    description: ""
+    required: true
+  container:
+    description: ""
+    required: true
+  container-suffix:
+    description: ""
+    required: true
+  runner:
+    description: ""
+    required: true
+  build-depends-repos:
+    description: ""
+    required: true
+  build-pre-command:
+    description: ""
+    required: true
+  codecov-token:
+    description: ""
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Show disk space before the tasks
+      run: df -h
+      shell: bash
+
+    - name: Show machine specs
+      run: lscpu && free -h
+      shell: bash
+
+    - name: Remove exec_depend
+      uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
+
+    - name: Get modified packages
+      id: get-modified-packages
+      uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1
+
+    - name: Create ccache directory
+      run: |
+        mkdir -p ${CCACHE_DIR}
+        du -sh ${CCACHE_DIR} && ccache -s
+      shell: bash
+
+    - name: Attempt to restore ccache
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          /root/.ccache
+        key: ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}-${{ github.event.pull_request.base.sha }}
+        restore-keys: |
+          ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}-
+
+    - name: Show ccache stats before build and reset stats
+      run: |
+        du -sh ${CCACHE_DIR} && ccache -s
+        ccache --zero-stats
+      shell: bash
+
+    - name: Export CUDA state as a variable for adding to cache key
+      run: |
+        build_type_cuda_state=nocuda
+        if [[ "${{ inputs.container-suffix }}" == "-cuda" ]]; then
+          build_type_cuda_state=cuda
+        fi
+        echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}"
+        echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state"
+      shell: bash
+
+    - name: Build
+      if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }}
+      uses: autowarefoundation/autoware-github-actions/colcon-build@v1
+      with:
+        rosdistro: ${{ inputs.rosdistro }}
+        target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
+        build-depends-repos: ${{ inputs.build-depends-repos }}
+        cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }}
+        build-pre-command: ${{ inputs.build-pre-command }}
+
+    - name: Show ccache stats after build
+      run: du -sh ${CCACHE_DIR} && ccache -s
+      shell: bash
+
+    - name: Test
+      id: test
+      if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }}
+      uses: autowarefoundation/autoware-github-actions/colcon-test@v1
+      with:
+        rosdistro: ${{ inputs.rosdistro }}
+        target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
+        build-depends-repos: ${{ inputs.build-depends-repos }}
+
+    - name: Upload coverage to CodeCov
+      if: ${{ steps.test.outputs.coverage-report-files != '' }}
+      uses: codecov/codecov-action@v4
+      with:
+        files: ${{ steps.test.outputs.coverage-report-files }}
+        fail_ci_if_error: false
+        verbose: true
+        flags: differential
+        token: ${{ inputs.codecov-token }}
+
+    - name: Show disk space after the tasks
+      run: df -h
+      shell: bash
diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml
@@ -2,68 +2,93 @@ name: build-and-test-differential
 
 on:
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CC: /usr/lib/ccache/gcc
+  CXX: /usr/lib/ccache/g++
 
 jobs:
+  make-sure-label-is-present:
+    uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1
+    with:
+      label: tag:run-build-and-test-differential
+
+  make-sure-require-cuda-label-is-present:
+    uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1
+    with:
+      label: tag:require-cuda-build-and-test
+
   build-and-test-differential:
-    runs-on: ubuntu-latest
-    container: ${{ matrix.container }}
+    needs: [make-sure-label-is-present, make-sure-require-cuda-label-is-present]
+    if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }}
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}${{ matrix.container-suffix }}
     strategy:
       fail-fast: false
       matrix:
         rosdistro:
           - humble
+        container-suffix:
+          - ""
+          - -cuda
         include:
           - rosdistro: humble
-            container: ros:humble
+            container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe
             build-depends-repos: build_depends.repos
+          - container-suffix: -cuda
+            runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large
+            build-pre-command: taskset --cpu-list 0-6
+          - container-suffix: ""
+            runner: ubuntu-latest
+            build-pre-command: ""
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Remove exec_depend
-        uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
-
-      - name: Get modified packages
-        id: get-modified-packages
-        uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1
+      - name: Set PR fetch depth
+        run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+        shell: bash
 
-      - name: Build
-        if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }}
-        uses: autowarefoundation/autoware-github-actions/colcon-build@v1
+      - name: Checkout PR branch and all PR commits
+        uses: actions/checkout@v4
         with:
-          rosdistro: ${{ matrix.rosdistro }}
-          target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
-          build-depends-repos: ${{ matrix.build-depends-repos }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: ${{ env.PR_FETCH_DEPTH }}
 
-      - name: Test
-        id: test
-        if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }}
-        uses: autowarefoundation/autoware-github-actions/colcon-test@v1
+      - name: Run build-and-test-differential action
+        if: ${{ !(matrix.container-suffix == '-cuda') || needs.make-sure-require-cuda-label-is-present.outputs.result == 'true' }}
+        uses: ./.github/actions/build-and-test-differential
         with:
           rosdistro: ${{ matrix.rosdistro }}
-          target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
+          container: ${{ matrix.container }}
+          container-suffix: ${{ matrix.container-suffix }}
+          runner: ${{ matrix.runner }}
           build-depends-repos: ${{ matrix.build-depends-repos }}
-
-      - name: Upload coverage to CodeCov
-        if: ${{ steps.test.outputs.coverage-report-files != '' }}
-        uses: codecov/codecov-action@v4
-        with:
-          files: ${{ steps.test.outputs.coverage-report-files }}
-          fail_ci_if_error: false
-          verbose: true
-          flags: differential
+          build-pre-command: ${{ matrix.build-pre-command }}
+          codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
   clang-tidy-differential:
-    runs-on: ubuntu-latest
-    container: ros:humble
     needs: build-and-test-differential
+    runs-on: ubuntu-22.04
+    container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe-cuda
     steps:
-      - name: Check out repository
+      - name: Set PR fetch depth
+        run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+
+      - name: Checkout PR branch and all PR commits
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: ${{ env.PR_FETCH_DEPTH }}
+
+      - name: Show disk space before the tasks
+        run: df -h
 
       - name: Remove exec_depend
         uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1
@@ -72,20 +97,22 @@ jobs:
         id: get-modified-packages
         uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1
 
-      - name: Get modified files
-        id: get-modified-files
-        uses: tj-actions/changed-files@v35
-        with:
-          files: |
-            **/*.cpp
-            **/*.hpp
+      - name: Get changed files (existing files only)
+        id: get-changed-files
+        run: |
+          echo "changed-files=$(git diff --name-only "origin/${{ github.base_ref }}"...HEAD | grep -E '\.(cpp|hpp)$' | while read -r file; do [ -e "$file" ] && echo -n "$file "; done)" >> $GITHUB_OUTPUT
+        shell: bash
 
       - name: Run clang-tidy
-        if: ${{ steps.get-modified-files.outputs.all_changed_files != '' }}
+        if: ${{ steps.get-changed-files.outputs.changed-files != '' }}
         uses: autowarefoundation/autoware-github-actions/clang-tidy@v1
         with:
           rosdistro: humble
           target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}
-          target-files: ${{ steps.get-modified-files.outputs.all_changed_files }}
-          clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy
+          target-files: ${{ steps.get-changed-files.outputs.changed-files }}
+          clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy-ci
           build-depends-repos: build_depends.repos
+          cache-key-element: cuda
+
+      - name: Show disk space after the tasks
+        run: df -h
diff --git a/build_depends.repos b/build_depends.repos
@@ -1,8 +1,8 @@
 repositories:
   autoware_msgs:
     type: git
-    url: https://github.com/tier4/autoware_auto_msgs.git # TODO(Tier IV): Move to autowarefoundation/autoware_msgs
-    version: tier4/main
+    url: https://github.com/autowarefoundation/autoware_msgs.git
+    version: main
   autoware_common:
     type: git
     url: https://github.com/autowarefoundation/autoware_common.git