diff --git a/.github/actions/colcon-build-docker/README.md b/.github/actions/colcon-build-docker/README.md new file mode 100644 index 0000000000000..2c98d9dec2b1f --- /dev/null +++ b/.github/actions/colcon-build-docker/README.md @@ -0,0 +1,43 @@ +# colcon-build + +This action runs `colcon build`. + +## Usage + +```yaml +jobs: + build: + runs-on: ubuntu-latest + container: ros:galactic + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + + - name: Build + if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + with: + rosdistro: galactic + target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + build-depends-repos: build_depends.repos +``` + +## Inputs + +| Name | Required | Description | +| ------------------- | -------- | --------------------------------------------------- | +| rosdistro | true | ROS distro. | +| target-packages | true | The target packages to build. | +| build-depends-repos | false | The `.repos` file that includes build dependencies. | +| cmake-build-type | false | The value for `CMAKE_BUILD_TYPE`. | +| token | false | The token for build dependencies. | + +## Outputs + +None. diff --git a/.github/actions/colcon-build-docker/action.yaml b/.github/actions/colcon-build-docker/action.yaml new file mode 100644 index 0000000000000..4ae9c573a3e6f --- /dev/null +++ b/.github/actions/colcon-build-docker/action.yaml @@ -0,0 +1,98 @@ +name: colcon-build-docker +description: "" + +inputs: + rosdistro: + description: "" + required: true + target-packages: + description: "" + required: true + build-depends-repos: + description: "" + required: false + cmake-build-type: + description: "" + required: false + default: Release + token: + description: "" + required: false + default: ${{ github.token }} + include-eol-distros: + description: "" + required: false + cache-build-artifacts: + description: "Caches the build and install folders for clang-tidy to use" + required: false + default: "false" + +runs: + using: composite + steps: + - name: Show target packages + run: | + echo "target packages: ${{ inputs.target-packages }}" + shell: bash + + - name: Install pip for rosdep + run: | + sudo apt-get -yqq update + sudo apt-get -yqq install python3-pip + shell: bash + + - name: Set git config + if: ${{ inputs.build-depends-repos != '' }} + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ inputs.token }} + + - name: Clone dependency packages + if: ${{ inputs.build-depends-repos != '' }} + run: | + mkdir -p dependency_ws + vcs import dependency_ws < ${{ inputs.build-depends-repos }} + shell: bash + + - name: Run rosdep install + run: | + package_paths=$(colcon list -p --packages-above-and-dependencies ${{ inputs.target-packages }} --base-paths . dependency_ws) + sudo apt-get -yqq update + if [ "${{ inputs.include-eol-distros }}" = "true" ]; then + rosdep update --include-eol-distros + else + rosdep update + fi + DEBIAN_FRONTEND=noninteractive rosdep install -yqq --from-paths ${package_paths} --ignore-src --rosdistro ${{ inputs.rosdistro }} + shell: bash + + - name: Set up colcon-mixin + run: | + colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml || true + colcon mixin update default + shell: bash + + - name: Build + run: | + . /opt/ros/${{ inputs.rosdistro }}/setup.sh + # Cannot use unshare inside a Docker container, so we just disable + # DNS lookups instead. + cat /etc/nsswitch.conf + sed -e 's#hosts:\(.*\)dns\(.*\)#hosts:\1\2#g' -i.bak /etc/nsswitch.conf + cat /etc/nsswitch.conf + colcon build --event-handlers console_cohesion+ \ + --packages-above-and-dependencies ${{ inputs.target-packages }} \ + --cmake-args -DCMAKE_BUILD_TYPE=${{ inputs.cmake-build-type }} \ + --mixin coverage-gcc coverage-pytest compile-commands + mv /etc/nsswitch.conf.bak /etc/nsswitch.conf + cat /etc/nsswitch.conf + shell: bash + + - name: Cache build artifacts + if: ${{ inputs.cache-build-artifacts == 'true' }} + uses: actions/cache@v4 + with: + path: | + ./build + ./install + key: build-${{ inputs.rosdistro }}-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cmake-build-type }}-${{ github.sha }} diff --git a/.github/actions/colcon-test-docker/README.md b/.github/actions/colcon-test-docker/README.md new file mode 100644 index 0000000000000..262cd993ec936 --- /dev/null +++ b/.github/actions/colcon-test-docker/README.md @@ -0,0 +1,86 @@ +# colcon-test + +This action runs `colcon test` with labels specified in a regex format. +Note that you need to build target packages before running this action. + +## Usage + +```yaml +jobs: + build: + runs-on: ubuntu-latest + container: ros:galactic + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + + - name: Build + if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + with: + rosdistro: galactic + target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + build-depends-repos: build_depends.repos + + test: + needs: build + runs-on: ubuntu-latest + container: ros:galactic + strategy: + matrix: + include: + - test-label: gtest + codecov-flags: gtest + - test-label: launch_test + codecov-flags: launch_test + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + + - name: Test + id: test + if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-test@v1 + with: + rosdistro: galactic + target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} + build-depends-repos: build_depends.repos + label-regex: ${{ matrix.test-label }} + + - name: Upload coverage to Codecov + if: ${{ steps.test.outputs.coverage-report-files != '' }} + uses: codecov/codecov-action@v2 + with: + files: ${{ steps.test.outputs.coverage-report-files }} + fail_ci_if_error: false + verbose: true + flags: ${{ matrix.codecov-flags }} +``` + +## Inputs + +| Name | Required | Description | +| ------------------- | -------- | --------------------------------------------------- | +| rosdistro | true | ROS distro. | +| target-packages | true | The target packages to test. | +| build-depends-repos | false | The `.repos` file that includes build dependencies. | +| label-regex | false | The regex pattern of test labels to be run. | +| token | false | The token for build dependencies. | + +## Outputs + +| Name | Description | +| --------------------- | -------------------------------- | +| coverage-report-files | Generated coverage report files. | diff --git a/.github/actions/colcon-test-docker/action.yaml b/.github/actions/colcon-test-docker/action.yaml new file mode 100644 index 0000000000000..7e0057780c400 --- /dev/null +++ b/.github/actions/colcon-test-docker/action.yaml @@ -0,0 +1,145 @@ +name: colcon-test +description: "" + +inputs: + rosdistro: + description: "" + required: true + target-packages: + description: "" + required: true + build-depends-repos: + description: "" + required: false + label-regex: + description: "" + required: false + default: .* # Run all tests + token: + description: "" + required: false + default: ${{ github.token }} + include-eol-distros: + description: "" + required: false +outputs: + coverage-report-files: + description: "" + value: ${{ steps.test.outputs.coverage-reports }} + +runs: + using: composite + steps: + - name: Show target packages + run: | + echo "target packages: ${{ inputs.target-packages }}" + shell: bash + + - name: Set up colcon-mixin + run: | + colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml || true + colcon mixin update default + shell: bash + + - name: Install packages for coverage commands + run: | + sudo apt-get -yqq update + sudo apt-get -yqq install \ + lcov \ + python3-pip \ + python3-colcon-coveragepy-result \ + python3-colcon-lcov-result \ + python3-pytest-cov + shell: bash + + - name: Set git config + if: ${{ inputs.build-depends-repos != '' }} + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ inputs.token }} + + - name: Clone dependency packages + if: ${{ inputs.build-depends-repos != '' }} + run: | + mkdir -p dependency_ws + vcs import dependency_ws < ${{ inputs.build-depends-repos }} + shell: bash + + - name: Run rosdep install + run: | + package_paths=$(colcon list -p --packages-above-and-dependencies ${{ inputs.target-packages }} --base-paths . dependency_ws) + sudo apt-get -yqq update + if [ "${{ inputs.include-eol-distros }}" = "true" ]; then + rosdep update --include-eol-distros + else + rosdep update + fi + DEBIAN_FRONTEND=noninteractive rosdep install -yqq --from-paths ${package_paths} --ignore-src --rosdistro ${{ inputs.rosdistro }} + shell: bash + + - name: Restore build files from cache + id: restore-build-files + uses: actions/cache@v4 + with: + path: | + ./build + ./install + key: build-${{ inputs.rosdistro }}-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cmake-build-type }}-${{ github.sha }} + + - name: Test + run: | + set -e + . /opt/ros/${{ inputs.rosdistro }}/setup.sh + + colcon lcov-result --zero-counters \ + --verbose \ + --packages-above ${{ inputs.target-packages }} + + colcon lcov-result --initial \ + --verbose \ + --packages-above ${{ inputs.target-packages }} + + colcon test --event-handlers console_cohesion+ \ + --mixin coverage-pytest \ + --packages-above ${{ inputs.target-packages }} \ + --ctest-args -L "${{ inputs.label-regex }}" \ + --executor sequential \ + --return-code-on-test-failure + + colcon lcov-result \ + --verbose \ + --packages-above ${{ inputs.target-packages }} || true + + colcon coveragepy-result \ + --verbose \ + --packages-above ${{ inputs.target-packages }} || true + shell: bash + + - name: Check if the lcov coverage file exists + id: lcov-coverage-file-existence + uses: autowarefoundation/autoware-github-actions/check-file-existence@v1 + with: + files: lcov/total_coverage.info + + - name: Check if the coveragepy coverage file exists + id: coveragepy-coverage-file-existence + uses: autowarefoundation/autoware-github-actions/check-file-existence@v1 + with: + files: coveragepy/.coverage + + - name: Generate output + id: test + run: | + coverage_files=() + if [[ "${{ steps.lcov-coverage-file-existence.outputs.exists }}" == "true" ]]; then + coverage_files+=("lcov/total_coverage.info") + fi + + if [[ "${{ steps.coveragepy-coverage-file-existence.outputs.exists}}" == "true" ]]; then + coverage_files+=("coveragepy/.coverage") + fi + + # Join coverage file names + coverage_files_str="$(IFS=,; echo "${coverage_files[*]}")" + echo "coverage-reports=$coverage_files_str" >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/actions/get-modified-packages-docker/README.md b/.github/actions/get-modified-packages-docker/README.md new file mode 100644 index 0000000000000..0e00193b47d9c --- /dev/null +++ b/.github/actions/get-modified-packages-docker/README.md @@ -0,0 +1,55 @@ +# get-modified-packages + +This action get the list of ROS packages modified in the pull request. + +## Usage + +### Basic + +```yaml +jobs: + get-modified-packages: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 +``` + +### Fetch only the PR branch and all PR commits + +```yaml +jobs: + get-modified-packages: + runs-on: ubuntu-latest + steps: + - name: Set PR fetch depth + run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" + + - name: Checkout PR branch and all PR commits + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.PR_FETCH_DEPTH }} + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 +``` + +## Inputs + +| Name | Required | Description | +| ----------- | -------- | ------------------------------------------------ | +| base-branch | false | The base branch to search for modified packages. | + +## Outputs + +| Name | Description | +| ----------------- | ------------------------------------------------------ | +| modified-packages | The list of ROS packages modified in the pull request. | diff --git a/.github/actions/get-modified-packages-docker/action.yaml b/.github/actions/get-modified-packages-docker/action.yaml new file mode 100644 index 0000000000000..46596bcd3a9c6 --- /dev/null +++ b/.github/actions/get-modified-packages-docker/action.yaml @@ -0,0 +1,46 @@ +name: get-modified-packages +description: "" + +inputs: + base-ref: + description: "" + required: false + default: ${{ github.base_ref }} + +outputs: + modified-packages: + description: "" + value: ${{ steps.get-modified-packages.outputs.modified-packages }} + +runs: + using: composite + steps: + - name: Workaround for https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + shell: bash + + - name: Fetch the base branch with enough history for a common merge-base commit + run: | + git fetch origin ${{ inputs.base-ref }} + shell: bash + + - name: printenv + run: printenv + shell: bash + + - name: Get modified packages + id: get-modified-packages + run: | + modified_packages=${{ DOCKER_RUN_NO_BASH }} \ + -v ${GITHUB_ACTION_PATH}:${GITHUB_ACTION_PATH} \ + /bin/bash -c \ + ${GITHUB_ACTION_PATH}/get-modified-packages.sh origin/${{ inputs.base-ref }} + + echo "modified-packages=$modified_packages" >> $GITHUB_OUTPUT + shell: bash + + - name: Show result + run: | + echo "modified-packages: ${{ steps.get-modified-packages.outputs.modified-packages }}" + shell: bash diff --git a/.github/actions/get-modified-packages-docker/get-modified-packages.sh b/.github/actions/get-modified-packages-docker/get-modified-packages.sh new file mode 100755 index 0000000000000..834115f5a621e --- /dev/null +++ b/.github/actions/get-modified-packages-docker/get-modified-packages.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Search for packages that have been modified from the base branch. +# Usage: get-modified-packages.sh + +# Parse arguments +args=() +while [ "${1-}" != "" ]; do + case "$1" in + *) + args+=("$1") + ;; + esac + shift +done + +# example: base_branch="origin/main" +base_branch="${args[0]}" + +# Check args +if [ "$base_branch" = "" ]; then + echo -e "\e[31mPlease input a valid base_branch as the 1st argument.\e[m" + exit 1 +fi + +function find_package_dir() { + [ "$1" == "" ] && return 1 + + target_dir=$(dirname "$1") + while true; do + # Output package name if package.xml found + if [ -f "$target_dir/package.xml" ]; then + if [ ! -f "$target_dir/COLCON_IGNORE" ]; then + echo "$target_dir" + return 0 + fi + fi + + # Exit if no parent found + parent_dir=$(dirname "$target_dir") + if [ "$parent_dir" = "$target_dir" ]; then + return 0 + fi + + # Move to parent dir + target_dir=$parent_dir + done + + return 1 +} + +# Find modified files from the base branch +if ! modified_files=$(git diff --name-only "$base_branch"...HEAD); then + echo -e "\e[31mFailed to determine modified files. Please check if the base branch exists and fetch depth is sufficient.\e[m" + exit 1 +fi + +# Find modified packages +modified_package_dirs=() +for modified_file in $modified_files; do + modified_package_dir=$(find_package_dir "$modified_file") + + if [ "$modified_package_dir" != "" ]; then + modified_package_dirs+=("$modified_package_dir") + fi +done + +# Get package names from paths +if [ "${#modified_package_dirs[@]}" != "0" ]; then + modified_packages=$(colcon list --names-only --paths "${modified_package_dirs[@]}") +fi + +# Output +# shellcheck disable=SC2086 +printf "%s " $modified_packages | sed 's/\s*$//' diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index e7ceba3de319b..3dc41685d02c5 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -11,6 +11,12 @@ on: env: CC: /usr/lib/ccache/gcc CXX: /usr/lib/ccache/g++ + DOCKER_RUN_NO_BASH: docker run --shm-size 1G \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ${{ matrix.container }}${{ matrix.container-suffix }} + DOCKER_RUN: ${{ DOCKER_RUN_NO_BASH }} \ + /bin/bash -c jobs: build-and-test-differential: @@ -27,14 +33,14 @@ jobs: container: ghcr.io/autowarefoundation/autoware:latest-prebuilt build-depends-repos: build_depends.repos steps: -# - name: Set PR fetch depth -# run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" -# -# - name: Checkout PR branch and all PR commits -# uses: actions/checkout@v4 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# fetch-depth: ${{ env.PR_FETCH_DEPTH }} + - name: Set PR fetch depth + run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" + + - name: Checkout PR branch and all PR commits + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.PR_FETCH_DEPTH }} - name: Show disk space before the tasks run: df -h @@ -46,7 +52,22 @@ jobs: run: printenv - name: tree - run: tree ../../ + run: tree $HOME + + - name: Remove exec_depend + uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 + + - name: Get modified packages + id: get-modified-packages + uses: ./.github/actions/get-modified-packages-docker + +# - name: docker run +# run: docker run --shm-size 1G \ +# -v ${{ github.workspace }}:/workspace \ +# -w /workspace \ +# ${{ matrix.container }}${{ matrix.container-suffix }} \ +# /bin/bash -c \ + # - name: Docker # run: |