From 0dd842cb823c6c8d93ece70d814d91f7eff7f779 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 30 Apr 2024 16:03:48 +0300 Subject: [PATCH 01/46] add modular containers Signed-off-by: Oguz Ozturk --- .devcontainer/base/devcontainer.json | 2 +- .devcontainer/cuda/devcontainer.json | 2 +- .../docker-build-and-push-openadk/action.yaml | 201 ++++++++++++++++++ .../actions/docker-build-and-push/action.yaml | 33 +-- .github/workflows/build-main-self-hosted.yaml | 4 +- .github/workflows/build-main.yaml | 4 +- ...ocker-build-and-push-main-self-hosted.yaml | 22 +- .../workflows/docker-build-and-push-main.yaml | 22 +- .github/workflows/update-docker-manifest.yaml | 5 + docker/autoware-openadk/build.sh | 123 +++++++++++ docker/autoware-openadk/docker-bake.hcl | 40 ++-- .../modules/planning-control/Dockerfile | 36 ++++ .../planning_control.launch.xml | 69 ++++++ .../modules/simulator/Dockerfile | 45 ++++ .../modules/visualizer/Dockerfile | 33 +++ .../test/run-test-scenario.sh | 19 ++ .../test/test-scenario.docker-compose.yaml | 38 ++++ .../{autoware-openadk => autoware}/Dockerfile | 6 +- docker/{ => autoware}/build.sh | 10 +- docker/autoware/docker-bake.hcl | 27 +++ .../etc/.bash_aliases | 0 docker/autoware/etc/ros_entrypoint.sh | 32 +++ docker/{ => autoware}/run.sh | 6 +- 23 files changed, 717 insertions(+), 62 deletions(-) create mode 100644 .github/actions/docker-build-and-push-openadk/action.yaml create mode 100755 docker/autoware-openadk/build.sh create mode 100644 docker/autoware-openadk/modules/planning-control/Dockerfile create mode 100644 docker/autoware-openadk/modules/planning-control/planning_control.launch.xml create mode 100644 docker/autoware-openadk/modules/simulator/Dockerfile create mode 100644 docker/autoware-openadk/modules/visualizer/Dockerfile create mode 100755 docker/autoware-openadk/test/run-test-scenario.sh create mode 100644 docker/autoware-openadk/test/test-scenario.docker-compose.yaml rename docker/{autoware-openadk => autoware}/Dockerfile (96%) rename docker/{ => autoware}/build.sh (88%) mode change 100755 => 100644 create mode 100644 docker/autoware/docker-bake.hcl rename docker/{autoware-openadk => autoware}/etc/.bash_aliases (100%) create mode 100644 docker/autoware/etc/ros_entrypoint.sh rename docker/{ => autoware}/run.sh (96%) diff --git a/.devcontainer/base/devcontainer.json b/.devcontainer/base/devcontainer.json index 939cc741f5e..f3b5ffe9984 100644 --- a/.devcontainer/base/devcontainer.json +++ b/.devcontainer/base/devcontainer.json @@ -2,7 +2,7 @@ "name": "Autoware", "build": { "dockerfile": "../Dockerfile", - "args": { "BASE_IMAGE": "ghcr.io/autowarefoundation/autoware-openadk:latest-devel" } + "args": { "BASE_IMAGE": "ghcr.io/autowarefoundation/autoware:latest-devel" } }, "remoteUser": "autoware", "runArgs": [ diff --git a/.devcontainer/cuda/devcontainer.json b/.devcontainer/cuda/devcontainer.json index bc0efde100f..bb0b6ad849e 100644 --- a/.devcontainer/cuda/devcontainer.json +++ b/.devcontainer/cuda/devcontainer.json @@ -2,7 +2,7 @@ "name": "Autoware-cuda", "build": { "dockerfile": "../Dockerfile", - "args": { "BASE_IMAGE": "ghcr.io/autowarefoundation/autoware-openadk:latest-devel-cuda" } + "args": { "BASE_IMAGE": "ghcr.io/autowarefoundation/autoware:latest-devel-cuda" } }, "remoteUser": "autoware", "hostRequirements": { diff --git a/.github/actions/docker-build-and-push-openadk/action.yaml b/.github/actions/docker-build-and-push-openadk/action.yaml new file mode 100644 index 00000000000..69e9517cd46 --- /dev/null +++ b/.github/actions/docker-build-and-push-openadk/action.yaml @@ -0,0 +1,201 @@ +name: docker-build-and-push-openadk +description: "" + +inputs: + bake-target: + description: "" + required: true + build-args: + description: "" + required: false + tag-prefix: + description: "" + required: false + tag-suffix: + description: "" + required: false + tag-arch: + description: "" + required: false + allow-push: + description: "" + default: "true" + required: false + +runs: + using: composite + steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Install jq + run: | + sudo apt-get -y update + sudo apt-get -y install jq + shell: bash + + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref_type }}" == "tag" ]; then + tags+=("$(echo "${{ github.ref }}" | sed -E 's/.*([vV][0-9]+\.[0-9]+\.[0-9]+).*/\1/')") + fi + + tags+=("{{date 'YYYYMMDD'}}") + tags+=("latest") + tags+=("latest-${{ inputs.tag-prefix }}") + + # Output multiline strings: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "tags<<$EOF" >> $GITHUB_OUTPUT + echo "$(printf "%s\n" "${tags[@]}")" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + shell: bash + + # openadk modular images meta + - name: Docker meta for openadk:base + id: meta-base + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base-${{ inputs.tag-arch }} + + - name: Docker meta for openadk:planning-control + id: meta-planning-control + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-planning-control + flavor: | + latest=false + suffix=-planning-control-${{ inputs.tag-arch }} + + - name: Docker meta for openadk:visualizer + id: meta-visualizer + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-visualizer + flavor: | + latest=false + suffix=-visualizer-${{ inputs.tag-arch }} + + - name: Docker meta for openadk:simulator + id: meta-simulator + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-simulator + flavor: | + latest=false + suffix=-simulator-${{ inputs.tag-arch }} + + - name: Login to GitHub Container Registry + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and Push to GitHub Container Registry + if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || ( github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'registry') }} + uses: docker/bake-action@v3 + with: + push: ${{ inputs.allow-push == 'true' }} + files: | + docker/${{ inputs.bake-target }}/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-planning-control.outputs.bake-file }} + ${{ steps.meta-visualizer.outputs.bake-file }} + ${{ steps.meta-simulator.outputs.bake-file }} + provenance: false + targets: | + base + set: | + ${{ inputs.build-args }} + + - name: Build and Save Artifacts + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + uses: docker/bake-action@v3 + with: + push: false + files: | + docker/${{ inputs.bake-target }}/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} + ${{ steps.meta-planning-control.outputs.bake-file }} + ${{ steps.meta-visualizer.outputs.bake-file }} + ${{ steps.meta-simulator.outputs.bake-file }} + provenance: false + set: | + ${{ inputs.build-args }} + base.output=type=docker,dest=/tmp/base.tar + planning-control.output=type=docker,dest=/tmp/planning-control.tar + visualizer.output=type=docker,dest=/tmp/visualizer.tar + simulator.output=type=docker,dest=/tmp/simulator.tar + + - name: Upload Artifact - base + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: artifact-upload-step-base + uses: actions/upload-artifact@v4 + with: + name: base-image${{ inputs.tag-suffix }} + path: /tmp/base.tar + retention-days: 7 + compression-level: 6 + overwrite: true + if-no-files-found: error + + - name: Upload Artifact - planning-control + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: artifact-upload-step-planning-control + uses: actions/upload-artifact@v4 + with: + name: planning-control-image${{ inputs.tag-suffix }} + path: /tmp/planning-control.tar + retention-days: 7 + compression-level: 6 + overwrite: true + if-no-files-found: error + + - name: Upload Artifact - visualizer + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: artifact-upload-step-visualizer + uses: actions/upload-artifact@v4 + with: + name: visualizer-image${{ inputs.tag-suffix }} + path: /tmp/visualizer.tar + retention-days: 7 + compression-level: 6 + overwrite: true + if-no-files-found: error + + - name: Upload Artifact - simulator + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: artifact-upload-step-simulator + uses: actions/upload-artifact@v4 + with: + name: simulator-image${{ inputs.tag-suffix }} + path: /tmp/simulator.tar + retention-days: 7 + compression-level: 6 + overwrite: true + if-no-files-found: error + + - name: Output artifact URLs + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: output-artifact-urls + run: | + echo 'base URL ${{ steps.artifact-upload-step-base.outputs.artifact-url }}' + echo 'planning-control URL ${{ steps.artifact-upload-step-planning-control.outputs.artifact-url }}' + echo 'visualizer URL ${{ steps.artifact-upload-step-visualizer.outputs.artifact-url }}' + echo 'simulator URL ${{ steps.artifact-upload-step-simulator.outputs.artifact-url }}' + shell: bash diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 53a95eb8fe7..faa5cb2104d 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -14,6 +14,9 @@ inputs: tag-suffix: description: "" required: false + tag-arch: + description: "" + required: false allow-push: description: "" default: "true" @@ -59,7 +62,7 @@ runs: bake-target: docker-metadata-action-prebuilt flavor: | latest=false - suffix=-prebuilt${{ inputs.tag-suffix }} + suffix=-prebuilt${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} - name: Docker meta for devel id: meta-devel @@ -70,10 +73,9 @@ runs: bake-target: docker-metadata-action-devel flavor: | latest=false - suffix=-devel${{ inputs.tag-suffix }} + suffix=-devel${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} - name: Docker meta for runtime - if: ${{ github.event_name == 'workflow_dispatch' }} || ${{ (github.event_name == 'push' && github.ref_type == 'tag') }} id: meta-runtime uses: docker/metadata-action@v4 with: @@ -82,7 +84,7 @@ runs: bake-target: docker-metadata-action-runtime flavor: | latest=${{ github.event_name == 'push' && github.ref_type == 'tag' }} - suffix=-runtime${{ inputs.tag-suffix }} + suffix=-runtime${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} @@ -92,8 +94,8 @@ runs: username: ${{ github.repository_owner }} password: ${{ github.token }} - - name: Build and Push - prebuilt and devel - if: ${{ (github.event_name == 'push' && github.ref_type == 'branch') || github.event_name == 'schedule' }} + - name: Build and Push to GitHub Container Registry + if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || ( github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'registry') }} uses: docker/bake-action@v3 with: push: ${{ inputs.allow-push == 'true' }} @@ -101,22 +103,6 @@ runs: docker/${{ inputs.bake-target }}/docker-bake.hcl ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-devel.outputs.bake-file }} - targets: | - prebuilt - devel - provenance: false - set: | - ${{ inputs.build-args }} - - - name: Build and Publish to GitHub Container Registry - if: ${{ ( github.event_name == 'push' && github.ref_type == 'tag' ) || ( github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'registry') }} - uses: docker/bake-action@v3 - with: - push: true - files: | - docker/${{ inputs.bake-target }}/docker-bake.hcl - ${{ steps.meta-devel.outputs.bake-file }} - ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} provenance: false set: | @@ -129,8 +115,8 @@ runs: push: false files: | docker/${{ inputs.bake-target }}/docker-bake.hcl - ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-prebuilt.outputs.bake-file }} + ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} provenance: false set: | @@ -194,6 +180,7 @@ runs: shell: bash - name: Output artifact URLs + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: output-artifact-urls run: | echo 'prebuilt URL ${{ steps.artifact-upload-step-prebuilt.outputs.artifact-url }}' diff --git a/.github/workflows/build-main-self-hosted.yaml b/.github/workflows/build-main-self-hosted.yaml index 9c40bcfd5b2..86a0bd3811a 100644 --- a/.github/workflows/build-main-self-hosted.yaml +++ b/.github/workflows/build-main-self-hosted.yaml @@ -48,9 +48,9 @@ jobs: uses: ./.github/actions/free-disk-space - name: Build 'autoware-universe' - uses: ./.github/actions/docker-build-and-push + uses: ./.github/actions/docker-build-push with: - bake-target: autoware-openadk + bake-target: autoware build-args: | *.platform=linux/arm64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 1fff60ce250..e49c4a06acf 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -43,9 +43,9 @@ jobs: uses: ./.github/actions/free-disk-space - name: Build 'autoware-universe' - uses: ./.github/actions/docker-build-and-push + uses: ./.github/actions/docker-build-push with: - bake-target: autoware-openadk + bake-target: autoware build-args: | *.platform=linux/amd64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index ff5ae7a16c6..3523ce4ad16 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -75,10 +75,10 @@ jobs: mkdir src vcs import src < autoware.repos - - name: Build 'autoware-openadk' + - name: Build 'Autoware' uses: ./.github/actions/docker-build-and-push with: - bake-target: autoware-openadk + bake-target: autoware build-args: | *.platform=linux/arm64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} @@ -87,7 +87,23 @@ jobs: *.args.LIB_DIR=${{ matrix.lib_dir }} *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-arm64 + tag-suffix: ${{ matrix.additional-tag-suffix }} + tag-arch: arm64 + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + allow-push: true + + - name: Build 'Openadk' + uses: ./.github/actions/docker-build-and-push-openadk + with: + bake-target: openadk + build-args: | + *.platform=linux/arm64 + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + tag-suffix: ${{ matrix.additional-tag-suffix }} + tag-arch: arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index ee132df8666..a48be674ceb 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -70,8 +70,23 @@ jobs: mkdir src vcs import src < autoware.repos - - name: Build 'autoware-openadk' - uses: ./.github/actions/docker-build-and-push + - name: Build 'Autoware' + uses: ./.github/actions/docker-build-push + with: + bake-target: autoware + build-args: | + *.platform=linux/amd64 + *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} + *.args.SETUP_ARGS=${{ matrix.setup-args }} + *.args.LIB_DIR=${{ matrix.lib_dir }} + tag-suffix: ${{ matrix.additional-tag-suffix }} + tag-arch: amd64 + tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + allow-push: true + + - name: Build 'Openadk' + uses: ./.github/actions/docker-build-and-push-openadk with: bake-target: autoware-openadk build-args: | @@ -82,7 +97,8 @@ jobs: *.args.LIB_DIR=${{ matrix.lib_dir }} *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max - tag-suffix: ${{ matrix.additional-tag-suffix }}-amd64 + tag-suffix: ${{ matrix.additional-tag-suffix }} + tag-arch: amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml index 7c63a54dcb5..a3b4cbf70bf 100644 --- a/.github/workflows/update-docker-manifest.yaml +++ b/.github/workflows/update-docker-manifest.yaml @@ -12,6 +12,11 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Combine multi arch images for 'autoware' + uses: ./.github/actions/combine-multi-arch-images + with: + package-name: autoware + - name: Combine multi arch images for 'autoware-openadk' uses: ./.github/actions/combine-multi-arch-images with: diff --git a/docker/autoware-openadk/build.sh b/docker/autoware-openadk/build.sh new file mode 100755 index 00000000000..58db2e6731c --- /dev/null +++ b/docker/autoware-openadk/build.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -e + +# Function to print help message +print_help() { + echo "Usage: build.sh [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -h Display this help message" + echo " --no-cuda Disable CUDA support" + echo " --platform Specify the platform (default: current platform)" + echo "" + echo "Note: The --platform option should be one of 'linux/amd64' or 'linux/arm64'." +} + +SCRIPT_DIR=$(readlink -f "$(dirname "$0")") +WORKSPACE_ROOT="$SCRIPT_DIR/../.." + +# Parse arguments +parse_arguments() { + while [ "$1" != "" ]; do + case "$1" in + --help | -h) + print_help + exit 1 + ;; + --no-cuda) + option_no_cuda=true + ;; + --platform) + option_platform="$2" + shift + ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; + esac + shift + done +} + +# Set platform +set_platform() { + if [ -n "$option_platform" ]; then + platform="$option_platform" + else + platform="linux/amd64" + if [ "$(uname -m)" = "aarch64" ]; then + platform="linux/arm64" + fi + fi +} + +# Set arch lib dir +set_arch_lib_dir() { + if [ "$platform" = "linux/arm64" ]; then + lib_dir="aarch64" + elif [ "$platform" = "linux/amd64" ]; then + lib_dir="x86_64" + else + echo "Unsupported platform: $platform" + exit 1 + fi +} + +# Load env +load_env() { + source "$WORKSPACE_ROOT/amd64.env" + if [ "$platform" = "linux/arm64" ]; then + source "$WORKSPACE_ROOT/arm64.env" + fi +} + +# Set CUDA options +set_cuda_options() { + if [ "$option_no_cuda" = "true" ]; then + setup_args="--no-nvidia" + image_name_suffix="" + else + image_name_suffix="-cuda" + fi +} + +# Build images +build_images() { + # https://github.com/docker/buildx/issues/484 + export BUILDKIT_STEP_LOG_MAX_SIZE=10000000 + + echo "Building images for platform: $platform" + echo "ROS distro: $rosdistro" + echo "Base image: $base_image" + echo "Setup args: $setup_args" + echo "Lib dir: $lib_dir" + echo "Image name suffix: $image_name_suffix" + echo "Targets: ${targets[*]}" + + set -x + docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/docker-bake.hcl" \ + --set "*.context=$WORKSPACE_ROOT" \ + --set "*.ssh=default" \ + --set "*.platform=$platform" \ + --set "*.args.ROS_DISTRO=$rosdistro" \ + --set "*.args.BASE_IMAGE=$base_image" \ + --set "*.args.SETUP_ARGS=$setup_args" \ + --set "*.args.LIB_DIR=$lib_dir" \ + --set "base.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-base" \ + --set "planning-control.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-planning-control" \ + --set "visualizer.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-visualizer" \ + --set "simulator.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-simulator" \ + "${targets[@]}" + set +x +} + +# Main script execution +parse_arguments "$@" +set_cuda_options +set_platform +set_arch_lib_dir +load_env +build_images diff --git a/docker/autoware-openadk/docker-bake.hcl b/docker/autoware-openadk/docker-bake.hcl index d472a0df008..175d5ee80f5 100644 --- a/docker/autoware-openadk/docker-bake.hcl +++ b/docker/autoware-openadk/docker-bake.hcl @@ -1,26 +1,34 @@ group "default" { - targets = ["prebuilt", "devel", "runtime"] + targets = ["base", "planning-control", "simulator", "visualizer"] } // For docker/metadata-action -target "docker-metadata-action-prebuilt" {} -target "docker-metadata-action-devel" {} -target "docker-metadata-action-runtime" {} +target "docker-metadata-action-base" {} +target "docker-metadata-action-planning-control" {} +target "docker-metadata-action-visualizer" {} +target "docker-metadata-action-simulator" {} -target "prebuilt" { - inherits = ["docker-metadata-action-prebuilt"] - dockerfile = "docker/autoware-openadk/Dockerfile" - target = "prebuilt" +// Autoware openadk modules +target "base" { + inherits = ["docker-metadata-action-base"] + dockerfile = "docker/autoware/Dockerfile" + target = "base" } -target "devel" { - inherits = ["docker-metadata-action-devel"] - dockerfile = "docker/autoware-openadk/Dockerfile" - target = "devel" +target "planning-control" { + inherits = ["docker-metadata-action-planning-control"] + dockerfile = "docker/autoware-openadk/modules/planning-control/Dockerfile" + target = "planning-control" } -target "runtime" { - inherits = ["docker-metadata-action-runtime"] - dockerfile = "docker/autoware-openadk/Dockerfile" - target = "runtime" +target "visualizer" { + inherits = ["docker-metadata-action-visualizer"] + dockerfile = "docker/autoware-openadk/modules/visualizer/Dockerfile" + target = "visualizer" +} + +target "simulator" { + inherits = ["docker-metadata-action-simulator"] + dockerfile = "docker/autoware-openadk/modules/simulator/Dockerfile" + target = "simulator" } diff --git a/docker/autoware-openadk/modules/planning-control/Dockerfile b/docker/autoware-openadk/modules/planning-control/Dockerfile new file mode 100644 index 00000000000..3eded4df478 --- /dev/null +++ b/docker/autoware-openadk/modules/planning-control/Dockerfile @@ -0,0 +1,36 @@ +FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as planning-control +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG ROS_DISTRO +ARG LIB_DIR + +# Copy install folder from prebuilt +COPY --from=ghcr.io/autowarefoundation/autoware:latest-prebuilt /autoware/install/ /autoware/install/ +COPY autoware.repos /autoware/ + +# Dependencies +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ + && pip uninstall -y ansible ansible-core \ + && mkdir src \ + && vcs import src < autoware.repos \ + && apt-get update \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ + && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ + && find / -name "*.o" -type f -delete \ + && find / -name "*.h" -type f -delete \ + && find / -name "*.hpp" -type f -delete \ + && rm -rf /autoware/src /autoware/ansible /autoware/autoware.repos \ + /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ + /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + +# Copy launch file +COPY docker/autoware-openadk/modules/planning-control/planning_control.launch.xml /autoware/launch/planning_control.launch.xml + +# Create entrypoint +COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +RUN chmod +x /ros_entrypoint.sh +ENTRYPOINT ["/ros_entrypoint.sh"] +CMD ["bash"] diff --git a/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml b/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml new file mode 100644 index 00000000000..e5210e69b2a --- /dev/null +++ b/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/autoware-openadk/modules/simulator/Dockerfile b/docker/autoware-openadk/modules/simulator/Dockerfile new file mode 100644 index 00000000000..50e9094d7c7 --- /dev/null +++ b/docker/autoware-openadk/modules/simulator/Dockerfile @@ -0,0 +1,45 @@ +FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as simulator +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG PLATFORM +ARG LIB_DIR +ARG ROS_DISTRO + +COPY simulator.repos /autoware/ +COPY autoware.repos /autoware/ + +RUN --mount=type=ssh \ + mkdir src \ + && vcs import src < simulator.repos \ + && vcs import src/simulator/scenario_simulator/external < src/simulator/scenario_simulator/dependency_humble.repos \ + && apt-get update \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --rosdistro "$ROS_DISTRO" --from-paths src --ignore-src + +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ + && pip uninstall -y ansible ansible-core \ + && source /opt/ros/"$ROS_DISTRO"/setup.bash \ + && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \ + && find /autoware/install -type d -exec chmod -R 777 {} \; \ + && rm -rf src/* \ + && vcs import src < autoware.repos \ + && apt-get update \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --rosdistro "$ROS_DISTRO" --from-paths src --ignore-src \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ + && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ + && find / -name "*.o" -type f -delete \ + && find / -name "*.h" -type f -delete \ + && find / -name "*.hpp" -type f -delete \ + && rm -rf /autoware/src /autoware/ansible /autoware/simulator.repos \ + /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ + /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + +COPY --from=ghcr.io/autowarefoundation/autoware-openadk:latest-prebuilt /autoware/install /autoware/install + +# Create entrypoint +COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +RUN chmod +x /ros_entrypoint.sh +ENTRYPOINT ["/ros_entrypoint.sh"] +CMD ["bash"] diff --git a/docker/autoware-openadk/modules/visualizer/Dockerfile b/docker/autoware-openadk/modules/visualizer/Dockerfile new file mode 100644 index 00000000000..e1e7d844fe1 --- /dev/null +++ b/docker/autoware-openadk/modules/visualizer/Dockerfile @@ -0,0 +1,33 @@ +FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as visualizer +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG ROS_DISTRO +ARG LIB_DIR + +# Copy install folder from prebuilt +COPY --from=ghcr.io/autowarefoundation/autoware:latest-prebuilt /autoware/install/ /autoware/install/ +COPY autoware.repos /autoware/ + +# Dependencies +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ + && pip uninstall -y ansible ansible-core \ + && mkdir src \ + && vcs import src < autoware.repos \ + && apt-get update \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ + && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ + && find / -name "*.o" -type f -delete \ + && find / -name "*.h" -type f -delete \ + && find / -name "*.hpp" -type f -delete \ + && rm -rf /autoware/src /autoware/ansible /autoware/autoware.repos \ + /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ + /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + +# Create entrypoint +COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +RUN chmod +x /ros_entrypoint.sh +ENTRYPOINT ["/ros_entrypoint.sh"] +CMD ["bash"] diff --git a/docker/autoware-openadk/test/run-test-scenario.sh b/docker/autoware-openadk/test/run-test-scenario.sh new file mode 100755 index 00000000000..5fa04497401 --- /dev/null +++ b/docker/autoware-openadk/test/run-test-scenario.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e + +# download test artifacts from google drive +if [ ! -d ~/autoware_artifacts/test-artifacts ]; then + echo "Downloading test artifacts from google drive..." + gdown -O ~/autoware_artifacts/ 'https://drive.google.com/uc?export=download&id=1gTCWcye4JxqR7rSxl2b-iiLqgya6ShBR' + unzip -d ~/autoware_artifacts ~/autoware_artifacts/test-artifacts.zip + rm ~/autoware_artifacts/test-artifacts.zip +else + echo "Test artifacts are already downloaded." +fi + +# enable xhost +xhost + + +# start containers +echo "Starting containers..." +docker compose -f test-scenario.docker-compose.yaml up diff --git a/docker/autoware-openadk/test/test-scenario.docker-compose.yaml b/docker/autoware-openadk/test/test-scenario.docker-compose.yaml new file mode 100644 index 00000000000..e3c68dd195e --- /dev/null +++ b/docker/autoware-openadk/test/test-scenario.docker-compose.yaml @@ -0,0 +1,38 @@ +version: "2.17" + +services: + planning-control: + image: ghcr.io/autowarefoundation/autoware-openadk:latest-planning-control + volumes: + - /dev/shm:/dev/shm + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + network_mode: "host" + environment: + - ROS_LOCALHOST_ONLY=1 + command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-artifacts/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false + + simulator: + image: ghcr.io/autowarefoundation/autoware-openadk:latest-simulator + volumes: + - /dev/shm:/dev/shm + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + network_mode: "host" + environment: + - ROS_LOCALHOST_ONLY=1 + depends_on: + - planning-control + command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-artifacts/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-artifacts/map/ launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 + + visualizer: + image: ghcr.io/autowarefoundation/autoware-openadk:latest-visualizer + volumes: + - /dev/shm:/dev/shm + - /tmp/.X11-unix:/tmp/.X11-unix + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + network_mode: "host" + environment: + - DISPLAY=$DISPLAY + - ROS_LOCALHOST_ONLY=1 + depends_on: + - simulator + command: /autoware/test-artifacts/rviz/launch_rviz.sh diff --git a/docker/autoware-openadk/Dockerfile b/docker/autoware/Dockerfile similarity index 96% rename from docker/autoware-openadk/Dockerfile rename to docker/autoware/Dockerfile index 0ee7ebfa0d2..96bccf8b409 100644 --- a/docker/autoware-openadk/Dockerfile +++ b/docker/autoware/Dockerfile @@ -110,7 +110,7 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Create entrypoint -COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/autoware/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -143,11 +143,11 @@ RUN --mount=type=ssh \ COPY --from=prebuilt /autoware/install/ /autoware/install/ # Copy bash aliases -COPY docker/autoware-openadk/etc/.bash_aliases /root/.bash_aliases +COPY docker/autoware/etc/.bash_aliases /root/.bash_aliases RUN echo "source /autoware/install/setup.bash" > /etc/bash.bashrc # Create entrypoint -COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/autoware/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/docker/build.sh b/docker/autoware/build.sh old mode 100755 new mode 100644 similarity index 88% rename from docker/build.sh rename to docker/autoware/build.sh index fe9917b7575..ccdea679866 --- a/docker/build.sh +++ b/docker/autoware/build.sh @@ -16,7 +16,7 @@ print_help() { } SCRIPT_DIR=$(readlink -f "$(dirname "$0")") -WORKSPACE_ROOT="$SCRIPT_DIR/../" +WORKSPACE_ROOT="$SCRIPT_DIR/../.." # Parse arguments parse_arguments() { @@ -111,7 +111,7 @@ build_images() { echo "Targets: ${targets[*]}" set -x - docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/autoware-openadk/docker-bake.hcl" \ + docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/docker-bake.hcl" \ --set "*.context=$WORKSPACE_ROOT" \ --set "*.ssh=default" \ --set "*.platform=$platform" \ @@ -119,9 +119,9 @@ build_images() { --set "*.args.BASE_IMAGE=$base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ - --set "devel.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-devel$image_name_suffix" \ - --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-prebuilt$image_name_suffix" \ - --set "runtime.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-runtime$image_name_suffix" \ + --set "devel.tags=ghcr.io/autowarefoundation/autoware:latest-devel$image_name_suffix" \ + --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware:latest-prebuilt$image_name_suffix" \ + --set "runtime.tags=ghcr.io/autowarefoundation/autoware:latest-runtime$image_name_suffix" \ "${targets[@]}" set +x } diff --git a/docker/autoware/docker-bake.hcl b/docker/autoware/docker-bake.hcl new file mode 100644 index 00000000000..6c4ee66adf2 --- /dev/null +++ b/docker/autoware/docker-bake.hcl @@ -0,0 +1,27 @@ +group "default" { + targets = ["prebuilt", "devel", "runtime"] +} + +// For docker/metadata-action +target "docker-metadata-action-prebuilt" {} +target "docker-metadata-action-devel" {} +target "docker-metadata-action-runtime" {} + +// Autoware monolithic +target "prebuilt" { + inherits = ["docker-metadata-action-prebuilt"] + dockerfile = "docker/autoware/Dockerfile" + target = "prebuilt" +} + +target "devel" { + inherits = ["docker-metadata-action-devel"] + dockerfile = "docker/autoware/Dockerfile" + target = "devel" +} + +target "runtime" { + inherits = ["docker-metadata-action-runtime"] + dockerfile = "docker/autoware/Dockerfile" + target = "runtime" +} diff --git a/docker/autoware-openadk/etc/.bash_aliases b/docker/autoware/etc/.bash_aliases similarity index 100% rename from docker/autoware-openadk/etc/.bash_aliases rename to docker/autoware/etc/.bash_aliases diff --git a/docker/autoware/etc/ros_entrypoint.sh b/docker/autoware/etc/ros_entrypoint.sh new file mode 100644 index 00000000000..cbe45ef3bdb --- /dev/null +++ b/docker/autoware/etc/ros_entrypoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 + +# Get the user ID and group ID of the local user +USER_ID=${LOCAL_UID} +USER_NAME=${LOCAL_USER} +GROUP_ID=${LOCAL_GID} +GROUP_NAME=${LOCAL_GROUP} + +# Check if any of the variables are empty +if [[ -z $USER_ID || -z $USER_NAME || -z $GROUP_ID || -z $GROUP_NAME ]]; then + source "/opt/ros/$ROS_DISTRO/setup.bash" + source /autoware/install/setup.bash + exec "$@" +else + echo "Starting with user: $USER_NAME >> UID $USER_ID, GID: $GROUP_ID" + + # Create group and user with GID/UID + groupadd -g "$GROUP_ID" "$GROUP_NAME" + useradd -u "$USER_ID" -g "$GROUP_ID" -s /bin/bash -m -d /home/"$USER_NAME" "$USER_NAME" + + # Add sudo privileges to the user + echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers + + # Source ROS2 + # hadolint ignore=SC1090 + source "/opt/ros/$ROS_DISTRO/setup.bash" + source /autoware/install/setup.bash + + # Execute the command as the user + exec /usr/sbin/gosu "$USER_NAME" "$@" +fi diff --git a/docker/run.sh b/docker/autoware/run.sh similarity index 96% rename from docker/run.sh rename to docker/autoware/run.sh index ebcd32ff29e..1bb1a53ae45 100755 --- a/docker/run.sh +++ b/docker/autoware/run.sh @@ -10,7 +10,7 @@ GREEN='\033[0;32m' NC='\033[0m' # No Color SCRIPT_DIR=$(readlink -f "$(dirname "$0")") -WORKSPACE_ROOT="$SCRIPT_DIR/../" +WORKSPACE_ROOT="$SCRIPT_DIR/../.." source "$WORKSPACE_ROOT/amd64.env" if [ "$(uname -m)" = "aarch64" ]; then source "$WORKSPACE_ROOT/arm64.env" @@ -124,9 +124,9 @@ set_variables() { # Set image based on option if [ "$option_devel" == "true" ]; then - IMAGE="ghcr.io/autowarefoundation/autoware-openadk:latest-devel" + IMAGE="ghcr.io/autowarefoundation/autoware:latest-devel" else - IMAGE="ghcr.io/autowarefoundation/autoware-openadk:latest-runtime" + IMAGE="ghcr.io/autowarefoundation/autoware:latest-runtime" fi } From 990bf3e8449bb0416cc220028eb22f5009ee1864 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 30 Apr 2024 16:55:05 +0300 Subject: [PATCH 02/46] . Signed-off-by: Oguz Ozturk --- .github/workflows/build-main-self-hosted.yaml | 2 +- .github/workflows/build-main.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-main-self-hosted.yaml b/.github/workflows/build-main-self-hosted.yaml index 86a0bd3811a..9e85f01bd3e 100644 --- a/.github/workflows/build-main-self-hosted.yaml +++ b/.github/workflows/build-main-self-hosted.yaml @@ -48,7 +48,7 @@ jobs: uses: ./.github/actions/free-disk-space - name: Build 'autoware-universe' - uses: ./.github/actions/docker-build-push + uses: ./.github/actions/docker-build-and-push with: bake-target: autoware build-args: | diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index e49c4a06acf..1c8123f360e 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -43,7 +43,7 @@ jobs: uses: ./.github/actions/free-disk-space - name: Build 'autoware-universe' - uses: ./.github/actions/docker-build-push + uses: ./.github/actions/docker-build-and-push with: bake-target: autoware build-args: | diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index a48be674ceb..5b6dab8c5ef 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -71,7 +71,7 @@ jobs: vcs import src < autoware.repos - name: Build 'Autoware' - uses: ./.github/actions/docker-build-push + uses: ./.github/actions/docker-build-and-push with: bake-target: autoware build-args: | From ba155451ad605f2ba4c3e8c158a560ee53f6221f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:10:56 +0000 Subject: [PATCH 03/46] style(pre-commit): autofix --- .../modules/planning-control/planning_control.launch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml b/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml index e5210e69b2a..0788fbdef77 100644 --- a/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml +++ b/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml @@ -66,4 +66,4 @@ - \ No newline at end of file + From 242ace089c7e6303e9102d7e73e12c529ba10edf Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 3 May 2024 17:43:50 +0300 Subject: [PATCH 04/46] refine dockerfiles Signed-off-by: Oguz Ozturk --- ansible/playbooks/openadk.yaml | 4 +- .../modules/planning-control/Dockerfile | 9 +-- .../planning_control.launch.xml | 69 ------------------- .../modules/simulator/Dockerfile | 6 +- .../modules/visualizer/Dockerfile | 6 +- .../test/test-scenario.docker-compose.yaml | 14 ++-- 6 files changed, 13 insertions(+), 95 deletions(-) delete mode 100644 docker/autoware-openadk/modules/planning-control/planning_control.launch.xml diff --git a/ansible/playbooks/openadk.yaml b/ansible/playbooks/openadk.yaml index f343d9cef55..46a8757229f 100644 --- a/ansible/playbooks/openadk.yaml +++ b/ansible/playbooks/openadk.yaml @@ -24,6 +24,8 @@ when: module == 'base' - role: autoware.dev_env.kisak_mesa when: module == 'base' + - role: autoware.dev_env.pacmod + when: module == 'base' - role: autoware.dev_env.build_tools when: module == 'all' and install_devel=='y' @@ -34,8 +36,6 @@ when: (module == 'perception-localization' or module == 'all') and prompt_install_nvidia=='y' - role: autoware.dev_env.tensorrt when: (module == 'perception-localization' or module == 'all') and prompt_install_nvidia=='y' - - role: autoware.dev_env.pacmod - when: module == 'planning-control' or module == 'perception-localization' or module == 'all' # Development environment - role: autoware.dev_env.dev_tools diff --git a/docker/autoware-openadk/modules/planning-control/Dockerfile b/docker/autoware-openadk/modules/planning-control/Dockerfile index 3eded4df478..d1486985a1a 100644 --- a/docker/autoware-openadk/modules/planning-control/Dockerfile +++ b/docker/autoware-openadk/modules/planning-control/Dockerfile @@ -9,13 +9,11 @@ COPY autoware.repos /autoware/ # Dependencies RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ - && pip uninstall -y ansible ansible-core \ - && mkdir src \ + mkdir src \ && vcs import src < autoware.repos \ && apt-get update \ && rosdep update \ - && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y -t exec --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ && find / -name "*.o" -type f -delete \ @@ -26,9 +24,6 @@ RUN --mount=type=ssh \ /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* -# Copy launch file -COPY docker/autoware-openadk/modules/planning-control/planning_control.launch.xml /autoware/launch/planning_control.launch.xml - # Create entrypoint COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh diff --git a/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml b/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml deleted file mode 100644 index 0788fbdef77..00000000000 --- a/docker/autoware-openadk/modules/planning-control/planning_control.launch.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docker/autoware-openadk/modules/simulator/Dockerfile b/docker/autoware-openadk/modules/simulator/Dockerfile index 50e9094d7c7..98a53b81bac 100644 --- a/docker/autoware-openadk/modules/simulator/Dockerfile +++ b/docker/autoware-openadk/modules/simulator/Dockerfile @@ -13,12 +13,8 @@ RUN --mount=type=ssh \ && vcs import src/simulator/scenario_simulator/external < src/simulator/scenario_simulator/dependency_humble.repos \ && apt-get update \ && rosdep update \ - && DEBIAN_FRONTEND=noninteractive rosdep install -y --rosdistro "$ROS_DISTRO" --from-paths src --ignore-src - -RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ - && pip uninstall -y ansible ansible-core \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y -t exec --rosdistro "$ROS_DISTRO" --from-paths src --ignore-src \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \ && find /autoware/install -type d -exec chmod -R 777 {} \; \ && rm -rf src/* \ diff --git a/docker/autoware-openadk/modules/visualizer/Dockerfile b/docker/autoware-openadk/modules/visualizer/Dockerfile index e1e7d844fe1..6852880dda0 100644 --- a/docker/autoware-openadk/modules/visualizer/Dockerfile +++ b/docker/autoware-openadk/modules/visualizer/Dockerfile @@ -9,13 +9,11 @@ COPY autoware.repos /autoware/ # Dependencies RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module planning-control --no-cuda-drivers --runtime openadk \ - && pip uninstall -y ansible ansible-core \ - && mkdir src \ + mkdir src \ && vcs import src < autoware.repos \ && apt-get update \ && rosdep update \ - && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y -t exec --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ && find / -name "*.o" -type f -delete \ diff --git a/docker/autoware-openadk/test/test-scenario.docker-compose.yaml b/docker/autoware-openadk/test/test-scenario.docker-compose.yaml index e3c68dd195e..1e03a8e4175 100644 --- a/docker/autoware-openadk/test/test-scenario.docker-compose.yaml +++ b/docker/autoware-openadk/test/test-scenario.docker-compose.yaml @@ -5,34 +5,32 @@ services: image: ghcr.io/autowarefoundation/autoware-openadk:latest-planning-control volumes: - /dev/shm:/dev/shm - - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ network_mode: "host" environment: - ROS_LOCALHOST_ONLY=1 - command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-artifacts/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false + command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-scenario/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false simulator: image: ghcr.io/autowarefoundation/autoware-openadk:latest-simulator volumes: - /dev/shm:/dev/shm - - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ network_mode: "host" environment: - ROS_LOCALHOST_ONLY=1 - depends_on: - - planning-control - command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-artifacts/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-artifacts/map/ launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 + command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-scenario/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-scenario/map launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 visualizer: image: ghcr.io/autowarefoundation/autoware-openadk:latest-visualizer volumes: - /dev/shm:/dev/shm - /tmp/.X11-unix:/tmp/.X11-unix - - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-artifacts/ + - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ network_mode: "host" environment: - DISPLAY=$DISPLAY - ROS_LOCALHOST_ONLY=1 depends_on: - simulator - command: /autoware/test-artifacts/rviz/launch_rviz.sh + command: /autoware/test-scenario/rviz/launch_rviz.sh From 68a82a1c8e4083a49a3bc5e860163cabf1fa4595 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Thu, 16 May 2024 16:05:39 +0300 Subject: [PATCH 05/46] Simplify run.sh Signed-off-by: Oguz Ozturk --- docker/autoware/run.sh | 49 +++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/docker/autoware/run.sh b/docker/autoware/run.sh index 1bb1a53ae45..af8cb71bef9 100755 --- a/docker/autoware/run.sh +++ b/docker/autoware/run.sh @@ -6,7 +6,7 @@ set -e # Define terminal colors RED='\033[0;31m' GREEN='\033[0;32m' -# BLUE='\033[0;34m' +BLUE='\033[0;34m' NC='\033[0m' # No Color SCRIPT_DIR=$(readlink -f "$(dirname "$0")") @@ -30,7 +30,7 @@ DEFAULT_LAUNCH_CMD="ros2 launch autoware_launch autoware.launch.xml data_path:=/ # Function to print help message print_help() { echo -e "\n------------------------------------------------------------" - echo -e "${RED}Note:${NC} The --map-path option is mandatory if not custom launch command given. Please provide exact path to the map files." + echo -e "${RED}Note:${NC} The --map-path option is mandatory for the runtime. For development environment with shell access, use --devel option." echo -e " Default launch command: ${GREEN}${DEFAULT_LAUNCH_CMD}${NC}" echo -e "------------------------------------------------------------" echo -e "${RED}Usage:${NC} run.sh [OPTIONS] [LAUNCH_CMD](optional)" @@ -39,7 +39,7 @@ print_help() { echo -e " ${GREEN}--data-path${NC} Specify to mount data files into /autoware_data" echo -e " ${GREEN}--map-path${NC} Specify to mount map files into /autoware_map (mandatory if no custom launch command is provided)" echo -e " ${GREEN}--no-nvidia${NC} Disable NVIDIA GPU support" - echo -e " ${GREEN}--devel${NC} Use the latest development version of Autoware" + echo -e " ${GREEN}--devel${NC} Launch the latest Autoware development environment with shell access" echo -e " ${GREEN}--headless${NC} Run Autoware in headless mode (default: false)" echo -e " ${GREEN}--workspace${NC} Specify to mount the workspace into /workspace" echo "" @@ -93,13 +93,11 @@ parse_arguments() { done } -# Set image and workspace variables +# Set docker image used and workspace variables set_variables() { - # Check if map path is provided for default launch command - if [ "$MAP_PATH" == "" ] && [ "$LAUNCH_CMD" == "" ]; then - print_help - exit 1 - fi + if [ "$option_devel" = "true" ]; then + # Set image based on option + IMAGE="ghcr.io/autowarefoundation/autoware:latest-devel" # Mount data path DATA="-v ${DATA_PATH}:/autoware_data:ro" @@ -118,15 +116,30 @@ set_variables() { if [ "$WORKSPACE_PATH" != "" ]; then LAUNCH_CMD="/bin/bash" else - LAUNCH_CMD=${DEFAULT_LAUNCH_CMD} + USER_ID="-e LOCAL_UID=$(id -u) -e LOCAL_GID=$(id -g) -e LOCAL_USER=$(id -un) -e LOCAL_GROUP=$(id -gn)" + WORKSPACE="-v ${WORKSPACE_PATH}:/workspace" fi - fi - # Set image based on option - if [ "$option_devel" == "true" ]; then - IMAGE="ghcr.io/autowarefoundation/autoware:latest-devel" + # Set launch command + LAUNCH_CMD="/bin/bash" else + # Set image based on option IMAGE="ghcr.io/autowarefoundation/autoware:latest-runtime" + + # Set map path + if [ "$MAP_PATH" = "" ]; then + echo -e "\n------------------------------------------------------------" + echo -e "${RED}Note:${NC} The --map-path option is mandatory for the runtime. For development environment with shell access, use --devel option." + echo -e "------------------------------------------------------------" + exit 1 + else + MAP="-v ${MAP_PATH}:/autoware_map:ro" + fi + + # Set default launch command if not provided + if [ "$LAUNCH_CMD" = "" ]; then + LAUNCH_CMD=${DEFAULT_LAUNCH_CMD} + fi fi } @@ -157,7 +170,13 @@ main() { set_gpu_flag set_x_display - echo -e "${GREEN}\n-----------------------LAUNCHING CONTAINER-----------------------" + if [ "$option_devel" = "true" ]; then + echo -e "${GREEN}-----------------------------------------------------------------${NC}" + echo -e "${BLUE}Launching Autoware development environment${NC}" + else + echo -e "${GREEN}-----------------------------------------------------------------${NC}" + echo -e "${GREEN}Launching Autoware${NC}" + fi echo -e "${GREEN}IMAGE:${NC} ${IMAGE}" echo -e "${GREEN}DATA PATH(mounted):${NC} ${DATA_PATH}:/autoware_data" echo -e "${GREEN}MAP PATH(mounted):${NC} ${MAP_PATH}:/autoware_map" From b4ba9c21a0eeb203b253d85e6204ff99fa6afbc9 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Thu, 16 May 2024 16:19:19 +0300 Subject: [PATCH 06/46] Simply workspace argument Signed-off-by: Oguz Ozturk --- docker/autoware/run.sh | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/docker/autoware/run.sh b/docker/autoware/run.sh index af8cb71bef9..4e4da08bb78 100755 --- a/docker/autoware/run.sh +++ b/docker/autoware/run.sh @@ -38,10 +38,10 @@ print_help() { echo -e " ${GREEN}--help/-h${NC} Display this help message" echo -e " ${GREEN}--data-path${NC} Specify to mount data files into /autoware_data" echo -e " ${GREEN}--map-path${NC} Specify to mount map files into /autoware_map (mandatory if no custom launch command is provided)" - echo -e " ${GREEN}--no-nvidia${NC} Disable NVIDIA GPU support" echo -e " ${GREEN}--devel${NC} Launch the latest Autoware development environment with shell access" + echo -e " ${GREEN}--workspace${NC} (--devel only)Specify the directory to mount into /workspace, by default it uses current directory (pwd) as workspace" + echo -e " ${GREEN}--no-nvidia${NC} Disable NVIDIA GPU support" echo -e " ${GREEN}--headless${NC} Run Autoware in headless mode (default: false)" - echo -e " ${GREEN}--workspace${NC} Specify to mount the workspace into /workspace" echo "" } @@ -99,29 +99,24 @@ set_variables() { # Set image based on option IMAGE="ghcr.io/autowarefoundation/autoware:latest-devel" - # Mount data path - DATA="-v ${DATA_PATH}:/autoware_data:ro" - - # Mount map path if provided - MAP="-v ${MAP_PATH}:/autoware_map:ro" + # Set workspace path, if not provided use the current directory + if [ "$WORKSPACE_PATH" = "" ]; then + WORKSPACE_PATH=$(pwd) + fi + WORKSPACE="-v ${WORKSPACE_PATH}:/workspace" - # Set workspace path if provided and login with local user - if [ "$WORKSPACE_PATH" != "" ]; then + # Set user ID and group ID to match the local user USER_ID="-e LOCAL_UID=$(id -u) -e LOCAL_GID=$(id -g) -e LOCAL_USER=$(id -un) -e LOCAL_GROUP=$(id -gn)" - WORKSPACE="-v ${WORKSPACE_PATH}:/workspace" - fi - # Set default launch command if not provided - if [ "$LAUNCH_CMD" == "" ]; then - if [ "$WORKSPACE_PATH" != "" ]; then - LAUNCH_CMD="/bin/bash" - else - USER_ID="-e LOCAL_UID=$(id -u) -e LOCAL_GID=$(id -g) -e LOCAL_USER=$(id -un) -e LOCAL_GROUP=$(id -gn)" - WORKSPACE="-v ${WORKSPACE_PATH}:/workspace" + # Set map path + if [ "$MAP_PATH" != "" ]; then + MAP="-v ${MAP_PATH}:/autoware_map:ro" fi # Set launch command - LAUNCH_CMD="/bin/bash" + if [ "$LAUNCH_CMD" = "" ]; then + LAUNCH_CMD="/bin/bash" + fi else # Set image based on option IMAGE="ghcr.io/autowarefoundation/autoware:latest-runtime" From 4aaaff9d722744de80b9953763902ddd6f3996c8 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Thu, 16 May 2024 16:59:43 +0300 Subject: [PATCH 07/46] Simplify directories Signed-off-by: Oguz Ozturk --- .../docker-build-and-push-openadk/action.yaml | 12 ++++++------ .../docker-build-and-push-main-self-hosted.yaml | 6 +++--- .github/workflows/docker-build-and-push-main.yaml | 6 +++--- .github/workflows/update-docker-manifest.yaml | 4 ++-- ansible/playbooks/{openadk.yaml => openadkit.yaml} | 0 docker/{autoware => }/Dockerfile | 10 +++++----- docker/{autoware => }/build.sh | 2 +- docker/{autoware => }/docker-bake.hcl | 6 +++--- docker/{autoware => }/etc/.bash_aliases | 0 docker/{autoware-openadk => }/etc/ros_entrypoint.sh | 0 docker/{autoware-openadk => openadkit}/build.sh | 8 ++++---- .../{autoware-openadk => openadkit}/docker-bake.hcl | 8 ++++---- docker/{autoware => openadkit}/etc/ros_entrypoint.sh | 0 .../modules/planning-control/Dockerfile | 4 ++-- .../modules/simulator/Dockerfile | 6 +++--- .../modules/visualizer/Dockerfile | 4 ++-- .../test/run-test-scenario.sh | 0 .../test/test-scenario.docker-compose.yaml | 6 +++--- docker/{autoware => }/run.sh | 2 +- setup-dev-env.sh | 2 +- 20 files changed, 43 insertions(+), 43 deletions(-) rename ansible/playbooks/{openadk.yaml => openadkit.yaml} (100%) rename docker/{autoware => }/Dockerfile (95%) rename docker/{autoware => }/build.sh (98%) mode change 100644 => 100755 rename docker/{autoware => }/docker-bake.hcl (79%) rename docker/{autoware => }/etc/.bash_aliases (100%) rename docker/{autoware-openadk => }/etc/ros_entrypoint.sh (100%) rename docker/{autoware-openadk => openadkit}/build.sh (95%) rename docker/{autoware-openadk => openadkit}/docker-bake.hcl (75%) rename docker/{autoware => openadkit}/etc/ros_entrypoint.sh (100%) rename docker/{autoware-openadk => openadkit}/modules/planning-control/Dockerfile (86%) rename docker/{autoware-openadk => openadkit}/modules/simulator/Dockerfile (84%) rename docker/{autoware-openadk => openadkit}/modules/visualizer/Dockerfile (87%) rename docker/{autoware-openadk => openadkit}/test/run-test-scenario.sh (100%) rename docker/{autoware-openadk => openadkit}/test/test-scenario.docker-compose.yaml (86%) rename docker/{autoware => }/run.sh (99%) diff --git a/.github/actions/docker-build-and-push-openadk/action.yaml b/.github/actions/docker-build-and-push-openadk/action.yaml index 69e9517cd46..10c3d3cb3db 100644 --- a/.github/actions/docker-build-and-push-openadk/action.yaml +++ b/.github/actions/docker-build-and-push-openadk/action.yaml @@ -1,4 +1,4 @@ -name: docker-build-and-push-openadk +name: docker-build-and-push-openadkit description: "" inputs: @@ -53,8 +53,8 @@ runs: echo "$EOF" >> $GITHUB_OUTPUT shell: bash - # openadk modular images meta - - name: Docker meta for openadk:base + # openadkit modular images meta + - name: Docker meta for openadkit:base id: meta-base uses: docker/metadata-action@v4 with: @@ -65,7 +65,7 @@ runs: latest=false suffix=-base-${{ inputs.tag-arch }} - - name: Docker meta for openadk:planning-control + - name: Docker meta for openadkit:planning-control id: meta-planning-control uses: docker/metadata-action@v4 with: @@ -76,7 +76,7 @@ runs: latest=false suffix=-planning-control-${{ inputs.tag-arch }} - - name: Docker meta for openadk:visualizer + - name: Docker meta for openadkit:visualizer id: meta-visualizer uses: docker/metadata-action@v4 with: @@ -87,7 +87,7 @@ runs: latest=false suffix=-visualizer-${{ inputs.tag-arch }} - - name: Docker meta for openadk:simulator + - name: Docker meta for openadkit:simulator id: meta-simulator uses: docker/metadata-action@v4 with: diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 3523ce4ad16..2e7fe758342 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -92,10 +92,10 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true - - name: Build 'Openadk' - uses: ./.github/actions/docker-build-and-push-openadk + - name: Build 'Openadkit' + uses: ./.github/actions/docker-build-and-push-openadkit with: - bake-target: openadk + bake-target: openadkit build-args: | *.platform=linux/arm64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 5b6dab8c5ef..53453ecbad1 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -85,10 +85,10 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true - - name: Build 'Openadk' - uses: ./.github/actions/docker-build-and-push-openadk + - name: Build 'Openadkit' + uses: ./.github/actions/docker-build-and-push-openadkit with: - bake-target: autoware-openadk + bake-target: openadkit build-args: | *.platform=linux/amd64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml index a3b4cbf70bf..c98378d7041 100644 --- a/.github/workflows/update-docker-manifest.yaml +++ b/.github/workflows/update-docker-manifest.yaml @@ -17,7 +17,7 @@ jobs: with: package-name: autoware - - name: Combine multi arch images for 'autoware-openadk' + - name: Combine multi arch images for 'openadkit' uses: ./.github/actions/combine-multi-arch-images with: - package-name: autoware-openadk + package-name: autoware-openadkit diff --git a/ansible/playbooks/openadk.yaml b/ansible/playbooks/openadkit.yaml similarity index 100% rename from ansible/playbooks/openadk.yaml rename to ansible/playbooks/openadkit.yaml diff --git a/docker/autoware/Dockerfile b/docker/Dockerfile similarity index 95% rename from docker/autoware/Dockerfile rename to docker/Dockerfile index 96bccf8b409..6049154adaa 100644 --- a/docker/autoware/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ WORKDIR /autoware # Set up base environment RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module base --runtime openadk \ + ./setup-dev-env.sh -y --module base --runtime openadkit \ && pip uninstall -y ansible ansible-core \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc @@ -72,7 +72,7 @@ ENV CXX="/usr/lib/ccache/g++" # cspell: ignore libcu libnv # Set up development environment RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadk \ + ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && find / -name 'libcu*.a' -delete \ @@ -110,7 +110,7 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Create entrypoint -COPY docker/autoware/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -143,11 +143,11 @@ RUN --mount=type=ssh \ COPY --from=prebuilt /autoware/install/ /autoware/install/ # Copy bash aliases -COPY docker/autoware/etc/.bash_aliases /root/.bash_aliases +COPY docker/etc/.bash_aliases /root/.bash_aliases RUN echo "source /autoware/install/setup.bash" > /etc/bash.bashrc # Create entrypoint -COPY docker/autoware/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/docker/autoware/build.sh b/docker/build.sh old mode 100644 new mode 100755 similarity index 98% rename from docker/autoware/build.sh rename to docker/build.sh index ccdea679866..3d0a417c952 --- a/docker/autoware/build.sh +++ b/docker/build.sh @@ -16,7 +16,7 @@ print_help() { } SCRIPT_DIR=$(readlink -f "$(dirname "$0")") -WORKSPACE_ROOT="$SCRIPT_DIR/../.." +WORKSPACE_ROOT="$SCRIPT_DIR/.." # Parse arguments parse_arguments() { diff --git a/docker/autoware/docker-bake.hcl b/docker/docker-bake.hcl similarity index 79% rename from docker/autoware/docker-bake.hcl rename to docker/docker-bake.hcl index 6c4ee66adf2..c716e3e95b6 100644 --- a/docker/autoware/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -10,18 +10,18 @@ target "docker-metadata-action-runtime" {} // Autoware monolithic target "prebuilt" { inherits = ["docker-metadata-action-prebuilt"] - dockerfile = "docker/autoware/Dockerfile" + dockerfile = "docker/Dockerfile" target = "prebuilt" } target "devel" { inherits = ["docker-metadata-action-devel"] - dockerfile = "docker/autoware/Dockerfile" + dockerfile = "docker/Dockerfile" target = "devel" } target "runtime" { inherits = ["docker-metadata-action-runtime"] - dockerfile = "docker/autoware/Dockerfile" + dockerfile = "docker/Dockerfile" target = "runtime" } diff --git a/docker/autoware/etc/.bash_aliases b/docker/etc/.bash_aliases similarity index 100% rename from docker/autoware/etc/.bash_aliases rename to docker/etc/.bash_aliases diff --git a/docker/autoware-openadk/etc/ros_entrypoint.sh b/docker/etc/ros_entrypoint.sh similarity index 100% rename from docker/autoware-openadk/etc/ros_entrypoint.sh rename to docker/etc/ros_entrypoint.sh diff --git a/docker/autoware-openadk/build.sh b/docker/openadkit/build.sh similarity index 95% rename from docker/autoware-openadk/build.sh rename to docker/openadkit/build.sh index 58db2e6731c..0abaa223b4e 100755 --- a/docker/autoware-openadk/build.sh +++ b/docker/openadkit/build.sh @@ -106,10 +106,10 @@ build_images() { --set "*.args.BASE_IMAGE=$base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ - --set "base.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-base" \ - --set "planning-control.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-planning-control" \ - --set "visualizer.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-visualizer" \ - --set "simulator.tags=ghcr.io/autowarefoundation/autoware-openadk:latest-simulator" \ + --set "base.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-base" \ + --set "planning-control.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-planning-control" \ + --set "visualizer.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-visualizer" \ + --set "simulator.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-simulator" \ "${targets[@]}" set +x } diff --git a/docker/autoware-openadk/docker-bake.hcl b/docker/openadkit/docker-bake.hcl similarity index 75% rename from docker/autoware-openadk/docker-bake.hcl rename to docker/openadkit/docker-bake.hcl index 175d5ee80f5..8c5d1f5d3f6 100644 --- a/docker/autoware-openadk/docker-bake.hcl +++ b/docker/openadkit/docker-bake.hcl @@ -8,7 +8,7 @@ target "docker-metadata-action-planning-control" {} target "docker-metadata-action-visualizer" {} target "docker-metadata-action-simulator" {} -// Autoware openadk modules +// Autoware openadkit modules target "base" { inherits = ["docker-metadata-action-base"] dockerfile = "docker/autoware/Dockerfile" @@ -17,18 +17,18 @@ target "base" { target "planning-control" { inherits = ["docker-metadata-action-planning-control"] - dockerfile = "docker/autoware-openadk/modules/planning-control/Dockerfile" + dockerfile = "docker/openadkit/modules/planning-control/Dockerfile" target = "planning-control" } target "visualizer" { inherits = ["docker-metadata-action-visualizer"] - dockerfile = "docker/autoware-openadk/modules/visualizer/Dockerfile" + dockerfile = "docker/openadkit/modules/visualizer/Dockerfile" target = "visualizer" } target "simulator" { inherits = ["docker-metadata-action-simulator"] - dockerfile = "docker/autoware-openadk/modules/simulator/Dockerfile" + dockerfile = "docker/openadkit/modules/simulator/Dockerfile" target = "simulator" } diff --git a/docker/autoware/etc/ros_entrypoint.sh b/docker/openadkit/etc/ros_entrypoint.sh similarity index 100% rename from docker/autoware/etc/ros_entrypoint.sh rename to docker/openadkit/etc/ros_entrypoint.sh diff --git a/docker/autoware-openadk/modules/planning-control/Dockerfile b/docker/openadkit/modules/planning-control/Dockerfile similarity index 86% rename from docker/autoware-openadk/modules/planning-control/Dockerfile rename to docker/openadkit/modules/planning-control/Dockerfile index d1486985a1a..b0cf18666f7 100644 --- a/docker/autoware-openadk/modules/planning-control/Dockerfile +++ b/docker/openadkit/modules/planning-control/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as planning-control +FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as planning-control SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR @@ -25,7 +25,7 @@ RUN --mount=type=ssh \ /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* # Create entrypoint -COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/openadkit/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/docker/autoware-openadk/modules/simulator/Dockerfile b/docker/openadkit/modules/simulator/Dockerfile similarity index 84% rename from docker/autoware-openadk/modules/simulator/Dockerfile rename to docker/openadkit/modules/simulator/Dockerfile index 98a53b81bac..b8f38daacf9 100644 --- a/docker/autoware-openadk/modules/simulator/Dockerfile +++ b/docker/openadkit/modules/simulator/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as simulator +FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as simulator SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG PLATFORM ARG LIB_DIR @@ -32,10 +32,10 @@ RUN --mount=type=ssh \ /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* -COPY --from=ghcr.io/autowarefoundation/autoware-openadk:latest-prebuilt /autoware/install /autoware/install +COPY --from=ghcr.io/autowarefoundation/autoware-openadkit:latest-prebuilt /autoware/install /autoware/install # Create entrypoint -COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/openadkit/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/docker/autoware-openadk/modules/visualizer/Dockerfile b/docker/openadkit/modules/visualizer/Dockerfile similarity index 87% rename from docker/autoware-openadk/modules/visualizer/Dockerfile rename to docker/openadkit/modules/visualizer/Dockerfile index 6852880dda0..39d64d4ae4c 100644 --- a/docker/autoware-openadk/modules/visualizer/Dockerfile +++ b/docker/openadkit/modules/visualizer/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadk:latest-base as visualizer +FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as visualizer SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR @@ -25,7 +25,7 @@ RUN --mount=type=ssh \ /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* # Create entrypoint -COPY docker/autoware-openadk/etc/ros_entrypoint.sh /ros_entrypoint.sh +COPY docker/openadkit/etc/ros_entrypoint.sh /ros_entrypoint.sh RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/docker/autoware-openadk/test/run-test-scenario.sh b/docker/openadkit/test/run-test-scenario.sh similarity index 100% rename from docker/autoware-openadk/test/run-test-scenario.sh rename to docker/openadkit/test/run-test-scenario.sh diff --git a/docker/autoware-openadk/test/test-scenario.docker-compose.yaml b/docker/openadkit/test/test-scenario.docker-compose.yaml similarity index 86% rename from docker/autoware-openadk/test/test-scenario.docker-compose.yaml rename to docker/openadkit/test/test-scenario.docker-compose.yaml index 1e03a8e4175..adf16968c13 100644 --- a/docker/autoware-openadk/test/test-scenario.docker-compose.yaml +++ b/docker/openadkit/test/test-scenario.docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.17" services: planning-control: - image: ghcr.io/autowarefoundation/autoware-openadk:latest-planning-control + image: ghcr.io/autowarefoundation/autoware-openadkit:latest-planning-control volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ @@ -12,7 +12,7 @@ services: command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-scenario/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false simulator: - image: ghcr.io/autowarefoundation/autoware-openadk:latest-simulator + image: ghcr.io/autowarefoundation/autoware-openadkit:latest-simulator volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ @@ -22,7 +22,7 @@ services: command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-scenario/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-scenario/map launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 visualizer: - image: ghcr.io/autowarefoundation/autoware-openadk:latest-visualizer + image: ghcr.io/autowarefoundation/autoware-openadkit:latest-visualizer volumes: - /dev/shm:/dev/shm - /tmp/.X11-unix:/tmp/.X11-unix diff --git a/docker/autoware/run.sh b/docker/run.sh similarity index 99% rename from docker/autoware/run.sh rename to docker/run.sh index 4e4da08bb78..08efee36888 100755 --- a/docker/autoware/run.sh +++ b/docker/run.sh @@ -10,7 +10,7 @@ BLUE='\033[0;34m' NC='\033[0m' # No Color SCRIPT_DIR=$(readlink -f "$(dirname "$0")") -WORKSPACE_ROOT="$SCRIPT_DIR/../.." +WORKSPACE_ROOT="$SCRIPT_DIR/.." source "$WORKSPACE_ROOT/amd64.env" if [ "$(uname -m)" = "aarch64" ]; then source "$WORKSPACE_ROOT/arm64.env" diff --git a/setup-dev-env.sh b/setup-dev-env.sh index 48afecc4444..0c5cf960165 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -133,7 +133,7 @@ if [ "$option_yes" = "true" ] || [ "$option_download_artifacts" = "true" ]; then fi # Check downloading artifacts -if [ "$target_playbook" = "autoware.dev_env.docker" ] || [ "$target_playbook" = "autoware.dev_env.openadk" ]; then +if [ "$target_playbook" = "autoware.dev_env.openadkit" ]; then if [ "$option_download_artifacts" = "true" ]; then echo -e "\e[36mArtifacts will be downloaded to $option_data_dir\e[m" ansible_args+=("--extra-vars" "prompt_download_artifacts=y") From 4df70c2ec12b8a9c71f8b3f9c4145c6a3d45ac17 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 17 May 2024 18:21:14 +0300 Subject: [PATCH 08/46] switch from autoware-openadkit to openadkit for simplification Signed-off-by: Oguz Ozturk --- .../action.yaml | 0 .github/workflows/update-docker-manifest.yaml | 2 +- docker/docker-bake.hcl | 10 ++++++++-- docker/openadkit/build.sh | 8 ++++---- docker/openadkit/docker-bake.hcl | 9 +-------- docker/openadkit/modules/planning-control/Dockerfile | 2 +- docker/openadkit/modules/simulator/Dockerfile | 4 ++-- docker/openadkit/modules/visualizer/Dockerfile | 2 +- .../openadkit/test/test-scenario.docker-compose.yaml | 6 +++--- 9 files changed, 21 insertions(+), 22 deletions(-) rename .github/actions/{docker-build-and-push-openadk => docker-build-and-push-openadkit}/action.yaml (100%) diff --git a/.github/actions/docker-build-and-push-openadk/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml similarity index 100% rename from .github/actions/docker-build-and-push-openadk/action.yaml rename to .github/actions/docker-build-and-push-openadkit/action.yaml diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml index c98378d7041..e90ed2aa9fb 100644 --- a/.github/workflows/update-docker-manifest.yaml +++ b/.github/workflows/update-docker-manifest.yaml @@ -20,4 +20,4 @@ jobs: - name: Combine multi arch images for 'openadkit' uses: ./.github/actions/combine-multi-arch-images with: - package-name: autoware-openadkit + package-name: openadkit diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index c716e3e95b6..95aa124fb4b 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -1,13 +1,19 @@ group "default" { - targets = ["prebuilt", "devel", "runtime"] + targets = ["base", "prebuilt", "devel", "runtime"] } // For docker/metadata-action +target "docker-metadata-action-base" {} target "docker-metadata-action-prebuilt" {} target "docker-metadata-action-devel" {} target "docker-metadata-action-runtime" {} -// Autoware monolithic +target "base" { + inherits = ["docker-metadata-action-base"] + dockerfile = "docker/Dockerfile" + target = "base" +} + target "prebuilt" { inherits = ["docker-metadata-action-prebuilt"] dockerfile = "docker/Dockerfile" diff --git a/docker/openadkit/build.sh b/docker/openadkit/build.sh index 0abaa223b4e..984046e3f17 100755 --- a/docker/openadkit/build.sh +++ b/docker/openadkit/build.sh @@ -106,10 +106,10 @@ build_images() { --set "*.args.BASE_IMAGE=$base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ - --set "base.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-base" \ - --set "planning-control.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-planning-control" \ - --set "visualizer.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-visualizer" \ - --set "simulator.tags=ghcr.io/autowarefoundation/autoware-openadkit:latest-simulator" \ + --set "base.tags=ghcr.io/autowarefoundation/openadkit:latest-base" \ + --set "planning-control.tags=ghcr.io/autowarefoundation/openadkit:latest-planning-control" \ + --set "visualizer.tags=ghcr.io/autowarefoundation/openadkit:latest-visualizer" \ + --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" \ "${targets[@]}" set +x } diff --git a/docker/openadkit/docker-bake.hcl b/docker/openadkit/docker-bake.hcl index 8c5d1f5d3f6..eb589d9872d 100644 --- a/docker/openadkit/docker-bake.hcl +++ b/docker/openadkit/docker-bake.hcl @@ -1,20 +1,13 @@ group "default" { - targets = ["base", "planning-control", "simulator", "visualizer"] + targets = ["planning-control", "simulator", "visualizer"] } // For docker/metadata-action -target "docker-metadata-action-base" {} target "docker-metadata-action-planning-control" {} target "docker-metadata-action-visualizer" {} target "docker-metadata-action-simulator" {} // Autoware openadkit modules -target "base" { - inherits = ["docker-metadata-action-base"] - dockerfile = "docker/autoware/Dockerfile" - target = "base" -} - target "planning-control" { inherits = ["docker-metadata-action-planning-control"] dockerfile = "docker/openadkit/modules/planning-control/Dockerfile" diff --git a/docker/openadkit/modules/planning-control/Dockerfile b/docker/openadkit/modules/planning-control/Dockerfile index b0cf18666f7..48e6eea52b5 100644 --- a/docker/openadkit/modules/planning-control/Dockerfile +++ b/docker/openadkit/modules/planning-control/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as planning-control +FROM ghcr.io/autowarefoundation/autoware:latest-base as planning-control SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR diff --git a/docker/openadkit/modules/simulator/Dockerfile b/docker/openadkit/modules/simulator/Dockerfile index b8f38daacf9..099c46b299b 100644 --- a/docker/openadkit/modules/simulator/Dockerfile +++ b/docker/openadkit/modules/simulator/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as simulator +FROM ghcr.io/autowarefoundation/autoware:latest-base as simulator SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG PLATFORM ARG LIB_DIR @@ -32,7 +32,7 @@ RUN --mount=type=ssh \ /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* -COPY --from=ghcr.io/autowarefoundation/autoware-openadkit:latest-prebuilt /autoware/install /autoware/install +COPY --from=ghcr.io/autowarefoundation/autoware:latest-prebuilt /autoware/install /autoware/install # Create entrypoint COPY docker/openadkit/etc/ros_entrypoint.sh /ros_entrypoint.sh diff --git a/docker/openadkit/modules/visualizer/Dockerfile b/docker/openadkit/modules/visualizer/Dockerfile index 39d64d4ae4c..1c804d501e0 100644 --- a/docker/openadkit/modules/visualizer/Dockerfile +++ b/docker/openadkit/modules/visualizer/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/autowarefoundation/autoware-openadkit:latest-base as visualizer +FROM ghcr.io/autowarefoundation/autoware:latest-base as visualizer SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR diff --git a/docker/openadkit/test/test-scenario.docker-compose.yaml b/docker/openadkit/test/test-scenario.docker-compose.yaml index adf16968c13..94a038109c5 100644 --- a/docker/openadkit/test/test-scenario.docker-compose.yaml +++ b/docker/openadkit/test/test-scenario.docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.17" services: planning-control: - image: ghcr.io/autowarefoundation/autoware-openadkit:latest-planning-control + image: ghcr.io/autowarefoundation/openadkit:latest-planning-control volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ @@ -12,7 +12,7 @@ services: command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-scenario/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false simulator: - image: ghcr.io/autowarefoundation/autoware-openadkit:latest-simulator + image: ghcr.io/autowarefoundation/openadkit:latest-simulator volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ @@ -22,7 +22,7 @@ services: command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-scenario/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-scenario/map launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 visualizer: - image: ghcr.io/autowarefoundation/autoware-openadkit:latest-visualizer + image: ghcr.io/autowarefoundation/openadkit:latest-visualizer volumes: - /dev/shm:/dev/shm - /tmp/.X11-unix:/tmp/.X11-unix From 39b3cd025c157bb974bebffede4128f94f40078a Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 17 May 2024 18:45:30 +0300 Subject: [PATCH 09/46] update documentation Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push/action.yaml | 15 +++++++++++++-- docker/README.md | 12 +++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index faa5cb2104d..df56e3e8823 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -53,6 +53,17 @@ runs: echo "$EOF" >> $GITHUB_OUTPUT shell: bash + - name: Docker meta for base + id: meta-base + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-base + flavor: | + latest=false + suffix=-base${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} + - name: Docker meta for prebuilt id: meta-prebuilt uses: docker/metadata-action@v4 @@ -100,7 +111,7 @@ runs: with: push: ${{ inputs.allow-push == 'true' }} files: | - docker/${{ inputs.bake-target }}/docker-bake.hcl + docker/docker-bake.hcl ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} @@ -114,7 +125,7 @@ runs: with: push: false files: | - docker/${{ inputs.bake-target }}/docker-bake.hcl + docker/docker-bake.hcl ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} diff --git a/docker/README.md b/docker/README.md index a569443bf01..57e1c1370dc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,11 @@ -# Open AD Kit: containerized workloads for Autoware +## Autoware Docker 🐳 -[Open AD Kit](https://autoware.org/open-ad-kit/) offers containers for Autoware to simplify the development and deployment of Autoware and its dependencies. This directory contains scripts to build and run the containers. +This directory contains scripts to build and run the containers for Autoware both for development(devel image) and deployment(runtime image). -Detailed instructions on how to use the containers can be found in the [Open AD Kit documentation](https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/docker-installation/). +## Open AD Kit: containerized workloads for Autoware + +[Open AD Kit](https://autoware.org/open-ad-kit/) is a set of containerized workloads for Autoware that aims to enable modular and scalable deployment of Autoware. It is a part of the Autoware Foundation's effort to make Autoware software-defined and cloud-native. + +**Detailed instructions on how to use the containers can be found in the [Open AD Kit documentation](https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/docker-installation/).** + +> ℹ️ Open AD Kit is a work in progress, and aiming its first official release Q3/2024. We welcome contributions and feedback. Please join the discussion on the [Autoware Foundation Discord](https://discord.com/channels/953808765935816715/953877118415151205). From efb80e5fbe547b9dd305ab0bfce0137f60b5aab2 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 17 May 2024 18:55:10 +0300 Subject: [PATCH 10/46] add base metadata into autoware Signed-off-by: Oguz Ozturk --- .../action.yaml | 31 +------------------ .../actions/docker-build-and-push/action.yaml | 15 +++++++++ 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 10c3d3cb3db..7659bf53da2 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -53,18 +53,7 @@ runs: echo "$EOF" >> $GITHUB_OUTPUT shell: bash - # openadkit modular images meta - - name: Docker meta for openadkit:base - id: meta-base - uses: docker/metadata-action@v4 - with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} - tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-base - flavor: | - latest=false - suffix=-base-${{ inputs.tag-arch }} - + # openadkit modular images metadata - name: Docker meta for openadkit:planning-control id: meta-planning-control uses: docker/metadata-action@v4 @@ -113,13 +102,10 @@ runs: push: ${{ inputs.allow-push == 'true' }} files: | docker/${{ inputs.bake-target }}/docker-bake.hcl - ${{ steps.meta-base.outputs.bake-file }} ${{ steps.meta-planning-control.outputs.bake-file }} ${{ steps.meta-visualizer.outputs.bake-file }} ${{ steps.meta-simulator.outputs.bake-file }} provenance: false - targets: | - base set: | ${{ inputs.build-args }} @@ -130,30 +116,16 @@ runs: push: false files: | docker/${{ inputs.bake-target }}/docker-bake.hcl - ${{ steps.meta-base.outputs.bake-file }} ${{ steps.meta-planning-control.outputs.bake-file }} ${{ steps.meta-visualizer.outputs.bake-file }} ${{ steps.meta-simulator.outputs.bake-file }} provenance: false set: | ${{ inputs.build-args }} - base.output=type=docker,dest=/tmp/base.tar planning-control.output=type=docker,dest=/tmp/planning-control.tar visualizer.output=type=docker,dest=/tmp/visualizer.tar simulator.output=type=docker,dest=/tmp/simulator.tar - - name: Upload Artifact - base - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-base - uses: actions/upload-artifact@v4 - with: - name: base-image${{ inputs.tag-suffix }} - path: /tmp/base.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: error - - name: Upload Artifact - planning-control if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: artifact-upload-step-planning-control @@ -194,7 +166,6 @@ runs: if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: output-artifact-urls run: | - echo 'base URL ${{ steps.artifact-upload-step-base.outputs.artifact-url }}' echo 'planning-control URL ${{ steps.artifact-upload-step-planning-control.outputs.artifact-url }}' echo 'visualizer URL ${{ steps.artifact-upload-step-visualizer.outputs.artifact-url }}' echo 'simulator URL ${{ steps.artifact-upload-step-simulator.outputs.artifact-url }}' diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index df56e3e8823..c030c7f5a45 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -112,6 +112,7 @@ runs: push: ${{ inputs.allow-push == 'true' }} files: | docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} @@ -126,6 +127,7 @@ runs: push: false files: | docker/docker-bake.hcl + ${{ steps.meta-base.outputs.bake-file }} ${{ steps.meta-prebuilt.outputs.bake-file }} ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} @@ -136,6 +138,18 @@ runs: devel.output=type=docker,dest=/tmp/devel.tar runtime.output=type=docker,dest=/tmp/runtime.tar + - name: Upload Artifact - base + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + id: artifact-upload-step-base + uses: actions/upload-artifact@v4 + with: + name: base-image${{ inputs.tag-suffix }} + path: /tmp/base.tar + retention-days: 7 + compression-level: 6 + overwrite: true + if-no-files-found: error + - name: Upload Artifact - prebuilt if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: artifact-upload-step-prebuilt @@ -194,6 +208,7 @@ runs: if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: output-artifact-urls run: | + echo 'base URL ${{ steps.artifact-upload-step-base.outputs.artifact-url }}' echo 'prebuilt URL ${{ steps.artifact-upload-step-prebuilt.outputs.artifact-url }}' echo 'devel URL ${{ steps.artifact-upload-step-devel.outputs.artifact-url }}' echo 'runtime URL ${{ steps.artifact-upload-step-runtime.outputs.artifact-url }}' From ff54c31229f303f56a11b3e43ed7892224b9920b Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 17 May 2024 19:35:17 +0300 Subject: [PATCH 11/46] add base to build.sh Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push/action.yaml | 2 +- docker/build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index c030c7f5a45..4a67ff3e6e9 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -62,7 +62,7 @@ runs: bake-target: docker-metadata-action-base flavor: | latest=false - suffix=-base${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} + suffix=-base$-${{ inputs.tag-arch }} - name: Docker meta for prebuilt id: meta-prebuilt diff --git a/docker/build.sh b/docker/build.sh index 3d0a417c952..6ee035b8b76 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -119,6 +119,7 @@ build_images() { --set "*.args.BASE_IMAGE=$base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ + --set "base.tags=ghcr.io/autowarefoundation/autoware:latest-base" \ --set "devel.tags=ghcr.io/autowarefoundation/autoware:latest-devel$image_name_suffix" \ --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware:latest-prebuilt$image_name_suffix" \ --set "runtime.tags=ghcr.io/autowarefoundation/autoware:latest-runtime$image_name_suffix" \ From 450ce3e18b78622cf78de0c0685a056f7035dc2d Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 11:34:45 +0300 Subject: [PATCH 12/46] fix docker tags Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push/action.yaml | 8 ++++---- .../workflows/docker-build-and-push-main-self-hosted.yaml | 5 +++-- .github/workflows/docker-build-and-push-main.yaml | 4 ++-- docker/openadkit/modules/planning-control/Dockerfile | 3 ++- docker/openadkit/modules/simulator/Dockerfile | 3 ++- docker/openadkit/modules/visualizer/Dockerfile | 3 ++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 4a67ff3e6e9..63e9c156849 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -62,7 +62,7 @@ runs: bake-target: docker-metadata-action-base flavor: | latest=false - suffix=-base$-${{ inputs.tag-arch }} + suffix=-base${{ inputs.tag-arch }} - name: Docker meta for prebuilt id: meta-prebuilt @@ -73,7 +73,7 @@ runs: bake-target: docker-metadata-action-prebuilt flavor: | latest=false - suffix=-prebuilt${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} + suffix=-prebuilt${{ inputs.tag-suffix }}${{ inputs.tag-arch }} - name: Docker meta for devel id: meta-devel @@ -84,7 +84,7 @@ runs: bake-target: docker-metadata-action-devel flavor: | latest=false - suffix=-devel${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} + suffix=-devel${{ inputs.tag-suffix }}${{ inputs.tag-arch }} - name: Docker meta for runtime id: meta-runtime @@ -95,7 +95,7 @@ runs: bake-target: docker-metadata-action-runtime flavor: | latest=${{ github.event_name == 'push' && github.ref_type == 'tag' }} - suffix=-runtime${{ inputs.tag-suffix }}-${{ inputs.tag-arch }} + suffix=-runtime${{ inputs.tag-suffix }}${{ inputs.tag-arch }} - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 2e7fe758342..1e6df865601 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -88,7 +88,7 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max tag-suffix: ${{ matrix.additional-tag-suffix }} - tag-arch: arm64 + tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true @@ -102,8 +102,9 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} + *.args.ARCH=$(echo "$platform" | cut -d'/' -f2) tag-suffix: ${{ matrix.additional-tag-suffix }} - tag-arch: arm64 + tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 53453ecbad1..1c0224be56d 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -81,7 +81,7 @@ jobs: *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} tag-suffix: ${{ matrix.additional-tag-suffix }} - tag-arch: amd64 + tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true @@ -98,7 +98,7 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max tag-suffix: ${{ matrix.additional-tag-suffix }} - tag-arch: amd64 + tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true diff --git a/docker/openadkit/modules/planning-control/Dockerfile b/docker/openadkit/modules/planning-control/Dockerfile index 48e6eea52b5..b3d3f3dcd1f 100644 --- a/docker/openadkit/modules/planning-control/Dockerfile +++ b/docker/openadkit/modules/planning-control/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/autowarefoundation/autoware:latest-base as planning-control +ARG ARCH +FROM ghcr.io/autowarefoundation/autoware:latest-base-${ARCH} as planning-control SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR diff --git a/docker/openadkit/modules/simulator/Dockerfile b/docker/openadkit/modules/simulator/Dockerfile index 099c46b299b..494390e52c4 100644 --- a/docker/openadkit/modules/simulator/Dockerfile +++ b/docker/openadkit/modules/simulator/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/autowarefoundation/autoware:latest-base as simulator +ARG ARCH +FROM ghcr.io/autowarefoundation/autoware:latest-base-${ARCH} as simulator SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG PLATFORM ARG LIB_DIR diff --git a/docker/openadkit/modules/visualizer/Dockerfile b/docker/openadkit/modules/visualizer/Dockerfile index 1c804d501e0..5223f2cac99 100644 --- a/docker/openadkit/modules/visualizer/Dockerfile +++ b/docker/openadkit/modules/visualizer/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/autowarefoundation/autoware:latest-base as visualizer +ARG ARCH +FROM ghcr.io/autowarefoundation/autoware:latest-base-${ARCH} as visualizer SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR From 471e60e1d1835d2f908077da75fa03e0f92116e5 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 14:04:52 +0300 Subject: [PATCH 13/46] update readme Signed-off-by: Oguz Ozturk --- docker/README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docker/README.md b/docker/README.md index 57e1c1370dc..81329731727 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,11 +1,5 @@ -## Autoware Docker 🐳 +## 🐳 Open AD Kit: containerized workloads for Autoware -This directory contains scripts to build and run the containers for Autoware both for development(devel image) and deployment(runtime image). +This directory contains scripts to build and run the containers for Autoware both for development(devel image) and deployment(runtime image). Detailed instructions on how to use the containers can be found in the [Open AD Kit documentation](https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/docker-installation/). -## Open AD Kit: containerized workloads for Autoware - -[Open AD Kit](https://autoware.org/open-ad-kit/) is a set of containerized workloads for Autoware that aims to enable modular and scalable deployment of Autoware. It is a part of the Autoware Foundation's effort to make Autoware software-defined and cloud-native. - -**Detailed instructions on how to use the containers can be found in the [Open AD Kit documentation](https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/docker-installation/).** - -> ℹ️ Open AD Kit is a work in progress, and aiming its first official release Q3/2024. We welcome contributions and feedback. Please join the discussion on the [Autoware Foundation Discord](https://discord.com/channels/953808765935816715/953877118415151205). +> ℹ️ [Open AD Kit](https://autoware.org/open-ad-kit/) offers a set of modular workloads designed to enable scalable deployment of Autoware. This initiative is part of the Autoware Foundation's efforts to transition Autoware into a software-defined and cloud-native solution . Currently, the project is in progress, with its first official release targeted for Q3/2024. We welcome contributions and feedback, please join our discussions on the [Autoware Foundation Discord](https://discord.com/channels/953808765935816715/953877118415151205). From fba279e55e69f50a5f3fe34b585497c6cd3ce63a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 11:35:49 +0000 Subject: [PATCH 14/46] style(pre-commit): autofix --- docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run.sh b/docker/run.sh index 08efee36888..23fbd89f35c 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -129,7 +129,7 @@ set_variables() { exit 1 else MAP="-v ${MAP_PATH}:/autoware_map:ro" - fi + fi # Set default launch command if not provided if [ "$LAUNCH_CMD" = "" ]; then From 96b7b9b5e1256d81eb0c715bb7d49910306d6667 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 14:46:21 +0300 Subject: [PATCH 15/46] . Signed-off-by: Oguz Ozturk --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 1e6df865601..906a915ba2a 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -102,7 +102,7 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} - *.args.ARCH=$(echo "$platform" | cut -d'/' -f2) + *.args.ARCH: ${{ env.platform.split('/')[1] }} tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 1c0224be56d..ffa87c52b81 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -97,6 +97,7 @@ jobs: *.args.LIB_DIR=${{ matrix.lib_dir }} *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max + *.args.ARCH: ${{ env.platform.split('/')[1] }} tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} From 0c4571aa564327726913c1bc25e15e327268fc5f Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 14:51:38 +0300 Subject: [PATCH 16/46] . Signed-off-by: Oguz Ozturk --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 906a915ba2a..0ee376d6f6f 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -102,7 +102,7 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} - *.args.ARCH: ${{ env.platform.split('/')[1] }} + *.args.ARCH: arm64 tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index ffa87c52b81..7948dc70672 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -97,7 +97,7 @@ jobs: *.args.LIB_DIR=${{ matrix.lib_dir }} *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max - *.args.ARCH: ${{ env.platform.split('/')[1] }} + *.args.ARCH: amd64 tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} From 1788b7fa7f84a7133ff5f2d92218912380455067 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 14:55:25 +0300 Subject: [PATCH 17/46] . Signed-off-by: Oguz Ozturk --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 0ee376d6f6f..5bc526c5d1b 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -9,7 +9,7 @@ name: docker-build-and-push-main-self-hosted on: push: tags: - - adkit-v*.*.* + - openadkit-v*.*.* branches: - main schedule: diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 7948dc70672..70025f848f5 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -9,7 +9,7 @@ name: docker-build-and-push-main on: push: tags: - - adkit-v*.*.* + - openadkit-v*.*.* branches: - main schedule: From 1a9c784dbb16f0341df27d43a119e605c1f9048b Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 17:32:46 +0300 Subject: [PATCH 18/46] add base tar Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 63e9c156849..c770f1e331a 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -134,6 +134,7 @@ runs: provenance: false set: | ${{ inputs.build-args }} + base.output=type=docker,dest=/tmp/base.tar prebuilt.output=type=docker,dest=/tmp/prebuilt.tar devel.output=type=docker,dest=/tmp/devel.tar runtime.output=type=docker,dest=/tmp/runtime.tar From f0bd4fb8692b3dc91c5cfe344236bb732a4bb291 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 20 May 2024 20:35:38 +0300 Subject: [PATCH 19/46] fix typo Signed-off-by: Oguz Ozturk --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 5bc526c5d1b..29a8d197e29 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -102,7 +102,7 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} - *.args.ARCH: arm64 + *.args.ARCH=arm64 tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 70025f848f5..640dfa4281d 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -97,7 +97,7 @@ jobs: *.args.LIB_DIR=${{ matrix.lib_dir }} *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max - *.args.ARCH: amd64 + *.args.ARCH=amd64 tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} From 4b34a4a2e84343e66dcbe37c88ae005cc295cf3c Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 21 May 2024 13:07:26 +0300 Subject: [PATCH 20/46] ommit building all images in all matrix threads Signed-off-by: Oguz Ozturk --- .../actions/docker-build-and-push-openadkit/action.yaml | 7 +++++++ .github/actions/docker-build-and-push/action.yaml | 7 +++++++ .../workflows/docker-build-and-push-main-self-hosted.yaml | 5 +++++ .github/workflows/docker-build-and-push-main.yaml | 5 +++++ 4 files changed, 24 insertions(+) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 7659bf53da2..e67cbd85e07 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -5,6 +5,9 @@ inputs: bake-target: description: "" required: true + bake-images: + description: "" + required: false build-args: description: "" required: false @@ -106,6 +109,8 @@ runs: ${{ steps.meta-visualizer.outputs.bake-file }} ${{ steps.meta-simulator.outputs.bake-file }} provenance: false + targets: | + ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} @@ -120,6 +125,8 @@ runs: ${{ steps.meta-visualizer.outputs.bake-file }} ${{ steps.meta-simulator.outputs.bake-file }} provenance: false + targets: | + ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} planning-control.output=type=docker,dest=/tmp/planning-control.tar diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index c770f1e331a..76cfecf405c 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -5,6 +5,9 @@ inputs: bake-target: description: "" required: true + bake-images: + description: "" + required: false build-args: description: "" required: false @@ -117,6 +120,8 @@ runs: ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} provenance: false + targets: | + ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} @@ -132,6 +137,8 @@ runs: ${{ steps.meta-devel.outputs.bake-file }} ${{ steps.meta-runtime.outputs.bake-file }} provenance: false + targets: | + ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} base.output=type=docker,dest=/tmp/base.tar diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 29a8d197e29..e5f16884c32 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -43,10 +43,12 @@ jobs: lib_dir: aarch64 setup-args: --no-nvidia additional-tag-suffix: "" + bake-images: [default] - name: cuda base_image_env: base_image lib_dir: aarch64 additional-tag-suffix: -cuda + bake-images: [prebuilt, devel, runtime] steps: # https://github.com/actions/checkout/issues/211 - name: Change permission of workspace @@ -90,9 +92,11 @@ jobs: tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + bake-images: ${{ matrix.bake-images }} allow-push: true - name: Build 'Openadkit' + if: ${{ matrix.name == 'no-cuda' }} uses: ./.github/actions/docker-build-and-push-openadkit with: bake-target: openadkit @@ -106,6 +110,7 @@ jobs: tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -arm64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + bake-images: ${{ matrix.bake-images }} allow-push: true - name: Show disk space diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 640dfa4281d..4238b36cd68 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -43,10 +43,12 @@ jobs: lib_dir: x86_64 setup-args: --no-nvidia additional-tag-suffix: "" + bake-images: [default] - name: cuda base_image_env: base_image lib_dir: x86_64 additional-tag-suffix: -cuda + bake-images: [prebuilt, devel, runtime] steps: - name: Check out repository uses: actions/checkout@v4 @@ -83,9 +85,11 @@ jobs: tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + bake-images: ${{ matrix.bake-images }} allow-push: true - name: Build 'Openadkit' + if: ${{ matrix.name == 'no-cuda' }} uses: ./.github/actions/docker-build-and-push-openadkit with: bake-target: openadkit @@ -101,6 +105,7 @@ jobs: tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} + bake-images: ${{ matrix.bake-images }} allow-push: true - name: Show disk space From 31f1a70f82b3052f2d1c1ef1b0b64b5fbf933f17 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 21 May 2024 13:10:02 +0300 Subject: [PATCH 21/46] . Signed-off-by: Oguz Ozturk --- .../actions/docker-build-and-push-openadkit/action.yaml | 6 +++--- .github/actions/docker-build-and-push/action.yaml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index e67cbd85e07..082dbb3b0d4 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -143,7 +143,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Upload Artifact - visualizer if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} @@ -155,7 +155,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Upload Artifact - simulator if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} @@ -167,7 +167,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 76cfecf405c..4c73b2840c8 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -156,7 +156,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Upload Artifact - prebuilt if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} @@ -168,7 +168,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Remove tar file - prebuilt if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} @@ -186,7 +186,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Remove tar file - devel if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} @@ -204,7 +204,7 @@ runs: retention-days: 7 compression-level: 6 overwrite: true - if-no-files-found: error + if-no-files-found: warn - name: Remove tar file - runtime if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} From 39fb79151e3a62d6014a7aa961c349e9de2ace5b Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 21 May 2024 15:18:13 +0300 Subject: [PATCH 22/46] fix typo Signed-off-by: Oguz Ozturk --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 4 ++-- .github/workflows/docker-build-and-push-main.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index e5f16884c32..18f27689cbc 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -43,12 +43,12 @@ jobs: lib_dir: aarch64 setup-args: --no-nvidia additional-tag-suffix: "" - bake-images: [default] + bake-images: default - name: cuda base_image_env: base_image lib_dir: aarch64 additional-tag-suffix: -cuda - bake-images: [prebuilt, devel, runtime] + bake-images: prebuilt,devel,runtime steps: # https://github.com/actions/checkout/issues/211 - name: Change permission of workspace diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 4238b36cd68..1c889f4a7bb 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -43,12 +43,12 @@ jobs: lib_dir: x86_64 setup-args: --no-nvidia additional-tag-suffix: "" - bake-images: [default] + bake-images: default - name: cuda base_image_env: base_image lib_dir: x86_64 additional-tag-suffix: -cuda - bake-images: [prebuilt, devel, runtime] + bake-images: prebuilt,devel,runtime steps: - name: Check out repository uses: actions/checkout@v4 From 803df77a27eaee38c69a6e2cf32d2168e6be1f00 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 21 May 2024 16:01:33 +0300 Subject: [PATCH 23/46] update readme Signed-off-by: Oguz Ozturk --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 81329731727..067ebd970ed 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,4 +2,4 @@ This directory contains scripts to build and run the containers for Autoware both for development(devel image) and deployment(runtime image). Detailed instructions on how to use the containers can be found in the [Open AD Kit documentation](https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/docker-installation/). -> ℹ️ [Open AD Kit](https://autoware.org/open-ad-kit/) offers a set of modular workloads designed to enable scalable deployment of Autoware. This initiative is part of the Autoware Foundation's efforts to transition Autoware into a software-defined and cloud-native solution . Currently, the project is in progress, with its first official release targeted for Q3/2024. We welcome contributions and feedback, please join our discussions on the [Autoware Foundation Discord](https://discord.com/channels/953808765935816715/953877118415151205). +> ℹ️ [Open AD Kit](https://autoware.org/open-ad-kit/) also offers a set of modular workloads designed to enable scalable deployment of Autoware. This initiative is part of the Autoware Foundation's efforts to transition Autoware into a software-defined and cloud-native solution . Currently, the project is in progress, with its first official release targeted for Q3/2024. We welcome contributions and feedback, please join our discussions on the [Autoware Foundation Discord](https://discord.com/channels/953808765935816715/953877118415151205). From cc841e259e2e8de19b66b3c899321b6cb7e8cda6 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Wed, 22 May 2024 17:22:24 +0300 Subject: [PATCH 24/46] fix ansible style-check Signed-off-by: Oguz Ozturk --- docker/openadkit/test/test-scenario.docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/openadkit/test/test-scenario.docker-compose.yaml b/docker/openadkit/test/test-scenario.docker-compose.yaml index 94a038109c5..f3aca988c59 100644 --- a/docker/openadkit/test/test-scenario.docker-compose.yaml +++ b/docker/openadkit/test/test-scenario.docker-compose.yaml @@ -6,7 +6,7 @@ services: volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ - network_mode: "host" + network_mode: host environment: - ROS_LOCALHOST_ONLY=1 command: ros2 launch autoware_launch planning_simulator.launch.xml map_path:=/autoware/test-scenario/map/ vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit scenario_simulation:=true use_foa:=false perception/enable_traffic_light:=false rviz:=false @@ -16,7 +16,7 @@ services: volumes: - /dev/shm:/dev/shm - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ - network_mode: "host" + network_mode: host environment: - ROS_LOCALHOST_ONLY=1 command: ros2 launch scenario_test_runner scenario_test_runner.launch.py architecture_type:=awf/universe record:=false scenario:=/autoware/test-scenario/yield_maneuver_demo.yaml sensor_model:=sample_sensor_kit vehicle_model:=sample_vehicle map_path:=/autoware/test-scenario/map launch_autoware:=False initialize_duration:=60 global_frame_rate:=12 @@ -27,7 +27,7 @@ services: - /dev/shm:/dev/shm - /tmp/.X11-unix:/tmp/.X11-unix - /home/${USER}/autoware_artifacts/test-artifacts:/autoware/test-scenario/ - network_mode: "host" + network_mode: host environment: - DISPLAY=$DISPLAY - ROS_LOCALHOST_ONLY=1 From ef5d1a6daa32a90aacdfe64af068a4d3834c0169 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Wed, 22 May 2024 17:26:36 +0300 Subject: [PATCH 25/46] fix stylecheck Signed-off-by: Oguz Ozturk --- docker/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 6ee035b8b76..d92e783bf75 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -58,10 +58,9 @@ set_cuda_options() { # Set build options set_build_options() { + targets=() if [ "$option_devel_only" = "true" ]; then targets=("devel") - else - targets=() fi } From 1df309840d6003e7281ce46358de056973bb7f49 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Wed, 22 May 2024 17:32:25 +0300 Subject: [PATCH 26/46] . Signed-off-by: Oguz Ozturk --- docker/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/build.sh b/docker/build.sh index d92e783bf75..f700965763d 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -109,6 +109,7 @@ build_images() { echo "Image name suffix: $image_name_suffix" echo "Targets: ${targets[*]}" + # shellcheck disable=SC2154 set -x docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/docker-bake.hcl" \ --set "*.context=$WORKSPACE_ROOT" \ From 99514cfda3b0fb1dabfe86612b94b54533f087cf Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Wed, 22 May 2024 17:22:43 +0300 Subject: [PATCH 27/46] remove targets from openadki build.sh Signed-off-by: Oguz Ozturk --- docker/openadkit/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/openadkit/build.sh b/docker/openadkit/build.sh index 984046e3f17..910c3801b2a 100755 --- a/docker/openadkit/build.sh +++ b/docker/openadkit/build.sh @@ -95,7 +95,6 @@ build_images() { echo "Setup args: $setup_args" echo "Lib dir: $lib_dir" echo "Image name suffix: $image_name_suffix" - echo "Targets: ${targets[*]}" set -x docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/docker-bake.hcl" \ @@ -109,8 +108,7 @@ build_images() { --set "base.tags=ghcr.io/autowarefoundation/openadkit:latest-base" \ --set "planning-control.tags=ghcr.io/autowarefoundation/openadkit:latest-planning-control" \ --set "visualizer.tags=ghcr.io/autowarefoundation/openadkit:latest-visualizer" \ - --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" \ - "${targets[@]}" + --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" set +x } From fc237f8b4b79215233b68a64ce044b716169c14e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 17:22:59 +0000 Subject: [PATCH 28/46] style(pre-commit): autofix --- docker/openadkit/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/openadkit/build.sh b/docker/openadkit/build.sh index 910c3801b2a..4eed829f328 100755 --- a/docker/openadkit/build.sh +++ b/docker/openadkit/build.sh @@ -108,7 +108,7 @@ build_images() { --set "base.tags=ghcr.io/autowarefoundation/openadkit:latest-base" \ --set "planning-control.tags=ghcr.io/autowarefoundation/openadkit:latest-planning-control" \ --set "visualizer.tags=ghcr.io/autowarefoundation/openadkit:latest-visualizer" \ - --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" + --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" set +x } From 6608bfeb32e7a20e9c2f10f6b16430a1379a9e9e Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Thu, 23 May 2024 14:00:09 +0300 Subject: [PATCH 29/46] fix https://github.com/autowarefoundation/autoware/issues/4765 Signed-off-by: Oguz Ozturk --- docker/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6049154adaa..6247c18666e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -74,9 +74,7 @@ ENV CXX="/usr/lib/ccache/g++" RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ - && find / -name 'libcu*.a' -delete \ - && find / -name 'libnv*.a' -delete + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Install rosdep dependencies COPY --from=src-imported /rosdep-all-depend-packages.txt /tmp/rosdep-all-depend-packages.txt From eb8b55aeee755c2ca71f67578d1d334897f92977 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 14:00:25 +0000 Subject: [PATCH 30/46] style(pre-commit): autofix --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6247c18666e..f2e5f5c25f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -74,7 +74,7 @@ ENV CXX="/usr/lib/ccache/g++" RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Install rosdep dependencies COPY --from=src-imported /rosdep-all-depend-packages.txt /tmp/rosdep-all-depend-packages.txt From e0d0a3112e802ef1d0a6d5d155f6b4f2e1c69638 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 24 May 2024 12:59:11 +0300 Subject: [PATCH 31/46] Update from reviewers Signed-off-by: Oguz Ozturk --- .../action.yaml | 6 ------ .../actions/docker-build-and-push/action.yaml | 20 ------------------- ...ocker-build-and-push-main-self-hosted.yaml | 4 ++-- .../workflows/docker-build-and-push-main.yaml | 4 ++-- docker/build.sh | 1 - docker/etc/ros_entrypoint.sh | 2 +- docker/openadkit/etc/ros_entrypoint.sh | 2 +- docker/run.sh | 15 ++++++++------ 8 files changed, 15 insertions(+), 39 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 082dbb3b0d4..430c869827b 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -31,12 +31,6 @@ runs: - name: Setup Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Install jq - run: | - sudo apt-get -y update - sudo apt-get -y install jq - shell: bash - - name: Set Docker tags id: set-docker-tags run: | diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 4c73b2840c8..65779cb6f38 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -31,12 +31,6 @@ runs: - name: Setup Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Install jq - run: | - sudo apt-get -y update - sudo apt-get -y install jq - shell: bash - - name: Set Docker tags id: set-docker-tags run: | @@ -141,23 +135,10 @@ runs: ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} - base.output=type=docker,dest=/tmp/base.tar prebuilt.output=type=docker,dest=/tmp/prebuilt.tar devel.output=type=docker,dest=/tmp/devel.tar runtime.output=type=docker,dest=/tmp/runtime.tar - - name: Upload Artifact - base - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-base - uses: actions/upload-artifact@v4 - with: - name: base-image${{ inputs.tag-suffix }} - path: /tmp/base.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - name: Upload Artifact - prebuilt if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: artifact-upload-step-prebuilt @@ -216,7 +197,6 @@ runs: if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: output-artifact-urls run: | - echo 'base URL ${{ steps.artifact-upload-step-base.outputs.artifact-url }}' echo 'prebuilt URL ${{ steps.artifact-upload-step-prebuilt.outputs.artifact-url }}' echo 'devel URL ${{ steps.artifact-upload-step-devel.outputs.artifact-url }}' echo 'runtime URL ${{ steps.artifact-upload-step-runtime.outputs.artifact-url }}' diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 18f27689cbc..4685b4cee3e 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -48,7 +48,7 @@ jobs: base_image_env: base_image lib_dir: aarch64 additional-tag-suffix: -cuda - bake-images: prebuilt,devel,runtime + bake-images: default steps: # https://github.com/actions/checkout/issues/211 - name: Change permission of workspace @@ -95,7 +95,7 @@ jobs: bake-images: ${{ matrix.bake-images }} allow-push: true - - name: Build 'Openadkit' + - name: Build 'Open AD Kit' if: ${{ matrix.name == 'no-cuda' }} uses: ./.github/actions/docker-build-and-push-openadkit with: diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 1c889f4a7bb..e01b5c6406f 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -48,7 +48,7 @@ jobs: base_image_env: base_image lib_dir: x86_64 additional-tag-suffix: -cuda - bake-images: prebuilt,devel,runtime + bake-images: default steps: - name: Check out repository uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: bake-images: ${{ matrix.bake-images }} allow-push: true - - name: Build 'Openadkit' + - name: Build 'Open AD Kit' if: ${{ matrix.name == 'no-cuda' }} uses: ./.github/actions/docker-build-and-push-openadkit with: diff --git a/docker/build.sh b/docker/build.sh index f700965763d..d92e783bf75 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -109,7 +109,6 @@ build_images() { echo "Image name suffix: $image_name_suffix" echo "Targets: ${targets[*]}" - # shellcheck disable=SC2154 set -x docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/docker-bake.hcl" \ --set "*.context=$WORKSPACE_ROOT" \ diff --git a/docker/etc/ros_entrypoint.sh b/docker/etc/ros_entrypoint.sh index cbe45ef3bdb..40dbbee02e9 100644 --- a/docker/etc/ros_entrypoint.sh +++ b/docker/etc/ros_entrypoint.sh @@ -22,7 +22,7 @@ else # Add sudo privileges to the user echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers - # Source ROS2 + # Source ROS 2 # hadolint ignore=SC1090 source "/opt/ros/$ROS_DISTRO/setup.bash" source /autoware/install/setup.bash diff --git a/docker/openadkit/etc/ros_entrypoint.sh b/docker/openadkit/etc/ros_entrypoint.sh index cbe45ef3bdb..40dbbee02e9 100644 --- a/docker/openadkit/etc/ros_entrypoint.sh +++ b/docker/openadkit/etc/ros_entrypoint.sh @@ -22,7 +22,7 @@ else # Add sudo privileges to the user echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers - # Source ROS2 + # Source ROS 2 # hadolint ignore=SC1090 source "/opt/ros/$ROS_DISTRO/setup.bash" source /autoware/install/setup.bash diff --git a/docker/run.sh b/docker/run.sh index 23fbd89f35c..f14cf8a1208 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -37,9 +37,9 @@ print_help() { echo -e "Options:" echo -e " ${GREEN}--help/-h${NC} Display this help message" echo -e " ${GREEN}--data-path${NC} Specify to mount data files into /autoware_data" - echo -e " ${GREEN}--map-path${NC} Specify to mount map files into /autoware_map (mandatory if no custom launch command is provided)" + echo -e " ${GREEN}--map-path${NC} Specify to mount map files into /autoware_map (mandatory for runtime)" echo -e " ${GREEN}--devel${NC} Launch the latest Autoware development environment with shell access" - echo -e " ${GREEN}--workspace${NC} (--devel only)Specify the directory to mount into /workspace, by default it uses current directory (pwd) as workspace" + echo -e " ${GREEN}--workspace${NC} (--devel only)Specify the directory to mount into /workspace, by default it uses current directory (pwd)" echo -e " ${GREEN}--no-nvidia${NC} Disable NVIDIA GPU support" echo -e " ${GREEN}--headless${NC} Run Autoware in headless mode (default: false)" echo "" @@ -93,7 +93,7 @@ parse_arguments() { done } -# Set docker image used and workspace variables +# Set the docker image and workspace variables set_variables() { if [ "$option_devel" = "true" ]; then # Set image based on option @@ -173,9 +173,12 @@ main() { echo -e "${GREEN}Launching Autoware${NC}" fi echo -e "${GREEN}IMAGE:${NC} ${IMAGE}" - echo -e "${GREEN}DATA PATH(mounted):${NC} ${DATA_PATH}:/autoware_data" - echo -e "${GREEN}MAP PATH(mounted):${NC} ${MAP_PATH}:/autoware_map" - echo -e "${GREEN}WORKSPACE(mounted):${NC} ${WORKSPACE_PATH}:/workspace" + if [ "$option_devel" = "true" ]; then + echo -e "${GREEN}WORKSPACE PATH(mounted):${NC} ${WORKSPACE_PATH}:/workspace" + fi + if [ "$MAP_PATH" != "" ]; then + echo -e "${GREEN}MAP PATH(mounted):${NC} ${MAP_PATH}:/autoware_map" + fi echo -e "${GREEN}LAUNCH CMD:${NC} ${LAUNCH_CMD}" echo -e "${GREEN}-----------------------------------------------------------------${NC}" From 24d84f14a17dbfb213efbe54e27ca2509b3af8a6 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Fri, 24 May 2024 18:19:43 +0300 Subject: [PATCH 32/46] remove upload artifacts because of the disk space issue Signed-off-by: Oguz Ozturk --- .../action.yaml | 118 ++++++++------ .../actions/docker-build-and-push/action.yaml | 153 ++++++++++-------- 2 files changed, 158 insertions(+), 113 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 430c869827b..f85bd2a78f1 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -107,8 +107,8 @@ runs: ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} - - - name: Build and Save Artifacts + + - name: Build only if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} uses: docker/bake-action@v3 with: @@ -121,53 +121,75 @@ runs: provenance: false targets: | ${{ inputs.bake-images }} - set: | - ${{ inputs.build-args }} - planning-control.output=type=docker,dest=/tmp/planning-control.tar - visualizer.output=type=docker,dest=/tmp/visualizer.tar - simulator.output=type=docker,dest=/tmp/simulator.tar - - - name: Upload Artifact - planning-control - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-planning-control - uses: actions/upload-artifact@v4 - with: - name: planning-control-image${{ inputs.tag-suffix }} - path: /tmp/planning-control.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - - name: Upload Artifact - visualizer - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-visualizer - uses: actions/upload-artifact@v4 - with: - name: visualizer-image${{ inputs.tag-suffix }} - path: /tmp/visualizer.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - - name: Upload Artifact - simulator - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-simulator - uses: actions/upload-artifact@v4 - with: - name: simulator-image${{ inputs.tag-suffix }} - path: /tmp/simulator.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - + - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: output-artifact-urls + id: warn-no-artifacts run: | - echo 'planning-control URL ${{ steps.artifact-upload-step-planning-control.outputs.artifact-url }}' - echo 'visualizer URL ${{ steps.artifact-upload-step-visualizer.outputs.artifact-url }}' - echo 'simulator URL ${{ steps.artifact-upload-step-simulator.outputs.artifact-url }}' + echo 'No artifacts uploaded because of disk space issue' shell: bash + + # TODO:Enable after solving the issue with the disk space + # - name: Build and Save Artifacts + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # uses: docker/bake-action@v3 + # with: + # push: false + # files: | + # docker/${{ inputs.bake-target }}/docker-bake.hcl + # ${{ steps.meta-planning-control.outputs.bake-file }} + # ${{ steps.meta-visualizer.outputs.bake-file }} + # ${{ steps.meta-simulator.outputs.bake-file }} + # provenance: false + # targets: | + # ${{ inputs.bake-images }} + # set: | + # ${{ inputs.build-args }} + # planning-control.output=type=docker,dest=/tmp/planning-control.tar + # visualizer.output=type=docker,dest=/tmp/visualizer.tar + # simulator.output=type=docker,dest=/tmp/simulator.tar + + # - name: Upload Artifact - planning-control + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-planning-control + # uses: actions/upload-artifact@v4 + # with: + # name: planning-control-image${{ inputs.tag-suffix }} + # path: /tmp/planning-control.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Upload Artifact - visualizer + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-visualizer + # uses: actions/upload-artifact@v4 + # with: + # name: visualizer-image${{ inputs.tag-suffix }} + # path: /tmp/visualizer.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Upload Artifact - simulator + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-simulator + # uses: actions/upload-artifact@v4 + # with: + # name: simulator-image${{ inputs.tag-suffix }} + # path: /tmp/simulator.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Output artifact URLs + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: output-artifact-urls + # run: | + # echo 'planning-control URL ${{ steps.artifact-upload-step-planning-control.outputs.artifact-url }}' + # echo 'visualizer URL ${{ steps.artifact-upload-step-visualizer.outputs.artifact-url }}' + # echo 'simulator URL ${{ steps.artifact-upload-step-simulator.outputs.artifact-url }}' + # shell: bash diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 65779cb6f38..269f607339e 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -118,8 +118,8 @@ runs: ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} - - - name: Build and Save Artifacts + + - name: Build Only if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} uses: docker/bake-action@v3 with: @@ -133,71 +133,94 @@ runs: provenance: false targets: | ${{ inputs.bake-images }} - set: | - ${{ inputs.build-args }} - prebuilt.output=type=docker,dest=/tmp/prebuilt.tar - devel.output=type=docker,dest=/tmp/devel.tar - runtime.output=type=docker,dest=/tmp/runtime.tar - - - name: Upload Artifact - prebuilt - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-prebuilt - uses: actions/upload-artifact@v4 - with: - name: prebuilt-image${{ inputs.tag-suffix }} - path: /tmp/prebuilt.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - - name: Remove tar file - prebuilt - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - run: | - rm /tmp/prebuilt.tar - shell: bash - - - name: Upload Artifact - devel - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-devel - uses: actions/upload-artifact@v4 - with: - name: devel-image${{ inputs.tag-suffix }} - path: /tmp/devel.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - - name: Remove tar file - devel - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - run: | - rm /tmp/devel.tar - shell: bash - - - name: Upload Artifact - runtime - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: artifact-upload-step-runtime - uses: actions/upload-artifact@v4 - with: - name: runtime-image${{ inputs.tag-suffix }} - path: /tmp/runtime.tar - retention-days: 7 - compression-level: 6 - overwrite: true - if-no-files-found: warn - - - name: Remove tar file - runtime - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - run: | - rm /tmp/runtime.tar - shell: bash - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} - id: output-artifact-urls + id: warn-no-artifacts run: | - echo 'prebuilt URL ${{ steps.artifact-upload-step-prebuilt.outputs.artifact-url }}' - echo 'devel URL ${{ steps.artifact-upload-step-devel.outputs.artifact-url }}' - echo 'runtime URL ${{ steps.artifact-upload-step-runtime.outputs.artifact-url }}' + echo 'No artifacts uploaded because of disk space issue' shell: bash + + # TODO:Enable after solving the issue with the disk space + # - name: Build and Save Artifacts + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # uses: docker/bake-action@v3 + # with: + # push: false + # files: | + # docker/docker-bake.hcl + # ${{ steps.meta-base.outputs.bake-file }} + # ${{ steps.meta-prebuilt.outputs.bake-file }} + # ${{ steps.meta-devel.outputs.bake-file }} + # ${{ steps.meta-runtime.outputs.bake-file }} + # provenance: false + # targets: | + # ${{ inputs.bake-images }} + # set: | + # ${{ inputs.build-args }} + # prebuilt.output=type=docker,dest=/tmp/prebuilt.tar + # devel.output=type=docker,dest=/tmp/devel.tar + # runtime.output=type=docker,dest=/tmp/runtime.tar + + # - name: Upload Artifact - prebuilt + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-prebuilt + # uses: actions/upload-artifact@v4 + # with: + # name: prebuilt-image${{ inputs.tag-suffix }} + # path: /tmp/prebuilt.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Remove tar file - prebuilt + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # run: | + # rm /tmp/prebuilt.tar + # shell: bash + + # - name: Upload Artifact - devel + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-devel + # uses: actions/upload-artifact@v4 + # with: + # name: devel-image${{ inputs.tag-suffix }} + # path: /tmp/devel.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Remove tar file - devel + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # run: | + # rm /tmp/devel.tar + # shell: bash + + # - name: Upload Artifact - runtime + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: artifact-upload-step-runtime + # uses: actions/upload-artifact@v4 + # with: + # name: runtime-image${{ inputs.tag-suffix }} + # path: /tmp/runtime.tar + # retention-days: 7 + # compression-level: 6 + # overwrite: true + # if-no-files-found: warn + + # - name: Remove tar file - runtime + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # run: | + # rm /tmp/runtime.tar + # shell: bash + + # - name: Output artifact URLs + # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} + # id: output-artifact-urls + # run: | + # echo 'prebuilt URL ${{ steps.artifact-upload-step-prebuilt.outputs.artifact-url }}' + # echo 'devel URL ${{ steps.artifact-upload-step-devel.outputs.artifact-url }}' + # echo 'runtime URL ${{ steps.artifact-upload-step-runtime.outputs.artifact-url }}' + # shell: bash From 43824c5d71a45ac99eb039f1d63b034b572b547f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 15:20:01 +0000 Subject: [PATCH 33/46] style(pre-commit): autofix --- .github/actions/docker-build-and-push-openadkit/action.yaml | 4 ++-- .github/actions/docker-build-and-push/action.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index f85bd2a78f1..9da682d2e0a 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -107,7 +107,7 @@ runs: ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} - + - name: Build only if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} uses: docker/bake-action@v3 @@ -121,7 +121,7 @@ runs: provenance: false targets: | ${{ inputs.bake-images }} - + - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} id: warn-no-artifacts diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 269f607339e..c436ac2446d 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -118,7 +118,7 @@ runs: ${{ inputs.bake-images }} set: | ${{ inputs.build-args }} - + - name: Build Only if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} uses: docker/bake-action@v3 From cf43ae9dc03f29aed72011210d58589cb579ad83 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:04:41 +0300 Subject: [PATCH 34/46] . Signed-off-by: Oguz Ozturk --- .github/workflows/update-docker-manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml index e90ed2aa9fb..a8097dafda7 100644 --- a/.github/workflows/update-docker-manifest.yaml +++ b/.github/workflows/update-docker-manifest.yaml @@ -12,12 +12,12 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Combine multi arch images for 'autoware' + - name: Combine multi arch images for 'Autoware' uses: ./.github/actions/combine-multi-arch-images with: package-name: autoware - - name: Combine multi arch images for 'openadkit' + - name: Combine multi arch images for 'Open AD Kit' uses: ./.github/actions/combine-multi-arch-images with: package-name: openadkit From 170524670706b47aa7a997d782cd9aea18728aae Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:23:27 +0300 Subject: [PATCH 35/46] update yamllint settings Signed-off-by: Oguz Ozturk --- .yamllint.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.yamllint.yaml b/.yamllint.yaml index 2c7bd088e26..fecac4e02ba 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -10,6 +10,8 @@ rules: comments: level: error min-spaces-from-content: 1 # To be compatible with C++ and Python + comments-indentation: # Set to warning as it's sometimes fails to detect vscode auto-commenting https://github.com/adrienverge/yamllint/issues/384 + level: warning document-start: level: error present: false # Don't need document start markers From c32b00f41bb70574834beb65bf3435370ce9adda Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:29:30 +0300 Subject: [PATCH 36/46] Remove data_path as it is included in the image again Signed-off-by: Oguz Ozturk --- docker/run.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docker/run.sh b/docker/run.sh index f14cf8a1208..17be654d8cc 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -20,12 +20,11 @@ fi option_no_nvidia=false option_devel=false option_headless=false -DATA_PATH="${HOME}/autoware_data" MAP_PATH="" WORKSPACE_PATH="" USER_ID="" WORKSPACE="" -DEFAULT_LAUNCH_CMD="ros2 launch autoware_launch autoware.launch.xml data_path:=/autoware_data map_path:=/autoware_map vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit" +DEFAULT_LAUNCH_CMD="ros2 launch autoware_launch autoware.launch.xml map_path:=/autoware_map vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit" # Function to print help message print_help() { @@ -36,7 +35,6 @@ print_help() { echo -e "${RED}Usage:${NC} run.sh [OPTIONS] [LAUNCH_CMD](optional)" echo -e "Options:" echo -e " ${GREEN}--help/-h${NC} Display this help message" - echo -e " ${GREEN}--data-path${NC} Specify to mount data files into /autoware_data" echo -e " ${GREEN}--map-path${NC} Specify to mount map files into /autoware_map (mandatory for runtime)" echo -e " ${GREEN}--devel${NC} Launch the latest Autoware development environment with shell access" echo -e " ${GREEN}--workspace${NC} (--devel only)Specify the directory to mount into /workspace, by default it uses current directory (pwd)" @@ -66,10 +64,6 @@ parse_arguments() { WORKSPACE_PATH="$2" shift ;; - --data-path) - DATA_PATH="$2" - shift - ;; --map-path) MAP_PATH="$2" shift @@ -186,7 +180,7 @@ main() { set -x docker run -it --rm --net=host ${GPU_FLAG} ${USER_ID} ${MOUNT_X} \ -e XAUTHORITY=${XAUTHORITY} -e XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR -e NVIDIA_DRIVER_CAPABILITIES=all -v /etc/localtime:/etc/localtime:ro \ - ${WORKSPACE} ${DATA} ${MAP} ${IMAGE} \ + ${WORKSPACE} ${MAP} ${IMAGE} \ ${LAUNCH_CMD} } From 29d1a47026fcf4fd59aa661a8f42efc5a19bf6e7 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:30:32 +0300 Subject: [PATCH 37/46] download artifacts inside the dockerfile again Signed-off-by: Oguz Ozturk --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f2e5f5c25f4..b79de8e4088 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -72,7 +72,7 @@ ENV CXX="/usr/lib/ccache/g++" # cspell: ignore libcu libnv # Set up development environment RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadkit \ + ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache @@ -123,7 +123,7 @@ ARG SETUP_ARGS COPY --from=src-imported /rosdep-exec-depend-packages.txt /tmp/rosdep-exec-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers --runtime openadk \ + ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers --runtime openadk \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ From 83dad77d18b379931696844f25b59863d98f15d1 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:33:06 +0300 Subject: [PATCH 38/46] . Signed-off-by: Oguz Ozturk --- .yamllint.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.yamllint.yaml b/.yamllint.yaml index fecac4e02ba..0788c113194 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -10,8 +10,7 @@ rules: comments: level: error min-spaces-from-content: 1 # To be compatible with C++ and Python - comments-indentation: # Set to warning as it's sometimes fails to detect vscode auto-commenting https://github.com/adrienverge/yamllint/issues/384 - level: warning + comments-indentation: disable # Disable as it's sometimes fails to detect vscode auto-commenting https://github.com/adrienverge/yamllint/issues/384 document-start: level: error present: false # Don't need document start markers From 6293ea2d649bb3f3a2cd33dff1b59c8f9c819bae Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 14:47:07 +0300 Subject: [PATCH 39/46] fix playbook naming Signed-off-by: Oguz Ozturk --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b79de8e4088..1f6cadcad86 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -103,7 +103,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install development tools and artifacts RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module dev-tools openadk \ + ./setup-dev-env.sh -y --module dev-tools openadkit \ && pip uninstall -y ansible ansible-core \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache @@ -123,7 +123,7 @@ ARG SETUP_ARGS COPY --from=src-imported /rosdep-exec-depend-packages.txt /tmp/rosdep-exec-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers --runtime openadk \ + ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ From 9ddc39996fed7dd0556e4ed8fb4eedbc42021309 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 15:01:25 +0300 Subject: [PATCH 40/46] add caching Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push-openadkit/action.yaml | 2 ++ .github/actions/docker-build-and-push/action.yaml | 2 ++ .../workflows/docker-build-and-push-main-self-hosted.yaml | 1 + .github/workflows/docker-build-and-push-main.yaml | 5 +++-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 9da682d2e0a..43d2d448cb0 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -121,6 +121,8 @@ runs: provenance: false targets: | ${{ inputs.bake-images }} + set: | + ${{ inputs.build-args }} - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index c436ac2446d..f01881f2b2b 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -133,6 +133,8 @@ runs: provenance: false targets: | ${{ inputs.bake-images }} + set: | + ${{ inputs.build-args }} - name: Output artifact URLs if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }} diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 4685b4cee3e..227f6b54752 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -87,6 +87,7 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} + *.args.ARCH=arm64 *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max tag-suffix: ${{ matrix.additional-tag-suffix }} diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index e01b5c6406f..a5c4ae036ef 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -82,6 +82,9 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} + *.args.ARCH=amd64 + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 tag-prefix: ${{ needs.load-env.outputs.rosdistro }} @@ -99,8 +102,6 @@ jobs: *.args.BASE_IMAGE=${{ needs.load-env.outputs[format('{0}', matrix.base_image_env)] }} *.args.SETUP_ARGS=${{ matrix.setup-args }} *.args.LIB_DIR=${{ matrix.lib_dir }} - *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }} - *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.name }},mode=max *.args.ARCH=amd64 tag-suffix: ${{ matrix.additional-tag-suffix }} tag-arch: -amd64 From e6d49b5975113f5715474efa48f742854f39d61d Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Mon, 27 May 2024 19:56:05 +0300 Subject: [PATCH 41/46] remove base from build.sh Signed-off-by: Oguz Ozturk --- docker/openadkit/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/openadkit/build.sh b/docker/openadkit/build.sh index 4eed829f328..c0faf717396 100755 --- a/docker/openadkit/build.sh +++ b/docker/openadkit/build.sh @@ -105,7 +105,6 @@ build_images() { --set "*.args.BASE_IMAGE=$base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ - --set "base.tags=ghcr.io/autowarefoundation/openadkit:latest-base" \ --set "planning-control.tags=ghcr.io/autowarefoundation/openadkit:latest-planning-control" \ --set "visualizer.tags=ghcr.io/autowarefoundation/openadkit:latest-visualizer" \ --set "simulator.tags=ghcr.io/autowarefoundation/openadkit:latest-simulator" From d0a6d206bf784fcf35e1c52f0ee93d0b79484dcc Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Tue, 28 May 2024 09:54:47 +0300 Subject: [PATCH 42/46] remove extra dash'-' from docker image tagging Signed-off-by: Oguz Ozturk --- .github/actions/docker-build-and-push-openadkit/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/docker-build-and-push-openadkit/action.yaml b/.github/actions/docker-build-and-push-openadkit/action.yaml index 43d2d448cb0..638bcf2c034 100644 --- a/.github/actions/docker-build-and-push-openadkit/action.yaml +++ b/.github/actions/docker-build-and-push-openadkit/action.yaml @@ -60,7 +60,7 @@ runs: bake-target: docker-metadata-action-planning-control flavor: | latest=false - suffix=-planning-control-${{ inputs.tag-arch }} + suffix=-planning-control${{ inputs.tag-arch }} - name: Docker meta for openadkit:visualizer id: meta-visualizer @@ -71,7 +71,7 @@ runs: bake-target: docker-metadata-action-visualizer flavor: | latest=false - suffix=-visualizer-${{ inputs.tag-arch }} + suffix=-visualizer${{ inputs.tag-arch }} - name: Docker meta for openadkit:simulator id: meta-simulator @@ -82,7 +82,7 @@ runs: bake-target: docker-metadata-action-simulator flavor: | latest=false - suffix=-simulator-${{ inputs.tag-arch }} + suffix=-simulator${{ inputs.tag-arch }} - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} From 95f1bb5740dfbd2f10431d5074c405bed4086216 Mon Sep 17 00:00:00 2001 From: Oguz Ozturk Date: Wed, 29 May 2024 00:26:12 +0300 Subject: [PATCH 43/46] add clone repositories into build.sh Signed-off-by: Oguz Ozturk --- docker/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/build.sh b/docker/build.sh index d92e783bf75..284225fb36b 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -96,6 +96,13 @@ load_env() { fi } +# Clone repositories +clone_repositories() { + cd "$WORKSPACE_ROOT" + mkdir -p src + vcs import src < autoware/autoware.repos +} + # Build images build_images() { # https://github.com/docker/buildx/issues/484 @@ -133,4 +140,5 @@ set_build_options set_platform set_arch_lib_dir load_env +clone_repositories build_images From 0b28c0eb1a0acdd8a41aafe52a37db513285ec90 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 21:26:27 +0000 Subject: [PATCH 44/46] style(pre-commit): autofix --- docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build.sh b/docker/build.sh index 284225fb36b..347a5f37008 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -100,7 +100,7 @@ load_env() { clone_repositories() { cd "$WORKSPACE_ROOT" mkdir -p src - vcs import src < autoware/autoware.repos + vcs import src Date: Wed, 29 May 2024 00:28:23 +0300 Subject: [PATCH 45/46] . Signed-off-by: Oguz Ozturk --- docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build.sh b/docker/build.sh index 347a5f37008..a9104f5920f 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -99,7 +99,7 @@ load_env() { # Clone repositories clone_repositories() { cd "$WORKSPACE_ROOT" - mkdir -p src + mkdir src vcs import src Date: Wed, 29 May 2024 09:27:57 +0000 Subject: [PATCH 46/46] style(pre-commit): autofix --- .github/actions/docker-build-and-push/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index eafe16af77e..c08dba3c356 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -134,4 +134,4 @@ runs: targets: | ${{ inputs.bake-images }} set: | - ${{ inputs.build-args }} \ No newline at end of file + ${{ inputs.build-args }}