feat(docker): disable --download-artifacts (#19)

* introduce src-imported stage Signed-off-by: Yutaka Kondo <[email protected]> * empty Signed-off-by: Yutaka Kondo <[email protected]> * chore(docker): separate `rosdep install` and `colcon build` RUN commands (autowarefoundation#4719) * refactor(ci): reuse `load-env.yaml` to load env (autowarefoundation#4734) * reuse load-env.yaml to load env * update base_image Signed-off-by: Yutaka Kondo <[email protected]> * fix base_image Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> * generate install package lists * append || true Signed-off-by: Yutaka Kondo <[email protected]> * cat packages Signed-off-by: Yutaka Kondo <[email protected]> * chore(docker): separate `rosdep install` and `colcon build` RUN commands (autowarefoundation#4719) * wip Signed-off-by: Yutaka Kondo <[email protected]> * wip Signed-off-by: Yutaka Kondo <[email protected]> * Update Dockerfile * Update Dockerfile * install pacmod * rename * Revert "install pacmod" This reverts commit ebff10f. * install pacmod3 * register pamod3 repos * ignore hadolint * revert * run set-dev-env.sh * pip install * copy * rosdep update * vcs import outside dockerfile Signed-off-by: Yutaka Kondo <[email protected]> * refactor(docker): introduce `src-imported` stage (autowarefoundation#4712) * introduce src-imported stage Signed-off-by: Yutaka Kondo <[email protected]> * empty Signed-off-by: Yutaka Kondo <[email protected]> * generate install package lists * append || true Signed-off-by: Yutaka Kondo <[email protected]> * cat packages Signed-off-by: Yutaka Kondo <[email protected]> * chore(docker): separate `rosdep install` and `colcon build` RUN commands (autowarefoundation#4719) * wip Signed-off-by: Yutaka Kondo <[email protected]> * wip Signed-off-by: Yutaka Kondo <[email protected]> * Update Dockerfile * Update Dockerfile * install pacmod * rename * Revert "install pacmod" This reverts commit ebff10f. * install pacmod3 * register pamod3 repos * ignore hadolint * revert * run set-dev-env.sh * pip install * copy * rosdep update --------- Signed-off-by: Yutaka Kondo <[email protected]> * refine order and remove /autoware/src at last * fix hadolint * remove tar files Signed-off-by: Yutaka Kondo <[email protected]> * shell: bash Signed-off-by: Yutaka Kondo <[email protected]> * refactor(ci): reuse `load-env.yaml` to load env (autowarefoundation#4734) * reuse load-env.yaml to load env * update base_image Signed-off-by: Yutaka Kondo <[email protected]> * fix base_image Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> * remove --download-artifacts Signed-off-by: Yutaka Kondo <[email protected]> * remove --download-artifacts Signed-off-by: Yutaka Kondo <[email protected]> * add --data-path option Signed-off-by: Yutaka Kondo <[email protected]> * wip Signed-off-by: Yutaka Kondo <[email protected]> * wip Signed-off-by: Yutaka Kondo <[email protected]> * run download_artifacts Signed-off-by: Yutaka Kondo <[email protected]> * remove todo Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]>
youtalk · May 25, 2024 · 0a11090 · 0a11090
1 parent a021f3f
commit 0a11090
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 5 deletions.
diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml
@@ -143,6 +143,12 @@ runs:
         overwrite: true
         if-no-files-found: error
 
+    - name: Remove tar file - prebuilt
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }}
+      run: |
+        rm /tmp/prebuilt.tar
+      shell: bash
+
     - name: Upload Artifact - devel
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }}
       id: artifact-upload-step-devel
@@ -155,6 +161,12 @@ runs:
         overwrite: true
         if-no-files-found: error
 
+    - name: Remove tar file - devel
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }}
+      run: |
+        rm /tmp/devel.tar
+      shell: bash
+
     - name: Upload Artifact - runtime
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }}
       id: artifact-upload-step-runtime
@@ -167,6 +179,12 @@ runs:
         overwrite: true
         if-no-files-found: error
 
+    - name: Remove tar file - runtime
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.artifacts-destination == 'tarball' }}
+      run: |
+        rm /tmp/runtime.tar
+      shell: bash
+
     - name: Output artifact URLs
       id: output-artifact-urls
       run: |

diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml
@@ -90,3 +90,7 @@ jobs:
           tag-suffix: ${{ matrix.additional-tag-suffix }}-amd64
           tag-prefix: ${{ needs.load-env.outputs.rosdistro }}
           allow-push: true
+
+      - name: Show disk space
+        run: |
+          df -h
diff --git a/ansible/playbooks/docker.yaml b/ansible/playbooks/docker.yaml
@@ -24,3 +24,7 @@
       when: prompt_install_nvidia == 'y'
     - role: autoware.dev_env.docker_engine
     - role: autoware.dev_env.nvidia_container_toolkit
+
+    # ONNX files and other artifacts
+    - role: autoware.dev_env.artifacts
+      when: prompt_download_artifacts == 'y'
diff --git a/docker/autoware-openadk/Dockerfile b/docker/autoware-openadk/Dockerfile
@@ -105,7 +105,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Install development tools and artifacts
 RUN --mount=type=ssh \
-  ./setup-dev-env.sh -y --module dev-tools --download-artifacts openadk \
+  ./setup-dev-env.sh -y --module dev-tools openadk \
   && pip uninstall -y ansible ansible-core \
   && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache
 
@@ -125,7 +125,7 @@ ARG SETUP_ARGS
 COPY --from=src-imported /rosdep-exec-depend-packages.txt /tmp/rosdep-exec-depend-packages.txt
 # hadolint ignore=SC2002
 RUN --mount=type=ssh \
-  ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers --runtime openadk \
+  ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers --runtime openadk \
   && pip uninstall -y ansible ansible-core \
   && apt-get update \
   && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \

diff --git a/docker/run.sh b/docker/run.sh
@@ -20,11 +20,12 @@ fi
 option_no_nvidia=false
 option_devel=false
 option_headless=false
+DATA_PATH="${HOME}/autoware_data"
 MAP_PATH=""
 WORKSPACE_PATH=""
 USER_ID=""
 WORKSPACE=""
-DEFAULT_LAUNCH_CMD="ros2 launch autoware_launch autoware.launch.xml map_path:=/autoware_map vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit"
+DEFAULT_LAUNCH_CMD="ros2 launch autoware_launch autoware.launch.xml data_path:=/autoware_data map_path:=/autoware_map vehicle_model:=sample_vehicle sensor_model:=sample_sensor_kit"
 
 # Function to print help message
 print_help() {
@@ -35,6 +36,7 @@ print_help() {
     echo -e "${RED}Usage:${NC} run.sh [OPTIONS] [LAUNCH_CMD](optional)"
     echo -e "Options:"
     echo -e "  ${GREEN}--help/-h${NC}       Display this help message"
+    echo -e "  ${GREEN}--data-path${NC}     Specify to mount data files into /autoware_data"
     echo -e "  ${GREEN}--map-path${NC}      Specify to mount map files into /autoware_map (mandatory if no custom launch command is provided)"
     echo -e "  ${GREEN}--no-nvidia${NC}     Disable NVIDIA GPU support"
     echo -e "  ${GREEN}--devel${NC}         Use the latest development version of Autoware"
@@ -64,6 +66,10 @@ parse_arguments() {
             WORKSPACE_PATH="$2"
             shift
             ;;
+        --data-path)
+            DATA_PATH="$2"
+            shift
+            ;;
         --map-path)
             MAP_PATH="$2"
             shift
@@ -95,6 +101,9 @@ set_variables() {
         exit 1
     fi
 
+    # Mount data path if provided
+    DATA="-v ${DATA_PATH}:/autoware_data:ro"
+
     # Mount map path if provided
     MAP="-v ${MAP_PATH}:/autoware_map:ro"
 
@@ -150,6 +159,7 @@ main() {
 
     echo -e "${GREEN}\n-----------------------LAUNCHING CONTAINER-----------------------"
     echo -e "${GREEN}IMAGE:${NC} ${IMAGE}"
+    echo -e "${GREEN}DATA PATH(mounted):${NC} ${DATA_PATH}:/autoware_data"
     echo -e "${GREEN}MAP PATH(mounted):${NC} ${MAP_PATH}:/autoware_map"
     echo -e "${GREEN}WORKSPACE(mounted):${NC} ${WORKSPACE_PATH}:/workspace"
     echo -e "${GREEN}LAUNCH CMD:${NC} ${LAUNCH_CMD}"
@@ -159,7 +169,7 @@ main() {
     set -x
     docker run -it --rm --net=host ${GPU_FLAG} ${USER_ID} ${MOUNT_X} \
         -e XAUTHORITY=${XAUTHORITY} -e XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR -e NVIDIA_DRIVER_CAPABILITIES=all -v /etc/localtime:/etc/localtime:ro \
-        ${WORKSPACE} ${MAP} ${IMAGE} \
+        ${WORKSPACE} ${DATA} ${MAP} ${IMAGE} \
         ${LAUNCH_CMD}
 }
 

diff --git a/setup-dev-env.sh b/setup-dev-env.sh
@@ -133,7 +133,7 @@ if [ "$option_yes" = "true" ] || [ "$option_download_artifacts" = "true" ]; then
 fi
 
 # Check downloading artifacts
-if [ "$target_playbook" = "autoware.dev_env.openadk" ]; then
+if [ "$target_playbook" = "autoware.dev_env.docker" ] || [ "$target_playbook" = "autoware.dev_env.openadk" ]; then
     if [ "$option_download_artifacts" = "true" ]; then
         echo -e "\e[36mArtifacts will be downloaded to $option_data_dir\e[m"
         ansible_args+=("--extra-vars" "prompt_download_artifacts=y")