refactor(docker): introduce src-imported stage (#4712)

* introduce src-imported stage Signed-off-by: Yutaka Kondo <[email protected]> * empty Signed-off-by: Yutaka Kondo <[email protected]> * generate install package lists * append || true Signed-off-by: Yutaka Kondo <[email protected]> * cat packages Signed-off-by: Yutaka Kondo <[email protected]> * chore(docker): separate `rosdep install` and `colcon build` RUN commands (#4719) * wip Signed-off-by: Yutaka Kondo <[email protected]> * wip Signed-off-by: Yutaka Kondo <[email protected]> * Update Dockerfile * Update Dockerfile * install pacmod * rename * Revert "install pacmod" This reverts commit ebff10f. * install pacmod3 * register pamod3 repos * ignore hadolint * revert * run set-dev-env.sh * pip install * copy * rosdep update --------- Signed-off-by: Yutaka Kondo <[email protected]>
autowarefoundation · May 20, 2024 · 5ee2907 · 5ee2907
1 parent 837b486
commit 5ee2907
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 15 deletions.
diff --git a/ansible/playbooks/rosdep.yaml b/ansible/playbooks/rosdep.yaml
@@ -0,0 +1,10 @@
+- name: Register rosdep package repositories
+  hosts: localhost
+  connection: local
+  pre_tasks:
+    - name: Verify OS
+      ansible.builtin.fail:
+        msg: Only Ubuntu 22.04 is supported for this branch. Please refer to https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/source-installation/.
+      when: ansible_distribution != 'Ubuntu' or ansible_distribution_version != '22.04'
+  roles:
+    - role: autoware.dev_env.pacmod
diff --git a/docker/autoware-openadk/Dockerfile b/docker/autoware-openadk/Dockerfile
@@ -32,13 +32,49 @@ WORKDIR /autoware
 RUN --mount=type=ssh \
   ./setup-dev-env.sh -y --module base --runtime openadk \
   && pip uninstall -y ansible ansible-core \
-  && pip install --no-cache-dir vcstool \
   && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \
   && echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc
 
 # Create entrypoint
 CMD ["/bin/bash"]
 
+FROM $BASE_IMAGE as src-imported
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ARG ROS_DISTRO
+
+# Copy files
+COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/
+COPY ansible/ /autoware/ansible/
+WORKDIR /autoware
+
+RUN --mount=type=ssh \
+  ./setup-dev-env.sh -y rosdep \
+  && pip uninstall -y ansible ansible-core \
+  && pip install --no-cache-dir vcstool \
+  && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache
+
+WORKDIR /autoware
+COPY autoware.repos /autoware/
+RUN --mount=type=ssh \
+  mkdir src \
+  && vcs import src < autoware.repos
+
+# Generate install package lists
+RUN rosdep update && rosdep keys --ignore-src --from-paths src \
+    | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \
+    | grep -v '^#' \
+    | sed 's/ \+/\n/g'\
+    | sort \
+    > /rosdep-all-depend-packages.txt \
+    && cat /rosdep-all-depend-packages.txt
+RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \
+    | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \
+    | grep -v '^#' \
+    | sed 's/ \+/\n/g'\
+    | sort \
+    > /rosdep-exec-depend-packages.txt \
+    && cat /rosdep-exec-depend-packages.txt
+
 FROM base as prebuilt
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 ARG ROS_DISTRO
@@ -56,16 +92,13 @@ RUN --mount=type=ssh \
   && find / -name 'libcu*.a' -delete \
   && find / -name 'libnv*.a' -delete
 
-# Copy repository files
-COPY autoware.repos /autoware/
-
 # Install rosdep dependencies
+COPY --from=src-imported /autoware/src /autoware/src
+COPY --from=src-imported /rosdep-all-depend-packages.txt /tmp/rosdep-all-depend-packages.txt
+# hadolint ignore=SC2002
 RUN --mount=type=ssh \
-  mkdir src \
-  && vcs import src < autoware.repos \
-  && apt-get update \
-  && rosdep update \
-  && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \
+  apt-get update \
+  && cat /tmp/rosdep-all-depend-packages.txt | xargs apt-get install -y --no-install-recommends \
   && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache
 
 # Build Autoware
@@ -103,20 +136,19 @@ ARG LIB_DIR
 ARG SETUP_ARGS
 
 # Set up runtime environment and artifacts
-COPY autoware.repos /autoware/
+COPY --from=src-imported /rosdep-exec-depend-packages.txt /tmp/rosdep-exec-depend-packages.txt
+# hadolint ignore=SC2002
 RUN --mount=type=ssh \
   ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --download-artifacts --no-cuda-drivers --runtime openadk \
   && pip uninstall -y ansible ansible-core \
-  && mkdir src \
-  && vcs import src < autoware.repos \
-  && rosdep update \
-  && DEBIAN_FRONTEND=noninteractive rosdep install -y --dependency-types=exec --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \
+  && apt-get update \
+  && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \
   && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \
   && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \
   && find / -name "*.o" -type f -delete \
   && find / -name "*.h" -type f -delete \
   && find / -name "*.hpp" -type f -delete \
-  && rm -rf /autoware/src /autoware/ansible /autoware/autoware.repos \
+  && rm -rf /autoware/ansible \
     /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \
     /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \
     /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm*