Backport Dockerfile changes (moveit#3201)

... from noetic-devel branch and build docker images via GHA.
shadow-robot · Sep 1, 2022 · 2433d8f · 2433d8f
1 parent bb005c2
commit 2433d8f
Show file tree

Hide file tree

Showing 8 changed files with 236 additions and 38 deletions.
diff --git a/.docker/README.md b/.docker/README.md
@@ -1,3 +1,3 @@
-# MoveIt! Docker Containers
+# MoveIt Docker Containers
 
-For more information see [Continuous Integration and Docker](http://moveit.ros.org/documentation/contributing/continuous_integration.html) documentation.
+For more information see the pages [Continuous Integration and Docker](http://moveit.ros.org/documentation/contributing/continuous_integration.html) and [Using Docker Containers with MoveIt](https://moveit.ros.org/install/docker/).
diff --git a/.docker/ci-shadow-fixed/Dockerfile → .docker/ci-testing/Dockerfile b/.docker/ci-shadow-fixed/Dockerfile → .docker/ci-testing/Dockerfile
@@ -1,8 +1,8 @@
-# moveit/moveit:melodic-ci-shadow-fixed
+# moveit/moveit:melodic-ci-testing
 # Sets up a base image to use for running Continuous Integration on Travis
 
 FROM moveit/moveit:melodic-ci
-MAINTAINER Dave Coleman [email protected]
+MAINTAINER Robert Haschke [email protected]
 
 # Switch to ros-shadow-fixed
 RUN echo "deb http://packages.ros.org/ros-shadow-fixed/ubuntu `lsb_release -cs` main" | tee /etc/apt/sources.list.d/ros-latest.list

diff --git a/.docker/ci/Dockerfile b/.docker/ci/Dockerfile
@@ -2,32 +2,34 @@
 # Sets up a base image to use for running Continuous Integration on Travis
 
 FROM ros:melodic-ros-base
-MAINTAINER Dave Coleman [email protected]
+MAINTAINER Robert Haschke [email protected]
 
 ENV TERM xterm
 
 # Setup (temporary) ROS workspace
 WORKDIR /root/ws_moveit
 
+# Copy MoveIt sources from docker context
+COPY . src/moveit
+
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#minimize-the-number-of-layers
 RUN \
     # Update apt package list as previous containers clear the cache
-    apt-get -qq update && \
-    apt-get -qq dist-upgrade && \
+    apt-get -q update && \
+    apt-get -q -y dist-upgrade && \
     #
     # Install some base dependencies
-    apt-get -qq install --no-install-recommends -y \
+    apt-get -q install --no-install-recommends -y \
         # Some basic requirements
         wget git sudo \
         # Preferred build tools
-        python-catkin-tools \
+        python$(test "${ROS_DISTRO}" = "noetic" && echo 3)-catkin-tools \
         clang clang-format-10 clang-tidy clang-tools \
         ccache && \
     #
-    # Download MoveIt source, so that we can fetch all necessary dependencies
-    wstool init --shallow src https://raw.githubusercontent.com/ros-planning/moveit/${ROS_DISTRO}-devel/.github/workflows/upstream.rosinstall && \
-    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit src/moveit && \
+    # Download MoveIt sources, so that we can fetch all necessary dependencies
+    wstool init --shallow src src/moveit/.github/workflows/upstream.rosinstall && \
     git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit_resources src/moveit_resources && \
     #
     # Download all dependencies of MoveIt
@@ -38,7 +40,10 @@ RUN \
     rm -rf src && \
     #
     # Clear apt-cache to reduce image size
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    # Globally disable git security
+    # https://github.blog/2022-04-12-git-security-vulnerability-announced
+    git config --global --add safe.directory "*"
 
 # Continous Integration Setting
 ENV IN_DOCKER 1
diff --git a/.docker/release/Dockerfile b/.docker/release/Dockerfile
@@ -1,10 +1,11 @@
 # moveit/moveit:melodic-release
-# Full debian-based install of MoveIt! using apt-get
+# Full debian-based install of MoveIt using apt-get
 
 FROM ros:melodic-ros-base
 MAINTAINER Dave Coleman [email protected]
 
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
-RUN apt-get update && \
+RUN apt-get update -q && \
+    apt-get dist-upgrade -q -y && \
     apt-get install -y ros-${ROS_DISTRO}-moveit-* && \
     rm -rf /var/lib/apt/lists/*
diff --git a/.docker/source/Dockerfile b/.docker/source/Dockerfile
@@ -1,38 +1,35 @@
+# syntax = docker/dockerfile:1.3
+
 # moveit/moveit:melodic-source
 # Downloads the moveit source code and install remaining debian dependencies
 
-FROM moveit/moveit:melodic-ci-shadow-fixed
-MAINTAINER Dave Coleman [email protected]
+FROM moveit/moveit:melodic-ci-testing
+MAINTAINER Robert Haschke [email protected]
 
+ENV PYTHONIOENCODING UTF-8
+# Export ROS_UNDERLAY for downstream docker containers
 ENV ROS_UNDERLAY /root/ws_moveit/install
-WORKDIR $ROS_UNDERLAY/../src
+# Environment variable used in instructions on moveit.ros.org website for running clang-tidy
+ENV CATKIN_WS $(realpath $ROS_UNDERLAY/..)
+WORKDIR $ROS_UNDERLAY/..
+
+# Copy MoveIt sources from docker context
+COPY . src/moveit
 
 # Commands are combined in single RUN statement with "apt/lists" folder removal to reduce image size
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#minimize-the-number-of-layers
-RUN \
-    # Download moveit source so that we can get necessary dependencies
-    wstool init --shallow . https://raw.githubusercontent.com/ros-planning/moveit/${ROS_DISTRO}-devel/.github/workflows/upstream.rosinstall && \
-    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit && \
-    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit_resources && \
-    #
-    # Update apt package list as cache is cleared in previous container
-    # Usually upgrading involves a few packages only (if container builds became out-of-sync)
-    apt-get -qq update && \
-    apt-get -qq dist-upgrade && \
+RUN --mount=type=cache,target=/root/.ccache/ \
+    # Enable ccache
+    PATH=/usr/lib/ccache:$PATH && \
+    # Fetch required upstream sources for building
+    wstool init --shallow src src/moveit/.github/workflows/upstream.rosinstall && \
+    git clone --depth 1 --branch ${ROS_DISTRO}-devel https://github.com/ros-planning/moveit_resources src/moveit_resources && \
     #
-    rosdep update && \
-    rosdep install -y --from-paths . --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV PYTHONIOENCODING UTF-8
-RUN cd .. && \
     catkin config --extend /opt/ros/$ROS_DISTRO --install --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON && \
     # Status rate is limited so that just enough info is shown to keep Docker from timing out,
     # but not too much such that the Docker log gets too long (another form of timeout)
     catkin build --limit-status-rate 0.001 --no-notify && \
+    ccache -s && \
     #
     # Update /ros_entrypoint.sh to source our new workspace
     sed -i "s#/opt/ros/\$ROS_DISTRO/setup.bash#$ROS_UNDERLAY/setup.sh#g" /ros_entrypoint.sh
-
-# Environment variable used in instructions on moveit.ros.org website for running clang-tidy
-ENV CATKIN_WS $(realpath $ROS_UNDERLAY/..)
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,37 @@
+# ignore everything
+*
+# but include these:
+!.github/workflows/upstream.rosinstall
+!**/package.xml
+!**/CATKIN_IGNORE
+
+# https://github.com/moby/moby/issues/42788
+!moveit_plugins/moveit_plugins/package.xml
+!moveit_plugins/moveit_ros_control_interface/package.xml
+!moveit_plugins/moveit_simple_controller_manager/package.xml
+!moveit_plugins/moveit_fake_controller_manager/package.xml
+!moveit_kinematics/package.xml
+!moveit_setup_assistant/package.xml
+!moveit_core/package.xml
+!moveit_commander/package.xml
+!moveit_planners/ompl/package.xml
+!moveit_planners/chomp/chomp_motion_planner/package.xml
+!moveit_planners/chomp/chomp_interface/package.xml
+!moveit_planners/chomp/chomp_optimizer_adapter/package.xml
+!moveit_planners/pilz_industrial_motion_planner_testutils/package.xml
+!moveit_planners/pilz_industrial_motion_planner/package.xml
+!moveit_planners/moveit_planners/package.xml
+!moveit_runtime/package.xml
+!moveit/package.xml
+!moveit_ros/warehouse/package.xml
+!moveit_ros/moveit_servo/package.xml
+!moveit_ros/occupancy_map_monitor/package.xml
+!moveit_ros/perception/package.xml
+!moveit_ros/move_group/package.xml
+!moveit_ros/robot_interaction/package.xml
+!moveit_ros/visualization/package.xml
+!moveit_ros/manipulation/package.xml
+!moveit_ros/planning/package.xml
+!moveit_ros/planning_interface/package.xml
+!moveit_ros/benchmarks/package.xml
+!moveit_ros/moveit_ros/package.xml
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -18,7 +18,7 @@ jobs:
         env:
           - IMAGE: melodic-ci
             CCOV: true
-          - IMAGE: melodic-ci-shadow-fixed
+          - IMAGE: melodic-ci-testing
             IKFAST_TEST: true
             CATKIN_LINT: true
             CLANG_TIDY: pedantic

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -0,0 +1,158 @@
+name: docker
+
+on:
+  schedule:
+    # 6 AM UTC every Sunday
+    - cron: "0 6 * * 6"
+  workflow_dispatch:
+  push:
+    branches:
+      - melodic-devel
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    env:
+      IMAGE: moveit/moveit:melodic-${{ github.job }}
+
+    steps:
+      - uses: addnab/docker-run-action@v3
+        name: Check for apt updates
+        continue-on-error: true
+        id: apt
+        with:
+          image: ${{ env.IMAGE }}
+          run: |
+            apt-get update
+            have_updates=$(apt-get --simulate upgrade | grep -q "^0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.$" && echo false || echo true)
+            echo "::set-output name=no_cache::$have_updates"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+      - name: Login to Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+        with:
+          file: .docker/${{ github.job }}/Dockerfile
+          push: true
+          no-cache: ${{ steps.apt.outputs.no_cache || github.event_name == 'workflow_dispatch' }}
+          cache-from: type=registry,ref=${{ env.IMAGE }}
+          cache-to: type=inline
+          tags: ${{ env.IMAGE }}
+
+  ci:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    env:
+      IMAGE: moveit/moveit:melodic-${{ github.job }}
+
+    steps:
+      - uses: addnab/docker-run-action@v3
+        name: Check for apt updates
+        continue-on-error: true
+        id: apt
+        with:
+          image: ${{ env.IMAGE }}
+          run: |
+            apt-get update
+            have_updates=$(apt-get --simulate upgrade | grep -q "^0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.$" && echo false || echo true)
+            echo "::set-output name=no_cache::$have_updates"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+      - name: Login to Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+        with:
+          file: .docker/${{ github.job }}/Dockerfile
+          push: true
+          no-cache: ${{ steps.apt.outputs.no_cache || github.event_name == 'workflow_dispatch' }}
+          cache-from: type=registry,ref=${{ env.IMAGE }}
+          cache-to: type=inline
+          tags: ${{ env.IMAGE }}
+
+  ci-testing:
+    needs: ci
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    env:
+      IMAGE: moveit/moveit:melodic-${{ github.job }}
+
+    steps:
+      - uses: addnab/docker-run-action@v3
+        name: Check for apt updates
+        continue-on-error: true
+        id: apt
+        with:
+          image: ${{ env.IMAGE }}
+          run: |
+            apt-get update
+            have_updates=$(apt-get --simulate upgrade | grep -q "^0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.$" && echo false || echo true)
+            echo "::set-output name=no_cache::$have_updates"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+      - name: Login to Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name != 'schedule' || steps.apt.outputs.no_cache }}
+        with:
+          file: .docker/${{ github.job }}/Dockerfile
+          push: true
+          no-cache: ${{ steps.apt.outputs.no_cache || github.event_name == 'workflow_dispatch' }}
+          cache-from: type=registry,ref=${{ env.IMAGE }}
+          cache-to: type=inline
+          tags: |
+            ${{ env.IMAGE }}
+            moveit/moveit:melodic-ci-shadow-fixed
+
+  source:
+    needs: ci-testing
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    env:
+      IMAGE: moveit/moveit:melodic-${{ github.job }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Remove .dockerignore"
+        run: rm .dockerignore  # enforce full source context
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: .docker/${{ github.job }}/Dockerfile
+          push: true
+          cache-from: type=registry,ref=${{ env.IMAGE }}
+          cache-to: type=inline
+          tags: ${{ env.IMAGE }}