docker/ubuntu-*: cache deb lists+packages

Use the docker cache for package lists and deb files. This avoids updating the same lists multiple times in the same build, and enables consecutive builds to install packages without downloading them from internet.
OSGeo · Oct 9, 2024 · 797bd6c · 797bd6c
1 parent dce6dfd
commit 797bd6c
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 70 deletions.
diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile
@@ -21,7 +21,12 @@ ARG TARGET_ARCH=
 RUN echo ${TARGET_ARCH}
 COPY ./bh-set-envvars.sh /buildscripts/bh-set-envvars.sh
 
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN rm -f /etc/apt/apt.conf.d/docker-clean \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && if test "${TARGET_ARCH}" != ""; then \
     rm -f /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources \
     && echo "deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu/ noble main restricted universe" >> /etc/apt/sources.list \
@@ -35,25 +40,27 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y g++-13-${GCC_ARCH}-linux-gnu \
     && ln -s ${GCC_ARCH}-linux-gnu-gcc-13 /usr/bin/${GCC_ARCH}-linux-gnu-gcc \
-    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++ \
-    && rm -rf /var/lib/apt/lists/*; \
+    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++; \
     fi
 
 # Setup build env for PROJ
 USER root
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
             build-essential ca-certificates ccache\
             git make ninja-build cmake wget unzip libtool automake \
             zlib1g-dev${APT_ARCH_SUFFIX} libsqlite3-dev${APT_ARCH_SUFFIX} pkg-config sqlite3 libcurl4-openssl-dev${APT_ARCH_SUFFIX} \
             libtiff-dev${APT_ARCH_SUFFIX} \
-            rsync \
-    && rm -rf /var/lib/apt/lists/*
+            rsync
 
 ARG JAVA_VERSION=17
 # Setup build env for GDAL
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
        libopenjp2-7-dev${APT_ARCH_SUFFIX} libcairo2-dev${APT_ARCH_SUFFIX} \
@@ -73,8 +80,7 @@ RUN . /buildscripts/bh-set-envvars.sh \
        libbrotli-dev${APT_ARCH_SUFFIX} \
        libarchive-dev${APT_ARCH_SUFFIX} \
        libaec-dev${APT_ARCH_SUFFIX} \
-       libavif-dev${APT_ARCH_SUFFIX} \
-    && rm -rf /var/lib/apt/lists/*
+       libavif-dev${APT_ARCH_SUFFIX}
 
 # Build likbkea
 ARG KEA_VERSION=1.5.2
@@ -133,13 +139,16 @@ RUN . /buildscripts/bh-set-envvars.sh \
 
 # Build tiledb
 ARG TILEDB_VERSION=2.23.0
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
+    && apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        libspdlog-dev${APT_ARCH_SUFFIX} libmagic-dev${APT_ARCH_SUFFIX}
+
 COPY ./tiledb-FindLZ4_EP.cmake.patch /buildscripts/tiledb-FindLZ4_EP.cmake.patch
 COPY ./tiledb-FindOpenSSL_EP.cmake.patch /buildscripts/tiledb-FindOpenSSL_EP.cmake.patch
 RUN . /buildscripts/bh-set-envvars.sh \
-    && apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        libspdlog-dev${APT_ARCH_SUFFIX} libmagic-dev${APT_ARCH_SUFFIX} \
-    && rm -rf /var/lib/apt/lists/* \
     && mkdir tiledb \
     && curl -L -fsS https://github.com/TileDB-Inc/TileDB/archive/${TILEDB_VERSION}.tar.gz \
         | tar xz -C tiledb --strip-components=1 \
@@ -226,7 +235,9 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && rm -rf QB3
 
 ARG WITH_PDFIUM=yes
-RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
   curl -LO -fsS https://github.com/rouault/pdfium_build_gdal_3_10/releases/download/pdfium_6677_v1/install-ubuntu2004-rev6677.tar.gz \
   && tar -xzf install-ubuntu2004-rev6677.tar.gz \
   && chown -R root:root install \
@@ -235,14 +246,15 @@ RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
   && rm -rf install-ubuntu2004-rev6677.tar.gz install \
   && apt-get update -y \
   && apt-get install -y --fix-missing --no-install-recommends liblcms2-dev${APT_ARCH_SUFFIX} \
-  && rm -rf /var/lib/apt/lists/* \
   ) ; fi
 
 # Build libjxl
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
-    && apt-get install -y --fix-missing --no-install-recommends libgflags-dev${APT_ARCH_SUFFIX} \
-    && git clone https://github.com/libjxl/libjxl.git --recursive \
+    && apt-get install -y --fix-missing --no-install-recommends libgflags-dev${APT_ARCH_SUFFIX}
+RUN git clone https://github.com/libjxl/libjxl.git --recursive \
     && cd libjxl \
     && mkdir build \
     && cd build \
@@ -253,14 +265,15 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && rm -f /lib/${GCC_ARCH}-linux-gnu/libjxl*.a \
     && rm -f /build_thirdparty/lib/${GCC_ARCH}-linux-gnu/libjxl*.a \
     && cd ../.. \
-    && rm -rf libjxl \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf libjxl
 
 # Install Arrow C++
 ARG ARROW_VERSION=16.1.0-1
 # ARROW_SOVERSION to be updated in the "Build final image" section too
 ARG ARROW_SOVERSION=1600
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ca-certificates lsb-release wget \
     && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
@@ -273,7 +286,6 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-acero-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
-    && rm -rf /var/lib/apt/lists/* \
     && rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
 
 ARG RSYNC_REMOTE
@@ -297,6 +309,11 @@ RUN . /buildscripts/bh-set-envvars.sh \
      && rm -rf /build_tmp_proj
 
 # Build PROJ
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y patchelf
+
 ARG PROJ_VERSION=master
 RUN --mount=type=cache,id=ubuntu-full-proj,target=$HOME/.cache \
     . /buildscripts/bh-set-envvars.sh \
@@ -321,18 +338,19 @@ RUN date
 ARG JAVA_VERSION=17
 ARG ARROW_SOVERSION=1600
 
-# Update distro
-RUN apt-get update -y && apt-get upgrade -y \
-    && rm -rf /var/lib/apt/lists/*
-
 ARG TARGET_ARCH=
-RUN apt-get update \
-# PROJ dependencies
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+# Update distro
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    # PROJ dependencies
+    && apt-get install -y \
         libsqlite3-0 libtiff6 libcurl4 \
         wget curl unzip ca-certificates \
-# GDAL dependencies
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    # GDAL dependencies
+    && apt-get install -y \
         libopenjp2-7 libcairo2 python3-numpy \
         libpng16-16 libjpeg-turbo8 libgif7 liblzma5 libgeos3.12.1 libgeos-c1v5 \
         libxml2 libexpat1 \
@@ -353,18 +371,17 @@ RUN apt-get update \
         # pil for antialias option of gdal2tiles
         python3-pil \
     # Install JRE with --no-install-recommends, otherwise it draws default-jre, which draws systemd, which fails to install when running the arm64v8/ubuntu:24.04 image on a 64bit host
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-"$JAVA_VERSION"-jre \
+    && apt-get install -y --no-install-recommends openjdk-"$JAVA_VERSION"-jre \
     # Worksround OGDI packaging bug for Ubuntu 24.04
     && ln -s /usr/lib/$(uname -p)-linux-gnu/ogdi/4.1/libvrf.so /usr/lib/$(uname -p)-linux-gnu \
     # Install Arrow C++
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ca-certificates lsb-release wget \
+    && apt-get install -y -V ca-certificates lsb-release wget \
     && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
+    && apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
     && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow${ARROW_SOVERSION} \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet${ARROW_SOVERSION} \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset${ARROW_SOVERSION} \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y -V libarrow${ARROW_SOVERSION} \
+    && apt-get install -y -V libparquet${ARROW_SOVERSION} \
+    && apt-get install -y -V libarrow-dataset${ARROW_SOVERSION}
 
 # Attempt to order layers starting with less frequently varying ones
 

diff --git a/docker/ubuntu-full/bh-proj.sh b/docker/ubuntu-full/bh-proj.sh
@@ -100,9 +100,6 @@ else
     done;
 fi
 
-apt-get update -y
-DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y patchelf
-rm -rf /var/lib/apt/lists/*
 patchelf --set-soname libinternalproj.so.${PROJ_SO_FIRST} ${DESTDIR}${PROJ_INSTALL_PREFIX}/lib/libinternalproj.so.${PROJ_SO}
 for i in "${DESTDIR}${PROJ_INSTALL_PREFIX}/bin"/*; do
   patchelf --replace-needed libproj.so.${PROJ_SO_FIRST} libinternalproj.so.${PROJ_SO_FIRST} $i;

diff --git a/docker/ubuntu-small/Dockerfile b/docker/ubuntu-small/Dockerfile
@@ -22,7 +22,12 @@ ARG TARGET_ARCH=
 RUN echo ${TARGET_ARCH}
 COPY ./bh-set-envvars.sh /buildscripts/bh-set-envvars.sh
 
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN rm -f /etc/apt/apt.conf.d/docker-clean \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && if test "${TARGET_ARCH}" != ""; then \
     rm -f /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources \
     && echo "deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu/ noble main restricted universe" >> /etc/apt/sources.list \
@@ -36,24 +41,26 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y g++-13-${GCC_ARCH}-linux-gnu \
     && ln -s ${GCC_ARCH}-linux-gnu-gcc-13 /usr/bin/${GCC_ARCH}-linux-gnu-gcc \
-    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++ \
-    && rm -rf /var/lib/apt/lists/*; \
+    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++; \
     fi
 
 # Setup build env for PROJ
 USER root
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
             build-essential ca-certificates ccache \
             git make ninja-build cmake wget unzip libtool automake \
             zlib1g-dev${APT_ARCH_SUFFIX} libsqlite3-dev${APT_ARCH_SUFFIX} pkg-config sqlite3 libcurl4-openssl-dev${APT_ARCH_SUFFIX} \
             libtiff-dev${APT_ARCH_SUFFIX} \
-            rsync \
-    && rm -rf /var/lib/apt/lists/*
+            rsync
 
 # Setup build env for GDAL
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
        python3-dev${APT_ARCH_SUFFIX} python3-numpy${APT_ARCH_SUFFIX} python3-setuptools${APT_ARCH_SUFFIX} \
@@ -64,8 +71,7 @@ RUN . /buildscripts/bh-set-envvars.sh \
        libzstd-dev${APT_ARCH_SUFFIX} bash zip curl \
        libpq-dev${APT_ARCH_SUFFIX} libssl-dev${APT_ARCH_SUFFIX} libopenjp2-7-dev${APT_ARCH_SUFFIX} \
        libspatialite-dev${APT_ARCH_SUFFIX} \
-       autoconf automake sqlite3 bash-completion swig \
-    && rm -rf /var/lib/apt/lists/*
+       autoconf automake sqlite3 bash-completion swig
 
 # Build openjpeg
 ARG OPENJPEG_VERSION=
@@ -99,6 +105,11 @@ ARG WITH_CCACHE
 
 # Build PROJ
 ARG PROJ_VERSION=master
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y patchelf
+
 RUN --mount=type=cache,id=ubuntu-small-proj,target=$HOME/.cache \
     . /buildscripts/bh-set-envvars.sh \
     && mkdir proj \
@@ -147,9 +158,6 @@ RUN --mount=type=cache,id=ubuntu-small-proj,target=$HOME/.cache \
     && rm /build${PROJ_INSTALL_PREFIX}/lib/libproj.*  \
     && ${GCC_ARCH}-linux-gnu-strip -s /build${PROJ_INSTALL_PREFIX}/lib/libinternalproj.so.${PROJ_SO} \
     && for i in /build${PROJ_INSTALL_PREFIX}/bin/*; do ${GCC_ARCH}-linux-gnu-strip -s $i 2>/dev/null || /bin/true; done \
-    && apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y patchelf \
-    && rm -rf /var/lib/apt/lists/* \
     && patchelf --set-soname libinternalproj.so.${PROJ_SO_FIRST} /build${PROJ_INSTALL_PREFIX}/lib/libinternalproj.so.${PROJ_SO} \
     && for i in /build${PROJ_INSTALL_PREFIX}/bin/*; do patchelf --replace-needed libproj.so.${PROJ_SO_FIRST} libinternalproj.so.${PROJ_SO_FIRST} $i; done
 
@@ -234,19 +242,17 @@ USER root
 RUN date
 
 # Update distro
-RUN apt-get update -y && apt-get upgrade -y \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update \
-# PROJ dependencies
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y  --no-install-recommends \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    # PROJ dependencies
+    && apt-get install -y --no-install-recommends \
         libsqlite3-0 libtiff6 libcurl4 \
         curl unzip ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-
-# GDAL dependencies
-RUN apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y  --no-install-recommends \
+    # GDAL dependencies
+    && apt-get install -y --no-install-recommends \
         python3-numpy libpython3.12 \
         libjpeg-turbo8 libgeos3.12.1 libgeos-c1v5 \
         libexpat1 \
@@ -256,12 +262,7 @@ RUN apt-get update -y \
         libzstd1 bash libpq5 libssl3 libopenjp2-7 libspatialite8 \
         # pil for antialias option of gdal2tiles
         python3-pil \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        python-is-python3 \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y python-is-python3
 
 # Order layers starting with less frequently varying ones
 # Only used for custom libopenjp2