Merge pull request #10974 from pjonsson/docker-more-cache

docker: enable more caching

docker/alpine-normal/Dockerfile

@@ -82,6 +82,7 @@ RUN apk add --no-cache \
     py3-setuptools \
     python3-dev \
     qhull-dev \
+    rsync \
     sfcgal-dev \
     snappy-dev \
     sqlite-dev \
@@ -140,8 +141,8 @@ RUN if test "${OPENDRIVE_VERSION}" != ""; then ( \
     && rm -rf libOpenDRIVE-${OPENDRIVE_VERSION} \
     ); fi
 
-RUN apk add --no-cache rsync ccache
 ARG RSYNC_REMOTE
+ARG WITH_CCACHE
 
 # Build PROJ
 ARG PROJ_VERSION=master
@@ -154,6 +155,8 @@ RUN --mount=type=cache,id=alpine-normal-proj,target=$HOME/.cache \
         echo "Downloading cache..."; \
         rsync -ra ${RSYNC_REMOTE}/proj/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
         export CC="ccache gcc"; \
         export CXX="ccache g++"; \
         export PROJ_DB_CACHE_PARAM="-DPROJ_DB_CACHE_DIR=$HOME/.cache"; \
@@ -172,10 +175,12 @@ RUN --mount=type=cache,id=alpine-normal-proj,target=$HOME/.cache \
     && ninja install \
     && DESTDIR="/build_proj" ninja install \
     && if test "${RSYNC_REMOTE}" != ""; then \
-        ccache -s; \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/proj/$(uname -m)/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
+        ccache -s; \
         unset CC; \
         unset CXX; \
     fi \
@@ -197,6 +202,8 @@ RUN --mount=type=cache,id=alpine-normal-spatialite,target=$HOME/.cache \
         echo "Downloading cache..."; \
         rsync -ra ${RSYNC_REMOTE}/spatialite/ $HOME/.cache/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
         export CC="ccache gcc"; \
         export CXX="ccache g++"; \
         ccache -M 100M; \
@@ -205,10 +212,12 @@ RUN --mount=type=cache,id=alpine-normal-spatialite,target=$HOME/.cache \
     && make -j$(nproc) \
     && make install \
     && if test "${RSYNC_REMOTE}" != ""; then \
-        ccache -s; \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/spatialite/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
+        ccache -s; \
         unset CC; \
         unset CXX; \
     fi \
@@ -247,6 +256,8 @@ RUN --mount=type=cache,id=alpine-normal-gdal,target=$HOME/.cache \
         echo "Downloading cache..."; \
         rsync -ra ${RSYNC_REMOTE}/gdal/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
         # Little trick to avoid issues with Python bindings
         printf "#!/bin/sh\nccache gcc \$*" > ccache_gcc.sh; \
         chmod +x ccache_gcc.sh; \
@@ -274,10 +285,12 @@ RUN --mount=type=cache,id=alpine-normal-gdal,target=$HOME/.cache \
     && (ninja multireadtest && cp apps/multireadtest /build/usr/bin) \
     && cd .. \
     && if test "${RSYNC_REMOTE}" != ""; then \
-        ccache -s; \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/gdal/$(uname -m)/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
+        ccache -s; \
         unset CC; \
         unset CXX; \
     fi \

docker/alpine-small/Dockerfile

@@ -20,11 +20,15 @@ RUN apk add --no-cache wget curl unzip make libtool autoconf automake pkgconfig
 
 # For PROJ and GDAL
 RUN apk add --no-cache \
+    ccache \
     cmake ninja-build \
     linux-headers \
     curl-dev tiff-dev \
     zlib-dev zstd-dev lz4-dev libdeflate-dev libarchive-dev \
-    libjpeg-turbo-dev libpng-dev libwebp-dev expat-dev postgresql-dev openjpeg-dev
+    libjpeg-turbo-dev libpng-dev libwebp-dev expat-dev \
+    postgresql-dev \
+    rsync \
+    openjpeg-dev
 
 # Ninja is not in the default path on Alpine.
 ENV PATH=/usr/lib/ninja-build/bin:$PATH
@@ -47,8 +51,6 @@ RUN if test "${OPENJPEG_VERSION}" != ""; then ( \
     && rm -rf openjpeg-${OPENJPEG_VERSION} \
     ); fi
 
-RUN apk add --no-cache rsync ccache
-
 ARG PROJ_DATUMGRID_LATEST_LAST_MODIFIED
 RUN \
     mkdir -p /build_projgrids/usr/share/proj \
@@ -57,6 +59,7 @@ RUN \
     && rm -f *.zip
 
 ARG RSYNC_REMOTE
+ARG WITH_CCACHE
 
 # Build PROJ
 ARG PROJ_VERSION=master
@@ -69,6 +72,8 @@ RUN --mount=type=cache,id=alpine-small-proj,target=$HOME/.cache \
         echo "Downloading cache..."; \
         rsync -ra ${RSYNC_REMOTE}/proj/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
         export CC="ccache gcc"; \
         export CXX="ccache g++"; \
         export PROJ_DB_CACHE_PARAM="-DPROJ_DB_CACHE_DIR=$HOME/.cache"; \
@@ -87,10 +92,12 @@ RUN --mount=type=cache,id=alpine-small-proj,target=$HOME/.cache \
     && ninja install \
     && DESTDIR="/build_proj" ninja install \
     && if test "${RSYNC_REMOTE}" != ""; then \
-        ccache -s; \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/proj/$(uname -m)/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
+        ccache -s; \
         unset CC; \
         unset CXX; \
     fi \
@@ -117,6 +124,8 @@ RUN --mount=type=cache,id=alpine-small-gdal,target=$HOME/.cache \
         echo "Downloading cache..."; \
         rsync -ra ${RSYNC_REMOTE}/gdal/$(uname -m)/ $HOME/.cache/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
         export CC="ccache gcc"; \
         export CXX="ccache g++"; \
         ccache -M 1G; \
@@ -138,10 +147,12 @@ RUN --mount=type=cache,id=alpine-small-gdal,target=$HOME/.cache \
     && DESTDIR="/build" ninja install \
     && cd .. \
     && if test "${RSYNC_REMOTE}" != ""; then \
-        ccache -s; \
         echo "Uploading cache..."; \
         rsync -ra --delete $HOME/.cache/ ${RSYNC_REMOTE}/gdal/$(uname -m)/; \
         echo "Finished"; \
+    fi \
+    && if [ -n "${WITH_CCACHE:-}" ]; then \
+        ccache -s; \
         unset CC; \
         unset CXX; \
     fi \

docker/ubuntu-full/Dockerfile

@@ -21,7 +21,12 @@ ARG TARGET_ARCH=
 RUN echo ${TARGET_ARCH}
 COPY ./bh-set-envvars.sh /buildscripts/bh-set-envvars.sh
 
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN rm -f /etc/apt/apt.conf.d/docker-clean \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && if test "${TARGET_ARCH}" != ""; then \
     rm -f /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources \
     && echo "deb [arch=amd64] http://us.archive.ubuntu.com/ubuntu/ noble main restricted universe" >> /etc/apt/sources.list \
@@ -35,24 +40,27 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y g++-13-${GCC_ARCH}-linux-gnu \
     && ln -s ${GCC_ARCH}-linux-gnu-gcc-13 /usr/bin/${GCC_ARCH}-linux-gnu-gcc \
-    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++ \
-    && rm -rf /var/lib/apt/lists/*; \
+    && ln -s ${GCC_ARCH}-linux-gnu-g++-13 /usr/bin/${GCC_ARCH}-linux-gnu-g++; \
     fi
 
 # Setup build env for PROJ
 USER root
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
-            build-essential ca-certificates \
+            build-essential ca-certificates ccache\
             git make ninja-build cmake wget unzip libtool automake \
             zlib1g-dev${APT_ARCH_SUFFIX} libsqlite3-dev${APT_ARCH_SUFFIX} pkg-config sqlite3 libcurl4-openssl-dev${APT_ARCH_SUFFIX} \
             libtiff-dev${APT_ARCH_SUFFIX} \
-    && rm -rf /var/lib/apt/lists/*
+            rsync
 
 ARG JAVA_VERSION=17
 # Setup build env for GDAL
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \
        libopenjp2-7-dev${APT_ARCH_SUFFIX} libcairo2-dev${APT_ARCH_SUFFIX} \
@@ -72,8 +80,7 @@ RUN . /buildscripts/bh-set-envvars.sh \
        libbrotli-dev${APT_ARCH_SUFFIX} \
        libarchive-dev${APT_ARCH_SUFFIX} \
        libaec-dev${APT_ARCH_SUFFIX} \
-       libavif-dev${APT_ARCH_SUFFIX} \
-    && rm -rf /var/lib/apt/lists/*
+       libavif-dev${APT_ARCH_SUFFIX}
 
 # Build likbkea
 ARG KEA_VERSION=1.5.2
@@ -132,13 +139,16 @@ RUN . /buildscripts/bh-set-envvars.sh \
 
 # Build tiledb
 ARG TILEDB_VERSION=2.23.0
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
+    && apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        libspdlog-dev${APT_ARCH_SUFFIX} libmagic-dev${APT_ARCH_SUFFIX}
+
 COPY ./tiledb-FindLZ4_EP.cmake.patch /buildscripts/tiledb-FindLZ4_EP.cmake.patch
 COPY ./tiledb-FindOpenSSL_EP.cmake.patch /buildscripts/tiledb-FindOpenSSL_EP.cmake.patch
 RUN . /buildscripts/bh-set-envvars.sh \
-    && apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        libspdlog-dev${APT_ARCH_SUFFIX} libmagic-dev${APT_ARCH_SUFFIX} \
-    && rm -rf /var/lib/apt/lists/* \
     && mkdir tiledb \
     && curl -L -fsS https://github.com/TileDB-Inc/TileDB/archive/${TILEDB_VERSION}.tar.gz \
         | tar xz -C tiledb --strip-components=1 \
@@ -225,7 +235,9 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && rm -rf QB3
 
 ARG WITH_PDFIUM=yes
-RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
   curl -LO -fsS https://github.com/rouault/pdfium_build_gdal_3_10/releases/download/pdfium_6677_v1/install-ubuntu2004-rev6677.tar.gz \
   && tar -xzf install-ubuntu2004-rev6677.tar.gz \
   && chown -R root:root install \
@@ -234,14 +246,15 @@ RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$"  ; then ( \
   && rm -rf install-ubuntu2004-rev6677.tar.gz install \
   && apt-get update -y \
   && apt-get install -y --fix-missing --no-install-recommends liblcms2-dev${APT_ARCH_SUFFIX} \
-  && rm -rf /var/lib/apt/lists/* \
   ) ; fi
 
 # Build libjxl
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
-    && apt-get install -y --fix-missing --no-install-recommends libgflags-dev${APT_ARCH_SUFFIX} \
-    && git clone https://github.com/libjxl/libjxl.git --recursive \
+    && apt-get install -y --fix-missing --no-install-recommends libgflags-dev${APT_ARCH_SUFFIX}
+RUN git clone https://github.com/libjxl/libjxl.git --recursive \
     && cd libjxl \
     && mkdir build \
     && cd build \
@@ -252,14 +265,15 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && rm -f /lib/${GCC_ARCH}-linux-gnu/libjxl*.a \
     && rm -f /build_thirdparty/lib/${GCC_ARCH}-linux-gnu/libjxl*.a \
     && cd ../.. \
-    && rm -rf libjxl \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf libjxl
 
 # Install Arrow C++
 ARG ARROW_VERSION=16.1.0-1
 # ARROW_SOVERSION to be updated in the "Build final image" section too
 ARG ARROW_SOVERSION=1600
-RUN . /buildscripts/bh-set-envvars.sh \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    . /buildscripts/bh-set-envvars.sh \
     && apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ca-certificates lsb-release wget \
     && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
@@ -272,13 +286,10 @@ RUN . /buildscripts/bh-set-envvars.sh \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-acero-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \
-    && rm -rf /var/lib/apt/lists/* \
     && rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
 
-RUN apt-get update -y \
-    && apt-get install -y --fix-missing --no-install-recommends rsync ccache \
-    && rm -rf /var/lib/apt/lists/*
 ARG RSYNC_REMOTE
+ARG WITH_CCACHE
 
 ARG WITH_DEBUG_SYMBOLS=no
 
@@ -298,6 +309,11 @@ RUN . /buildscripts/bh-set-envvars.sh \
      && rm -rf /build_tmp_proj
 
 # Build PROJ
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y patchelf
+
 ARG PROJ_VERSION=master
 RUN --mount=type=cache,id=ubuntu-full-proj,target=$HOME/.cache \
     . /buildscripts/bh-set-envvars.sh \
@@ -322,18 +338,19 @@ RUN date
 ARG JAVA_VERSION=17
 ARG ARROW_SOVERSION=1600
 
-# Update distro
-RUN apt-get update -y && apt-get upgrade -y \
-    && rm -rf /var/lib/apt/lists/*
-
 ARG TARGET_ARCH=
-RUN apt-get update \
-# PROJ dependencies
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+# Update distro
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    # PROJ dependencies
+    && apt-get install -y \
         libsqlite3-0 libtiff6 libcurl4 \
         wget curl unzip ca-certificates \
-# GDAL dependencies
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    # GDAL dependencies
+    && apt-get install -y \
         libopenjp2-7 libcairo2 python3-numpy \
         libpng16-16 libjpeg-turbo8 libgif7 liblzma5 libgeos3.12.1 libgeos-c1v5 \
         libxml2 libexpat1 \
@@ -354,18 +371,17 @@ RUN apt-get update \
         # pil for antialias option of gdal2tiles
         python3-pil \
     # Install JRE with --no-install-recommends, otherwise it draws default-jre, which draws systemd, which fails to install when running the arm64v8/ubuntu:24.04 image on a 64bit host
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-"$JAVA_VERSION"-jre \
+    && apt-get install -y --no-install-recommends openjdk-"$JAVA_VERSION"-jre \
     # Worksround OGDI packaging bug for Ubuntu 24.04
     && ln -s /usr/lib/$(uname -p)-linux-gnu/ogdi/4.1/libvrf.so /usr/lib/$(uname -p)-linux-gnu \
     # Install Arrow C++
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ca-certificates lsb-release wget \
+    && apt-get install -y -V ca-certificates lsb-release wget \
     && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
+    && apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
     && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow${ARROW_SOVERSION} \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet${ARROW_SOVERSION} \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset${ARROW_SOVERSION} \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y -V libarrow${ARROW_SOVERSION} \
+    && apt-get install -y -V libparquet${ARROW_SOVERSION} \
+    && apt-get install -y -V libarrow-dataset${ARROW_SOVERSION}
 
 # Attempt to order layers starting with less frequently varying ones