Add CMake options EMBED_RESOURCE_FILES and USE_ONLY_EMBEDDED_RESOURCE_FILES for proj.db and proj.ini embedding

.github/workflows/clang_linux.yml

@@ -31,6 +31,10 @@ jobs:
             id: regular
             PROJ_CMAKE_BUILD_OPTIONS: ""
 
+          - name: EMBED_RESOURCE_FILES
+            id: EMBED_RESOURCE_FILES
+            PROJ_CMAKE_BUILD_OPTIONS: "-DEMBED_RESOURCE_FILES=ON"
+
           - name: Without TIFF
             id: without_tiff
             PROJ_CMAKE_BUILD_OPTIONS: "-DENABLE_TIFF=OFF"

.github/workflows/fedora_rawhide.yml

@@ -0,0 +1,37 @@
+name: Fedora Rawhide
+
+on:
+  push:
+      paths-ignore:
+          - 'docs/**'
+  pull_request:
+      paths-ignore:
+          - 'docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+
+  fedora_rawhide:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Cache
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+                ${{ github.workspace }}/ccache.tar.gz
+          key: ${{ runner.os }}-cache-fedora_rawhide-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-cache-fedora_rawhide-
+
+      - name: Run
+        run: docker run -e CI -e WORK_DIR="$PWD" -v $PWD:$PWD fedora:rawhide $PWD/.github/workflows/fedora_rawhide/start.sh

.github/workflows/fedora_rawhide/start.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils
+
+cd "$WORK_DIR"
+
+if test -f "$WORK_DIR/ccache.tar.gz"; then
+    echo "Restoring ccache..."
+    (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+fi
+
+export CCACHE_CPP2=yes
+
+ccache -M 500M
+ccache -s
+
+mkdir build
+cd build
+CC=clang CXX=clang++ cmake .. \
+    -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
+make -j$(nproc)
+ctest -j$(nproc)
+cd ..
+
+ccache -s
+
+echo "Saving ccache..."
+rm -f "$WORK_DIR/ccache.tar.gz"
+(cd $HOME && tar czf "$WORK_DIR/ccache.tar.gz" .cache)

CMakeLists.txt

@@ -349,6 +349,45 @@ else()
   message(STATUS "Testing disabled")
 endif()
 
+################################################################################
+# Whether we should embed resources
+################################################################################
+
+function (is_sharp_embed_available res)
+    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.21 AND
+        ((CMAKE_C_COMPILER_ID STREQUAL "GNU") OR (CMAKE_C_COMPILER_ID STREQUAL "Clang")))
+        # CMAKE_C_STANDARD=23 only supported since CMake 3.21
+        set(TEST_SHARP_EMBED
+          "static const unsigned char embedded[] = {\n#embed __FILE__\n};\nint main() { (void)embedded; return 0;}"
+        )
+        set(CMAKE_C_STANDARD_BACKUP "${CMAKE_C_STANDARD}")
+        set(CMAKE_C_STANDARD "23")
+        check_c_source_compiles("${TEST_SHARP_EMBED}" _TEST_SHARP_EMBED)
+        set(CMAKE_C_STANDARD "${CMAKE_C_STANDARD_BACKUP}")
+        if (_TEST_SHARP_EMBED)
+            set(${res} ON PARENT_SCOPE)
+        else()
+            set(${res} OFF PARENT_SCOPE)
+        endif()
+    else()
+        set(${res} OFF PARENT_SCOPE)
+    endif()
+endfunction()
+
+is_sharp_embed_available(IS_SHARP_EMBED_AVAILABLE_RES)
+if (NOT BUILD_SHARED_LIBS)
+    set(DEFAULT_EMBED_RESOURCE_FILES ON)
+else()
+    set(DEFAULT_EMBED_RESOURCE_FILES OFF)
+endif()
+option(EMBED_RESOURCE_FILES "Whether resource files (limited to proj.db) should be embedded into the PROJ library" ${DEFAULT_EMBED_RESOURCE_FILES})
+
+option(USE_ONLY_EMBEDDED_RESOURCE_FILES "Whether embedded resource files (limited to proj.db) should be used (should nominally be used together with EMBED_RESOURCE_FILES=ON, otherwise this will result in non-functional builds)" OFF)
+
+if (USE_ONLY_EMBEDDED_RESOURCE_FILES AND NOT EMBED_RESOURCE_FILES)
+  message(WARNING "USE_ONLY_EMBEDDED_RESOURCE_FILES=ON set but EMBED_RESOURCE_FILES=OFF: some drivers will lack required resource files")
+endif()
+
 ################################################################################
 # Build configured components
 ################################################################################
@@ -432,3 +471,5 @@ endif()
 
 configure_file(cmake/uninstall.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake @ONLY)
 add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake)
+
+message(STATUS "EMBED_RESOURCE_FILES=${EMBED_RESOURCE_FILES}")

cmake/FileEmbed.cmake

@@ -0,0 +1,99 @@
+# Derived from https://gitlab.com/jhamberg/cmake-examples/-/blob/master/cmake/FileEmbed.cmake
+# MIT licensed
+# Copyright (c) 2022 Jonathan Hamberg
+
+function(FileEmbedSetup)
+
+    if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed)
+        file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed)
+    endif ()
+
+    if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c)
+        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c "")
+    endif ()
+
+    add_library(file_embed ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c)
+    target_include_directories(file_embed PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/file_embed)
+
+endfunction()
+
+function(FileEmbedAdd file)
+    FileEmbedGenerate(${file} var)
+    target_sources(file_embed PUBLIC ${var})
+
+    add_custom_command(
+            OUTPUT ${var}
+            COMMAND ${CMAKE_COMMAND}
+            -DRUN_FILE_EMBED_GENERATE=1
+            "-DFILE_EMBED_GENERATE_PATH=${file}"
+            -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake
+            MAIN_DEPENDENCY ${file}
+    )
+endfunction()
+
+function(FileEmbedGenerate file generated_c)
+
+    get_filename_component(base_filename ${file} NAME)
+    set(output_filename "${base_filename}.c")
+    string(MAKE_C_IDENTIFIER ${base_filename} c_name)
+    file(READ ${file} content HEX)
+
+    string(LENGTH "${content}" size)
+    math(EXPR size_mult_16 "${size} - (${size} % 16)")
+    string(SUBSTRING "${content}" 0 ${size_mult_16} content_mult_16)
+
+    string(REGEX REPLACE
+           "([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])\
+([A-Fa-f0-9][A-Fa-f0-9])"
+           "0x\\1,0x\\2,0x\\3,0x\\4,0x\\5,0x\\6,0x\\7,0x\\8,\n" SEPARATED_HEX "${content_mult_16}")
+
+    set(SEPARATED_HEX_REMAINDER "")
+    if (NOT ${size_mult_16} EQUAL ${size})
+        string(SUBSTRING "${content}" ${size_mult_16} 16 content_remainder)
+        string(REGEX REPLACE "([A-Fa-f0-9][A-Fa-f0-9])" "0x\\1," SEPARATED_HEX_REMAINDER "${content_remainder}")
+    endif()
+
+    set(output_c "${SEPARATED_HEX}${SEPARATED_HEX_REMAINDER}")
+
+    set(output_c "
+#include \"${c_name}.h\"
+const uint8_t ${c_name}_data[] = {
+    ${output_c}
+}\;
+const unsigned ${c_name}_size = sizeof(${c_name}_data)\;
+")
+
+    set(output_h "
+#ifndef ${c_name}_H
+#define ${c_name}_H
+#include <stdint.h>
+extern const uint8_t ${c_name}_data[]\;
+extern const unsigned ${c_name}_size\;
+#endif // ${c_name}_H
+    ")
+
+
+    if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed)
+        file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed)
+    endif ()
+
+
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c
+            ${output_c})
+
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.h
+            ${output_h})
+
+    set(${generated_c} ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c PARENT_SCOPE)
+
+endfunction()
+
+if (RUN_FILE_EMBED_GENERATE)
+    FileEmbedGenerate(${FILE_EMBED_GENERATE_PATH} var)
+endif ()

data/CMakeLists.txt

@@ -11,7 +11,7 @@ if(NOT IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
   endif()
 endif()
 
-set(CONFIG_FILES
+set(PROJ_INI
   proj.ini
 )
 
@@ -60,7 +60,7 @@ add_custom_command(
 add_custom_target(generate_proj_db ALL DEPENDS ${PROJ_DB})
 
 if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
-  foreach(FILE ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
+  foreach(FILE ${PROJ_INI} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
     configure_file(${FILE} ${FILE} COPYONLY)
   endforeach()
 endif()
@@ -78,7 +78,7 @@ set(DATA_FOR_TESTS
 execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests)
 execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests)
 
-foreach(FILE ${DATA_FOR_TESTS} ${CONFIG_FILES})
+foreach(FILE ${DATA_FOR_TESTS} ${PROJ_INI})
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/${FILE} COPYONLY)
 endforeach()
 
@@ -109,13 +109,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D
 #install
 #
 set(ALL_DATA_FILE
-  ${CONFIG_FILES}
   ${PROJ_DICTIONARY}
   ${GRIDSHIFT_FILES}
-  ${PROJ_DB}
   ${SCHEMA_FILES}
   ${GEOTIFF_FILES}
 )
+
+if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES)
+    list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB})
+endif()
+
 install(
   FILES ${ALL_DATA_FILE}
   DESTINATION ${PROJ_DATA_PATH}

data/generate_proj_db.cmake

@@ -14,6 +14,8 @@ file(READ ${ALL_SQL_IN} CONTENTS)
 string(REPLACE "\${PROJ_VERSION}" "${PROJ_VERSION}" CONTENTS_MOD "${CONTENTS}")
 file(WRITE "${ALL_SQL_IN}" "${CONTENTS_MOD}")
 
+set(generate_proj_db ON)
+
 if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
   set(USE_PROJ_DB_CACHE_DIR TRUE)
   set(PROJ_DB_SQL_MD5_FILE "${PROJ_DB_CACHE_DIR}/proj.db.sql.md5")
@@ -26,18 +28,21 @@ if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
     message(STATUS "Reusing cached proj.db from ${PROJ_DB_CACHE_DIR}")
     get_filename_component(PROJ_DB_DIR "${PROJ_DB}" DIRECTORY)
     file(COPY "${CACHED_PROJ_DB}" DESTINATION "${PROJ_DB_DIR}")
-    return()
+    file(TOUCH "${PROJ_DB}")
+    set(generate_proj_db OFF)
   endif()
 endif()
 
-execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}"
-                INPUT_FILE "${ALL_SQL_IN}"
-                RESULT_VARIABLE STATUS)
+if (generate_proj_db)
+    execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}"
+                    INPUT_FILE "${ALL_SQL_IN}"
+                    RESULT_VARIABLE STATUS)
 
-if(STATUS AND NOT STATUS EQUAL 0)
-  message(FATAL_ERROR "Build of proj.db failed")
-elseif(USE_PROJ_DB_CACHE_DIR)
-  message(STATUS "Saving cache: ${CACHED_PROJ_DB}")
-  file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}")
-  file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n")
+    if(STATUS AND NOT STATUS EQUAL 0)
+      message(FATAL_ERROR "Build of proj.db failed")
+    elseif(USE_PROJ_DB_CACHE_DIR)
+      message(STATUS "Saving cache: ${CACHED_PROJ_DB}")
+      file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}")
+      file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n")
+    endif()
 endif()

docs/source/install.rst

@@ -446,6 +446,26 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.
 
     Embed ``PROJ_DATA`` hard-coded alternative path for data files location. Disable to avoid setting this non-relocatable hard-coded path. Default ON.
 
+.. option:: EMBED_RESOURCE_FILES=ON/OFF
+
+    .. versionadded:: 9.6
+
+    Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON
+    for static library builds (BUILD_SHARED_LIBS=OFF).
+    When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.
+
+.. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF
+
+    .. versionadded:: 9.6
+
+    Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate
+    :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the
+    embedded version if not found.
+    By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating
+    those files on the file system is made. Default is OFF.
+    Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting
+    USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF.
+
 
 Building on Windows with vcpkg and Visual Studio 2017 or 2019
 --------------------------------------------------------------------------------

src/embedded_resources.c

@@ -0,0 +1,34 @@
+#include "embedded_resources.h"
+
+#if USE_SHARP_EMBED
+const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
+    static const unsigned char proj_db[] = {
+#embed PROJ_DB
+    };
+    *pnSize = (unsigned int)sizeof(proj_db);
+    return proj_db;
+}
+
+const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
+    static const char proj_ini[] = {
+#embed PROJ_INI
+    };
+    *pnSize = (unsigned int)sizeof(proj_ini);
+    return proj_ini;
+}
+
+#else
+
+#include "file_embed/proj_db.h"
+const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
+    *pnSize = proj_db_size;
+    return proj_db_data;
+}
+
+#include "file_embed/proj_ini.h"
+const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
+    *pnSize = proj_ini_size;
+    return (const char *)proj_ini_data;
+}
+
+#endif

src/embedded_resources.h

@@ -0,0 +1,15 @@
+#ifndef EMBEDDED_RESOURCES_H
+#define EMBEDDED_RESOURCES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize);
+const char *pj_get_embedded_proj_ini(unsigned int *pnSize);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* EMBEDDED_RESOURCES_H */