ssl: conditionally enable SSL Engine APIs

Conditionally enable ssl_socket methods that use the deprecated OpenSSL ENGINE APIs. The APIs are enabled when the OpenSSL being compiled against has the ENGINE APIs enabled. In addition these APIs can be disabled by passing -DENABLE_SSL_ENGINE_API=OFF to CMake at build-time. Fixed: #795 Fixed: #713 Signed-off-by: GitHub <[email protected]>
alanxz · Dec 24, 2023 · 497480c · 497480c
1 parent a2d04a4
commit 497480c
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 6 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -42,6 +42,7 @@ endif()
 
 include(CheckSymbolExists)
 include(CheckLibraryExists)
+include(CMakeDependentOption)
 include(CMakePushCheckState)
 include(GNUInstallDirs)
 
@@ -120,14 +121,19 @@ if (ENABLE_SSL_SUPPORT)
   set(THREADS_PREFER_PTHREAD_FLAG ON)
   find_package(Threads REQUIRED)
   cmake_pop_check_state()
+
+  cmake_push_check_state()
+  set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL)
+  check_symbol_exists(ENGINE_new openssl/engine.h HAS_OPENSSL_ENGINE)
+  cmake_pop_check_state()
+
+  cmake_dependent_option(ENABLE_SSL_ENGINE_API "Enable support for deprecated OpenSSL ENGINE feature" ON "HAS_OPENSSL_ENGINE" OFF)
 endif()
 
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   include(CTest)
 endif()
 
-include(CMakeDependentOption)
-
 option(BUILD_SHARED_LIBS "Build rabbitmq-c as a shared library" ON)
 option(BUILD_STATIC_LIBS "Build rabbitmq-c as a static library" ON)
 option(INSTALL_STATIC_LIBS "Install rabbitmq-c static library" ON)

diff --git a/cmake/config.h.in b/cmake/config.h.in
@@ -7,4 +7,6 @@
 
 #define AMQ_PLATFORM "@CMAKE_SYSTEM_NAME@"
 
+#cmakedefine ENABLE_SSL_ENGINE_API
+
 #endif /* CONFIG_H */
diff --git a/include/rabbitmq-c/amqp.h b/include/rabbitmq-c/amqp.h
@@ -670,7 +670,8 @@ typedef enum amqp_status_enum_ {
                                                          certificate failed. */
   AMQP_STATUS_SSL_CONNECTION_FAILED = -0x0203, /**< SSL handshake failed. */
   AMQP_STATUS_SSL_SET_ENGINE_FAILED = -0x0204, /**< SSL setting engine failed */
-  _AMQP_STATUS_SSL_NEXT_VALUE = -0x0205        /**< Internal value */
+  AMQP_STATUS_SSL_UNIMPLEMENTED = -0x0205, /**< SSL API is not implemented. */
+  _AMQP_STATUS_SSL_NEXT_VALUE = -0x0206        /**< Internal value */
 } amqp_status_enum;
 
 /**

diff --git a/include/rabbitmq-c/ssl_socket.h b/include/rabbitmq-c/ssl_socket.h
@@ -115,7 +115,8 @@ int AMQP_CALL amqp_ssl_socket_set_key(amqp_socket_t *self, const char *cert,
  * \param [in] the key ID.
  *
  * \return \ref AMQP_STATUS_OK on success an \ref amqp_status_enum value on
- *  failure.
+ *  failure. May return \ref AMQP_STATUS_SSL_UNIMPLEMENTED if OpenSSL does 
+ *  not support the ENGINE API.
  *
  * \since v0.11.0
  */
@@ -278,7 +279,8 @@ int AMQP_CALL amqp_initialize_ssl_library(void);
  * has been called.
  *
  * \param [in] engine the engine ID
- * \return AMQP_STATUS_OK on success.
+ * \return AMQP_STATUS_OK on success. May return \ref AMQP_STATUS_SSL_UNIMPLEMENTED
+ *   if OpenSSL does not support the ENGINE API.
  *
  * \since v0.11.0
  */

diff --git a/librabbitmq/amqp_api.c b/librabbitmq/amqp_api.c
@@ -85,7 +85,9 @@ static const char *ssl_error_strings[] = {
     /* AMQP_STATUS_SSL_CONNECTION_FAILED      -0x0203 */
     "SSL handshake failed",
     /* AMQP_STATUS_SSL_SET_ENGINE_FAILED      -0x0204 */
-    "SSL setting engine failed"};
+    "SSL setting engine failed",
+    /* AMQP_STATUS_SSL_UNIMPLEMENTED          -0x0204 */
+    "SSL API is not implemented"};
 
 static const char *unknown_error_string = "(unknown error)";
 

diff --git a/librabbitmq/amqp_openssl.c b/librabbitmq/amqp_openssl.c
@@ -23,7 +23,9 @@
 #include <limits.h>
 #include <openssl/bio.h>
 #include <openssl/conf.h>
+#ifdef ENABLE_SSL_ENGINE_API
 #include <openssl/engine.h>
+#endif
 #include <openssl/err.h>
 #include <openssl/rsa.h>
 #include <openssl/ssl.h>
@@ -37,7 +39,9 @@ static int decrement_ssl_connections(void);
 static pthread_mutex_t openssl_init_mutex = PTHREAD_MUTEX_INITIALIZER;
 static amqp_boolean_t openssl_bio_initialized = 0;
 static int openssl_connections = 0;
+#ifdef ENABLE_SSL_ENGINE_API
 static ENGINE *openssl_engine = NULL;
+#endif
 
 #define CHECK_SUCCESS(condition)                                            \
   do {                                                                      \
@@ -407,6 +411,7 @@ int amqp_ssl_socket_set_key(amqp_socket_t *base, const char *cert,
 
 int amqp_ssl_socket_set_key_engine(amqp_socket_t *base, const char *cert,
                                    const char *key) {
+#ifdef ENABLE_SSL_ENGINE_API
   int status;
   struct amqp_ssl_socket_t *self;
   EVP_PKEY *pkey = NULL;
@@ -431,6 +436,9 @@ int amqp_ssl_socket_set_key_engine(amqp_socket_t *base, const char *cert,
     return AMQP_STATUS_SSL_ERROR;
   }
   return AMQP_STATUS_OK;
+#else
+  return AMQP_STATUS_SSL_UNIMPLEMENTED;
+#endif
 }
 
 static int password_cb(AMQP_UNUSED char *buffer, AMQP_UNUSED int length,
@@ -584,6 +592,7 @@ void amqp_set_initialize_ssl_library(amqp_boolean_t do_initialize) {
 int amqp_initialize_ssl_library(void) { return AMQP_STATUS_OK; }
 
 int amqp_set_ssl_engine(const char *engine) {
+#ifdef ENABLE_SSL_ENGINE_API
   int status = AMQP_STATUS_OK;
   CHECK_SUCCESS(pthread_mutex_lock(&openssl_init_mutex));
 
@@ -613,6 +622,9 @@ int amqp_set_ssl_engine(const char *engine) {
 out:
   CHECK_SUCCESS(pthread_mutex_unlock(&openssl_init_mutex));
   return status;
+#else
+  return AMQP_STATUS_SSL_UNIMPLEMENTED;
+#endif
 }
 
 static int initialize_ssl_and_increment_connections() {