Skip to content

Commit

Permalink
Add OpenSSL 3.x support to openssl-dynamic (netty#867)
Browse files Browse the repository at this point in the history 
### Motivation:
The `openssl-dynamic` module only compiles against OpenSSL 1.x.

### Modifications:
The most significant change is that one no longer needs to provide DH
parameters manually. By calling `SSL_CTX_set_dh_auto()`, OpenSSL will
use the built-in parameters that match the key pair size. E.g. RSA 2048
will use DH 2048 and so on.

The property `jdk.tls.ephemeralDHKeySize`, which was used to affect the
size of the DH parameters before, will be ignored when using the
dynamically linked OpenSSL. Instead, the system OpenSSL library's config
will be used.

Another change is how FIPS is managed because OpenSSL 3 introduces a
"Provider" concept. The `FIPS_mode_set` is removed and is now replaced
with `EVP_default_properties_enable_fips` which instructs OpenSSL to
load the FIPS compliant provider, if no provider is explicitly
requested.


### Result:
The `openssl-dynamic` module can now successfully compile and run with
OpenSSL 3.x.
  • Loading branch information
@klaren @jennyowen
klaren authored and jennyowen committed Jul 30, 2024
1 parent 593db1e commit 182239a
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 13 deletions.
26 changes: 18 additions & 8 deletions openssl-dynamic/src/main/c/ssl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ static UI_METHOD *ui_method = NULL;
#endif // OPENSSL_NO_ENGINE


#if defined(OPENSSL_FIPS) && (OPENSSL_VERSION_NUMBER < 0x30000000L)
#define tcn_enable_fips(to) FIPS_mode_set((to))
#else
#define tcn_enable_fips(to) EVP_default_properties_enable_fips(NULL, (to))
#endif

#if OPENSSL_VERSION_NUMBER < 0x10100000L || (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x2090000fL)

/* Global reference to the pool used by the dynamic mutexes */
Expand Down Expand Up @@ -470,10 +476,12 @@ static BIO_METHOD* BIO_java_bytebuffer() {
#endif
}

#if OPENSSL_VERSION_NUMBER < 0x30000000L
static int ssl_tmp_key_init_dh(int bits, int idx)
{
return (SSL_temp_keys[idx] = tcn_SSL_dh_get_tmp_param(bits)) ? 0 : 1;
}
#endif

TCN_IMPLEMENT_CALL(jint, SSL, version)(TCN_STDARGS)
{
Expand Down Expand Up @@ -520,9 +528,9 @@ static apr_status_t ssl_init_cleanup(void *data)
free_bio_methods();
#endif

// Reset fips mode to the default.
#ifdef OPENSSL_FIPS
FIPS_mode_set(0);
#if defined(OPENSSL_FIPS) && (OPENSSL_VERSION_NUMBER < 0x30000000L)
// Reset fips mode to the default.
tcn_enable_fips(0);
#endif

#ifndef OPENSSL_NO_ENGINE
Expand Down Expand Up @@ -850,7 +858,9 @@ TCN_IMPLEMENT_CALL(jint, SSL, initialize)(TCN_STDARGS, jstring engine)
init_bio_methods();
#endif

#if OPENSSL_VERSION_NUMBER < 0x30000000L
SSL_TMP_KEYS_INIT(r);
#endif
if (r) {
// TODO: Should we really do this as the user may want to inspect the error stack ?
ERR_clear_error();
Expand Down Expand Up @@ -957,7 +967,7 @@ TCN_IMPLEMENT_CALL(jlong /* SSL * */, SSL, newSSL)(TCN_STDARGS,
tcn_ThrowException(e, "cannot create new ssl state struct");
return 0;
}

// Set the app_data2 before all the others because it may be used in SSL_free.
tcn_SSL_set_app_state(ssl, state);

Expand Down Expand Up @@ -1325,7 +1335,7 @@ TCN_IMPLEMENT_CALL(jobjectArray, SSL, getPeerCertChain)(TCN_STDARGS,
// Out of memory
return NULL;
}

for(i = 0; i < len; i++) {

#ifdef OPENSSL_IS_BORINGSSL
Expand Down Expand Up @@ -1848,7 +1858,7 @@ TCN_IMPLEMENT_CALL(jbyteArray, SSL, getSessionId)(TCN_STDARGS, jlong ssl)
return NULL;
}


if ((bArray = (*e)->NewByteArray(e, len)) == NULL) {
return NULL;
}
Expand Down Expand Up @@ -2490,8 +2500,8 @@ TCN_IMPLEMENT_CALL(jbyteArray, SSL, getOcspResponse)(TCN_STDARGS, jlong ssl) {

TCN_IMPLEMENT_CALL(void, SSL, fipsModeSet)(TCN_STDARGS, jint mode)
{
#ifdef OPENSSL_FIPS
if (FIPS_mode_set((int) mode) == 0) {
#if defined(OPENSSL_FIPS) || (OPENSSL_VERSION_NUMBER >= 0x30000000L)
if (tcn_enable_fips((int) mode) == 0) {
char err[ERR_LEN];
ERR_error_string_n(ERR_get_error(), err, ERR_LEN);
ERR_clear_error();
Expand Down
11 changes: 10 additions & 1 deletion openssl-dynamic/src/main/c/ssl_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@
#include <openssl/x509v3.h>
#include <openssl/hmac.h>
#include <openssl/dh.h>
#if OPENSSL_VERSION_NUMBER >= 0x30000000
#include <openssl/core_names.h>
#endif

#define ERR_LEN 256

Expand Down Expand Up @@ -290,7 +293,11 @@ typedef struct tcn_ssl_ctxt_t tcn_ssl_ctxt_t;

typedef struct {
unsigned char key_name[SSL_SESSION_TICKET_KEY_NAME_LEN];
unsigned char hmac_key[SSL_SESSION_TICKET_HMAC_KEY_LEN];
#if OPENSSL_VERSION_NUMBER < 0x30000000L
unsigned char hmac_key[SSL_SESSION_TICKET_HMAC_KEY_LEN];
#else
OSSL_PARAM mac_params[3];
#endif
unsigned char aes_key[SSL_SESSION_TICKET_AES_KEY_LEN];
} tcn_ssl_ticket_key_t;

Expand Down Expand Up @@ -422,6 +429,7 @@ void *tcn_SSL_CTX_get_app_state(const SSL_CTX *);
void tcn_SSL_CTX_set_app_state(SSL_CTX *, void *);

int tcn_SSL_password_callback(char *, int, int, void *);
#if OPENSSL_VERSION_NUMBER < 0x30000000
DH *tcn_SSL_dh_get_tmp_param(int);
DH *tcn_SSL_callback_tmp_DH(SSL *, int, int);
// The following provided callbacks will always return DH of a given length.
Expand All @@ -430,6 +438,7 @@ DH *tcn_SSL_callback_tmp_DH_512(SSL *, int, int);
DH *tcn_SSL_callback_tmp_DH_1024(SSL *, int, int);
DH *tcn_SSL_callback_tmp_DH_2048(SSL *, int, int);
DH *tcn_SSL_callback_tmp_DH_4096(SSL *, int, int);
#endif
int tcn_SSL_CTX_use_certificate_chain(SSL_CTX *, const char *, bool);
int tcn_SSL_CTX_use_certificate_chain_bio(SSL_CTX *, BIO *, bool);
int tcn_SSL_CTX_use_client_CA_bio(SSL_CTX *, BIO *);
Expand Down
39 changes: 36 additions & 3 deletions openssl-dynamic/src/main/c/sslcontext.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,11 @@ TCN_IMPLEMENT_CALL(jlong, SSLContext, make)(TCN_STDARGS, jint protocol, jint mod
EC_KEY_free(ecdh);
#endif

#if OPENSSL_VERSION_NUMBER < 0x30000000L
SSL_CTX_set_tmp_dh_callback(c->ctx, tcn_SSL_callback_tmp_DH);
#else
SSL_CTX_set_dh_auto(c->ctx, 1);
#endif
}

// Default depth is 100 and disabled according to https://www.openssl.org/docs/man1.0.2/ssl/SSL_set_verify.html.
Expand Down Expand Up @@ -647,6 +651,7 @@ TCN_IMPLEMENT_CALL(jboolean, SSLContext, setNumTickets)(TCN_STDARGS, jlong ctx,

TCN_IMPLEMENT_CALL(void, SSLContext, setTmpDHLength)(TCN_STDARGS, jlong ctx, jint length)
{
#if OPENSSL_VERSION_NUMBER < 0x30000000L
tcn_ssl_ctxt_t *c = J2P(ctx, tcn_ssl_ctxt_t *);

TCN_CHECK_NULL(c, ctx, /* void */);
Expand All @@ -668,6 +673,7 @@ TCN_IMPLEMENT_CALL(void, SSLContext, setTmpDHLength)(TCN_STDARGS, jlong ctx, jin
tcn_Throw(e, "Unsupported length %s", length);
return;
}
#endif // OPENSSL_VERSION_NUMBER < 0x30000000L
}

#ifndef OPENSSL_IS_BORINGSSL
Expand Down Expand Up @@ -1271,7 +1277,16 @@ static int find_session_key(tcn_ssl_ctxt_t *c, unsigned char key_name[16], tcn_s
return result;
}

static int ssl_tlsext_ticket_key_cb(SSL *s, unsigned char key_name[16], unsigned char *iv, EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) {
static int ssl_tlsext_ticket_key_cb(SSL *s,
unsigned char key_name[16],
unsigned char *iv,
EVP_CIPHER_CTX *ctx,
#if OPENSSL_VERSION_NUMBER < 0x30000000L
HMAC_CTX *hmac_ctx,
#else
EVP_MAC_CTX *mac_ctx,
#endif
int enc) {
tcn_ssl_ctxt_t *c = NULL;
tcn_ssl_ticket_key_t key;
int is_current_key;
Expand All @@ -1289,15 +1304,23 @@ static int ssl_tlsext_ticket_key_cb(SSL *s, unsigned char key_name[16], unsigned
memcpy(key_name, key.key_name, 16);

EVP_EncryptInit_ex(ctx, EVP_aes_128_cbc(), NULL, key.aes_key, iv);
HMAC_Init_ex(hctx, key.hmac_key, 16, EVP_sha256(), NULL);
#if OPENSSL_VERSION_NUMBER < 0x30000000L
HMAC_Init_ex(hmac_ctx, key.hmac_key, 16, EVP_sha256(), NULL);
#else
EVP_MAC_CTX_set_params(mac_ctx, key.mac_params);
#endif
apr_atomic_inc32(&c->ticket_keys_new);
return 1;
}
// No ticket configured
return 0;
} else { /* retrieve session */
if (find_session_key(c, key_name, &key, &is_current_key)) {
HMAC_Init_ex(hctx, key.hmac_key, 16, EVP_sha256(), NULL);
#if OPENSSL_VERSION_NUMBER < 0x30000000L
HMAC_Init_ex(hmac_ctx, key.hmac_key, 16, EVP_sha256(), NULL);
#else
EVP_MAC_CTX_set_params(mac_ctx, key.mac_params);
#endif
EVP_DecryptInit_ex(ctx, EVP_aes_128_cbc(), NULL, key.aes_key, iv );
if (!is_current_key) {
// The ticket matched a key in the list, and we want to upgrade it to the current
Expand Down Expand Up @@ -1337,7 +1360,13 @@ TCN_IMPLEMENT_CALL(void, SSLContext, setSessionTicketKeys0)(TCN_STDARGS, jlong c
for (i = 0; i < cnt; ++i) {
key = b + (SSL_SESSION_TICKET_KEY_SIZE * i);
memcpy(ticket_keys[i].key_name, key, 16);
#if OPENSSL_VERSION_NUMBER < 0x30000000L
memcpy(ticket_keys[i].hmac_key, key + 16, 16);
#else
ticket_keys[i].mac_params[0] = OSSL_PARAM_construct_octet_string(OSSL_MAC_PARAM_KEY, key + 16, 16);
ticket_keys[i].mac_params[1] = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, "sha256", 0);
ticket_keys[i].mac_params[2] = OSSL_PARAM_construct_end();
#endif
memcpy(ticket_keys[i].aes_key, key + 32, 16);
}
(*e)->ReleaseByteArrayElements(e, keys, b, 0);
Expand All @@ -1350,7 +1379,11 @@ TCN_IMPLEMENT_CALL(void, SSLContext, setSessionTicketKeys0)(TCN_STDARGS, jlong c
c->ticket_keys = ticket_keys;
apr_thread_rwlock_unlock(c->mutex);

#if OPENSSL_VERSION_NUMBER < 0x30000000L
SSL_CTX_set_tlsext_ticket_key_cb(c->ctx, ssl_tlsext_ticket_key_cb);
#else
SSL_CTX_set_tlsext_ticket_key_evp_cb(c->ctx, ssl_tlsext_ticket_key_cb);
#endif
}

static const char* authentication_method(const SSL* ssl) {
Expand Down
2 changes: 2 additions & 0 deletions openssl-dynamic/src/main/c/sslutils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ int tcn_SSL_password_callback(char *buf, int bufsiz, int verify,
return (int)strlen(buf);
}

#if OPENSSL_VERSION_NUMBER < 0x30000000L
static unsigned char dh0512_p[]={
0xD9,0xBA,0xBF,0xFD,0x69,0x38,0xC9,0x51,0x2D,0x19,0x37,0x39,
0xD7,0x7D,0x7E,0x3E,0x25,0x58,0x55,0x94,0x90,0x60,0x93,0x7A,
Expand Down Expand Up @@ -434,6 +435,7 @@ DH *tcn_SSL_callback_tmp_DH_4096(SSL *ssl, int export, int keylen)
{
return (DH *)SSL_temp_keys[SSL_TMP_KEY_DH_4096];
}
#endif // OPENSSL_VERSION_NUMBER < 0x30000000L

/*
* Read a file that optionally contains the server certificate in PEM
Expand Down
2 changes: 1 addition & 1 deletion openssl-dynamic/src/main/native-package/m4/tcnative.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ int main() {
OPENSSL_VERSION_NUMBER < 0x00908000L) ||
OPENSSL_VERSION_NUMBER >= 0x0090801fL)
return (0);
printf("\n\nFound OPENSSL_VERSION_NUMBER %#010x\n",
printf("\n\nFound OPENSSL_VERSION_NUMBER %#010lx\n",
OPENSSL_VERSION_NUMBER);
printf("Require OPENSSL_VERSION_NUMBER 0x0090701f or greater (0.9.7a)\n"
"Require OPENSSL_VERSION_NUMBER 0x0090801f or greater (0.9.8a)\n\n");
Expand Down

0 comments on commit 182239a

Please sign in to comment.