rsa: Fix build with OpenSSL 1.1.x

The rsa_st struct has been made opaque in 1.1.x, add forward compatible
code to access the n, e, d members of rsa_struct.

EVP_MD_CTX_cleanup has been removed in 1.1.x and EVP_MD_CTX_reset should be
called to reinitialise an already created structure.

diff --git a/lib/rsa/rsa-sign.c b/lib/rsa/rsa-sign.c
index 8c6637e3287e5d3fd069c3523bdd7b196c11544c..1da4ef7fffbdd04acfce068664df87267c473cec 100644 (file)
--- a/lib/rsa/rsa-sign.c
+++ b/lib/rsa/rsa-sign.c
@@ -9,6 +9,7 @@
 #include <string.h>
 #include <image.h>
 #include <time.h>
+#include <openssl/bn.h>
 #include <openssl/rsa.h>
 #include <openssl/pem.h>
 #include <openssl/err.h>
@@ -20,6 +21,19 @@
 #define HAVE_ERR_REMOVE_THREAD_STATE
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+static void RSA_get0_key(const RSA *r,
+                 const BIGNUM **n, const BIGNUM **e, const BIGNUM **d)
+{
+   if (n != NULL)
+       *n = r->n;
+   if (e != NULL)
+       *e = r->e;
+   if (d != NULL)
+       *d = r->d;
+}
+#endif
+
 static int rsa_err(const char *msg)
 {
        unsigned long sslErr = ERR_get_error();
@@ -286,16 +300,22 @@ static int rsa_init(void)
 {
        int ret;
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        ret = SSL_library_init();
+#else
+       ret = OPENSSL_init_ssl(0, NULL);
+#endif
        if (!ret) {
                fprintf(stderr, "Failure to init SSL library\n");
                return -1;
        }
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        SSL_load_error_strings();
 
        OpenSSL_add_all_algorithms();
        OpenSSL_add_all_digests();
        OpenSSL_add_all_ciphers();
+#endif
 
        return 0;
 }
@@ -335,12 +355,15 @@ err_set_rsa:
 err_engine_init:
        ENGINE_free(e);
 err_engine_by_id:
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        ENGINE_cleanup();
+#endif
        return ret;
 }
 
 static void rsa_remove(void)
 {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        CRYPTO_cleanup_all_ex_data();
        ERR_free_strings();
 #ifdef HAVE_ERR_REMOVE_THREAD_STATE
@@ -349,6 +372,7 @@ static void rsa_remove(void)
        ERR_remove_state(0);
 #endif
        EVP_cleanup();
+#endif
 }
 
 static void rsa_engine_remove(ENGINE *e)
@@ -409,7 +433,11 @@ static int rsa_sign_with_key(RSA *rsa, struct checksum_algo *checksum_algo,
                ret = rsa_err("Could not obtain signature");
                goto err_sign;
        }
-       EVP_MD_CTX_cleanup(context);
+       #if OPENSSL_VERSION_NUMBER < 0x10100000L
+               EVP_MD_CTX_cleanup(context);
+       #else
+               EVP_MD_CTX_reset(context);
+       #endif
        EVP_MD_CTX_destroy(context);
        EVP_PKEY_free(key);
 
@@ -479,6 +507,7 @@ static int rsa_get_exponent(RSA *key, uint64_t *e)
 {
        int ret;
        BIGNUM *bn_te;
+       const BIGNUM *key_e;
        uint64_t te;
 
        ret = -EINVAL;
@@ -487,17 +516,18 @@ static int rsa_get_exponent(RSA *key, uint64_t *e)
        if (!e)
                goto cleanup;
 
-       if (BN_num_bits(key->e) > 64)
+       RSA_get0_key(key, NULL, &key_e, NULL);
+       if (BN_num_bits(key_e) > 64)
                goto cleanup;
 
-       *e = BN_get_word(key->e);
+       *e = BN_get_word(key_e);
 
-       if (BN_num_bits(key->e) < 33) {
+       if (BN_num_bits(key_e) < 33) {
                ret = 0;
                goto cleanup;
        }
 
-       bn_te = BN_dup(key->e);
+       bn_te = BN_dup(key_e);
        if (!bn_te)
                goto cleanup;
 
@@ -527,6 +557,7 @@ int rsa_get_params(RSA *key, uint64_t *exponent, uint32_t *n0_invp,
 {
        BIGNUM *big1, *big2, *big32, *big2_32;
        BIGNUM *n, *r, *r_squared, *tmp;
+       const BIGNUM *key_n;
        BN_CTX *bn_ctx = BN_CTX_new();
        int ret = 0;
 
@@ -548,7 +579,8 @@ int rsa_get_params(RSA *key, uint64_t *exponent, uint32_t *n0_invp,
        if (0 != rsa_get_exponent(key, exponent))
                ret = -1;
 
-       if (!BN_copy(n, key->n) || !BN_set_word(big1, 1L) ||
+       RSA_get0_key(key, &key_n, NULL, NULL);
+       if (!BN_copy(n, key_n) || !BN_set_word(big1, 1L) ||
            !BN_set_word(big2, 2L) || !BN_set_word(big32, 32L))
                ret = -1;