rsa: Fix two errors in the implementation

1. Failure to set the return code correctly
2. Failure to detect the loop end condition when the value is equal to
the modulus.

Reported-by: Jeroen Hofstee <jeroen@myspectrum.nl>
Signed-off-by: Simon Glass <sjg@chromium.org>

diff --git a/lib/rsa/rsa-sign.c b/lib/rsa/rsa-sign.c
index f4d433867a2d28ebcf2827c5ead4a8c8c92d6cc2..5d9716f01349f4074653f2735fa3e6204ddba8e7 100644 (file)
--- a/lib/rsa/rsa-sign.c
+++ b/lib/rsa/rsa-sign.c
@@ -76,6 +76,7 @@ static int rsa_get_pub_key(const char *keydir, const char *name, RSA **rsap)
        rsa = EVP_PKEY_get1_RSA(key);
        if (!rsa) {
                rsa_err("Couldn't convert to a RSA style key");
+               ret = -EINVAL;
                goto err_rsa;
        }
        fclose(f);

diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index c5bcdb60e814ee2ad01c6fb5fe548f761397d86f..4ef19b66f4b12588f2bbe0a978500a0718177336 100644 (file)
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -57,9 +57,9 @@ static void subtract_modulus(const struct rsa_public_key *key, uint32_t num[])
 static int greater_equal_modulus(const struct rsa_public_key *key,
                                 uint32_t num[])
 {
-       uint32_t i;
+       int i;
 
-       for (i = key->len - 1; i >= 0; i--) {
+       for (i = (int)key->len - 1; i >= 0; i--) {
                if (num[i] < key->modulus[i])
                        return 0;
                if (num[i] > key->modulus[i])