rsa: Fix two errors in the implementation

1. Failure to set the return code correctly
2. Failure to detect the loop end condition when the value is equal to
the modulus.

Reported-by: Jeroen Hofstee <jeroen@myspectrum.nl>
Signed-off-by: Simon Glass <sjg@chromium.org>

lib/rsa/rsa-sign.c

@@ -76,6 +76,7 @@ static int rsa_get_pub_key(const char *keydir, const char *name, RSA **rsap)
 	rsa = EVP_PKEY_get1_RSA(key);
 	if (!rsa) {
 		rsa_err("Couldn't convert to a RSA style key");
+		ret = -EINVAL;
 		goto err_rsa;
 	}
 	fclose(f);

lib/rsa/rsa-verify.c

@@ -57,9 +57,9 @@ static void subtract_modulus(const struct rsa_public_key *key, uint32_t num[])
 static int greater_equal_modulus(const struct rsa_public_key *key,
 				 uint32_t num[])
 {
-	uint32_t i;
+	int i;
 
-	for (i = key->len - 1; i >= 0; i--) {
+	for (i = (int)key->len - 1; i >= 0; i--) {
 		if (num[i] < key->modulus[i])
 			return 0;
 		if (num[i] > key->modulus[i])