Add input check to rsa_decrypt family of functions.

ChangeLog

+2021-05-17  Niels Möller  <nisse@lysator.liu.se>
+
+	* rsa-decrypt-tr.c (rsa_decrypt_tr): Check up-front that input is
+	in range.
+	* rsa-sec-decrypt.c (rsa_sec_decrypt): Likewise.
+	* rsa-decrypt.c (rsa_decrypt): Likewise.
+	* testsuite/rsa-encrypt-test.c (test_main): Add tests with input > n.
+
 2021-05-14  Niels Möller  <nisse@lysator.liu.se>
 
 	* rsa-sign-tr.c (rsa_sec_blind): Delete mn argument.

rsa-decrypt-tr.c

@@ -52,6 +52,10 @@ rsa_decrypt_tr(const struct rsa_public_key *pub,
   mp_size_t key_limb_size;
   int res;
 
+  /* First check that input is in range. */
+  if (mpz_sgn (gibberish) < 0 || mpz_cmp (gibberish, pub->n) >= 0)
+    return 0;
+
   key_limb_size = mpz_size(pub->n);
 
   TMP_GMP_ALLOC (m, key_limb_size);

rsa-decrypt.c

@@ -48,6 +48,16 @@ rsa_decrypt(const struct rsa_private_key *key,
   int res;
 
   mpz_init(m);
+
+  /* First check that input is in range. Since we don't have the
+     public key available here, we need to reconstruct n. */
+  mpz_mul (m, key->p, key->q);
+  if (mpz_sgn (gibberish) < 0 || mpz_cmp (gibberish, m) >= 0)
+    {
+      mpz_clear (m);
+      return 0;
+    }
+
   rsa_compute_root(key, m, gibberish);
 
   res = pkcs1_decrypt (key->size, m, length, message);

rsa-sec-decrypt.c

@@ -55,6 +55,10 @@ rsa_sec_decrypt(const struct rsa_public_key *pub,
   TMP_GMP_DECL (em, uint8_t);
   int res;
 
+  /* First check that input is in range. */
+  if (mpz_sgn (gibberish) < 0 || mpz_cmp (gibberish, pub->n) >= 0)
+    return 0;
+
   TMP_GMP_ALLOC (m, mpz_size(pub->n));
   TMP_GMP_ALLOC (em, key->size);
 

rsa.h

@@ -428,13 +428,14 @@ rsa_sec_decrypt(const struct rsa_public_key *pub,
 	        size_t length, uint8_t *message,
 	        const mpz_t gibberish);
 
-/* Compute x, the e:th root of m. Calling it with x == m is allowed. */
+/* Compute x, the e:th root of m. Calling it with x == m is allowed.
+   It is required that 0 <= m < n. */
 void
 rsa_compute_root(const struct rsa_private_key *key,
 		 mpz_t x, const mpz_t m);
 
 /* Safer variant, using RSA blinding, and checking the result after
-   CRT. */
+   CRT. It is required that 0 <= m < n. */
 int
 rsa_compute_root_tr(const struct rsa_public_key *pub,
 		    const struct rsa_private_key *key,

testsuite/rsa-encrypt-test.c

@@ -19,11 +19,12 @@ test_main(void)
   uint8_t after;
 
   mpz_t gibberish;
-  mpz_t zero;
+  mpz_t bad_input;
 
   rsa_private_key_init(&key);
   rsa_public_key_init(&pub);
   mpz_init(gibberish);
+  mpz_init(bad_input);
 
   knuth_lfib_init(&lfib, 17);
   
@@ -103,15 +104,40 @@ test_main(void)
   ASSERT(decrypted[0] == 'A');
 
   /* Test zero input. */
-  mpz_init_set_ui (zero, 0);
+  mpz_set_ui (bad_input, 0);
   decrypted_length = msg_length;
-  ASSERT(!rsa_decrypt(&key, &decrypted_length, decrypted, zero));
+  ASSERT(!rsa_decrypt(&key, &decrypted_length, decrypted, bad_input));
   ASSERT(!rsa_decrypt_tr(&pub, &key,
 			 &lfib, (nettle_random_func *) knuth_lfib_random,
-			 &decrypted_length, decrypted, zero));
+			 &decrypted_length, decrypted, bad_input));
   ASSERT(!rsa_sec_decrypt(&pub, &key,
 			  &lfib, (nettle_random_func *) knuth_lfib_random,
-			  decrypted_length, decrypted, zero));
+			  decrypted_length, decrypted, bad_input));
+  ASSERT(decrypted_length == msg_length);
+
+  /* Test input that is slightly larger than n */
+  mpz_add(bad_input, gibberish, pub.n);
+  decrypted_length = msg_length;
+  ASSERT(!rsa_decrypt(&key, &decrypted_length, decrypted, bad_input));
+  ASSERT(!rsa_decrypt_tr(&pub, &key,
+			 &lfib, (nettle_random_func *) knuth_lfib_random,
+			 &decrypted_length, decrypted, bad_input));
+  ASSERT(!rsa_sec_decrypt(&pub, &key,
+			  &lfib, (nettle_random_func *) knuth_lfib_random,
+			  decrypted_length, decrypted, bad_input));
+  ASSERT(decrypted_length == msg_length);
+
+  /* Test input that is considerably larger than n */
+  mpz_mul_2exp (bad_input, pub.n, 100);
+  mpz_add (bad_input, bad_input, gibberish);
+  decrypted_length = msg_length;
+  ASSERT(!rsa_decrypt(&key, &decrypted_length, decrypted, bad_input));
+  ASSERT(!rsa_decrypt_tr(&pub, &key,
+			 &lfib, (nettle_random_func *) knuth_lfib_random,
+			 &decrypted_length, decrypted, bad_input));
+  ASSERT(!rsa_sec_decrypt(&pub, &key,
+			  &lfib, (nettle_random_func *) knuth_lfib_random,
+			  decrypted_length, decrypted, bad_input));
   ASSERT(decrypted_length == msg_length);
 
   /* Test invalid key. */
@@ -124,6 +150,6 @@ test_main(void)
   rsa_private_key_clear(&key);
   rsa_public_key_clear(&pub);
   mpz_clear(gibberish);
-  mpz_clear(zero);
+  mpz_clear(bad_input);
   free(decrypted);
 }