Changes to pkcs1_signature_prefix interface.

ChangeLog

+2012-06-03  Niels Möller  <nisse@lysator.liu.se>
+
+	* testsuite/pkcs1-test.c (test_main): Include leading zero in
+	expected result.
+
+	* pkcs1.c (pkcs1_signature_prefix): Return pointer to where the
+	digest should be written. Let the size input be the key size in
+	octets, rather then key size - 1.
+	* pkcs1-rsa-*.c: Updated for above.
+	* rsa-*-sign.c, rsa-*-verify.c: Pass key->size, not key->size - 1.
+
 2012-05-18  Niels Möller  <nisse@lysator.liu.se>
 
 	* pkcs1-encrypt.c (pkcs1_encrypt): New file and function.

pkcs1-rsa-md5.c

@@ -62,18 +62,20 @@ md5_prefix[] =
 };
 
 int
-pkcs1_rsa_md5_encode(mpz_t m, unsigned size, struct md5_ctx *hash)
+pkcs1_rsa_md5_encode(mpz_t m, unsigned key_size, struct md5_ctx *hash)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(md5_prefix),
 			     md5_prefix,
-			     MD5_DIGEST_SIZE))
+			     MD5_DIGEST_SIZE);
+  if (p)
     {
-      md5_digest(hash, MD5_DIGEST_SIZE, em + size - MD5_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      md5_digest(hash, MD5_DIGEST_SIZE, p);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else
@@ -81,18 +83,20 @@ pkcs1_rsa_md5_encode(mpz_t m, unsigned size, struct md5_ctx *hash)
 }
 
 int
-pkcs1_rsa_md5_encode_digest(mpz_t m, unsigned size, const uint8_t *digest)
+pkcs1_rsa_md5_encode_digest(mpz_t m, unsigned key_size, const uint8_t *digest)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(md5_prefix),
 			     md5_prefix,
-			     MD5_DIGEST_SIZE))
+			     MD5_DIGEST_SIZE);
+  if (p)
     {
-      memcpy(em + size - MD5_DIGEST_SIZE, digest, MD5_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      memcpy(p, digest, MD5_DIGEST_SIZE);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else

pkcs1-rsa-sha1.c

@@ -62,18 +62,20 @@ sha1_prefix[] =
 };
 
 int
-pkcs1_rsa_sha1_encode(mpz_t m, unsigned size, struct sha1_ctx *hash)
+pkcs1_rsa_sha1_encode(mpz_t m, unsigned key_size, struct sha1_ctx *hash)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha1_prefix),
 			     sha1_prefix,
-			     SHA1_DIGEST_SIZE))
+			     SHA1_DIGEST_SIZE);
+  if (p)
     {
-      sha1_digest(hash, SHA1_DIGEST_SIZE, em + size - SHA1_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      sha1_digest(hash, SHA1_DIGEST_SIZE, p);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else
@@ -81,18 +83,20 @@ pkcs1_rsa_sha1_encode(mpz_t m, unsigned size, struct sha1_ctx *hash)
 }
 
 int
-pkcs1_rsa_sha1_encode_digest(mpz_t m, unsigned size, const uint8_t *digest)
+pkcs1_rsa_sha1_encode_digest(mpz_t m, unsigned key_size, const uint8_t *digest)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha1_prefix),
 			     sha1_prefix,
-			     SHA1_DIGEST_SIZE))
+			     SHA1_DIGEST_SIZE);
+  if (p)
     {
-      memcpy(em + size - SHA1_DIGEST_SIZE, digest, SHA1_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      memcpy(p, digest, SHA1_DIGEST_SIZE);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else

pkcs1-rsa-sha256.c

@@ -60,18 +60,20 @@ sha256_prefix[] =
 };
 
 int
-pkcs1_rsa_sha256_encode(mpz_t m, unsigned size, struct sha256_ctx *hash)
+pkcs1_rsa_sha256_encode(mpz_t m, unsigned key_size, struct sha256_ctx *hash)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha256_prefix),
 			     sha256_prefix,
-			     SHA256_DIGEST_SIZE))
+			     SHA256_DIGEST_SIZE);
+  if (p)
     {
-      sha256_digest(hash, SHA256_DIGEST_SIZE, em + size - SHA256_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      sha256_digest(hash, SHA256_DIGEST_SIZE, p);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else
@@ -79,18 +81,20 @@ pkcs1_rsa_sha256_encode(mpz_t m, unsigned size, struct sha256_ctx *hash)
 }
 
 int
-pkcs1_rsa_sha256_encode_digest(mpz_t m, unsigned size, const uint8_t *digest)
+pkcs1_rsa_sha256_encode_digest(mpz_t m, unsigned key_size, const uint8_t *digest)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha256_prefix),
 			     sha256_prefix,
-			     SHA256_DIGEST_SIZE))
+			     SHA256_DIGEST_SIZE);
+  if (p)
     {
-      memcpy(em + size - SHA256_DIGEST_SIZE, digest, SHA256_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      memcpy(p, digest, SHA256_DIGEST_SIZE);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else

pkcs1-rsa-sha512.c

@@ -60,19 +60,20 @@ sha512_prefix[] =
 };
 
 int
-pkcs1_rsa_sha512_encode(mpz_t m, unsigned size, struct sha512_ctx *hash)
+pkcs1_rsa_sha512_encode(mpz_t m, unsigned key_size, struct sha512_ctx *hash)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha512_prefix),
 			     sha512_prefix,
-			     SHA512_DIGEST_SIZE))
+			     SHA512_DIGEST_SIZE);
+  if (p)
     {
-      sha512_digest(hash, SHA512_DIGEST_SIZE,
-		    em + size - SHA512_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      sha512_digest(hash, SHA512_DIGEST_SIZE, p);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else
@@ -80,18 +81,20 @@ pkcs1_rsa_sha512_encode(mpz_t m, unsigned size, struct sha512_ctx *hash)
 }
 
 int
-pkcs1_rsa_sha512_encode_digest(mpz_t m, unsigned size, const uint8_t *digest)
+pkcs1_rsa_sha512_encode_digest(mpz_t m, unsigned key_size, const uint8_t *digest)
 {
+  uint8_t *p;
   TMP_DECL(em, uint8_t, NETTLE_MAX_BIGNUM_SIZE);
-  TMP_ALLOC(em, size);
+  TMP_ALLOC(em, key_size);
 
-  if (pkcs1_signature_prefix(size, em,
+  p = pkcs1_signature_prefix(key_size, em,
 			     sizeof(sha512_prefix),
 			     sha512_prefix,
-			     SHA512_DIGEST_SIZE))
+			     SHA512_DIGEST_SIZE);
+  if (p)
     {
-      memcpy(em + size - SHA512_DIGEST_SIZE, digest, SHA512_DIGEST_SIZE);
-      nettle_mpz_set_str_256_u(m, size, em);
+      memcpy(p, digest, SHA512_DIGEST_SIZE);
+      nettle_mpz_set_str_256_u(m, key_size, em);
       return 1;
     }
   else

pkcs1.c

@@ -34,13 +34,13 @@
 
 /* Formats the PKCS#1 padding, of the form
  *
- *   0x01 0xff ... 0xff 0x00 id ...digest...
+ *   0x00 0x01 0xff ... 0xff 0x00 id ...digest...
  *
  * where the 0xff ... 0xff part consists of at least 8 octets. The 
- * total size should be one less than the octet size of n.
+ * total size equals the octet size of n.
  */
-int
-pkcs1_signature_prefix(unsigned size,
+uint8_t *
+pkcs1_signature_prefix(unsigned key_size,
 		       uint8_t *buffer,
 		       unsigned id_size,
 		       const uint8_t *id,
@@ -48,17 +48,18 @@ pkcs1_signature_prefix(unsigned size,
 {
   unsigned j;
   
-  if (size < 10 + id_size + digest_size)
-    return 0;
+  if (key_size < 11 + id_size + digest_size)
+    return NULL;
 
-  j = size - digest_size - id_size;
+  j = key_size - digest_size - id_size;
 
   memcpy (buffer + j, id, id_size);
-  buffer[0] = 1;
-  buffer[--j] = 0;
+  buffer[0] = 0;
+  buffer[1] = 1;
+  buffer[j-1] = 0;
 
-  assert(j >= 9);
-  memset(buffer + 1, 0xff, j - 1);
+  assert(j >= 11);
+  memset(buffer + 2, 0xff, j - 3);
 
-  return 1;
+  return buffer + j + id_size;
 }

pkcs1.h

@@ -51,8 +51,8 @@ struct sha1_ctx;
 struct sha256_ctx;
 struct sha512_ctx;
 
-int
-pkcs1_signature_prefix(unsigned size,
+uint8_t *
+pkcs1_signature_prefix(unsigned key_size,
 		       uint8_t *buffer,
 		       unsigned id_size,
 		       const uint8_t *id,

rsa-md5-sign.c

@@ -39,9 +39,7 @@ rsa_md5_sign(const struct rsa_private_key *key,
              struct md5_ctx *hash,
              mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_md5_encode(s, key->size - 1, hash))
+  if (pkcs1_rsa_md5_encode(s, key->size, hash))
     {
       rsa_compute_root(key, s, s);
       return 1;
@@ -58,9 +56,7 @@ rsa_md5_sign_digest(const struct rsa_private_key *key,
 		    const uint8_t *digest,
 		    mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_md5_encode_digest(s, key->size - 1, digest))
+  if (pkcs1_rsa_md5_encode_digest(s, key->size, digest))
     {
       rsa_compute_root(key, s, s);
       return 1;

rsa-md5-verify.c

@@ -42,10 +42,9 @@ rsa_md5_verify(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
 
-  res = (pkcs1_rsa_md5_encode(m, key->size - 1, hash)
+  res = (pkcs1_rsa_md5_encode(m, key->size, hash)
 	 && _rsa_verify(key, m, s));
 
   mpz_clear(m);
@@ -61,10 +60,9 @@ rsa_md5_verify_digest(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_md5_encode_digest(m, key->size - 1, digest)
+  res = (pkcs1_rsa_md5_encode_digest(m, key->size, digest)
 	 && _rsa_verify(key, m, s));
 
   mpz_clear(m);

rsa-sha1-sign.c

@@ -39,9 +39,7 @@ rsa_sha1_sign(const struct rsa_private_key *key,
               struct sha1_ctx *hash,
               mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha1_encode(s, key->size - 1, hash))
+  if (pkcs1_rsa_sha1_encode(s, key->size, hash))
     {
       rsa_compute_root(key, s, s);
       return 1;
@@ -58,9 +56,7 @@ rsa_sha1_sign_digest(const struct rsa_private_key *key,
 		     const uint8_t *digest,
 		     mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha1_encode_digest(s, key->size - 1, digest))
+  if (pkcs1_rsa_sha1_encode_digest(s, key->size, digest))
     {
       rsa_compute_root(key, s, s);
       return 1;

rsa-sha1-verify.c

@@ -42,10 +42,9 @@ rsa_sha1_verify(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_sha1_encode(m, key->size - 1, hash)
+  res = (pkcs1_rsa_sha1_encode(m, key->size, hash)
 	 && _rsa_verify(key, m, s));
   
   mpz_clear(m);
@@ -61,10 +60,9 @@ rsa_sha1_verify_digest(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_sha1_encode_digest(m, key->size - 1, digest)
+  res = (pkcs1_rsa_sha1_encode_digest(m, key->size, digest)
 	 && _rsa_verify(key, m, s));
   
   mpz_clear(m);

rsa-sha256-sign.c

@@ -39,9 +39,7 @@ rsa_sha256_sign(const struct rsa_private_key *key,
 		struct sha256_ctx *hash,
 		mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha256_encode(s, key->size - 1, hash))
+  if (pkcs1_rsa_sha256_encode(s, key->size, hash))
     {
       rsa_compute_root(key, s, s);
       return 1;
@@ -58,9 +56,7 @@ rsa_sha256_sign_digest(const struct rsa_private_key *key,
 		       const uint8_t *digest,
 		       mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha256_encode_digest(s, key->size - 1, digest))
+  if (pkcs1_rsa_sha256_encode_digest(s, key->size, digest))
     {
       rsa_compute_root(key, s, s);
       return 1;

rsa-sha256-verify.c

@@ -42,10 +42,9 @@ rsa_sha256_verify(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
 
-  res = (pkcs1_rsa_sha256_encode(m, key->size - 1, hash)
+  res = (pkcs1_rsa_sha256_encode(m, key->size, hash)
 	 &&_rsa_verify(key, m, s));
   
   mpz_clear(m);
@@ -61,10 +60,9 @@ rsa_sha256_verify_digest(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_sha256_encode_digest(m, key->size - 1, digest)
+  res = (pkcs1_rsa_sha256_encode_digest(m, key->size, digest)
 	 && _rsa_verify(key, m, s));
   
   mpz_clear(m);

rsa-sha512-sign.c

@@ -39,9 +39,7 @@ rsa_sha512_sign(const struct rsa_private_key *key,
 		struct sha512_ctx *hash,
 		mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha512_encode(s, key->size - 1, hash))
+  if (pkcs1_rsa_sha512_encode(s, key->size, hash))
     {
       rsa_compute_root(key, s, s);
       return 1;
@@ -58,9 +56,7 @@ rsa_sha512_sign_digest(const struct rsa_private_key *key,
 		       const uint8_t *digest,
 		       mpz_t s)
 {
-  assert(key->size > 0);
-
-  if (pkcs1_rsa_sha512_encode_digest(s, key->size - 1, digest))
+  if (pkcs1_rsa_sha512_encode_digest(s, key->size, digest))
     {
       rsa_compute_root(key, s, s);
       return 1;

rsa-sha512-verify.c

@@ -42,10 +42,9 @@ rsa_sha512_verify(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_sha512_encode(m, key->size - 1, hash) 
+  res = (pkcs1_rsa_sha512_encode(m, key->size, hash) 
 	 && _rsa_verify(key, m, s));
   
   mpz_clear(m);
@@ -61,10 +60,9 @@ rsa_sha512_verify_digest(const struct rsa_public_key *key,
   int res;
   mpz_t m;
 
-  assert(key->size > 0);
   mpz_init(m);
   
-  res = (pkcs1_rsa_sha512_encode_digest(m, key->size - 1, digest)
+  res = (pkcs1_rsa_sha512_encode_digest(m, key->size, digest)
 	 && _rsa_verify(key, m, s));
   
   mpz_clear(m);

testsuite/pkcs1-test.c

@@ -6,7 +6,7 @@ int
 test_main(void)
 {
   uint8_t buffer[16];
-  uint8_t expected[16] = {    1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  uint8_t expected[16] = { 0,    1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 			   0xff, 0xff, 0xff, 0xff, 0,    'a',  'b',  'c' };
 
   pkcs1_signature_prefix(sizeof(buffer), buffer,