rsa: Verify RSA padding programatically

Padding verification was done against static SHA/RSA pair arrays which
take up a lot of static memory, are mostly 0xff, and cannot be reused
for additional SHA/RSA pairings. The padding can be easily computed
according to PKCS#1v2.1 as:

  EM = 0x00 || 0x01 || PS || 0x00 || T

where PS is (emLen - tLen - 3) octets of 0xff and T is DER encoding
of the hash.

Store DER prefix in checksum_algo and create rsa_verify_padding
function to handle verification of a message for any SHA/RSA pairing.

Signed-off-by: Andrew Duda <aduda@meraki.com>
Signed-off-by: aduda <aduda@meraki.com>
Reviewed-by: Simon Glass <sjg@chromium.org>

common/image-sig.c

@@ -34,32 +34,35 @@ struct checksum_algo checksum_algos[] = {
 	{
 		"sha1",
 		SHA1_SUM_LEN,
+		SHA1_DER_LEN,
+		sha1_der_prefix,
 		RSA2048_BYTES,
 #if IMAGE_ENABLE_SIGN
 		EVP_sha1,
 #endif
 		hash_calculate,
-		padding_sha1_rsa2048,
 	},
 	{
 		"sha256",
 		SHA256_SUM_LEN,
+		SHA256_DER_LEN,
+		sha256_der_prefix,
 		RSA2048_BYTES,
 #if IMAGE_ENABLE_SIGN
 		EVP_sha256,
 #endif
 		hash_calculate,
-		padding_sha256_rsa2048,
 	},
 	{
 		"sha256",
 		SHA256_SUM_LEN,
+		SHA256_DER_LEN,
+		sha256_der_prefix,
 		RSA4096_BYTES,
 #if IMAGE_ENABLE_SIGN
 		EVP_sha256,
 #endif
 		hash_calculate,
-		padding_sha256_rsa4096,
 	}
 
 };

include/image.h

@@ -1070,6 +1070,8 @@ struct image_region {
 struct checksum_algo {
 	const char *name;
 	const int checksum_len;
+	const int der_len;
+	const uint8_t *der_prefix;
 	const int key_len;
 #if IMAGE_ENABLE_SIGN
 	const EVP_MD *(*calculate_sign)(void);
@@ -1077,7 +1079,6 @@ struct checksum_algo {
 	int (*calculate)(const char *name,
 			 const struct image_region region[],
 			 int region_count, uint8_t *checksum);
-	const uint8_t *rsa_padding;
 };
 
 struct image_sig_algo {

include/u-boot/rsa-checksum.h

@@ -12,10 +12,6 @@
 #include <u-boot/sha1.h>
 #include <u-boot/sha256.h>
 
-extern const uint8_t padding_sha256_rsa4096[];
-extern const uint8_t padding_sha256_rsa2048[];
-extern const uint8_t padding_sha1_rsa2048[];
-
 /**
  * hash_calculate() - Calculate hash over the data
  *

include/u-boot/sha1.h

@@ -21,6 +21,9 @@ extern "C" {
 
 #define SHA1_SUM_POS	-0x20
 #define SHA1_SUM_LEN	20
+#define SHA1_DER_LEN	15
+
+extern const uint8_t sha1_der_prefix[];
 
 /**
  * \brief	   SHA-1 context structure

include/u-boot/sha256.h

@@ -2,6 +2,9 @@
 #define _SHA256_H
 
 #define SHA256_SUM_LEN	32
+#define SHA256_DER_LEN	19
+
+extern const uint8_t sha256_der_prefix[];
 
 /* Reset watchdog each time we process this many bytes */
 #define CHUNKSZ_SHA256	(64 * 1024)

lib/rsa/rsa-checksum.c

@@ -13,130 +13,9 @@
 #include <hash.h>
 #else
 #include "fdt_host.h"
-#include <u-boot/sha1.h>
-#include <u-boot/sha256.h>
 #endif
 #include <u-boot/rsa.h>
 
-/* PKCS 1.5 paddings as described in the RSA PKCS#1 v2.1 standard. */
-
-const uint8_t padding_sha256_rsa2048[RSA2048_BYTES - SHA256_SUM_LEN] = {
-0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x30, 0x31, 0x30,
-0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
-0x00, 0x04, 0x20
-};
-
-const uint8_t padding_sha1_rsa2048[RSA2048_BYTES - SHA1_SUM_LEN] = {
-	0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0x00, 0x30, 0x21, 0x30,
-	0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a,
-	0x05, 0x00, 0x04, 0x14
-};
-
-const uint8_t padding_sha256_rsa4096[RSA4096_BYTES - SHA256_SUM_LEN] = {
-	0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0x00, 0x30, 0x31, 0x30,
-	0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65,
-	0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20
-};
-
 int hash_calculate(const char *name,
 		    const struct image_region region[],
 		    int region_count, uint8_t *checksum)

lib/rsa/rsa-verify.c

@@ -24,6 +24,40 @@
 /* Default public exponent for backward compatibility */
 #define RSA_DEFAULT_PUBEXP	65537
 
+/**
+ * rsa_verify_padding() - Verify RSA message padding is valid
+ *
+ * Verify a RSA message's padding is consistent with PKCS1.5
+ * padding as described in the RSA PKCS#1 v2.1 standard.
+ *
+ * @msg:	Padded message
+ * @pad_len:	Number of expected padding bytes
+ * @algo:	Checksum algo structure having information on DER encoding etc.
+ * @return 0 on success, != 0 on failure
+ */
+static int rsa_verify_padding(const uint8_t *msg, const int pad_len,
+			      struct checksum_algo *algo)
+{
+	int ff_len;
+	int ret;
+
+	/* first byte must be 0x00 */
+	ret = *msg++;
+	/* second byte must be 0x01 */
+	ret |= *msg++ ^ 0x01;
+	/* next ff_len bytes must be 0xff */
+	ff_len = pad_len - algo->der_len - 3;
+	ret |= *msg ^ 0xff;
+	ret |= memcmp(msg, msg+1, ff_len-1);
+	msg += ff_len;
+	/* next byte must be 0x00 */
+	ret |= *msg++;
+	/* next der_len bytes must match der_prefix */
+	ret |= memcmp(msg, algo->der_prefix, algo->der_len);
+
+	return ret;
+}
+
 /**
  * rsa_verify_key() - Verify a signature against some data using RSA Key
  *
@@ -83,11 +117,11 @@ static int rsa_verify_key(struct key_prop *prop, const uint8_t *sig,
 		return ret;
 	}
 
-	padding = algo->rsa_padding;
 	pad_len = algo->key_len - algo->checksum_len;
 
 	/* Check pkcs1.5 padding bytes. */
-	if (memcmp(buf, padding, pad_len)) {
+	ret = rsa_verify_padding(buf, pad_len, algo);
+	if (ret) {
 		debug("In RSAVerify(): Padding check failed!\n");
 		return -EINVAL;
 	}

lib/sha1.c

@@ -26,6 +26,11 @@
 #include <watchdog.h>
 #include <u-boot/sha1.h>
 
+const uint8_t sha1_der_prefix[SHA1_DER_LEN] = {
+	0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e,
+	0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14
+};
+
 /*
  * 32-bit integer manipulation macros (big endian)
  */

lib/sha256.c

@@ -15,6 +15,12 @@
 #include <watchdog.h>
 #include <u-boot/sha256.h>
 
+const uint8_t sha256_der_prefix[SHA256_DER_LEN] = {
+	0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86,
+	0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
+	0x00, 0x04, 0x20
+};
+
 /*
  * 32-bit integer manipulation macros (big endian)
  */