commit 217416a76e2505e8af6f384fb2b413f9df8f7508
author Yanray Wang <yanray.wang@arm.com> Fri Aug 11 15:03:51 2023 +0800
committer Yanray Wang <yanray.wang@arm.com> Wed Sep 06 10:28:09 2023 +0800
tree 0843d3476616efbcb9e4c55c10b09715c61a4aac
parent 06c466d869094390e1d48cb05649ea118b273f61

pkwrite.c: save stack usage for pk_write_pubkey_pem

mbedtls_pk_write_pubkey_pem would allocate 2086 bytes in writing a DER
encoded RSA public key. To save stack usage significantly, we use
heap memory instead.

Signed-off-by: Yanray Wang <yanray.wang@arm.com>

library/pkwrite.c

diff --git a/library/pkwrite.c b/library/pkwrite.c
index 88e6855..7253c6e 100644
--- a/library/pkwrite.c
+++ b/library/pkwrite.c
@@ -571,23 +571,30 @@
 int mbedtls_pk_write_pubkey_pem(mbedtls_pk_context *key, unsigned char *buf, size_t size)
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
-    unsigned char output_buf[PUB_DER_MAX_BYTES];
+    unsigned char *output_buf = NULL;
+    output_buf = calloc(1, PUB_DER_MAX_BYTES);
+    if (output_buf == NULL) {
+        return MBEDTLS_ERR_PK_ALLOC_FAILED;
+    }
     size_t olen = 0;
 
     PK_VALIDATE_RET(key != NULL);
     PK_VALIDATE_RET(buf != NULL || size == 0);
 
     if ((ret = mbedtls_pk_write_pubkey_der(key, output_buf,
-                                           sizeof(output_buf))) < 0) {
+                                           PUB_DER_MAX_BYTES)) < 0) {
+        free(output_buf);
         return ret;
     }
 
     if ((ret = mbedtls_pem_write_buffer(PEM_BEGIN_PUBLIC_KEY, PEM_END_PUBLIC_KEY,
-                                        output_buf + sizeof(output_buf) - ret,
+                                        output_buf + PUB_DER_MAX_BYTES - ret,
                                         ret, buf, size, &olen)) != 0) {
+        free(output_buf);
         return ret;
     }
 
+    free(output_buf);
     return 0;
 }