Squashed commit upgrading to mbedtls-3.6.0

Squash merging branch import/mbedtls-3.6.0

0fc9291f4 ("libmbedtls: bignum: restore mbedtls_mpi_exp_mod() from v3.5.2")
0ef87b1e6 ("libmbedtls: reset minimum rsa key size")
70b079496 ("libmbedtls: adjust use of rsa pk_wrap API")
6cf76464f ("libmbedtls: allow inclusion of arm_neon.h")
27df5c911 ("libmbedtls: fix cipher_wrap.c for NIST AES Key Wrap mode")
aa584f9ed ("libmbedtls: fix cipher_wrap.c for chacha20 and chachapoly")
523ae957e ("libmbedtls: add fault mitigation in mbedtls_rsa_rsassa_pkcs1_v15_verify()")
30bdb1bbf ("libmbedtls: add fault mitigation in mbedtls_rsa_rsassa_pss_verify_ext()")
e45cdab62 ("libmbedtls: add SM2 curve")
d2fda4fc2 ("libmbedtls: fix no CRT issue")
ab0eb5515 ("libmbedtls: add interfaces in mbedtls for context memory operation")
7925a6f26 ("libmedtls: mpi_miller_rabin: increase count limit")
8eaf69279 ("libmbedtls: add mbedtls_mpi_init_mempool()")
12e83fc8d ("libmbedtls: make mbedtls_mpi_mont*() available")
f9e261da5 ("mbedtls: configure mbedtls to reach for config")
7b6f378d7 ("mbedtls: remove default include/mbedtls/config.h")
c16331743 ("Import mbedtls-3.6.0")

Signed-off-by: Tom Van Eyck <tom.vaneyck@kuleuven.be>
Acked-by: Jerome Forissier <jerome.forissier@linaro.org>
diff --git a/lib/libmbedtls/mbedtls/library/aes.c b/lib/libmbedtls/mbedtls/library/aes.c
index 69da582..b1a5c3e 100644
--- a/lib/libmbedtls/mbedtls/library/aes.c
+++ b/lib/libmbedtls/mbedtls/library/aes.c
@@ -2,24 +2,12 @@
  *  FIPS-197 compliant AES implementation
  *
  *  Copyright The Mbed TLS Contributors
- *  SPDX-License-Identifier: Apache-2.0
- *
- *  Licensed under the Apache License, Version 2.0 (the "License"); you may
- *  not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
+ *  SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
  */
 /*
  *  The AES block cipher was designed by Vincent Rijmen and Joan Daemen.
  *
- *  http://csrc.nist.gov/encryption/aes/rijndael/Rijndael.pdf
+ *  https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/aes-development/rijndael-ammended.pdf
  *  http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf
  */
 
@@ -33,6 +21,27 @@
 #include "mbedtls/platform.h"
 #include "mbedtls/platform_util.h"
 #include "mbedtls/error.h"
+
+#if defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+#if !((defined(MBEDTLS_ARCH_IS_ARMV8_A) && defined(MBEDTLS_AESCE_C)) || \
+    (defined(MBEDTLS_ARCH_IS_X64)       && defined(MBEDTLS_AESNI_C)) || \
+    (defined(MBEDTLS_ARCH_IS_X86)       && defined(MBEDTLS_AESNI_C)))
+#error "MBEDTLS_AES_USE_HARDWARE_ONLY defined, but not all prerequisites"
+#endif
+#endif
+
+#if defined(MBEDTLS_ARCH_IS_X86)
+#if defined(MBEDTLS_PADLOCK_C)
+#if !defined(MBEDTLS_HAVE_ASM)
+#error "MBEDTLS_PADLOCK_C defined, but not all prerequisites"
+#endif
+#if defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+#error "MBEDTLS_AES_USE_HARDWARE_ONLY cannot be defined when " \
+    "MBEDTLS_PADLOCK_C is set"
+#endif
+#endif
+#endif
+
 #if defined(MBEDTLS_PADLOCK_C)
 #include "padlock.h"
 #endif
@@ -44,10 +53,21 @@
 #endif
 
 #include "mbedtls/platform.h"
+#include "ctr.h"
+
+/*
+ * This is a convenience shorthand macro to check if we need reverse S-box and
+ * reverse tables. It's private and only defined in this file.
+ */
+#if (!defined(MBEDTLS_AES_DECRYPT_ALT) || \
+    (!defined(MBEDTLS_AES_SETKEY_DEC_ALT) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY))) && \
+    !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
+#define MBEDTLS_AES_NEED_REVERSE_TABLES
+#endif
 
 #if !defined(MBEDTLS_AES_ALT)
 
-#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
+#if defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)
 static int aes_padlock_ace = -1;
 #endif
 
@@ -55,7 +75,7 @@
 /*
  * Forward S-box
  */
-static const unsigned char FSb[256] =
+MBEDTLS_MAYBE_UNUSED static const unsigned char FSb[256] =
 {
     0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5,
     0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
@@ -162,31 +182,27 @@
     V(CB, B0, B0, 7B), V(FC, 54, 54, A8), V(D6, BB, BB, 6D), V(3A, 16, 16, 2C)
 
 #define V(a, b, c, d) 0x##a##b##c##d
-static const uint32_t FT0[256] = { FT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t FT0[256] = { FT };
 #undef V
 
-#if !defined(MBEDTLS_AES_FEWER_TABLES)
-
 #define V(a, b, c, d) 0x##b##c##d##a
-static const uint32_t FT1[256] = { FT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t FT1[256] = { FT };
 #undef V
 
 #define V(a, b, c, d) 0x##c##d##a##b
-static const uint32_t FT2[256] = { FT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t FT2[256] = { FT };
 #undef V
 
 #define V(a, b, c, d) 0x##d##a##b##c
-static const uint32_t FT3[256] = { FT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t FT3[256] = { FT };
 #undef V
 
-#endif /* !MBEDTLS_AES_FEWER_TABLES */
-
 #undef FT
 
 /*
  * Reverse S-box
  */
-static const unsigned char RSb[256] =
+MBEDTLS_MAYBE_UNUSED static const unsigned char RSb[256] =
 {
     0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38,
     0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB,
@@ -292,32 +308,29 @@
     V(71, 01, A8, 39), V(DE, B3, 0C, 08), V(9C, E4, B4, D8), V(90, C1, 56, 64), \
     V(61, 84, CB, 7B), V(70, B6, 32, D5), V(74, 5C, 6C, 48), V(42, 57, B8, D0)
 
+
 #define V(a, b, c, d) 0x##a##b##c##d
-static const uint32_t RT0[256] = { RT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t RT0[256] = { RT };
 #undef V
 
-#if !defined(MBEDTLS_AES_FEWER_TABLES)
-
 #define V(a, b, c, d) 0x##b##c##d##a
-static const uint32_t RT1[256] = { RT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t RT1[256] = { RT };
 #undef V
 
 #define V(a, b, c, d) 0x##c##d##a##b
-static const uint32_t RT2[256] = { RT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t RT2[256] = { RT };
 #undef V
 
 #define V(a, b, c, d) 0x##d##a##b##c
-static const uint32_t RT3[256] = { RT };
+MBEDTLS_MAYBE_UNUSED static const uint32_t RT3[256] = { RT };
 #undef V
 
-#endif /* !MBEDTLS_AES_FEWER_TABLES */
-
 #undef RT
 
 /*
  * Round constants
  */
-static const uint32_t RCON[10] =
+MBEDTLS_MAYBE_UNUSED static const uint32_t round_constants[10] =
 {
     0x00000001, 0x00000002, 0x00000004, 0x00000008,
     0x00000010, 0x00000020, 0x00000040, 0x00000080,
@@ -329,29 +342,26 @@
 /*
  * Forward S-box & tables
  */
-static unsigned char FSb[256];
-static uint32_t FT0[256];
-#if !defined(MBEDTLS_AES_FEWER_TABLES)
-static uint32_t FT1[256];
-static uint32_t FT2[256];
-static uint32_t FT3[256];
-#endif /* !MBEDTLS_AES_FEWER_TABLES */
+MBEDTLS_MAYBE_UNUSED static unsigned char FSb[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t FT0[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t FT1[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t FT2[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t FT3[256];
 
 /*
  * Reverse S-box & tables
  */
-static unsigned char RSb[256];
-static uint32_t RT0[256];
-#if !defined(MBEDTLS_AES_FEWER_TABLES)
-static uint32_t RT1[256];
-static uint32_t RT2[256];
-static uint32_t RT3[256];
-#endif /* !MBEDTLS_AES_FEWER_TABLES */
+MBEDTLS_MAYBE_UNUSED static unsigned char RSb[256];
+
+MBEDTLS_MAYBE_UNUSED static uint32_t RT0[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t RT1[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t RT2[256];
+MBEDTLS_MAYBE_UNUSED static uint32_t RT3[256];
 
 /*
  * Round constants
  */
-static uint32_t RCON[10];
+MBEDTLS_MAYBE_UNUSED static uint32_t round_constants[10];
 
 /*
  * Tables generation code
@@ -360,48 +370,53 @@
 #define XTIME(x) (((x) << 1) ^ (((x) & 0x80) ? 0x1B : 0x00))
 #define MUL(x, y) (((x) && (y)) ? pow[(log[(x)]+log[(y)]) % 255] : 0)
 
-static int aes_init_done = 0;
+MBEDTLS_MAYBE_UNUSED static int aes_init_done = 0;
 
-static void aes_gen_tables(void)
+MBEDTLS_MAYBE_UNUSED static void aes_gen_tables(void)
 {
-    int i, x, y, z;
-    int pow[256];
-    int log[256];
+    int i;
+    uint8_t x, y, z;
+    uint8_t pow[256];
+    uint8_t log[256];
 
     /*
      * compute pow and log tables over GF(2^8)
      */
     for (i = 0, x = 1; i < 256; i++) {
         pow[i] = x;
-        log[x] = i;
-        x = MBEDTLS_BYTE_0(x ^ XTIME(x));
+        log[x] = (uint8_t) i;
+        x ^= XTIME(x);
     }
 
     /*
      * calculate the round constants
      */
     for (i = 0, x = 1; i < 10; i++) {
-        RCON[i] = (uint32_t) x;
-        x = MBEDTLS_BYTE_0(XTIME(x));
+        round_constants[i] = x;
+        x = XTIME(x);
     }
 
     /*
      * generate the forward and reverse S-boxes
      */
     FSb[0x00] = 0x63;
+#if defined(MBEDTLS_AES_NEED_REVERSE_TABLES)
     RSb[0x63] = 0x00;
+#endif
 
     for (i = 1; i < 256; i++) {
         x = pow[255 - log[i]];
 
-        y  = x; y = MBEDTLS_BYTE_0((y << 1) | (y >> 7));
-        x ^= y; y = MBEDTLS_BYTE_0((y << 1) | (y >> 7));
-        x ^= y; y = MBEDTLS_BYTE_0((y << 1) | (y >> 7));
-        x ^= y; y = MBEDTLS_BYTE_0((y << 1) | (y >> 7));
+        y  = x; y = (y << 1) | (y >> 7);
+        x ^= y; y = (y << 1) | (y >> 7);
+        x ^= y; y = (y << 1) | (y >> 7);
+        x ^= y; y = (y << 1) | (y >> 7);
         x ^= y ^ 0x63;
 
-        FSb[i] = (unsigned char) x;
+        FSb[i] = x;
+#if defined(MBEDTLS_AES_NEED_REVERSE_TABLES)
         RSb[x] = (unsigned char) i;
+#endif
     }
 
     /*
@@ -409,8 +424,8 @@
      */
     for (i = 0; i < 256; i++) {
         x = FSb[i];
-        y = MBEDTLS_BYTE_0(XTIME(x));
-        z = MBEDTLS_BYTE_0(y ^ x);
+        y = XTIME(x);
+        z = y ^ x;
 
         FT0[i] = ((uint32_t) y) ^
                  ((uint32_t) x <<  8) ^
@@ -423,6 +438,7 @@
         FT3[i] = ROTL8(FT2[i]);
 #endif /* !MBEDTLS_AES_FEWER_TABLES */
 
+#if defined(MBEDTLS_AES_NEED_REVERSE_TABLES)
         x = RSb[i];
 
         RT0[i] = ((uint32_t) MUL(0x0E, x)) ^
@@ -435,6 +451,7 @@
         RT2[i] = ROTL8(RT1[i]);
         RT3[i] = ROTL8(RT2[i]);
 #endif /* !MBEDTLS_AES_FEWER_TABLES */
+#endif /* MBEDTLS_AES_NEED_REVERSE_TABLES */
     }
 }
 
@@ -510,16 +527,17 @@
  * Note that the offset is in units of elements of buf, i.e. 32-bit words,
  * i.e. an offset of 1 means 4 bytes and so on.
  */
-#if (defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)) ||        \
+#if (defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)) ||        \
     (defined(MBEDTLS_AESNI_C) && MBEDTLS_AESNI_HAVE_CODE == 2)
 #define MAY_NEED_TO_ALIGN
 #endif
-static unsigned mbedtls_aes_rk_offset(uint32_t *buf)
+
+MBEDTLS_MAYBE_UNUSED static unsigned mbedtls_aes_rk_offset(uint32_t *buf)
 {
 #if defined(MAY_NEED_TO_ALIGN)
     int align_16_bytes = 0;
 
-#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
+#if defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)
     if (aes_padlock_ace == -1) {
         aes_padlock_ace = mbedtls_padlock_has_support(MBEDTLS_PADLOCK_ACE);
     }
@@ -558,13 +576,14 @@
 int mbedtls_aes_setkey_enc(mbedtls_aes_context *ctx, const unsigned char *key,
                            unsigned int keybits)
 {
-    unsigned int i;
     uint32_t *RK;
 
     switch (keybits) {
         case 128: ctx->nr = 10; break;
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
         case 192: ctx->nr = 12; break;
         case 256: ctx->nr = 14; break;
+#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
         default: return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH;
     }
 
@@ -584,21 +603,22 @@
     }
 #endif
 
-#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64)
-    if (mbedtls_aesce_has_support()) {
+#if defined(MBEDTLS_AESCE_HAVE_CODE)
+    if (MBEDTLS_AESCE_HAS_SUPPORT()) {
         return mbedtls_aesce_setkey_enc((unsigned char *) RK, key, keybits);
     }
 #endif
 
-    for (i = 0; i < (keybits >> 5); i++) {
+#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+    for (unsigned int i = 0; i < (keybits >> 5); i++) {
         RK[i] = MBEDTLS_GET_UINT32_LE(key, i << 2);
     }
 
     switch (ctx->nr) {
         case 10:
 
-            for (i = 0; i < 10; i++, RK += 4) {
-                RK[4]  = RK[0] ^ RCON[i] ^
+            for (unsigned int i = 0; i < 10; i++, RK += 4) {
+                RK[4]  = RK[0] ^ round_constants[i] ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_1(RK[3])]) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_2(RK[3])] <<  8) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_3(RK[3])] << 16) ^
@@ -610,10 +630,11 @@
             }
             break;
 
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
         case 12:
 
-            for (i = 0; i < 8; i++, RK += 6) {
-                RK[6]  = RK[0] ^ RCON[i] ^
+            for (unsigned int i = 0; i < 8; i++, RK += 6) {
+                RK[6]  = RK[0] ^ round_constants[i] ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_1(RK[5])]) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_2(RK[5])] <<  8) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_3(RK[5])] << 16) ^
@@ -629,8 +650,8 @@
 
         case 14:
 
-            for (i = 0; i < 7; i++, RK += 8) {
-                RK[8]  = RK[0] ^ RCON[i] ^
+            for (unsigned int i = 0; i < 7; i++, RK += 8) {
+                RK[8]  = RK[0] ^ round_constants[i] ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_1(RK[7])]) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_2(RK[7])] <<  8) ^
                          ((uint32_t) FSb[MBEDTLS_BYTE_3(RK[7])] << 16) ^
@@ -651,23 +672,28 @@
                 RK[15] = RK[7] ^ RK[14];
             }
             break;
+#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
     }
 
     return 0;
+#endif /* !MBEDTLS_AES_USE_HARDWARE_ONLY */
 }
 #endif /* !MBEDTLS_AES_SETKEY_ENC_ALT */
 
 /*
  * AES key schedule (decryption)
  */
-#if !defined(MBEDTLS_AES_SETKEY_DEC_ALT)
+#if !defined(MBEDTLS_AES_SETKEY_DEC_ALT) && !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
 int mbedtls_aes_setkey_dec(mbedtls_aes_context *ctx, const unsigned char *key,
                            unsigned int keybits)
 {
-    int i, j, ret;
+#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+    uint32_t *SK;
+#endif
+    int ret;
     mbedtls_aes_context cty;
     uint32_t *RK;
-    uint32_t *SK;
+
 
     mbedtls_aes_init(&cty);
 
@@ -689,8 +715,8 @@
     }
 #endif
 
-#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64)
-    if (mbedtls_aesce_has_support()) {
+#if defined(MBEDTLS_AESCE_HAVE_CODE)
+    if (MBEDTLS_AESCE_HAS_SUPPORT()) {
         mbedtls_aesce_inverse_key(
             (unsigned char *) RK,
             (const unsigned char *) (cty.buf + cty.rk_offset),
@@ -699,15 +725,16 @@
     }
 #endif
 
+#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
     SK = cty.buf + cty.rk_offset + cty.nr * 4;
 
     *RK++ = *SK++;
     *RK++ = *SK++;
     *RK++ = *SK++;
     *RK++ = *SK++;
-
-    for (i = ctx->nr - 1, SK -= 8; i > 0; i--, SK -= 8) {
-        for (j = 0; j < 4; j++, SK++) {
+    SK -= 8;
+    for (int i = ctx->nr - 1; i > 0; i--, SK -= 8) {
+        for (int j = 0; j < 4; j++, SK++) {
             *RK++ = AES_RT0(FSb[MBEDTLS_BYTE_0(*SK)]) ^
                     AES_RT1(FSb[MBEDTLS_BYTE_1(*SK)]) ^
                     AES_RT2(FSb[MBEDTLS_BYTE_2(*SK)]) ^
@@ -719,13 +746,13 @@
     *RK++ = *SK++;
     *RK++ = *SK++;
     *RK++ = *SK++;
-
+#endif /* !MBEDTLS_AES_USE_HARDWARE_ONLY */
 exit:
     mbedtls_aes_free(&cty);
 
     return ret;
 }
-#endif /* !MBEDTLS_AES_SETKEY_DEC_ALT */
+#endif /* !MBEDTLS_AES_SETKEY_DEC_ALT && !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */
 
 #if defined(MBEDTLS_CIPHER_MODE_XTS)
 static int mbedtls_aes_xts_decode_keys(const unsigned char *key,
@@ -914,7 +941,7 @@
 /*
  * AES-ECB block decryption
  */
-#if !defined(MBEDTLS_AES_DECRYPT_ALT)
+#if !defined(MBEDTLS_AES_DECRYPT_ALT) && !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
 int mbedtls_internal_aes_decrypt(mbedtls_aes_context *ctx,
                                  const unsigned char input[16],
                                  unsigned char output[16])
@@ -971,9 +998,8 @@
 
     return 0;
 }
-#endif /* !MBEDTLS_AES_DECRYPT_ALT */
+#endif /* !MBEDTLS_AES_DECRYPT_ALT && !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */
 
-#if defined(MAY_NEED_TO_ALIGN)
 /* VIA Padlock and our intrinsics-based implementation of AESNI require
  * the round keys to be aligned on a 16-byte boundary. We take care of this
  * before creating them, but the AES context may have moved (this can happen
@@ -981,7 +1007,7 @@
  * calls it might have a different alignment with respect to 16-byte memory.
  * So we may need to realign.
  */
-static void aes_maybe_realign(mbedtls_aes_context *ctx)
+MBEDTLS_MAYBE_UNUSED static void aes_maybe_realign(mbedtls_aes_context *ctx)
 {
     unsigned new_offset = mbedtls_aes_rk_offset(ctx->buf);
     if (new_offset != ctx->rk_offset) {
@@ -991,7 +1017,6 @@
         ctx->rk_offset = new_offset;
     }
 }
-#endif
 
 /*
  * AES-ECB block encryption/decryption
@@ -1015,26 +1040,32 @@
     }
 #endif
 
-#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64)
-    if (mbedtls_aesce_has_support()) {
+#if defined(MBEDTLS_AESCE_HAVE_CODE)
+    if (MBEDTLS_AESCE_HAS_SUPPORT()) {
         return mbedtls_aesce_crypt_ecb(ctx, mode, input, output);
     }
 #endif
 
-#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
+#if defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)
     if (aes_padlock_ace > 0) {
         return mbedtls_padlock_xcryptecb(ctx, mode, input, output);
     }
 #endif
 
-    if (mode == MBEDTLS_AES_ENCRYPT) {
-        return mbedtls_internal_aes_encrypt(ctx, input, output);
-    } else {
+#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
+    if (mode == MBEDTLS_AES_DECRYPT) {
         return mbedtls_internal_aes_decrypt(ctx, input, output);
+    } else
+#endif
+    {
+        return mbedtls_internal_aes_encrypt(ctx, input, output);
     }
+#endif /* !MBEDTLS_AES_USE_HARDWARE_ONLY */
 }
 
 #if defined(MBEDTLS_CIPHER_MODE_CBC)
+
 /*
  * AES-CBC buffer encryption/decryption
  */
@@ -1052,11 +1083,16 @@
         return MBEDTLS_ERR_AES_BAD_INPUT_DATA;
     }
 
+    /* Nothing to do if length is zero. */
+    if (length == 0) {
+        return 0;
+    }
+
     if (length % 16) {
         return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH;
     }
 
-#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
+#if defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)
     if (aes_padlock_ace > 0) {
         if (mbedtls_padlock_xcryptcbc(ctx, mode, length, iv, input, output) == 0) {
             return 0;
@@ -1068,6 +1104,8 @@
     }
 #endif
 
+    const unsigned char *ivp = iv;
+
     if (mode == MBEDTLS_AES_DECRYPT) {
         while (length > 0) {
             memcpy(temp, input, 16);
@@ -1075,8 +1113,10 @@
             if (ret != 0) {
                 goto exit;
             }
-
-            mbedtls_xor(output, output, iv, 16);
+            /* Avoid using the NEON implementation of mbedtls_xor. Because of the dependency on
+             * the result for the next block in CBC, and the cost of transferring that data from
+             * NEON registers, NEON is slower on aarch64. */
+            mbedtls_xor_no_simd(output, output, iv, 16);
 
             memcpy(iv, temp, 16);
 
@@ -1086,18 +1126,19 @@
         }
     } else {
         while (length > 0) {
-            mbedtls_xor(output, input, iv, 16);
+            mbedtls_xor_no_simd(output, input, ivp, 16);
 
             ret = mbedtls_aes_crypt_ecb(ctx, mode, output, output);
             if (ret != 0) {
                 goto exit;
             }
-            memcpy(iv, output, 16);
+            ivp = output;
 
             input  += 16;
             output += 16;
             length -= 16;
         }
+        memcpy(iv, ivp, 16);
     }
     ret = 0;
 
@@ -1118,8 +1159,11 @@
  * for machine endianness and hence works correctly on both big and little
  * endian machines.
  */
-static void mbedtls_gf128mul_x_ble(unsigned char r[16],
-                                   const unsigned char x[16])
+#if defined(MBEDTLS_AESCE_C) || defined(MBEDTLS_AESNI_C)
+MBEDTLS_OPTIMIZE_FOR_PERFORMANCE
+#endif
+static inline void mbedtls_gf128mul_x_ble(unsigned char r[16],
+                                          const unsigned char x[16])
 {
     uint64_t a, b, ra, rb;
 
@@ -1135,7 +1179,13 @@
 
 /*
  * AES-XTS buffer encryption/decryption
+ *
+ * Use of MBEDTLS_OPTIMIZE_FOR_PERFORMANCE here and for mbedtls_gf128mul_x_ble()
+ * is a 3x performance improvement for gcc -Os, if we have hardware AES support.
  */
+#if defined(MBEDTLS_AESCE_C) || defined(MBEDTLS_AESNI_C)
+MBEDTLS_OPTIMIZE_FOR_PERFORMANCE
+#endif
 int mbedtls_aes_crypt_xts(mbedtls_aes_xts_context *ctx,
                           int mode,
                           size_t length,
@@ -1172,7 +1222,7 @@
     }
 
     while (blocks--) {
-        if (leftover && (mode == MBEDTLS_AES_DECRYPT) && blocks == 0) {
+        if (MBEDTLS_UNLIKELY(leftover && (mode == MBEDTLS_AES_DECRYPT) && blocks == 0)) {
             /* We are on the last block in a decrypt operation that has
              * leftover bytes, so we need to use the next tweak for this block,
              * and this tweak for the leftover bytes. Save the current tweak for
@@ -1392,36 +1442,38 @@
                           const unsigned char *input,
                           unsigned char *output)
 {
-    int c, i;
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
-    size_t n;
 
-    n = *nc_off;
+    size_t offset = *nc_off;
 
-    if (n > 0x0F) {
+    if (offset > 0x0F) {
         return MBEDTLS_ERR_AES_BAD_INPUT_DATA;
     }
 
-    while (length--) {
-        if (n == 0) {
+    for (size_t i = 0; i < length;) {
+        size_t n = 16;
+        if (offset == 0) {
             ret = mbedtls_aes_crypt_ecb(ctx, MBEDTLS_AES_ENCRYPT, nonce_counter, stream_block);
             if (ret != 0) {
                 goto exit;
             }
-
-            for (i = 16; i > 0; i--) {
-                if (++nonce_counter[i - 1] != 0) {
-                    break;
-                }
-            }
+            mbedtls_ctr_increment_counter(nonce_counter);
+        } else {
+            n -= offset;
         }
-        c = *input++;
-        *output++ = (unsigned char) (c ^ stream_block[n]);
 
-        n = (n + 1) & 0x0F;
+        if (n > (length - i)) {
+            n = (length - i);
+        }
+        mbedtls_xor(&output[i], &input[i], &stream_block[offset], n);
+        // offset might be non-zero for the last block, but in that case, we don't use it again
+        offset = 0;
+        i += n;
     }
 
-    *nc_off = n;
+    // capture offset for future resumption
+    *nc_off = (*nc_off + length) % 16;
+
     ret = 0;
 
 exit:
@@ -1437,45 +1489,55 @@
  *
  * http://csrc.nist.gov/archive/aes/rijndael/rijndael-vals.zip
  */
-static const unsigned char aes_test_ecb_dec[3][16] =
+#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
+static const unsigned char aes_test_ecb_dec[][16] =
 {
     { 0x44, 0x41, 0x6A, 0xC2, 0xD1, 0xF5, 0x3C, 0x58,
       0x33, 0x03, 0x91, 0x7E, 0x6B, 0xE9, 0xEB, 0xE0 },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0x48, 0xE3, 0x1E, 0x9E, 0x25, 0x67, 0x18, 0xF2,
       0x92, 0x29, 0x31, 0x9C, 0x19, 0xF1, 0x5B, 0xA4 },
     { 0x05, 0x8C, 0xCF, 0xFD, 0xBB, 0xCB, 0x38, 0x2D,
       0x1F, 0x6F, 0x56, 0x58, 0x5D, 0x8A, 0x4A, 0xDE }
+#endif
 };
+#endif
 
-static const unsigned char aes_test_ecb_enc[3][16] =
+static const unsigned char aes_test_ecb_enc[][16] =
 {
     { 0xC3, 0x4C, 0x05, 0x2C, 0xC0, 0xDA, 0x8D, 0x73,
       0x45, 0x1A, 0xFE, 0x5F, 0x03, 0xBE, 0x29, 0x7F },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0xF3, 0xF6, 0x75, 0x2A, 0xE8, 0xD7, 0x83, 0x11,
       0x38, 0xF0, 0x41, 0x56, 0x06, 0x31, 0xB1, 0x14 },
     { 0x8B, 0x79, 0xEE, 0xCC, 0x93, 0xA0, 0xEE, 0x5D,
       0xFF, 0x30, 0xB4, 0xEA, 0x21, 0x63, 0x6D, 0xA4 }
+#endif
 };
 
 #if defined(MBEDTLS_CIPHER_MODE_CBC)
-static const unsigned char aes_test_cbc_dec[3][16] =
+static const unsigned char aes_test_cbc_dec[][16] =
 {
     { 0xFA, 0xCA, 0x37, 0xE0, 0xB0, 0xC8, 0x53, 0x73,
       0xDF, 0x70, 0x6E, 0x73, 0xF7, 0xC9, 0xAF, 0x86 },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0x5D, 0xF6, 0x78, 0xDD, 0x17, 0xBA, 0x4E, 0x75,
       0xB6, 0x17, 0x68, 0xC6, 0xAD, 0xEF, 0x7C, 0x7B },
     { 0x48, 0x04, 0xE1, 0x81, 0x8F, 0xE6, 0x29, 0x75,
       0x19, 0xA3, 0xE8, 0x8C, 0x57, 0x31, 0x04, 0x13 }
+#endif
 };
 
-static const unsigned char aes_test_cbc_enc[3][16] =
+static const unsigned char aes_test_cbc_enc[][16] =
 {
     { 0x8A, 0x05, 0xFC, 0x5E, 0x09, 0x5A, 0xF4, 0x84,
       0x8A, 0x08, 0xD3, 0x28, 0xD3, 0x68, 0x8E, 0x3D },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0x7B, 0xD9, 0x66, 0xD5, 0x3A, 0xD8, 0xC1, 0xBB,
       0x85, 0xD2, 0xAD, 0xFA, 0xE8, 0x7B, 0xB1, 0x04 },
     { 0xFE, 0x3C, 0x53, 0x65, 0x3E, 0x2F, 0x45, 0xB5,
       0x6F, 0xCD, 0x88, 0xB2, 0xCC, 0x89, 0x8F, 0xF0 }
+#endif
 };
 #endif /* MBEDTLS_CIPHER_MODE_CBC */
 
@@ -1485,10 +1547,11 @@
  *
  * http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
  */
-static const unsigned char aes_test_cfb128_key[3][32] =
+static const unsigned char aes_test_cfb128_key[][32] =
 {
     { 0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6,
       0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0x8E, 0x73, 0xB0, 0xF7, 0xDA, 0x0E, 0x64, 0x52,
       0xC8, 0x10, 0xF3, 0x2B, 0x80, 0x90, 0x79, 0xE5,
       0x62, 0xF8, 0xEA, 0xD2, 0x52, 0x2C, 0x6B, 0x7B },
@@ -1496,6 +1559,7 @@
       0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81,
       0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7,
       0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 }
+#endif
 };
 
 static const unsigned char aes_test_cfb128_iv[16] =
@@ -1516,7 +1580,7 @@
     0xAD, 0x2B, 0x41, 0x7B, 0xE6, 0x6C, 0x37, 0x10
 };
 
-static const unsigned char aes_test_cfb128_ct[3][64] =
+static const unsigned char aes_test_cfb128_ct[][64] =
 {
     { 0x3B, 0x3F, 0xD9, 0x2E, 0xB7, 0x2D, 0xAD, 0x20,
       0x33, 0x34, 0x49, 0xF8, 0xE8, 0x3C, 0xFB, 0x4A,
@@ -1526,6 +1590,7 @@
       0xB1, 0x80, 0x8C, 0xF1, 0x87, 0xA4, 0xF4, 0xDF,
       0xC0, 0x4B, 0x05, 0x35, 0x7C, 0x5D, 0x1C, 0x0E,
       0xEA, 0xC4, 0xC6, 0x6F, 0x9F, 0xF7, 0xF2, 0xE6 },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0xCD, 0xC8, 0x0D, 0x6F, 0xDD, 0xF1, 0x8C, 0xAB,
       0x34, 0xC2, 0x59, 0x09, 0xC9, 0x9A, 0x41, 0x74,
       0x67, 0xCE, 0x7F, 0x7F, 0x81, 0x17, 0x36, 0x21,
@@ -1542,6 +1607,7 @@
       0xA1, 0x3E, 0xD0, 0xA8, 0x26, 0x7A, 0xE2, 0xF9,
       0x75, 0xA3, 0x85, 0x74, 0x1A, 0xB9, 0xCE, 0xF8,
       0x20, 0x31, 0x62, 0x3D, 0x55, 0xB1, 0xE4, 0x71 }
+#endif
 };
 #endif /* MBEDTLS_CIPHER_MODE_CFB */
 
@@ -1551,10 +1617,11 @@
  *
  * https://csrc.nist.gov/publications/detail/sp/800-38a/final
  */
-static const unsigned char aes_test_ofb_key[3][32] =
+static const unsigned char aes_test_ofb_key[][32] =
 {
     { 0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6,
       0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0x8E, 0x73, 0xB0, 0xF7, 0xDA, 0x0E, 0x64, 0x52,
       0xC8, 0x10, 0xF3, 0x2B, 0x80, 0x90, 0x79, 0xE5,
       0x62, 0xF8, 0xEA, 0xD2, 0x52, 0x2C, 0x6B, 0x7B },
@@ -1562,6 +1629,7 @@
       0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81,
       0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7,
       0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 }
+#endif
 };
 
 static const unsigned char aes_test_ofb_iv[16] =
@@ -1582,7 +1650,7 @@
     0xAD, 0x2B, 0x41, 0x7B, 0xE6, 0x6C, 0x37, 0x10
 };
 
-static const unsigned char aes_test_ofb_ct[3][64] =
+static const unsigned char aes_test_ofb_ct[][64] =
 {
     { 0x3B, 0x3F, 0xD9, 0x2E, 0xB7, 0x2D, 0xAD, 0x20,
       0x33, 0x34, 0x49, 0xF8, 0xE8, 0x3C, 0xFB, 0x4A,
@@ -1592,6 +1660,7 @@
       0x43, 0x44, 0xf7, 0xa8, 0x22, 0x60, 0xed, 0xcc,
       0x30, 0x4c, 0x65, 0x28, 0xf6, 0x59, 0xc7, 0x78,
       0x66, 0xa5, 0x10, 0xd9, 0xc1, 0xd6, 0xae, 0x5e },
+#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
     { 0xCD, 0xC8, 0x0D, 0x6F, 0xDD, 0xF1, 0x8C, 0xAB,
       0x34, 0xC2, 0x59, 0x09, 0xC9, 0x9A, 0x41, 0x74,
       0xfc, 0xc2, 0x8b, 0x8d, 0x4c, 0x63, 0x83, 0x7c,
@@ -1608,6 +1677,7 @@
       0xf3, 0x9d, 0x1c, 0x5b, 0xba, 0x97, 0xc4, 0x08,
       0x01, 0x26, 0x14, 0x1d, 0x67, 0xf3, 0x7b, 0xe8,
       0x53, 0x8f, 0x5a, 0x8b, 0xe7, 0x40, 0xe4, 0x84 }
+#endif
 };
 #endif /* MBEDTLS_CIPHER_MODE_OFB */
 
@@ -1618,7 +1688,7 @@
  * http://www.faqs.org/rfcs/rfc3686.html
  */
 
-static const unsigned char aes_test_ctr_key[3][16] =
+static const unsigned char aes_test_ctr_key[][16] =
 {
     { 0xAE, 0x68, 0x52, 0xF8, 0x12, 0x10, 0x67, 0xCC,
       0x4B, 0xF7, 0xA5, 0x76, 0x55, 0x77, 0xF3, 0x9E },
@@ -1628,7 +1698,7 @@
       0xAC, 0x6E, 0x61, 0x85, 0x29, 0xF9, 0xA0, 0xDC }
 };
 
-static const unsigned char aes_test_ctr_nonce_counter[3][16] =
+static const unsigned char aes_test_ctr_nonce_counter[][16] =
 {
     { 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 },
@@ -1638,11 +1708,10 @@
       0x4A, 0x17, 0x86, 0xF0, 0x00, 0x00, 0x00, 0x01 }
 };
 
-static const unsigned char aes_test_ctr_pt[3][48] =
+static const unsigned char aes_test_ctr_pt[][48] =
 {
     { 0x53, 0x69, 0x6E, 0x67, 0x6C, 0x65, 0x20, 0x62,
       0x6C, 0x6F, 0x63, 0x6B, 0x20, 0x6D, 0x73, 0x67 },
-
     { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
       0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
       0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
@@ -1655,7 +1724,7 @@
       0x20, 0x21, 0x22, 0x23 }
 };
 
-static const unsigned char aes_test_ctr_ct[3][48] =
+static const unsigned char aes_test_ctr_ct[][48] =
 {
     { 0xE4, 0x09, 0x5D, 0x4F, 0xB7, 0xA7, 0xB3, 0x79,
       0x2D, 0x61, 0x75, 0xA3, 0x26, 0x13, 0x11, 0xB8 },
@@ -1779,312 +1848,359 @@
 #if defined(MBEDTLS_AES_ALT)
         mbedtls_printf("  AES note: alternative implementation.\n");
 #else /* MBEDTLS_AES_ALT */
-#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
-        if (mbedtls_padlock_has_support(MBEDTLS_PADLOCK_ACE)) {
-            mbedtls_printf("  AES note: using VIA Padlock.\n");
-        } else
-#endif
 #if defined(MBEDTLS_AESNI_HAVE_CODE)
+#if MBEDTLS_AESNI_HAVE_CODE == 1
+        mbedtls_printf("  AES note: AESNI code present (assembly implementation).\n");
+#elif MBEDTLS_AESNI_HAVE_CODE == 2
+        mbedtls_printf("  AES note: AESNI code present (intrinsics implementation).\n");
+#else
+#error "Unrecognised value for MBEDTLS_AESNI_HAVE_CODE"
+#endif
         if (mbedtls_aesni_has_support(MBEDTLS_AESNI_AES)) {
             mbedtls_printf("  AES note: using AESNI.\n");
         } else
 #endif
-#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64)
-        if (mbedtls_aesce_has_support()) {
+#if defined(MBEDTLS_VIA_PADLOCK_HAVE_CODE)
+        if (mbedtls_padlock_has_support(MBEDTLS_PADLOCK_ACE)) {
+            mbedtls_printf("  AES note: using VIA Padlock.\n");
+        } else
+#endif
+#if defined(MBEDTLS_AESCE_HAVE_CODE)
+        if (MBEDTLS_AESCE_HAS_SUPPORT()) {
             mbedtls_printf("  AES note: using AESCE.\n");
         } else
 #endif
-        mbedtls_printf("  AES note: built-in implementation.\n");
+        {
+#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
+            mbedtls_printf("  AES note: built-in implementation.\n");
+#endif
+        }
 #endif /* MBEDTLS_AES_ALT */
     }
 
     /*
      * ECB mode
      */
-    for (i = 0; i < 6; i++) {
-        u = i >> 1;
-        keybits = 128 + u * 64;
-        mode = i & 1;
+    {
+        static const int num_tests =
+            sizeof(aes_test_ecb_enc) / sizeof(*aes_test_ecb_enc);
 
-        if (verbose != 0) {
-            mbedtls_printf("  AES-ECB-%3u (%s): ", keybits,
-                           (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
-        }
+        for (i = 0; i < num_tests << 1; i++) {
+            u = i >> 1;
+            keybits = 128 + u * 64;
+            mode = i & 1;
 
-        memset(buf, 0, 16);
+            if (verbose != 0) {
+                mbedtls_printf("  AES-ECB-%3u (%s): ", keybits,
+                               (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
+            }
+#if defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                if (verbose != 0) {
+                    mbedtls_printf("skipped\n");
+                }
+                continue;
+            }
+#endif
 
-        if (mode == MBEDTLS_AES_DECRYPT) {
-            ret = mbedtls_aes_setkey_dec(&ctx, key, keybits);
-            aes_tests = aes_test_ecb_dec[u];
-        } else {
-            ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
-            aes_tests = aes_test_ecb_enc[u];
-        }
+            memset(buf, 0, 16);
 
-        /*
-         * AES-192 is an optional feature that may be unavailable when
-         * there is an alternative underlying implementation i.e. when
-         * MBEDTLS_AES_ALT is defined.
-         */
-        if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
-            mbedtls_printf("skipped\n");
-            continue;
-        } else if (ret != 0) {
-            goto exit;
-        }
+#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                ret = mbedtls_aes_setkey_dec(&ctx, key, keybits);
+                aes_tests = aes_test_ecb_dec[u];
+            } else
+#endif
+            {
+                ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
+                aes_tests = aes_test_ecb_enc[u];
+            }
 
-        for (j = 0; j < 10000; j++) {
-            ret = mbedtls_aes_crypt_ecb(&ctx, mode, buf, buf);
-            if (ret != 0) {
+            /*
+             * AES-192 is an optional feature that may be unavailable when
+             * there is an alternative underlying implementation i.e. when
+             * MBEDTLS_AES_ALT is defined.
+             */
+            if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
+                mbedtls_printf("skipped\n");
+                continue;
+            } else if (ret != 0) {
                 goto exit;
             }
+
+            for (j = 0; j < 10000; j++) {
+                ret = mbedtls_aes_crypt_ecb(&ctx, mode, buf, buf);
+                if (ret != 0) {
+                    goto exit;
+                }
+            }
+
+            if (memcmp(buf, aes_tests, 16) != 0) {
+                ret = 1;
+                goto exit;
+            }
+
+            if (verbose != 0) {
+                mbedtls_printf("passed\n");
+            }
         }
 
-        if (memcmp(buf, aes_tests, 16) != 0) {
-            ret = 1;
-            goto exit;
-        }
-
         if (verbose != 0) {
-            mbedtls_printf("passed\n");
+            mbedtls_printf("\n");
         }
     }
 
-    if (verbose != 0) {
-        mbedtls_printf("\n");
-    }
-
 #if defined(MBEDTLS_CIPHER_MODE_CBC)
     /*
      * CBC mode
      */
-    for (i = 0; i < 6; i++) {
-        u = i >> 1;
-        keybits = 128 + u * 64;
-        mode = i & 1;
+    {
+        static const int num_tests =
+            sizeof(aes_test_cbc_dec) / sizeof(*aes_test_cbc_dec);
 
-        if (verbose != 0) {
-            mbedtls_printf("  AES-CBC-%3u (%s): ", keybits,
-                           (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
-        }
+        for (i = 0; i < num_tests << 1; i++) {
+            u = i >> 1;
+            keybits = 128 + u * 64;
+            mode = i & 1;
 
-        memset(iv, 0, 16);
-        memset(prv, 0, 16);
-        memset(buf, 0, 16);
-
-        if (mode == MBEDTLS_AES_DECRYPT) {
-            ret = mbedtls_aes_setkey_dec(&ctx, key, keybits);
-            aes_tests = aes_test_cbc_dec[u];
-        } else {
-            ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
-            aes_tests = aes_test_cbc_enc[u];
-        }
-
-        /*
-         * AES-192 is an optional feature that may be unavailable when
-         * there is an alternative underlying implementation i.e. when
-         * MBEDTLS_AES_ALT is defined.
-         */
-        if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
-            mbedtls_printf("skipped\n");
-            continue;
-        } else if (ret != 0) {
-            goto exit;
-        }
-
-        for (j = 0; j < 10000; j++) {
-            if (mode == MBEDTLS_AES_ENCRYPT) {
-                unsigned char tmp[16];
-
-                memcpy(tmp, prv, 16);
-                memcpy(prv, buf, 16);
-                memcpy(buf, tmp, 16);
+            if (verbose != 0) {
+                mbedtls_printf("  AES-CBC-%3u (%s): ", keybits,
+                               (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
             }
 
-            ret = mbedtls_aes_crypt_cbc(&ctx, mode, 16, iv, buf, buf);
-            if (ret != 0) {
+            memset(iv, 0, 16);
+            memset(prv, 0, 16);
+            memset(buf, 0, 16);
+
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                ret = mbedtls_aes_setkey_dec(&ctx, key, keybits);
+                aes_tests = aes_test_cbc_dec[u];
+            } else {
+                ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
+                aes_tests = aes_test_cbc_enc[u];
+            }
+
+            /*
+             * AES-192 is an optional feature that may be unavailable when
+             * there is an alternative underlying implementation i.e. when
+             * MBEDTLS_AES_ALT is defined.
+             */
+            if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
+                mbedtls_printf("skipped\n");
+                continue;
+            } else if (ret != 0) {
                 goto exit;
             }
 
-        }
+            for (j = 0; j < 10000; j++) {
+                if (mode == MBEDTLS_AES_ENCRYPT) {
+                    unsigned char tmp[16];
 
-        if (memcmp(buf, aes_tests, 16) != 0) {
-            ret = 1;
-            goto exit;
+                    memcpy(tmp, prv, 16);
+                    memcpy(prv, buf, 16);
+                    memcpy(buf, tmp, 16);
+                }
+
+                ret = mbedtls_aes_crypt_cbc(&ctx, mode, 16, iv, buf, buf);
+                if (ret != 0) {
+                    goto exit;
+                }
+
+            }
+
+            if (memcmp(buf, aes_tests, 16) != 0) {
+                ret = 1;
+                goto exit;
+            }
+
+            if (verbose != 0) {
+                mbedtls_printf("passed\n");
+            }
         }
 
         if (verbose != 0) {
-            mbedtls_printf("passed\n");
+            mbedtls_printf("\n");
         }
     }
-
-    if (verbose != 0) {
-        mbedtls_printf("\n");
-    }
 #endif /* MBEDTLS_CIPHER_MODE_CBC */
 
 #if defined(MBEDTLS_CIPHER_MODE_CFB)
     /*
      * CFB128 mode
      */
-    for (i = 0; i < 6; i++) {
-        u = i >> 1;
-        keybits = 128 + u * 64;
-        mode = i & 1;
+    {
+        static const int num_tests =
+            sizeof(aes_test_cfb128_key) / sizeof(*aes_test_cfb128_key);
 
-        if (verbose != 0) {
-            mbedtls_printf("  AES-CFB128-%3u (%s): ", keybits,
-                           (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
-        }
+        for (i = 0; i < num_tests << 1; i++) {
+            u = i >> 1;
+            keybits = 128 + u * 64;
+            mode = i & 1;
 
-        memcpy(iv,  aes_test_cfb128_iv, 16);
-        memcpy(key, aes_test_cfb128_key[u], keybits / 8);
+            if (verbose != 0) {
+                mbedtls_printf("  AES-CFB128-%3u (%s): ", keybits,
+                               (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
+            }
 
-        offset = 0;
-        ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
-        /*
-         * AES-192 is an optional feature that may be unavailable when
-         * there is an alternative underlying implementation i.e. when
-         * MBEDTLS_AES_ALT is defined.
-         */
-        if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
-            mbedtls_printf("skipped\n");
-            continue;
-        } else if (ret != 0) {
-            goto exit;
-        }
+            memcpy(iv,  aes_test_cfb128_iv, 16);
+            memcpy(key, aes_test_cfb128_key[u], keybits / 8);
 
-        if (mode == MBEDTLS_AES_DECRYPT) {
-            memcpy(buf, aes_test_cfb128_ct[u], 64);
-            aes_tests = aes_test_cfb128_pt;
-        } else {
-            memcpy(buf, aes_test_cfb128_pt, 64);
-            aes_tests = aes_test_cfb128_ct[u];
-        }
+            offset = 0;
+            ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
+            /*
+             * AES-192 is an optional feature that may be unavailable when
+             * there is an alternative underlying implementation i.e. when
+             * MBEDTLS_AES_ALT is defined.
+             */
+            if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
+                mbedtls_printf("skipped\n");
+                continue;
+            } else if (ret != 0) {
+                goto exit;
+            }
 
-        ret = mbedtls_aes_crypt_cfb128(&ctx, mode, 64, &offset, iv, buf, buf);
-        if (ret != 0) {
-            goto exit;
-        }
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                memcpy(buf, aes_test_cfb128_ct[u], 64);
+                aes_tests = aes_test_cfb128_pt;
+            } else {
+                memcpy(buf, aes_test_cfb128_pt, 64);
+                aes_tests = aes_test_cfb128_ct[u];
+            }
 
-        if (memcmp(buf, aes_tests, 64) != 0) {
-            ret = 1;
-            goto exit;
+            ret = mbedtls_aes_crypt_cfb128(&ctx, mode, 64, &offset, iv, buf, buf);
+            if (ret != 0) {
+                goto exit;
+            }
+
+            if (memcmp(buf, aes_tests, 64) != 0) {
+                ret = 1;
+                goto exit;
+            }
+
+            if (verbose != 0) {
+                mbedtls_printf("passed\n");
+            }
         }
 
         if (verbose != 0) {
-            mbedtls_printf("passed\n");
+            mbedtls_printf("\n");
         }
     }
-
-    if (verbose != 0) {
-        mbedtls_printf("\n");
-    }
 #endif /* MBEDTLS_CIPHER_MODE_CFB */
 
 #if defined(MBEDTLS_CIPHER_MODE_OFB)
     /*
      * OFB mode
      */
-    for (i = 0; i < 6; i++) {
-        u = i >> 1;
-        keybits = 128 + u * 64;
-        mode = i & 1;
+    {
+        static const int num_tests =
+            sizeof(aes_test_ofb_key) / sizeof(*aes_test_ofb_key);
 
-        if (verbose != 0) {
-            mbedtls_printf("  AES-OFB-%3u (%s): ", keybits,
-                           (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
-        }
+        for (i = 0; i < num_tests << 1; i++) {
+            u = i >> 1;
+            keybits = 128 + u * 64;
+            mode = i & 1;
 
-        memcpy(iv,  aes_test_ofb_iv, 16);
-        memcpy(key, aes_test_ofb_key[u], keybits / 8);
+            if (verbose != 0) {
+                mbedtls_printf("  AES-OFB-%3u (%s): ", keybits,
+                               (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
+            }
 
-        offset = 0;
-        ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
-        /*
-         * AES-192 is an optional feature that may be unavailable when
-         * there is an alternative underlying implementation i.e. when
-         * MBEDTLS_AES_ALT is defined.
-         */
-        if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
-            mbedtls_printf("skipped\n");
-            continue;
-        } else if (ret != 0) {
-            goto exit;
-        }
+            memcpy(iv,  aes_test_ofb_iv, 16);
+            memcpy(key, aes_test_ofb_key[u], keybits / 8);
 
-        if (mode == MBEDTLS_AES_DECRYPT) {
-            memcpy(buf, aes_test_ofb_ct[u], 64);
-            aes_tests = aes_test_ofb_pt;
-        } else {
-            memcpy(buf, aes_test_ofb_pt, 64);
-            aes_tests = aes_test_ofb_ct[u];
-        }
+            offset = 0;
+            ret = mbedtls_aes_setkey_enc(&ctx, key, keybits);
+            /*
+             * AES-192 is an optional feature that may be unavailable when
+             * there is an alternative underlying implementation i.e. when
+             * MBEDTLS_AES_ALT is defined.
+             */
+            if (ret == MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED && keybits == 192) {
+                mbedtls_printf("skipped\n");
+                continue;
+            } else if (ret != 0) {
+                goto exit;
+            }
 
-        ret = mbedtls_aes_crypt_ofb(&ctx, 64, &offset, iv, buf, buf);
-        if (ret != 0) {
-            goto exit;
-        }
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                memcpy(buf, aes_test_ofb_ct[u], 64);
+                aes_tests = aes_test_ofb_pt;
+            } else {
+                memcpy(buf, aes_test_ofb_pt, 64);
+                aes_tests = aes_test_ofb_ct[u];
+            }
 
-        if (memcmp(buf, aes_tests, 64) != 0) {
-            ret = 1;
-            goto exit;
+            ret = mbedtls_aes_crypt_ofb(&ctx, 64, &offset, iv, buf, buf);
+            if (ret != 0) {
+                goto exit;
+            }
+
+            if (memcmp(buf, aes_tests, 64) != 0) {
+                ret = 1;
+                goto exit;
+            }
+
+            if (verbose != 0) {
+                mbedtls_printf("passed\n");
+            }
         }
 
         if (verbose != 0) {
-            mbedtls_printf("passed\n");
+            mbedtls_printf("\n");
         }
     }
-
-    if (verbose != 0) {
-        mbedtls_printf("\n");
-    }
 #endif /* MBEDTLS_CIPHER_MODE_OFB */
 
 #if defined(MBEDTLS_CIPHER_MODE_CTR)
     /*
      * CTR mode
      */
-    for (i = 0; i < 6; i++) {
-        u = i >> 1;
-        mode = i & 1;
+    {
+        static const int num_tests =
+            sizeof(aes_test_ctr_key) / sizeof(*aes_test_ctr_key);
 
-        if (verbose != 0) {
-            mbedtls_printf("  AES-CTR-128 (%s): ",
-                           (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
-        }
+        for (i = 0; i < num_tests << 1; i++) {
+            u = i >> 1;
+            mode = i & 1;
 
-        memcpy(nonce_counter, aes_test_ctr_nonce_counter[u], 16);
-        memcpy(key, aes_test_ctr_key[u], 16);
+            if (verbose != 0) {
+                mbedtls_printf("  AES-CTR-128 (%s): ",
+                               (mode == MBEDTLS_AES_DECRYPT) ? "dec" : "enc");
+            }
 
-        offset = 0;
-        if ((ret = mbedtls_aes_setkey_enc(&ctx, key, 128)) != 0) {
-            goto exit;
-        }
+            memcpy(nonce_counter, aes_test_ctr_nonce_counter[u], 16);
+            memcpy(key, aes_test_ctr_key[u], 16);
 
-        len = aes_test_ctr_len[u];
+            offset = 0;
+            if ((ret = mbedtls_aes_setkey_enc(&ctx, key, 128)) != 0) {
+                goto exit;
+            }
 
-        if (mode == MBEDTLS_AES_DECRYPT) {
-            memcpy(buf, aes_test_ctr_ct[u], len);
-            aes_tests = aes_test_ctr_pt[u];
-        } else {
-            memcpy(buf, aes_test_ctr_pt[u], len);
-            aes_tests = aes_test_ctr_ct[u];
-        }
+            len = aes_test_ctr_len[u];
 
-        ret = mbedtls_aes_crypt_ctr(&ctx, len, &offset, nonce_counter,
-                                    stream_block, buf, buf);
-        if (ret != 0) {
-            goto exit;
-        }
+            if (mode == MBEDTLS_AES_DECRYPT) {
+                memcpy(buf, aes_test_ctr_ct[u], len);
+                aes_tests = aes_test_ctr_pt[u];
+            } else {
+                memcpy(buf, aes_test_ctr_pt[u], len);
+                aes_tests = aes_test_ctr_ct[u];
+            }
 
-        if (memcmp(buf, aes_tests, len) != 0) {
-            ret = 1;
-            goto exit;
-        }
+            ret = mbedtls_aes_crypt_ctr(&ctx, len, &offset, nonce_counter,
+                                        stream_block, buf, buf);
+            if (ret != 0) {
+                goto exit;
+            }
 
-        if (verbose != 0) {
-            mbedtls_printf("passed\n");
+            if (memcmp(buf, aes_tests, len) != 0) {
+                ret = 1;
+                goto exit;
+            }
+
+            if (verbose != 0) {
+                mbedtls_printf("passed\n");
+            }
         }
     }
 
@@ -2094,14 +2210,14 @@
 #endif /* MBEDTLS_CIPHER_MODE_CTR */
 
 #if defined(MBEDTLS_CIPHER_MODE_XTS)
+    /*
+     * XTS mode
+     */
     {
         static const int num_tests =
             sizeof(aes_test_xts_key) / sizeof(*aes_test_xts_key);
         mbedtls_aes_xts_context ctx_xts;
 
-        /*
-         * XTS mode
-         */
         mbedtls_aes_xts_init(&ctx_xts);
 
         for (i = 0; i < num_tests << 1; i++) {