perf(cpus): make reset errata do fewer branches
Errata application is painful for performance. For a start, it's done
when the core has just come out of reset, which means branch predictors
and caches will be empty so a branch to a workaround function must be
fetched from memory and that round trip is very slow. Then it also runs
with the I-cache off, which means that the loop to iterate over the
workarounds must also be fetched from memory on each iteration.
We can remove both branches. First, we can simply apply every erratum
directly instead of defining a workaround function and jumping to it.
Currently, no errata that need to be applied at both reset and runtime,
with the same workaround function, exist. If the need arose in future,
this should be achievable with a reset + runtime wrapper combo.
Then, we can construct a function that applies each erratum linearly
instead of looping over the list. If this function is part of the reset
function, then the only "far" branches at reset will be for the checker
functions. Importantly, this mitigates the slowdown even when an erratum
is disabled.
The result is ~50% speedup on N1SDP and ~20% on AArch64 Juno on wakeup
from PSCI calls that end in powerdown. This is roughly back to the
baseline of v2.9, before the errata framework regressed on performance
(or a little better). It is important to note that there are other
slowdowns since then that remain unknown.
Change-Id: Ie4d5288a331b11fd648e5c4a0b652b74160b07b9
Signed-off-by: Boyan Karatotev <boyan.karatotev@arm.com>
diff --git a/lib/cpus/aarch64/aem_generic.S b/lib/cpus/aarch64/aem_generic.S
index 9002da6..9843943 100644
--- a/lib/cpus/aarch64/aem_generic.S
+++ b/lib/cpus/aarch64/aem_generic.S
@@ -8,6 +8,8 @@
#include <asm_macros.S>
#include <cpu_macros.S>
+cpu_reset_prologue aem_generic
+
func aem_generic_core_pwr_dwn
/* ---------------------------------------------
* Disable the Data Cache.
diff --git a/lib/cpus/aarch64/cortex_a35.S b/lib/cpus/aarch64/cortex_a35.S
index c3d8c8d..40e6200 100644
--- a/lib/cpus/aarch64/cortex_a35.S
+++ b/lib/cpus/aarch64/cortex_a35.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2016-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -11,6 +11,7 @@
#include <cpu_macros.S>
#include <plat_macros.S>
+cpu_reset_prologue cortex_a35
/* ---------------------------------------------
* Disable L1 data cache and unified L2 cache
* ---------------------------------------------
diff --git a/lib/cpus/aarch64/cortex_a510.S b/lib/cpus/aarch64/cortex_a510.S
index b399bbc..cbeeb2b 100644
--- a/lib/cpus/aarch64/cortex_a510.S
+++ b/lib/cpus/aarch64/cortex_a510.S
@@ -22,6 +22,8 @@
#error "Cortex-A510 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a510
+
workaround_reset_start cortex_a510, ERRATUM(1922240), ERRATA_A510_1922240
/* Apply the workaround by setting IMP_CMPXACTLR_EL1[11:10] = 0b11. */
sysreg_bitfield_insert CORTEX_A510_CMPXACTLR_EL1, CORTEX_A510_CMPXACTLR_EL1_SNPPREFERUNIQUE_DISABLE, \
diff --git a/lib/cpus/aarch64/cortex_a520.S b/lib/cpus/aarch64/cortex_a520.S
index d9e654b..7d63dbb 100644
--- a/lib/cpus/aarch64/cortex_a520.S
+++ b/lib/cpus/aarch64/cortex_a520.S
@@ -24,6 +24,8 @@
#error "Cortex A520 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a520
+
workaround_reset_start cortex_a520, ERRATUM(2630792), ERRATA_A520_2630792
sysreg_bit_set CORTEX_A520_CPUACTLR_EL1, BIT(38)
workaround_reset_end cortex_a520, ERRATUM(2630792)
diff --git a/lib/cpus/aarch64/cortex_a53.S b/lib/cpus/aarch64/cortex_a53.S
index 4a5b318..dbfff87 100644
--- a/lib/cpus/aarch64/cortex_a53.S
+++ b/lib/cpus/aarch64/cortex_a53.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2014-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -12,6 +12,8 @@
#include <plat_macros.S>
#include <lib/cpus/errata.h>
+cpu_reset_prologue cortex_a53
+
/* ---------------------------------------------
* Disable L1 data cache and unified L2 cache
* ---------------------------------------------
@@ -36,12 +38,12 @@
/* Due to the nature of the errata it is applied unconditionally when chosen */
check_erratum_ls cortex_a53, ERRATUM(819472), CPU_REV(0, 1)
/* erratum workaround is interleaved with generic code */
-add_erratum_entry cortex_a53, ERRATUM(819472), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(819472), ERRATUM_ALWAYS_CHOSEN
/* Due to the nature of the errata it is applied unconditionally when chosen */
check_erratum_ls cortex_a53, ERRATUM(824069), CPU_REV(0, 2)
/* erratum workaround is interleaved with generic code */
-add_erratum_entry cortex_a53, ERRATUM(824069), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(824069), ERRATUM_ALWAYS_CHOSEN
workaround_reset_start cortex_a53, ERRATUM(826319), ERRATA_A53_826319
mrs x1, CORTEX_A53_L2ACTLR_EL1
@@ -55,7 +57,7 @@
/* Due to the nature of the errata it is applied unconditionally when chosen */
check_erratum_ls cortex_a53, ERRATUM(827319), CPU_REV(0, 2)
/* erratum workaround is interleaved with generic code */
-add_erratum_entry cortex_a53, ERRATUM(827319), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(827319), ERRATUM_ALWAYS_CHOSEN
check_erratum_custom_start cortex_a53, ERRATUM(835769)
cmp x0, CPU_REV(0, 4)
@@ -78,7 +80,7 @@
check_erratum_custom_end cortex_a53, ERRATUM(835769)
/* workaround at build time */
-add_erratum_entry cortex_a53, ERRATUM(835769), ERRATA_A53_835769, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(835769), ERRATA_A53_835769
/*
* Disable the cache non-temporal hint.
@@ -114,7 +116,7 @@
check_erratum_custom_end cortex_a53, ERRATUM(843419)
/* workaround at build time */
-add_erratum_entry cortex_a53, ERRATUM(843419), ERRATA_A53_843419, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(843419), ERRATA_A53_843419
/*
* Earlier revisions of the core are affected as well, but don't
@@ -131,7 +133,7 @@
check_erratum_chosen cortex_a53, ERRATUM(1530924), ERRATA_A53_1530924
/* erratum has no workaround in the cpu. Generic code must take care */
-add_erratum_entry cortex_a53, ERRATUM(1530924), ERRATA_A53_1530924, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a53, ERRATUM(1530924), ERRATA_A53_1530924
cpu_reset_func_start cortex_a53
/* Enable the SMP bit. */
diff --git a/lib/cpus/aarch64/cortex_a55.S b/lib/cpus/aarch64/cortex_a55.S
index 2267d66..cf91431 100644
--- a/lib/cpus/aarch64/cortex_a55.S
+++ b/lib/cpus/aarch64/cortex_a55.S
@@ -20,6 +20,8 @@
.globl cortex_a55_reset_func
.globl cortex_a55_core_pwr_dwn
+cpu_reset_prologue cortex_a55
+
workaround_reset_start cortex_a55, ERRATUM(798953), ERRATA_DSU_798953
errata_dsu_798953_wa_impl
workaround_reset_end cortex_a55, ERRATUM(798953)
@@ -111,7 +113,7 @@
check_erratum_chosen cortex_a55, ERRATUM(1530923), ERRATA_A55_1530923
/* erratum has no workaround in the cpu. Generic code must take care */
-add_erratum_entry cortex_a55, ERRATUM(1530923), ERRATA_A55_1530923, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a55, ERRATUM(1530923), ERRATA_A55_1530923
cpu_reset_func_start cortex_a55
cpu_reset_func_end cortex_a55
diff --git a/lib/cpus/aarch64/cortex_a57.S b/lib/cpus/aarch64/cortex_a57.S
index aac9c51..adacc5c 100644
--- a/lib/cpus/aarch64/cortex_a57.S
+++ b/lib/cpus/aarch64/cortex_a57.S
@@ -13,6 +13,8 @@
#include <cpu_macros.S>
#include <plat_macros.S>
+cpu_reset_prologue cortex_a57
+
/* ---------------------------------------------
* Disable L1 data cache and unified L2 cache
* ---------------------------------------------
@@ -81,7 +83,7 @@
/* erratum always worked around, but report it correctly */
check_erratum_ls cortex_a57, ERRATUM(813419), CPU_REV(0, 0)
-add_erratum_entry cortex_a57, ERRATUM(813419), ERRATUM_ALWAYS_CHOSEN, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a57, ERRATUM(813419), ERRATUM_ALWAYS_CHOSEN
workaround_reset_start cortex_a57, ERRATUM(813420), ERRATA_A57_813420
sysreg_bit_set CORTEX_A57_CPUACTLR_EL1, CORTEX_A57_CPUACTLR_EL1_DCC_AS_DCCI
@@ -150,7 +152,7 @@
check_erratum_chosen cortex_a57, ERRATUM(1319537), ERRATA_A57_1319537
/* erratum has no workaround in the cpu. Generic code must take care */
-add_erratum_entry cortex_a57, ERRATUM(1319537), ERRATA_A57_1319537, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a57, ERRATUM(1319537), ERRATA_A57_1319537
workaround_reset_start cortex_a57, CVE(2017, 5715), WORKAROUND_CVE_2017_5715
#if IMAGE_BL31
diff --git a/lib/cpus/aarch64/cortex_a65.S b/lib/cpus/aarch64/cortex_a65.S
index 064e6f0..3c32adb 100644
--- a/lib/cpus/aarch64/cortex_a65.S
+++ b/lib/cpus/aarch64/cortex_a65.S
@@ -23,6 +23,8 @@
#error "Cortex-A65 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a65
+
workaround_reset_start cortex_a65, ERRATUM(936184), ERRATA_DSU_936184
errata_dsu_936184_wa_impl
workaround_reset_end cortex_a65, ERRATUM(936184)
diff --git a/lib/cpus/aarch64/cortex_a65ae.S b/lib/cpus/aarch64/cortex_a65ae.S
index d2f9e49..f1a63b0 100644
--- a/lib/cpus/aarch64/cortex_a65ae.S
+++ b/lib/cpus/aarch64/cortex_a65ae.S
@@ -23,6 +23,8 @@
#error "Cortex-A65AE supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a65ae
+
workaround_reset_start cortex_a65ae, ERRATUM(936184), ERRATA_DSU_936184
errata_dsu_936184_wa_impl
workaround_reset_end cortex_a65ae, ERRATUM(936184)
diff --git a/lib/cpus/aarch64/cortex_a710.S b/lib/cpus/aarch64/cortex_a710.S
index 17163a1..cb24aa1 100644
--- a/lib/cpus/aarch64/cortex_a710.S
+++ b/lib/cpus/aarch64/cortex_a710.S
@@ -29,6 +29,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A710_BHB_LOOP_COUNT, cortex_a710
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a710
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_a710, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_A710_CPUECTLR_EL1, BIT(46)
@@ -223,7 +225,7 @@
check_erratum_chosen cortex_a710, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
-add_erratum_entry cortex_a710, ERRATUM(3701772), ERRATA_A710_3701772, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a710, ERRATUM(3701772), ERRATA_A710_3701772
check_erratum_ls cortex_a710, ERRATUM(3701772), CPU_REV(2, 1)
diff --git a/lib/cpus/aarch64/cortex_a715.S b/lib/cpus/aarch64/cortex_a715.S
index fbc73ed..e50764d 100644
--- a/lib/cpus/aarch64/cortex_a715.S
+++ b/lib/cpus/aarch64/cortex_a715.S
@@ -28,6 +28,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A715_BHB_LOOP_COUNT, cortex_a715
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a715
+
workaround_reset_start cortex_a715, ERRATUM(2331818), ERRATA_A715_2331818
sysreg_bit_set CORTEX_A715_CPUACTLR2_EL1, BIT(20)
workaround_reset_end cortex_a715, ERRATUM(2331818)
@@ -129,7 +131,7 @@
check_erratum_chosen cortex_a715, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
-add_erratum_entry cortex_a715, ERRATUM(3699560), ERRATA_A715_3699560, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a715, ERRATUM(3699560), ERRATA_A715_3699560
check_erratum_ls cortex_a715, ERRATUM(3699560), CPU_REV(1, 3)
diff --git a/lib/cpus/aarch64/cortex_a72.S b/lib/cpus/aarch64/cortex_a72.S
index c300ea7..fee28ee 100644
--- a/lib/cpus/aarch64/cortex_a72.S
+++ b/lib/cpus/aarch64/cortex_a72.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2015-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -15,6 +15,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A72_BHB_LOOP_COUNT, cortex_a72
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a72
+
/* ---------------------------------------------
* Disable L1 data cache and unified L2 cache
* ---------------------------------------------
@@ -92,7 +94,7 @@
/* Due to the nature of the errata it is applied unconditionally when chosen */
check_erratum_chosen cortex_a72, ERRATUM(1319367), ERRATA_A72_1319367
/* erratum workaround is interleaved with generic code */
-add_erratum_entry cortex_a72, ERRATUM(1319367), ERRATA_A72_1319367, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a72, ERRATUM(1319367), ERRATA_A72_1319367
workaround_reset_start cortex_a72, CVE(2017, 5715), WORKAROUND_CVE_2017_5715
#if IMAGE_BL31
diff --git a/lib/cpus/aarch64/cortex_a720.S b/lib/cpus/aarch64/cortex_a720.S
index ab2c12f..2991f93 100644
--- a/lib/cpus/aarch64/cortex_a720.S
+++ b/lib/cpus/aarch64/cortex_a720.S
@@ -22,6 +22,8 @@
#error "Cortex A720 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a720
+
.global check_erratum_cortex_a720_3699561
#if WORKAROUND_CVE_2022_23960
@@ -74,7 +76,7 @@
check_erratum_chosen cortex_a720, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
-add_erratum_entry cortex_a720, ERRATUM(3699561), ERRATA_A720_3699561, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a720, ERRATUM(3699561), ERRATA_A720_3699561
check_erratum_ls cortex_a720, ERRATUM(3699561), CPU_REV(0, 2)
diff --git a/lib/cpus/aarch64/cortex_a720_ae.S b/lib/cpus/aarch64/cortex_a720_ae.S
index 57a5030..c72a29e 100644
--- a/lib/cpus/aarch64/cortex_a720_ae.S
+++ b/lib/cpus/aarch64/cortex_a720_ae.S
@@ -21,9 +21,11 @@
#error "Cortex-A720AE supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a720_ae
+
.global check_erratum_cortex_a720_ae_3699562
-add_erratum_entry cortex_a720_ae, ERRATUM(3699562), ERRATA_A720_AE_3699562, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a720_ae, ERRATUM(3699562), ERRATA_A720_AE_3699562
check_erratum_ls cortex_a720_ae, ERRATUM(3699562), CPU_REV(0, 0)
diff --git a/lib/cpus/aarch64/cortex_a725.S b/lib/cpus/aarch64/cortex_a725.S
index c4d6034..a8c0db2 100644
--- a/lib/cpus/aarch64/cortex_a725.S
+++ b/lib/cpus/aarch64/cortex_a725.S
@@ -21,9 +21,11 @@
#error "Cortex-A725 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a725
+
.global check_erratum_cortex_a725_3699564
-add_erratum_entry cortex_a725, ERRATUM(3699564), ERRATA_A725_3699564, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a725, ERRATUM(3699564), ERRATA_A725_3699564
check_erratum_ls cortex_a725, ERRATUM(3699564), CPU_REV(0, 1)
diff --git a/lib/cpus/aarch64/cortex_a73.S b/lib/cpus/aarch64/cortex_a73.S
index 2130ceb..d1fc6d4 100644
--- a/lib/cpus/aarch64/cortex_a73.S
+++ b/lib/cpus/aarch64/cortex_a73.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2016-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -10,6 +10,8 @@
#include <cpu_macros.S>
#include <plat_macros.S>
+cpu_reset_prologue cortex_a73
+
/* ---------------------------------------------
* Disable L1 data cache
* ---------------------------------------------
diff --git a/lib/cpus/aarch64/cortex_a75.S b/lib/cpus/aarch64/cortex_a75.S
index 336e00e..13599ca 100644
--- a/lib/cpus/aarch64/cortex_a75.S
+++ b/lib/cpus/aarch64/cortex_a75.S
@@ -18,6 +18,8 @@
#error "Cortex-A75 must be compiled with HW_ASSISTED_COHERENCY enabled"
#endif
+cpu_reset_prologue cortex_a75
+
workaround_reset_start cortex_a75, ERRATUM(764081), ERRATA_A75_764081
sysreg_bit_set sctlr_el3, SCTLR_IESB_BIT
workaround_reset_end cortex_a75, ERRATUM(764081)
diff --git a/lib/cpus/aarch64/cortex_a76.S b/lib/cpus/aarch64/cortex_a76.S
index 589edce..822ef05 100644
--- a/lib/cpus/aarch64/cortex_a76.S
+++ b/lib/cpus/aarch64/cortex_a76.S
@@ -30,6 +30,8 @@
#define ESR_EL3_A64_SMC0 0x5e000000
#define ESR_EL3_A32_SMC0 0x4e000000
+cpu_reset_prologue cortex_a76
+
#if DYNAMIC_WORKAROUND_CVE_2018_3639
/*
* This macro applies the mitigation for CVE-2018-3639.
@@ -428,7 +430,7 @@
check_erratum_chosen cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
/* erratum has no workaround in the cpu. Generic code must take care */
-add_erratum_entry cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960, NO_APPLY_AT_RESET
+add_erratum_entry cortex_a76, CVE(2022, 23960), WORKAROUND_CVE_2022_23960
workaround_reset_start cortex_a76, ERRATUM(798953), ERRATA_DSU_798953
errata_dsu_798953_wa_impl
diff --git a/lib/cpus/aarch64/cortex_a76ae.S b/lib/cpus/aarch64/cortex_a76ae.S
index 2fe3dbc..54af9a0 100644
--- a/lib/cpus/aarch64/cortex_a76ae.S
+++ b/lib/cpus/aarch64/cortex_a76ae.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2019-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -21,6 +21,8 @@
#error "Cortex-A76AE supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_a76ae
+
#if WORKAROUND_CVE_2022_23960
wa_cve_2022_23960_bhb_vector_table CORTEX_A76AE_BHB_LOOP_COUNT, cortex_a76ae
#endif /* WORKAROUND_CVE_2022_23960 */
diff --git a/lib/cpus/aarch64/cortex_a77.S b/lib/cpus/aarch64/cortex_a77.S
index 1759b77..7fb964d 100644
--- a/lib/cpus/aarch64/cortex_a77.S
+++ b/lib/cpus/aarch64/cortex_a77.S
@@ -17,6 +17,8 @@
#error "Cortex-A77 must be compiled with HW_ASSISTED_COHERENCY enabled"
#endif
+cpu_reset_prologue cortex_a77
+
/* 64-bit only core */
#if CTX_INCLUDE_AARCH32_REGS == 1
#error "Cortex-A77 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
diff --git a/lib/cpus/aarch64/cortex_a78.S b/lib/cpus/aarch64/cortex_a78.S
index 9f2ffdf..a66214b 100644
--- a/lib/cpus/aarch64/cortex_a78.S
+++ b/lib/cpus/aarch64/cortex_a78.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2019-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -24,6 +24,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A78_BHB_LOOP_COUNT, cortex_a78
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a78
+
/* Disable hardware page aggregation.Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_a78, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_A78_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_a78_ae.S b/lib/cpus/aarch64/cortex_a78_ae.S
index 7fa1f9b..c537967 100644
--- a/lib/cpus/aarch64/cortex_a78_ae.S
+++ b/lib/cpus/aarch64/cortex_a78_ae.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2019-2025, Arm Limited. All rights reserved.
* Copyright (c) 2021-2023, NVIDIA Corporation. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -22,6 +22,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A78_AE_BHB_LOOP_COUNT, cortex_a78_ae
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a78_ae
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_a78_ae, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_A78_AE_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_a78c.S b/lib/cpus/aarch64/cortex_a78c.S
index 3f6944a..aba7d25 100644
--- a/lib/cpus/aarch64/cortex_a78c.S
+++ b/lib/cpus/aarch64/cortex_a78c.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2021-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -21,6 +21,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_A78C_BHB_LOOP_COUNT, cortex_a78c
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_a78c
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_a78c, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_A78C_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_alto.S b/lib/cpus/aarch64/cortex_alto.S
index 1422563..97192a6 100644
--- a/lib/cpus/aarch64/cortex_alto.S
+++ b/lib/cpus/aarch64/cortex_alto.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2024-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -25,6 +25,8 @@
#error "Travis needs ERRATA_SME_POWER_DOWN=1 to powerdown correctly"
#endif
+cpu_reset_prologue cortex_alto
+
cpu_reset_func_start cortex_alto
/* Disable speculative loads */
msr SSBS, xzr
diff --git a/lib/cpus/aarch64/cortex_arcadia.S b/lib/cpus/aarch64/cortex_arcadia.S
index c97d87d..ae8eb91 100644
--- a/lib/cpus/aarch64/cortex_arcadia.S
+++ b/lib/cpus/aarch64/cortex_arcadia.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2024-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -21,6 +21,8 @@
#error "Cortex-ARCADIA supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_arcadia
+
cpu_reset_func_start cortex_arcadia
/* Disable speculative loads */
msr SSBS, xzr
diff --git a/lib/cpus/aarch64/cortex_gelas.S b/lib/cpus/aarch64/cortex_gelas.S
index 709bb12..cdf6284 100644
--- a/lib/cpus/aarch64/cortex_gelas.S
+++ b/lib/cpus/aarch64/cortex_gelas.S
@@ -29,6 +29,8 @@
#error "Gelas needs ERRATA_SME_POWER_DOWN=1 to powerdown correctly"
#endif
+cpu_reset_prologue cortex_gelas
+
cpu_reset_func_start cortex_gelas
/* ----------------------------------------------------
* Disable speculative loads
diff --git a/lib/cpus/aarch64/cortex_x1.S b/lib/cpus/aarch64/cortex_x1.S
index 5bd020c..27d181a 100644
--- a/lib/cpus/aarch64/cortex_x1.S
+++ b/lib/cpus/aarch64/cortex_x1.S
@@ -23,6 +23,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_X1_BHB_LOOP_COUNT, cortex_x1
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_x1
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_x1, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_X1_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_x2.S b/lib/cpus/aarch64/cortex_x2.S
index 547c430..549beec 100644
--- a/lib/cpus/aarch64/cortex_x2.S
+++ b/lib/cpus/aarch64/cortex_x2.S
@@ -25,7 +25,7 @@
.global check_erratum_cortex_x2_3701772
-add_erratum_entry cortex_x2, ERRATUM(3701772), ERRATA_X2_3701772, NO_APPLY_AT_RESET
+add_erratum_entry cortex_x2, ERRATUM(3701772), ERRATA_X2_3701772
check_erratum_ls cortex_x2, ERRATUM(3701772), CPU_REV(2, 1)
@@ -33,6 +33,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_X2_BHB_LOOP_COUNT, cortex_x2
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_x2
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_x2, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_X2_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_x3.S b/lib/cpus/aarch64/cortex_x3.S
index f57a113..da9e306 100644
--- a/lib/cpus/aarch64/cortex_x3.S
+++ b/lib/cpus/aarch64/cortex_x3.S
@@ -24,7 +24,7 @@
.global check_erratum_cortex_x3_3701769
-add_erratum_entry cortex_x3, ERRATUM(3701769), ERRATA_X3_3701769, NO_APPLY_AT_RESET
+add_erratum_entry cortex_x3, ERRATUM(3701769), ERRATA_X3_3701769
check_erratum_ls cortex_x3, ERRATUM(3701769), CPU_REV(1, 2)
@@ -32,6 +32,8 @@
wa_cve_2022_23960_bhb_vector_table CORTEX_X3_BHB_LOOP_COUNT, cortex_x3
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue cortex_x3
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start cortex_x3, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set CORTEX_X3_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/cortex_x4.S b/lib/cpus/aarch64/cortex_x4.S
index 4b1cf91..79a8d54 100644
--- a/lib/cpus/aarch64/cortex_x4.S
+++ b/lib/cpus/aarch64/cortex_x4.S
@@ -22,6 +22,8 @@
#error "Cortex X4 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue cortex_x4
+
.global check_erratum_cortex_x4_2726228
.global check_erratum_cortex_x4_3701758
@@ -122,7 +124,7 @@
check_erratum_chosen cortex_x4, CVE(2024, 7881), WORKAROUND_CVE_2024_7881
-add_erratum_entry cortex_x4, ERRATUM(3701758), ERRATA_X4_3701758, NO_APPLY_AT_RESET
+add_erratum_entry cortex_x4, ERRATUM(3701758), ERRATA_X4_3701758
check_erratum_ls cortex_x4, ERRATUM(3701758), CPU_REV(0, 3)
diff --git a/lib/cpus/aarch64/cortex_x925.S b/lib/cpus/aarch64/cortex_x925.S
index 5974c18..7dec375 100644
--- a/lib/cpus/aarch64/cortex_x925.S
+++ b/lib/cpus/aarch64/cortex_x925.S
@@ -21,9 +21,9 @@
#error "Cortex-X925 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
-.global check_erratum_cortex_x925_3701747
+cpu_reset_prologue cortex_x925
-add_erratum_entry cortex_x925, ERRATUM(3701747), ERRATA_X925_3701747, NO_APPLY_AT_RESET
+add_erratum_entry cortex_x925, ERRATUM(3701747), ERRATA_X925_3701747
check_erratum_ls cortex_x925, ERRATUM(3701747), CPU_REV(0, 1)
diff --git a/lib/cpus/aarch64/denver.S b/lib/cpus/aarch64/denver.S
index ca250d3..64158e7 100644
--- a/lib/cpus/aarch64/denver.S
+++ b/lib/cpus/aarch64/denver.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2024, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2015-2025, Arm Limited and Contributors. All rights reserved.
* Copyright (c) 2020-2022, NVIDIA Corporation. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -13,6 +13,8 @@
#include <cpu_macros.S>
#include <plat_macros.S>
+cpu_reset_prologue denver
+
/* -------------------------------------------------
* CVE-2017-5715 mitigation
*
diff --git a/lib/cpus/aarch64/generic.S b/lib/cpus/aarch64/generic.S
index 849056f..0a10eed 100644
--- a/lib/cpus/aarch64/generic.S
+++ b/lib/cpus/aarch64/generic.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2020-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -11,6 +11,8 @@
#include <cpu_macros.S>
#include <plat_macros.S>
+cpu_reset_prologue generic
+
/* ---------------------------------------------
* Disable L1 data cache and unified L2 cache
* ---------------------------------------------
diff --git a/lib/cpus/aarch64/neoverse_e1.S b/lib/cpus/aarch64/neoverse_e1.S
index c6dd117..f37bb28 100644
--- a/lib/cpus/aarch64/neoverse_e1.S
+++ b/lib/cpus/aarch64/neoverse_e1.S
@@ -22,6 +22,8 @@
#error "Neoverse-E1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue neoverse_e1
+
workaround_reset_start neoverse_e1, ERRATUM(936184), ERRATA_DSU_936184
errata_dsu_936184_wa_impl
workaround_reset_end neoverse_e1, ERRATUM(936184)
diff --git a/lib/cpus/aarch64/neoverse_n1.S b/lib/cpus/aarch64/neoverse_n1.S
index 638d0d3..1ad9557 100644
--- a/lib/cpus/aarch64/neoverse_n1.S
+++ b/lib/cpus/aarch64/neoverse_n1.S
@@ -28,6 +28,8 @@
wa_cve_2022_23960_bhb_vector_table NEOVERSE_N1_BHB_LOOP_COUNT, neoverse_n1
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue neoverse_n1
+
workaround_reset_start neoverse_n1, ERRATUM(936184), ERRATA_DSU_936184
errata_dsu_936184_wa_impl
workaround_reset_end neoverse_n1, ERRATUM(936184)
diff --git a/lib/cpus/aarch64/neoverse_n2.S b/lib/cpus/aarch64/neoverse_n2.S
index fd6877d..9c6f01a 100644
--- a/lib/cpus/aarch64/neoverse_n2.S
+++ b/lib/cpus/aarch64/neoverse_n2.S
@@ -23,7 +23,7 @@
.global check_erratum_neoverse_n2_3701773
-add_erratum_entry neoverse_n2, ERRATUM(3701773), ERRATA_N2_3701773, NO_APPLY_AT_RESET
+add_erratum_entry neoverse_n2, ERRATUM(3701773), ERRATA_N2_3701773
check_erratum_ls neoverse_n2, ERRATUM(3701773), CPU_REV(0, 3)
@@ -31,6 +31,8 @@
wa_cve_2022_23960_bhb_vector_table NEOVERSE_N2_BHB_LOOP_COUNT, neoverse_n2
#endif /* WORKAROUND_CVE_2022_23960 */
+cpu_reset_prologue neoverse_n2
+
workaround_reset_start neoverse_n2, ERRATUM(2313941), ERRATA_DSU_2313941
errata_dsu_2313941_wa_impl
workaround_reset_end neoverse_n2, ERRATUM(2313941)
diff --git a/lib/cpus/aarch64/neoverse_n3.S b/lib/cpus/aarch64/neoverse_n3.S
index 8abcafe..1b7a3e1 100644
--- a/lib/cpus/aarch64/neoverse_n3.S
+++ b/lib/cpus/aarch64/neoverse_n3.S
@@ -21,9 +21,11 @@
#error "Neoverse-N3 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue neoverse_n3
+
.global check_erratum_neoverse_n3_3699563
-add_erratum_entry neoverse_n3, ERRATUM(3699563), ERRATA_N3_3699563, NO_APPLY_AT_RESET
+add_erratum_entry neoverse_n3, ERRATUM(3699563), ERRATA_N3_3699563
check_erratum_ls neoverse_n3, ERRATUM(3699563), CPU_REV(0, 0)
diff --git a/lib/cpus/aarch64/neoverse_v1.S b/lib/cpus/aarch64/neoverse_v1.S
index f16f7e7..e1e8214 100644
--- a/lib/cpus/aarch64/neoverse_v1.S
+++ b/lib/cpus/aarch64/neoverse_v1.S
@@ -22,6 +22,8 @@
#error "Neoverse-V1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue neoverse_v1
+
#if WORKAROUND_CVE_2022_23960
wa_cve_2022_23960_bhb_vector_table NEOVERSE_V1_BHB_LOOP_COUNT, neoverse_v1
#endif /* WORKAROUND_CVE_2022_23960 */
diff --git a/lib/cpus/aarch64/neoverse_v2.S b/lib/cpus/aarch64/neoverse_v2.S
index b43f6dd..06521ec 100644
--- a/lib/cpus/aarch64/neoverse_v2.S
+++ b/lib/cpus/aarch64/neoverse_v2.S
@@ -22,6 +22,8 @@
#error "Neoverse V2 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue neoverse_v2
+
/* Disable hardware page aggregation. Enables mitigation for `CVE-2024-5660` */
workaround_reset_start neoverse_v2, CVE(2024, 5660), WORKAROUND_CVE_2024_5660
sysreg_bit_set NEOVERSE_V2_CPUECTLR_EL1, BIT(46)
diff --git a/lib/cpus/aarch64/neoverse_v3.S b/lib/cpus/aarch64/neoverse_v3.S
index dad3709..29bfd0e 100644
--- a/lib/cpus/aarch64/neoverse_v3.S
+++ b/lib/cpus/aarch64/neoverse_v3.S
@@ -22,9 +22,11 @@
#error "Neoverse V3 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue neoverse_v3
+
.global check_erratum_neoverse_v3_3701767
-add_erratum_entry neoverse_v3, ERRATUM(3701767), ERRATA_V3_3701767, NO_APPLY_AT_RESET
+add_erratum_entry neoverse_v3, ERRATUM(3701767), ERRATA_V3_3701767
check_erratum_ls neoverse_v3, ERRATUM(3701767), CPU_REV(0, 2)
diff --git a/lib/cpus/aarch64/nevis.S b/lib/cpus/aarch64/nevis.S
index 0180ab7..0d04e65 100644
--- a/lib/cpus/aarch64/nevis.S
+++ b/lib/cpus/aarch64/nevis.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2023-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -21,6 +21,8 @@
#error "Nevis supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue nevis
+
cpu_reset_func_start nevis
/* ----------------------------------------------------
* Disable speculative loads
diff --git a/lib/cpus/aarch64/qemu_max.S b/lib/cpus/aarch64/qemu_max.S
index 529bb4f..a727379 100644
--- a/lib/cpus/aarch64/qemu_max.S
+++ b/lib/cpus/aarch64/qemu_max.S
@@ -8,6 +8,8 @@
#include <cpu_macros.S>
#include <qemu_max.h>
+cpu_reset_prologue qemu_max
+
func qemu_max_core_pwr_dwn
/* ---------------------------------------------
* Disable the Data Cache.
diff --git a/lib/cpus/aarch64/rainier.S b/lib/cpus/aarch64/rainier.S
index ea687be..9ad9362 100644
--- a/lib/cpus/aarch64/rainier.S
+++ b/lib/cpus/aarch64/rainier.S
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2024, Arm Limited. All rights reserved.
+ * Copyright (c) 2020-2025, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -21,6 +21,8 @@
#error "Rainier CPU supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
#endif
+cpu_reset_prologue rainier
+
/* --------------------------------------------------
* Disable speculative loads if Rainier supports
* SSBS.
diff --git a/lib/cpus/aarch64/travis.S b/lib/cpus/aarch64/travis.S
index 2e41668..0a95e80 100644
--- a/lib/cpus/aarch64/travis.S
+++ b/lib/cpus/aarch64/travis.S
@@ -29,6 +29,8 @@
#error "Travis needs ERRATA_SME_POWER_DOWN=1 to powerdown correctly"
#endif
+cpu_reset_prologue travis
+
cpu_reset_func_start travis
/* ----------------------------------------------------
* Disable speculative loads