feat(sme): add sme helper routines and add streaming sve support
This patch adds a few helper routines to set the Streaming SVE vector
length (SVL) in the SMCR_EL2 register, to enable/disable FEAT_SME_FA64
and to get CPU's Streaming SVE mode status.
This patch also makes SVE compare routines compatible for both normal
SVE and streaming SVE mode.
Signed-off-by: Arunachalam Ganapathy <arunachalam.ganapathy@arm.com>
Change-Id: I7294bb17a85de395a321e99241704066662c90e8
diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h
index d607029..13d7063 100644
--- a/include/lib/aarch64/arch.h
+++ b/include/lib/aarch64/arch.h
@@ -967,7 +967,13 @@
/* SMCR_ELx definitions */
#define SMCR_ELX_LEN_SHIFT U(0)
-#define SMCR_ELX_LEN_MASK U(0x1ff)
+#define SMCR_ELX_LEN_WIDTH U(4)
+/*
+ * SMCR_ELX_RAZ_LEN is defined to find the architecturally permitted SVL. This
+ * is a combination of RAZ and LEN bit fields.
+ */
+#define SMCR_ELX_RAZ_LEN_SHIFT UL(0)
+#define SMCR_ELX_RAZ_LEN_WIDTH UL(9)
#define SMCR_ELX_EZT0_BIT (U(1) << 30)
#define SMCR_ELX_FA64_BIT (U(1) << 31)
#define SMCR_EL2_RESET_VAL (SMCR_ELX_EZT0_BIT | SMCR_ELX_FA64_BIT)
diff --git a/include/lib/extensions/sme.h b/include/lib/extensions/sme.h
index 99372e4..4a7e9b7 100644
--- a/include/lib/extensions/sme.h
+++ b/include/lib/extensions/sme.h
@@ -7,9 +7,14 @@
#ifndef SME_H
#define SME_H
+#include <stdlib.h> /* for rand() */
+
#define MAX_VL (512)
#define MAX_VL_B (MAX_VL / 8)
-#define SME_SMCR_LEN_MAX U(0x1FF)
+#define SME_SVQ_ARCH_MAX (MASK(SMCR_ELX_LEN) >> SMCR_ELX_LEN_SHIFT)
+
+/* get a random Streaming SVE VQ b/w 0 to SME_SVQ_ARCH_MAX */
+#define SME_GET_RANDOM_SVQ (rand() % (SME_SVQ_ARCH_MAX + 1))
typedef enum {
SMSTART, /* enters streaming sve mode and enables SME ZA array */
@@ -34,5 +39,10 @@
void sme_ZA_to_vector(const uint64_t *output_vector);
void sme2_load_zt0_instruction(const uint64_t *inputbuf);
void sme2_store_zt0_instruction(const uint64_t *outputbuf);
+void sme_config_svq(uint32_t svq);
+void sme_enable_fa64(void);
+void sme_disable_fa64(void);
+bool sme_smstat_sm(void);
+bool sme_feat_fa64_enabled(void);
#endif /* SME_H */
diff --git a/include/lib/extensions/sve.h b/include/lib/extensions/sve.h
index 4458001..5670afc 100644
--- a/include/lib/extensions/sve.h
+++ b/include/lib/extensions/sve.h
@@ -9,6 +9,7 @@
#include <arch.h>
#include <stdlib.h> /* for rand() */
+#include <lib/extensions/sme.h>
#define fill_sve_helper(num) "ldr z"#num", [%0, #"#num", MUL VL];"
#define read_sve_helper(num) "str z"#num", [%0, #"#num", MUL VL];"
diff --git a/lib/extensions/sme/aarch64/sme.c b/lib/extensions/sme/aarch64/sme.c
index b4eaa2f..ee21578 100644
--- a/lib/extensions/sme/aarch64/sme.c
+++ b/lib/extensions/sme/aarch64/sme.c
@@ -10,6 +10,7 @@
#include <arch.h>
#include <arch_features.h>
#include <arch_helpers.h>
+#include <assert.h>
#include <debug.h>
#include <lib/extensions/sme.h>
@@ -84,3 +85,63 @@
isb();
}
+
+/* Set the Streaming SVE vector length (SVL) in the SMCR_EL2 register */
+void sme_config_svq(uint32_t svq)
+{
+ u_register_t smcr_el2_val;
+
+ /* cap svq to arch supported max value */
+ if (svq > SME_SVQ_ARCH_MAX) {
+ svq = SME_SVQ_ARCH_MAX;
+ }
+
+ smcr_el2_val = read_smcr_el2();
+
+ smcr_el2_val &= ~(MASK(SMCR_ELX_LEN));
+ smcr_el2_val |= INPLACE(SMCR_ELX_LEN, svq);
+
+ write_smcr_el2(smcr_el2_val);
+ isb();
+}
+
+static void set_smcr_fa64(bool enable)
+{
+ if (enable) {
+ write_smcr_el2(read_smcr_el2() | SMCR_ELX_FA64_BIT);
+ } else {
+ write_smcr_el2(read_smcr_el2() & ~SMCR_ELX_FA64_BIT);
+ }
+
+ isb();
+}
+
+/*
+ * Enable FEAT_SME_FA64, This control causes all implemented A64 instructions
+ * to be treated as legal in Streaming SVE mode at EL2, if they are treated as
+ * legal at EL3.
+ */
+void sme_enable_fa64(void)
+{
+ return set_smcr_fa64(true);
+}
+
+/*
+ * Disable FEAT_SME_FA64, This control does not cause any instruction to be
+ * treated as legal in Streaming SVE mode.
+ */
+void sme_disable_fa64(void)
+{
+ return set_smcr_fa64(false);
+}
+
+/* Returns 'true' if the CPU is in Streaming SVE mode */
+bool sme_smstat_sm(void)
+{
+ return ((read_svcr() & SVCR_SM_BIT) != 0U);
+}
+
+bool sme_feat_fa64_enabled(void)
+{
+ return ((read_smcr_el2() & SMCR_ELX_FA64_BIT) != 0U);
+}
diff --git a/lib/extensions/sve/aarch64/sve.c b/lib/extensions/sve/aarch64/sve.c
index b6b4182..8811af1 100644
--- a/lib/extensions/sve/aarch64/sve.c
+++ b/lib/extensions/sve/aarch64/sve.c
@@ -85,7 +85,10 @@
return vl_bitmap;
}
-/* Write SVE Z[0-31] registers passed in 'z_regs' */
+/*
+ * Write SVE Z[0-31] registers passed in 'z_regs' for Normal SVE or Streaming
+ * SVE mode
+ */
void sve_z_regs_write(const sve_z_regs_t *z_regs)
{
__asm__ volatile(
@@ -126,7 +129,9 @@
: : "r" (z_regs));
}
-/* Read SVE Z[0-31] and store it in 'zregs' */
+/*
+ * Read SVE Z[0-31] and store it in 'zregs' for Normal SVE or Streaming SVE mode
+ */
void sve_z_regs_read(sve_z_regs_t *z_regs)
{
__asm__ volatile(
@@ -167,7 +172,10 @@
: : "r" (z_regs));
}
-/* Write SVE P[0-15] registers passed in 'p_regs' */
+/*
+ * Write SVE P[0-15] registers passed in 'p_regs' for Normal SVE or Streaming
+ * SVE mode
+ */
void sve_p_regs_write(const sve_p_regs_t *p_regs)
{
__asm__ volatile(
@@ -192,7 +200,10 @@
: : "r" (p_regs));
}
-/* Read SVE P[0-15] registers and store it in 'p_regs' */
+/*
+ * Read SVE P[0-15] registers and store it in 'p_regs' for Normal SVE or
+ * Streaming SVE mode
+ */
void sve_p_regs_read(sve_p_regs_t *p_regs)
{
__asm__ volatile(
@@ -217,7 +228,10 @@
: : "r" (p_regs));
}
-/* Write SVE FFR registers passed in 'ffr_regs' */
+/*
+ * Write SVE FFR registers passed in 'ffr_regs' for Normal SVE or Streaming SVE
+ * mode
+ */
void sve_ffr_regs_write(const sve_ffr_regs_t *ffr_regs)
{
uint8_t sve_p_reg[SVE_P_REG_LEN_BYTES];
@@ -235,7 +249,10 @@
: "memory");
}
-/* Read SVE FFR registers and store it in 'ffr_regs' */
+/*
+ * Read SVE FFR registers and store it in 'ffr_regs' for Normal SVE or Streaming
+ * SVE mode
+ */
void sve_ffr_regs_read(sve_ffr_regs_t *ffr_regs)
{
uint8_t sve_p_reg[SVE_P_REG_LEN_BYTES];
@@ -255,7 +272,7 @@
/*
* Generate random values and write it to 'z_regs', then write it to SVE Z
- * registers.
+ * registers for Normal SVE or Streaming SVE mode.
*/
void sve_z_regs_write_rand(sve_z_regs_t *z_regs)
{
@@ -278,7 +295,7 @@
/*
* Generate random values and write it to 'p_regs', then write it to SVE P
- * registers.
+ * registers for Normal SVE or Streaming SVE mode.
*/
void sve_p_regs_write_rand(sve_p_regs_t *p_regs)
{
@@ -301,7 +318,7 @@
/*
* Generate random values and write it to 'ffr_regs', then write it to SVE FFR
- * registers.
+ * registers for Normal SVE or Streaming SVE mode.
*/
void sve_ffr_regs_write_rand(sve_ffr_regs_t *ffr_regs)
{
@@ -323,6 +340,7 @@
/*
* Compare Z registers passed in 's1' (old values) with 's2' (new values).
+ * This routine works for Normal SVE or Streaming SVE mode.
*
* Returns:
* 0 : All Z[0-31] registers in 's1' and 's2' are equal
@@ -333,6 +351,10 @@
uint32_t z_size;
uint64_t cmp_bitmap = 0UL;
+ /*
+ * 'rdvl' returns Streaming SVE VL if PSTATE.SM=1 else returns normal
+ * SVE VL
+ */
z_size = (uint32_t)sve_rdvl_1();
for (uint32_t i = 0U; i < SVE_NUM_VECTORS; i++) {
@@ -352,6 +374,7 @@
/*
* Compare P registers passed in 's1' (old values) with 's2' (new values).
+ * This routine works for Normal SVE or Streaming SVE mode.
*
* Returns:
* 0 : All P[0-15] registers in 's1' and 's2' are equal
@@ -382,6 +405,7 @@
/*
* Compare FFR register passed in 's1' (old values) with 's2' (new values).
+ * This routine works for Normal SVE or Streaming SVE mode.
*
* Returns:
* 0 : FFR register in 's1' and 's2' are equal
diff --git a/realm/realm.mk b/realm/realm.mk
index 9b0a230..5e658ed 100644
--- a/realm/realm.mk
+++ b/realm/realm.mk
@@ -47,7 +47,9 @@
lib/delay/delay.c \
lib/extensions/fpu/fpu.c \
lib/extensions/sve/aarch64/sve.c \
- lib/extensions/sve/aarch64/sve_helpers.S
+ lib/extensions/sve/aarch64/sve_helpers.S \
+ lib/extensions/sme/aarch64/sme.c \
+ lib/extensions/sme/aarch64/sme_helpers.S
# TODO: Remove dependency on TFTF files.
REALM_SOURCES += \
diff --git a/tftf/tests/extensions/sme/test_sme.c b/tftf/tests/extensions/sme/test_sme.c
index 43cd8b0..39c6457 100644
--- a/tftf/tests/extensions/sme/test_sme.c
+++ b/tftf/tests/extensions/sme/test_sme.c
@@ -98,23 +98,23 @@
saved_smcr = read_smcr_el2();
/* Write SMCR_EL2 with the LEN max to find implemented width. */
- write_smcr_el2(SME_SMCR_LEN_MAX);
+ write_smcr_el2(MASK(SMCR_ELX_RAZ_LEN));
isb();
len_max = (unsigned int)read_smcr_el2();
VERBOSE("Maximum SMCR_EL2.LEN value: 0x%x\n", len_max);
VERBOSE("Enumerating supported vector lengths...\n");
for (unsigned int i = 0; i <= len_max; i++) {
- /* Load new value into SMCR_EL2.LEN */
+ /* Load new value into SMCR_EL2.RAZ_LEN */
reg = read_smcr_el2();
- reg &= ~(SMCR_ELX_LEN_MASK << SMCR_ELX_LEN_SHIFT);
- reg |= (i << SMCR_ELX_LEN_SHIFT);
+ reg &= ~(MASK(SMCR_ELX_RAZ_LEN));
+ reg |= INPLACE(SMCR_ELX_RAZ_LEN, i);
write_smcr_el2(reg);
isb();
/* Compute current and requested vector lengths in bits. */
current_vector_len = ((unsigned int)sme_rdsvl_1() * 8U);
- requested_vector_len = (i+1U)*128U;
+ requested_vector_len = (i + 1U) * 128U;
/*
* We count down from the maximum SMLEN value, so if the values
@@ -122,13 +122,16 @@
*/
if (current_vector_len == requested_vector_len) {
svl_max = current_vector_len;
- VERBOSE("SUPPORTED: %u bits (LEN=%u)\n", requested_vector_len, i);
+ VERBOSE("SUPPORTED: %u bits (LEN=%u)\n",
+ requested_vector_len, i);
} else {
- VERBOSE("NOT SUPPORTED: %u bits (LEN=%u)\n", requested_vector_len, i);
+ VERBOSE("NOT SUPPORTED: %u bits (LEN=%u)\n",
+ requested_vector_len, i);
}
}
- INFO("Largest Supported Streaming Vector Length(SVL): %u bits \n", svl_max);
+ INFO("Largest Supported Streaming Vector Length(SVL): %u bits\n",
+ svl_max);
/* Exiting Streaming SVE mode */
sme_smstop(SMSTOP_SM);