feat(runtime/rmi): implement RMI_DEV_MEM_(UN)MAP
- Implement RMI_DEV_MEM_MAP and RMI_DEV_MEM_UNMAP
commands as per RMM Specification 1.1-alp12.
Update RMI_RTT_READ_ENTRY and RMI_RTT_FOLD to
support entries with HIPAS=ASSIGNED_DEV.
- Fix comment for VTCR_SH0_IS in VTCR_FLAGS macro
definition.
Change-Id: I830c37b89bf1e4355ad1e3bbe8696dfc6220b6d0
Signed-off-by: AlexeiFedorov <Alexei.Fedorov@arm.com>
diff --git a/configs/fvp_defcfg.cmake b/configs/fvp_defcfg.cmake
index cb22a0e..ede45b1 100644
--- a/configs/fvp_defcfg.cmake
+++ b/configs/fvp_defcfg.cmake
@@ -18,7 +18,7 @@
# Maximum number of translation tables allocated by the runtime context
# for the translation library.
#
-arm_config_option_override(NAME PLAT_CMN_CTX_MAX_XLAT_TABLES DEFAULT 12)
+arm_config_option_override(NAME PLAT_CMN_CTX_MAX_XLAT_TABLES DEFAULT 13)
#
# Maximum number of granules supported, enough to cover
diff --git a/docs/design/locking.rst b/docs/design/locking.rst
index 11ac059..7ac2151 100644
--- a/docs/design/locking.rst
+++ b/docs/design/locking.rst
@@ -532,7 +532,7 @@
- DEV_GRANULE_STATE_DELEGATED
#. **Internal**: A granule state belongs to the `internal` class iff it is not
- an `external`. These are objects which are referenced from another
+ an `external`. These are objects which are referenced from another
object after that object is locked. Each `internal` object should be
referenced from exactly one place. The following granule states are
`internal`:
@@ -549,6 +549,9 @@
#. Granules expected to be in an `external` state must be locked in order of
their physical address, starting with the lowest address.
+#. Memory granules expected to be in an `external` state must be locked before
+ locking any device memory granules in `external` state.
+
#. Once a granule expected to be in an `external` state has been locked, its
state must be checked against the expected state. If these do not match, the
granule must be unlocked and no further granules may be locked within the
diff --git a/lib/arch/include/arch.h b/lib/arch/include/arch.h
index e36ffab..cecdf2a 100644
--- a/lib/arch/include/arch.h
+++ b/lib/arch/include/arch.h
@@ -778,13 +778,13 @@
#define VTCR_RES1 (UL(1) << 31)
#define VTCR_FLAGS ( \
- VTCR_IRGN0_WBRAWA | /* PTW inner cache attr. is WB RAWA*/ \
- VTCR_ORGN0_WBRAWA | /* PTW outer cache attr. is WB RAWA*/ \
- VTCR_SH0_IS | /* PTW shareability attr. is Outer Sharable*/\
- VTCR_TG0_4K | /* 4K granule size in non-secure PT*/ \
+ VTCR_IRGN0_WBRAWA | /* PTW inner cache attr. is WB RAWA */ \
+ VTCR_ORGN0_WBRAWA | /* PTW outer cache attr. is WB RAWA */ \
+ VTCR_SH0_IS | /* PTW shareability attr. is Inner Sharable */ \
+ VTCR_TG0_4K | /* 4K granule size in non-secure PT */ \
/* VS = 0 size(VMID) = 8 */ \
- /* NSW = 0 non-secure s2 is made of secure pages*/ \
- VTCR_NSA | /* non-secure IPA maps to non-secure PA */ \
+ /* NSW = 0 non-secure s2 is made of secure pages */ \
+ VTCR_NSA | /* non-secure IPA maps to non-secure PA */ \
VTCR_RES1 \
)
diff --git a/lib/common/include/platform_api.h b/lib/common/include/platform_api.h
index 1ba1fed..02643e1 100644
--- a/lib/common/include/platform_api.h
+++ b/lib/common/include/platform_api.h
@@ -24,8 +24,8 @@
* Takes an aligned dev_granule address, validates it and if valid returns the
* index in the struct dev_granules array or UINT64_MAX in case of an error.
*
- * This function also validates that the dev_granule address is a valid
- * page address and returns device granule type if the addr is valid.
+ * This function also validates that the dev_granule address is a valid page
+ * address and returns device granule coherency type if the addr is valid.
*/
unsigned long plat_dev_granule_addr_to_idx(unsigned long addr, enum dev_coh_type *type);
diff --git a/lib/granule/include/dev_granule.h b/lib/granule/include/dev_granule.h
index de8d8cf..bfc6b84 100644
--- a/lib/granule/include/dev_granule.h
+++ b/lib/granule/include/dev_granule.h
@@ -171,8 +171,8 @@
}
/*
- * Takes a valid pointer to a struct dev_granule, corresponding to dev_type
- * and returns the dev_granule physical address.
+ * Takes a valid pointer to a struct dev_granule, corresponding to device memory
+ * coherency type and returns the dev_granule physical address.
*
* This is purely a lookup, and provides no guarantees about the attributes of
* the dev_granule (i.e. whether it is locked, its state or its reference count).
@@ -181,7 +181,8 @@
/*
* Takes an aligned dev_granule address, returns a pointer to the corresponding
- * struct dev_granule and sets device granule type in address passed in @type.
+ * struct dev_granule and sets device granule coherency type in address passed
+ * in @type.
*
* This is purely a lookup, and provides no guarantees about the attributes of
* the granule (i.e. whether it is locked, its state or its reference count).
@@ -197,7 +198,7 @@
*
* Returns:
* Pointer to the struct dev_granule if @addr is a valid dev_granule physical
- * address and device granule type in address passed in @type.
+ * address and device granule coherency type in address passed in @type.
* NULL if any of:
* - @addr is not aligned to the size of a granule.
* - @addr is out of range.
@@ -207,7 +208,7 @@
/*
* Obtain a pointer to a locked dev_granule at @addr if @addr is a valid dev_granule
* physical address and the state of the dev_granule at @addr is @expected_state and
- * set device granule type.
+ * set device granule coherency type.
*
* Returns:
* A valid dev_granule pointer if @addr is a valid dev_granule physical address
@@ -216,10 +217,38 @@
* - @addr is not aligned to the size of a granule.
* - @addr is out of range.
* - if the state of the dev_granule at @addr is not @expected_state.
+ * The system coherent memory space associated with dev_granule is returned in
+ * @type output parameter.
*/
struct dev_granule *find_lock_dev_granule(unsigned long addr,
unsigned char expected_state,
enum dev_coh_type *type);
+
+/*
+ * Obtain a pointer to an array of @n locked dev_granules at @addr if @addr is a
+ * valid dev_granule physical address and the states of all @n dev_granules in
+ * array at @addr are @expected_state.
+ *
+ * Returns:
+ * A valid pointer to the 1st dev_granule in array if @addr is a valid
+ * dev_granule physical address.
+ * NULL if any of:
+ * - @addr is not aligned to the size of a granule.
+ * - @addr is out of range.
+ * - if the states of all dev_granules in array at @addr are not @expected_state.
+ * - if not all @n dev_granules in array have the same coherency type.
+ * The coherency type associated with dev_granules is returned in
+ * @type output parameter.
+ *
+ * Locking only succeeds if all @n the dev_granules are in their expected states and
+ * have the same coherency memory type.
+ * If the function fails, no lock is held.
+ */
+struct dev_granule *find_lock_dev_granules(unsigned long addr,
+ unsigned char expected_state,
+ unsigned long n,
+ enum dev_coh_type *type);
+
/*
* Refcount field occupies LSB bits of the dev_granule descriptor,
* and functions which modify its value can operate directly on
diff --git a/lib/granule/src/dev_granule.c b/lib/granule/src/dev_granule.c
index efec5c2..be0203e 100644
--- a/lib/granule/src/dev_granule.c
+++ b/lib/granule/src/dev_granule.c
@@ -15,15 +15,14 @@
IF_NCBMC(__section("granules_memory"));
/*
- * Takes a valid pointer to a struct dev_granule, corresponding to dev_type
- * and returns the dev_granule physical address.
+ * Takes a valid pointer to a struct dev_granule, corresponding to device memory
+ * coherency type and returns the dev_granule physical address.
*
* This is purely a lookup, and provides no guarantees about the attributes of
* the dev_granule (i.e. whether it is locked, its state or its reference count).
*/
unsigned long dev_granule_addr(const struct dev_granule *g, enum dev_coh_type type)
{
- (void)type;
unsigned long idx;
assert(g != NULL);
@@ -55,7 +54,8 @@
/*
* Takes an aligned dev_granule address, returns a pointer to the corresponding
- * struct dev_granule and sets device granule type in address passed in @type.
+ * struct dev_granule and sets device granule coherency type in address passed
+ * in @type.
*
* This is purely a lookup, and provides no guarantees about the attributes of
* the granule (i.e. whether it is locked, its state or its reference count).
@@ -72,7 +72,8 @@
/*
* Verifies whether @addr is a valid dev_granule physical address, returns
- * a pointer to the corresponding struct dev_granule and sets device granule type.
+ * a pointer to the corresponding struct dev_granule and sets device granule
+ * coherency type.
*
* This is purely a lookup, and provides no guarantees w.r.t the state of the
* granule (e.g. locking).
@@ -103,7 +104,7 @@
/*
* Obtain a pointer to a locked dev_granule at @addr if @addr is a valid dev_granule
* physical address and the state of the dev_granule at @addr is @expected_state and
- * set device granule type.
+ * set device granule coherency type.
*
* Returns:
* A valid dev_granule pointer if @addr is a valid dev_granule physical address
@@ -112,6 +113,8 @@
* - @addr is not aligned to the size of a granule.
* - @addr is out of range.
* - if the state of the dev_granule at @addr is not @expected_state.
+ * The system coherent memory space associated with dev_granule is returned in
+ * @type output parameter.
*/
struct dev_granule *find_lock_dev_granule(unsigned long addr,
unsigned char expected_state,
@@ -129,3 +132,57 @@
return g;
}
+
+/*
+ * Obtain a pointer to an array of @n locked dev_granules at @addr if @addr is a
+ * valid dev_granule physical address and the states of all @n dev_granules in
+ * array at @addr are @expected_state.
+ *
+ * Returns:
+ * A valid pointer to the 1st dev_granule in array if @addr is a valid
+ * dev_granule physical address.
+ * NULL if any of:
+ * - @addr is not aligned to the size of a granule.
+ * - @addr is out of range.
+ * - if the states of all dev_granules in array at @addr are not @expected_state.
+ * - if not all @n dev_granules in array have the same coherency type.
+ * The coherency type associated with dev_granules is returned in
+ * @type output parameter.
+ *
+ * Locking only succeeds if all @n the dev_granules are in their expected states and
+ * have the same coherency memory type.
+ * If the function fails, no lock is held.
+ */
+struct dev_granule *find_lock_dev_granules(unsigned long addr,
+ unsigned char expected_state,
+ unsigned long n,
+ enum dev_coh_type *type)
+{
+ struct dev_granule *g;
+
+ /*
+ * Find and lock the 1st dev_granule in array.
+ * Get its coherency type.
+ */
+ g = find_lock_dev_granule(addr, expected_state, type);
+ if (g == NULL) {
+ return NULL;
+ }
+
+ for (unsigned long i = 1UL; i < n; i++) {
+ enum dev_coh_type g_type;
+ struct dev_granule *g_dev;
+
+ addr += GRANULE_SIZE;
+
+ g_dev = find_lock_dev_granule(addr, expected_state, &g_type);
+ if ((g_dev == NULL) || (g_type != *type)) {
+ while (i != 0UL) {
+ dev_granule_unlock(&g[--i]);
+ }
+ return NULL;
+ }
+ }
+
+ return g;
+}
diff --git a/lib/granule/src/granule.c b/lib/granule/src/granule.c
index 6ff1b64..7869fcc 100644
--- a/lib/granule/src/granule.c
+++ b/lib/granule/src/granule.c
@@ -150,18 +150,18 @@
/*
* Find a set of granules and lock them in order of their address.
*
- * @granules: Pointer to array of @n items. Each item must be pre-populated
+ * @gs: Pointer to array of @n items. Each item must be pre-populated
* with ->addr set to the granule's address, and ->state set to
* the expected state of the granule, and ->g_ret pointing to
* a valid 'struct granule *'.
* This function sorts the supplied array in place.
- * @n: Number of struct granule_set in array pointed to by @granules
+ * @n: Number of struct granule_set in array pointed to by @gs
*
* Returns:
- * True if all granules in @granules were successfully locked.
+ * True if all granules in @gs were successfully locked.
*
* False if any two entries in @granules have the same ->addr, or
- * if, for any entry in @granules, any of the following is true:
+ * if, for any entry in @gs, any of the following is true:
* - entry->addr is not aligned to the size of a granule
* - entry->addr is out of range
* - the state of the granule at entry->addr is not entry->state
@@ -169,7 +169,7 @@
* Locking only succeeds if the granules are in their expected states as per the
* locking rules in granule_types.h.
*
- * If the function succeeds, for all items in @granules, ->g points to a locked
+ * If the function succeeds, for all items in @gs, ->g points to a locked
* granule in ->state and *->g_ret is set to the pointer value.
*
* If the function fails, no lock is held and no *->g_ret pointers are
diff --git a/lib/s2tt/include/ripas.h b/lib/s2tt/include/ripas.h
index a445672..2d22d4a 100644
--- a/lib/s2tt/include/ripas.h
+++ b/lib/s2tt/include/ripas.h
@@ -18,7 +18,7 @@
RIPAS_EMPTY = RMI_EMPTY, /* Unused IPA for Realm */
RIPAS_RAM = RMI_RAM, /* IPA used for Code/Data by Realm */
RIPAS_DESTROYED = RMI_DESTROYED,/* IPA is inaccessible to the Realm */
- RIPAS_DEV /* Address where memory of an assigned
+ RIPAS_DEV = RMI_DEV /* IPA where memory of an assigned
Realm device is mapped */
};
diff --git a/lib/s2tt/include/s2tt.h b/lib/s2tt/include/s2tt.h
index 89a30fc..c893168 100644
--- a/lib/s2tt/include/s2tt.h
+++ b/lib/s2tt/include/s2tt.h
@@ -7,6 +7,7 @@
#define S2TT_H
#include <arch_features.h>
+#include <dev_type.h>
#include <granule_types.h>
#include <memory.h>
#include <stdbool.h>
@@ -49,6 +50,7 @@
#define S2TT_MIN_STARTING_LEVEL (0)
#define S2TT_MIN_STARTING_LEVEL_LPA2 (-1)
#define S2TT_PAGE_LEVEL (3)
+#define S2TT_MIN_DEV_BLOCK_LEVEL (2)
#define S2TT_MIN_BLOCK_LEVEL (1)
/*
@@ -113,6 +115,19 @@
unsigned long s2tte,
unsigned long pa,
long level);
+unsigned long s2tte_create_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long pa, long level);
+unsigned long s2tte_create_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long pa, long level);
+unsigned long s2tte_create_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level);
+unsigned long s2tte_create_assigned_dev_dev_coh_type(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level,
+ enum dev_coh_type type);
+unsigned long s2tte_create_assigned_dev_unchanged(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte,
+ unsigned long pa,
+ long level);
unsigned long s2tte_create_table(const struct s2tt_context *s2_ctx,
unsigned long pa, long level);
@@ -138,11 +153,19 @@
unsigned long s2tte, long level);
bool s2tte_is_assigned_ns(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level);
-bool s2tte_is_table(const struct s2tt_context *s2_ctx,
- unsigned long s2tte, long level);
bool s2tte_is_assigned_destroyed(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level);
+bool s2tte_is_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level);
+bool s2tte_is_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level);
+bool s2tte_is_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level);
+
+bool s2tte_is_table(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level);
+
unsigned long s2tte_pa(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level);
bool s2tte_is_addr_lvl_aligned(const struct s2tt_context *s2_ctx,
@@ -175,6 +198,13 @@
void s2tt_init_assigned_destroyed(const struct s2tt_context *s2_ctx,
unsigned long *s2tt, unsigned long pa,
long level);
+void s2tt_init_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long pa, long level);
+void s2tt_init_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long pa, long level);
+void s2tt_init_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long s2tte,
+ unsigned long pa, long level);
void s2tt_invalidate_page(const struct s2tt_context *s2_ctx, unsigned long addr);
void s2tt_invalidate_block(const struct s2tt_context *s2_ctx, unsigned long addr);
@@ -198,6 +228,12 @@
unsigned long *table, long level);
bool s2tt_maps_assigned_destroyed_block(const struct s2tt_context *s2_ctx,
unsigned long *table, long level);
+bool s2tt_maps_assigned_dev_empty_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level);
+bool s2tt_maps_assigned_dev_destroyed_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level);
+bool s2tt_maps_assigned_dev_dev_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level);
struct s2tt_walk {
struct granule *g_llt;
diff --git a/lib/s2tt/src/s2tt.c b/lib/s2tt/src/s2tt.c
index 6149ead..74c52c9 100644
--- a/lib/s2tt/src/s2tt.c
+++ b/lib/s2tt/src/s2tt.c
@@ -7,6 +7,7 @@
#include <assert.h>
#include <bitmap.h>
#include <buffer.h>
+#include <dev_granule.h>
#include <granule.h>
#include <ripas.h>
#include <s2tt.h>
@@ -125,11 +126,9 @@
/*
* Returns true if s2tte has 'output address' field, namely, if it is one of:
- * - assigned_empty
- * - assigned_ram
- * - assigned_ns
- * - assigned_destroyed
- * - table
+ * - valid TTE
+ * - HIPAS = assigned
+ * - HIPAS = assigned_dev
*/
static bool s2tte_has_pa(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level)
@@ -139,7 +138,8 @@
bool valid_desc = ((s2tte & S2TT_DESC_VALID_MASK) == S2TTE_VALID);
return (valid_desc || /* block, page or table */
- s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED));
+ s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED) ||
+ s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED_DEV));
}
/*
@@ -433,7 +433,7 @@
}
/*
- * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED and
+ * Creates s2tte with output address @pa, HIPAS=ASSIGNED and
* RIPAS=@s2tte_ripas, at level @level.
*/
static unsigned long s2tte_create_assigned(const struct s2tt_context *s2_ctx,
@@ -442,11 +442,10 @@
{
unsigned long tte;
+ assert(s2_ctx != NULL);
assert(level >= S2TT_MIN_BLOCK_LEVEL);
assert(level <= S2TT_PAGE_LEVEL);
- assert(EXTRACT(S2TTE_INVALID_RIPAS, s2tte_ripas)
- <= EXTRACT(S2TTE_INVALID_RIPAS, S2TTE_INVALID_RIPAS_DESTROYED));
- assert(s2_ctx != NULL);
+ assert(s2tte_ripas <= S2TTE_INVALID_RIPAS_DESTROYED);
assert(s2tte_is_addr_lvl_aligned(s2_ctx, pa, level));
tte = pa_to_s2tte(pa, s2_ctx->enable_lpa2);
@@ -514,10 +513,160 @@
{
unsigned long current_ripas = s2tte & S2TTE_INVALID_RIPAS_MASK;
+ assert((s2tte & S2TT_DESC_VALID_MASK) == S2TTE_INVALID);
+
return s2tte_create_assigned(s2_ctx, pa, level, current_ripas);
}
/*
+ * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED_DEV and
+ * RIPAS=@s2tte_ripas, at level @level.
+ * This function is only called for @s2tte_ripas values corresponding to
+ * RIPAS_EMPTY and RIPAS_DESTROYED.
+ */
+static unsigned long s2tte_create_assigned_dev(const struct s2tt_context *s2_ctx,
+ unsigned long pa, long level,
+ unsigned long s2tte_ripas)
+{
+ (void)level;
+ unsigned long tte;
+
+ assert(s2_ctx != NULL);
+ assert(level >= S2TT_MIN_DEV_BLOCK_LEVEL);
+ assert((s2tte_ripas == S2TTE_INVALID_RIPAS_EMPTY) ||
+ (s2tte_ripas == S2TTE_INVALID_RIPAS_DESTROYED));
+ assert(s2tte_is_addr_lvl_aligned(s2_ctx, pa, level));
+
+ tte = pa_to_s2tte(pa, s2_ctx->enable_lpa2);
+
+ return (tte | S2TTE_INVALID_HIPAS_ASSIGNED_DEV | s2tte_ripas);
+}
+
+/*
+ * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED_DEV
+ * and RIPAS=EMPTY, at level @level.
+ */
+unsigned long s2tte_create_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long pa, long level)
+{
+ return s2tte_create_assigned_dev(s2_ctx, pa, level,
+ S2TTE_INVALID_RIPAS_EMPTY);
+}
+
+/*
+ * Creates an invalid s2tte with output address @pa, HIPAS=ASSIGNED_DEV and
+ * RIPAS=DESTROYED, at level @level.
+ */
+unsigned long s2tte_create_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long pa, long level)
+{
+ return s2tte_create_assigned_dev(s2_ctx, pa, level,
+ S2TTE_INVALID_RIPAS_DESTROYED);
+}
+
+/*
+ * Creates an dev_assigned s2tte with output address @pa and the same
+ * RIPAS as passed on @s2tte.
+ */
+unsigned long s2tte_create_assigned_dev_unchanged(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte,
+ unsigned long pa,
+ long level)
+{
+ unsigned long current_ripas = s2tte & S2TTE_INVALID_RIPAS_MASK;
+
+ assert((s2tte & S2TT_DESC_VALID_MASK) == S2TTE_INVALID);
+
+ return s2tte_create_assigned_dev(s2_ctx, pa, level, current_ripas);
+}
+
+/*
+ * Creates a valid assigned_dev_dev s2tte at @level with attributes passed in
+ * @attr.
+ */
+static unsigned long create_assigned_dev_dev_attr(unsigned long s2tte,
+ unsigned long attr, long level,
+ bool lpa2)
+{
+ unsigned long pa, new_s2tte;
+
+ assert(level >= S2TT_MIN_DEV_BLOCK_LEVEL);
+
+ pa = s2tte_to_pa(s2tte, level, lpa2);
+
+ new_s2tte = attr | pa_to_s2tte(pa, lpa2);
+
+ if (level == S2TT_PAGE_LEVEL) {
+ return (new_s2tte | S2TTE_L3_PAGE);
+ }
+
+ return (new_s2tte | S2TTE_L012_BLOCK);
+}
+
+/*
+ * Creates a valid assigned_dev_dev s2tte at @level with attributes passed in
+ * @s2tte.
+ */
+unsigned long s2tte_create_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level)
+{
+ unsigned long s2tte_mask = (S2TTE_DEV_ATTRS_MASK | S2TTE_MEMATTR_MASK);
+ unsigned long attr;
+ bool lpa2;
+
+ assert(s2_ctx != NULL);
+
+ lpa2 = s2_ctx->enable_lpa2;
+
+ /* Add Shareability bits if FEAT_LPA2 is not enabled */
+ if (!lpa2) {
+ s2tte_mask |= S2TTE_SH_MASK;
+ }
+
+ /* Get attributes */
+ attr = s2tte & s2tte_mask;
+
+ return create_assigned_dev_dev_attr(s2tte, attr, level, lpa2);
+}
+
+/*
+ * Creates a valid assigned_dev_dev s2tte at @level with attributes based on
+ * device coherency passed in @type.
+ */
+unsigned long s2tte_create_assigned_dev_dev_coh_type(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level,
+ enum dev_coh_type type)
+{
+ unsigned long attr;
+ bool lpa2;
+
+ assert(s2_ctx != NULL);
+ assert(type < DEV_MEM_MAX);
+
+ lpa2 = s2_ctx->enable_lpa2;
+
+ if (type == DEV_MEM_COHERENT) {
+ /* Coherent device */
+ attr = S2TTE_DEV_COH_ATTRS;
+
+ /* Add Shareability bits if FEAT_LPA2 is not enabled */
+ if (!lpa2) {
+ attr |= S2TTE_SH_IS; /* Inner Shareable */
+ }
+ } else {
+ /* Non-coherent device */
+ attr = S2TTE_DEV_NCOH_ATTRS;
+
+ /* Add Shareability bits if FEAT_LPA2 is not enabled */
+ if (!lpa2) {
+ attr |= S2TTE_SH_OS; /* Outer Shareable */
+ }
+ }
+
+ return create_assigned_dev_dev_attr(s2tte, attr, level, lpa2);
+}
+
+/*
* Creates an assigned_ns s2tte at level @level.
*
* The following S2 TTE fields are provided through @s2tte argument:
@@ -611,9 +760,8 @@
unsigned long s2tte_create_table(const struct s2tt_context *s2_ctx,
unsigned long pa, long level)
{
- __unused long min_starting_level;
-
(void)level;
+ __unused long min_starting_level;
assert(s2_ctx != NULL);
@@ -659,11 +807,26 @@
static inline bool s2tte_has_assigned_ripas(unsigned long s2tte,
unsigned long ripas)
{
+ assert((s2tte & S2TT_DESC_VALID_MASK) == S2TTE_INVALID);
+ assert(ripas != S2TTE_INVALID_RIPAS_RAM);
+
return (((s2tte & S2TTE_INVALID_RIPAS_MASK) == ripas) &&
s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED));
}
/*
+ * Returns true if @s2tte has HIPAS=ASSIGNED_DEV and RIPAS=@ripas.
+ */
+static bool s2tte_has_assigned_dev_ripas(unsigned long s2tte, unsigned long ripas)
+{
+ assert((s2tte & S2TT_DESC_VALID_MASK) == S2TTE_INVALID);
+ assert(ripas != S2TTE_INVALID_RIPAS_DEV);
+
+ return ((s2tte & S2TTE_INVALID_RIPAS_MASK) == ripas) &&
+ s2tte_has_hipas(s2tte, S2TTE_INVALID_HIPAS_ASSIGNED_DEV);
+}
+
+/*
* Returns true if @s2tte has HIPAS=UNASSIGNED.
*/
bool s2tte_is_unassigned(const struct s2tt_context *s2_ctx, unsigned long s2tte)
@@ -725,8 +888,12 @@
bool s2tte_is_assigned_destroyed(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level)
{
- (void)level;
(void)s2_ctx;
+ (void)level;
+
+ if ((s2tte & S2TT_DESC_VALID_MASK) != S2TTE_INVALID) {
+ return false;
+ }
return s2tte_has_assigned_ripas(s2tte, S2TTE_INVALID_RIPAS_DESTROYED);
}
@@ -737,8 +904,12 @@
bool s2tte_is_assigned_empty(const struct s2tt_context *s2_ctx,
unsigned long s2tte, long level)
{
- (void)level;
(void)s2_ctx;
+ (void)level;
+
+ if ((s2tte & S2TT_DESC_VALID_MASK) != S2TTE_INVALID) {
+ return false;
+ }
return s2tte_has_assigned_ripas(s2tte, S2TTE_INVALID_RIPAS_EMPTY);
}
@@ -746,9 +917,8 @@
static bool s2tte_check(const struct s2tt_context *s2_ctx, unsigned long s2tte,
long level, unsigned long ns)
{
- unsigned long desc_type;
-
(void)s2_ctx;
+ unsigned long desc_type;
if ((s2tte & S2TTE_NS) != ns) {
return false;
@@ -787,6 +957,80 @@
}
/*
+ * Returns true if @s2tte has HIPAS=ASSIGNED_DEV and RIPAS=RIPAS_DEV.
+ */
+bool s2tte_is_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level)
+{
+ unsigned long attr;
+ unsigned long desc_type = s2tte & S2TT_DESC_TYPE_MASK;
+ bool lpa2;
+
+ assert(s2_ctx != NULL);
+
+ /* Only pages at L3 and valid blocks at L2 allowed */
+ if (!(((level == S2TT_PAGE_LEVEL) && (desc_type == S2TTE_L3_PAGE)) ||
+ ((level == S2TT_MIN_DEV_BLOCK_LEVEL) && (desc_type == S2TTE_L012_BLOCK)))) {
+ return false;
+ }
+
+ attr = s2tte & (S2TTE_DEV_ATTRS_MASK | S2TTE_MEMATTR_MASK);
+ lpa2 = s2_ctx->enable_lpa2;
+
+ /*
+ * Check that NS, XN, S2AP, AF and MemAttr[3:0] match with provided
+ * attributes. When LPA2 is not enabled, assert if shareability
+ * attrubutes are not set correctly.
+ */
+ if (attr == S2TTE_DEV_COH_ATTRS) {
+ if (!lpa2) {
+ /* Coherent device, Inner Shareable */
+ assert((s2tte & S2TTE_SH_MASK) == S2TTE_SH_IS);
+ }
+ return true;
+ } else if (attr == S2TTE_DEV_NCOH_ATTRS) {
+ if (!lpa2) {
+ /* Non-coherent device, Outer Shareable */
+ assert((s2tte & S2TTE_SH_MASK) == S2TTE_SH_OS);
+ }
+ return true;
+ }
+ return false;
+}
+
+/*
+ * Returns true if @s2tte has HIPAS=ASSIGNED_DEV and RIPAS=RIPAS_EMPTY.
+ */
+bool s2tte_is_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level)
+{
+ (void)s2_ctx;
+ (void)level;
+
+ if ((s2tte & S2TT_DESC_VALID_MASK) != S2TTE_INVALID) {
+ return false;
+ }
+
+ return s2tte_has_assigned_dev_ripas(s2tte, S2TTE_INVALID_RIPAS_EMPTY);
+}
+
+/*
+ * Returns true if @s2tte is an dev_assigned_destroyed.
+ */
+bool s2tte_is_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long s2tte, long level)
+{
+ (void)s2_ctx;
+ (void)level;
+
+ if ((s2tte & S2TT_DESC_VALID_MASK) != S2TTE_INVALID) {
+ return false;
+ }
+
+ return s2tte_has_assigned_dev_ripas(s2tte, S2TTE_INVALID_RIPAS_DESTROYED);
+}
+
+/*
* Returns true if @s2tte is a table at level @level.
*/
bool s2tte_is_table(const struct s2tt_context *s2_ctx, unsigned long s2tte,
@@ -840,11 +1084,11 @@
void s2tt_init_unassigned_empty(const struct s2tt_context *s2_ctx,
unsigned long *s2tt)
{
- assert(s2tt != NULL);
-
unsigned long s2tte =
s2tte_create_unassigned_empty(s2_ctx);
+ assert(s2tt != NULL);
+
for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
s2tt[i] = s2tte;
}
@@ -861,10 +1105,10 @@
void s2tt_init_unassigned_ram(const struct s2tt_context *s2_ctx,
unsigned long *s2tt)
{
- assert(s2tt != NULL);
-
unsigned long s2tte = s2tte_create_unassigned_ram(s2_ctx);
+ assert(s2tt != NULL);
+
for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
s2tt[i] = s2tte;
}
@@ -881,10 +1125,10 @@
void s2tt_init_unassigned_ns(const struct s2tt_context *s2_ctx,
unsigned long *s2tt)
{
- assert(s2tt != NULL);
-
unsigned long s2tte = s2tte_create_unassigned_ns(s2_ctx);
+ assert(s2tt != NULL);
+
for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
s2tt[i] = s2tte;
}
@@ -901,14 +1145,15 @@
void s2tt_init_unassigned_destroyed(const struct s2tt_context *s2_ctx,
unsigned long *s2tt)
{
- assert(s2tt != NULL);
-
unsigned long s2tte =
s2tte_create_unassigned_destroyed(s2_ctx);
+ assert(s2tt != NULL);
+
for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
s2tt[i] = s2tte;
}
+
dsb(ish);
}
@@ -1018,6 +1263,78 @@
}
/*
+ * Populates @s2tt with HIPAS=ASSIGNED_DEV, RIPAS=EMPTY s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_assigned_dev_empty(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+
+ for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_assigned_dev_empty(s2_ctx, pa, level);
+ pa += map_size;
+ }
+
+ dsb(ish);
+}
+
+/*
+ * Populates @s2tt with HIPAS=ASSIGNED_DEV, RIPAS=DESTROYED s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_assigned_dev_destroyed(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+
+ for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_assigned_dev_destroyed(s2_ctx, pa, level);
+ pa += map_size;
+ }
+
+ dsb(ish);
+}
+
+/*
+ * Populates @s2tt with assigned_dev_dev s2ttes that refer to a
+ * contiguous memory block starting at @pa, and mapped at level @level.
+ *
+ * The granule is populated before it is made a table,
+ * hence, don't use s2tte_write for access.
+ */
+void s2tt_init_assigned_dev_dev(const struct s2tt_context *s2_ctx,
+ unsigned long *s2tt, unsigned long s2tte,
+ unsigned long pa, long level)
+{
+ const unsigned long map_size = s2tte_map_size(level);
+ unsigned long s2tte_mask = (S2TTE_DEV_ATTRS_MASK | S2TTE_MEMATTR_MASK);
+
+ assert(s2_ctx != NULL);
+ assert(level >= S2TT_MIN_DEV_BLOCK_LEVEL);
+
+ /* Add Shareability bits if FEAT_LPA2 is not enabled */
+ if (!s2_ctx->enable_lpa2) {
+ s2tte_mask |= S2TTE_SH_MASK;
+ }
+
+ s2tte &= s2tte_mask;
+
+ for (unsigned int i = 0U; i < S2TTES_PER_S2TT; i++) {
+ s2tt[i] = s2tte_create_assigned_dev_dev(s2_ctx, s2tte | pa, level);
+ pa += map_size;
+ }
+
+ dsb(ish);
+}
+
+/*
* Returns true if s2tte is a live RTTE entry. i.e.,
* HIPAS is ASSIGNED.
*
@@ -1256,6 +1573,39 @@
}
/*
+ * Returns true if all s2ttes are assigned_dev_empty and
+ * refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool s2tt_maps_assigned_dev_empty_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level)
+{
+ return table_maps_block(s2_ctx, table, level,
+ s2tte_is_assigned_dev_empty, false);
+}
+
+/*
+ * Returns true if all s2ttes are assigned_dev_destroyed and
+ * refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool s2tt_maps_assigned_dev_destroyed_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level)
+{
+ return table_maps_block(s2_ctx, table, level,
+ s2tte_is_assigned_dev_destroyed, false);
+}
+
+/*
+ * Returns true if all s2ttes are assigned_dev_dev and
+ * refer to a contiguous block of granules aligned to @level - 1.
+ */
+bool s2tt_maps_assigned_dev_dev_block(const struct s2tt_context *s2_ctx,
+ unsigned long *table, long level)
+{
+ return table_maps_block(s2_ctx, table, level,
+ s2tte_is_assigned_dev_dev, false);
+}
+
+/*
* Scan the RTT @s2tt (which is @wi.level), from the entry (@wi.index) and
* skip the non-live entries (i.e., HIPAS=UNASSIGNED).
* In other words, the scanning stops when a live RTTE is encountered or we
diff --git a/lib/s2tt/src/s2tt_pvt_defs.h b/lib/s2tt/src/s2tt_pvt_defs.h
index 9f52ecd..152c21a 100644
--- a/lib/s2tt/src/s2tt_pvt_defs.h
+++ b/lib/s2tt/src/s2tt_pvt_defs.h
@@ -17,36 +17,44 @@
* RIPAS and DESC_TYPE fields share bit[1], bit[0] set to 1 applies to a valid descriptor
* with HIPAS and RIPAS fields N/A.
*
- * ========================================================================================
- * s2tte type level NS[55] HIPAS[5:3] RIPAS[2:1] DESC_TYPE[1:0] OA alignment
- * ========================================================================================
- * unassigned_empty any 0 unassigned[0] empty[0] invalid[0] n/a
- * ----------------------------------------------------------------------------------------
- * unassigned_ram any 0 unassigned[0] ram[1] invalid[2] n/a
- * ----------------------------------------------------------------------------------------
- * unassigned_destroyed any 0 unassigned[0] destroyed[2] invalid[0] n/a
- * ----------------------------------------------------------------------------------------
- * assigned_empty 2,3 0 assigned[1] empty[0] invalid[0] to level
- * ----------------------------------------------------------------------------------------
- * assigned_ram 3 0 n/a n/a page[3] to level
- * 2 0 n/a n/a block[1] to level
- * ----------------------------------------------------------------------------------------
- * assigned_destroyed any 0 assigned[1] destroyed[2] invalid[0] n/a
- * ========================================================================================
- * unassigned_ns any 1 unassigned[0] n/a invalid[0] n/a
- * ----------------------------------------------------------------------------------------
- * assigned_ns 3 1 n/a n/a page[3] to level
- * 2 1 n/a n/a block[1] to level
- * ========================================================================================
- * table <=2 n/a n/a n/a table[3] to 4KB
- * ========================================================================================
+ * ===========================================================================================
+ * s2tte type level NS[55] HIPAS[5:3] RIPAS[2:1] DESC_TYPE[1:0] OA alignment
+ * ==========================================================================================
+ * unassigned_empty any 0 unassigned[0] empty[0] invalid[0] n/a
+ * ------------------------------------------------------------------------------------------
+ * unassigned_ram any 0 unassigned[0] ram[1] invalid[2] n/a
+ * ------------------------------------------------------------------------------------------
+ * unassigned_destroyed any 0 unassigned[0] destroyed[2] invalid[0] n/a
+ * ------------------------------------------------------------------------------------------
+ * assigned_empty 2,3 0 assigned[1] empty[0] invalid[0] to level
+ * ------------------------------------------------------------------------------------------
+ * assigned_ram 3 0 n/a n/a page[3] to level
+ * 2 0 n/a n/a block[1] to level
+ * ------------------------------------------------------------------------------------------
+ * assigned_destroyed any 0 assigned[1] destroyed[2] invalid[0] n/a
+ * ------------------------------------------------------------------------------------------
+ * assigned_dev_empty any 0 assigned_dev[3] empty[0] invalid[0] to level
+ * ------------------------------------------------------------------------------------------
+ * assigned_dev_destroyed any 0 assigned_dev[3] destroyed[2] invalid[0] to level
+ * ------------------------------------------------------------------------------------------
+ * assigned_dev_dev 3 0 n/a n/a page[3] to level
+ * 2 0 n/a n/a block[1] to level
+ * ==========================================================================================
+ * unassigned_ns any 1 unassigned[0] n/a invalid[0] n/a
+ * ------------------------------------------------------------------------------------------
+ * assigned_ns 3 1 n/a n/a page[3] to level
+ * 2 1 n/a n/a block[1] to level
+ * ==========================================================================================
+ * table <=2 n/a n/a n/a table[3] to 4KB
+ * ==========================================================================================
*/
#define S2TTE_INVALID_HIPAS_SHIFT 5
#define S2TTE_INVALID_HIPAS_WIDTH 3U
#define S2TTE_INVALID_HIPAS_MASK MASK(S2TTE_INVALID_HIPAS)
-#define S2TTE_INVALID_HIPAS_UNASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, RMI_UNASSIGNED))
-#define S2TTE_INVALID_HIPAS_ASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, RMI_ASSIGNED))
+#define S2TTE_INVALID_HIPAS_UNASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, RMI_UNASSIGNED))
+#define S2TTE_INVALID_HIPAS_ASSIGNED (INPLACE(S2TTE_INVALID_HIPAS, RMI_ASSIGNED))
+#define S2TTE_INVALID_HIPAS_ASSIGNED_DEV (INPLACE(S2TTE_INVALID_HIPAS, RMI_ASSIGNED_DEV))
#define S2TTE_INVALID_RIPAS_SHIFT 1
#define S2TTE_INVALID_RIPAS_WIDTH 2U
@@ -55,6 +63,7 @@
#define S2TTE_INVALID_RIPAS_EMPTY (INPLACE(S2TTE_INVALID_RIPAS, RMI_EMPTY))
#define S2TTE_INVALID_RIPAS_RAM (INPLACE(S2TTE_INVALID_RIPAS, RMI_RAM))
#define S2TTE_INVALID_RIPAS_DESTROYED (INPLACE(S2TTE_INVALID_RIPAS, RMI_DESTROYED))
+#define S2TTE_INVALID_RIPAS_DEV (INPLACE(S2TTE_INVALID_RIPAS, RMI_DEV))
#define S2TTE_NS (1UL << 55)
#define S2TTE_AF (1UL << 10)
@@ -101,7 +110,9 @@
*/
#define S2TTE_MEMATTR_SHIFT 2
#define S2TTE_MEMATTR_MASK (0x7UL << S2TTE_MEMATTR_SHIFT)
+#define S2TTE_MEMATTR_FWB_NC ((1UL << 4) | (1UL << 2))
#define S2TTE_MEMATTR_FWB_NORMAL_WB ((1UL << 4) | (2UL << 2))
+#define S2TTE_MEMATTR_DEV_COH ((1UL << 4) | (3UL << 2))
#define S2TTE_MEMATTR_FWB_RESERVED ((1UL << 4) | (0UL << 2))
#define S2TTE_AP_SHIFT 6
@@ -125,6 +136,12 @@
#define S2TTE_ATTRS (S2TTE_ATTRS_LPA2 | S2TTE_SH_IS)
#define S2TTE_ATTRS_MASK (S2TTE_ATTRS_LPA2_MASK | S2TTE_SH_MASK)
+#define S2TTE_DEV_ATTRS (S2TTE_AP_RW | S2TTE_AF | S2TTE_XN)
+#define S2TTE_DEV_ATTRS_MASK (S2TTE_NS | S2TTE_AP_MASK | S2TTE_AF | S2TTE_XN)
+
+#define S2TTE_DEV_COH_ATTRS (S2TTE_DEV_ATTRS | S2TTE_MEMATTR_DEV_COH)
+#define S2TTE_DEV_NCOH_ATTRS (S2TTE_DEV_ATTRS | S2TTE_MEMATTR_FWB_NC)
+
/* NS attributes controlled by the host */
#define S2TTE_NS_ATTR_MASK (S2TTE_MEMATTR_MASK | S2TTE_AP_MASK)
@@ -140,6 +157,7 @@
#define S2TTE_BLOCK_LPA2 (S2TTE_ATTRS_LPA2 | S2TTE_L012_BLOCK)
#define S2TTE_PAGE (S2TTE_ATTRS | S2TTE_L3_PAGE)
#define S2TTE_PAGE_LPA2 (S2TTE_ATTRS_LPA2 | S2TTE_L3_PAGE)
+
#define S2TTE_BLOCK_NS ((S2TTE_NS_ATTR_RMM) | S2TTE_L012_BLOCK)
#define S2TTE_PAGE_NS ((S2TTE_NS_ATTR_RMM) | S2TTE_L3_PAGE)
#define S2TTE_INVALID S2TTE_Lx_INVALID
diff --git a/lib/smc/include/smc-rmi.h b/lib/smc/include/smc-rmi.h
index cfdd473..ff463f2 100644
--- a/lib/smc/include/smc-rmi.h
+++ b/lib/smc/include/smc-rmi.h
@@ -124,6 +124,7 @@
#define RMI_UNASSIGNED UL(0)
#define RMI_ASSIGNED UL(1)
#define RMI_TABLE UL(2)
+#define RMI_ASSIGNED_DEV UL(3)
/* RmiFeature enumerations */
#define RMI_FEATURE_FALSE UL(0)
@@ -188,8 +189,8 @@
/* Address which is inaccessible to the Realm due to an action taken by the Host */
#define RMI_DESTROYED UL(2)
-/* Address where MMIO of an assigned Realm device is mapped. */
-#define RMI_IO UL(3)
+/* Address where memory of an assigned Realm device is mapped */
+#define RMI_DEV UL(3)
/* RmiPmuOverflowStatus enumeration representing PMU overflow status */
#define RMI_PMU_OVERFLOW_NOT_ACTIVE U(0)
@@ -375,6 +376,9 @@
* arg0 == RD address
* arg1 == map address
* arg2 == level
+ *
+ * ret1 == Top of the non-live address region. Only valid
+ * if ret0 == RMI_SUCCESS or ret0 == (RMI_ERROR_RTT, x)
*/
#define SMC_RMI_RTT_UNMAP_UNPROTECTED SMC64_RMI_FID(U(0x12))
@@ -442,13 +446,27 @@
/*
* FID: 0xC4000172
+ *
+ * arg0 == RD address
+ * arg1 == map address
+ * arg2 == level
+ * arg3 == PA of the target device memory
*/
-#define SMC_RMI_DEV_MAP SMC64_RMI_FID(U(0x22))
+#define SMC_RMI_DEV_MEM_MAP SMC64_RMI_FID(U(0x22))
/*
* FID: 0xC4000173
+ *
+ * arg0 == RD address
+ * arg1 == map address
+ * arg2 == level
+ *
+ * ret1 == Address (PA) of the device memory granule, if ret0 == RMI_SUCCESS
+ * Otherwise, undefined.
+ * ret2 == Top of the non-live address region. Only valid
+ * if ret0 == RMI_SUCCESS or ret0 == (RMI_ERROR_RTT, x)
*/
-#define SMC_RMI_DEV_UNMAP SMC64_RMI_FID(U(0x23))
+#define SMC_RMI_DEV_MEM_UNMAP SMC64_RMI_FID(U(0x23))
/*
* FID: 0xC4000174
diff --git a/plat/arm/src/arm_granule.c b/plat/arm/src/arm_granule.c
index 69cf0c7..d5b09b6 100644
--- a/plat/arm/src/arm_granule.c
+++ b/plat/arm/src/arm_granule.c
@@ -98,7 +98,6 @@
}
}
- *type = DEV_MEM_MAX;
return UINT64_MAX;
}
diff --git a/runtime/core/handler.c b/runtime/core/handler.c
index 07e2616..fad1c6a 100644
--- a/runtime/core/handler.c
+++ b/runtime/core/handler.c
@@ -158,8 +158,8 @@
HANDLER(REC_AUX_COUNT, 1, 1, smc_rec_aux_count, true, true),
HANDLER(RTT_INIT_RIPAS, 3, 1, smc_rtt_init_ripas, false, true),
HANDLER(RTT_SET_RIPAS, 4, 1, smc_rtt_set_ripas, false, true),
- HANDLER(DEV_MAP, 0, 0, NULL, true, true),
- HANDLER(DEV_UNMAP, 0, 0, NULL, true, true),
+ HANDLER(DEV_MEM_MAP, 4, 0, smc_dev_mem_map, false, true),
+ HANDLER(DEV_MEM_UNMAP, 3, 2, smc_dev_mem_unmap, false, true),
HANDLER(PDEV_ABORT, 0, 0, NULL, true, true),
HANDLER(PDEV_COMMUNICATE, 2, 0, NULL, true, true),
HANDLER(PDEV_CREATE, 2, 0, NULL, true, true),
diff --git a/runtime/include/smc-handler.h b/runtime/include/smc-handler.h
index 5965078..c24099b 100644
--- a/runtime/include/smc-handler.h
+++ b/runtime/include/smc-handler.h
@@ -96,4 +96,14 @@
unsigned long top,
struct smc_result *res);
+unsigned long smc_dev_mem_map(unsigned long rd_addr,
+ unsigned long map_addr,
+ unsigned long ulevel,
+ unsigned long dev_mem_addr);
+
+void smc_dev_mem_unmap(unsigned long rd_addr,
+ unsigned long map_addr,
+ unsigned long ulevel,
+ struct smc_result *res);
+
#endif /* SMC_HANDLER_H */
diff --git a/runtime/rmi/rtt.c b/runtime/rmi/rtt.c
index 64f1388..5d302f3 100644
--- a/runtime/rmi/rtt.c
+++ b/runtime/rmi/rtt.c
@@ -6,6 +6,7 @@
#include <assert.h>
#include <buffer.h>
+#include <dev_granule.h>
#include <errno.h>
#include <granule.h>
#include <measurement.h>
@@ -20,7 +21,8 @@
#include <string.h>
/*
- * Validate the map_addr value passed to RMI_RTT_* and RMI_DATA_* commands.
+ * Validate the map_addr value passed to
+ * RMI_RTT_*, RMI_DATA_* and RMI_DEV_MEM_* commands.
*/
static bool validate_map_addr(unsigned long map_addr,
long level,
@@ -47,7 +49,7 @@
}
/*
- * Map/Unmap commands can operate up to a level 2 block entry so min_level is
+ * Map/Unmap commands can operate up to a level 1 block entry so min_level is
* the smallest block size.
*/
static bool validate_rtt_map_cmds(unsigned long map_addr,
@@ -260,6 +262,69 @@
*/
atomic_granule_get(wi.g_llt);
+ } else if (s2tte_is_assigned_dev_empty(&s2_ctx, parent_s2tte, level - 1L)) {
+ unsigned long block_pa;
+
+ /*
+ * We should observe parent assigned s2tte only when
+ * we create tables above this level.
+ */
+ assert(level > S2TT_MIN_DEV_BLOCK_LEVEL);
+
+ block_pa = s2tte_pa(&s2_ctx, parent_s2tte, level - 1L);
+
+ s2tt_init_assigned_dev_empty(&s2_ctx, s2tt, block_pa, level);
+
+ /*
+ * Increase the refcount to mark the granule as in-use. refcount
+ * is incremented by S2TTES_PER_S2TT (ref RTT unfolding).
+ */
+ granule_refcount_inc(g_tbl, (unsigned short)S2TTES_PER_S2TT);
+
+ } else if (s2tte_is_assigned_dev_destroyed(&s2_ctx, parent_s2tte, level - 1L)) {
+ unsigned long block_pa;
+
+ /*
+ * We should observe parent assigned s2tte only when
+ * we create tables above this level.
+ */
+ assert(level > S2TT_MIN_DEV_BLOCK_LEVEL);
+
+ block_pa = s2tte_pa(&s2_ctx, parent_s2tte, level - 1L);
+
+ s2tt_init_assigned_dev_destroyed(&s2_ctx, s2tt, block_pa, level);
+
+ /*
+ * Increase the refcount to mark the granule as in-use. refcount
+ * is incremented by S2TTES_PER_S2TT (ref RTT unfolding).
+ */
+ granule_refcount_inc(g_tbl, (unsigned short)S2TTES_PER_S2TT);
+
+ } else if (s2tte_is_assigned_dev_dev(&s2_ctx, parent_s2tte, level - 1L)) {
+ unsigned long block_pa;
+
+ /*
+ * We should observe parent valid s2tte only when
+ * we create tables above this level.
+ */
+ assert(level > S2TT_MIN_DEV_BLOCK_LEVEL);
+
+ /*
+ * Break before make. This may cause spurious S2 aborts.
+ */
+ s2tte_write(&parent_s2tt[wi.index], 0UL);
+ s2tt_invalidate_block(&s2_ctx, map_addr);
+
+ block_pa = s2tte_pa(&s2_ctx, parent_s2tte, level - 1L);
+
+ s2tt_init_assigned_dev_dev(&s2_ctx, s2tt, parent_s2tte, block_pa, level);
+
+ /*
+ * Increase the refcount to mark the granule as in-use. refcount
+ * is incremented by S2TTES_PER_S2TT (ref RTT unfolding).
+ */
+ granule_refcount_inc(g_tbl, (unsigned short)S2TTES_PER_S2TT);
+
} else if (s2tte_is_table(&s2_ctx, parent_s2tte, level - 1L)) {
ret = pack_return_code(RMI_ERROR_RTT,
(unsigned char)(level - 1L));
@@ -432,7 +497,22 @@
} else if (s2tt_maps_assigned_destroyed_block(&s2_ctx,
table, level)) {
parent_s2tte = s2tte_create_assigned_destroyed(&s2_ctx,
- block_pa, level - 1L);
+ block_pa, level - 1L);
+ } else if (s2tt_maps_assigned_dev_empty_block(&s2_ctx,
+ table, level)) {
+ parent_s2tte = s2tte_create_assigned_dev_empty(&s2_ctx,
+ block_pa,
+ level - 1L);
+ } else if (s2tt_maps_assigned_dev_destroyed_block(&s2_ctx,
+ table, level)) {
+ parent_s2tte = s2tte_create_assigned_dev_destroyed(&s2_ctx,
+ block_pa,
+ level - 1L);
+ } else if (s2tt_maps_assigned_dev_dev_block(&s2_ctx, table, level)) {
+ parent_s2tte = s2tte_create_assigned_dev_dev(&s2_ctx,
+ s2tte,
+ level - 1L);
+
/* The table contains mixed entries that cannot be folded */
} else {
ret = pack_return_code(RMI_ERROR_RTT,
@@ -459,7 +539,8 @@
s2tte_write(&parent_s2tt[wi.index], 0UL);
if (s2tte_is_assigned_ram(&s2_ctx, parent_s2tte, level - 1L) ||
- s2tte_is_assigned_ns(&s2_ctx, parent_s2tte, level - 1L)) {
+ s2tte_is_assigned_ns(&s2_ctx, parent_s2tte, level - 1L) ||
+ s2tte_is_assigned_dev_dev(&s2_ctx, parent_s2tte, level - 1L)) {
s2tt_invalidate_pages_in_block(&s2_ctx, map_addr);
} else {
s2tt_invalidate_block(&s2_ctx, map_addr);
@@ -832,6 +913,19 @@
res->x[2] = RMI_ASSIGNED;
res->x[3] = s2tte_pa(&s2_ctx, s2tte, wi.last_level);
res->x[4] = (unsigned long)RIPAS_DESTROYED;
+ } else if (s2tte_is_assigned_dev_empty(&s2_ctx, s2tte, wi.last_level)) {
+ res->x[2] = RMI_ASSIGNED_DEV;
+ res->x[3] = s2tte_pa(&s2_ctx, s2tte, wi.last_level);
+ res->x[4] = (unsigned long)RIPAS_EMPTY;
+ } else if (s2tte_is_assigned_dev_destroyed(&s2_ctx, s2tte,
+ wi.last_level)) {
+ res->x[2] = RMI_ASSIGNED_DEV;
+ res->x[3] = 0UL;
+ res->x[4] = (unsigned long)RIPAS_DESTROYED;
+ } else if (s2tte_is_assigned_dev_dev(&s2_ctx, s2tte, wi.last_level)) {
+ res->x[2] = RMI_ASSIGNED_DEV;
+ res->x[3] = s2tte_pa(&s2_ctx, s2tte, wi.last_level);
+ res->x[4] = (unsigned long)RIPAS_DEV;
} else if (s2tte_is_unassigned_ns(&s2_ctx, s2tte)) {
res->x[2] = RMI_UNASSIGNED;
res->x[3] = 0UL;
@@ -1474,3 +1568,214 @@
granule_unlock(g_rec);
granule_unlock(g_rd);
}
+
+unsigned long smc_dev_mem_map(unsigned long rd_addr,
+ unsigned long map_addr,
+ unsigned long ulevel,
+ unsigned long dev_mem_addr)
+{
+ struct dev_granule *g_dev;
+ struct granule *g_rd;
+ struct rd *rd;
+ struct s2tt_walk wi;
+ struct s2tt_context s2_ctx;
+ unsigned long s2tte, *s2tt, num_granules;
+ long level = (long)ulevel;
+ unsigned long ret;
+ __unused enum dev_coh_type type;
+
+ /* Dev_Mem_Map/Unmap commands can operate up to a level 2 block entry */
+ if ((level < S2TT_MIN_DEV_BLOCK_LEVEL) || (level > S2TT_PAGE_LEVEL)) {
+ return RMI_ERROR_INPUT;
+ }
+
+ /*
+ * The code below assumes that "external" granules are always
+ * locked before "external" dev_granules, hence, RD GRANULE is locked
+ * before DELEGATED DEV_GRANULE.
+ *
+ * The alternative scheme is that all external granules and device
+ * granules are locked together in the order of their physical
+ * addresses. For that scheme, however, we need primitives similar to
+ * 'find_lock_two_granules' that would work with different object
+ * types (struct granule and struct dev_granule).
+ */
+
+ g_rd = find_lock_granule(rd_addr, GRANULE_STATE_RD);
+ if (g_rd == NULL) {
+ return RMI_ERROR_INPUT;
+ }
+
+ if (level == S2TT_PAGE_LEVEL) {
+ num_granules = 1UL;
+ } else {
+ assert(level == (S2TT_PAGE_LEVEL - 1L));
+ num_granules = S2TTES_PER_S2TT;
+ }
+
+ g_dev = find_lock_dev_granules(dev_mem_addr,
+ DEV_GRANULE_STATE_DELEGATED,
+ num_granules,
+ &type);
+ if (g_dev == NULL) {
+ granule_unlock(g_rd);
+ return RMI_ERROR_INPUT;
+ }
+
+ rd = buffer_granule_map(g_rd, SLOT_RD);
+ assert(rd != NULL);
+
+ if (!addr_in_par(rd, map_addr) ||
+ !validate_map_addr(map_addr, level, rd)) {
+ buffer_unmap(rd);
+ granule_unlock(g_rd);
+ ret = RMI_ERROR_INPUT;
+ goto out_unlock_granules;
+ }
+
+ s2_ctx = rd->s2_ctx;
+ buffer_unmap(rd);
+ granule_lock(s2_ctx.g_rtt, GRANULE_STATE_RTT);
+ granule_unlock(g_rd);
+
+ s2tt_walk_lock_unlock(&s2_ctx, map_addr, level, &wi);
+ if (wi.last_level != level) {
+ ret = pack_return_code(RMI_ERROR_RTT,
+ (unsigned char)wi.last_level);
+ goto out_unlock_ll_table;
+ }
+
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
+ assert(s2tt != NULL);
+
+ s2tte = s2tte_read(&s2tt[wi.index]);
+
+ if (!s2tte_is_unassigned(&s2_ctx, s2tte)) {
+ ret = pack_return_code(RMI_ERROR_RTT, (unsigned char)level);
+ goto out_unmap_ll_table;
+ }
+
+ s2tte = s2tte_create_assigned_dev_unchanged(&s2_ctx, s2tte,
+ dev_mem_addr, level);
+ s2tte_write(&s2tt[wi.index], s2tte);
+ atomic_granule_get(wi.g_llt);
+
+ ret = RMI_SUCCESS;
+
+out_unmap_ll_table:
+ buffer_unmap(s2tt);
+out_unlock_ll_table:
+ granule_unlock(wi.g_llt);
+out_unlock_granules:
+ while (num_granules != 0UL) {
+ if (ret == RMI_SUCCESS) {
+ dev_granule_unlock_transition(&g_dev[--num_granules],
+ DEV_GRANULE_STATE_MAPPED);
+ } else {
+ dev_granule_unlock(&g_dev[--num_granules]);
+ }
+ }
+ return ret;
+}
+
+void smc_dev_mem_unmap(unsigned long rd_addr,
+ unsigned long map_addr,
+ unsigned long ulevel,
+ struct smc_result *res)
+{
+ struct granule *g_rd;
+ struct rd *rd;
+ struct s2tt_walk wi;
+ struct s2tt_context s2_ctx;
+ unsigned long dev_mem_addr, dev_addr, s2tte, *s2tt, num_granules;
+ long level = (long)ulevel;
+ __unused enum dev_coh_type type;
+
+ /* Dev_Mem_Map/Unmap commands can operate up to a level 2 block entry */
+ if ((level < S2TT_MIN_DEV_BLOCK_LEVEL) || (level > S2TT_PAGE_LEVEL)) {
+ res->x[0] = RMI_ERROR_INPUT;
+ res->x[2] = 0UL;
+ return;
+ }
+
+ g_rd = find_lock_granule(rd_addr, GRANULE_STATE_RD);
+ if (g_rd == NULL) {
+ res->x[0] = RMI_ERROR_INPUT;
+ res->x[2] = 0UL;
+ return;
+ }
+
+ rd = buffer_granule_map(g_rd, SLOT_RD);
+ assert(rd != NULL);
+
+ if (!addr_in_par(rd, map_addr) ||
+ !validate_map_addr(map_addr, level, rd)) {
+ buffer_unmap(rd);
+ granule_unlock(g_rd);
+ res->x[0] = RMI_ERROR_INPUT;
+ res->x[2] = 0UL;
+ return;
+ }
+
+ s2_ctx = rd->s2_ctx;
+ buffer_unmap(rd);
+
+ granule_lock(s2_ctx.g_rtt, GRANULE_STATE_RTT);
+ granule_unlock(g_rd);
+
+ s2tt_walk_lock_unlock(&s2_ctx, map_addr, level, &wi);
+ s2tt = buffer_granule_map(wi.g_llt, SLOT_RTT);
+ assert(s2tt != NULL);
+
+ if (wi.last_level != level) {
+ res->x[0] = pack_return_code(RMI_ERROR_RTT,
+ (unsigned char)wi.last_level);
+ goto out_unmap_ll_table;
+ }
+
+ s2tte = s2tte_read(&s2tt[wi.index]);
+
+ if (s2tte_is_assigned_dev_dev(&s2_ctx, s2tte, level)) {
+ dev_mem_addr = s2tte_pa(&s2_ctx, s2tte, level);
+ s2tte = s2tte_create_unassigned_destroyed(&s2_ctx);
+ s2tte_write(&s2tt[wi.index], s2tte);
+ if (level == S2TT_PAGE_LEVEL) {
+ s2tt_invalidate_page(&s2_ctx, map_addr);
+ } else {
+ s2tt_invalidate_block(&s2_ctx, map_addr);
+ }
+ } else if (s2tte_is_assigned_dev_empty(&s2_ctx, s2tte, level)) {
+ dev_mem_addr = s2tte_pa(&s2_ctx, s2tte, level);
+ s2tte = s2tte_create_unassigned_empty(&s2_ctx);
+ s2tte_write(&s2tt[wi.index], s2tte);
+ } else if (s2tte_is_assigned_dev_destroyed(&s2_ctx, s2tte, level)) {
+ dev_mem_addr = s2tte_pa(&s2_ctx, s2tte, level);
+ s2tte = s2tte_create_unassigned_destroyed(&s2_ctx);
+ s2tte_write(&s2tt[wi.index], s2tte);
+ } else {
+ res->x[0] = pack_return_code(RMI_ERROR_RTT,
+ (unsigned char)level);
+ goto out_unmap_ll_table;
+ }
+
+ atomic_granule_put(wi.g_llt);
+
+ num_granules = (level == S2TT_PAGE_LEVEL) ? 1UL : S2TTES_PER_S2TT;
+ dev_addr = dev_mem_addr;
+
+ for (unsigned long i = 0UL; i < num_granules; i++) {
+ struct dev_granule *g_dev;
+
+ g_dev = find_lock_dev_granule(dev_addr, DEV_GRANULE_STATE_MAPPED, &type);
+ assert(g_dev != NULL);
+ dev_granule_unlock_transition(g_dev, DEV_GRANULE_STATE_DELEGATED);
+ dev_addr += GRANULE_SIZE;
+ }
+
+ res->x[0] = RMI_SUCCESS;
+ res->x[1] = dev_mem_addr;
+out_unmap_ll_table:
+ res->x[2] = s2tt_skip_non_live_entries(&s2_ctx, map_addr, s2tt, &wi);
+ buffer_unmap(s2tt);
+ granule_unlock(wi.g_llt);
+}