Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index d5912f1..1edc993 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1,18 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
* Author: Marc Zyngier <marc.zyngier@arm.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define pr_fmt(fmt) "GICv3: " fmt
@@ -27,6 +16,7 @@
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/percpu.h>
+#include <linux/refcount.h>
#include <linux/slab.h>
#include <linux/irqchip.h>
@@ -41,6 +31,10 @@
#include "irq-gic-common.h"
+#define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
+
+#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
+
struct redist_region {
void __iomem *redist_base;
phys_addr_t phys_base;
@@ -55,14 +49,47 @@
struct irq_domain *domain;
u64 redist_stride;
u32 nr_redist_regions;
+ u64 flags;
bool has_rss;
- unsigned int irq_nr;
- struct partition_desc *ppi_descs[16];
+ unsigned int ppi_nr;
+ struct partition_desc **ppi_descs;
};
static struct gic_chip_data gic_data __read_mostly;
static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
+#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
+#define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
+#define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
+
+/*
+ * The behaviours of RPR and PMR registers differ depending on the value of
+ * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
+ * distributor and redistributors depends on whether security is enabled in the
+ * GIC.
+ *
+ * When security is enabled, non-secure priority values from the (re)distributor
+ * are presented to the GIC CPUIF as follow:
+ * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
+ *
+ * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
+ * EL1 are subject to a similar operation thus matching the priorities presented
+ * from the (re)distributor when security is enabled.
+ *
+ * see GICv3/GICv4 Architecture Specification (IHI0069D):
+ * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
+ * priorities.
+ * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
+ * interrupt.
+ *
+ * For now, we only support pseudo-NMIs if we have non-secure view of
+ * priorities.
+ */
+static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
+
+/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
+static refcount_t *ppi_nmi_refs;
+
static struct gic_kvm_info gic_v3_kvm_info;
static DEFINE_PER_CPU(bool, has_rss);
@@ -74,6 +101,38 @@
/* Our default, arbitrary priority value. Linux only uses one anyway. */
#define DEFAULT_PMR_VALUE 0xf0
+enum gic_intid_range {
+ PPI_RANGE,
+ SPI_RANGE,
+ EPPI_RANGE,
+ ESPI_RANGE,
+ LPI_RANGE,
+ __INVALID_RANGE__
+};
+
+static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
+{
+ switch (hwirq) {
+ case 16 ... 31:
+ return PPI_RANGE;
+ case 32 ... 1019:
+ return SPI_RANGE;
+ case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
+ return EPPI_RANGE;
+ case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
+ return ESPI_RANGE;
+ case 8192 ... GENMASK(23, 0):
+ return LPI_RANGE;
+ default:
+ return __INVALID_RANGE__;
+ }
+}
+
+static enum gic_intid_range get_intid_range(struct irq_data *d)
+{
+ return __get_intid_range(d->hwirq);
+}
+
static inline unsigned int gic_irq(struct irq_data *d)
{
return d->hwirq;
@@ -81,18 +140,26 @@
static inline int gic_irq_in_rdist(struct irq_data *d)
{
- return gic_irq(d) < 32;
+ enum gic_intid_range range = get_intid_range(d);
+ return range == PPI_RANGE || range == EPPI_RANGE;
}
static inline void __iomem *gic_dist_base(struct irq_data *d)
{
- if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ case EPPI_RANGE:
+ /* SGI+PPI -> SGI_base for this CPU */
return gic_data_rdist_sgi_base();
- if (d->hwirq <= 1023) /* SPI -> dist_base */
+ case SPI_RANGE:
+ case ESPI_RANGE:
+ /* SPI -> dist_base */
return gic_data.dist_base;
- return NULL;
+ default:
+ return NULL;
+ }
}
static void gic_do_wait_for_rwp(void __iomem *base)
@@ -139,6 +206,9 @@
u32 count = 1000000; /* 1s! */
u32 val;
+ if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
+ return;
+
rbase = gic_data_rdist_rd_base();
val = readl_relaxed(rbase + GICR_WAKER);
@@ -170,24 +240,79 @@
/*
* Routines to disable, enable, EOI and route interrupts
*/
+static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
+{
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ case SPI_RANGE:
+ *index = d->hwirq;
+ return offset;
+ case EPPI_RANGE:
+ /*
+ * Contrary to the ESPI range, the EPPI range is contiguous
+ * to the PPI range in the registers, so let's adjust the
+ * displacement accordingly. Consistency is overrated.
+ */
+ *index = d->hwirq - EPPI_BASE_INTID + 32;
+ return offset;
+ case ESPI_RANGE:
+ *index = d->hwirq - ESPI_BASE_INTID;
+ switch (offset) {
+ case GICD_ISENABLER:
+ return GICD_ISENABLERnE;
+ case GICD_ICENABLER:
+ return GICD_ICENABLERnE;
+ case GICD_ISPENDR:
+ return GICD_ISPENDRnE;
+ case GICD_ICPENDR:
+ return GICD_ICPENDRnE;
+ case GICD_ISACTIVER:
+ return GICD_ISACTIVERnE;
+ case GICD_ICACTIVER:
+ return GICD_ICACTIVERnE;
+ case GICD_IPRIORITYR:
+ return GICD_IPRIORITYRnE;
+ case GICD_ICFGR:
+ return GICD_ICFGRnE;
+ case GICD_IROUTER:
+ return GICD_IROUTERnE;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+
+ WARN_ON(1);
+ *index = d->hwirq;
+ return offset;
+}
+
static int gic_peek_irq(struct irq_data *d, u32 offset)
{
- u32 mask = 1 << (gic_irq(d) % 32);
void __iomem *base;
+ u32 index, mask;
+
+ offset = convert_offset_index(d, offset, &index);
+ mask = 1 << (index % 32);
if (gic_irq_in_rdist(d))
base = gic_data_rdist_sgi_base();
else
base = gic_data.dist_base;
- return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
+ return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
}
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
- u32 mask = 1 << (gic_irq(d) % 32);
void (*rwp_wait)(void);
void __iomem *base;
+ u32 index, mask;
+
+ offset = convert_offset_index(d, offset, &index);
+ mask = 1 << (index % 32);
if (gic_irq_in_rdist(d)) {
base = gic_data_rdist_sgi_base();
@@ -197,7 +322,7 @@
rwp_wait = gic_dist_wait_for_rwp;
}
- writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
+ writel_relaxed(mask, base + offset + (index / 32) * 4);
rwp_wait();
}
@@ -226,12 +351,18 @@
gic_poke_irq(d, GICD_ISENABLER);
}
+static inline bool gic_supports_nmi(void)
+{
+ return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
+ static_branch_likely(&supports_pseudo_nmis);
+}
+
static int gic_irq_set_irqchip_state(struct irq_data *d,
enum irqchip_irq_state which, bool val)
{
u32 reg;
- if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
+ if (d->hwirq >= 8192) /* PPI/SPI only */
return -EINVAL;
switch (which) {
@@ -258,7 +389,7 @@
static int gic_irq_get_irqchip_state(struct irq_data *d,
enum irqchip_irq_state which, bool *val)
{
- if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
+ if (d->hwirq >= 8192) /* PPI/SPI only */
return -EINVAL;
switch (which) {
@@ -281,6 +412,98 @@
return 0;
}
+static void gic_irq_set_prio(struct irq_data *d, u8 prio)
+{
+ void __iomem *base = gic_dist_base(d);
+ u32 offset, index;
+
+ offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
+
+ writeb_relaxed(prio, base + offset + index);
+}
+
+static u32 gic_get_ppi_index(struct irq_data *d)
+{
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ return d->hwirq - 16;
+ case EPPI_RANGE:
+ return d->hwirq - EPPI_BASE_INTID + 16;
+ default:
+ unreachable();
+ }
+}
+
+static int gic_irq_nmi_setup(struct irq_data *d)
+{
+ struct irq_desc *desc = irq_to_desc(d->irq);
+
+ if (!gic_supports_nmi())
+ return -EINVAL;
+
+ if (gic_peek_irq(d, GICD_ISENABLER)) {
+ pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
+ return -EINVAL;
+ }
+
+ /*
+ * A secondary irq_chip should be in charge of LPI request,
+ * it should not be possible to get there
+ */
+ if (WARN_ON(gic_irq(d) >= 8192))
+ return -EINVAL;
+
+ /* desc lock should already be held */
+ if (gic_irq_in_rdist(d)) {
+ u32 idx = gic_get_ppi_index(d);
+
+ /* Setting up PPI as NMI, only switch handler for first NMI */
+ if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
+ refcount_set(&ppi_nmi_refs[idx], 1);
+ desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
+ }
+ } else {
+ desc->handle_irq = handle_fasteoi_nmi;
+ }
+
+ gic_irq_set_prio(d, GICD_INT_NMI_PRI);
+
+ return 0;
+}
+
+static void gic_irq_nmi_teardown(struct irq_data *d)
+{
+ struct irq_desc *desc = irq_to_desc(d->irq);
+
+ if (WARN_ON(!gic_supports_nmi()))
+ return;
+
+ if (gic_peek_irq(d, GICD_ISENABLER)) {
+ pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
+ return;
+ }
+
+ /*
+ * A secondary irq_chip should be in charge of LPI request,
+ * it should not be possible to get there
+ */
+ if (WARN_ON(gic_irq(d) >= 8192))
+ return;
+
+ /* desc lock should already be held */
+ if (gic_irq_in_rdist(d)) {
+ u32 idx = gic_get_ppi_index(d);
+
+ /* Tearing down NMI, only switch handler for last NMI */
+ if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
+ desc->handle_irq = handle_percpu_devid_irq;
+ } else {
+ desc->handle_irq = handle_fasteoi_irq;
+ }
+
+ gic_irq_set_prio(d, GICD_INT_DEF_PRI);
+}
+
static void gic_eoi_irq(struct irq_data *d)
{
gic_write_eoir(gic_irq(d));
@@ -299,17 +522,22 @@
static int gic_set_type(struct irq_data *d, unsigned int type)
{
+ enum gic_intid_range range;
unsigned int irq = gic_irq(d);
void (*rwp_wait)(void);
void __iomem *base;
+ u32 offset, index;
+ int ret;
/* Interrupt configuration for SGIs can't be changed */
if (irq < 16)
return -EINVAL;
+ range = get_intid_range(d);
+
/* SPIs have restrictions on the supported types */
- if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
- type != IRQ_TYPE_EDGE_RISING)
+ if ((range == SPI_RANGE || range == ESPI_RANGE) &&
+ type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
return -EINVAL;
if (gic_irq_in_rdist(d)) {
@@ -320,7 +548,16 @@
rwp_wait = gic_dist_wait_for_rwp;
}
- return gic_configure_irq(irq, type, base, rwp_wait);
+ offset = convert_offset_index(d, GICD_ICFGR, &index);
+
+ ret = gic_configure_irq(index, type, base + offset, rwp_wait);
+ if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
+ /* Misconfigured PPIs are usually not fatal */
+ pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
+ ret = 0;
+ }
+
+ return ret;
}
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
@@ -344,52 +581,132 @@
return aff;
}
+static void gic_deactivate_unhandled(u32 irqnr)
+{
+ if (static_branch_likely(&supports_deactivate_key)) {
+ if (irqnr < 8192)
+ gic_write_dir(irqnr);
+ } else {
+ gic_write_eoir(irqnr);
+ }
+}
+
+static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
+{
+ bool irqs_enabled = interrupts_enabled(regs);
+ int err;
+
+ if (irqs_enabled)
+ nmi_enter();
+
+ if (static_branch_likely(&supports_deactivate_key))
+ gic_write_eoir(irqnr);
+ /*
+ * Leave the PSR.I bit set to prevent other NMIs to be
+ * received while handling this one.
+ * PSR.I will be restored when we ERET to the
+ * interrupted context.
+ */
+ err = handle_domain_nmi(gic_data.domain, irqnr, regs);
+ if (err)
+ gic_deactivate_unhandled(irqnr);
+
+ if (irqs_enabled)
+ nmi_exit();
+}
+
static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
{
u32 irqnr;
- do {
- irqnr = gic_read_iar();
+ irqnr = gic_read_iar();
- if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
- int err;
+ if (gic_supports_nmi() &&
+ unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) {
+ gic_handle_nmi(irqnr, regs);
+ return;
+ }
- if (static_branch_likely(&supports_deactivate_key))
- gic_write_eoir(irqnr);
- else
- isb();
+ if (gic_prio_masking_enabled()) {
+ gic_pmr_mask_irqs();
+ gic_arch_enable_irqs();
+ }
- err = handle_domain_irq(gic_data.domain, irqnr, regs);
- if (err) {
- WARN_ONCE(true, "Unexpected interrupt received!\n");
- if (static_branch_likely(&supports_deactivate_key)) {
- if (irqnr < 8192)
- gic_write_dir(irqnr);
- } else {
- gic_write_eoir(irqnr);
- }
- }
- continue;
- }
- if (irqnr < 16) {
+ /* Check for special IDs first */
+ if ((irqnr >= 1020 && irqnr <= 1023))
+ return;
+
+ /* Treat anything but SGIs in a uniform way */
+ if (likely(irqnr > 15)) {
+ int err;
+
+ if (static_branch_likely(&supports_deactivate_key))
gic_write_eoir(irqnr);
- if (static_branch_likely(&supports_deactivate_key))
- gic_write_dir(irqnr);
-#ifdef CONFIG_SMP
- /*
- * Unlike GICv2, we don't need an smp_rmb() here.
- * The control dependency from gic_read_iar to
- * the ISB in gic_write_eoir is enough to ensure
- * that any shared data read by handle_IPI will
- * be read after the ACK.
- */
- handle_IPI(irqnr, regs);
-#else
- WARN_ONCE(true, "Unexpected SGI received!\n");
-#endif
- continue;
+ else
+ isb();
+
+ err = handle_domain_irq(gic_data.domain, irqnr, regs);
+ if (err) {
+ WARN_ONCE(true, "Unexpected interrupt received!\n");
+ gic_deactivate_unhandled(irqnr);
}
- } while (irqnr != ICC_IAR1_EL1_SPURIOUS);
+ return;
+ }
+ if (irqnr < 16) {
+ gic_write_eoir(irqnr);
+ if (static_branch_likely(&supports_deactivate_key))
+ gic_write_dir(irqnr);
+#ifdef CONFIG_SMP
+ /*
+ * Unlike GICv2, we don't need an smp_rmb() here.
+ * The control dependency from gic_read_iar to
+ * the ISB in gic_write_eoir is enough to ensure
+ * that any shared data read by handle_IPI will
+ * be read after the ACK.
+ */
+ handle_IPI(irqnr, regs);
+#else
+ WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+ }
+}
+
+static u32 gic_get_pribits(void)
+{
+ u32 pribits;
+
+ pribits = gic_read_ctlr();
+ pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
+ pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
+ pribits++;
+
+ return pribits;
+}
+
+static bool gic_has_group0(void)
+{
+ u32 val;
+ u32 old_pmr;
+
+ old_pmr = gic_read_pmr();
+
+ /*
+ * Let's find out if Group0 is under control of EL3 or not by
+ * setting the highest possible, non-zero priority in PMR.
+ *
+ * If SCR_EL3.FIQ is set, the priority gets shifted down in
+ * order for the CPU interface to set bit 7, and keep the
+ * actual priority in the non-secure range. In the process, it
+ * looses the least significant bit and the actual priority
+ * becomes 0x80. Reading it back returns 0, indicating that
+ * we're don't have access to Group0.
+ */
+ gic_write_pmr(BIT(8 - gic_get_pribits()));
+ val = gic_read_pmr();
+
+ gic_write_pmr(old_pmr);
+
+ return val != 0;
}
static void __init gic_dist_init(void)
@@ -408,10 +725,26 @@
* do the right thing if the kernel is running in secure mode,
* but that's not the intended use case anyway.
*/
- for (i = 32; i < gic_data.irq_nr; i += 32)
+ for (i = 32; i < GIC_LINE_NR; i += 32)
writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
- gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
+ /* Extended SPI range, not handled by the GICv2/GICv3 common code */
+ for (i = 0; i < GIC_ESPI_NR; i += 32) {
+ writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
+ writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
+ }
+
+ for (i = 0; i < GIC_ESPI_NR; i += 32)
+ writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
+
+ for (i = 0; i < GIC_ESPI_NR; i += 16)
+ writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
+
+ for (i = 0; i < GIC_ESPI_NR; i += 4)
+ writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
+
+ /* Now do the common stuff, and wait for the distributor to drain */
+ gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
/* Enable distributor with ARE, Group1 */
writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
@@ -422,8 +755,11 @@
* enabled.
*/
affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
- for (i = 32; i < gic_data.irq_nr; i++)
+ for (i = 32; i < GIC_LINE_NR; i++)
gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
+
+ for (i = 0; i < GIC_ESPI_NR; i++)
+ gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
}
static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
@@ -509,31 +845,42 @@
return -ENODEV;
}
-static int __gic_update_vlpi_properties(struct redist_region *region,
- void __iomem *ptr)
+static int __gic_update_rdist_properties(struct redist_region *region,
+ void __iomem *ptr)
{
u64 typer = gic_read_typer(ptr + GICR_TYPER);
gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS);
+ gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
return 1;
}
-static void gic_update_vlpi_properties(void)
+static void gic_update_rdist_properties(void)
{
- gic_iterate_rdists(__gic_update_vlpi_properties);
+ gic_data.ppi_nr = UINT_MAX;
+ gic_iterate_rdists(__gic_update_rdist_properties);
+ if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
+ gic_data.ppi_nr = 0;
+ pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
pr_info("%sVLPI support, %sdirect LPI support\n",
!gic_data.rdists.has_vlpis ? "no " : "",
!gic_data.rdists.has_direct_lpi ? "no " : "");
}
+/* Check whether it's single security state view */
+static inline bool gic_dist_security_disabled(void)
+{
+ return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
+}
+
static void gic_cpu_sys_reg_init(void)
{
int i, cpu = smp_processor_id();
u64 mpidr = cpu_logical_map(cpu);
u64 need_rss = MPIDR_RS(mpidr);
bool group0;
- u32 val, pribits;
+ u32 pribits;
/*
* Need to check that the SRE bit has actually been set. If
@@ -545,28 +892,22 @@
if (!gic_enable_sre())
pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
- pribits = gic_read_ctlr();
- pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
- pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
- pribits++;
+ pribits = gic_get_pribits();
- /*
- * Let's find out if Group0 is under control of EL3 or not by
- * setting the highest possible, non-zero priority in PMR.
- *
- * If SCR_EL3.FIQ is set, the priority gets shifted down in
- * order for the CPU interface to set bit 7, and keep the
- * actual priority in the non-secure range. In the process, it
- * looses the least significant bit and the actual priority
- * becomes 0x80. Reading it back returns 0, indicating that
- * we're don't have access to Group0.
- */
- write_gicreg(BIT(8 - pribits), ICC_PMR_EL1);
- val = read_gicreg(ICC_PMR_EL1);
- group0 = val != 0;
+ group0 = gic_has_group0();
/* Set priority mask register */
- write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
+ if (!gic_prio_masking_enabled()) {
+ write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
+ } else {
+ /*
+ * Mismatch configuration with boot CPU, the system is likely
+ * to die as interrupt masking will not work properly on all
+ * CPUs
+ */
+ WARN_ON(gic_supports_nmi() && group0 &&
+ !gic_dist_security_disabled());
+ }
/*
* Some firmwares hand over to the kernel with the BPR changed from
@@ -591,8 +932,10 @@
case 7:
write_gicreg(0, ICC_AP0R3_EL1);
write_gicreg(0, ICC_AP0R2_EL1);
+ /* Fall through */
case 6:
write_gicreg(0, ICC_AP0R1_EL1);
+ /* Fall through */
case 5:
case 4:
write_gicreg(0, ICC_AP0R0_EL1);
@@ -606,8 +949,10 @@
case 7:
write_gicreg(0, ICC_AP1R3_EL1);
write_gicreg(0, ICC_AP1R2_EL1);
+ /* Fall through */
case 6:
write_gicreg(0, ICC_AP1R1_EL1);
+ /* Fall through */
case 5:
case 4:
write_gicreg(0, ICC_AP1R0_EL1);
@@ -653,12 +998,15 @@
static int gic_dist_supports_lpis(void)
{
- return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && !gicv3_nolpi;
+ return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
+ !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
+ !gicv3_nolpi);
}
static void gic_cpu_init(void)
{
void __iomem *rbase;
+ int i;
/* Register ourselves with the rest of the world */
if (gic_populate_rdist())
@@ -666,16 +1014,18 @@
gic_enable_redist(true);
+ WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
+ !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
+ "Distributor has extended ranges, but CPU%d doesn't\n",
+ smp_processor_id());
+
rbase = gic_data_rdist_sgi_base();
/* Configure SGIs/PPIs as non-secure Group-1 */
- writel_relaxed(~0, rbase + GICR_IGROUPR0);
+ for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
+ writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
- gic_cpu_config(rbase, gic_redist_wait_for_rwp);
-
- /* Give LPIs a spin */
- if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
- its_cpu_init();
+ gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
/* initialise system registers */
gic_cpu_sys_reg_init();
@@ -689,6 +1039,10 @@
static int gic_starting_cpu(unsigned int cpu)
{
gic_cpu_init();
+
+ if (gic_dist_supports_lpis())
+ its_cpu_init();
+
return 0;
}
@@ -775,6 +1129,7 @@
bool force)
{
unsigned int cpu;
+ u32 offset, index;
void __iomem *reg;
int enabled;
u64 val;
@@ -795,7 +1150,8 @@
if (enabled)
gic_mask_irq(d);
- reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
+ offset = convert_offset_index(d, GICD_IROUTER, &index);
+ reg = gic_dist_base(d) + offset + (index * 8);
val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
gic_write_irouter(val, reg);
@@ -819,12 +1175,6 @@
#endif
#ifdef CONFIG_CPU_PM
-/* Check whether it's single security state view */
-static bool gic_dist_security_disabled(void)
-{
- return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
-}
-
static int gic_cpu_pm_notifier(struct notifier_block *self,
unsigned long cmd, void *v)
{
@@ -861,6 +1211,8 @@
.irq_set_affinity = gic_set_affinity,
.irq_get_irqchip_state = gic_irq_get_irqchip_state,
.irq_set_irqchip_state = gic_irq_set_irqchip_state,
+ .irq_nmi_setup = gic_irq_nmi_setup,
+ .irq_nmi_teardown = gic_irq_nmi_teardown,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
@@ -876,13 +1228,13 @@
.irq_get_irqchip_state = gic_irq_get_irqchip_state,
.irq_set_irqchip_state = gic_irq_set_irqchip_state,
.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
+ .irq_nmi_setup = gic_irq_nmi_setup,
+ .irq_nmi_teardown = gic_irq_nmi_teardown,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
};
-#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
-
static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hw)
{
@@ -891,36 +1243,32 @@
if (static_branch_likely(&supports_deactivate_key))
chip = &gic_eoimode1_chip;
- /* SGIs are private to the core kernel */
- if (hw < 16)
- return -EPERM;
- /* Nothing here */
- if (hw >= gic_data.irq_nr && hw < 8192)
- return -EPERM;
- /* Off limits */
- if (hw >= GIC_ID_NR)
- return -EPERM;
-
- /* PPIs */
- if (hw < 32) {
+ switch (__get_intid_range(hw)) {
+ case PPI_RANGE:
+ case EPPI_RANGE:
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_percpu_devid_irq, NULL, NULL);
irq_set_status_flags(irq, IRQ_NOAUTOEN);
- }
- /* SPIs */
- if (hw >= 32 && hw < gic_data.irq_nr) {
+ break;
+
+ case SPI_RANGE:
+ case ESPI_RANGE:
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
irq_set_probe(irq);
irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
- }
- /* LPIs */
- if (hw >= 8192 && hw < GIC_ID_NR) {
+ break;
+
+ case LPI_RANGE:
if (!gic_dist_supports_lpis())
return -EPERM;
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
+ break;
+
+ default:
+ return -EPERM;
}
return 0;
@@ -942,12 +1290,24 @@
*hwirq = fwspec->param[1] + 32;
break;
case 1: /* PPI */
- case GIC_IRQ_TYPE_PARTITION:
*hwirq = fwspec->param[1] + 16;
break;
+ case 2: /* ESPI */
+ *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
+ break;
+ case 3: /* EPPI */
+ *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
+ break;
case GIC_IRQ_TYPE_LPI: /* LPI */
*hwirq = fwspec->param[1];
break;
+ case GIC_IRQ_TYPE_PARTITION:
+ *hwirq = fwspec->param[1];
+ if (fwspec->param[1] >= 16)
+ *hwirq += EPPI_BASE_INTID - 16;
+ else
+ *hwirq += 16;
+ break;
default:
return -EINVAL;
}
@@ -1027,7 +1387,8 @@
* then we need to match the partition domain.
*/
if (fwspec->param_count >= 4 &&
- fwspec->param[0] == 1 && fwspec->param[3] != 0)
+ fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
+ gic_data.ppi_descs)
return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
return d == gic_data.domain;
@@ -1048,6 +1409,9 @@
struct device_node *np;
int ret;
+ if (!gic_data.ppi_descs)
+ return -ENOMEM;
+
np = of_find_node_by_phandle(fwspec->param[3]);
if (WARN_ON(!np))
return -EINVAL;
@@ -1068,6 +1432,84 @@
.select = gic_irq_domain_select,
};
+static bool gic_enable_quirk_msm8996(void *data)
+{
+ struct gic_chip_data *d = data;
+
+ d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
+
+ return true;
+}
+
+static bool gic_enable_quirk_hip06_07(void *data)
+{
+ struct gic_chip_data *d = data;
+
+ /*
+ * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
+ * not being an actual ARM implementation). The saving grace is
+ * that GIC-600 doesn't have ESPI, so nothing to do in that case.
+ * HIP07 doesn't even have a proper IIDR, and still pretends to
+ * have ESPI. In both cases, put them right.
+ */
+ if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
+ /* Zero both ESPI and the RES0 field next to it... */
+ d->rdists.gicd_typer &= ~GENMASK(9, 8);
+ return true;
+ }
+
+ return false;
+}
+
+static const struct gic_quirk gic_quirks[] = {
+ {
+ .desc = "GICv3: Qualcomm MSM8996 broken firmware",
+ .compatible = "qcom,msm8996-gic-v3",
+ .init = gic_enable_quirk_msm8996,
+ },
+ {
+ .desc = "GICv3: HIP06 erratum 161010803",
+ .iidr = 0x0204043b,
+ .mask = 0xffffffff,
+ .init = gic_enable_quirk_hip06_07,
+ },
+ {
+ .desc = "GICv3: HIP07 erratum 161010803",
+ .iidr = 0x00000000,
+ .mask = 0xffffffff,
+ .init = gic_enable_quirk_hip06_07,
+ },
+ {
+ }
+};
+
+static void gic_enable_nmi_support(void)
+{
+ int i;
+
+ if (!gic_prio_masking_enabled())
+ return;
+
+ if (gic_has_group0() && !gic_dist_security_disabled()) {
+ pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
+ return;
+ }
+
+ ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
+ if (!ppi_nmi_refs)
+ return;
+
+ for (i = 0; i < gic_data.ppi_nr; i++)
+ refcount_set(&ppi_nmi_refs[i], 0);
+
+ static_branch_enable(&supports_pseudo_nmis);
+
+ if (static_branch_likely(&supports_deactivate_key))
+ gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+ else
+ gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
+}
+
static int __init gic_init_bases(void __iomem *dist_base,
struct redist_region *rdist_regs,
u32 nr_redist_regions,
@@ -1075,7 +1517,6 @@
struct fwnode_handle *handle)
{
u32 typer;
- int gic_irqs;
int err;
if (!is_hyp_mode_available())
@@ -1092,15 +1533,15 @@
/*
* Find out how many interrupts are supported.
- * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
*/
typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
gic_data.rdists.gicd_typer = typer;
- gic_irqs = GICD_TYPER_IRQS(typer);
- if (gic_irqs > 1020)
- gic_irqs = 1020;
- gic_data.irq_nr = gic_irqs;
+ gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
+ gic_quirks, &gic_data);
+
+ pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
+ pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
&gic_data);
irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
@@ -1125,16 +1566,23 @@
set_handle_irq(gic_handle_irq);
- gic_update_vlpi_properties();
-
- if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
- its_init(handle, &gic_data.rdists, gic_data.domain);
+ gic_update_rdist_properties();
gic_smp_init();
gic_dist_init();
gic_cpu_init();
gic_cpu_pm_init();
+ if (gic_dist_supports_lpis()) {
+ its_init(handle, &gic_data.rdists, gic_data.domain);
+ its_cpu_init();
+ } else {
+ if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
+ gicv2m_init(handle, gic_data.domain);
+ }
+
+ gic_enable_nmi_support();
+
return 0;
out_free:
@@ -1166,6 +1614,10 @@
if (!parts_node)
return;
+ gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
+ if (!gic_data.ppi_descs)
+ return;
+
nr_parts = of_get_child_count(parts_node);
if (!nr_parts)
@@ -1183,8 +1635,8 @@
part->partition_id = of_node_to_fwnode(child_part);
- pr_info("GIC: PPI partition %s[%d] { ",
- child_part->name, part_idx);
+ pr_info("GIC: PPI partition %pOFn[%d] { ",
+ child_part, part_idx);
n = of_property_count_elems_of_size(child_part, "affinity",
sizeof(u32));
@@ -1217,7 +1669,7 @@
part_idx++;
}
- for (i = 0; i < 16; i++) {
+ for (i = 0; i < gic_data.ppi_nr; i++) {
unsigned int irq;
struct partition_desc *desc;
struct irq_fwspec ppi_fwspec = {
@@ -1317,6 +1769,8 @@
if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
redist_stride = 0;
+ gic_enable_of_quirks(node, gic_quirks, &gic_data);
+
err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
redist_stride, &node->fwnode);
if (err)
@@ -1364,7 +1818,7 @@
}
static int __init
-gic_acpi_parse_madt_redist(struct acpi_subtable_header *header,
+gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_madt_generic_redistributor *redist =
@@ -1382,7 +1836,7 @@
}
static int __init
-gic_acpi_parse_madt_gicc(struct acpi_subtable_header *header,
+gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_madt_generic_interrupt *gicc =
@@ -1424,14 +1878,14 @@
return -ENODEV;
}
-static int __init gic_acpi_match_gicr(struct acpi_subtable_header *header,
+static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
const unsigned long end)
{
/* Subtable presence means that redist exists, that's it */
return 0;
}
-static int __init gic_acpi_match_gicc(struct acpi_subtable_header *header,
+static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_madt_generic_interrupt *gicc =
@@ -1497,7 +1951,7 @@
return true;
}
-static int __init gic_acpi_parse_virt_madt_gicc(struct acpi_subtable_header *header,
+static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_madt_generic_interrupt *gicc =
@@ -1613,7 +2067,7 @@
if (err)
goto out_redist_unmap;
- domain_handle = irq_domain_alloc_fwnode(acpi_data.dist_base);
+ domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
if (!domain_handle) {
err = -ENOMEM;
goto out_redist_unmap;