Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index e062ab9..568c523 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -1,22 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
* Author: Joerg Roedel <jroedel@suse.de>
* Leo Duran <leo.duran@amd.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
+#define pr_fmt(fmt) "AMD-Vi: " fmt
+#define dev_fmt(fmt) pr_fmt(fmt)
+
#include <linux/pci.h>
#include <linux/acpi.h>
#include <linux/list.h>
@@ -32,6 +23,8 @@
#include <linux/mem_encrypt.h>
#include <asm/pci-direct.h>
#include <asm/iommu.h>
+#include <asm/apic.h>
+#include <asm/msidef.h>
#include <asm/gart.h>
#include <asm/x86_init.h>
#include <asm/iommu_table.h>
@@ -39,6 +32,7 @@
#include <asm/irq_remapping.h>
#include <linux/crash_dump.h>
+#include "amd_iommu.h"
#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
#include "irq_remapping.h"
@@ -186,12 +180,6 @@
bool amd_iommu_force_isolation __read_mostly;
/*
- * List of protection domains - used during resume
- */
-LIST_HEAD(amd_iommu_pd_list);
-spinlock_t amd_iommu_pd_lock;
-
-/*
* Pointer to the device table which is shared by all AMD IOMMUs
* it is indexed by the PCI device id or the HT unit id and contains
* information about the domain the device belongs to as well as the
@@ -356,7 +344,7 @@
static void iommu_set_exclusion_range(struct amd_iommu *iommu)
{
u64 start = iommu->exclusion_start & PAGE_MASK;
- u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
+ u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
u64 entry;
if (!iommu->exclusion_start)
@@ -421,6 +409,9 @@
static void iommu_disable(struct amd_iommu *iommu)
{
+ if (!iommu->mmio_base)
+ return;
+
/* Disable command buffer */
iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
@@ -443,9 +434,9 @@
static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
{
if (!request_mem_region(address, end, "amd_iommu")) {
- pr_err("AMD-Vi: Can not reserve memory region %llx-%llx for mmio\n",
+ pr_err("Can not reserve memory region %llx-%llx for mmio\n",
address, end);
- pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
+ pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
return NULL;
}
@@ -512,7 +503,7 @@
u32 ivhd_size = get_ivhd_header_size(h);
if (!ivhd_size) {
- pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type);
+ pr_err("Unsupported IVHD type %#x\n", h->type);
return -EINVAL;
}
@@ -553,7 +544,7 @@
checksum += p[i];
if (checksum != 0) {
/* ACPI table corrupt */
- pr_err(FW_BUG "AMD-Vi: IVRS invalid checksum\n");
+ pr_err(FW_BUG "IVRS invalid checksum\n");
return -ENODEV;
}
@@ -903,12 +894,22 @@
}
}
- old_devtb_phys = entry & PAGE_MASK;
+ /*
+ * When SME is enabled in the first kernel, the entry includes the
+ * memory encryption mask(sme_me_mask), we must remove the memory
+ * encryption mask to obtain the true physical address in kdump kernel.
+ */
+ old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
+
if (old_devtb_phys >= 0x100000000ULL) {
pr_err("The address of old device table is above 4G, not trustworthy!\n");
return false;
}
- old_devtb = memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB);
+ old_devtb = (sme_active() && is_kdump_kernel())
+ ? (__force void *)ioremap_encrypted(old_devtb_phys,
+ dev_table_size)
+ : memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB);
+
if (!old_devtb)
return false;
@@ -1002,7 +1003,7 @@
set_iommu_for_device(iommu, devid);
}
-static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line)
+int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line)
{
struct devid_map *entry;
struct list_head *list;
@@ -1018,7 +1019,7 @@
if (!(entry->id == id && entry->cmd_line))
continue;
- pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n",
+ pr_info("Command-line override present for %s id %d - ignoring\n",
type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
*devid = entry->devid;
@@ -1051,7 +1052,7 @@
!entry->cmd_line)
continue;
- pr_info("AMD-Vi: Command-line override for hid:%s uid:%s\n",
+ pr_info("Command-line override for hid:%s uid:%s\n",
hid, uid);
*devid = entry->devid;
return 0;
@@ -1067,7 +1068,7 @@
entry->cmd_line = cmd_line;
entry->root_devid = (entry->devid & (~0x7));
- pr_info("AMD-Vi:%s, add hid:%s, uid:%s, rdevid:%d\n",
+ pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n",
entry->cmd_line ? "cmd" : "ivrs",
entry->hid, entry->uid, entry->root_devid);
@@ -1153,6 +1154,8 @@
if (ret)
return ret;
+ amd_iommu_apply_ivrs_quirks();
+
/*
* First save the recommended feature enable bits from ACPI
*/
@@ -1163,7 +1166,7 @@
*/
ivhd_size = get_ivhd_header_size(h);
if (!ivhd_size) {
- pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type);
+ pr_err("Unsupported IVHD type %#x\n", h->type);
return -EINVAL;
}
@@ -1445,8 +1448,7 @@
pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
- pr_info("AMD-Vi: Applying erratum 746 workaround for IOMMU at %s\n",
- dev_name(&iommu->dev->dev));
+ pci_info(iommu->dev, "Applying erratum 746 workaround\n");
/* Clear the enable writing bit */
pci_write_config_dword(iommu->dev, 0xf0, 0x90);
@@ -1476,8 +1478,7 @@
/* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
iommu_write_l2(iommu, 0x47, value | BIT(0));
- pr_info("AMD-Vi: Applying ATS write check workaround for IOMMU at %s\n",
- dev_name(&iommu->dev->dev));
+ pci_info(iommu->dev, "Applying ATS write check workaround\n");
}
/*
@@ -1496,7 +1497,7 @@
iommu->index = amd_iommus_present++;
if (unlikely(iommu->index >= MAX_IOMMUS)) {
- WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
+ WARN(1, "System has more IOMMUs than supported by this driver\n");
return -ENOSYS;
}
@@ -1653,6 +1654,7 @@
static void init_iommu_perf_ctr(struct amd_iommu *iommu)
{
+ struct pci_dev *pdev = iommu->dev;
u64 val = 0xabcd, val2 = 0;
if (!iommu_feature(iommu, FEATURE_PC))
@@ -1664,12 +1666,12 @@
if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) ||
(iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) ||
(val != val2)) {
- pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n");
+ pci_err(pdev, "Unable to write to IOMMU perf counter.\n");
amd_iommu_pc_present = false;
return;
}
- pr_info("AMD-Vi: IOMMU performance counters supported\n");
+ pci_info(pdev, "IOMMU performance counters supported\n");
val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
iommu->max_banks = (u8) ((val >> 12) & 0x3f);
@@ -1710,7 +1712,7 @@
NULL,
};
-static int iommu_init_pci(struct amd_iommu *iommu)
+static int __init iommu_init_pci(struct amd_iommu *iommu)
{
int cap_ptr = iommu->cap_ptr;
u32 range, misc, low, high;
@@ -1828,14 +1830,14 @@
struct amd_iommu *iommu;
for_each_iommu(iommu) {
+ struct pci_dev *pdev = iommu->dev;
int i;
- pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
- dev_name(&iommu->dev->dev), iommu->cap_ptr);
+ pci_info(pdev, "Found IOMMU cap 0x%hx\n", iommu->cap_ptr);
if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
- pr_info("AMD-Vi: Extended features (%#llx):\n",
- iommu->features);
+ pci_info(pdev, "Extended features (%#llx):\n",
+ iommu->features);
for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
if (iommu_feature(iommu, (1ULL << i)))
pr_cont(" %s", feat_str[i]);
@@ -1848,11 +1850,11 @@
}
}
if (irq_remapping_enabled) {
- pr_info("AMD-Vi: Interrupt remapping enabled\n");
+ pr_info("Interrupt remapping enabled\n");
if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
- pr_info("AMD-Vi: virtual APIC enabled\n");
+ pr_info("Virtual APIC enabled\n");
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
- pr_info("AMD-Vi: X2APIC enabled\n");
+ pr_info("X2APIC enabled\n");
}
}
@@ -1923,6 +1925,90 @@
return 0;
}
+#define XT_INT_DEST_MODE(x) (((x) & 0x1ULL) << 2)
+#define XT_INT_DEST_LO(x) (((x) & 0xFFFFFFULL) << 8)
+#define XT_INT_VEC(x) (((x) & 0xFFULL) << 32)
+#define XT_INT_DEST_HI(x) ((((x) >> 24) & 0xFFULL) << 56)
+
+/**
+ * Setup the IntCapXT registers with interrupt routing information
+ * based on the PCI MSI capability block registers, accessed via
+ * MMIO MSI address low/hi and MSI data registers.
+ */
+static void iommu_update_intcapxt(struct amd_iommu *iommu)
+{
+ u64 val;
+ u32 addr_lo = readl(iommu->mmio_base + MMIO_MSI_ADDR_LO_OFFSET);
+ u32 addr_hi = readl(iommu->mmio_base + MMIO_MSI_ADDR_HI_OFFSET);
+ u32 data = readl(iommu->mmio_base + MMIO_MSI_DATA_OFFSET);
+ bool dm = (addr_lo >> MSI_ADDR_DEST_MODE_SHIFT) & 0x1;
+ u32 dest = ((addr_lo >> MSI_ADDR_DEST_ID_SHIFT) & 0xFF);
+
+ if (x2apic_enabled())
+ dest |= MSI_ADDR_EXT_DEST_ID(addr_hi);
+
+ val = XT_INT_VEC(data & 0xFF) |
+ XT_INT_DEST_MODE(dm) |
+ XT_INT_DEST_LO(dest) |
+ XT_INT_DEST_HI(dest);
+
+ /**
+ * Current IOMMU implemtation uses the same IRQ for all
+ * 3 IOMMU interrupts.
+ */
+ writeq(val, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
+ writeq(val, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
+ writeq(val, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
+}
+
+static void _irq_notifier_notify(struct irq_affinity_notify *notify,
+ const cpumask_t *mask)
+{
+ struct amd_iommu *iommu;
+
+ for_each_iommu(iommu) {
+ if (iommu->dev->irq == notify->irq) {
+ iommu_update_intcapxt(iommu);
+ break;
+ }
+ }
+}
+
+static void _irq_notifier_release(struct kref *ref)
+{
+}
+
+static int iommu_init_intcapxt(struct amd_iommu *iommu)
+{
+ int ret;
+ struct irq_affinity_notify *notify = &iommu->intcapxt_notify;
+
+ /**
+ * IntCapXT requires XTSup=1, which can be inferred
+ * amd_iommu_xt_mode.
+ */
+ if (amd_iommu_xt_mode != IRQ_REMAP_X2APIC_MODE)
+ return 0;
+
+ /**
+ * Also, we need to setup notifier to update the IntCapXT registers
+ * whenever the irq affinity is changed from user-space.
+ */
+ notify->irq = iommu->dev->irq;
+ notify->notify = _irq_notifier_notify,
+ notify->release = _irq_notifier_release,
+ ret = irq_set_affinity_notifier(iommu->dev->irq, notify);
+ if (ret) {
+ pr_err("Failed to register irq affinity notifier (devid=%#x, irq %d)\n",
+ iommu->devid, iommu->dev->irq);
+ return ret;
+ }
+
+ iommu_update_intcapxt(iommu);
+ iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
+ return ret;
+}
+
static int iommu_init_msi(struct amd_iommu *iommu)
{
int ret;
@@ -1939,6 +2025,10 @@
return ret;
enable_faults:
+ ret = iommu_init_intcapxt(iommu);
+ if (ret)
+ return ret;
+
iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
if (iommu->ppr_log != NULL)
@@ -2001,6 +2091,9 @@
if (e == NULL)
return -ENOMEM;
+ if (m->flags & IVMD_FLAG_EXCL_RANGE)
+ init_exclusion_range(m);
+
switch (m->type) {
default:
kfree(e);
@@ -2047,9 +2140,7 @@
while (p < end) {
m = (struct ivmd_header *)p;
- if (m->flags & IVMD_FLAG_EXCL_RANGE)
- init_exclusion_range(m);
- else if (m->flags & IVMD_FLAG_UNITY_MAP)
+ if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
init_unity_map_range(m);
p += m->length;
@@ -2330,15 +2421,6 @@
amd_iommu_dev_table = NULL;
free_iommu_all();
-
-#ifdef CONFIG_GART_IOMMU
- /*
- * We failed to initialize the AMD IOMMU - try fallback to GART
- * if possible.
- */
- gart_iommu_init();
-
-#endif
}
/* SB IOAPIC is always on this device in AMD systems */
@@ -2366,7 +2448,7 @@
devid = get_ioapic_devid(id);
if (devid < 0) {
- pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n",
+ pr_err("%s: IOAPIC[%d] not in IVRS table\n",
fw_bug, id);
ret = false;
} else if (devid == IOAPIC_SB_DEVID) {
@@ -2384,11 +2466,11 @@
* when the BIOS is buggy and provides us the wrong
* device id for the IOAPIC in the system.
*/
- pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug);
+ pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
}
if (!ret)
- pr_err("AMD-Vi: Disabling interrupt remapping\n");
+ pr_err("Disabling interrupt remapping\n");
return ret;
}
@@ -2443,7 +2525,7 @@
return -ENODEV;
else if (ACPI_FAILURE(status)) {
const char *err = acpi_format_exception(status);
- pr_err("AMD-Vi: IVRS table error: %s\n", err);
+ pr_err("IVRS table error: %s\n", err);
return -EINVAL;
}
@@ -2513,8 +2595,6 @@
*/
__set_bit(0, amd_iommu_pd_alloc_bitmap);
- spin_lock_init(&amd_iommu_pd_lock);
-
/*
* now the data structures are allocated and basically initialized
* start the real acpi table scan
@@ -2596,7 +2676,7 @@
return false;
else if (ACPI_FAILURE(status)) {
const char *err = acpi_format_exception(status);
- pr_err("AMD-Vi: IVRS table error: %s\n", err);
+ pr_err("IVRS table error: %s\n", err);
return false;
}
@@ -2631,9 +2711,7 @@
ret = early_amd_iommu_init();
init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
if (init_state == IOMMU_ACPI_FINISHED && amd_iommu_disabled) {
- pr_info("AMD-Vi: AMD IOMMU disabled on kernel command-line\n");
- free_dma_resources();
- free_iommu_resources();
+ pr_info("AMD IOMMU disabled on kernel command-line\n");
init_state = IOMMU_CMDLINE_DISABLED;
ret = -EINVAL;
}
@@ -2674,6 +2752,19 @@
BUG();
}
+ if (ret) {
+ free_dma_resources();
+ if (!irq_remapping_enabled) {
+ disable_iommus();
+ free_iommu_resources();
+ } else {
+ struct amd_iommu *iommu;
+
+ uninit_device_table_dma();
+ for_each_iommu(iommu)
+ iommu_flush_all_caches(iommu);
+ }
+ }
return ret;
}
@@ -2747,17 +2838,15 @@
int ret;
ret = iommu_go_to_state(IOMMU_INITIALIZED);
- if (ret) {
- free_dma_resources();
- if (!irq_remapping_enabled) {
- disable_iommus();
- free_iommu_resources();
- } else {
- uninit_device_table_dma();
- for_each_iommu(iommu)
- iommu_flush_all_caches(iommu);
- }
+#ifdef CONFIG_GART_IOMMU
+ if (ret && list_empty(&amd_iommu_list)) {
+ /*
+ * We failed to initialize the AMD IOMMU - try fallback
+ * to GART if possible.
+ */
+ gart_iommu_init();
}
+#endif
for_each_iommu(iommu)
amd_iommu_debugfs_setup(iommu);
@@ -2778,7 +2867,7 @@
(boot_cpu_data.microcode <= 0x080011ff))
return true;
- pr_notice("AMD-Vi: IOMMU not currently supported when SME is active\n");
+ pr_notice("IOMMU not currently supported when SME is active\n");
return false;
}
@@ -2863,12 +2952,12 @@
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
- pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str);
+ pr_err("Invalid command line: ivrs_ioapic%s\n", str);
return 1;
}
if (early_ioapic_map_size == EARLY_MAP_SIZE) {
- pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
+ pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
str);
return 1;
}
@@ -2893,12 +2982,12 @@
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
- pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str);
+ pr_err("Invalid command line: ivrs_hpet%s\n", str);
return 1;
}
if (early_hpet_map_size == EARLY_MAP_SIZE) {
- pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n",
+ pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
str);
return 1;
}
@@ -2923,7 +3012,7 @@
ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid);
if (ret != 4) {
- pr_err("AMD-Vi: Invalid command line: ivrs_acpihid(%s)\n", str);
+ pr_err("Invalid command line: ivrs_acpihid(%s)\n", str);
return 1;
}
@@ -2932,7 +3021,7 @@
uid = p;
if (!hid || !(*hid) || !uid) {
- pr_err("AMD-Vi: Invalid command line: hid or uid\n");
+ pr_err("Invalid command line: hid or uid\n");
return 1;
}