Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | #include <linux/kernel.h> |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 3 | #include <linux/pgtable.h> |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 4 | |
| 5 | #include <linux/string.h> |
| 6 | #include <linux/bitops.h> |
| 7 | #include <linux/smp.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/sched/clock.h> |
| 10 | #include <linux/thread_info.h> |
| 11 | #include <linux/init.h> |
| 12 | #include <linux/uaccess.h> |
| 13 | |
| 14 | #include <asm/cpufeature.h> |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 15 | #include <asm/msr.h> |
| 16 | #include <asm/bugs.h> |
| 17 | #include <asm/cpu.h> |
| 18 | #include <asm/intel-family.h> |
| 19 | #include <asm/microcode_intel.h> |
| 20 | #include <asm/hwcap2.h> |
| 21 | #include <asm/elf.h> |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 22 | #include <asm/cpu_device_id.h> |
| 23 | #include <asm/cmdline.h> |
| 24 | #include <asm/traps.h> |
| 25 | #include <asm/resctrl.h> |
| 26 | #include <asm/numa.h> |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 27 | |
| 28 | #ifdef CONFIG_X86_64 |
| 29 | #include <linux/topology.h> |
| 30 | #endif |
| 31 | |
| 32 | #include "cpu.h" |
| 33 | |
| 34 | #ifdef CONFIG_X86_LOCAL_APIC |
| 35 | #include <asm/mpspec.h> |
| 36 | #include <asm/apic.h> |
| 37 | #endif |
| 38 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 39 | enum split_lock_detect_state { |
| 40 | sld_off = 0, |
| 41 | sld_warn, |
| 42 | sld_fatal, |
| 43 | }; |
| 44 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 45 | /* |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 46 | * Default to sld_off because most systems do not support split lock detection |
| 47 | * split_lock_setup() will switch this to sld_warn on systems that support |
| 48 | * split lock detect, unless there is a command line override. |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 49 | */ |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 50 | static enum split_lock_detect_state sld_state __ro_after_init = sld_off; |
| 51 | static u64 msr_test_ctrl_cache __ro_after_init; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 52 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 53 | /* |
| 54 | * With a name like MSR_TEST_CTL it should go without saying, but don't touch |
| 55 | * MSR_TEST_CTL unless the CPU is one of the whitelisted models. Writing it |
| 56 | * on CPUs that do not support SLD can cause fireworks, even when writing '0'. |
| 57 | */ |
| 58 | static bool cpu_model_supports_sld __ro_after_init; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 59 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 60 | /* |
| 61 | * Processors which have self-snooping capability can handle conflicting |
| 62 | * memory type across CPUs by snooping its own cache. However, there exists |
| 63 | * CPU models in which having conflicting memory types still leads to |
| 64 | * unpredictable behavior, machine check errors, or hangs. Clear this |
| 65 | * feature to prevent its use on machines with known erratas. |
| 66 | */ |
| 67 | static void check_memory_type_self_snoop_errata(struct cpuinfo_x86 *c) |
| 68 | { |
| 69 | switch (c->x86_model) { |
| 70 | case INTEL_FAM6_CORE_YONAH: |
| 71 | case INTEL_FAM6_CORE2_MEROM: |
| 72 | case INTEL_FAM6_CORE2_MEROM_L: |
| 73 | case INTEL_FAM6_CORE2_PENRYN: |
| 74 | case INTEL_FAM6_CORE2_DUNNINGTON: |
| 75 | case INTEL_FAM6_NEHALEM: |
| 76 | case INTEL_FAM6_NEHALEM_G: |
| 77 | case INTEL_FAM6_NEHALEM_EP: |
| 78 | case INTEL_FAM6_NEHALEM_EX: |
| 79 | case INTEL_FAM6_WESTMERE: |
| 80 | case INTEL_FAM6_WESTMERE_EP: |
| 81 | case INTEL_FAM6_SANDYBRIDGE: |
| 82 | setup_clear_cpu_cap(X86_FEATURE_SELFSNOOP); |
| 83 | } |
| 84 | } |
| 85 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 86 | static bool ring3mwait_disabled __read_mostly; |
| 87 | |
| 88 | static int __init ring3mwait_disable(char *__unused) |
| 89 | { |
| 90 | ring3mwait_disabled = true; |
| 91 | return 0; |
| 92 | } |
| 93 | __setup("ring3mwait=disable", ring3mwait_disable); |
| 94 | |
| 95 | static void probe_xeon_phi_r3mwait(struct cpuinfo_x86 *c) |
| 96 | { |
| 97 | /* |
| 98 | * Ring 3 MONITOR/MWAIT feature cannot be detected without |
| 99 | * cpu model and family comparison. |
| 100 | */ |
| 101 | if (c->x86 != 6) |
| 102 | return; |
| 103 | switch (c->x86_model) { |
| 104 | case INTEL_FAM6_XEON_PHI_KNL: |
| 105 | case INTEL_FAM6_XEON_PHI_KNM: |
| 106 | break; |
| 107 | default: |
| 108 | return; |
| 109 | } |
| 110 | |
| 111 | if (ring3mwait_disabled) |
| 112 | return; |
| 113 | |
| 114 | set_cpu_cap(c, X86_FEATURE_RING3MWAIT); |
| 115 | this_cpu_or(msr_misc_features_shadow, |
| 116 | 1UL << MSR_MISC_FEATURES_ENABLES_RING3MWAIT_BIT); |
| 117 | |
| 118 | if (c == &boot_cpu_data) |
| 119 | ELF_HWCAP2 |= HWCAP2_RING3MWAIT; |
| 120 | } |
| 121 | |
| 122 | /* |
| 123 | * Early microcode releases for the Spectre v2 mitigation were broken. |
| 124 | * Information taken from; |
| 125 | * - https://newsroom.intel.com/wp-content/uploads/sites/11/2018/03/microcode-update-guidance.pdf |
| 126 | * - https://kb.vmware.com/s/article/52345 |
| 127 | * - Microcode revisions observed in the wild |
| 128 | * - Release note from 20180108 microcode release |
| 129 | */ |
| 130 | struct sku_microcode { |
| 131 | u8 model; |
| 132 | u8 stepping; |
| 133 | u32 microcode; |
| 134 | }; |
| 135 | static const struct sku_microcode spectre_bad_microcodes[] = { |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 136 | { INTEL_FAM6_KABYLAKE, 0x0B, 0x80 }, |
| 137 | { INTEL_FAM6_KABYLAKE, 0x0A, 0x80 }, |
| 138 | { INTEL_FAM6_KABYLAKE, 0x09, 0x80 }, |
| 139 | { INTEL_FAM6_KABYLAKE_L, 0x0A, 0x80 }, |
| 140 | { INTEL_FAM6_KABYLAKE_L, 0x09, 0x80 }, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 141 | { INTEL_FAM6_SKYLAKE_X, 0x03, 0x0100013e }, |
| 142 | { INTEL_FAM6_SKYLAKE_X, 0x04, 0x0200003c }, |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 143 | { INTEL_FAM6_BROADWELL, 0x04, 0x28 }, |
| 144 | { INTEL_FAM6_BROADWELL_G, 0x01, 0x1b }, |
| 145 | { INTEL_FAM6_BROADWELL_D, 0x02, 0x14 }, |
| 146 | { INTEL_FAM6_BROADWELL_D, 0x03, 0x07000011 }, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 147 | { INTEL_FAM6_BROADWELL_X, 0x01, 0x0b000025 }, |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 148 | { INTEL_FAM6_HASWELL_L, 0x01, 0x21 }, |
| 149 | { INTEL_FAM6_HASWELL_G, 0x01, 0x18 }, |
| 150 | { INTEL_FAM6_HASWELL, 0x03, 0x23 }, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 151 | { INTEL_FAM6_HASWELL_X, 0x02, 0x3b }, |
| 152 | { INTEL_FAM6_HASWELL_X, 0x04, 0x10 }, |
| 153 | { INTEL_FAM6_IVYBRIDGE_X, 0x04, 0x42a }, |
| 154 | /* Observed in the wild */ |
| 155 | { INTEL_FAM6_SANDYBRIDGE_X, 0x06, 0x61b }, |
| 156 | { INTEL_FAM6_SANDYBRIDGE_X, 0x07, 0x712 }, |
| 157 | }; |
| 158 | |
| 159 | static bool bad_spectre_microcode(struct cpuinfo_x86 *c) |
| 160 | { |
| 161 | int i; |
| 162 | |
| 163 | /* |
| 164 | * We know that the hypervisor lie to us on the microcode version so |
| 165 | * we may as well hope that it is running the correct version. |
| 166 | */ |
| 167 | if (cpu_has(c, X86_FEATURE_HYPERVISOR)) |
| 168 | return false; |
| 169 | |
| 170 | if (c->x86 != 6) |
| 171 | return false; |
| 172 | |
| 173 | for (i = 0; i < ARRAY_SIZE(spectre_bad_microcodes); i++) { |
| 174 | if (c->x86_model == spectre_bad_microcodes[i].model && |
| 175 | c->x86_stepping == spectre_bad_microcodes[i].stepping) |
| 176 | return (c->microcode <= spectre_bad_microcodes[i].microcode); |
| 177 | } |
| 178 | return false; |
| 179 | } |
| 180 | |
| 181 | static void early_init_intel(struct cpuinfo_x86 *c) |
| 182 | { |
| 183 | u64 misc_enable; |
| 184 | |
| 185 | /* Unmask CPUID levels if masked: */ |
| 186 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { |
| 187 | if (msr_clear_bit(MSR_IA32_MISC_ENABLE, |
| 188 | MSR_IA32_MISC_ENABLE_LIMIT_CPUID_BIT) > 0) { |
| 189 | c->cpuid_level = cpuid_eax(0); |
| 190 | get_cpu_cap(c); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | if ((c->x86 == 0xf && c->x86_model >= 0x03) || |
| 195 | (c->x86 == 0x6 && c->x86_model >= 0x0e)) |
| 196 | set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); |
| 197 | |
| 198 | if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64)) |
| 199 | c->microcode = intel_get_microcode_revision(); |
| 200 | |
| 201 | /* Now if any of them are set, check the blacklist and clear the lot */ |
| 202 | if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) || |
| 203 | cpu_has(c, X86_FEATURE_INTEL_STIBP) || |
| 204 | cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) || |
| 205 | cpu_has(c, X86_FEATURE_STIBP)) && bad_spectre_microcode(c)) { |
| 206 | pr_warn("Intel Spectre v2 broken microcode detected; disabling Speculation Control\n"); |
| 207 | setup_clear_cpu_cap(X86_FEATURE_IBRS); |
| 208 | setup_clear_cpu_cap(X86_FEATURE_IBPB); |
| 209 | setup_clear_cpu_cap(X86_FEATURE_STIBP); |
| 210 | setup_clear_cpu_cap(X86_FEATURE_SPEC_CTRL); |
| 211 | setup_clear_cpu_cap(X86_FEATURE_MSR_SPEC_CTRL); |
| 212 | setup_clear_cpu_cap(X86_FEATURE_INTEL_STIBP); |
| 213 | setup_clear_cpu_cap(X86_FEATURE_SSBD); |
| 214 | setup_clear_cpu_cap(X86_FEATURE_SPEC_CTRL_SSBD); |
| 215 | } |
| 216 | |
| 217 | /* |
| 218 | * Atom erratum AAE44/AAF40/AAG38/AAH41: |
| 219 | * |
| 220 | * A race condition between speculative fetches and invalidating |
| 221 | * a large page. This is worked around in microcode, but we |
| 222 | * need the microcode to have already been loaded... so if it is |
| 223 | * not, recommend a BIOS update and disable large pages. |
| 224 | */ |
| 225 | if (c->x86 == 6 && c->x86_model == 0x1c && c->x86_stepping <= 2 && |
| 226 | c->microcode < 0x20e) { |
| 227 | pr_warn("Atom PSE erratum detected, BIOS microcode update recommended\n"); |
| 228 | clear_cpu_cap(c, X86_FEATURE_PSE); |
| 229 | } |
| 230 | |
| 231 | #ifdef CONFIG_X86_64 |
| 232 | set_cpu_cap(c, X86_FEATURE_SYSENTER32); |
| 233 | #else |
| 234 | /* Netburst reports 64 bytes clflush size, but does IO in 128 bytes */ |
| 235 | if (c->x86 == 15 && c->x86_cache_alignment == 64) |
| 236 | c->x86_cache_alignment = 128; |
| 237 | #endif |
| 238 | |
| 239 | /* CPUID workaround for 0F33/0F34 CPU */ |
| 240 | if (c->x86 == 0xF && c->x86_model == 0x3 |
| 241 | && (c->x86_stepping == 0x3 || c->x86_stepping == 0x4)) |
| 242 | c->x86_phys_bits = 36; |
| 243 | |
| 244 | /* |
| 245 | * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate |
| 246 | * with P/T states and does not stop in deep C-states. |
| 247 | * |
| 248 | * It is also reliable across cores and sockets. (but not across |
| 249 | * cabinets - we turn it off in that case explicitly.) |
| 250 | */ |
| 251 | if (c->x86_power & (1 << 8)) { |
| 252 | set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); |
| 253 | set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); |
| 254 | } |
| 255 | |
| 256 | /* Penwell and Cloverview have the TSC which doesn't sleep on S3 */ |
| 257 | if (c->x86 == 6) { |
| 258 | switch (c->x86_model) { |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 259 | case INTEL_FAM6_ATOM_SALTWELL_MID: |
| 260 | case INTEL_FAM6_ATOM_SALTWELL_TABLET: |
| 261 | case INTEL_FAM6_ATOM_SILVERMONT_MID: |
| 262 | case INTEL_FAM6_ATOM_AIRMONT_NP: |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 263 | set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC_S3); |
| 264 | break; |
| 265 | default: |
| 266 | break; |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | /* |
| 271 | * There is a known erratum on Pentium III and Core Solo |
| 272 | * and Core Duo CPUs. |
| 273 | * " Page with PAT set to WC while associated MTRR is UC |
| 274 | * may consolidate to UC " |
| 275 | * Because of this erratum, it is better to stick with |
| 276 | * setting WC in MTRR rather than using PAT on these CPUs. |
| 277 | * |
| 278 | * Enable PAT WC only on P4, Core 2 or later CPUs. |
| 279 | */ |
| 280 | if (c->x86 == 6 && c->x86_model < 15) |
| 281 | clear_cpu_cap(c, X86_FEATURE_PAT); |
| 282 | |
| 283 | /* |
| 284 | * If fast string is not enabled in IA32_MISC_ENABLE for any reason, |
| 285 | * clear the fast string and enhanced fast string CPU capabilities. |
| 286 | */ |
| 287 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { |
| 288 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); |
| 289 | if (!(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) { |
| 290 | pr_info("Disabled fast string operations\n"); |
| 291 | setup_clear_cpu_cap(X86_FEATURE_REP_GOOD); |
| 292 | setup_clear_cpu_cap(X86_FEATURE_ERMS); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | /* |
| 297 | * Intel Quark Core DevMan_001.pdf section 6.4.11 |
| 298 | * "The operating system also is required to invalidate (i.e., flush) |
| 299 | * the TLB when any changes are made to any of the page table entries. |
| 300 | * The operating system must reload CR3 to cause the TLB to be flushed" |
| 301 | * |
| 302 | * As a result, boot_cpu_has(X86_FEATURE_PGE) in arch/x86/include/asm/tlbflush.h |
| 303 | * should be false so that __flush_tlb_all() causes CR3 insted of CR4.PGE |
| 304 | * to be modified. |
| 305 | */ |
| 306 | if (c->x86 == 5 && c->x86_model == 9) { |
| 307 | pr_info("Disabling PGE capability bit\n"); |
| 308 | setup_clear_cpu_cap(X86_FEATURE_PGE); |
| 309 | } |
| 310 | |
| 311 | if (c->cpuid_level >= 0x00000001) { |
| 312 | u32 eax, ebx, ecx, edx; |
| 313 | |
| 314 | cpuid(0x00000001, &eax, &ebx, &ecx, &edx); |
| 315 | /* |
| 316 | * If HTT (EDX[28]) is set EBX[16:23] contain the number of |
| 317 | * apicids which are reserved per package. Store the resulting |
| 318 | * shift value for the package management code. |
| 319 | */ |
| 320 | if (edx & (1U << 28)) |
| 321 | c->x86_coreid_bits = get_count_order((ebx >> 16) & 0xff); |
| 322 | } |
| 323 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 324 | check_memory_type_self_snoop_errata(c); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 325 | |
| 326 | /* |
| 327 | * Get the number of SMT siblings early from the extended topology |
| 328 | * leaf, if available. Otherwise try the legacy SMT detection. |
| 329 | */ |
| 330 | if (detect_extended_topology_early(c) < 0) |
| 331 | detect_ht_early(c); |
| 332 | } |
| 333 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 334 | static void bsp_init_intel(struct cpuinfo_x86 *c) |
| 335 | { |
| 336 | resctrl_cpu_detect(c); |
| 337 | } |
| 338 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 339 | #ifdef CONFIG_X86_32 |
| 340 | /* |
| 341 | * Early probe support logic for ppro memory erratum #50 |
| 342 | * |
| 343 | * This is called before we do cpu ident work |
| 344 | */ |
| 345 | |
| 346 | int ppro_with_ram_bug(void) |
| 347 | { |
| 348 | /* Uses data from early_cpu_detect now */ |
| 349 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && |
| 350 | boot_cpu_data.x86 == 6 && |
| 351 | boot_cpu_data.x86_model == 1 && |
| 352 | boot_cpu_data.x86_stepping < 8) { |
| 353 | pr_info("Pentium Pro with Errata#50 detected. Taking evasive action.\n"); |
| 354 | return 1; |
| 355 | } |
| 356 | return 0; |
| 357 | } |
| 358 | |
| 359 | static void intel_smp_check(struct cpuinfo_x86 *c) |
| 360 | { |
| 361 | /* calling is from identify_secondary_cpu() ? */ |
| 362 | if (!c->cpu_index) |
| 363 | return; |
| 364 | |
| 365 | /* |
| 366 | * Mask B, Pentium, but not Pentium MMX |
| 367 | */ |
| 368 | if (c->x86 == 5 && |
| 369 | c->x86_stepping >= 1 && c->x86_stepping <= 4 && |
| 370 | c->x86_model <= 3) { |
| 371 | /* |
| 372 | * Remember we have B step Pentia with bugs |
| 373 | */ |
| 374 | WARN_ONCE(1, "WARNING: SMP operation may be unreliable" |
| 375 | "with B stepping processors.\n"); |
| 376 | } |
| 377 | } |
| 378 | |
| 379 | static int forcepae; |
| 380 | static int __init forcepae_setup(char *__unused) |
| 381 | { |
| 382 | forcepae = 1; |
| 383 | return 1; |
| 384 | } |
| 385 | __setup("forcepae", forcepae_setup); |
| 386 | |
| 387 | static void intel_workarounds(struct cpuinfo_x86 *c) |
| 388 | { |
| 389 | #ifdef CONFIG_X86_F00F_BUG |
| 390 | /* |
| 391 | * All models of Pentium and Pentium with MMX technology CPUs |
| 392 | * have the F0 0F bug, which lets nonprivileged users lock up the |
| 393 | * system. Announce that the fault handler will be checking for it. |
| 394 | * The Quark is also family 5, but does not have the same bug. |
| 395 | */ |
| 396 | clear_cpu_bug(c, X86_BUG_F00F); |
| 397 | if (c->x86 == 5 && c->x86_model < 9) { |
| 398 | static int f00f_workaround_enabled; |
| 399 | |
| 400 | set_cpu_bug(c, X86_BUG_F00F); |
| 401 | if (!f00f_workaround_enabled) { |
| 402 | pr_notice("Intel Pentium with F0 0F bug - workaround enabled.\n"); |
| 403 | f00f_workaround_enabled = 1; |
| 404 | } |
| 405 | } |
| 406 | #endif |
| 407 | |
| 408 | /* |
| 409 | * SEP CPUID bug: Pentium Pro reports SEP but doesn't have it until |
| 410 | * model 3 mask 3 |
| 411 | */ |
| 412 | if ((c->x86<<8 | c->x86_model<<4 | c->x86_stepping) < 0x633) |
| 413 | clear_cpu_cap(c, X86_FEATURE_SEP); |
| 414 | |
| 415 | /* |
| 416 | * PAE CPUID issue: many Pentium M report no PAE but may have a |
| 417 | * functionally usable PAE implementation. |
| 418 | * Forcefully enable PAE if kernel parameter "forcepae" is present. |
| 419 | */ |
| 420 | if (forcepae) { |
| 421 | pr_warn("PAE forced!\n"); |
| 422 | set_cpu_cap(c, X86_FEATURE_PAE); |
| 423 | add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE); |
| 424 | } |
| 425 | |
| 426 | /* |
| 427 | * P4 Xeon erratum 037 workaround. |
| 428 | * Hardware prefetcher may cause stale data to be loaded into the cache. |
| 429 | */ |
| 430 | if ((c->x86 == 15) && (c->x86_model == 1) && (c->x86_stepping == 1)) { |
| 431 | if (msr_set_bit(MSR_IA32_MISC_ENABLE, |
| 432 | MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE_BIT) > 0) { |
| 433 | pr_info("CPU: C0 stepping P4 Xeon detected.\n"); |
| 434 | pr_info("CPU: Disabling hardware prefetching (Erratum 037)\n"); |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | /* |
| 439 | * See if we have a good local APIC by checking for buggy Pentia, |
| 440 | * i.e. all B steppings and the C2 stepping of P54C when using their |
| 441 | * integrated APIC (see 11AP erratum in "Pentium Processor |
| 442 | * Specification Update"). |
| 443 | */ |
| 444 | if (boot_cpu_has(X86_FEATURE_APIC) && (c->x86<<8 | c->x86_model<<4) == 0x520 && |
| 445 | (c->x86_stepping < 0x6 || c->x86_stepping == 0xb)) |
| 446 | set_cpu_bug(c, X86_BUG_11AP); |
| 447 | |
| 448 | |
| 449 | #ifdef CONFIG_X86_INTEL_USERCOPY |
| 450 | /* |
| 451 | * Set up the preferred alignment for movsl bulk memory moves |
| 452 | */ |
| 453 | switch (c->x86) { |
| 454 | case 4: /* 486: untested */ |
| 455 | break; |
| 456 | case 5: /* Old Pentia: untested */ |
| 457 | break; |
| 458 | case 6: /* PII/PIII only like movsl with 8-byte alignment */ |
| 459 | movsl_mask.mask = 7; |
| 460 | break; |
| 461 | case 15: /* P4 is OK down to 8-byte alignment */ |
| 462 | movsl_mask.mask = 7; |
| 463 | break; |
| 464 | } |
| 465 | #endif |
| 466 | |
| 467 | intel_smp_check(c); |
| 468 | } |
| 469 | #else |
| 470 | static void intel_workarounds(struct cpuinfo_x86 *c) |
| 471 | { |
| 472 | } |
| 473 | #endif |
| 474 | |
| 475 | static void srat_detect_node(struct cpuinfo_x86 *c) |
| 476 | { |
| 477 | #ifdef CONFIG_NUMA |
| 478 | unsigned node; |
| 479 | int cpu = smp_processor_id(); |
| 480 | |
| 481 | /* Don't do the funky fallback heuristics the AMD version employs |
| 482 | for now. */ |
| 483 | node = numa_cpu_node(cpu); |
| 484 | if (node == NUMA_NO_NODE || !node_online(node)) { |
| 485 | /* reuse the value from init_cpu_to_node() */ |
| 486 | node = cpu_to_node(cpu); |
| 487 | } |
| 488 | numa_set_node(cpu, node); |
| 489 | #endif |
| 490 | } |
| 491 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 492 | #define MSR_IA32_TME_ACTIVATE 0x982 |
| 493 | |
| 494 | /* Helpers to access TME_ACTIVATE MSR */ |
| 495 | #define TME_ACTIVATE_LOCKED(x) (x & 0x1) |
| 496 | #define TME_ACTIVATE_ENABLED(x) (x & 0x2) |
| 497 | |
| 498 | #define TME_ACTIVATE_POLICY(x) ((x >> 4) & 0xf) /* Bits 7:4 */ |
| 499 | #define TME_ACTIVATE_POLICY_AES_XTS_128 0 |
| 500 | |
| 501 | #define TME_ACTIVATE_KEYID_BITS(x) ((x >> 32) & 0xf) /* Bits 35:32 */ |
| 502 | |
| 503 | #define TME_ACTIVATE_CRYPTO_ALGS(x) ((x >> 48) & 0xffff) /* Bits 63:48 */ |
| 504 | #define TME_ACTIVATE_CRYPTO_AES_XTS_128 1 |
| 505 | |
| 506 | /* Values for mktme_status (SW only construct) */ |
| 507 | #define MKTME_ENABLED 0 |
| 508 | #define MKTME_DISABLED 1 |
| 509 | #define MKTME_UNINITIALIZED 2 |
| 510 | static int mktme_status = MKTME_UNINITIALIZED; |
| 511 | |
| 512 | static void detect_tme(struct cpuinfo_x86 *c) |
| 513 | { |
| 514 | u64 tme_activate, tme_policy, tme_crypto_algs; |
| 515 | int keyid_bits = 0, nr_keyids = 0; |
| 516 | static u64 tme_activate_cpu0 = 0; |
| 517 | |
| 518 | rdmsrl(MSR_IA32_TME_ACTIVATE, tme_activate); |
| 519 | |
| 520 | if (mktme_status != MKTME_UNINITIALIZED) { |
| 521 | if (tme_activate != tme_activate_cpu0) { |
| 522 | /* Broken BIOS? */ |
| 523 | pr_err_once("x86/tme: configuration is inconsistent between CPUs\n"); |
| 524 | pr_err_once("x86/tme: MKTME is not usable\n"); |
| 525 | mktme_status = MKTME_DISABLED; |
| 526 | |
| 527 | /* Proceed. We may need to exclude bits from x86_phys_bits. */ |
| 528 | } |
| 529 | } else { |
| 530 | tme_activate_cpu0 = tme_activate; |
| 531 | } |
| 532 | |
| 533 | if (!TME_ACTIVATE_LOCKED(tme_activate) || !TME_ACTIVATE_ENABLED(tme_activate)) { |
| 534 | pr_info_once("x86/tme: not enabled by BIOS\n"); |
| 535 | mktme_status = MKTME_DISABLED; |
| 536 | return; |
| 537 | } |
| 538 | |
| 539 | if (mktme_status != MKTME_UNINITIALIZED) |
| 540 | goto detect_keyid_bits; |
| 541 | |
| 542 | pr_info("x86/tme: enabled by BIOS\n"); |
| 543 | |
| 544 | tme_policy = TME_ACTIVATE_POLICY(tme_activate); |
| 545 | if (tme_policy != TME_ACTIVATE_POLICY_AES_XTS_128) |
| 546 | pr_warn("x86/tme: Unknown policy is active: %#llx\n", tme_policy); |
| 547 | |
| 548 | tme_crypto_algs = TME_ACTIVATE_CRYPTO_ALGS(tme_activate); |
| 549 | if (!(tme_crypto_algs & TME_ACTIVATE_CRYPTO_AES_XTS_128)) { |
| 550 | pr_err("x86/mktme: No known encryption algorithm is supported: %#llx\n", |
| 551 | tme_crypto_algs); |
| 552 | mktme_status = MKTME_DISABLED; |
| 553 | } |
| 554 | detect_keyid_bits: |
| 555 | keyid_bits = TME_ACTIVATE_KEYID_BITS(tme_activate); |
| 556 | nr_keyids = (1UL << keyid_bits) - 1; |
| 557 | if (nr_keyids) { |
| 558 | pr_info_once("x86/mktme: enabled by BIOS\n"); |
| 559 | pr_info_once("x86/mktme: %d KeyIDs available\n", nr_keyids); |
| 560 | } else { |
| 561 | pr_info_once("x86/mktme: disabled by BIOS\n"); |
| 562 | } |
| 563 | |
| 564 | if (mktme_status == MKTME_UNINITIALIZED) { |
| 565 | /* MKTME is usable */ |
| 566 | mktme_status = MKTME_ENABLED; |
| 567 | } |
| 568 | |
| 569 | /* |
| 570 | * KeyID bits effectively lower the number of physical address |
| 571 | * bits. Update cpuinfo_x86::x86_phys_bits accordingly. |
| 572 | */ |
| 573 | c->x86_phys_bits -= keyid_bits; |
| 574 | } |
| 575 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 576 | static void init_cpuid_fault(struct cpuinfo_x86 *c) |
| 577 | { |
| 578 | u64 msr; |
| 579 | |
| 580 | if (!rdmsrl_safe(MSR_PLATFORM_INFO, &msr)) { |
| 581 | if (msr & MSR_PLATFORM_INFO_CPUID_FAULT) |
| 582 | set_cpu_cap(c, X86_FEATURE_CPUID_FAULT); |
| 583 | } |
| 584 | } |
| 585 | |
| 586 | static void init_intel_misc_features(struct cpuinfo_x86 *c) |
| 587 | { |
| 588 | u64 msr; |
| 589 | |
| 590 | if (rdmsrl_safe(MSR_MISC_FEATURES_ENABLES, &msr)) |
| 591 | return; |
| 592 | |
| 593 | /* Clear all MISC features */ |
| 594 | this_cpu_write(msr_misc_features_shadow, 0); |
| 595 | |
| 596 | /* Check features and update capabilities and shadow control bits */ |
| 597 | init_cpuid_fault(c); |
| 598 | probe_xeon_phi_r3mwait(c); |
| 599 | |
| 600 | msr = this_cpu_read(msr_misc_features_shadow); |
| 601 | wrmsrl(MSR_MISC_FEATURES_ENABLES, msr); |
| 602 | } |
| 603 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 604 | static void split_lock_init(void); |
| 605 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 606 | static void init_intel(struct cpuinfo_x86 *c) |
| 607 | { |
| 608 | early_init_intel(c); |
| 609 | |
| 610 | intel_workarounds(c); |
| 611 | |
| 612 | /* |
| 613 | * Detect the extended topology information if available. This |
| 614 | * will reinitialise the initial_apicid which will be used |
| 615 | * in init_intel_cacheinfo() |
| 616 | */ |
| 617 | detect_extended_topology(c); |
| 618 | |
| 619 | if (!cpu_has(c, X86_FEATURE_XTOPOLOGY)) { |
| 620 | /* |
| 621 | * let's use the legacy cpuid vector 0x1 and 0x4 for topology |
| 622 | * detection. |
| 623 | */ |
| 624 | detect_num_cpu_cores(c); |
| 625 | #ifdef CONFIG_X86_32 |
| 626 | detect_ht(c); |
| 627 | #endif |
| 628 | } |
| 629 | |
| 630 | init_intel_cacheinfo(c); |
| 631 | |
| 632 | if (c->cpuid_level > 9) { |
| 633 | unsigned eax = cpuid_eax(10); |
| 634 | /* Check for version and the number of counters */ |
| 635 | if ((eax & 0xff) && (((eax>>8) & 0xff) > 1)) |
| 636 | set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON); |
| 637 | } |
| 638 | |
| 639 | if (cpu_has(c, X86_FEATURE_XMM2)) |
| 640 | set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); |
| 641 | |
| 642 | if (boot_cpu_has(X86_FEATURE_DS)) { |
| 643 | unsigned int l1, l2; |
| 644 | |
| 645 | rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); |
| 646 | if (!(l1 & (1<<11))) |
| 647 | set_cpu_cap(c, X86_FEATURE_BTS); |
| 648 | if (!(l1 & (1<<12))) |
| 649 | set_cpu_cap(c, X86_FEATURE_PEBS); |
| 650 | } |
| 651 | |
| 652 | if (c->x86 == 6 && boot_cpu_has(X86_FEATURE_CLFLUSH) && |
| 653 | (c->x86_model == 29 || c->x86_model == 46 || c->x86_model == 47)) |
| 654 | set_cpu_bug(c, X86_BUG_CLFLUSH_MONITOR); |
| 655 | |
| 656 | if (c->x86 == 6 && boot_cpu_has(X86_FEATURE_MWAIT) && |
| 657 | ((c->x86_model == INTEL_FAM6_ATOM_GOLDMONT))) |
| 658 | set_cpu_bug(c, X86_BUG_MONITOR); |
| 659 | |
| 660 | #ifdef CONFIG_X86_64 |
| 661 | if (c->x86 == 15) |
| 662 | c->x86_cache_alignment = c->x86_clflush_size * 2; |
| 663 | if (c->x86 == 6) |
| 664 | set_cpu_cap(c, X86_FEATURE_REP_GOOD); |
| 665 | #else |
| 666 | /* |
| 667 | * Names for the Pentium II/Celeron processors |
| 668 | * detectable only by also checking the cache size. |
| 669 | * Dixon is NOT a Celeron. |
| 670 | */ |
| 671 | if (c->x86 == 6) { |
| 672 | unsigned int l2 = c->x86_cache_size; |
| 673 | char *p = NULL; |
| 674 | |
| 675 | switch (c->x86_model) { |
| 676 | case 5: |
| 677 | if (l2 == 0) |
| 678 | p = "Celeron (Covington)"; |
| 679 | else if (l2 == 256) |
| 680 | p = "Mobile Pentium II (Dixon)"; |
| 681 | break; |
| 682 | |
| 683 | case 6: |
| 684 | if (l2 == 128) |
| 685 | p = "Celeron (Mendocino)"; |
| 686 | else if (c->x86_stepping == 0 || c->x86_stepping == 5) |
| 687 | p = "Celeron-A"; |
| 688 | break; |
| 689 | |
| 690 | case 8: |
| 691 | if (l2 == 128) |
| 692 | p = "Celeron (Coppermine)"; |
| 693 | break; |
| 694 | } |
| 695 | |
| 696 | if (p) |
| 697 | strcpy(c->x86_model_id, p); |
| 698 | } |
| 699 | |
| 700 | if (c->x86 == 15) |
| 701 | set_cpu_cap(c, X86_FEATURE_P4); |
| 702 | if (c->x86 == 6) |
| 703 | set_cpu_cap(c, X86_FEATURE_P3); |
| 704 | #endif |
| 705 | |
| 706 | /* Work around errata */ |
| 707 | srat_detect_node(c); |
| 708 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 709 | init_ia32_feat_ctl(c); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 710 | |
| 711 | if (cpu_has(c, X86_FEATURE_TME)) |
| 712 | detect_tme(c); |
| 713 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 714 | init_intel_misc_features(c); |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 715 | |
| 716 | if (tsx_ctrl_state == TSX_CTRL_ENABLE) |
| 717 | tsx_enable(); |
| 718 | if (tsx_ctrl_state == TSX_CTRL_DISABLE) |
| 719 | tsx_disable(); |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 720 | |
| 721 | split_lock_init(); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 722 | } |
| 723 | |
| 724 | #ifdef CONFIG_X86_32 |
| 725 | static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) |
| 726 | { |
| 727 | /* |
| 728 | * Intel PIII Tualatin. This comes in two flavours. |
| 729 | * One has 256kb of cache, the other 512. We have no way |
| 730 | * to determine which, so we use a boottime override |
| 731 | * for the 512kb model, and assume 256 otherwise. |
| 732 | */ |
| 733 | if ((c->x86 == 6) && (c->x86_model == 11) && (size == 0)) |
| 734 | size = 256; |
| 735 | |
| 736 | /* |
| 737 | * Intel Quark SoC X1000 contains a 4-way set associative |
| 738 | * 16K cache with a 16 byte cache line and 256 lines per tag |
| 739 | */ |
| 740 | if ((c->x86 == 5) && (c->x86_model == 9)) |
| 741 | size = 16; |
| 742 | return size; |
| 743 | } |
| 744 | #endif |
| 745 | |
| 746 | #define TLB_INST_4K 0x01 |
| 747 | #define TLB_INST_4M 0x02 |
| 748 | #define TLB_INST_2M_4M 0x03 |
| 749 | |
| 750 | #define TLB_INST_ALL 0x05 |
| 751 | #define TLB_INST_1G 0x06 |
| 752 | |
| 753 | #define TLB_DATA_4K 0x11 |
| 754 | #define TLB_DATA_4M 0x12 |
| 755 | #define TLB_DATA_2M_4M 0x13 |
| 756 | #define TLB_DATA_4K_4M 0x14 |
| 757 | |
| 758 | #define TLB_DATA_1G 0x16 |
| 759 | |
| 760 | #define TLB_DATA0_4K 0x21 |
| 761 | #define TLB_DATA0_4M 0x22 |
| 762 | #define TLB_DATA0_2M_4M 0x23 |
| 763 | |
| 764 | #define STLB_4K 0x41 |
| 765 | #define STLB_4K_2M 0x42 |
| 766 | |
| 767 | static const struct _tlb_table intel_tlb_table[] = { |
| 768 | { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" }, |
| 769 | { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" }, |
| 770 | { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" }, |
| 771 | { 0x04, TLB_DATA_4M, 8, " TLB_DATA 4 MByte pages, 4-way set associative" }, |
| 772 | { 0x05, TLB_DATA_4M, 32, " TLB_DATA 4 MByte pages, 4-way set associative" }, |
| 773 | { 0x0b, TLB_INST_4M, 4, " TLB_INST 4 MByte pages, 4-way set associative" }, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 774 | { 0x4f, TLB_INST_4K, 32, " TLB_INST 4 KByte pages" }, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 775 | { 0x50, TLB_INST_ALL, 64, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, |
| 776 | { 0x51, TLB_INST_ALL, 128, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, |
| 777 | { 0x52, TLB_INST_ALL, 256, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, |
| 778 | { 0x55, TLB_INST_2M_4M, 7, " TLB_INST 2-MByte or 4-MByte pages, fully associative" }, |
| 779 | { 0x56, TLB_DATA0_4M, 16, " TLB_DATA0 4 MByte pages, 4-way set associative" }, |
| 780 | { 0x57, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, 4-way associative" }, |
| 781 | { 0x59, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, fully associative" }, |
| 782 | { 0x5a, TLB_DATA0_2M_4M, 32, " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" }, |
| 783 | { 0x5b, TLB_DATA_4K_4M, 64, " TLB_DATA 4 KByte and 4 MByte pages" }, |
| 784 | { 0x5c, TLB_DATA_4K_4M, 128, " TLB_DATA 4 KByte and 4 MByte pages" }, |
| 785 | { 0x5d, TLB_DATA_4K_4M, 256, " TLB_DATA 4 KByte and 4 MByte pages" }, |
| 786 | { 0x61, TLB_INST_4K, 48, " TLB_INST 4 KByte pages, full associative" }, |
| 787 | { 0x63, TLB_DATA_1G, 4, " TLB_DATA 1 GByte pages, 4-way set associative" }, |
| 788 | { 0x6b, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 8-way associative" }, |
| 789 | { 0x6c, TLB_DATA_2M_4M, 128, " TLB_DATA 2 MByte or 4 MByte pages, 8-way associative" }, |
| 790 | { 0x6d, TLB_DATA_1G, 16, " TLB_DATA 1 GByte pages, fully associative" }, |
| 791 | { 0x76, TLB_INST_2M_4M, 8, " TLB_INST 2-MByte or 4-MByte pages, fully associative" }, |
| 792 | { 0xb0, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 4-way set associative" }, |
| 793 | { 0xb1, TLB_INST_2M_4M, 4, " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" }, |
| 794 | { 0xb2, TLB_INST_4K, 64, " TLB_INST 4KByte pages, 4-way set associative" }, |
| 795 | { 0xb3, TLB_DATA_4K, 128, " TLB_DATA 4 KByte pages, 4-way set associative" }, |
| 796 | { 0xb4, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 4-way associative" }, |
| 797 | { 0xb5, TLB_INST_4K, 64, " TLB_INST 4 KByte pages, 8-way set associative" }, |
| 798 | { 0xb6, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 8-way set associative" }, |
| 799 | { 0xba, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way associative" }, |
| 800 | { 0xc0, TLB_DATA_4K_4M, 8, " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" }, |
| 801 | { 0xc1, STLB_4K_2M, 1024, " STLB 4 KByte and 2 MByte pages, 8-way associative" }, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 802 | { 0xc2, TLB_DATA_2M_4M, 16, " TLB_DATA 2 MByte/4MByte pages, 4-way associative" }, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 803 | { 0xca, STLB_4K, 512, " STLB 4 KByte pages, 4-way associative" }, |
| 804 | { 0x00, 0, 0 } |
| 805 | }; |
| 806 | |
| 807 | static void intel_tlb_lookup(const unsigned char desc) |
| 808 | { |
| 809 | unsigned char k; |
| 810 | if (desc == 0) |
| 811 | return; |
| 812 | |
| 813 | /* look up this descriptor in the table */ |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 814 | for (k = 0; intel_tlb_table[k].descriptor != desc && |
| 815 | intel_tlb_table[k].descriptor != 0; k++) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 816 | ; |
| 817 | |
| 818 | if (intel_tlb_table[k].tlb_type == 0) |
| 819 | return; |
| 820 | |
| 821 | switch (intel_tlb_table[k].tlb_type) { |
| 822 | case STLB_4K: |
| 823 | if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 824 | tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 825 | if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 826 | tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 827 | break; |
| 828 | case STLB_4K_2M: |
| 829 | if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 830 | tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 831 | if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 832 | tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 833 | if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries) |
| 834 | tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries; |
| 835 | if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries) |
| 836 | tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries; |
| 837 | if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 838 | tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 839 | if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 840 | tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 841 | break; |
| 842 | case TLB_INST_ALL: |
| 843 | if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 844 | tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 845 | if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries) |
| 846 | tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries; |
| 847 | if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 848 | tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 849 | break; |
| 850 | case TLB_INST_4K: |
| 851 | if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 852 | tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 853 | break; |
| 854 | case TLB_INST_4M: |
| 855 | if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 856 | tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 857 | break; |
| 858 | case TLB_INST_2M_4M: |
| 859 | if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries) |
| 860 | tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries; |
| 861 | if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 862 | tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 863 | break; |
| 864 | case TLB_DATA_4K: |
| 865 | case TLB_DATA0_4K: |
| 866 | if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 867 | tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 868 | break; |
| 869 | case TLB_DATA_4M: |
| 870 | case TLB_DATA0_4M: |
| 871 | if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 872 | tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 873 | break; |
| 874 | case TLB_DATA_2M_4M: |
| 875 | case TLB_DATA0_2M_4M: |
| 876 | if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries) |
| 877 | tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries; |
| 878 | if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 879 | tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 880 | break; |
| 881 | case TLB_DATA_4K_4M: |
| 882 | if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) |
| 883 | tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; |
| 884 | if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) |
| 885 | tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; |
| 886 | break; |
| 887 | case TLB_DATA_1G: |
| 888 | if (tlb_lld_1g[ENTRIES] < intel_tlb_table[k].entries) |
| 889 | tlb_lld_1g[ENTRIES] = intel_tlb_table[k].entries; |
| 890 | break; |
| 891 | } |
| 892 | } |
| 893 | |
| 894 | static void intel_detect_tlb(struct cpuinfo_x86 *c) |
| 895 | { |
| 896 | int i, j, n; |
| 897 | unsigned int regs[4]; |
| 898 | unsigned char *desc = (unsigned char *)regs; |
| 899 | |
| 900 | if (c->cpuid_level < 2) |
| 901 | return; |
| 902 | |
| 903 | /* Number of times to iterate */ |
| 904 | n = cpuid_eax(2) & 0xFF; |
| 905 | |
| 906 | for (i = 0 ; i < n ; i++) { |
| 907 | cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); |
| 908 | |
| 909 | /* If bit 31 is set, this is an unknown format */ |
| 910 | for (j = 0 ; j < 3 ; j++) |
| 911 | if (regs[j] & (1 << 31)) |
| 912 | regs[j] = 0; |
| 913 | |
| 914 | /* Byte 0 is level count, not a descriptor */ |
| 915 | for (j = 1 ; j < 16 ; j++) |
| 916 | intel_tlb_lookup(desc[j]); |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | static const struct cpu_dev intel_cpu_dev = { |
| 921 | .c_vendor = "Intel", |
| 922 | .c_ident = { "GenuineIntel" }, |
| 923 | #ifdef CONFIG_X86_32 |
| 924 | .legacy_models = { |
| 925 | { .family = 4, .model_names = |
| 926 | { |
| 927 | [0] = "486 DX-25/33", |
| 928 | [1] = "486 DX-50", |
| 929 | [2] = "486 SX", |
| 930 | [3] = "486 DX/2", |
| 931 | [4] = "486 SL", |
| 932 | [5] = "486 SX/2", |
| 933 | [7] = "486 DX/2-WB", |
| 934 | [8] = "486 DX/4", |
| 935 | [9] = "486 DX/4-WB" |
| 936 | } |
| 937 | }, |
| 938 | { .family = 5, .model_names = |
| 939 | { |
| 940 | [0] = "Pentium 60/66 A-step", |
| 941 | [1] = "Pentium 60/66", |
| 942 | [2] = "Pentium 75 - 200", |
| 943 | [3] = "OverDrive PODP5V83", |
| 944 | [4] = "Pentium MMX", |
| 945 | [7] = "Mobile Pentium 75 - 200", |
| 946 | [8] = "Mobile Pentium MMX", |
| 947 | [9] = "Quark SoC X1000", |
| 948 | } |
| 949 | }, |
| 950 | { .family = 6, .model_names = |
| 951 | { |
| 952 | [0] = "Pentium Pro A-step", |
| 953 | [1] = "Pentium Pro", |
| 954 | [3] = "Pentium II (Klamath)", |
| 955 | [4] = "Pentium II (Deschutes)", |
| 956 | [5] = "Pentium II (Deschutes)", |
| 957 | [6] = "Mobile Pentium II", |
| 958 | [7] = "Pentium III (Katmai)", |
| 959 | [8] = "Pentium III (Coppermine)", |
| 960 | [10] = "Pentium III (Cascades)", |
| 961 | [11] = "Pentium III (Tualatin)", |
| 962 | } |
| 963 | }, |
| 964 | { .family = 15, .model_names = |
| 965 | { |
| 966 | [0] = "Pentium 4 (Unknown)", |
| 967 | [1] = "Pentium 4 (Willamette)", |
| 968 | [2] = "Pentium 4 (Northwood)", |
| 969 | [4] = "Pentium 4 (Foster)", |
| 970 | [5] = "Pentium 4 (Foster)", |
| 971 | } |
| 972 | }, |
| 973 | }, |
| 974 | .legacy_cache_size = intel_size_cache, |
| 975 | #endif |
| 976 | .c_detect_tlb = intel_detect_tlb, |
| 977 | .c_early_init = early_init_intel, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 978 | .c_bsp_init = bsp_init_intel, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 979 | .c_init = init_intel, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 980 | .c_x86_vendor = X86_VENDOR_INTEL, |
| 981 | }; |
| 982 | |
| 983 | cpu_dev_register(intel_cpu_dev); |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 984 | |
| 985 | #undef pr_fmt |
| 986 | #define pr_fmt(fmt) "x86/split lock detection: " fmt |
| 987 | |
| 988 | static const struct { |
| 989 | const char *option; |
| 990 | enum split_lock_detect_state state; |
| 991 | } sld_options[] __initconst = { |
| 992 | { "off", sld_off }, |
| 993 | { "warn", sld_warn }, |
| 994 | { "fatal", sld_fatal }, |
| 995 | }; |
| 996 | |
| 997 | static inline bool match_option(const char *arg, int arglen, const char *opt) |
| 998 | { |
| 999 | int len = strlen(opt); |
| 1000 | |
| 1001 | return len == arglen && !strncmp(arg, opt, len); |
| 1002 | } |
| 1003 | |
| 1004 | static bool split_lock_verify_msr(bool on) |
| 1005 | { |
| 1006 | u64 ctrl, tmp; |
| 1007 | |
| 1008 | if (rdmsrl_safe(MSR_TEST_CTRL, &ctrl)) |
| 1009 | return false; |
| 1010 | if (on) |
| 1011 | ctrl |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; |
| 1012 | else |
| 1013 | ctrl &= ~MSR_TEST_CTRL_SPLIT_LOCK_DETECT; |
| 1014 | if (wrmsrl_safe(MSR_TEST_CTRL, ctrl)) |
| 1015 | return false; |
| 1016 | rdmsrl(MSR_TEST_CTRL, tmp); |
| 1017 | return ctrl == tmp; |
| 1018 | } |
| 1019 | |
| 1020 | static void __init split_lock_setup(void) |
| 1021 | { |
| 1022 | enum split_lock_detect_state state = sld_warn; |
| 1023 | char arg[20]; |
| 1024 | int i, ret; |
| 1025 | |
| 1026 | if (!split_lock_verify_msr(false)) { |
| 1027 | pr_info("MSR access failed: Disabled\n"); |
| 1028 | return; |
| 1029 | } |
| 1030 | |
| 1031 | ret = cmdline_find_option(boot_command_line, "split_lock_detect", |
| 1032 | arg, sizeof(arg)); |
| 1033 | if (ret >= 0) { |
| 1034 | for (i = 0; i < ARRAY_SIZE(sld_options); i++) { |
| 1035 | if (match_option(arg, ret, sld_options[i].option)) { |
| 1036 | state = sld_options[i].state; |
| 1037 | break; |
| 1038 | } |
| 1039 | } |
| 1040 | } |
| 1041 | |
| 1042 | switch (state) { |
| 1043 | case sld_off: |
| 1044 | pr_info("disabled\n"); |
| 1045 | return; |
| 1046 | case sld_warn: |
| 1047 | pr_info("warning about user-space split_locks\n"); |
| 1048 | break; |
| 1049 | case sld_fatal: |
| 1050 | pr_info("sending SIGBUS on user-space split_locks\n"); |
| 1051 | break; |
| 1052 | } |
| 1053 | |
| 1054 | rdmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache); |
| 1055 | |
| 1056 | if (!split_lock_verify_msr(true)) { |
| 1057 | pr_info("MSR access failed: Disabled\n"); |
| 1058 | return; |
| 1059 | } |
| 1060 | |
| 1061 | sld_state = state; |
| 1062 | setup_force_cpu_cap(X86_FEATURE_SPLIT_LOCK_DETECT); |
| 1063 | } |
| 1064 | |
| 1065 | /* |
| 1066 | * MSR_TEST_CTRL is per core, but we treat it like a per CPU MSR. Locking |
| 1067 | * is not implemented as one thread could undo the setting of the other |
| 1068 | * thread immediately after dropping the lock anyway. |
| 1069 | */ |
| 1070 | static void sld_update_msr(bool on) |
| 1071 | { |
| 1072 | u64 test_ctrl_val = msr_test_ctrl_cache; |
| 1073 | |
| 1074 | if (on) |
| 1075 | test_ctrl_val |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; |
| 1076 | |
| 1077 | wrmsrl(MSR_TEST_CTRL, test_ctrl_val); |
| 1078 | } |
| 1079 | |
| 1080 | static void split_lock_init(void) |
| 1081 | { |
| 1082 | if (cpu_model_supports_sld) |
| 1083 | split_lock_verify_msr(sld_state != sld_off); |
| 1084 | } |
| 1085 | |
| 1086 | static void split_lock_warn(unsigned long ip) |
| 1087 | { |
| 1088 | pr_warn_ratelimited("#AC: %s/%d took a split_lock trap at address: 0x%lx\n", |
| 1089 | current->comm, current->pid, ip); |
| 1090 | |
| 1091 | /* |
| 1092 | * Disable the split lock detection for this task so it can make |
| 1093 | * progress and set TIF_SLD so the detection is re-enabled via |
| 1094 | * switch_to_sld() when the task is scheduled out. |
| 1095 | */ |
| 1096 | sld_update_msr(false); |
| 1097 | set_tsk_thread_flag(current, TIF_SLD); |
| 1098 | } |
| 1099 | |
| 1100 | bool handle_guest_split_lock(unsigned long ip) |
| 1101 | { |
| 1102 | if (sld_state == sld_warn) { |
| 1103 | split_lock_warn(ip); |
| 1104 | return true; |
| 1105 | } |
| 1106 | |
| 1107 | pr_warn_once("#AC: %s/%d %s split_lock trap at address: 0x%lx\n", |
| 1108 | current->comm, current->pid, |
| 1109 | sld_state == sld_fatal ? "fatal" : "bogus", ip); |
| 1110 | |
| 1111 | current->thread.error_code = 0; |
| 1112 | current->thread.trap_nr = X86_TRAP_AC; |
| 1113 | force_sig_fault(SIGBUS, BUS_ADRALN, NULL); |
| 1114 | return false; |
| 1115 | } |
| 1116 | EXPORT_SYMBOL_GPL(handle_guest_split_lock); |
| 1117 | |
| 1118 | bool handle_user_split_lock(struct pt_regs *regs, long error_code) |
| 1119 | { |
| 1120 | if ((regs->flags & X86_EFLAGS_AC) || sld_state == sld_fatal) |
| 1121 | return false; |
| 1122 | split_lock_warn(regs->ip); |
| 1123 | return true; |
| 1124 | } |
| 1125 | |
| 1126 | /* |
| 1127 | * This function is called only when switching between tasks with |
| 1128 | * different split-lock detection modes. It sets the MSR for the |
| 1129 | * mode of the new task. This is right most of the time, but since |
| 1130 | * the MSR is shared by hyperthreads on a physical core there can |
| 1131 | * be glitches when the two threads need different modes. |
| 1132 | */ |
| 1133 | void switch_to_sld(unsigned long tifn) |
| 1134 | { |
| 1135 | sld_update_msr(!(tifn & _TIF_SLD)); |
| 1136 | } |
| 1137 | |
| 1138 | /* |
| 1139 | * Bits in the IA32_CORE_CAPABILITIES are not architectural, so they should |
| 1140 | * only be trusted if it is confirmed that a CPU model implements a |
| 1141 | * specific feature at a particular bit position. |
| 1142 | * |
| 1143 | * The possible driver data field values: |
| 1144 | * |
| 1145 | * - 0: CPU models that are known to have the per-core split-lock detection |
| 1146 | * feature even though they do not enumerate IA32_CORE_CAPABILITIES. |
| 1147 | * |
| 1148 | * - 1: CPU models which may enumerate IA32_CORE_CAPABILITIES and if so use |
| 1149 | * bit 5 to enumerate the per-core split-lock detection feature. |
| 1150 | */ |
| 1151 | static const struct x86_cpu_id split_lock_cpu_ids[] __initconst = { |
| 1152 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, 0), |
| 1153 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, 0), |
| 1154 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, 0), |
| 1155 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, 1), |
| 1156 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, 1), |
| 1157 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, 1), |
| 1158 | X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, 1), |
| 1159 | X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, 1), |
| 1160 | X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, 1), |
| 1161 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, 1), |
| 1162 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, 1), |
| 1163 | {} |
| 1164 | }; |
| 1165 | |
| 1166 | void __init cpu_set_core_cap_bits(struct cpuinfo_x86 *c) |
| 1167 | { |
| 1168 | const struct x86_cpu_id *m; |
| 1169 | u64 ia32_core_caps; |
| 1170 | |
| 1171 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
| 1172 | return; |
| 1173 | |
| 1174 | m = x86_match_cpu(split_lock_cpu_ids); |
| 1175 | if (!m) |
| 1176 | return; |
| 1177 | |
| 1178 | switch (m->driver_data) { |
| 1179 | case 0: |
| 1180 | break; |
| 1181 | case 1: |
| 1182 | if (!cpu_has(c, X86_FEATURE_CORE_CAPABILITIES)) |
| 1183 | return; |
| 1184 | rdmsrl(MSR_IA32_CORE_CAPS, ia32_core_caps); |
| 1185 | if (!(ia32_core_caps & MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT)) |
| 1186 | return; |
| 1187 | break; |
| 1188 | default: |
| 1189 | return; |
| 1190 | } |
| 1191 | |
| 1192 | cpu_model_supports_sld = true; |
| 1193 | split_lock_setup(); |
| 1194 | } |