David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 2 | /* |
| 3 | * linux/arch/arm/mm/fault.c |
| 4 | * |
| 5 | * Copyright (C) 1995 Linus Torvalds |
| 6 | * Modifications for ARM processor (c) 1995-2004 Russell King |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 7 | */ |
| 8 | #include <linux/extable.h> |
| 9 | #include <linux/signal.h> |
| 10 | #include <linux/mm.h> |
| 11 | #include <linux/hardirq.h> |
| 12 | #include <linux/init.h> |
| 13 | #include <linux/kprobes.h> |
| 14 | #include <linux/uaccess.h> |
| 15 | #include <linux/page-flags.h> |
| 16 | #include <linux/sched/signal.h> |
| 17 | #include <linux/sched/debug.h> |
| 18 | #include <linux/highmem.h> |
| 19 | #include <linux/perf_event.h> |
| 20 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 21 | #include <asm/system_misc.h> |
| 22 | #include <asm/system_info.h> |
| 23 | #include <asm/tlbflush.h> |
| 24 | |
| 25 | #include "fault.h" |
| 26 | |
| 27 | #ifdef CONFIG_MMU |
| 28 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 29 | /* |
| 30 | * This is useful to dump out the page tables associated with |
| 31 | * 'addr' in mm 'mm'. |
| 32 | */ |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 33 | void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 34 | { |
| 35 | pgd_t *pgd; |
| 36 | |
| 37 | if (!mm) |
| 38 | mm = &init_mm; |
| 39 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 40 | printk("%spgd = %p\n", lvl, mm->pgd); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 41 | pgd = pgd_offset(mm, addr); |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 42 | printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd)); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 43 | |
| 44 | do { |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 45 | p4d_t *p4d; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 46 | pud_t *pud; |
| 47 | pmd_t *pmd; |
| 48 | pte_t *pte; |
| 49 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 50 | p4d = p4d_offset(pgd, addr); |
| 51 | if (p4d_none(*p4d)) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 52 | break; |
| 53 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 54 | if (p4d_bad(*p4d)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 55 | pr_cont("(bad)"); |
| 56 | break; |
| 57 | } |
| 58 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 59 | pud = pud_offset(p4d, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 60 | if (PTRS_PER_PUD != 1) |
| 61 | pr_cont(", *pud=%08llx", (long long)pud_val(*pud)); |
| 62 | |
| 63 | if (pud_none(*pud)) |
| 64 | break; |
| 65 | |
| 66 | if (pud_bad(*pud)) { |
| 67 | pr_cont("(bad)"); |
| 68 | break; |
| 69 | } |
| 70 | |
| 71 | pmd = pmd_offset(pud, addr); |
| 72 | if (PTRS_PER_PMD != 1) |
| 73 | pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd)); |
| 74 | |
| 75 | if (pmd_none(*pmd)) |
| 76 | break; |
| 77 | |
| 78 | if (pmd_bad(*pmd)) { |
| 79 | pr_cont("(bad)"); |
| 80 | break; |
| 81 | } |
| 82 | |
| 83 | /* We must not map this if we have highmem enabled */ |
| 84 | if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT))) |
| 85 | break; |
| 86 | |
| 87 | pte = pte_offset_map(pmd, addr); |
| 88 | pr_cont(", *pte=%08llx", (long long)pte_val(*pte)); |
| 89 | #ifndef CONFIG_ARM_LPAE |
| 90 | pr_cont(", *ppte=%08llx", |
| 91 | (long long)pte_val(pte[PTE_HWTABLE_PTRS])); |
| 92 | #endif |
| 93 | pte_unmap(pte); |
| 94 | } while(0); |
| 95 | |
| 96 | pr_cont("\n"); |
| 97 | } |
| 98 | #else /* CONFIG_MMU */ |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 99 | void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 100 | { } |
| 101 | #endif /* CONFIG_MMU */ |
| 102 | |
| 103 | /* |
| 104 | * Oops. The kernel tried to access some page that wasn't present. |
| 105 | */ |
| 106 | static void |
| 107 | __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, |
| 108 | struct pt_regs *regs) |
| 109 | { |
| 110 | /* |
| 111 | * Are we prepared to handle this kernel fault? |
| 112 | */ |
| 113 | if (fixup_exception(regs)) |
| 114 | return; |
| 115 | |
| 116 | /* |
| 117 | * No handler, we'll have to terminate things with extreme prejudice. |
| 118 | */ |
| 119 | bust_spinlocks(1); |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 120 | pr_alert("8<--- cut here ---\n"); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 121 | pr_alert("Unable to handle kernel %s at virtual address %08lx\n", |
| 122 | (addr < PAGE_SIZE) ? "NULL pointer dereference" : |
| 123 | "paging request", addr); |
| 124 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 125 | show_pte(KERN_ALERT, mm, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 126 | die("Oops", regs, fsr); |
| 127 | bust_spinlocks(0); |
| 128 | do_exit(SIGKILL); |
| 129 | } |
| 130 | |
| 131 | /* |
| 132 | * Something tried to access memory that isn't in our memory map.. |
| 133 | * User mode accesses just cause a SIGSEGV |
| 134 | */ |
| 135 | static void |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 136 | __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, |
| 137 | int code, struct pt_regs *regs) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 138 | { |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 139 | struct task_struct *tsk = current; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 140 | |
| 141 | if (addr > TASK_SIZE) |
| 142 | harden_branch_predictor(); |
| 143 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 144 | #ifdef CONFIG_DEBUG_USER |
| 145 | if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || |
| 146 | ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 147 | pr_err("8<--- cut here ---\n"); |
| 148 | pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 149 | tsk->comm, sig, addr, fsr); |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 150 | show_pte(KERN_ERR, tsk->mm, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 151 | show_regs(regs); |
| 152 | } |
| 153 | #endif |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 154 | #ifndef CONFIG_KUSER_HELPERS |
| 155 | if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000)) |
| 156 | printk_ratelimited(KERN_DEBUG |
| 157 | "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n", |
| 158 | tsk->comm, addr); |
| 159 | #endif |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 160 | |
| 161 | tsk->thread.address = addr; |
| 162 | tsk->thread.error_code = fsr; |
| 163 | tsk->thread.trap_no = 14; |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 164 | force_sig_fault(sig, code, (void __user *)addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 165 | } |
| 166 | |
| 167 | void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 168 | { |
| 169 | struct task_struct *tsk = current; |
| 170 | struct mm_struct *mm = tsk->active_mm; |
| 171 | |
| 172 | /* |
| 173 | * If we are in kernel mode at this point, we |
| 174 | * have no context to handle this fault with. |
| 175 | */ |
| 176 | if (user_mode(regs)) |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 177 | __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 178 | else |
| 179 | __do_kernel_fault(mm, addr, fsr, regs); |
| 180 | } |
| 181 | |
| 182 | #ifdef CONFIG_MMU |
| 183 | #define VM_FAULT_BADMAP 0x010000 |
| 184 | #define VM_FAULT_BADACCESS 0x020000 |
| 185 | |
| 186 | /* |
| 187 | * Check that the permissions on the VMA allow for the fault which occurred. |
| 188 | * If we encountered a write fault, we must have write permission, otherwise |
| 189 | * we allow any permission. |
| 190 | */ |
| 191 | static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma) |
| 192 | { |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 193 | unsigned int mask = VM_ACCESS_FLAGS; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 194 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 195 | if ((fsr & FSR_WRITE) && !(fsr & FSR_CM)) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 196 | mask = VM_WRITE; |
| 197 | if (fsr & FSR_LNX_PF) |
| 198 | mask = VM_EXEC; |
| 199 | |
| 200 | return vma->vm_flags & mask ? false : true; |
| 201 | } |
| 202 | |
| 203 | static vm_fault_t __kprobes |
| 204 | __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 205 | unsigned int flags, struct task_struct *tsk, |
| 206 | struct pt_regs *regs) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 207 | { |
| 208 | struct vm_area_struct *vma; |
| 209 | vm_fault_t fault; |
| 210 | |
| 211 | vma = find_vma(mm, addr); |
| 212 | fault = VM_FAULT_BADMAP; |
| 213 | if (unlikely(!vma)) |
| 214 | goto out; |
| 215 | if (unlikely(vma->vm_start > addr)) |
| 216 | goto check_stack; |
| 217 | |
| 218 | /* |
| 219 | * Ok, we have a good vm_area for this |
| 220 | * memory access, so we can handle it. |
| 221 | */ |
| 222 | good_area: |
| 223 | if (access_error(fsr, vma)) { |
| 224 | fault = VM_FAULT_BADACCESS; |
| 225 | goto out; |
| 226 | } |
| 227 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 228 | return handle_mm_fault(vma, addr & PAGE_MASK, flags, regs); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 229 | |
| 230 | check_stack: |
| 231 | /* Don't allow expansion below FIRST_USER_ADDRESS */ |
| 232 | if (vma->vm_flags & VM_GROWSDOWN && |
| 233 | addr >= FIRST_USER_ADDRESS && !expand_stack(vma, addr)) |
| 234 | goto good_area; |
| 235 | out: |
| 236 | return fault; |
| 237 | } |
| 238 | |
| 239 | static int __kprobes |
| 240 | do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 241 | { |
| 242 | struct task_struct *tsk; |
| 243 | struct mm_struct *mm; |
| 244 | int sig, code; |
| 245 | vm_fault_t fault; |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 246 | unsigned int flags = FAULT_FLAG_DEFAULT; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 247 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 248 | if (kprobe_page_fault(regs, fsr)) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 249 | return 0; |
| 250 | |
| 251 | tsk = current; |
| 252 | mm = tsk->mm; |
| 253 | |
| 254 | /* Enable interrupts if they were enabled in the parent context. */ |
| 255 | if (interrupts_enabled(regs)) |
| 256 | local_irq_enable(); |
| 257 | |
| 258 | /* |
| 259 | * If we're in an interrupt or have no user |
| 260 | * context, we must not take the fault.. |
| 261 | */ |
| 262 | if (faulthandler_disabled() || !mm) |
| 263 | goto no_context; |
| 264 | |
| 265 | if (user_mode(regs)) |
| 266 | flags |= FAULT_FLAG_USER; |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 267 | if ((fsr & FSR_WRITE) && !(fsr & FSR_CM)) |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 268 | flags |= FAULT_FLAG_WRITE; |
| 269 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 270 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); |
| 271 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 272 | /* |
| 273 | * As per x86, we may deadlock here. However, since the kernel only |
| 274 | * validly references user space from well defined areas of the code, |
| 275 | * we can bug out early if this is from code which shouldn't. |
| 276 | */ |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 277 | if (!mmap_read_trylock(mm)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 278 | if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc)) |
| 279 | goto no_context; |
| 280 | retry: |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 281 | mmap_read_lock(mm); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 282 | } else { |
| 283 | /* |
| 284 | * The above down_read_trylock() might have succeeded in |
| 285 | * which case, we'll have missed the might_sleep() from |
| 286 | * down_read() |
| 287 | */ |
| 288 | might_sleep(); |
| 289 | #ifdef CONFIG_DEBUG_VM |
| 290 | if (!user_mode(regs) && |
| 291 | !search_exception_tables(regs->ARM_pc)) |
| 292 | goto no_context; |
| 293 | #endif |
| 294 | } |
| 295 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 296 | fault = __do_page_fault(mm, addr, fsr, flags, tsk, regs); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 297 | |
| 298 | /* If we need to retry but a fatal signal is pending, handle the |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 299 | * signal first. We do not need to release the mmap_lock because |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 300 | * it would already be released in __lock_page_or_retry in |
| 301 | * mm/filemap.c. */ |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 302 | if (fault_signal_pending(fault, regs)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 303 | if (!user_mode(regs)) |
| 304 | goto no_context; |
| 305 | return 0; |
| 306 | } |
| 307 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 308 | if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 309 | if (fault & VM_FAULT_RETRY) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 310 | flags |= FAULT_FLAG_TRIED; |
| 311 | goto retry; |
| 312 | } |
| 313 | } |
| 314 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 315 | mmap_read_unlock(mm); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 316 | |
| 317 | /* |
| 318 | * Handle the "normal" case first - VM_FAULT_MAJOR |
| 319 | */ |
| 320 | if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) |
| 321 | return 0; |
| 322 | |
| 323 | /* |
| 324 | * If we are in kernel mode at this point, we |
| 325 | * have no context to handle this fault with. |
| 326 | */ |
| 327 | if (!user_mode(regs)) |
| 328 | goto no_context; |
| 329 | |
| 330 | if (fault & VM_FAULT_OOM) { |
| 331 | /* |
| 332 | * We ran out of memory, call the OOM killer, and return to |
| 333 | * userspace (which will retry the fault, or kill us if we |
| 334 | * got oom-killed) |
| 335 | */ |
| 336 | pagefault_out_of_memory(); |
| 337 | return 0; |
| 338 | } |
| 339 | |
| 340 | if (fault & VM_FAULT_SIGBUS) { |
| 341 | /* |
| 342 | * We had some memory, but were unable to |
| 343 | * successfully fix up this page fault. |
| 344 | */ |
| 345 | sig = SIGBUS; |
| 346 | code = BUS_ADRERR; |
| 347 | } else { |
| 348 | /* |
| 349 | * Something tried to access memory that |
| 350 | * isn't in our memory map.. |
| 351 | */ |
| 352 | sig = SIGSEGV; |
| 353 | code = fault == VM_FAULT_BADACCESS ? |
| 354 | SEGV_ACCERR : SEGV_MAPERR; |
| 355 | } |
| 356 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 357 | __do_user_fault(addr, fsr, sig, code, regs); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 358 | return 0; |
| 359 | |
| 360 | no_context: |
| 361 | __do_kernel_fault(mm, addr, fsr, regs); |
| 362 | return 0; |
| 363 | } |
| 364 | #else /* CONFIG_MMU */ |
| 365 | static int |
| 366 | do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 367 | { |
| 368 | return 0; |
| 369 | } |
| 370 | #endif /* CONFIG_MMU */ |
| 371 | |
| 372 | /* |
| 373 | * First Level Translation Fault Handler |
| 374 | * |
| 375 | * We enter here because the first level page table doesn't contain |
| 376 | * a valid entry for the address. |
| 377 | * |
| 378 | * If the address is in kernel space (>= TASK_SIZE), then we are |
| 379 | * probably faulting in the vmalloc() area. |
| 380 | * |
| 381 | * If the init_task's first level page tables contains the relevant |
| 382 | * entry, we copy the it to this task. If not, we send the process |
| 383 | * a signal, fixup the exception, or oops the kernel. |
| 384 | * |
| 385 | * NOTE! We MUST NOT take any locks for this case. We may be in an |
| 386 | * interrupt or a critical region, and should only copy the information |
| 387 | * from the master page table, nothing more. |
| 388 | */ |
| 389 | #ifdef CONFIG_MMU |
| 390 | static int __kprobes |
| 391 | do_translation_fault(unsigned long addr, unsigned int fsr, |
| 392 | struct pt_regs *regs) |
| 393 | { |
| 394 | unsigned int index; |
| 395 | pgd_t *pgd, *pgd_k; |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 396 | p4d_t *p4d, *p4d_k; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 397 | pud_t *pud, *pud_k; |
| 398 | pmd_t *pmd, *pmd_k; |
| 399 | |
| 400 | if (addr < TASK_SIZE) |
| 401 | return do_page_fault(addr, fsr, regs); |
| 402 | |
| 403 | if (user_mode(regs)) |
| 404 | goto bad_area; |
| 405 | |
| 406 | index = pgd_index(addr); |
| 407 | |
| 408 | pgd = cpu_get_pgd() + index; |
| 409 | pgd_k = init_mm.pgd + index; |
| 410 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 411 | p4d = p4d_offset(pgd, addr); |
| 412 | p4d_k = p4d_offset(pgd_k, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 413 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 414 | if (p4d_none(*p4d_k)) |
| 415 | goto bad_area; |
| 416 | if (!p4d_present(*p4d)) |
| 417 | set_p4d(p4d, *p4d_k); |
| 418 | |
| 419 | pud = pud_offset(p4d, addr); |
| 420 | pud_k = pud_offset(p4d_k, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 421 | |
| 422 | if (pud_none(*pud_k)) |
| 423 | goto bad_area; |
| 424 | if (!pud_present(*pud)) |
| 425 | set_pud(pud, *pud_k); |
| 426 | |
| 427 | pmd = pmd_offset(pud, addr); |
| 428 | pmd_k = pmd_offset(pud_k, addr); |
| 429 | |
| 430 | #ifdef CONFIG_ARM_LPAE |
| 431 | /* |
| 432 | * Only one hardware entry per PMD with LPAE. |
| 433 | */ |
| 434 | index = 0; |
| 435 | #else |
| 436 | /* |
| 437 | * On ARM one Linux PGD entry contains two hardware entries (see page |
| 438 | * tables layout in pgtable.h). We normally guarantee that we always |
| 439 | * fill both L1 entries. But create_mapping() doesn't follow the rule. |
| 440 | * It can create inidividual L1 entries, so here we have to call |
| 441 | * pmd_none() check for the entry really corresponded to address, not |
| 442 | * for the first of pair. |
| 443 | */ |
| 444 | index = (addr >> SECTION_SHIFT) & 1; |
| 445 | #endif |
| 446 | if (pmd_none(pmd_k[index])) |
| 447 | goto bad_area; |
| 448 | |
| 449 | copy_pmd(pmd, pmd_k); |
| 450 | return 0; |
| 451 | |
| 452 | bad_area: |
| 453 | do_bad_area(addr, fsr, regs); |
| 454 | return 0; |
| 455 | } |
| 456 | #else /* CONFIG_MMU */ |
| 457 | static int |
| 458 | do_translation_fault(unsigned long addr, unsigned int fsr, |
| 459 | struct pt_regs *regs) |
| 460 | { |
| 461 | return 0; |
| 462 | } |
| 463 | #endif /* CONFIG_MMU */ |
| 464 | |
| 465 | /* |
| 466 | * Some section permission faults need to be handled gracefully. |
| 467 | * They can happen due to a __{get,put}_user during an oops. |
| 468 | */ |
| 469 | #ifndef CONFIG_ARM_LPAE |
| 470 | static int |
| 471 | do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 472 | { |
| 473 | do_bad_area(addr, fsr, regs); |
| 474 | return 0; |
| 475 | } |
| 476 | #endif /* CONFIG_ARM_LPAE */ |
| 477 | |
| 478 | /* |
| 479 | * This abort handler always returns "fault". |
| 480 | */ |
| 481 | static int |
| 482 | do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 483 | { |
| 484 | return 1; |
| 485 | } |
| 486 | |
| 487 | struct fsr_info { |
| 488 | int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs); |
| 489 | int sig; |
| 490 | int code; |
| 491 | const char *name; |
| 492 | }; |
| 493 | |
| 494 | /* FSR definition */ |
| 495 | #ifdef CONFIG_ARM_LPAE |
| 496 | #include "fsr-3level.c" |
| 497 | #else |
| 498 | #include "fsr-2level.c" |
| 499 | #endif |
| 500 | |
| 501 | void __init |
| 502 | hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), |
| 503 | int sig, int code, const char *name) |
| 504 | { |
| 505 | if (nr < 0 || nr >= ARRAY_SIZE(fsr_info)) |
| 506 | BUG(); |
| 507 | |
| 508 | fsr_info[nr].fn = fn; |
| 509 | fsr_info[nr].sig = sig; |
| 510 | fsr_info[nr].code = code; |
| 511 | fsr_info[nr].name = name; |
| 512 | } |
| 513 | |
| 514 | /* |
| 515 | * Dispatch a data abort to the relevant handler. |
| 516 | */ |
| 517 | asmlinkage void |
| 518 | do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| 519 | { |
| 520 | const struct fsr_info *inf = fsr_info + fsr_fs(fsr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 521 | |
| 522 | if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) |
| 523 | return; |
| 524 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 525 | pr_alert("8<--- cut here ---\n"); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 526 | pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", |
| 527 | inf->name, fsr, addr); |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 528 | show_pte(KERN_ALERT, current->mm, addr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 529 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 530 | arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, |
| 531 | fsr, 0); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 532 | } |
| 533 | |
| 534 | void __init |
| 535 | hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), |
| 536 | int sig, int code, const char *name) |
| 537 | { |
| 538 | if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info)) |
| 539 | BUG(); |
| 540 | |
| 541 | ifsr_info[nr].fn = fn; |
| 542 | ifsr_info[nr].sig = sig; |
| 543 | ifsr_info[nr].code = code; |
| 544 | ifsr_info[nr].name = name; |
| 545 | } |
| 546 | |
| 547 | asmlinkage void |
| 548 | do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) |
| 549 | { |
| 550 | const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 551 | |
| 552 | if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) |
| 553 | return; |
| 554 | |
| 555 | pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", |
| 556 | inf->name, ifsr, addr); |
| 557 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 558 | arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, |
| 559 | ifsr, 0); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 560 | } |
| 561 | |
| 562 | /* |
| 563 | * Abort handler to be used only during first unmasking of asynchronous aborts |
| 564 | * on the boot CPU. This makes sure that the machine will not die if the |
| 565 | * firmware/bootloader left an imprecise abort pending for us to trip over. |
| 566 | */ |
| 567 | static int __init early_abort_handler(unsigned long addr, unsigned int fsr, |
| 568 | struct pt_regs *regs) |
| 569 | { |
| 570 | pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during " |
| 571 | "first unmask, this is most likely caused by a " |
| 572 | "firmware/bootloader bug.\n", fsr); |
| 573 | |
| 574 | return 0; |
| 575 | } |
| 576 | |
| 577 | void __init early_abt_enable(void) |
| 578 | { |
| 579 | fsr_info[FSR_FS_AEA].fn = early_abort_handler; |
| 580 | local_abt_enable(); |
| 581 | fsr_info[FSR_FS_AEA].fn = do_bad; |
| 582 | } |
| 583 | |
| 584 | #ifndef CONFIG_ARM_LPAE |
| 585 | static int __init exceptions_init(void) |
| 586 | { |
| 587 | if (cpu_architecture() >= CPU_ARCH_ARMv6) { |
| 588 | hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR, |
| 589 | "I-cache maintenance fault"); |
| 590 | } |
| 591 | |
| 592 | if (cpu_architecture() >= CPU_ARCH_ARMv7) { |
| 593 | /* |
| 594 | * TODO: Access flag faults introduced in ARMv6K. |
| 595 | * Runtime check for 'K' extension is needed |
| 596 | */ |
| 597 | hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR, |
| 598 | "section access flag fault"); |
| 599 | hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR, |
| 600 | "section access flag fault"); |
| 601 | } |
| 602 | |
| 603 | return 0; |
| 604 | } |
| 605 | |
| 606 | arch_initcall(exceptions_init); |
| 607 | #endif |