David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 2 | /* |
| 3 | * Kernel Probes Jump Optimization (Optprobes) |
| 4 | * |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 5 | * Copyright (C) IBM Corporation, 2002, 2004 |
| 6 | * Copyright (C) Hitachi Ltd., 2012 |
| 7 | */ |
| 8 | #include <linux/kprobes.h> |
| 9 | #include <linux/ptrace.h> |
| 10 | #include <linux/string.h> |
| 11 | #include <linux/slab.h> |
| 12 | #include <linux/hardirq.h> |
| 13 | #include <linux/preempt.h> |
| 14 | #include <linux/extable.h> |
| 15 | #include <linux/kdebug.h> |
| 16 | #include <linux/kallsyms.h> |
| 17 | #include <linux/ftrace.h> |
| 18 | #include <linux/frame.h> |
| 19 | |
| 20 | #include <asm/text-patching.h> |
| 21 | #include <asm/cacheflush.h> |
| 22 | #include <asm/desc.h> |
| 23 | #include <asm/pgtable.h> |
| 24 | #include <linux/uaccess.h> |
| 25 | #include <asm/alternative.h> |
| 26 | #include <asm/insn.h> |
| 27 | #include <asm/debugreg.h> |
| 28 | #include <asm/set_memory.h> |
| 29 | #include <asm/sections.h> |
| 30 | #include <asm/nospec-branch.h> |
| 31 | |
| 32 | #include "common.h" |
| 33 | |
| 34 | unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) |
| 35 | { |
| 36 | struct optimized_kprobe *op; |
| 37 | struct kprobe *kp; |
| 38 | long offs; |
| 39 | int i; |
| 40 | |
| 41 | for (i = 0; i < RELATIVEJUMP_SIZE; i++) { |
| 42 | kp = get_kprobe((void *)addr - i); |
| 43 | /* This function only handles jump-optimized kprobe */ |
| 44 | if (kp && kprobe_optimized(kp)) { |
| 45 | op = container_of(kp, struct optimized_kprobe, kp); |
| 46 | /* If op->list is not empty, op is under optimizing */ |
| 47 | if (list_empty(&op->list)) |
| 48 | goto found; |
| 49 | } |
| 50 | } |
| 51 | |
| 52 | return addr; |
| 53 | found: |
| 54 | /* |
| 55 | * If the kprobe can be optimized, original bytes which can be |
| 56 | * overwritten by jump destination address. In this case, original |
| 57 | * bytes must be recovered from op->optinsn.copied_insn buffer. |
| 58 | */ |
| 59 | if (probe_kernel_read(buf, (void *)addr, |
| 60 | MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) |
| 61 | return 0UL; |
| 62 | |
| 63 | if (addr == (unsigned long)kp->addr) { |
| 64 | buf[0] = kp->opcode; |
| 65 | memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); |
| 66 | } else { |
| 67 | offs = addr - (unsigned long)kp->addr - 1; |
| 68 | memcpy(buf, op->optinsn.copied_insn + offs, RELATIVE_ADDR_SIZE - offs); |
| 69 | } |
| 70 | |
| 71 | return (unsigned long)buf; |
| 72 | } |
| 73 | |
| 74 | /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ |
| 75 | static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) |
| 76 | { |
| 77 | #ifdef CONFIG_X86_64 |
| 78 | *addr++ = 0x48; |
| 79 | *addr++ = 0xbf; |
| 80 | #else |
| 81 | *addr++ = 0xb8; |
| 82 | #endif |
| 83 | *(unsigned long *)addr = val; |
| 84 | } |
| 85 | |
| 86 | asm ( |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 87 | ".pushsection .rodata\n" |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 88 | "optprobe_template_func:\n" |
| 89 | ".global optprobe_template_entry\n" |
| 90 | "optprobe_template_entry:\n" |
| 91 | #ifdef CONFIG_X86_64 |
| 92 | /* We don't bother saving the ss register */ |
| 93 | " pushq %rsp\n" |
| 94 | " pushfq\n" |
| 95 | SAVE_REGS_STRING |
| 96 | " movq %rsp, %rsi\n" |
| 97 | ".global optprobe_template_val\n" |
| 98 | "optprobe_template_val:\n" |
| 99 | ASM_NOP5 |
| 100 | ASM_NOP5 |
| 101 | ".global optprobe_template_call\n" |
| 102 | "optprobe_template_call:\n" |
| 103 | ASM_NOP5 |
| 104 | /* Move flags to rsp */ |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 105 | " movq 18*8(%rsp), %rdx\n" |
| 106 | " movq %rdx, 19*8(%rsp)\n" |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 107 | RESTORE_REGS_STRING |
| 108 | /* Skip flags entry */ |
| 109 | " addq $8, %rsp\n" |
| 110 | " popfq\n" |
| 111 | #else /* CONFIG_X86_32 */ |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 112 | " pushl %esp\n" |
| 113 | " pushfl\n" |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 114 | SAVE_REGS_STRING |
| 115 | " movl %esp, %edx\n" |
| 116 | ".global optprobe_template_val\n" |
| 117 | "optprobe_template_val:\n" |
| 118 | ASM_NOP5 |
| 119 | ".global optprobe_template_call\n" |
| 120 | "optprobe_template_call:\n" |
| 121 | ASM_NOP5 |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 122 | /* Move flags into esp */ |
| 123 | " movl 14*4(%esp), %edx\n" |
| 124 | " movl %edx, 15*4(%esp)\n" |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 125 | RESTORE_REGS_STRING |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 126 | /* Skip flags entry */ |
| 127 | " addl $4, %esp\n" |
| 128 | " popfl\n" |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 129 | #endif |
| 130 | ".global optprobe_template_end\n" |
| 131 | "optprobe_template_end:\n" |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 132 | ".popsection\n"); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 133 | |
| 134 | void optprobe_template_func(void); |
| 135 | STACK_FRAME_NON_STANDARD(optprobe_template_func); |
| 136 | |
| 137 | #define TMPL_MOVE_IDX \ |
| 138 | ((long)optprobe_template_val - (long)optprobe_template_entry) |
| 139 | #define TMPL_CALL_IDX \ |
| 140 | ((long)optprobe_template_call - (long)optprobe_template_entry) |
| 141 | #define TMPL_END_IDX \ |
| 142 | ((long)optprobe_template_end - (long)optprobe_template_entry) |
| 143 | |
| 144 | #define INT3_SIZE sizeof(kprobe_opcode_t) |
| 145 | |
| 146 | /* Optimized kprobe call back function: called from optinsn */ |
| 147 | static void |
| 148 | optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) |
| 149 | { |
| 150 | /* This is possible if op is under delayed unoptimizing */ |
| 151 | if (kprobe_disabled(&op->kp)) |
| 152 | return; |
| 153 | |
| 154 | preempt_disable(); |
| 155 | if (kprobe_running()) { |
| 156 | kprobes_inc_nmissed_count(&op->kp); |
| 157 | } else { |
| 158 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
| 159 | /* Save skipped registers */ |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 160 | regs->cs = __KERNEL_CS; |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 161 | #ifdef CONFIG_X86_32 |
| 162 | regs->cs |= get_kernel_rpl(); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 163 | regs->gs = 0; |
| 164 | #endif |
| 165 | regs->ip = (unsigned long)op->kp.addr + INT3_SIZE; |
| 166 | regs->orig_ax = ~0UL; |
| 167 | |
| 168 | __this_cpu_write(current_kprobe, &op->kp); |
| 169 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
| 170 | opt_pre_handler(&op->kp, regs); |
| 171 | __this_cpu_write(current_kprobe, NULL); |
| 172 | } |
| 173 | preempt_enable(); |
| 174 | } |
| 175 | NOKPROBE_SYMBOL(optimized_callback); |
| 176 | |
| 177 | static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) |
| 178 | { |
| 179 | struct insn insn; |
| 180 | int len = 0, ret; |
| 181 | |
| 182 | while (len < RELATIVEJUMP_SIZE) { |
| 183 | ret = __copy_instruction(dest + len, src + len, real + len, &insn); |
| 184 | if (!ret || !can_boost(&insn, src + len)) |
| 185 | return -EINVAL; |
| 186 | len += ret; |
| 187 | } |
| 188 | /* Check whether the address range is reserved */ |
| 189 | if (ftrace_text_reserved(src, src + len - 1) || |
| 190 | alternatives_text_reserved(src, src + len - 1) || |
| 191 | jump_label_text_reserved(src, src + len - 1)) |
| 192 | return -EBUSY; |
| 193 | |
| 194 | return len; |
| 195 | } |
| 196 | |
| 197 | /* Check whether insn is indirect jump */ |
| 198 | static int __insn_is_indirect_jump(struct insn *insn) |
| 199 | { |
| 200 | return ((insn->opcode.bytes[0] == 0xff && |
| 201 | (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ |
| 202 | insn->opcode.bytes[0] == 0xea); /* Segment based jump */ |
| 203 | } |
| 204 | |
| 205 | /* Check whether insn jumps into specified address range */ |
| 206 | static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) |
| 207 | { |
| 208 | unsigned long target = 0; |
| 209 | |
| 210 | switch (insn->opcode.bytes[0]) { |
| 211 | case 0xe0: /* loopne */ |
| 212 | case 0xe1: /* loope */ |
| 213 | case 0xe2: /* loop */ |
| 214 | case 0xe3: /* jcxz */ |
| 215 | case 0xe9: /* near relative jump */ |
| 216 | case 0xeb: /* short relative jump */ |
| 217 | break; |
| 218 | case 0x0f: |
| 219 | if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */ |
| 220 | break; |
| 221 | return 0; |
| 222 | default: |
| 223 | if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */ |
| 224 | break; |
| 225 | return 0; |
| 226 | } |
| 227 | target = (unsigned long)insn->next_byte + insn->immediate.value; |
| 228 | |
| 229 | return (start <= target && target <= start + len); |
| 230 | } |
| 231 | |
| 232 | static int insn_is_indirect_jump(struct insn *insn) |
| 233 | { |
| 234 | int ret = __insn_is_indirect_jump(insn); |
| 235 | |
| 236 | #ifdef CONFIG_RETPOLINE |
| 237 | /* |
| 238 | * Jump to x86_indirect_thunk_* is treated as an indirect jump. |
| 239 | * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with |
| 240 | * older gcc may use indirect jump. So we add this check instead of |
| 241 | * replace indirect-jump check. |
| 242 | */ |
| 243 | if (!ret) |
| 244 | ret = insn_jump_into_range(insn, |
| 245 | (unsigned long)__indirect_thunk_start, |
| 246 | (unsigned long)__indirect_thunk_end - |
| 247 | (unsigned long)__indirect_thunk_start); |
| 248 | #endif |
| 249 | return ret; |
| 250 | } |
| 251 | |
| 252 | /* Decode whole function to ensure any instructions don't jump into target */ |
| 253 | static int can_optimize(unsigned long paddr) |
| 254 | { |
| 255 | unsigned long addr, size = 0, offset = 0; |
| 256 | struct insn insn; |
| 257 | kprobe_opcode_t buf[MAX_INSN_SIZE]; |
| 258 | |
| 259 | /* Lookup symbol including addr */ |
| 260 | if (!kallsyms_lookup_size_offset(paddr, &size, &offset)) |
| 261 | return 0; |
| 262 | |
| 263 | /* |
| 264 | * Do not optimize in the entry code due to the unstable |
| 265 | * stack handling and registers setup. |
| 266 | */ |
| 267 | if (((paddr >= (unsigned long)__entry_text_start) && |
| 268 | (paddr < (unsigned long)__entry_text_end)) || |
| 269 | ((paddr >= (unsigned long)__irqentry_text_start) && |
| 270 | (paddr < (unsigned long)__irqentry_text_end))) |
| 271 | return 0; |
| 272 | |
| 273 | /* Check there is enough space for a relative jump. */ |
| 274 | if (size - offset < RELATIVEJUMP_SIZE) |
| 275 | return 0; |
| 276 | |
| 277 | /* Decode instructions */ |
| 278 | addr = paddr - offset; |
| 279 | while (addr < paddr - offset + size) { /* Decode until function end */ |
| 280 | unsigned long recovered_insn; |
| 281 | if (search_exception_tables(addr)) |
| 282 | /* |
| 283 | * Since some fixup code will jumps into this function, |
| 284 | * we can't optimize kprobe in this function. |
| 285 | */ |
| 286 | return 0; |
| 287 | recovered_insn = recover_probed_instruction(buf, addr); |
| 288 | if (!recovered_insn) |
| 289 | return 0; |
| 290 | kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE); |
| 291 | insn_get_length(&insn); |
| 292 | /* Another subsystem puts a breakpoint */ |
| 293 | if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) |
| 294 | return 0; |
| 295 | /* Recover address */ |
| 296 | insn.kaddr = (void *)addr; |
| 297 | insn.next_byte = (void *)(addr + insn.length); |
| 298 | /* Check any instructions don't jump into target */ |
| 299 | if (insn_is_indirect_jump(&insn) || |
| 300 | insn_jump_into_range(&insn, paddr + INT3_SIZE, |
| 301 | RELATIVE_ADDR_SIZE)) |
| 302 | return 0; |
| 303 | addr += insn.length; |
| 304 | } |
| 305 | |
| 306 | return 1; |
| 307 | } |
| 308 | |
| 309 | /* Check optimized_kprobe can actually be optimized. */ |
| 310 | int arch_check_optimized_kprobe(struct optimized_kprobe *op) |
| 311 | { |
| 312 | int i; |
| 313 | struct kprobe *p; |
| 314 | |
| 315 | for (i = 1; i < op->optinsn.size; i++) { |
| 316 | p = get_kprobe(op->kp.addr + i); |
| 317 | if (p && !kprobe_disabled(p)) |
| 318 | return -EEXIST; |
| 319 | } |
| 320 | |
| 321 | return 0; |
| 322 | } |
| 323 | |
| 324 | /* Check the addr is within the optimized instructions. */ |
| 325 | int arch_within_optimized_kprobe(struct optimized_kprobe *op, |
| 326 | unsigned long addr) |
| 327 | { |
| 328 | return ((unsigned long)op->kp.addr <= addr && |
| 329 | (unsigned long)op->kp.addr + op->optinsn.size > addr); |
| 330 | } |
| 331 | |
| 332 | /* Free optimized instruction slot */ |
| 333 | static |
| 334 | void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) |
| 335 | { |
| 336 | if (op->optinsn.insn) { |
| 337 | free_optinsn_slot(op->optinsn.insn, dirty); |
| 338 | op->optinsn.insn = NULL; |
| 339 | op->optinsn.size = 0; |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | void arch_remove_optimized_kprobe(struct optimized_kprobe *op) |
| 344 | { |
| 345 | __arch_remove_optimized_kprobe(op, 1); |
| 346 | } |
| 347 | |
| 348 | /* |
| 349 | * Copy replacing target instructions |
| 350 | * Target instructions MUST be relocatable (checked inside) |
| 351 | * This is called when new aggr(opt)probe is allocated or reused. |
| 352 | */ |
| 353 | int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, |
| 354 | struct kprobe *__unused) |
| 355 | { |
| 356 | u8 *buf = NULL, *slot; |
| 357 | int ret, len; |
| 358 | long rel; |
| 359 | |
| 360 | if (!can_optimize((unsigned long)op->kp.addr)) |
| 361 | return -EILSEQ; |
| 362 | |
| 363 | buf = kzalloc(MAX_OPTINSN_SIZE, GFP_KERNEL); |
| 364 | if (!buf) |
| 365 | return -ENOMEM; |
| 366 | |
| 367 | op->optinsn.insn = slot = get_optinsn_slot(); |
| 368 | if (!slot) { |
| 369 | ret = -ENOMEM; |
| 370 | goto out; |
| 371 | } |
| 372 | |
| 373 | /* |
| 374 | * Verify if the address gap is in 2GB range, because this uses |
| 375 | * a relative jump. |
| 376 | */ |
| 377 | rel = (long)slot - (long)op->kp.addr + RELATIVEJUMP_SIZE; |
| 378 | if (abs(rel) > 0x7fffffff) { |
| 379 | ret = -ERANGE; |
| 380 | goto err; |
| 381 | } |
| 382 | |
| 383 | /* Copy arch-dep-instance from template */ |
| 384 | memcpy(buf, optprobe_template_entry, TMPL_END_IDX); |
| 385 | |
| 386 | /* Copy instructions into the out-of-line buffer */ |
| 387 | ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr, |
| 388 | slot + TMPL_END_IDX); |
| 389 | if (ret < 0) |
| 390 | goto err; |
| 391 | op->optinsn.size = ret; |
| 392 | len = TMPL_END_IDX + op->optinsn.size; |
| 393 | |
| 394 | /* Set probe information */ |
| 395 | synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); |
| 396 | |
| 397 | /* Set probe function call */ |
| 398 | synthesize_relcall(buf + TMPL_CALL_IDX, |
| 399 | slot + TMPL_CALL_IDX, optimized_callback); |
| 400 | |
| 401 | /* Set returning jmp instruction at the tail of out-of-line buffer */ |
| 402 | synthesize_reljump(buf + len, slot + len, |
| 403 | (u8 *)op->kp.addr + op->optinsn.size); |
| 404 | len += RELATIVEJUMP_SIZE; |
| 405 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 406 | /* We have to use text_poke() for instruction buffer because it is RO */ |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 407 | text_poke(slot, buf, len); |
| 408 | ret = 0; |
| 409 | out: |
| 410 | kfree(buf); |
| 411 | return ret; |
| 412 | |
| 413 | err: |
| 414 | __arch_remove_optimized_kprobe(op, 0); |
| 415 | goto out; |
| 416 | } |
| 417 | |
| 418 | /* |
| 419 | * Replace breakpoints (int3) with relative jumps. |
| 420 | * Caller must call with locking kprobe_mutex and text_mutex. |
| 421 | */ |
| 422 | void arch_optimize_kprobes(struct list_head *oplist) |
| 423 | { |
| 424 | struct optimized_kprobe *op, *tmp; |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 425 | u8 insn_buff[RELATIVEJUMP_SIZE]; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 426 | |
| 427 | list_for_each_entry_safe(op, tmp, oplist, list) { |
| 428 | s32 rel = (s32)((long)op->optinsn.insn - |
| 429 | ((long)op->kp.addr + RELATIVEJUMP_SIZE)); |
| 430 | |
| 431 | WARN_ON(kprobe_disabled(&op->kp)); |
| 432 | |
| 433 | /* Backup instructions which will be replaced by jump address */ |
| 434 | memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, |
| 435 | RELATIVE_ADDR_SIZE); |
| 436 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 437 | insn_buff[0] = RELATIVEJUMP_OPCODE; |
| 438 | *(s32 *)(&insn_buff[1]) = rel; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 439 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 440 | text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 441 | op->optinsn.insn); |
| 442 | |
| 443 | list_del_init(&op->list); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | /* Replace a relative jump with a breakpoint (int3). */ |
| 448 | void arch_unoptimize_kprobe(struct optimized_kprobe *op) |
| 449 | { |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 450 | u8 insn_buff[RELATIVEJUMP_SIZE]; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 451 | |
| 452 | /* Set int3 to first byte for kprobes */ |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame^] | 453 | insn_buff[0] = BREAKPOINT_INSTRUCTION; |
| 454 | memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); |
| 455 | text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 456 | op->optinsn.insn); |
| 457 | } |
| 458 | |
| 459 | /* |
| 460 | * Recover original instructions and breakpoints from relative jumps. |
| 461 | * Caller must call with locking kprobe_mutex. |
| 462 | */ |
| 463 | extern void arch_unoptimize_kprobes(struct list_head *oplist, |
| 464 | struct list_head *done_list) |
| 465 | { |
| 466 | struct optimized_kprobe *op, *tmp; |
| 467 | |
| 468 | list_for_each_entry_safe(op, tmp, oplist, list) { |
| 469 | arch_unoptimize_kprobe(op); |
| 470 | list_move(&op->list, done_list); |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) |
| 475 | { |
| 476 | struct optimized_kprobe *op; |
| 477 | |
| 478 | if (p->flags & KPROBE_FLAG_OPTIMIZED) { |
| 479 | /* This kprobe is really able to run optimized path. */ |
| 480 | op = container_of(p, struct optimized_kprobe, kp); |
| 481 | /* Detour through copied instructions */ |
| 482 | regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; |
| 483 | if (!reenter) |
| 484 | reset_current_kprobe(); |
| 485 | return 1; |
| 486 | } |
| 487 | return 0; |
| 488 | } |
| 489 | NOKPROBE_SYMBOL(setup_detour_execution); |