David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 2 | /* |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2017 Zihao Yu |
| 5 | */ |
| 6 | |
| 7 | #include <linux/elf.h> |
| 8 | #include <linux/err.h> |
| 9 | #include <linux/errno.h> |
| 10 | #include <linux/moduleloader.h> |
Olivier Deprez | 0e64123 | 2021-09-23 10:07:05 +0200 | [diff] [blame] | 11 | #include <linux/vmalloc.h> |
| 12 | #include <linux/sizes.h> |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 13 | #include <linux/pgtable.h> |
Olivier Deprez | 0e64123 | 2021-09-23 10:07:05 +0200 | [diff] [blame] | 14 | #include <asm/sections.h> |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 15 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 16 | /* |
| 17 | * The auipc+jalr instruction pair can reach any PC-relative offset |
| 18 | * in the range [-2^31 - 2^11, 2^31 - 2^11) |
| 19 | */ |
| 20 | static bool riscv_insn_valid_32bit_offset(ptrdiff_t val) |
| 21 | { |
| 22 | #ifdef CONFIG_32BIT |
| 23 | return true; |
| 24 | #else |
| 25 | return (-(1L << 31) - (1L << 11)) <= val && val < ((1L << 31) - (1L << 11)); |
| 26 | #endif |
| 27 | } |
| 28 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 29 | static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v) |
| 30 | { |
| 31 | if (v != (u32)v) { |
| 32 | pr_err("%s: value %016llx out of range for 32-bit field\n", |
| 33 | me->name, (long long)v); |
| 34 | return -EINVAL; |
| 35 | } |
| 36 | *location = v; |
| 37 | return 0; |
| 38 | } |
| 39 | |
| 40 | static int apply_r_riscv_64_rela(struct module *me, u32 *location, Elf_Addr v) |
| 41 | { |
| 42 | *(u64 *)location = v; |
| 43 | return 0; |
| 44 | } |
| 45 | |
| 46 | static int apply_r_riscv_branch_rela(struct module *me, u32 *location, |
| 47 | Elf_Addr v) |
| 48 | { |
| 49 | ptrdiff_t offset = (void *)v - (void *)location; |
| 50 | u32 imm12 = (offset & 0x1000) << (31 - 12); |
| 51 | u32 imm11 = (offset & 0x800) >> (11 - 7); |
| 52 | u32 imm10_5 = (offset & 0x7e0) << (30 - 10); |
| 53 | u32 imm4_1 = (offset & 0x1e) << (11 - 4); |
| 54 | |
| 55 | *location = (*location & 0x1fff07f) | imm12 | imm11 | imm10_5 | imm4_1; |
| 56 | return 0; |
| 57 | } |
| 58 | |
| 59 | static int apply_r_riscv_jal_rela(struct module *me, u32 *location, |
| 60 | Elf_Addr v) |
| 61 | { |
| 62 | ptrdiff_t offset = (void *)v - (void *)location; |
| 63 | u32 imm20 = (offset & 0x100000) << (31 - 20); |
| 64 | u32 imm19_12 = (offset & 0xff000); |
| 65 | u32 imm11 = (offset & 0x800) << (20 - 11); |
| 66 | u32 imm10_1 = (offset & 0x7fe) << (30 - 10); |
| 67 | |
| 68 | *location = (*location & 0xfff) | imm20 | imm19_12 | imm11 | imm10_1; |
| 69 | return 0; |
| 70 | } |
| 71 | |
| 72 | static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, |
| 73 | Elf_Addr v) |
| 74 | { |
| 75 | ptrdiff_t offset = (void *)v - (void *)location; |
| 76 | u16 imm8 = (offset & 0x100) << (12 - 8); |
| 77 | u16 imm7_6 = (offset & 0xc0) >> (6 - 5); |
| 78 | u16 imm5 = (offset & 0x20) >> (5 - 2); |
| 79 | u16 imm4_3 = (offset & 0x18) << (12 - 5); |
| 80 | u16 imm2_1 = (offset & 0x6) << (12 - 10); |
| 81 | |
| 82 | *(u16 *)location = (*(u16 *)location & 0xe383) | |
| 83 | imm8 | imm7_6 | imm5 | imm4_3 | imm2_1; |
| 84 | return 0; |
| 85 | } |
| 86 | |
| 87 | static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location, |
| 88 | Elf_Addr v) |
| 89 | { |
| 90 | ptrdiff_t offset = (void *)v - (void *)location; |
| 91 | u16 imm11 = (offset & 0x800) << (12 - 11); |
| 92 | u16 imm10 = (offset & 0x400) >> (10 - 8); |
| 93 | u16 imm9_8 = (offset & 0x300) << (12 - 11); |
| 94 | u16 imm7 = (offset & 0x80) >> (7 - 6); |
| 95 | u16 imm6 = (offset & 0x40) << (12 - 11); |
| 96 | u16 imm5 = (offset & 0x20) >> (5 - 2); |
| 97 | u16 imm4 = (offset & 0x10) << (12 - 5); |
| 98 | u16 imm3_1 = (offset & 0xe) << (12 - 10); |
| 99 | |
| 100 | *(u16 *)location = (*(u16 *)location & 0xe003) | |
| 101 | imm11 | imm10 | imm9_8 | imm7 | imm6 | imm5 | imm4 | imm3_1; |
| 102 | return 0; |
| 103 | } |
| 104 | |
| 105 | static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location, |
| 106 | Elf_Addr v) |
| 107 | { |
| 108 | ptrdiff_t offset = (void *)v - (void *)location; |
| 109 | s32 hi20; |
| 110 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 111 | if (!riscv_insn_valid_32bit_offset(offset)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 112 | pr_err( |
| 113 | "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", |
| 114 | me->name, (long long)v, location); |
| 115 | return -EINVAL; |
| 116 | } |
| 117 | |
| 118 | hi20 = (offset + 0x800) & 0xfffff000; |
| 119 | *location = (*location & 0xfff) | hi20; |
| 120 | return 0; |
| 121 | } |
| 122 | |
| 123 | static int apply_r_riscv_pcrel_lo12_i_rela(struct module *me, u32 *location, |
| 124 | Elf_Addr v) |
| 125 | { |
| 126 | /* |
| 127 | * v is the lo12 value to fill. It is calculated before calling this |
| 128 | * handler. |
| 129 | */ |
| 130 | *location = (*location & 0xfffff) | ((v & 0xfff) << 20); |
| 131 | return 0; |
| 132 | } |
| 133 | |
| 134 | static int apply_r_riscv_pcrel_lo12_s_rela(struct module *me, u32 *location, |
| 135 | Elf_Addr v) |
| 136 | { |
| 137 | /* |
| 138 | * v is the lo12 value to fill. It is calculated before calling this |
| 139 | * handler. |
| 140 | */ |
| 141 | u32 imm11_5 = (v & 0xfe0) << (31 - 11); |
| 142 | u32 imm4_0 = (v & 0x1f) << (11 - 4); |
| 143 | |
| 144 | *location = (*location & 0x1fff07f) | imm11_5 | imm4_0; |
| 145 | return 0; |
| 146 | } |
| 147 | |
| 148 | static int apply_r_riscv_hi20_rela(struct module *me, u32 *location, |
| 149 | Elf_Addr v) |
| 150 | { |
| 151 | s32 hi20; |
| 152 | |
David Brazdil | 0f672f6 | 2019-12-10 10:32:29 +0000 | [diff] [blame] | 153 | if (IS_ENABLED(CONFIG_CMODEL_MEDLOW)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 154 | pr_err( |
| 155 | "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", |
| 156 | me->name, (long long)v, location); |
| 157 | return -EINVAL; |
| 158 | } |
| 159 | |
| 160 | hi20 = ((s32)v + 0x800) & 0xfffff000; |
| 161 | *location = (*location & 0xfff) | hi20; |
| 162 | return 0; |
| 163 | } |
| 164 | |
| 165 | static int apply_r_riscv_lo12_i_rela(struct module *me, u32 *location, |
| 166 | Elf_Addr v) |
| 167 | { |
| 168 | /* Skip medlow checking because of filtering by HI20 already */ |
| 169 | s32 hi20 = ((s32)v + 0x800) & 0xfffff000; |
| 170 | s32 lo12 = ((s32)v - hi20); |
| 171 | *location = (*location & 0xfffff) | ((lo12 & 0xfff) << 20); |
| 172 | return 0; |
| 173 | } |
| 174 | |
| 175 | static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location, |
| 176 | Elf_Addr v) |
| 177 | { |
| 178 | /* Skip medlow checking because of filtering by HI20 already */ |
| 179 | s32 hi20 = ((s32)v + 0x800) & 0xfffff000; |
| 180 | s32 lo12 = ((s32)v - hi20); |
| 181 | u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11); |
| 182 | u32 imm4_0 = (lo12 & 0x1f) << (11 - 4); |
| 183 | *location = (*location & 0x1fff07f) | imm11_5 | imm4_0; |
| 184 | return 0; |
| 185 | } |
| 186 | |
| 187 | static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location, |
| 188 | Elf_Addr v) |
| 189 | { |
| 190 | ptrdiff_t offset = (void *)v - (void *)location; |
| 191 | s32 hi20; |
| 192 | |
| 193 | /* Always emit the got entry */ |
| 194 | if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) { |
| 195 | offset = module_emit_got_entry(me, v); |
| 196 | offset = (void *)offset - (void *)location; |
| 197 | } else { |
| 198 | pr_err( |
| 199 | "%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n", |
| 200 | me->name, (long long)v, location); |
| 201 | return -EINVAL; |
| 202 | } |
| 203 | |
| 204 | hi20 = (offset + 0x800) & 0xfffff000; |
| 205 | *location = (*location & 0xfff) | hi20; |
| 206 | return 0; |
| 207 | } |
| 208 | |
| 209 | static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, |
| 210 | Elf_Addr v) |
| 211 | { |
| 212 | ptrdiff_t offset = (void *)v - (void *)location; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 213 | u32 hi20, lo12; |
| 214 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 215 | if (!riscv_insn_valid_32bit_offset(offset)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 216 | /* Only emit the plt entry if offset over 32-bit range */ |
| 217 | if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) { |
| 218 | offset = module_emit_plt_entry(me, v); |
| 219 | offset = (void *)offset - (void *)location; |
| 220 | } else { |
| 221 | pr_err( |
| 222 | "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", |
| 223 | me->name, (long long)v, location); |
| 224 | return -EINVAL; |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | hi20 = (offset + 0x800) & 0xfffff000; |
| 229 | lo12 = (offset - hi20) & 0xfff; |
| 230 | *location = (*location & 0xfff) | hi20; |
| 231 | *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20); |
| 232 | return 0; |
| 233 | } |
| 234 | |
| 235 | static int apply_r_riscv_call_rela(struct module *me, u32 *location, |
| 236 | Elf_Addr v) |
| 237 | { |
| 238 | ptrdiff_t offset = (void *)v - (void *)location; |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 239 | u32 hi20, lo12; |
| 240 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 241 | if (!riscv_insn_valid_32bit_offset(offset)) { |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 242 | pr_err( |
| 243 | "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n", |
| 244 | me->name, (long long)v, location); |
| 245 | return -EINVAL; |
| 246 | } |
| 247 | |
| 248 | hi20 = (offset + 0x800) & 0xfffff000; |
| 249 | lo12 = (offset - hi20) & 0xfff; |
| 250 | *location = (*location & 0xfff) | hi20; |
| 251 | *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20); |
| 252 | return 0; |
| 253 | } |
| 254 | |
| 255 | static int apply_r_riscv_relax_rela(struct module *me, u32 *location, |
| 256 | Elf_Addr v) |
| 257 | { |
| 258 | return 0; |
| 259 | } |
| 260 | |
| 261 | static int apply_r_riscv_align_rela(struct module *me, u32 *location, |
| 262 | Elf_Addr v) |
| 263 | { |
| 264 | pr_err( |
| 265 | "%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n", |
| 266 | me->name, location); |
| 267 | return -EINVAL; |
| 268 | } |
| 269 | |
| 270 | static int apply_r_riscv_add32_rela(struct module *me, u32 *location, |
| 271 | Elf_Addr v) |
| 272 | { |
| 273 | *(u32 *)location += (u32)v; |
| 274 | return 0; |
| 275 | } |
| 276 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 277 | static int apply_r_riscv_add64_rela(struct module *me, u32 *location, |
| 278 | Elf_Addr v) |
| 279 | { |
| 280 | *(u64 *)location += (u64)v; |
| 281 | return 0; |
| 282 | } |
| 283 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 284 | static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, |
| 285 | Elf_Addr v) |
| 286 | { |
| 287 | *(u32 *)location -= (u32)v; |
| 288 | return 0; |
| 289 | } |
| 290 | |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 291 | static int apply_r_riscv_sub64_rela(struct module *me, u32 *location, |
| 292 | Elf_Addr v) |
| 293 | { |
| 294 | *(u64 *)location -= (u64)v; |
| 295 | return 0; |
| 296 | } |
| 297 | |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 298 | static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, |
| 299 | Elf_Addr v) = { |
| 300 | [R_RISCV_32] = apply_r_riscv_32_rela, |
| 301 | [R_RISCV_64] = apply_r_riscv_64_rela, |
| 302 | [R_RISCV_BRANCH] = apply_r_riscv_branch_rela, |
| 303 | [R_RISCV_JAL] = apply_r_riscv_jal_rela, |
| 304 | [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela, |
| 305 | [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela, |
| 306 | [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela, |
| 307 | [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela, |
| 308 | [R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela, |
| 309 | [R_RISCV_HI20] = apply_r_riscv_hi20_rela, |
| 310 | [R_RISCV_LO12_I] = apply_r_riscv_lo12_i_rela, |
| 311 | [R_RISCV_LO12_S] = apply_r_riscv_lo12_s_rela, |
| 312 | [R_RISCV_GOT_HI20] = apply_r_riscv_got_hi20_rela, |
| 313 | [R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela, |
| 314 | [R_RISCV_CALL] = apply_r_riscv_call_rela, |
| 315 | [R_RISCV_RELAX] = apply_r_riscv_relax_rela, |
| 316 | [R_RISCV_ALIGN] = apply_r_riscv_align_rela, |
| 317 | [R_RISCV_ADD32] = apply_r_riscv_add32_rela, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 318 | [R_RISCV_ADD64] = apply_r_riscv_add64_rela, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 319 | [R_RISCV_SUB32] = apply_r_riscv_sub32_rela, |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 320 | [R_RISCV_SUB64] = apply_r_riscv_sub64_rela, |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 321 | }; |
| 322 | |
| 323 | int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
| 324 | unsigned int symindex, unsigned int relsec, |
| 325 | struct module *me) |
| 326 | { |
| 327 | Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; |
| 328 | int (*handler)(struct module *me, u32 *location, Elf_Addr v); |
| 329 | Elf_Sym *sym; |
| 330 | u32 *location; |
| 331 | unsigned int i, type; |
| 332 | Elf_Addr v; |
| 333 | int res; |
| 334 | |
| 335 | pr_debug("Applying relocate section %u to %u\n", relsec, |
| 336 | sechdrs[relsec].sh_info); |
| 337 | |
| 338 | for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { |
| 339 | /* This is where to make the change */ |
| 340 | location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr |
| 341 | + rel[i].r_offset; |
| 342 | /* This is the symbol it is referring to */ |
| 343 | sym = (Elf_Sym *)sechdrs[symindex].sh_addr |
| 344 | + ELF_RISCV_R_SYM(rel[i].r_info); |
| 345 | if (IS_ERR_VALUE(sym->st_value)) { |
| 346 | /* Ignore unresolved weak symbol */ |
| 347 | if (ELF_ST_BIND(sym->st_info) == STB_WEAK) |
| 348 | continue; |
Olivier Deprez | 157378f | 2022-04-04 15:47:50 +0200 | [diff] [blame^] | 349 | pr_warn("%s: Unknown symbol %s\n", |
| 350 | me->name, strtab + sym->st_name); |
Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame] | 351 | return -ENOENT; |
| 352 | } |
| 353 | |
| 354 | type = ELF_RISCV_R_TYPE(rel[i].r_info); |
| 355 | |
| 356 | if (type < ARRAY_SIZE(reloc_handlers_rela)) |
| 357 | handler = reloc_handlers_rela[type]; |
| 358 | else |
| 359 | handler = NULL; |
| 360 | |
| 361 | if (!handler) { |
| 362 | pr_err("%s: Unknown relocation type %u\n", |
| 363 | me->name, type); |
| 364 | return -EINVAL; |
| 365 | } |
| 366 | |
| 367 | v = sym->st_value + rel[i].r_addend; |
| 368 | |
| 369 | if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) { |
| 370 | unsigned int j; |
| 371 | |
| 372 | for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) { |
| 373 | unsigned long hi20_loc = |
| 374 | sechdrs[sechdrs[relsec].sh_info].sh_addr |
| 375 | + rel[j].r_offset; |
| 376 | u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info); |
| 377 | |
| 378 | /* Find the corresponding HI20 relocation entry */ |
| 379 | if (hi20_loc == sym->st_value |
| 380 | && (hi20_type == R_RISCV_PCREL_HI20 |
| 381 | || hi20_type == R_RISCV_GOT_HI20)) { |
| 382 | s32 hi20, lo12; |
| 383 | Elf_Sym *hi20_sym = |
| 384 | (Elf_Sym *)sechdrs[symindex].sh_addr |
| 385 | + ELF_RISCV_R_SYM(rel[j].r_info); |
| 386 | unsigned long hi20_sym_val = |
| 387 | hi20_sym->st_value |
| 388 | + rel[j].r_addend; |
| 389 | |
| 390 | /* Calculate lo12 */ |
| 391 | size_t offset = hi20_sym_val - hi20_loc; |
| 392 | if (IS_ENABLED(CONFIG_MODULE_SECTIONS) |
| 393 | && hi20_type == R_RISCV_GOT_HI20) { |
| 394 | offset = module_emit_got_entry( |
| 395 | me, hi20_sym_val); |
| 396 | offset = offset - hi20_loc; |
| 397 | } |
| 398 | hi20 = (offset + 0x800) & 0xfffff000; |
| 399 | lo12 = offset - hi20; |
| 400 | v = lo12; |
| 401 | |
| 402 | break; |
| 403 | } |
| 404 | } |
| 405 | if (j == sechdrs[relsec].sh_size / sizeof(*rel)) { |
| 406 | pr_err( |
| 407 | "%s: Can not find HI20 relocation information\n", |
| 408 | me->name); |
| 409 | return -EINVAL; |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | res = handler(me, location, v); |
| 414 | if (res) |
| 415 | return res; |
| 416 | } |
| 417 | |
| 418 | return 0; |
| 419 | } |
Olivier Deprez | 0e64123 | 2021-09-23 10:07:05 +0200 | [diff] [blame] | 420 | |
| 421 | #if defined(CONFIG_MMU) && defined(CONFIG_64BIT) |
| 422 | #define VMALLOC_MODULE_START \ |
| 423 | max(PFN_ALIGN((unsigned long)&_end - SZ_2G), VMALLOC_START) |
| 424 | void *module_alloc(unsigned long size) |
| 425 | { |
| 426 | return __vmalloc_node_range(size, 1, VMALLOC_MODULE_START, |
| 427 | VMALLOC_END, GFP_KERNEL, |
| 428 | PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, |
| 429 | __builtin_return_address(0)); |
| 430 | } |
| 431 | #endif |