1 // SPDX-License-Identifier: GPL-2.0+
3 * Author: Hanlu Li <lihanlu@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
9 #define pr_fmt(fmt) "kmod: " fmt
11 #include <linux/moduleloader.h>
12 #include <linux/elf.h>
14 #include <linux/numa.h>
15 #include <linux/vmalloc.h>
16 #include <linux/slab.h>
18 #include <linux/ftrace.h>
19 #include <linux/string.h>
20 #include <linux/kernel.h>
21 #include <asm/alternative.h>
24 static int rela_stack_push(s64 stack_value
, s64
*rela_stack
, size_t *rela_stack_top
)
26 if (*rela_stack_top
>= RELA_STACK_DEPTH
)
29 rela_stack
[(*rela_stack_top
)++] = stack_value
;
30 pr_debug("%s stack_value = 0x%llx\n", __func__
, stack_value
);
35 static int rela_stack_pop(s64
*stack_value
, s64
*rela_stack
, size_t *rela_stack_top
)
37 if (*rela_stack_top
== 0)
40 *stack_value
= rela_stack
[--(*rela_stack_top
)];
41 pr_debug("%s stack_value = 0x%llx\n", __func__
, *stack_value
);
46 static int apply_r_larch_none(struct module
*mod
, u32
*location
, Elf_Addr v
,
47 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
52 static int apply_r_larch_error(struct module
*me
, u32
*location
, Elf_Addr v
,
53 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
55 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me
->name
, type
);
59 static int apply_r_larch_32(struct module
*mod
, u32
*location
, Elf_Addr v
,
60 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
66 static int apply_r_larch_64(struct module
*mod
, u32
*location
, Elf_Addr v
,
67 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
69 *(Elf_Addr
*)location
= v
;
73 static int apply_r_larch_sop_push_pcrel(struct module
*mod
, u32
*location
, Elf_Addr v
,
74 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
76 return rela_stack_push(v
- (u64
)location
, rela_stack
, rela_stack_top
);
79 static int apply_r_larch_sop_push_absolute(struct module
*mod
, u32
*location
, Elf_Addr v
,
80 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
82 return rela_stack_push(v
, rela_stack
, rela_stack_top
);
85 static int apply_r_larch_sop_push_dup(struct module
*mod
, u32
*location
, Elf_Addr v
,
86 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
91 err
= rela_stack_pop(&opr1
, rela_stack
, rela_stack_top
);
94 err
= rela_stack_push(opr1
, rela_stack
, rela_stack_top
);
97 err
= rela_stack_push(opr1
, rela_stack
, rela_stack_top
);
104 static int apply_r_larch_sop_push_plt_pcrel(struct module
*mod
,
105 Elf_Shdr
*sechdrs
, u32
*location
, Elf_Addr v
,
106 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
108 ptrdiff_t offset
= (void *)v
- (void *)location
;
110 if (offset
>= SZ_128M
)
111 v
= module_emit_plt_entry(mod
, sechdrs
, v
);
113 if (offset
< -SZ_128M
)
114 v
= module_emit_plt_entry(mod
, sechdrs
, v
);
116 return apply_r_larch_sop_push_pcrel(mod
, location
, v
, rela_stack
, rela_stack_top
, type
);
119 static int apply_r_larch_sop(struct module
*mod
, u32
*location
, Elf_Addr v
,
120 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
123 s64 opr1
, opr2
, opr3
;
125 if (type
== R_LARCH_SOP_IF_ELSE
) {
126 err
= rela_stack_pop(&opr3
, rela_stack
, rela_stack_top
);
131 err
= rela_stack_pop(&opr2
, rela_stack
, rela_stack_top
);
134 err
= rela_stack_pop(&opr1
, rela_stack
, rela_stack_top
);
139 case R_LARCH_SOP_AND
:
140 err
= rela_stack_push(opr1
& opr2
, rela_stack
, rela_stack_top
);
142 case R_LARCH_SOP_ADD
:
143 err
= rela_stack_push(opr1
+ opr2
, rela_stack
, rela_stack_top
);
145 case R_LARCH_SOP_SUB
:
146 err
= rela_stack_push(opr1
- opr2
, rela_stack
, rela_stack_top
);
149 err
= rela_stack_push(opr1
<< opr2
, rela_stack
, rela_stack_top
);
152 err
= rela_stack_push(opr1
>> opr2
, rela_stack
, rela_stack_top
);
154 case R_LARCH_SOP_IF_ELSE
:
155 err
= rela_stack_push(opr1
? opr2
: opr3
, rela_stack
, rela_stack_top
);
158 pr_err("%s: Unsupport relocation type %u\n", mod
->name
, type
);
165 static int apply_r_larch_sop_imm_field(struct module
*mod
, u32
*location
, Elf_Addr v
,
166 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
170 union loongarch_instruction
*insn
= (union loongarch_instruction
*)location
;
172 err
= rela_stack_pop(&opr1
, rela_stack
, rela_stack_top
);
177 case R_LARCH_SOP_POP_32_U_10_12
:
178 if (!unsigned_imm_check(opr1
, 12))
181 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
182 insn
->reg2i12_format
.immediate
= opr1
& 0xfff;
184 case R_LARCH_SOP_POP_32_S_10_12
:
185 if (!signed_imm_check(opr1
, 12))
188 insn
->reg2i12_format
.immediate
= opr1
& 0xfff;
190 case R_LARCH_SOP_POP_32_S_10_16
:
191 if (!signed_imm_check(opr1
, 16))
194 insn
->reg2i16_format
.immediate
= opr1
& 0xffff;
196 case R_LARCH_SOP_POP_32_S_10_16_S2
:
200 if (!signed_imm_check(opr1
, 18))
203 insn
->reg2i16_format
.immediate
= (opr1
>> 2) & 0xffff;
205 case R_LARCH_SOP_POP_32_S_5_20
:
206 if (!signed_imm_check(opr1
, 20))
209 insn
->reg1i20_format
.immediate
= (opr1
) & 0xfffff;
211 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2
:
215 if (!signed_imm_check(opr1
, 23))
219 insn
->reg1i21_format
.immediate_l
= opr1
& 0xffff;
220 insn
->reg1i21_format
.immediate_h
= (opr1
>> 16) & 0x1f;
222 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2
:
226 if (!signed_imm_check(opr1
, 28))
230 insn
->reg0i26_format
.immediate_l
= opr1
& 0xffff;
231 insn
->reg0i26_format
.immediate_h
= (opr1
>> 16) & 0x3ff;
233 case R_LARCH_SOP_POP_32_U
:
234 if (!unsigned_imm_check(opr1
, 32))
237 /* (*(uint32_t *) PC) = opr */
238 *location
= (u32
)opr1
;
241 pr_err("%s: Unsupport relocation type %u\n", mod
->name
, type
);
246 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
247 mod
->name
, opr1
, __func__
, type
);
251 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
252 mod
->name
, opr1
, __func__
, type
);
256 static int apply_r_larch_add_sub(struct module
*mod
, u32
*location
, Elf_Addr v
,
257 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
261 *(s32
*)location
+= v
;
264 *(s64
*)location
+= v
;
267 *(s32
*)location
-= v
;
270 *(s64
*)location
-= v
;
273 pr_err("%s: Unsupport relocation type %u\n", mod
->name
, type
);
278 static int apply_r_larch_b26(struct module
*mod
,
279 Elf_Shdr
*sechdrs
, u32
*location
, Elf_Addr v
,
280 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
282 ptrdiff_t offset
= (void *)v
- (void *)location
;
283 union loongarch_instruction
*insn
= (union loongarch_instruction
*)location
;
285 if (offset
>= SZ_128M
)
286 v
= module_emit_plt_entry(mod
, sechdrs
, v
);
288 if (offset
< -SZ_128M
)
289 v
= module_emit_plt_entry(mod
, sechdrs
, v
);
291 offset
= (void *)v
- (void *)location
;
294 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
295 mod
->name
, (long long)offset
, type
);
299 if (!signed_imm_check(offset
, 28)) {
300 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
301 mod
->name
, (long long)offset
, type
);
306 insn
->reg0i26_format
.immediate_l
= offset
& 0xffff;
307 insn
->reg0i26_format
.immediate_h
= (offset
>> 16) & 0x3ff;
312 static int apply_r_larch_pcala(struct module
*mod
, u32
*location
, Elf_Addr v
,
313 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
315 union loongarch_instruction
*insn
= (union loongarch_instruction
*)location
;
316 /* Use s32 for a sign-extension deliberately. */
317 s32 offset_hi20
= (void *)((v
+ 0x800) & ~0xfff) -
318 (void *)((Elf_Addr
)location
& ~0xfff);
319 Elf_Addr anchor
= (((Elf_Addr
)location
) & ~0xfff) + offset_hi20
;
320 ptrdiff_t offset_rem
= (void *)v
- (void *)anchor
;
323 case R_LARCH_PCALA_LO12
:
324 insn
->reg2i12_format
.immediate
= v
& 0xfff;
326 case R_LARCH_PCALA_HI20
:
327 v
= offset_hi20
>> 12;
328 insn
->reg1i20_format
.immediate
= v
& 0xfffff;
330 case R_LARCH_PCALA64_LO20
:
331 v
= offset_rem
>> 32;
332 insn
->reg1i20_format
.immediate
= v
& 0xfffff;
334 case R_LARCH_PCALA64_HI12
:
335 v
= offset_rem
>> 52;
336 insn
->reg2i12_format
.immediate
= v
& 0xfff;
339 pr_err("%s: Unsupport relocation type %u\n", mod
->name
, type
);
346 static int apply_r_larch_got_pc(struct module
*mod
,
347 Elf_Shdr
*sechdrs
, u32
*location
, Elf_Addr v
,
348 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
)
350 Elf_Addr got
= module_emit_got_entry(mod
, sechdrs
, v
);
356 case R_LARCH_GOT_PC_LO12
:
357 type
= R_LARCH_PCALA_LO12
;
359 case R_LARCH_GOT_PC_HI20
:
360 type
= R_LARCH_PCALA_HI20
;
363 pr_err("%s: Unsupport relocation type %u\n", mod
->name
, type
);
367 return apply_r_larch_pcala(mod
, location
, got
, rela_stack
, rela_stack_top
, type
);
371 * reloc_handlers_rela() - Apply a particular relocation to a module
372 * @mod: the module to apply the reloc to
373 * @location: the address at which the reloc is to be applied
374 * @v: the value of the reloc, with addend for RELA-style
375 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
376 * @rela_stac_top: where the stack operation(pop/push) applies to
378 * Return: 0 upon success, else -ERRNO
380 typedef int (*reloc_rela_handler
)(struct module
*mod
, u32
*location
, Elf_Addr v
,
381 s64
*rela_stack
, size_t *rela_stack_top
, unsigned int type
);
383 /* The handlers for known reloc types */
384 static reloc_rela_handler reloc_rela_handlers
[] = {
385 [R_LARCH_NONE
... R_LARCH_RELAX
] = apply_r_larch_error
,
387 [R_LARCH_NONE
] = apply_r_larch_none
,
388 [R_LARCH_32
] = apply_r_larch_32
,
389 [R_LARCH_64
] = apply_r_larch_64
,
390 [R_LARCH_MARK_LA
] = apply_r_larch_none
,
391 [R_LARCH_MARK_PCREL
] = apply_r_larch_none
,
392 [R_LARCH_SOP_PUSH_PCREL
] = apply_r_larch_sop_push_pcrel
,
393 [R_LARCH_SOP_PUSH_ABSOLUTE
] = apply_r_larch_sop_push_absolute
,
394 [R_LARCH_SOP_PUSH_DUP
] = apply_r_larch_sop_push_dup
,
395 [R_LARCH_SOP_SUB
... R_LARCH_SOP_IF_ELSE
] = apply_r_larch_sop
,
396 [R_LARCH_SOP_POP_32_S_10_5
... R_LARCH_SOP_POP_32_U
] = apply_r_larch_sop_imm_field
,
397 [R_LARCH_ADD32
... R_LARCH_SUB64
] = apply_r_larch_add_sub
,
398 [R_LARCH_PCALA_HI20
...R_LARCH_PCALA64_HI12
] = apply_r_larch_pcala
,
401 int apply_relocate_add(Elf_Shdr
*sechdrs
, const char *strtab
,
402 unsigned int symindex
, unsigned int relsec
,
407 s64 rela_stack
[RELA_STACK_DEPTH
];
408 size_t rela_stack_top
= 0;
409 reloc_rela_handler handler
;
413 Elf_Rela
*rel
= (void *) sechdrs
[relsec
].sh_addr
;
415 pr_debug("%s: Applying relocate section %u to %u\n", __func__
, relsec
,
416 sechdrs
[relsec
].sh_info
);
419 for (i
= 0; i
< sechdrs
[relsec
].sh_size
/ sizeof(*rel
); i
++) {
420 /* This is where to make the change */
421 location
= (void *)sechdrs
[sechdrs
[relsec
].sh_info
].sh_addr
+ rel
[i
].r_offset
;
422 /* This is the symbol it is referring to */
423 sym
= (Elf_Sym
*)sechdrs
[symindex
].sh_addr
+ ELF_R_SYM(rel
[i
].r_info
);
424 if (IS_ERR_VALUE(sym
->st_value
)) {
425 /* Ignore unresolved weak symbol */
426 if (ELF_ST_BIND(sym
->st_info
) == STB_WEAK
)
428 pr_warn("%s: Unknown symbol %s\n", mod
->name
, strtab
+ sym
->st_name
);
432 type
= ELF_R_TYPE(rel
[i
].r_info
);
434 if (type
< ARRAY_SIZE(reloc_rela_handlers
))
435 handler
= reloc_rela_handlers
[type
];
440 pr_err("%s: Unknown relocation type %u\n", mod
->name
, type
);
444 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
445 (int)ELF_R_TYPE(rel
[i
].r_info
),
446 sym
->st_value
, rel
[i
].r_addend
, (u64
)location
);
448 v
= sym
->st_value
+ rel
[i
].r_addend
;
451 err
= apply_r_larch_b26(mod
, sechdrs
, location
,
452 v
, rela_stack
, &rela_stack_top
, type
);
454 case R_LARCH_GOT_PC_HI20
...R_LARCH_GOT_PC_LO12
:
455 err
= apply_r_larch_got_pc(mod
, sechdrs
, location
,
456 v
, rela_stack
, &rela_stack_top
, type
);
458 case R_LARCH_SOP_PUSH_PLT_PCREL
:
459 err
= apply_r_larch_sop_push_plt_pcrel(mod
, sechdrs
, location
,
460 v
, rela_stack
, &rela_stack_top
, type
);
463 err
= handler(mod
, location
, v
, rela_stack
, &rela_stack_top
, type
);
472 void *module_alloc(unsigned long size
)
474 return __vmalloc_node_range(size
, 1, MODULES_VADDR
, MODULES_END
,
475 GFP_KERNEL
, PAGE_KERNEL
, 0, NUMA_NO_NODE
, __builtin_return_address(0));
478 static void module_init_ftrace_plt(const Elf_Ehdr
*hdr
,
479 const Elf_Shdr
*sechdrs
, struct module
*mod
)
481 #ifdef CONFIG_DYNAMIC_FTRACE
482 struct plt_entry
*ftrace_plts
;
484 ftrace_plts
= (void *)sechdrs
->sh_addr
;
486 ftrace_plts
[FTRACE_PLT_IDX
] = emit_plt_entry(FTRACE_ADDR
);
488 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS
))
489 ftrace_plts
[FTRACE_REGS_PLT_IDX
] = emit_plt_entry(FTRACE_REGS_ADDR
);
491 mod
->arch
.ftrace_trampolines
= ftrace_plts
;
495 int module_finalize(const Elf_Ehdr
*hdr
,
496 const Elf_Shdr
*sechdrs
, struct module
*mod
)
498 const Elf_Shdr
*s
, *se
;
499 const char *secstrs
= (void *)hdr
+ sechdrs
[hdr
->e_shstrndx
].sh_offset
;
501 for (s
= sechdrs
, se
= sechdrs
+ hdr
->e_shnum
; s
< se
; s
++) {
502 if (!strcmp(".altinstructions", secstrs
+ s
->sh_name
))
503 apply_alternatives((void *)s
->sh_addr
, (void *)s
->sh_addr
+ s
->sh_size
);
504 if (!strcmp(".ftrace_trampoline", secstrs
+ s
->sh_name
))
505 module_init_ftrace_plt(hdr
, s
, mod
);