diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4b..705a9ab85f3a9 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -233,6 +233,7 @@ config X86 select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT + select HAVE_EBPF_JIT_KASAN if X86_64 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de7515ea1beae..9dbe3a52444e3 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -21,6 +21,17 @@ #include #include +#if IS_ENABLED(CONFIG_BPF_JIT_KASAN) +void __asan_load1(void *p); +void __asan_store1(void *p); +void __asan_load2(void *p); +void __asan_store2(void *p); +void __asan_load4(void *p); +void __asan_store4(void *p); +void __asan_load8(void *p); +void __asan_store8(void *p); +#endif + static bool all_callee_regs_used[4] = {true, true, true, true}; static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) @@ -1110,6 +1121,90 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, + u32 addr_reg, struct bpf_insn *insn, u8 *ip, + bool is_write, bool accesses_stack_only) +{ +#ifdef CONFIG_BPF_JIT_KASAN + u32 bpf_size = BPF_SIZE(insn->code); + s32 off = insn->off; + u8 *prog = *pprog; + void *kasan_func; + + if (!env) + return 0; + + if (accesses_stack_only) + return 0; + + /* Derive KASAN check function from access type and size */ + switch (bpf_size) { + case BPF_B: + kasan_func = is_write ? __asan_store1 : __asan_load1; + break; + case BPF_H: + kasan_func = is_write ? __asan_store2 : __asan_load2; + break; + case BPF_W: + kasan_func = is_write ? __asan_store4 : __asan_load4; + break; + case BPF_DW: + kasan_func = is_write ? __asan_store8 : __asan_load8; + break; + default: + return -EINVAL; + } + + /* Save rax */ + EMIT1(0x50); + /* Save rcx */ + EMIT1(0x51); + /* Save rdx */ + EMIT1(0x52); + /* Save rsi */ + EMIT1(0x56); + /* Save rdi */ + EMIT1(0x57); + /* Save r8 */ + EMIT2(0x41, 0x50); + /* Save r9 */ + EMIT2(0x41, 0x51); + + /* mov rdi, addr_reg */ + EMIT_mov(BPF_REG_1, addr_reg); + + /* add rdi, off (if offset is non-zero) */ + if (off) { + if (is_imm8(off)) { + /* add rdi, imm8 */ + EMIT4(0x48, 0x83, 0xC7, (u8)off); + } else { + /* add rdi, imm32 */ + EMIT3_off32(0x48, 0x81, 0xC7, off); + } + } + + /* Adjust ip to account for the instrumentation generated so far */ + ip += (prog - *pprog); + /* We emit a call, so update call depth counting */ + ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip); + /* call kasan_func */ + if (emit_call(&prog, kasan_func, ip)) + return -ERANGE; + + EMIT2(0x41, 0x59); + EMIT2(0x41, 0x58); + EMIT1(0x5F); + EMIT1(0x5E); + EMIT1(0x5A); + EMIT1(0x59); + EMIT1(0x58); + + *pprog = prog; +#endif /* CONFIG_BPF_JIT_KASAN */ + return 0; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1315,6 +1410,64 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int *pprog = prog; } +static void emit_st(u8 **pprog, struct bpf_insn *insn, int dst_reg, + s32 outgoing_arg_base, u16 outgoing_rsp) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + s32 insn_off; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { + /* Arg 6: store immediate in r9 register */ + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + (u32)imm32); + *pprog = prog; + return; + } + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + insn_off = insn->off; + if (dst_reg == BPF_REG_PARAMS) { + /* + * Args 7+: reverse BPF negative offsets to + * x86 positive rsp offsets. + * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... + */ + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; + dst_reg = BPF_REG_FP; + } + if (is_imm8(insn_off)) + EMIT2(add_1reg(0x40, dst_reg), insn_off); + else + EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -1423,17 +1576,31 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) +static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog, + struct bpf_insn *insn, u8 *ip, u32 dst_reg, + u32 src_reg, bool accesses_stack_only) { + u32 atomic_op = insn->imm; + int err; + switch (atomic_op) { case BPF_LOAD_ACQ: + err = emit_kasan_check(env, pprog, src_reg, insn, ip, false, + accesses_stack_only); + if (err) + return err; /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); + emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; case BPF_STORE_REL: + err = emit_kasan_check(env, pprog, dst_reg, insn, ip, true, + accesses_stack_only); + if (err) + return err; /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); + emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; default: pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", @@ -1811,10 +1978,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * const s32 imm32 = insn->imm; u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; + bool accesses_stack_only; u8 b2 = 0, b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; + int insn_idx; u8 jmp_cond; u8 *func; int nops; @@ -1831,6 +2000,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_ENDBR(); ip = image + addrs[i - 1] + (prog - temp); + insn_idx = i - 1 + bpf_prog->aux->subprog_start; + accesses_stack_only = + env ? !env->insn_aux_data[insn_idx].non_stack_access : + false; switch (insn->code) { /* ALU */ @@ -2207,49 +2380,17 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: - if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { - /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); - break; - } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; - if (dst_reg == BPF_REG_PARAMS) { - /* - * Args 7+: reverse BPF negative offsets to - * x86 positive rsp offsets. - * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... - */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; - dst_reg = BPF_REG_FP; - } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + err = emit_kasan_check(env, &prog, dst_reg, insn, ip, + true, accesses_stack_only); + if (err) + return err; - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, outgoing_arg_base, + outgoing_rsp); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ @@ -2267,6 +2408,10 @@ st: insn_off = insn->off; insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; dst_reg = BPF_REG_FP; } + err = emit_kasan_check(env, &prog, dst_reg, insn, ip, + true, accesses_stack_only); + if (err) + return err; emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); break; @@ -2428,6 +2573,12 @@ st: insn_off = insn->off; /* populate jmp_offset for JAE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; + } else { + err = emit_kasan_check(env, &prog, src_reg, + insn, ip, false, + accesses_stack_only); + if (err) + return err; } if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX || BPF_MODE(insn->code) == BPF_MEMSX) @@ -2489,15 +2640,16 @@ st: insn_off = insn->off; } fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; - u32 real_src_reg = src_reg; - u32 real_dst_reg = dst_reg; - u8 *branch_target; - + case BPF_STX | BPF_ATOMIC | BPF_DW: { + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + u32 real_src_reg = src_reg; + u32 real_dst_reg = dst_reg; + u8 *branch_target; + bool is_atomic_fetch = + (insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH)); + if (is_atomic_fetch) { /* * Can't be implemented with a single x86 insn. * Need to do a CMPXCHG loop. @@ -2510,7 +2662,17 @@ st: insn_off = insn->off; if (dst_reg == BPF_REG_0) real_dst_reg = BPF_REG_AX; + ip += 3; + } + if (!bpf_atomic_is_load_store(insn)) { + err = emit_kasan_check(env, &prog, real_dst_reg, + insn, ip, true, + accesses_stack_only); + if (err) + return err; branch_target = prog; + } + if (is_atomic_fetch) { /* Load old value */ emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, real_dst_reg, insn->off); @@ -2542,15 +2704,16 @@ st: insn_off = insn->off; } if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + err = emit_atomic_ld_st(env, &prog, insn, ip, + dst_reg, src_reg, + accesses_stack_only); else err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg, insn->off, BPF_SIZE(insn->code)); if (err) return err; break; - + } case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: if (!bpf_atomic_is_load_store(insn)) { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 317e99b9acc0a..abe8459cc7012 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -723,6 +723,8 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* instruction can access non-stack memory */ + bool non_stack_access; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 14acb2455746f..1ebcd247ef4e3 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1210,13 +1210,17 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, #ifdef CONFIG_BPF_SYSCALL struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len); + const struct bpf_insn *patch, u32 len, + s32 insn_off_in_patch); struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, struct bpf_insn_aux_data *orig_insn_aux); #else -static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, + u32 off, + const struct bpf_insn *patch, + u32 len, + s32 insn_off_in_patch) { return ERR_PTR(-ENOTSUPP); } diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f09..e1f3850d2f5a0 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -101,4 +105,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y if KASAN + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 47fe047ad30b8..cff0035f401a9 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1599,7 +1599,7 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp continue; if (env) - tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); + tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten, rewritten - 1); else tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d3be972714b22..0eda5512baece 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -152,12 +152,25 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +static bool is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + s32 insn_off_in_patch) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -171,8 +184,14 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = insn_has_def32(insn + off + cnt - 1); - if (cnt == 1) + if (cnt == 1) { + /* A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = is_mem_insn(insn + off); return; + } prog_len = new_prog->len; memmove(data + off + cnt - 1, data + off, @@ -182,7 +201,19 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = insn_has_def32(insn + i); + if (i == off + insn_off_in_patch) { + data[i].non_stack_access = data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX + */ + if (is_mem_insn(insn + off + cnt - 1) && insn_off_in_patch != cnt - 1) + data[off + cnt - 1].non_stack_access = true; /* * The indirect_target flag of the original instruction was moved to the last of the @@ -245,7 +276,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) } struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) + const struct bpf_insn *patch, u32 len, + s32 insn_off_in_patch) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; @@ -269,7 +301,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, insn_off_in_patch); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); @@ -668,7 +700,7 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, patch = zext_patch; patch_len = 2; apply_patch_buffer: - new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); + new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len, 0); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -713,7 +745,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt, 1); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -736,7 +768,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt, + cnt - 1); if (!new_prog) return -ENOMEM; @@ -768,7 +801,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) *patch++ = BPF_ST_NOSPEC(); *patch++ = *insn; cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -841,7 +875,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) *patch++ = *insn; *patch++ = BPF_ST_NOSPEC(); cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, 0); if (!new_prog) return -ENOMEM; @@ -971,7 +1006,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) size * 8, 0); patch_insn_buf: - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = + bpf_patch_insn_data(env, i + delta, insn_buf, cnt, -1); if (!new_prog) return -ENOMEM; @@ -1463,7 +1499,7 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat * ones for the hidden subprog. Hence all of the adjustment operations * in bpf_patch_insn_data are no-ops. */ - prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); + prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len, 0); if (!prog) return -ENOMEM; env->prog = prog; @@ -1548,7 +1584,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, 0); if (!new_prog) return -ENOMEM; @@ -1568,6 +1605,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) bool is_sdiv = isdiv && insn->off == 1; bool is_smod = !isdiv && insn->off == 1; struct bpf_insn *patch = insn_buf; + s32 insn_off_in_patch; if (is_sdiv) { /* [R,W]x sdiv 0 -> 0 @@ -1594,6 +1632,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn; cnt = patch - insn_buf; + insn_off_in_patch = cnt - 1; } else if (is_smod) { /* [R,W]x mod 0 -> [R,W]x */ /* [R,W]x mod -1 -> 0 */ @@ -1610,6 +1649,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_MOV32_IMM(insn->dst_reg, 0); *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn; + insn_off_in_patch = patch - insn_buf - 1; if (!is64) { *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); @@ -1625,12 +1665,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = *insn; cnt = patch - insn_buf; + insn_off_in_patch = cnt - 1; } else { /* [R,W]x mod 0 -> [R,W]x */ *patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | BPF_JEQ | BPF_K, insn->src_reg, 0, 1 + (is64 ? 0 : 1), 0); *patch++ = *insn; + insn_off_in_patch = patch - insn_buf - 1; if (!is64) { *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); @@ -1639,7 +1681,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = patch - insn_buf; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, insn_off_in_patch); if (!new_prog) return -ENOMEM; @@ -1655,6 +1698,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { struct bpf_insn *patch = insn_buf; u64 uaddress_limit = bpf_arch_uaddress_limit(); + s32 insn_off_in_patch; if (!uaddress_limit) goto next_insn; @@ -1665,11 +1709,13 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); *patch++ = *insn; + insn_off_in_patch = patch - insn_buf - 1; *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, insn_off_in_patch); if (!new_prog) return -ENOMEM; @@ -1689,7 +1735,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, 0); if (!new_prog) return -ENOMEM; @@ -1742,7 +1789,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -1787,7 +1835,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); cnt = 7; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, 3); if (!new_prog) return -ENOMEM; @@ -1808,7 +1857,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); cnt = 4; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, 2); if (!new_prog) return -ENOMEM; @@ -1829,7 +1879,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (cnt == 0) goto next_insn; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -1916,7 +1967,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) map)->index_mask); insn_buf[2] = *insn; cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -1949,7 +2001,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[2] = *insn; cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -1968,7 +2021,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[1] = *insn; cnt = 2; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, cnt - 1); if (!new_prog) return -ENOMEM; @@ -2008,8 +2062,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } - new_prog = bpf_patch_insn_data(env, i + delta, - insn_buf, cnt); + new_prog = bpf_patch_insn_data( + env, i + delta, insn_buf, cnt, -1); if (!new_prog) return -ENOMEM; @@ -2092,7 +2146,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 3; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, - cnt); + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2120,7 +2174,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); cnt = 1; #endif - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2138,7 +2193,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2172,7 +2228,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_JMP_A(1); insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2204,7 +2261,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 1; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2230,7 +2288,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 2; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2246,7 +2305,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Load IP address from ctx - 16 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + 1, -1); if (!new_prog) return -ENOMEM; @@ -2301,7 +2361,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); cnt = 11; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2319,7 +2380,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); cnt = 2; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, + cnt, -1); if (!new_prog) return -ENOMEM; @@ -2387,7 +2449,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Copy first actual insn to preserve it */ insn_buf[cnt++] = env->prog->insnsi[subprog_start]; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, cnt - 1); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; @@ -2486,7 +2548,7 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); *total_cnt = cnt; - new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); + new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt, -1); if (!new_prog) return new_prog; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 03e2202cca13d..658c4b1084cde 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3207,6 +3207,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + #define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) @@ -6412,6 +6417,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && reg->type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 0222a9a5d0761..815f3e04540ff 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -227,42 +227,6 @@ static void check_incorrect_index(void) check_mid_insn_index(); } -static int set_bpf_jit_harden(char *level) -{ - char old_level; - int err = -1; - int fd = -1; - - fd = open("/proc/sys/net/core/bpf_jit_harden", O_RDWR | O_NONBLOCK); - if (fd < 0) { - ASSERT_FAIL("open .../bpf_jit_harden returned %d (errno=%d)", fd, errno); - return -1; - } - - err = read(fd, &old_level, 1); - if (err != 1) { - ASSERT_FAIL("read from .../bpf_jit_harden returned %d (errno=%d)", err, errno); - err = -1; - goto end; - } - - lseek(fd, 0, SEEK_SET); - - err = write(fd, level, 1); - if (err != 1) { - ASSERT_FAIL("write to .../bpf_jit_harden returned %d (errno=%d)", err, errno); - err = -1; - goto end; - } - - err = 0; - *level = old_level; -end: - if (fd >= 0) - close(fd); - return err; -} - static void check_blindness(void) { struct bpf_insn insns[] = { @@ -272,7 +236,7 @@ static void check_blindness(void) BPF_MOV64_IMM(BPF_REG_0, 1), BPF_EXIT_INSN(), }; - int prog_fd = -1, map_fd; + int prog_fd = -1, map_fd, ret; struct bpf_insn_array_value val = {}; char bpf_jit_harden = '@'; /* non-exizsting value */ int i; @@ -291,7 +255,8 @@ static void check_blindness(void) goto cleanup; bpf_jit_harden = '2'; - if (set_bpf_jit_harden(&bpf_jit_harden)) { + ret = set_bpf_jit_harden(&bpf_jit_harden); + if (!ASSERT_OK(ret, "set bpf_jit_harden")) { bpf_jit_harden = '@'; /* open, read or write failed => no write was done */ goto cleanup; } @@ -313,7 +278,8 @@ static void check_blindness(void) cleanup: /* restore the old one */ if (bpf_jit_harden != '@') - set_bpf_jit_harden(&bpf_jit_harden); + ASSERT_OK(set_bpf_jit_harden(&bpf_jit_harden), + "restore hardening configuration"); close(prog_fd); close(map_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c new file mode 100644 index 0000000000000..2bc545e962595 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +/* + * Tests validating that KASAN reports are properly instrumented and + * generated on a wide variety of instructions. The running kernel needs + * kasan_multi_shot to run multiple kasan-generating subtests at once + */ +#include +#include +#include +#include +#include +#include +#include +#include "kasan.skel.h" +#include "kasan_harden.skel.h" + +#define SUBTEST_NAME_MAX_LEN 128 +#define PROG_NAME_MAX_LEN 128 + +#define MAX_LOG_SIZE (8 * 1024) +#define READ_CHUNK_SIZE 256 + +#define KASAN_PATTERN_SLAB_UAF "BUG: KASAN: slab-use-after-free " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_REPORT "%s of size %d at addr" + +static char klog_buffer[MAX_LOG_SIZE]; +static char record[MAX_LOG_SIZE]; + +struct test_spec { + char *prog_type; + bool is_write; + bool only_32_or_64; + bool needs_load_acq_store_rel; + bool skip_multi_size_testing; + bool skip_on_stack_testing; + int run_size; + bool expect_no_report; + bool rnd_hi32; +}; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct test_ctx { + __u8 prog_tag[BPF_TAG_SIZE]; + struct bpf_object *obj; + int *access_size; + bool skip_load_acq_store_rel; + struct bpf_program *prog; + char prog_name[SUBTEST_NAME_MAX_LEN]; + int klog_fd; +}; + +static int open_kernel_logs(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + + return fd; +} + +static void skip_kernel_logs(int fd) +{ + lseek(fd, 0, SEEK_END); +} + +static int read_kernel_logs(int fd, char *buf, size_t max_len) +{ + size_t total = 0; + ssize_t n; + + buf[0] = '\0'; + while (1) { + char *msg, *eol; + size_t len; + + n = read(fd, record, sizeof(record) - 1); + if (n < 0) { + if (errno == EAGAIN) + break; + return n; + } + record[n] = '\0'; + + /* + * Each kmsg record starts with some metadata, separated + * from the actual content by a semi-colon + */ + msg = strchr(record, ';'); + if (!msg) + continue; + msg++; + eol = strchr(msg, '\n'); + if (eol) + *eol = '\0'; + + len = strlen(msg); + if (total + len + 2 > max_len) + break; + memcpy(buf + total, msg, len); + total += len; + buf[total++] = '\n'; + buf[total] = '\0'; + } + + return total; +} + +static int check_kasan_report_in_kernel_logs(char *buf, struct test_ctx *ctx, + bool is_write, int size) +{ + char access_log[READ_CHUNK_SIZE]; + char *kasan_report_start; + int nsize; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_SLAB_UAF, + ctx->prog_tag[0], ctx->prog_tag[1], ctx->prog_tag[2], + ctx->prog_tag[3], ctx->prog_tag[4], ctx->prog_tag[5], + ctx->prog_tag[6], ctx->prog_tag[7], ctx->prog_name); + if (!ASSERT_GE(nsize, 0, "format kasan access header line")) + return nsize; + /* + * Searched kasan report is valid if + * - it contains the expected kasan pattern + * - the description of the faulty access is found somewhere + * after the header (not necessarily on the very next line, + * because other kernel messages may interleave) + * - faulty access properties match the tested type and size + */ + kasan_report_start = strstr(buf, access_log); + + if (!kasan_report_start) + return 1; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_REPORT, + is_write ? "Write" : "Read", size); + if (!ASSERT_GE(nsize, 0, "format kasan access report line")) + return nsize; + + if (!strstr(kasan_report_start, access_log)) + return 1; + + return 0; +} + +static void exec_subtest(struct test_ctx *ctx, struct test_spec *test, + int access_size, bool on_stack) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_info info; + uint8_t buf[ETH_HLEN] = {0}; + int ret, prog_fd; + __u32 info_len; + + ctx->prog = bpf_object__find_program_by_name(ctx->obj, + ctx->prog_name); + if (!ASSERT_OK_PTR(ctx->prog, "find test prog")) + return; + + info_len = sizeof(info); + memset(&info, 0, info_len); + prog_fd = bpf_program__fd(ctx->prog); + if (!ASSERT_OK_FD(prog_fd, "get prog fd")) + return; + ret = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (!ASSERT_OK(ret, "fetch loaded program info")) + return; + memcpy(ctx->prog_tag, info.tag, BPF_TAG_SIZE); + + skip_kernel_logs(ctx->klog_fd); + + topts.sz = sizeof(struct bpf_test_run_opts); + topts.data_size_in = ETH_HLEN; + topts.data_in = buf; + if (ctx->access_size) + *ctx->access_size = access_size; + ret = bpf_prog_test_run_opts(bpf_program__fd(ctx->prog), + &topts); + if (!ASSERT_OK(ret, "run prog")) + return; + + ret = read_kernel_logs(ctx->klog_fd, klog_buffer, MAX_LOG_SIZE); + if (!ASSERT_GE(ret, 0, "read kernel logs")) + return; + + ret = check_kasan_report_in_kernel_logs(klog_buffer, ctx, + test->is_write, access_size); + if (on_stack || test->expect_no_report) + ASSERT_NEQ(ret, 0, "no report should be generated"); + else + ASSERT_OK(ret, "report should be generated"); +} + +static void run_subtest_with_size_and_location(struct test_ctx *ctx, + struct test_spec *test, + int access_size, + bool on_stack) +{ + char subtest_name[SUBTEST_NAME_MAX_LEN]; + + if (test->skip_multi_size_testing) { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s", + test->prog_type); + strncpy(ctx->prog_name, test->prog_type, PROG_NAME_MAX_LEN); + } else { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s_%d_%s", + test->prog_type, access_size, + on_stack ? "on_stack" : "not_on_stack"); + snprintf(ctx->prog_name, PROG_NAME_MAX_LEN, "%s_%s", + test->prog_type, + on_stack ? "on_stack" : "not_on_stack"); + } + + if (!test__start_subtest(subtest_name)) + return; + + if (test->needs_load_acq_store_rel && ctx->skip_load_acq_store_rel) { + test__skip(); + return; + } + + exec_subtest(ctx, test, access_size, on_stack); +} + +static void run_subtest_with_size(struct test_ctx *ctx, struct test_spec *test, + int size) +{ + run_subtest_with_size_and_location(ctx, test, size, false); + if (!test->skip_on_stack_testing) + run_subtest_with_size_and_location(ctx, test, size, true); +} + +static void run_subtest(struct test_ctx *ctx, struct test_spec *test) +{ + if (test->skip_multi_size_testing) { + run_subtest_with_size(ctx, test, test->run_size); + return; + } + + if (!test->only_32_or_64) { + run_subtest_with_size(ctx, test, 1); + run_subtest_with_size(ctx, test, 2); + } + run_subtest_with_size(ctx, test, 4); + run_subtest_with_size(ctx, test, 8); +} + +static void run_blinding_subtest(void) +{ + struct test_spec blinding_spec = { + .prog_type = "st_blinded", + .is_write = true, + }; + char bpf_jit_harden = '2'; + struct kasan_harden *skel; + struct test_ctx *ctx; + + if (!test__start_subtest("st_blinded")) + return; + + ctx = calloc(1, sizeof(*ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc blinding ctx")) + return; + ctx->klog_fd = -1; + + if (set_bpf_jit_harden(&bpf_jit_harden)) + goto free_ctx; + + skel = kasan_harden__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load blinded prog")) + goto restore; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + ctx->obj = skel->obj; + strncpy(ctx->prog_name, "st_blinded", PROG_NAME_MAX_LEN); + + exec_subtest(ctx, &blinding_spec, 1, false); + +destroy: + close(ctx->klog_fd); + kasan_harden__destroy(skel); +restore: + set_bpf_jit_harden(&bpf_jit_harden); +free_ctx: + free(ctx); +} + +static struct test_spec tests[] = { + { + .prog_type = "st", + .is_write = true + }, + { + .prog_type = "stx", + .is_write = true + }, + { + .prog_type = "ldx", + .is_write = false + }, + { + .prog_type = "simple_atomic", + .is_write = true, + .only_32_or_64 = true + }, + { + .prog_type = "load_acquire", + .is_write = false, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "store_release", + .is_write = true, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "ldx_patched", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 4, + .rnd_hi32 = true + }, + { + .prog_type = "ldx_patched_on_stack", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 4, + .expect_no_report = true, + .rnd_hi32 = true + }, + { + .prog_type = "verifier_paths_stack_and_non_stack", + .is_write = true, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 1 + } +}; + +void test_kasan(void) +{ + struct kasan_write_val val; + struct test_spec *test; + struct test_ctx *ctx; + struct kasan *skel; + __u32 key = 0; + int i, ret; + + ctx = calloc(1, sizeof(struct test_ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc test ctx")) + return; + + if (!is_jit_enabled() || !get_kasan_jit_enabled() || + !get_kasan_multi_shot_enabled()) { + test__skip(); + goto end; + } + + skel = kasan__open(); + if (!ASSERT_OK_PTR(skel, "open prog")) + goto end; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + struct bpf_program *prog; + + if (!tests[i].rnd_hi32) + continue; + + prog = bpf_object__find_program_by_name(skel->obj, + tests[i].prog_type); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 prog")) + goto destroy; + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + } + + if (!ASSERT_OK(kasan__load(skel), "load prog")) + goto destroy; + + ctx->obj = skel->obj; + ctx->access_size = &skel->bss->access_size; + ctx->skip_load_acq_store_rel = skel->data->skip_load_acq_store_rel_tests; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + /* Fill map with recognizable values */ + ret = bpf_map__lookup_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "get map")) + goto close; + val.data_1 = 0xAA; + val.data_2 = 0xBBBB; + val.data_4 = 0xCCCCCCCC; + val.data_8 = 0xDDDDDDDDDDDDDDDD; + ret = bpf_map__update_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "set map")) + goto close; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test = &tests[i]; + run_subtest(ctx, test); + } + + /* + * Blinding subtest is handled differently as it needs the + * corresponding program to be loaded with bpf_jit_harden raised + */ + run_blinding_subtest(); + +close: + close(ctx->klog_fd); +destroy: + kasan__destroy(skel); +end: + free(ctx); +} diff --git a/tools/testing/selftests/bpf/progs/kasan.c b/tools/testing/selftests/bpf/progs/kasan.c new file mode 100644 index 0000000000000..82ddce3059aee --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include +#include "bpf_misc.h" +#include + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +int access_size; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_write_val); +} test_map SEC(".maps"); + +SEC("tcx/ingress") +int st_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + val.data_1 = 0xAA; + break; + case 2: + val.data_2 = 0xAA; + break; + case 4: + val.data_4 = 0xAA; + break; + case 8: + val.data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int st_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + val->data_1 = 0xAA; + break; + case 2: + val->data_2 = 0xAA; + break; + case 4: + val->data_4 = 0xAA; + break; + case 8: + val->data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + val.data_1 = access_size; + break; + case 2: + val.data_2 = access_size; + break; + case 4: + val.data_4 = access_size; + break; + case 8: + val.data_8 = access_size; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + val->data_1 = access_size; + break; + case 2: + val->data_2 = access_size; + break; + case 4: + val->data_4 = access_size; + break; + case 8: + val->data_8 = access_size; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __sink(val.data_1); + break; + case 2: + __sink(val.data_2); + break; + case 4: + __sink(val.data_4); + break; + case 8: + __sink(val.data_8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __sink(val->data_1); + break; + case 2: + __sink(val->data_2); + break; + case 4: + __sink(val->data_4); + break; + case 8: + __sink(val->data_8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_patched(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + __sink(val->data_4); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + __sink(val.data_4); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val.data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val.data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val->data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val->data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +bool skip_load_acq_store_rel_tests SEC(".data") = 0; + +SEC("tcx/ingress") +int load_acquire_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val.data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val.data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val.data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val.data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int load_acquire_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val->data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val->data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val->data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val->data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val.data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val.data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val.data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val.data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_write_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val->data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val->data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val->data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val->data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + return 0; +} +#else +bool skip_load_acq_store_rel_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int verifier_paths_stack_and_non_stack(struct __sk_buff *skb) +{ + struct kasan_write_val stack_val = {}; + struct kasan_write_val *val; + void *ptr; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + if (access_size) + ptr = val; + else + ptr = &stack_val; + + bpf_kfunc_kasan_poison(val, sizeof(*val)); + *(__u8 *)ptr = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(*val)); + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kasan_harden.c b/tools/testing/selftests/bpf/progs/kasan_harden.c new file mode 100644 index 0000000000000..808d7da644216 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan_harden.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_write_val); +} test_map SEC(".maps"); + +SEC("tcx/ingress") +int st_blinded(struct __sk_buff *skb) +{ + struct kasan_write_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_write_val)); + val->data_1 = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_write_val)); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 30f1cd23093cb..5d6a834f7d757 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -271,6 +271,26 @@ __bpf_kfunc void bpf_kfunc_put_default_trusted_ptr_test(struct prog_test_member */ } +#ifdef CONFIG_BPF_JIT_KASAN + +extern void kasan_poison(const void *addr, size_t size, u8 value, bool init); + +#define KASAN_SLAB_FREE 0xFB + +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, KASAN_SLAB_FREE, false); +} + +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, 0x00, false); +} +#else +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) { } +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) { } +#endif + __bpf_kfunc struct bpf_testmod_ctx * bpf_testmod_ctx_create(int *err) { @@ -740,6 +760,8 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); +BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) +BTF_ID_FLAGS(func, bpf_kfunc_kasan_unpoison) BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) BTF_ID_LIST(bpf_testmod_dtor_ids) diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index c970e7793dfcb..737f668b35e23 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -519,6 +519,38 @@ bool is_jit_enabled(void) return enabled; } +int set_bpf_jit_harden(char *level) +{ + char old_level; + int err = -1; + int fd = -1; + + fd = open("/proc/sys/net/core/bpf_jit_harden", O_RDWR | O_NONBLOCK); + if (fd < 0) + return -1; + + err = read(fd, &old_level, 1); + if (err != 1) { + err = -1; + goto end; + } + + lseek(fd, 0, SEEK_SET); + + err = write(fd, level, 1); + if (err != 1) { + err = -1; + goto end; + } + + err = 0; + *level = old_level; +end: + if (fd >= 0) + close(fd); + return err; +} + int stack_mprotect(void) { void *buf; diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 2edc6fb7fc521..e00642afe86f4 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -59,6 +59,7 @@ struct bpf_insn; int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt); int testing_prog_flags(void); bool is_jit_enabled(void); +int set_bpf_jit_harden(char *level); int stack_mprotect(void); #endif /* __TESTING_HELPERS_H */ diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index f997d7ec8fd08..95be839378837 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -142,3 +142,13 @@ bool get_unpriv_disabled(void) } return mitigations_off; } + +bool get_kasan_jit_enabled(void) +{ + return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; +} + +bool get_kasan_multi_shot_enabled(void) +{ + return cmdline_contains("kasan_multi_shot"); +} diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index 151f673296652..a7ceb51577cdc 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h @@ -5,3 +5,5 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_kasan_jit_enabled(void); +bool get_kasan_multi_shot_enabled(void);