Skip to content

bpf: Introduce global percpu data #12797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kernel-patches-daemon-bpf wants to merge 9 commits into
base: bpf-next_base
Choose a base branch
from
Open

bpf: Introduce global percpu data #12797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
11971c0
bpf: Drop duplicate blank lines in verifier
Asphaltt Jul 13, 2026
12e0360
bpf: Introduce global percpu data
Asphaltt Jul 13, 2026
8153690
libbpf: Probe percpu data feature
Asphaltt Jul 13, 2026
66f0c99
libbpf: Add support for global percpu data
Asphaltt Jul 13, 2026
7c1f43b
bpftool: Generate skeleton for global percpu data
Asphaltt Jul 13, 2026
ebf7d03
selftests/bpf: Add tests to verify global percpu data
Asphaltt Jul 13, 2026
6257003
selftests/bpf: Test direct reading/writing read-only percpu_array map
Asphaltt Jul 13, 2026
c12e9cd
selftests/bpf: Test verifier log for global percpu data
Asphaltt Jul 13, 2026
2931194
selftests/bpf: Verify bpf_iter for global percpu data
Asphaltt Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions kernel/bpf/arraymap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,37 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
return this_cpu_ptr(array->pptrs[index & array->index_mask]);
}

static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);

if (map->max_entries != 1)
return -EOPNOTSUPP;
if (off >= map->value_size)
return -EINVAL;
if (!bpf_jit_supports_percpu_insn())
return -EOPNOTSUPP;

*imm = (u64)(__force unsigned long) array->pptrs[0];
return 0;
}

static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);
u64 base = (u64)(__force unsigned long) array->pptrs[0];

if (map->max_entries != 1)
return -EOPNOTSUPP;
if (imm < base || imm >= base + array->elem_size)
return -ENOENT;
if (!bpf_jit_supports_percpu_insn())
return -EOPNOTSUPP;

*off = imm - base;
return 0;
}

/* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */
static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
{
Expand Down Expand Up @@ -551,9 +582,10 @@ static int array_map_check_btf(struct bpf_map *map,
const struct btf_type *key_type,
const struct btf_type *value_type)
{
/* One exception for keyless BTF: .bss/.data/.rodata map */
/* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */
if (btf_type_is_void(key_type)) {
if (map->map_type != BPF_MAP_TYPE_ARRAY ||
if ((map->map_type != BPF_MAP_TYPE_ARRAY &&
map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) ||
map->max_entries != 1)
return -EINVAL;

Expand Down Expand Up @@ -832,6 +864,8 @@ const struct bpf_map_ops percpu_array_map_ops = {
.map_get_next_key = bpf_array_get_next_key,
.map_lookup_elem = percpu_array_map_lookup_elem,
.map_gen_lookup = percpu_array_map_gen_lookup,
.map_direct_value_addr = percpu_array_map_direct_value_addr,
.map_direct_value_meta = percpu_array_map_direct_value_meta,
.map_update_elem = array_map_update_elem,
.map_delete_elem = array_map_delete_elem,
.map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem,
Expand Down
1 change: 0 additions & 1 deletion kernel/bpf/const_fold.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
u64 val = 0;

if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
map->map_type == BPF_MAP_TYPE_INSN_ARRAY ||
off < 0 || off + size > map->value_size ||
bpf_map_direct_read(map, off, size, &val, is_ldsx)) {
*dst = unknown;
Expand Down
38 changes: 38 additions & 0 deletions kernel/bpf/fixups.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1818,6 +1818,44 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
goto next_insn;
}

if (env->prog->jit_requested &&
bpf_jit_supports_percpu_insn() &&
insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
(insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) {
struct bpf_map *map;

aux = &env->insn_aux_data[i + delta];
map = env->used_maps[aux->map_index];
if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
goto next_insn;

prog->jit_required = true;

/*
* We are *skipping* first half of ld_imm64 insn
* with 'i++;', patching over second half of it
* with that same half + mov64_percpu_reg insn.
* All because bpf_patch_insn_data() can only
* replace one 8-byte insn, which does not work
* well for ld_imm64 insn.
*/

insn_buf[0] = insn[1];
insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg);
cnt = 2;

i++;
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
if (!new_prog)
return -ENOMEM;

delta += cnt - 1;
env->prog = prog = new_prog;
insn = new_prog->insnsi + i + delta;
goto next_insn;
}

if (insn->code != (BPF_JMP | BPF_CALL))
goto next_insn;
if (insn->src_reg == BPF_PSEUDO_CALL)
Expand Down
29 changes: 15 additions & 14 deletions kernel/bpf/verifier.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,6 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg,
enum bpf_dynptr_type type,
bool first_slot, int id, int parent_id);


static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
struct bpf_reg_state *sreg1,
struct bpf_reg_state *sreg2,
Expand Down Expand Up @@ -1692,7 +1691,6 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta
return true;
}


void bpf_free_backedges(struct bpf_scc_visit *visit)
{
struct bpf_scc_backedge *backedge, *next;
Expand Down Expand Up @@ -2312,7 +2310,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
return &elem->st;
}


static int cmp_subprogs(const void *a, const void *b)
{
return ((struct bpf_subprog_info *)a)->start -
Expand Down Expand Up @@ -4065,7 +4062,6 @@ static int check_stack_read(struct bpf_verifier_env *env,
return err;
}


/* check_stack_write dispatches to check_stack_write_fixed_off or
* check_stack_write_var_off.
*
Expand Down Expand Up @@ -4861,7 +4857,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
valid = false;
}


if (valid) {
env->insn_aux_data[insn_idx].ctx_field_size =
info.ctx_field_size;
Expand Down Expand Up @@ -5641,6 +5636,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
u64 addr;
int err;

if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
return -EINVAL;
err = map->ops->map_direct_value_addr(map, &addr, off);
if (err)
return err;
Expand Down Expand Up @@ -6200,6 +6197,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
if (tnum_is_const(reg->var_off) &&
bpf_map_is_rdonly(map) &&
map->ops->map_direct_value_addr &&
map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
map->map_type != BPF_MAP_TYPE_INSN_ARRAY) {
int map_off = off + reg->var_off.value;
u64 val = 0;
Expand Down Expand Up @@ -6702,7 +6700,6 @@ static int check_stack_range_initialized(
if (err)
return err;


if (tnum_is_const(reg->var_off)) {
min_off = max_off = reg->var_off.value + off;
} else {
Expand Down Expand Up @@ -7416,7 +7413,6 @@ static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
return meta->kfunc_flags & KF_ITER_NEW;
}


static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ITER_DESTROY;
Expand Down Expand Up @@ -8189,6 +8185,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
return -EACCES;
}

if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
verbose(env, "%s points to percpu_array map which cannot be used as const string\n",
reg_arg_name(env, argno));
return -EACCES;
}

if (!bpf_map_is_rdonly(map)) {
verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno));
return -EACCES;
Expand Down Expand Up @@ -11562,7 +11564,6 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *
return 0;
}


static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
struct btf_record *rec = reg_btf_record(reg);
Expand Down Expand Up @@ -16465,7 +16466,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
return 0;
}


static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range)
{
enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
Expand Down Expand Up @@ -18332,6 +18332,12 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
return -EINVAL;
}

if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY &&
!env->prog->jit_requested) {
verbose(env, "JIT is required to use global percpu data\n");
return -EOPNOTSUPP;
}

err = map->ops->map_direct_value_addr(map, &addr, off);
if (err) {
verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
Expand Down Expand Up @@ -18407,8 +18413,6 @@ static void release_insn_arrays(struct bpf_verifier_env *env)
bpf_insn_array_release(env->insn_array_maps[i]);
}



/* The verifier does more data flow analysis than llvm and will not
* explore branches that are dead at run time. Malicious programs can
* have dead code too. Therefore replace all dead at-run-time code
Expand Down Expand Up @@ -18436,8 +18440,6 @@ static void sanitize_dead_code(struct bpf_verifier_env *env)
}
}



static void free_states(struct bpf_verifier_env *env)
{
struct bpf_verifier_state_list *sl;
Expand Down Expand Up @@ -18719,7 +18721,6 @@ static int do_check_main(struct bpf_verifier_env *env)
return ret;
}


static void print_verification_stats(struct bpf_verifier_env *env)
{
/* Skip over hidden subprogs which are not verified. */
Expand Down
Loading
Loading