Skip to content

Commit

Permalink
bpf: Refactor map->off_arr handling
Browse files Browse the repository at this point in the history 
Refactor map->off_arr handling into generic functions that can work on
their own without hardcoding map specific code. The btf_fields_offs
structure is now returned from btf_parse_field_offs, which can be reused
later for types in program BTF.

All functions like copy_map_value, zero_map_value call generic
underlying functions so that they can also be reused later for copying
to values allocated in programs which encode specific fields.

Later, some helper functions will also require access to this
btf_field_offs structure to be able to skip over special fields at
runtime.

Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20221103191013.1236066-9-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
  • Loading branch information
Kumar Kartikeya Dwivedi authored and Alexei Starovoitov committed Nov 4, 2022
1 parent db55911 commit f71b2f6
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 81 deletions.
41 changes: 24 additions & 17 deletions include/linux/bpf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,57 +341,64 @@ static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
}

/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
static inline void __copy_map_value(struct bpf_map *map, void *dst, void *src, bool long_memcpy)
static inline void bpf_obj_memcpy(struct btf_field_offs *foffs,
void *dst, void *src, u32 size,
bool long_memcpy)
{
u32 curr_off = 0;
int i;

if (likely(!map->field_offs)) {
if (likely(!foffs)) {
if (long_memcpy)
bpf_long_memcpy(dst, src, round_up(map->value_size, 8));
bpf_long_memcpy(dst, src, round_up(size, 8));
else
memcpy(dst, src, map->value_size);
memcpy(dst, src, size);
return;
}

for (i = 0; i < map->field_offs->cnt; i++) {
u32 next_off = map->field_offs->field_off[i];
for (i = 0; i < foffs->cnt; i++) {
u32 next_off = foffs->field_off[i];
u32 sz = next_off - curr_off;

memcpy(dst + curr_off, src + curr_off, sz);
curr_off += map->field_offs->field_sz[i];
curr_off += foffs->field_sz[i];
}
memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off);
memcpy(dst + curr_off, src + curr_off, size - curr_off);
}

static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
{
__copy_map_value(map, dst, src, false);
bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, false);
}

static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
{
__copy_map_value(map, dst, src, true);
bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, true);
}

static inline void zero_map_value(struct bpf_map *map, void *dst)
static inline void bpf_obj_memzero(struct btf_field_offs *foffs, void *dst, u32 size)
{
u32 curr_off = 0;
int i;

if (likely(!map->field_offs)) {
memset(dst, 0, map->value_size);
if (likely(!foffs)) {
memset(dst, 0, size);
return;
}

for (i = 0; i < map->field_offs->cnt; i++) {
u32 next_off = map->field_offs->field_off[i];
for (i = 0; i < foffs->cnt; i++) {
u32 next_off = foffs->field_off[i];
u32 sz = next_off - curr_off;

memset(dst + curr_off, 0, sz);
curr_off += map->field_offs->field_sz[i];
curr_off += foffs->field_sz[i];
}
memset(dst + curr_off, 0, map->value_size - curr_off);
memset(dst + curr_off, 0, size - curr_off);
}

static inline void zero_map_value(struct bpf_map *map, void *dst)
{
bpf_obj_memzero(map->field_offs, dst, map->value_size);
}

void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
Expand Down
1 change: 1 addition & 0 deletions include/linux/btf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t);
int btf_find_timer(const struct btf *btf, const struct btf_type *t);
struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t,
u32 field_mask, u32 value_size);
struct btf_field_offs *btf_parse_field_offs(struct btf_record *rec);
bool btf_type_is_void(const struct btf_type *t);
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
Expand Down
55 changes: 55 additions & 0 deletions kernel/bpf/btf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3551,6 +3551,61 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
return ERR_PTR(ret);
}

static int btf_field_offs_cmp(const void *_a, const void *_b, const void *priv)
{
const u32 a = *(const u32 *)_a;
const u32 b = *(const u32 *)_b;

if (a < b)
return -1;
else if (a > b)
return 1;
return 0;
}

static void btf_field_offs_swap(void *_a, void *_b, int size, const void *priv)
{
struct btf_field_offs *foffs = (void *)priv;
u32 *off_base = foffs->field_off;
u32 *a = _a, *b = _b;
u8 *sz_a, *sz_b;

sz_a = foffs->field_sz + (a - off_base);
sz_b = foffs->field_sz + (b - off_base);

swap(*a, *b);
swap(*sz_a, *sz_b);
}

struct btf_field_offs *btf_parse_field_offs(struct btf_record *rec)
{
struct btf_field_offs *foffs;
u32 i, *off;
u8 *sz;

BUILD_BUG_ON(ARRAY_SIZE(foffs->field_off) != ARRAY_SIZE(foffs->field_sz));
if (IS_ERR_OR_NULL(rec) || WARN_ON_ONCE(rec->cnt > sizeof(foffs->field_off)))
return NULL;

foffs = kzalloc(sizeof(*foffs), GFP_KERNEL | __GFP_NOWARN);
if (!foffs)
return ERR_PTR(-ENOMEM);

off = foffs->field_off;
sz = foffs->field_sz;
for (i = 0; i < rec->cnt; i++) {
off[i] = rec->fields[i].offset;
sz[i] = btf_field_type_size(rec->fields[i].type);
}
foffs->cnt = rec->cnt;

if (foffs->cnt == 1)
return foffs;
sort_r(foffs->field_off, foffs->cnt, sizeof(foffs->field_off[0]),
btf_field_offs_cmp, btf_field_offs_swap, foffs);
return foffs;
}

static void __btf_struct_show(const struct btf *btf, const struct btf_type *t,
u32 type_id, void *data, u8 bits_offset,
struct btf_show *show)
Expand Down
73 changes: 9 additions & 64 deletions kernel/bpf/syscall.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -943,66 +943,6 @@ int map_check_no_btf(const struct bpf_map *map,
return -ENOTSUPP;
}

static int map_field_offs_cmp(const void *_a, const void *_b, const void *priv)
{
const u32 a = *(const u32 *)_a;
const u32 b = *(const u32 *)_b;

if (a < b)
return -1;
else if (a > b)
return 1;
return 0;
}

static void map_field_offs_swap(void *_a, void *_b, int size, const void *priv)
{
struct bpf_map *map = (struct bpf_map *)priv;
u32 *off_base = map->field_offs->field_off;
u32 *a = _a, *b = _b;
u8 *sz_a, *sz_b;

sz_a = map->field_offs->field_sz + (a - off_base);
sz_b = map->field_offs->field_sz + (b - off_base);

swap(*a, *b);
swap(*sz_a, *sz_b);
}

static int bpf_map_alloc_off_arr(struct bpf_map *map)
{
bool has_fields = !IS_ERR_OR_NULL(map->record);
struct btf_field_offs *fo;
struct btf_record *rec;
u32 i, *off;
u8 *sz;

if (!has_fields) {
map->field_offs = NULL;
return 0;
}

fo = kzalloc(sizeof(*map->field_offs), GFP_KERNEL | __GFP_NOWARN);
if (!fo)
return -ENOMEM;
map->field_offs = fo;

rec = map->record;
off = fo->field_off;
sz = fo->field_sz;
for (i = 0; i < rec->cnt; i++) {
*off++ = rec->fields[i].offset;
*sz++ = btf_field_type_size(rec->fields[i].type);
}
fo->cnt = rec->cnt;

if (fo->cnt == 1)
return 0;
sort_r(fo->field_off, fo->cnt, sizeof(fo->field_off[0]),
map_field_offs_cmp, map_field_offs_swap, map);
return 0;
}

static int map_check_btf(struct bpf_map *map, const struct btf *btf,
u32 btf_key_id, u32 btf_value_id)
{
Expand Down Expand Up @@ -1097,6 +1037,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
static int map_create(union bpf_attr *attr)
{
int numa_node = bpf_map_attr_numa_node(attr);
struct btf_field_offs *foffs;
struct bpf_map *map;
int f_flags;
int err;
Expand Down Expand Up @@ -1176,13 +1117,17 @@ static int map_create(union bpf_attr *attr)
attr->btf_vmlinux_value_type_id;
}

err = bpf_map_alloc_off_arr(map);
if (err)

foffs = btf_parse_field_offs(map->record);
if (IS_ERR(foffs)) {
err = PTR_ERR(foffs);
goto free_map;
}
map->field_offs = foffs;

err = security_bpf_map_alloc(map);
if (err)
goto free_map_off_arr;
goto free_map_field_offs;

err = bpf_map_alloc_id(map);
if (err)
Expand All @@ -1206,7 +1151,7 @@ static int map_create(union bpf_attr *attr)

free_map_sec:
security_bpf_map_free(map);
free_map_off_arr:
free_map_field_offs:
kfree(map->field_offs);
free_map:
btf_put(map->btf);
Expand Down

0 comments on commit f71b2f6

Please sign in to comment.