Skip to content

Commit

Permalink
nfp: bpf: relocate jump targets just before the load
Browse files Browse the repository at this point in the history 
Don't translate the program assuming it will be loaded at a given
address.  This will be required for sharing programs between ports
of the same NIC, tail calls and subprograms.  It will also make the
jump targets easier to understand when dumping the program to user
space.

Translate the program as if it was going to be loaded at address
zero.  When load happens add the load offset in and set addresses
of special branches.

Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
  • Loading branch information
Jakub Kicinski authored and Daniel Borkmann committed Jan 10, 2018
1 parent 488feea commit 2314fe9
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 87 deletions.
149 changes: 80 additions & 69 deletions drivers/net/ethernet/netronome/nfp/bpf/jit.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)

static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
{
return nfp_prog->start_off + nfp_prog->prog_len;
return nfp_prog->prog_len;
}

static bool
Expand All @@ -100,12 +100,6 @@ nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
}

static unsigned int
nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
{
return offset - nfp_prog->start_off;
}

/* --- Emitters --- */
static void
__emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
Expand Down Expand Up @@ -195,22 +189,28 @@ __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
nfp_prog_push(nfp_prog, insn);
}

static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
static void
emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
enum nfp_relo_type relo)
{
if (defer > 2) {
if (mask == BR_UNC && defer > 2) {
pr_err("BUG: branch defer out of bounds %d\n", defer);
nfp_prog->error = -EFAULT;
return;
}
__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);

__emit_br(nfp_prog, mask,
mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
BR_CSS_NONE, addr, defer);

nfp_prog->prog[nfp_prog->prog_len - 1] |=
FIELD_PREP(OP_RELO_TYPE, relo);
}

static void
emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
{
__emit_br(nfp_prog, mask,
mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
BR_CSS_NONE, addr, defer);
emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
}

static void
Expand Down Expand Up @@ -515,16 +515,6 @@ static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
emit_nop(nfp_prog);
}

static void
wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
enum br_special special)
{
emit_br(nfp_prog, mask, 0, 0);

nfp_prog->prog[nfp_prog->prog_len - 1] |=
FIELD_PREP(OP_BR_SPECIAL, special);
}

static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
{
emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
Expand Down Expand Up @@ -749,7 +739,7 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
emit_alu(nfp_prog, reg_none(),
plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
emit_br_relo(nfp_prog, BR_BLO, 0, 0, RELO_BR_GO_ABORT);

/* Load data */
return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
Expand All @@ -762,7 +752,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
/* Check packet length */
tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
emit_br_relo(nfp_prog, BR_BLO, 0, 0, RELO_BR_GO_ABORT);

/* Load data */
tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
Expand Down Expand Up @@ -1269,7 +1259,7 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);

/* Skip over the -EINVAL ret code (defer 2) */
emit_br_def(nfp_prog, end, 2);
emit_br(nfp_prog, BR_UNC, end, 2);

emit_alu(nfp_prog, plen_reg(nfp_prog),
plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
Expand Down Expand Up @@ -2036,7 +2026,7 @@ static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
emit_br_relo(nfp_prog, BR_UNC, 0, 0, RELO_BR_GO_OUT);

return 0;
}
Expand Down Expand Up @@ -2125,19 +2115,18 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
continue;

if (list_is_last(&meta->l, &nfp_prog->insns))
idx = nfp_prog->last_bpf_off;
br_idx = nfp_prog->last_bpf_off;
else
idx = list_next_entry(meta, l)->off - 1;

br_idx = nfp_prog_offset_to_index(nfp_prog, idx);
br_idx = list_next_entry(meta, l)->off - 1;

if (!nfp_is_br(nfp_prog->prog[br_idx])) {
pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
return -ELOOP;
}
/* Leave special branches for later */
if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
RELO_BR_REL)
continue;

if (!meta->jmp_dst) {
Expand All @@ -2152,38 +2141,13 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
return -ELOOP;
}

for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
idx <= br_idx; idx++) {
for (idx = meta->off; idx <= br_idx; idx++) {
if (!nfp_is_br(nfp_prog->prog[idx]))
continue;
br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
}
}

/* Fixup 'goto out's separately, they can be scattered around */
for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
enum br_special special;

if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
continue;

special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
switch (special) {
case OP_BR_NORMAL:
break;
case OP_BR_GO_OUT:
br_set_offset(&nfp_prog->prog[br_idx],
nfp_prog->tgt_out);
break;
case OP_BR_GO_ABORT:
br_set_offset(&nfp_prog->prog[br_idx],
nfp_prog->tgt_abort);
break;
}

nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
}

return 0;
}

Expand Down Expand Up @@ -2211,7 +2175,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
/* Target for aborts */
nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);

emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);

wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
Expand All @@ -2238,7 +2202,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
emit_shf(nfp_prog, reg_b(2),
reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);

emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);

emit_shf(nfp_prog, reg_b(2),
reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
Expand All @@ -2257,7 +2221,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
/* Target for aborts */
nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);

emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);

wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
Expand All @@ -2278,7 +2242,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
emit_shf(nfp_prog, reg_b(2),
reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);

emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
emit_br_relo(nfp_prog, BR_UNC, 0, 2, RELO_BR_NEXT_PKT);

wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
Expand Down Expand Up @@ -2694,20 +2658,19 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
return 0;
}

static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)
{
__le64 *ustore = (__force __le64 *)prog;
int i;

for (i = 0; i < nfp_prog->prog_len; i++) {
for (i = 0; i < len; i++) {
int err;

err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);
err = nfp_ustore_check_valid_no_ecc(prog[i]);
if (err)
return err;

nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);

ustore[i] = cpu_to_le64(nfp_prog->prog[i]);
ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i]));
}

return 0;
Expand All @@ -2728,7 +2691,7 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog)
return -EINVAL;
}

return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog);
return ret;
}

void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
Expand All @@ -2753,3 +2716,51 @@ void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
}
}
}

void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
{
unsigned int i;
u64 *prog;
int err;

prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),
GFP_KERNEL);
if (!prog)
return ERR_PTR(-ENOMEM);

for (i = 0; i < nfp_prog->prog_len; i++) {
enum nfp_relo_type special;

special = FIELD_GET(OP_RELO_TYPE, prog[i]);
switch (special) {
case RELO_NONE:
continue;
case RELO_BR_REL:
br_add_offset(&prog[i], bv->start_off);
break;
case RELO_BR_GO_OUT:
br_set_offset(&prog[i],
nfp_prog->tgt_out + bv->start_off);
break;
case RELO_BR_GO_ABORT:
br_set_offset(&prog[i],
nfp_prog->tgt_abort + bv->start_off);
break;
case RELO_BR_NEXT_PKT:
br_set_offset(&prog[i], bv->tgt_done);
break;
}

prog[i] &= ~OP_RELO_TYPE;
}

err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);
if (err)
goto err_free_prog;

return prog;

err_free_prog:
kfree(prog);
return ERR_PTR(err);
}
9 changes: 7 additions & 2 deletions drivers/net/ethernet/netronome/nfp/bpf/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,16 +87,21 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
static int
nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
{
struct nfp_bpf_vnic *bv;
int err;

nn->app_priv = kzalloc(sizeof(struct nfp_bpf_vnic), GFP_KERNEL);
if (!nn->app_priv)
bv = kzalloc(sizeof(*bv), GFP_KERNEL);
if (!bv)
return -ENOMEM;
nn->app_priv = bv;

err = nfp_app_nic_vnic_alloc(app, nn, id);
if (err)
goto err_free_priv;

bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);

return 0;
err_free_priv:
kfree(nn->app_priv);
Expand Down
29 changes: 18 additions & 11 deletions drivers/net/ethernet/netronome/nfp/bpf/main.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,20 @@

#include "../nfp_asm.h"

/* For branch fixup logic use up-most byte of branch instruction as scratch
/* For relocation logic use up-most byte of branch instruction as scratch
* area. Remember to clear this before sending instructions to HW!
*/
#define OP_BR_SPECIAL 0xff00000000000000ULL

enum br_special {
OP_BR_NORMAL = 0,
OP_BR_GO_OUT,
OP_BR_GO_ABORT,
#define OP_RELO_TYPE 0xff00000000000000ULL

enum nfp_relo_type {
RELO_NONE = 0,
/* standard internal jumps */
RELO_BR_REL,
/* internal jumps to parts of the outro */
RELO_BR_GO_OUT,
RELO_BR_GO_ABORT,
/* external jumps to fixed addresses */
RELO_BR_NEXT_PKT,
};

enum static_regs {
Expand Down Expand Up @@ -191,11 +196,9 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)
* @__prog_alloc_len: alloc size of @prog array
* @verifier_meta: temporary storage for verifier's insn meta
* @type: BPF program type
* @start_off: address of the first instruction in the memory
* @last_bpf_off: address of the last instruction translated from BPF
* @tgt_out: jump target for normal exit
* @tgt_abort: jump target for abort (e.g. access outside of packet buffer)
* @tgt_done: jump target to get the next packet
* @n_translated: number of successfully translated instructions (for errors)
* @error: error code if something went wrong
* @stack_depth: max stack depth from the verifier
Expand All @@ -213,11 +216,9 @@ struct nfp_prog {

enum bpf_prog_type type;

unsigned int start_off;
unsigned int last_bpf_off;
unsigned int tgt_out;
unsigned int tgt_abort;
unsigned int tgt_done;

unsigned int n_translated;
int error;
Expand All @@ -231,9 +232,13 @@ struct nfp_prog {
/**
* struct nfp_bpf_vnic - per-vNIC BPF priv structure
* @tc_prog: currently loaded cls_bpf program
* @start_off: address of the first instruction in the memory
* @tgt_done: jump target to get the next packet
*/
struct nfp_bpf_vnic {
struct bpf_prog *tc_prog;
unsigned int start_off;
unsigned int tgt_done;
};

void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);
Expand All @@ -257,4 +262,6 @@ int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
struct nfp_insn_meta *
nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
unsigned int insn_idx, unsigned int n_insns);

void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);
#endif
Loading

0 comments on commit 2314fe9

Please sign in to comment.