Skip to content

Commit

Permalink
nfp: bpf: move to new BPF program offload infrastructure
Browse files Browse the repository at this point in the history
Following steps are taken in the driver to offload an XDP program:

XDP_SETUP_PROG:
 * prepare:
   - allocate program state;
   - run verifier (bpf_analyzer());
   - run translation;
 * load:
   - stop old program if needed;
   - load program;
   - enable BPF if not enabled;
 * clean up:
   - free program image.

With new infrastructure the flow will look like this:

BPF_OFFLOAD_VERIFIER_PREP:
  - allocate program state;
BPF_OFFLOAD_TRANSLATE:
   - run translation;
XDP_SETUP_PROG:
   - stop old program if needed;
   - load program;
   - enable BPF if not enabled;
BPF_OFFLOAD_DESTROY:
   - free program image.

Take advantage of the new infrastructure.  Allocation of driver
metadata has to be moved from jit.c to offload.c since it's now
done at a different stage.  Since there is no separate driver
private data for verification step, move temporary nfp_meta
pointer into nfp_prog.  We will now use user space context
offsets.

Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Quentin Monnet <quentin.monnet@netronome.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Jakub Kicinski authored and David S. Miller committed Nov 5, 2017
1 parent 9314c44 commit c6c580d
Show file tree
Hide file tree
Showing 7 changed files with 121 additions and 106 deletions.
35 changes: 13 additions & 22 deletions drivers/net/ethernet/netronome/nfp/bpf/jit.c
Original file line number Diff line number Diff line change
Expand Up @@ -1427,19 +1427,18 @@ static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
swreg dst = reg_both(meta->insn.dst_reg * 2);

switch (meta->insn.off) {
case offsetof(struct sk_buff, len):
if (size != FIELD_SIZEOF(struct sk_buff, len))
case offsetof(struct __sk_buff, len):
if (size != FIELD_SIZEOF(struct __sk_buff, len))
return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
break;
case offsetof(struct sk_buff, data):
if (size != sizeof(void *))
case offsetof(struct __sk_buff, data):
if (size != FIELD_SIZEOF(struct __sk_buff, data))
return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
break;
case offsetof(struct sk_buff, cb) +
offsetof(struct bpf_skb_data_end, data_end):
if (size != sizeof(void *))
case offsetof(struct __sk_buff, data_end):
if (size != FIELD_SIZEOF(struct __sk_buff, data_end))
return -EOPNOTSUPP;
emit_alu(nfp_prog, dst,
plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
Expand All @@ -1458,14 +1457,15 @@ static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
{
swreg dst = reg_both(meta->insn.dst_reg * 2);

if (size != sizeof(void *))
return -EINVAL;

switch (meta->insn.off) {
case offsetof(struct xdp_buff, data):
case offsetof(struct xdp_md, data):
if (size != FIELD_SIZEOF(struct xdp_md, data))
return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
break;
case offsetof(struct xdp_buff, data_end):
case offsetof(struct xdp_md, data_end):
if (size != FIELD_SIZEOF(struct xdp_md, data_end))
return -EOPNOTSUPP;
emit_alu(nfp_prog, dst,
plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
break;
Expand Down Expand Up @@ -2243,19 +2243,10 @@ static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
return 0;
}

/**
* nfp_bpf_jit() - translate BPF code into NFP assembly
* @nfp_prog: nfp_prog prepared based on @filter
* @filter: kernel BPF filter struct
*/
int nfp_bpf_jit(struct nfp_prog *nfp_prog, struct bpf_prog *filter)
int nfp_bpf_jit(struct nfp_prog *nfp_prog)
{
int ret;

ret = nfp_prog_verify(nfp_prog, filter);
if (ret)
return ret;

ret = nfp_bpf_optimize(nfp_prog);
if (ret)
return ret;
Expand Down
4 changes: 4 additions & 0 deletions drivers/net/ethernet/netronome/nfp/bpf/main.c
Original file line number Diff line number Diff line change
Expand Up @@ -173,4 +173,8 @@ const struct nfp_app_type app_bpf = {
.setup_tc = nfp_bpf_setup_tc,
.tc_busy = nfp_bpf_tc_busy,
.xdp_offload = nfp_bpf_xdp_offload,

.bpf_verifier_prep = nfp_bpf_verifier_prep,
.bpf_translate = nfp_bpf_translate,
.bpf_destroy = nfp_bpf_destroy,
};
15 changes: 13 additions & 2 deletions drivers/net/ethernet/netronome/nfp/bpf/main.h
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
* @prog: machine code
* @prog_len: number of valid instructions in @prog array
* @__prog_alloc_len: alloc size of @prog array
* @verifier_meta: temporary storage for verifier's insn meta
* @type: BPF program type
* @start_off: address of the first instruction in the memory
* @tgt_out: jump target for normal exit
Expand All @@ -154,6 +155,8 @@ struct nfp_prog {
unsigned int prog_len;
unsigned int __prog_alloc_len;

struct nfp_insn_meta *verifier_meta;

enum bpf_prog_type type;

unsigned int start_off;
Expand All @@ -169,13 +172,21 @@ struct nfp_prog {
struct list_head insns;
};

int nfp_bpf_jit(struct nfp_prog *nfp_prog, struct bpf_prog *filter);
int nfp_bpf_jit(struct nfp_prog *prog);

int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog);
extern const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops;

struct netdev_bpf;
struct nfp_app;
struct nfp_net;

int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
bool old_prog);

int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
struct netdev_bpf *bpf);
int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
struct bpf_prog *prog);
int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
struct bpf_prog *prog);
#endif
85 changes: 38 additions & 47 deletions drivers/net/ethernet/netronome/nfp/bpf/offload.c
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,17 @@ static void nfp_prog_free(struct nfp_prog *nfp_prog)
kfree(nfp_prog);
}

static struct nfp_prog *nfp_bpf_verifier_prep(struct bpf_prog *prog)
int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
struct netdev_bpf *bpf)
{
struct bpf_prog *prog = bpf->verifier.prog;
struct nfp_prog *nfp_prog;
int ret;

nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
if (!nfp_prog)
return NULL;
return -ENOMEM;
prog->aux->offload->dev_priv = nfp_prog;

INIT_LIST_HEAD(&nfp_prog->insns);
nfp_prog->type = prog->type;
Expand All @@ -100,18 +103,21 @@ static struct nfp_prog *nfp_bpf_verifier_prep(struct bpf_prog *prog)
if (ret)
goto err_free;

return nfp_prog;
nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
bpf->verifier.ops = &nfp_bpf_analyzer_ops;

return 0;

err_free:
nfp_prog_free(nfp_prog);

return NULL;
return ret;
}

static int
nfp_bpf_translate(struct nfp_net *nn, struct nfp_prog *nfp_prog,
struct bpf_prog *prog)
int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
struct bpf_prog *prog)
{
struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
unsigned int stack_size;
unsigned int max_instr;

Expand All @@ -133,55 +139,38 @@ nfp_bpf_translate(struct nfp_net *nn, struct nfp_prog *nfp_prog,
if (!nfp_prog->prog)
return -ENOMEM;

return nfp_bpf_jit(nfp_prog, prog);
return nfp_bpf_jit(nfp_prog);
}

static void nfp_bpf_destroy(struct nfp_prog *nfp_prog)
int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
struct bpf_prog *prog)
{
struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

kfree(nfp_prog->prog);
nfp_prog_free(nfp_prog);

return 0;
}

static struct nfp_prog *
nfp_net_bpf_offload_prepare(struct nfp_net *nn, struct bpf_prog *prog,
dma_addr_t *dma_addr)
static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
{
struct nfp_prog *nfp_prog;
struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
unsigned int max_mtu;
dma_addr_t dma_addr;
int err;

max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
if (max_mtu < nn->dp.netdev->mtu) {
nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
return NULL;
return -EOPNOTSUPP;
}

nfp_prog = nfp_bpf_verifier_prep(prog);
if (!nfp_prog)
return NULL;

err = nfp_bpf_translate(nn, nfp_prog, prog);
if (err)
goto err_destroy_prog;

*dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
nfp_prog->prog_len * sizeof(u64),
DMA_TO_DEVICE);
if (dma_mapping_error(nn->dp.dev, *dma_addr))
goto err_destroy_prog;

return 0;

err_destroy_prog:
nfp_bpf_destroy(nfp_prog);
return NULL;
}

static void
nfp_net_bpf_load(struct nfp_net *nn, struct nfp_prog *nfp_prog,
dma_addr_t dma_addr)
{
int err;
dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
nfp_prog->prog_len * sizeof(u64),
DMA_TO_DEVICE);
if (dma_mapping_error(nn->dp.dev, dma_addr))
return -ENOMEM;

nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
Expand All @@ -193,7 +182,8 @@ nfp_net_bpf_load(struct nfp_net *nn, struct nfp_prog *nfp_prog,

dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
DMA_TO_DEVICE);
nfp_bpf_destroy(nfp_prog);

return err;
}

static void nfp_net_bpf_start(struct nfp_net *nn)
Expand Down Expand Up @@ -222,8 +212,10 @@ static int nfp_net_bpf_stop(struct nfp_net *nn)
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
bool old_prog)
{
struct nfp_prog *nfp_prog;
dma_addr_t dma_addr;
int err;

if (prog && !prog->aux->offload)
return -EINVAL;

if (prog && old_prog) {
u8 cap;
Expand All @@ -242,11 +234,10 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
if (old_prog && !prog)
return nfp_net_bpf_stop(nn);

nfp_prog = nfp_net_bpf_offload_prepare(nn, prog, &dma_addr);
if (!nfp_prog)
return -EINVAL;
err = nfp_net_bpf_load(nn, prog);
if (err)
return err;

nfp_net_bpf_load(nn, nfp_prog, dma_addr);
if (!old_prog)
nfp_net_bpf_start(nn);

Expand Down
43 changes: 8 additions & 35 deletions drivers/net/ethernet/netronome/nfp/bpf/verifier.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,6 @@

#include "main.h"

/* Analyzer/verifier definitions */
struct nfp_bpf_analyzer_priv {
struct nfp_prog *prog;
struct nfp_insn_meta *meta;
};

static struct nfp_insn_meta *
nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
unsigned int insn_idx, unsigned int n_insns)
Expand Down Expand Up @@ -171,11 +165,11 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
static int
nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
{
struct nfp_bpf_analyzer_priv *priv = env->analyzer_priv;
struct nfp_insn_meta *meta = priv->meta;
struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
struct nfp_insn_meta *meta = nfp_prog->verifier_meta;

meta = nfp_bpf_goto_meta(priv->prog, meta, insn_idx, env->prog->len);
priv->meta = meta;
meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len);
nfp_prog->verifier_meta = meta;

if (meta->insn.src_reg >= MAX_BPF_REG ||
meta->insn.dst_reg >= MAX_BPF_REG) {
Expand All @@ -184,39 +178,18 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
}

if (meta->insn.code == (BPF_JMP | BPF_EXIT))
return nfp_bpf_check_exit(priv->prog, env);
return nfp_bpf_check_exit(nfp_prog, env);

if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM))
return nfp_bpf_check_ptr(priv->prog, meta, env,
return nfp_bpf_check_ptr(nfp_prog, meta, env,
meta->insn.src_reg);
if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM))
return nfp_bpf_check_ptr(priv->prog, meta, env,
return nfp_bpf_check_ptr(nfp_prog, meta, env,
meta->insn.dst_reg);

return 0;
}

static const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
.insn_hook = nfp_verify_insn,
};

int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog)
{
struct nfp_bpf_analyzer_priv *priv;
int ret;

nfp_prog->stack_depth = prog->aux->stack_depth;

priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;

priv->prog = nfp_prog;
priv->meta = nfp_prog_first_meta(nfp_prog);

ret = bpf_analyzer(prog, &nfp_bpf_analyzer_ops, priv);

kfree(priv);

return ret;
}
Loading

0 comments on commit c6c580d

Please sign in to comment.