Skip to content

Commit

Permalink
nfp: bpf: eliminate zero extension code-gen
Browse files Browse the repository at this point in the history 
This patch eliminate zero extension code-gen for instructions including
both alu and load/store. The only exception is for ctx load, because
offload target doesn't go through host ctx convert logic so we do
customized load and ignores zext flag set by verifier.

Cc: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
  • Loading branch information
Jiong Wang authored and Alexei Starovoitov committed May 25, 2019
1 parent 66d0d5a commit 0b4de1f
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 48 deletions.
115 changes: 67 additions & 48 deletions drivers/net/ethernet/netronome/nfp/bpf/jit.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,13 @@ static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
}
}

static void
wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
{
if (meta->flags & FLAG_INSN_DO_ZEXT)
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
}

static void
wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
enum nfp_relo_type relo)
Expand Down Expand Up @@ -858,7 +865,8 @@ static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
}

static int
data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
u8 dst_gpr, int size)
{
unsigned int i;
u16 shift, sz;
Expand All @@ -881,14 +889,15 @@ data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));

if (i < 2)
wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
wrp_zext(nfp_prog, meta, dst_gpr);

return 0;
}

static int
data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
swreg lreg, swreg rreg, int size, enum cmd_mode mode)
data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u8 dst_gpr, swreg lreg, swreg rreg, int size,
enum cmd_mode mode)
{
unsigned int i;
u8 mask, sz;
Expand All @@ -911,33 +920,34 @@ data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));

if (i < 2)
wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
wrp_zext(nfp_prog, meta, dst_gpr);

return 0;
}

static int
data_ld_host_order_addr32(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
u8 dst_gpr, u8 size)
data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
{
return data_ld_host_order(nfp_prog, dst_gpr, reg_a(src_gpr), offset,
size, CMD_MODE_32b);
return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
offset, size, CMD_MODE_32b);
}

static int
data_ld_host_order_addr40(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
u8 dst_gpr, u8 size)
data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
{
swreg rega, regb;

addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);

return data_ld_host_order(nfp_prog, dst_gpr, rega, regb,
return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
size, CMD_MODE_40b_BA);
}

static int
construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u16 offset, u16 src, u8 size)
{
swreg tmp_reg;

Expand All @@ -953,10 +963,12 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);

/* Load data */
return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
}

static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
static int
construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u16 offset, u8 size)
{
swreg tmp_reg;

Expand All @@ -967,7 +979,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)

/* Load data */
tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
return data_ld(nfp_prog, tmp_reg, 0, size);
return data_ld(nfp_prog, meta, tmp_reg, 0, size);
}

static int
Expand Down Expand Up @@ -1204,7 +1216,7 @@ mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
}

if (clr_gpr && size < 8)
wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
wrp_zext(nfp_prog, meta, gpr);

while (size) {
u32 slice_end;
Expand Down Expand Up @@ -1305,9 +1317,10 @@ wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
enum alu_op alu_op)
{
const struct bpf_insn *insn = &meta->insn;
u8 dst = insn->dst_reg * 2;

wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
wrp_zext(nfp_prog, meta, dst);

return 0;
}
Expand All @@ -1319,7 +1332,7 @@ wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;

emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
wrp_zext(nfp_prog, meta, dst);

return 0;
}
Expand Down Expand Up @@ -2396,12 +2409,14 @@ static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
u8 dst = meta->insn.dst_reg * 2;

emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
wrp_zext(nfp_prog, meta, dst);

return 0;
}

static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
static int
__ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
u8 shift_amt)
{
if (shift_amt) {
/* Set signedness bit (MSB of result). */
Expand All @@ -2410,7 +2425,7 @@ static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
reg_b(dst), SHF_SC_R_SHF, shift_amt);
}
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);

return 0;
}
Expand All @@ -2425,7 +2440,7 @@ static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
umin = meta->umin_src;
umax = meta->umax_src;
if (umin == umax)
return __ashr_imm(nfp_prog, dst, umin);
return __ashr_imm(nfp_prog, meta, dst, umin);

src = insn->src_reg * 2;
/* NOTE: the first insn will set both indirect shift amount (source A)
Expand All @@ -2434,7 +2449,7 @@ static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
reg_b(dst), SHF_SC_R_SHF);
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);

return 0;
}
Expand All @@ -2444,15 +2459,17 @@ static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
const struct bpf_insn *insn = &meta->insn;
u8 dst = insn->dst_reg * 2;

return __ashr_imm(nfp_prog, dst, insn->imm);
return __ashr_imm(nfp_prog, meta, dst, insn->imm);
}

static int __shr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
static int
__shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
u8 shift_amt)
{
if (shift_amt)
emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
reg_b(dst), SHF_SC_R_SHF, shift_amt);
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);
return 0;
}

Expand All @@ -2461,7 +2478,7 @@ static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
const struct bpf_insn *insn = &meta->insn;
u8 dst = insn->dst_reg * 2;

return __shr_imm(nfp_prog, dst, insn->imm);
return __shr_imm(nfp_prog, meta, dst, insn->imm);
}

static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
Expand All @@ -2474,22 +2491,24 @@ static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
umin = meta->umin_src;
umax = meta->umax_src;
if (umin == umax)
return __shr_imm(nfp_prog, dst, umin);
return __shr_imm(nfp_prog, meta, dst, umin);

src = insn->src_reg * 2;
emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
reg_b(dst), SHF_SC_R_SHF);
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);
return 0;
}

static int __shl_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
static int
__shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
u8 shift_amt)
{
if (shift_amt)
emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
reg_b(dst), SHF_SC_L_SHF, shift_amt);
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);
return 0;
}

Expand All @@ -2498,7 +2517,7 @@ static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
const struct bpf_insn *insn = &meta->insn;
u8 dst = insn->dst_reg * 2;

return __shl_imm(nfp_prog, dst, insn->imm);
return __shl_imm(nfp_prog, meta, dst, insn->imm);
}

static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
Expand All @@ -2511,11 +2530,11 @@ static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
umin = meta->umin_src;
umax = meta->umax_src;
if (umin == umax)
return __shl_imm(nfp_prog, dst, umin);
return __shl_imm(nfp_prog, meta, dst, umin);

src = insn->src_reg * 2;
shl_reg64_lt32_low(nfp_prog, dst, src);
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
wrp_zext(nfp_prog, meta, dst);
return 0;
}

Expand Down Expand Up @@ -2577,34 +2596,34 @@ static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ld(nfp_prog, meta->insn.imm, 1);
return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
}

static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ld(nfp_prog, meta->insn.imm, 2);
return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
}

static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ld(nfp_prog, meta->insn.imm, 4);
return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
}

static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ind_ld(nfp_prog, meta->insn.imm,
return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
meta->insn.src_reg * 2, 1);
}

static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ind_ld(nfp_prog, meta->insn.imm,
return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
meta->insn.src_reg * 2, 2);
}

static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
return construct_data_ind_ld(nfp_prog, meta->insn.imm,
return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
meta->insn.src_reg * 2, 4);
}

Expand Down Expand Up @@ -2682,7 +2701,7 @@ mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));

return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2,
return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
tmp_reg, meta->insn.dst_reg * 2, size);
}

Expand All @@ -2694,7 +2713,7 @@ mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));

return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2,
return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
tmp_reg, meta->insn.dst_reg * 2, size);
}

Expand Down Expand Up @@ -2755,15 +2774,15 @@ mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);

if (!len_mid) {
wrp_immed(nfp_prog, dst_hi, 0);
wrp_zext(nfp_prog, meta, dst_gpr);
return 0;
}

src_mid = reg_xfer(idx + 1);

if (size <= REG_WIDTH) {
wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
wrp_immed(nfp_prog, dst_hi, 0);
wrp_zext(nfp_prog, meta, dst_gpr);
} else {
swreg src_hi = reg_xfer(idx + 2);

Expand Down Expand Up @@ -2794,10 +2813,10 @@ mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog,

if (size < REG_WIDTH) {
wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
wrp_immed(nfp_prog, dst_hi, 0);
wrp_zext(nfp_prog, meta, dst_gpr);
} else if (size == REG_WIDTH) {
wrp_mov(nfp_prog, dst_lo, src_lo);
wrp_immed(nfp_prog, dst_hi, 0);
wrp_zext(nfp_prog, meta, dst_gpr);
} else {
swreg src_hi = reg_xfer(idx + 1);

Expand Down
2 changes: 2 additions & 0 deletions drivers/net/ethernet/netronome/nfp/bpf/main.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,8 @@ struct nfp_bpf_reg_state {
#define FLAG_INSN_SKIP_PREC_DEPENDENT BIT(4)
/* Instruction is optimized by the verifier */
#define FLAG_INSN_SKIP_VERIFIER_OPT BIT(5)
/* Instruction needs to zero extend to high 32-bit */
#define FLAG_INSN_DO_ZEXT BIT(6)

#define FLAG_INSN_SKIP_MASK (FLAG_INSN_SKIP_NOOP | \
FLAG_INSN_SKIP_PREC_DEPENDENT | \
Expand Down
12 changes: 12 additions & 0 deletions drivers/net/ethernet/netronome/nfp/bpf/verifier.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,17 @@ static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)
goto continue_subprog;
}

static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog,
struct bpf_insn_aux_data *aux)
{
struct nfp_insn_meta *meta;

list_for_each_entry(meta, &nfp_prog->insns, l) {
if (aux[meta->n].zext_dst)
meta->flags |= FLAG_INSN_DO_ZEXT;
}
}

int nfp_bpf_finalize(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *info;
Expand Down Expand Up @@ -784,6 +795,7 @@ int nfp_bpf_finalize(struct bpf_verifier_env *env)
return -EOPNOTSUPP;
}

nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);
return 0;
}

Expand Down

0 comments on commit 0b4de1f

Please sign in to comment.