Skip to content

Commit

Permalink
mips, bpf: Add JIT workarounds for CPU errata
Browse files Browse the repository at this point in the history
This patch adds workarounds for the following CPU errata to the MIPS
eBPF JIT, if enabled in the kernel configuration.

  - R10000 ll/sc weak ordering
  - Loongson-3 ll/sc weak ordering
  - Loongson-2F jump hang

The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
so no additional mitigations are needed for that.

Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
Link: https://lore.kernel.org/bpf/20211005165408.2305108-6-johan.almbladh@anyfinetworks.com
  • Loading branch information
Johan Almbladh authored and Andrii Nakryiko committed Oct 6, 2021
1 parent fbc802d commit 7257022
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 7 deletions.
6 changes: 4 additions & 2 deletions arch/mips/net/bpf_jit_comp.c
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
/* Atomic read-modify-write (32-bit) */
void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
{
LLSC_sync(ctx);
emit(ctx, ll, MIPS_R_T9, off, dst);
switch (code) {
case BPF_ADD:
Expand All @@ -427,7 +428,7 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
break;
}
emit(ctx, sc, MIPS_R_T8, off, dst);
emit(ctx, beqz, MIPS_R_T8, -16);
emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
emit(ctx, nop); /* Delay slot */

if (code & BPF_FETCH) {
Expand All @@ -439,11 +440,12 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
/* Atomic compare-and-exchange (32-bit) */
void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
{
LLSC_sync(ctx);
emit(ctx, ll, MIPS_R_T9, off, dst);
emit(ctx, bne, MIPS_R_T9, res, 12);
emit(ctx, move, MIPS_R_T8, src); /* Delay slot */
emit(ctx, sc, MIPS_R_T8, off, dst);
emit(ctx, beqz, MIPS_R_T8, -20);
emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
emit(ctx, move, res, MIPS_R_T9); /* Delay slot */
clobber_reg(ctx, res);
}
Expand Down
26 changes: 25 additions & 1 deletion arch/mips/net/bpf_jit_comp.h
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,38 @@ struct jit_context {
};

/* Emit the instruction if the JIT memory space has been allocated */
#define emit(ctx, func, ...) \
#define __emit(ctx, func, ...) \
do { \
if ((ctx)->target != NULL) { \
u32 *p = &(ctx)->target[ctx->jit_index]; \
uasm_i_##func(&p, ##__VA_ARGS__); \
} \
(ctx)->jit_index++; \
} while (0)
#define emit(...) __emit(__VA_ARGS__)

/* Workaround for R10000 ll/sc errata */
#ifdef CONFIG_WAR_R10000
#define LLSC_beqz beqzl
#else
#define LLSC_beqz beqz
#endif

/* Workaround for Loongson-3 ll/sc errata */
#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
#define LLSC_sync(ctx) emit(ctx, sync, 0)
#define LLSC_offset 4
#else
#define LLSC_sync(ctx)
#define LLSC_offset 0
#endif

/* Workaround for Loongson-2F jump errata */
#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
#define JALR_MASK 0xffffffffcfffffffULL
#else
#define JALR_MASK (~0ULL)
#endif

/*
* Mark a BPF register as accessed, it needs to be
Expand Down
10 changes: 6 additions & 4 deletions arch/mips/net/bpf_jit_comp64.c
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
u8 t1 = MIPS_R_T6;
u8 t2 = MIPS_R_T7;

LLSC_sync(ctx);
emit(ctx, lld, t1, off, dst);
switch (code) {
case BPF_ADD:
Expand All @@ -398,7 +399,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
break;
}
emit(ctx, scd, t2, off, dst);
emit(ctx, beqz, t2, -16);
emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
emit(ctx, nop); /* Delay slot */

if (code & BPF_FETCH) {
Expand All @@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
u8 t1 = MIPS_R_T6;
u8 t2 = MIPS_R_T7;

LLSC_sync(ctx);
emit(ctx, lld, t1, off, dst);
emit(ctx, bne, t1, r0, 12);
emit(ctx, move, t2, src); /* Delay slot */
emit(ctx, scd, t2, off, dst);
emit(ctx, beqz, t2, -20);
emit(ctx, move, r0, t1); /* Delay slot */
emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
emit(ctx, move, r0, t1); /* Delay slot */

clobber_reg(ctx, r0);
}
Expand All @@ -443,7 +445,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);

/* Emit function call */
emit_mov_i64(ctx, tmp, addr);
emit_mov_i64(ctx, tmp, addr & JALR_MASK);
emit(ctx, jalr, MIPS_R_RA, tmp);
emit(ctx, nop); /* Delay slot */

Expand Down

0 comments on commit 7257022

Please sign in to comment.