Skip to content

Arena fault stream #9589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: bpf-next_base
Choose a base branch
from
Open

Arena fault stream #9589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9c8a0a9
bpf, x86: Add support for signed arena loads
kkdwivedi May 14, 2025
043dde0
bpf, arm64: Add support for signed arena loads
puranjaymohan Sep 12, 2025
2c9305a
selftests: bpf: Add tests for signed loads from arena
puranjaymohan Sep 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions arch/arm64/net/bpf_jit_comp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1133,12 +1133,14 @@ static int add_exception_handler(const struct bpf_insn *insn,
return 0;

if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
BPF_MODE(insn->code) != BPF_PROBE_MEM32SX &&
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
return 0;

is_arena = (BPF_MODE(insn->code) == BPF_PROBE_MEM32) ||
(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) ||
(BPF_MODE(insn->code) == BPF_PROBE_ATOMIC);

if (!ctx->prog->aux->extable ||
Expand Down Expand Up @@ -1659,7 +1661,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) {
emit(A64_ADD(1, tmp2, src, arena_vm_base), ctx);
src = tmp2;
}
Expand All @@ -1671,7 +1677,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
off_adj = off;
}
sign_extend = (BPF_MODE(insn->code) == BPF_MEMSX ||
BPF_MODE(insn->code) == BPF_PROBE_MEMSX);
BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX);
switch (BPF_SIZE(code)) {
case BPF_W:
if (is_lsi_offset(off_adj, 2)) {
Expand Down Expand Up @@ -1879,9 +1886,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
if (ret)
return ret;

ret = add_exception_handler(insn, ctx, dst);
if (ret)
return ret;
if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
ret = add_exception_handler(insn, ctx, dst);
if (ret)
return ret;
}
break;

default:
Expand Down
5 changes: 5 additions & 0 deletions arch/s390/net/bpf_jit_comp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2967,6 +2967,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (bpf_atomic_is_load_store(insn))
return false;
break;
case BPF_LDX | BPF_MEMSX | BPF_B:
case BPF_LDX | BPF_MEMSX | BPF_H:
case BPF_LDX | BPF_MEMSX | BPF_W:
return false;
}
return true;
}
Expand Down
35 changes: 34 additions & 1 deletion arch/x86/net/bpf_jit_comp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1152,11 +1152,38 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i
*pprog = prog;
}

static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)
{
u8 *prog = *pprog;

switch (size) {
case BPF_B:
/* movsx rax, byte ptr [rax + r12 + off] */
EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBE);
break;
case BPF_H:
/* movsx rax, word ptr [rax + r12 + off] */
EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBF);
break;
case BPF_W:
/* movsx rax, dword ptr [rax + r12 + off] */
EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63);
break;
}
emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off);
*pprog = prog;
}

static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);
}

static void emit_ldsx_r12(u8 **prog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
emit_ldsx_index(prog, size, dst_reg, src_reg, X86_REG_R12, off);
}

/* STX: *(u8*)(dst_reg + off) = src_reg */
static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
Expand Down Expand Up @@ -2109,13 +2136,19 @@ st: if (is_imm8(insn->off))
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
start_of_ldx = prog;
if (BPF_CLASS(insn->code) == BPF_LDX)
emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32SX)
emit_ldsx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
else
emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
else
emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
populate_extable:
Expand Down
3 changes: 3 additions & 0 deletions include/linux/filter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ struct ctl_table_header;
/* unused opcode to mark special atomic instruction */
#define BPF_PROBE_ATOMIC 0xe0

/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */
#define BPF_PROBE_MEM32SX 0xc0

/* unused opcode to mark call to interpreter with arguments */
#define BPF_CALL_ARGS 0xe0

Expand Down
11 changes: 8 additions & 3 deletions kernel/bpf/verifier.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21423,10 +21423,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
continue;
case PTR_TO_ARENA:
if (BPF_MODE(insn->code) == BPF_MEMSX) {
verbose(env, "sign extending loads from arena are not supported yet\n");
return -EOPNOTSUPP;
if (!bpf_jit_supports_insn(insn, true)) {
verbose(env, "sign extending loads from arena are not supported yet\n");
return -EOPNOTSUPP;
}
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code);
} else {
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
}
insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
env->prog->aux->num_exentries++;
continue;
default:
Expand Down Expand Up @@ -21632,6 +21636,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
if (BPF_CLASS(insn->code) == BPF_LDX &&
(BPF_MODE(insn->code) == BPF_PROBE_MEM ||
BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
BPF_MODE(insn->code) == BPF_PROBE_MEM32SX ||
BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
num_exentries++;
if ((BPF_CLASS(insn->code) == BPF_STX ||
Expand Down
176 changes: 176 additions & 0 deletions tools/testing/selftests/bpf/progs/verifier_ldsx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,20 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#include "bpf_arena_common.h"

#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
defined(__TARGET_ARCH_loongarch)) && \
__clang_major__ >= 18

struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 1);
} arena SEC(".maps");

SEC("socket")
__description("LDSX, S8")
__success __success_unpriv __retval(-2)
Expand Down Expand Up @@ -256,6 +263,175 @@ __naked void ldsx_ctx_8(void)
: __clobber_all);
}

SEC("syscall")
__description("Arena LDSX Disasm")
__success
__arch_x86_64
__jited("movslq 0x10(%rax,%r12), %r14")
__jited("movswq 0x18(%rax,%r12), %r14")
__jited("movsbq 0x20(%rax,%r12), %r14")
__jited("movslq 0x10(%rdi,%r12), %r15")
__jited("movswq 0x18(%rdi,%r12), %r15")
__jited("movsbq 0x20(%rdi,%r12), %r15")
__arch_arm64
__jited("add x11, x7, x28")
__jited("ldrsw x21, [x11, #0x10]")
__jited("add x11, x7, x28")
__jited("ldrsh x21, [x11, #0x18]")
__jited("add x11, x7, x28")
__jited("ldrsb x21, [x11, #0x20]")
__jited("add x11, x0, x28")
__jited("ldrsw x22, [x11, #0x10]")
__jited("add x11, x0, x28")
__jited("ldrsh x22, [x11, #0x18]")
__jited("add x11, x0, x28")
__jited("ldrsb x22, [x11, #0x20]")
__naked void arena_ldsx_disasm(void *ctx)
{
asm volatile (
"r1 = %[arena] ll;"
"r2 = 0;"
"r3 = 1;"
"r4 = %[numa_no_node];"
"r5 = 0;"
"call %[bpf_arena_alloc_pages];"
"r0 = addr_space_cast(r0, 0x0, 0x1);"
"r1 = r0;"
"r8 = *(s32 *)(r0 + 16);"
"r8 = *(s16 *)(r0 + 24);"
"r8 = *(s8 *)(r0 + 32);"
"r9 = *(s32 *)(r1 + 16);"
"r9 = *(s16 *)(r1 + 24);"
"r9 = *(s8 *)(r1 + 32);"
"r0 = 0;"
"exit;"
:: __imm(bpf_arena_alloc_pages),
__imm_addr(arena),
__imm_const(numa_no_node, NUMA_NO_NODE)
: __clobber_all
);
}

SEC("syscall")
__description("Arena LDSX Exception")
__success __retval(0)
__arch_x86_64
__arch_arm64
__naked void arena_ldsx_exception(void *ctx)
{
asm volatile (
"r1 = %[arena] ll;"
"r0 = 0xdeadbeef;"
"r0 = addr_space_cast(r0, 0x0, 0x1);"
"r1 = 0x3fe;"
"*(u64 *)(r0 + 0) = r1;"
"r0 = *(s8 *)(r0 + 0);"
"exit;"
:
: __imm_addr(arena)
: __clobber_all
);
}

SEC("syscall")
__description("Arena LDSX, S8")
__success __retval(-1)
__arch_x86_64
__arch_arm64
__naked void arena_ldsx_s8(void *ctx)
{
asm volatile (
"r1 = %[arena] ll;"
"r2 = 0;"
"r3 = 1;"
"r4 = %[numa_no_node];"
"r5 = 0;"
"call %[bpf_arena_alloc_pages];"
"r0 = addr_space_cast(r0, 0x0, 0x1);"
"r1 = 0x3fe;"
"*(u64 *)(r0 + 0) = r1;"
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
"r0 = *(s8 *)(r0 + 0);"
#else
"r0 = *(s8 *)(r0 + 7);"
#endif
"r0 >>= 1;"
"exit;"
:: __imm(bpf_arena_alloc_pages),
__imm_addr(arena),
__imm_const(numa_no_node, NUMA_NO_NODE)
: __clobber_all
);
}

SEC("syscall")
__description("Arena LDSX, S16")
__success __retval(-1)
__arch_x86_64
__arch_arm64
__naked void arena_ldsx_s16(void *ctx)
{
asm volatile (
"r1 = %[arena] ll;"
"r2 = 0;"
"r3 = 1;"
"r4 = %[numa_no_node];"
"r5 = 0;"
"call %[bpf_arena_alloc_pages];"
"r0 = addr_space_cast(r0, 0x0, 0x1);"
"r1 = 0x3fffe;"
"*(u64 *)(r0 + 0) = r1;"
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
"r0 = *(s16 *)(r0 + 0);"
#else
"r0 = *(s16 *)(r0 + 6);"
#endif
"r0 >>= 1;"
"exit;"
:: __imm(bpf_arena_alloc_pages),
__imm_addr(arena),
__imm_const(numa_no_node, NUMA_NO_NODE)
: __clobber_all
);
}

SEC("syscall")
__description("Arena LDSX, S32")
__success __retval(-1)
__arch_x86_64
__arch_arm64
__naked void arena_ldsx_s32(void *ctx)
{
asm volatile (
"r1 = %[arena] ll;"
"r2 = 0;"
"r3 = 1;"
"r4 = %[numa_no_node];"
"r5 = 0;"
"call %[bpf_arena_alloc_pages];"
"r0 = addr_space_cast(r0, 0x0, 0x1);"
"r1 = 0xfffffffe;"
"*(u64 *)(r0 + 0) = r1;"
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
"r0 = *(s32 *)(r0 + 0);"
#else
"r0 = *(s32 *)(r0 + 4);"
#endif
"r0 >>= 1;"
"exit;"
:: __imm(bpf_arena_alloc_pages),
__imm_addr(arena),
__imm_const(numa_no_node, NUMA_NO_NODE)
: __clobber_all
);
}

/* to retain debug info for BTF generation */
void kfunc_root(void)
{
bpf_arena_alloc_pages(0, 0, 0, 0, 0);
}

#else

SEC("socket")
Expand Down
Loading