summaryrefslogtreecommitdiff
path: root/arch/x86/net
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/net')
-rw-r--r--arch/x86/net/bpf_jit.S20
-rw-r--r--arch/x86/net/bpf_jit_comp.c66
2 files changed, 45 insertions, 41 deletions
diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S
index f2a7faf4706e..b33093f84528 100644
--- a/arch/x86/net/bpf_jit.S
+++ b/arch/x86/net/bpf_jit.S
@@ -19,9 +19,6 @@
*/
#define SKBDATA %r10
#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
-#define MAX_BPF_STACK (512 /* from filter.h */ + \
- 32 /* space for rbx,r13,r14,r15 */ + \
- 8 /* space for skb_copy_bits */)
#define FUNC(name) \
.globl name; \
@@ -66,7 +63,7 @@ FUNC(sk_load_byte_positive_offset)
/* rsi contains offset and can be scratched */
#define bpf_slow_path_common(LEN) \
- lea -MAX_BPF_STACK + 32(%rbp), %rdx;\
+ lea 32(%rbp), %rdx;\
FRAME_BEGIN; \
mov %rbx, %rdi; /* arg1 == skb */ \
push %r9; \
@@ -83,14 +80,14 @@ FUNC(sk_load_byte_positive_offset)
bpf_slow_path_word:
bpf_slow_path_common(4)
js bpf_error
- mov - MAX_BPF_STACK + 32(%rbp),%eax
+ mov 32(%rbp),%eax
bswap %eax
ret
bpf_slow_path_half:
bpf_slow_path_common(2)
js bpf_error
- mov - MAX_BPF_STACK + 32(%rbp),%ax
+ mov 32(%rbp),%ax
rol $8,%ax
movzwl %ax,%eax
ret
@@ -98,7 +95,7 @@ bpf_slow_path_half:
bpf_slow_path_byte:
bpf_slow_path_common(1)
js bpf_error
- movzbl - MAX_BPF_STACK + 32(%rbp),%eax
+ movzbl 32(%rbp),%eax
ret
#define sk_negative_common(SIZE) \
@@ -148,9 +145,10 @@ FUNC(sk_load_byte_negative_offset)
bpf_error:
# force a return 0 from jit handler
xor %eax,%eax
- mov - MAX_BPF_STACK(%rbp),%rbx
- mov - MAX_BPF_STACK + 8(%rbp),%r13
- mov - MAX_BPF_STACK + 16(%rbp),%r14
- mov - MAX_BPF_STACK + 24(%rbp),%r15
+ mov (%rbp),%rbx
+ mov 8(%rbp),%r13
+ mov 16(%rbp),%r14
+ mov 24(%rbp),%r15
+ add $40, %rbp
leaveq
ret
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index f58939393eef..e1324f280e06 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -197,17 +197,16 @@ struct jit_context {
#define BPF_MAX_INSN_SIZE 128
#define BPF_INSN_SAFETY 64
-#define STACKSIZE \
- (MAX_BPF_STACK + \
- 32 /* space for rbx, r13, r14, r15 */ + \
+#define AUX_STACK_SPACE \
+ (32 /* space for rbx, r13, r14, r15 */ + \
8 /* space for skb_copy_bits() buffer */)
-#define PROLOGUE_SIZE 48
+#define PROLOGUE_SIZE 37
/* emit x64 prologue code for BPF program and check it's size.
* bpf_tail_call helper will skip it while jumping into another program
*/
-static void emit_prologue(u8 **pprog)
+static void emit_prologue(u8 **pprog, u32 stack_depth)
{
u8 *prog = *pprog;
int cnt = 0;
@@ -215,13 +214,17 @@ static void emit_prologue(u8 **pprog)
EMIT1(0x55); /* push rbp */
EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */
- /* sub rsp, STACKSIZE */
- EMIT3_off32(0x48, 0x81, 0xEC, STACKSIZE);
+ /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */
+ EMIT3_off32(0x48, 0x81, 0xEC,
+ round_up(stack_depth, 8) + AUX_STACK_SPACE);
+
+ /* sub rbp, AUX_STACK_SPACE */
+ EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
/* all classic BPF filters use R6(rbx) save it */
- /* mov qword ptr [rbp-X],rbx */
- EMIT3_off32(0x48, 0x89, 0x9D, -STACKSIZE);
+ /* mov qword ptr [rbp+0],rbx */
+ EMIT4(0x48, 0x89, 0x5D, 0);
/* bpf_convert_filter() maps classic BPF register X to R7 and uses R8
* as temporary, so all tcpdump filters need to spill/fill R7(r13) and
@@ -231,12 +234,12 @@ static void emit_prologue(u8 **pprog)
* than synthetic ones. Therefore not worth adding complexity.
*/
- /* mov qword ptr [rbp-X],r13 */
- EMIT3_off32(0x4C, 0x89, 0xAD, -STACKSIZE + 8);
- /* mov qword ptr [rbp-X],r14 */
- EMIT3_off32(0x4C, 0x89, 0xB5, -STACKSIZE + 16);
- /* mov qword ptr [rbp-X],r15 */
- EMIT3_off32(0x4C, 0x89, 0xBD, -STACKSIZE + 24);
+ /* mov qword ptr [rbp+8],r13 */
+ EMIT4(0x4C, 0x89, 0x6D, 8);
+ /* mov qword ptr [rbp+16],r14 */
+ EMIT4(0x4C, 0x89, 0x75, 16);
+ /* mov qword ptr [rbp+24],r15 */
+ EMIT4(0x4C, 0x89, 0x7D, 24);
/* Clear the tail call counter (tail_call_cnt): for eBPF tail calls
* we need to reset the counter to 0. It's done in two instructions,
@@ -246,8 +249,8 @@ static void emit_prologue(u8 **pprog)
/* xor eax, eax */
EMIT2(0x31, 0xc0);
- /* mov qword ptr [rbp-X], rax */
- EMIT3_off32(0x48, 0x89, 0x85, -STACKSIZE + 32);
+ /* mov qword ptr [rbp+32], rax */
+ EMIT4(0x48, 0x89, 0x45, 32);
BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
*pprog = prog;
@@ -289,13 +292,13 @@ static void emit_bpf_tail_call(u8 **pprog)
/* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
* goto out;
*/
- EMIT2_off32(0x8B, 0x85, -STACKSIZE + 36); /* mov eax, dword ptr [rbp - 516] */
+ EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */
EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
#define OFFSET2 36
EMIT2(X86_JA, OFFSET2); /* ja out */
label2 = cnt;
EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
- EMIT2_off32(0x89, 0x85, -STACKSIZE + 36); /* mov dword ptr [rbp - 516], eax */
+ EMIT2_off32(0x89, 0x85, 36); /* mov dword ptr [rbp + 36], eax */
/* prog = array->ptrs[index]; */
EMIT4_off32(0x48, 0x8D, 0x84, 0xD6, /* lea rax, [rsi + rdx * 8 + offsetof(...)] */
@@ -361,7 +364,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
int proglen = 0;
u8 *prog = temp;
- emit_prologue(&prog);
+ emit_prologue(&prog, bpf_prog->aux->stack_depth);
if (seen_ld_abs)
emit_load_skb_data_hlen(&prog);
@@ -877,7 +880,7 @@ xadd: if (is_imm8(insn->off))
}
break;
- case BPF_JMP | BPF_CALL | BPF_X:
+ case BPF_JMP | BPF_TAIL_CALL:
emit_bpf_tail_call(&prog);
break;
@@ -1036,15 +1039,17 @@ common_load:
seen_exit = true;
/* update cleanup_addr */
ctx->cleanup_addr = proglen;
- /* mov rbx, qword ptr [rbp-X] */
- EMIT3_off32(0x48, 0x8B, 0x9D, -STACKSIZE);
- /* mov r13, qword ptr [rbp-X] */
- EMIT3_off32(0x4C, 0x8B, 0xAD, -STACKSIZE + 8);
- /* mov r14, qword ptr [rbp-X] */
- EMIT3_off32(0x4C, 0x8B, 0xB5, -STACKSIZE + 16);
- /* mov r15, qword ptr [rbp-X] */
- EMIT3_off32(0x4C, 0x8B, 0xBD, -STACKSIZE + 24);
-
+ /* mov rbx, qword ptr [rbp+0] */
+ EMIT4(0x48, 0x8B, 0x5D, 0);
+ /* mov r13, qword ptr [rbp+8] */
+ EMIT4(0x4C, 0x8B, 0x6D, 8);
+ /* mov r14, qword ptr [rbp+16] */
+ EMIT4(0x4C, 0x8B, 0x75, 16);
+ /* mov r15, qword ptr [rbp+24] */
+ EMIT4(0x4C, 0x8B, 0x7D, 24);
+
+ /* add rbp, AUX_STACK_SPACE */
+ EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
EMIT1(0xC9); /* leave */
EMIT1(0xC3); /* ret */
break;
@@ -1162,6 +1167,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
bpf_jit_binary_lock_ro(header);
prog->bpf_func = (void *)image;
prog->jited = 1;
+ prog->jited_len = proglen;
} else {
prog = orig_prog;
}
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-24 04:43:36 +0300