diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-06-04 02:27:18 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-06-04 02:27:18 +0300 |
commit | cb8e59cc87201af93dfbb6c3dccc8fcad72a09c2 (patch) | |
tree | a334db9022f89654b777bbce8c4c6632e65b9031 /arch/riscv | |
parent | 2e63f6ce7ed2c4ff83ba30ad9ccad422289a6c63 (diff) | |
parent | 065fcfd49763ec71ae345bb5c5a74f961031e70e (diff) | |
download | linux-cb8e59cc87201af93dfbb6c3dccc8fcad72a09c2.tar.xz |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next
Pull networking updates from David Miller:
1) Allow setting bluetooth L2CAP modes via socket option, from Luiz
Augusto von Dentz.
2) Add GSO partial support to igc, from Sasha Neftin.
3) Several cleanups and improvements to r8169 from Heiner Kallweit.
4) Add IF_OPER_TESTING link state and use it when ethtool triggers a
device self-test. From Andrew Lunn.
5) Start moving away from custom driver versions, use the globally
defined kernel version instead, from Leon Romanovsky.
6) Support GRO vis gro_cells in DSA layer, from Alexander Lobakin.
7) Allow hard IRQ deferral during NAPI, from Eric Dumazet.
8) Add sriov and vf support to hinic, from Luo bin.
9) Support Media Redundancy Protocol (MRP) in the bridging code, from
Horatiu Vultur.
10) Support netmap in the nft_nat code, from Pablo Neira Ayuso.
11) Allow UDPv6 encapsulation of ESP in the ipsec code, from Sabrina
Dubroca. Also add ipv6 support for espintcp.
12) Lots of ReST conversions of the networking documentation, from Mauro
Carvalho Chehab.
13) Support configuration of ethtool rxnfc flows in bcmgenet driver,
from Doug Berger.
14) Allow to dump cgroup id and filter by it in inet_diag code, from
Dmitry Yakunin.
15) Add infrastructure to export netlink attribute policies to
userspace, from Johannes Berg.
16) Several optimizations to sch_fq scheduler, from Eric Dumazet.
17) Fallback to the default qdisc if qdisc init fails because otherwise
a packet scheduler init failure will make a device inoperative. From
Jesper Dangaard Brouer.
18) Several RISCV bpf jit optimizations, from Luke Nelson.
19) Correct the return type of the ->ndo_start_xmit() method in several
drivers, it's netdev_tx_t but many drivers were using
'int'. From Yunjian Wang.
20) Add an ethtool interface for PHY master/slave config, from Oleksij
Rempel.
21) Add BPF iterators, from Yonghang Song.
22) Add cable test infrastructure, including ethool interfaces, from
Andrew Lunn. Marvell PHY driver is the first to support this
facility.
23) Remove zero-length arrays all over, from Gustavo A. R. Silva.
24) Calculate and maintain an explicit frame size in XDP, from Jesper
Dangaard Brouer.
25) Add CAP_BPF, from Alexei Starovoitov.
26) Support terse dumps in the packet scheduler, from Vlad Buslov.
27) Support XDP_TX bulking in dpaa2 driver, from Ioana Ciornei.
28) Add devm_register_netdev(), from Bartosz Golaszewski.
29) Minimize qdisc resets, from Cong Wang.
30) Get rid of kernel_getsockopt and kernel_setsockopt in order to
eliminate set_fs/get_fs calls. From Christoph Hellwig.
* git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (2517 commits)
selftests: net: ip_defrag: ignore EPERM
net_failover: fixed rollback in net_failover_open()
Revert "tipc: Fix potential tipc_aead refcnt leak in tipc_crypto_rcv"
Revert "tipc: Fix potential tipc_node refcnt leak in tipc_rcv"
vmxnet3: allow rx flow hash ops only when rss is enabled
hinic: add set_channels ethtool_ops support
selftests/bpf: Add a default $(CXX) value
tools/bpf: Don't use $(COMPILE.c)
bpf, selftests: Use bpf_probe_read_kernel
s390/bpf: Use bcr 0,%0 as tail call nop filler
s390/bpf: Maintain 8-byte stack alignment
selftests/bpf: Fix verifier test
selftests/bpf: Fix sample_cnt shared between two threads
bpf, selftests: Adapt cls_redirect to call csum_level helper
bpf: Add csum_level helper for fixing up csum levels
bpf: Fix up bpf_skb_adjust_room helper's skb csum setting
sfc: add missing annotation for efx_ef10_try_update_nic_stats_vf()
crypto/chtls: IPv6 support for inline TLS
Crypto/chcr: Fixes a coccinile check error
Crypto/chcr: Fixes compilations warnings
...
Diffstat (limited to 'arch/riscv')
-rw-r--r-- | arch/riscv/net/bpf_jit_comp32.c | 103 | ||||
-rw-r--r-- | arch/riscv/net/bpf_jit_comp64.c | 64 |
2 files changed, 112 insertions, 55 deletions
diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 302934177760..b198eaa74456 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -13,8 +13,35 @@ #include <linux/filter.h> #include "bpf_jit.h" +/* + * Stack layout during BPF program execution: + * + * high + * RV32 fp => +----------+ + * | saved ra | + * | saved fp | RV32 callee-saved registers + * | ... | + * +----------+ <= (fp - 4 * NR_SAVED_REGISTERS) + * | hi(R6) | + * | lo(R6) | + * | hi(R7) | JIT scratch space for BPF registers + * | lo(R7) | + * | ... | + * BPF_REG_FP => +----------+ <= (fp - 4 * NR_SAVED_REGISTERS + * | | - 4 * BPF_JIT_SCRATCH_REGS) + * | | + * | ... | BPF program stack + * | | + * RV32 sp => +----------+ + * | | + * | ... | Function call stack + * | | + * +----------+ + * low + */ + enum { - /* Stack layout - these are offsets from (top of stack - 4). */ + /* Stack layout - these are offsets from top of JIT scratch space. */ BPF_R6_HI, BPF_R6_LO, BPF_R7_HI, @@ -29,7 +56,11 @@ enum { BPF_JIT_SCRATCH_REGS, }; -#define STACK_OFFSET(k) (-4 - ((k) * 4)) +/* Number of callee-saved registers stored to stack: ra, fp, s1--s7. */ +#define NR_SAVED_REGISTERS 9 + +/* Offset from fp for BPF registers stored on stack. */ +#define STACK_OFFSET(k) (-4 - (4 * NR_SAVED_REGISTERS) - (4 * (k))) #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) @@ -111,11 +142,9 @@ static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo, static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) { - int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 4; + int stack_adjust = ctx->stack_size; const s8 *r0 = bpf2rv32[BPF_REG_0]; - store_offset -= 4 * BPF_JIT_SCRATCH_REGS; - /* Set return value if not tail call. */ if (!is_tail_call) { emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx); @@ -123,15 +152,15 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) } /* Restore callee-saved registers. */ - emit(rv_lw(RV_REG_RA, store_offset - 0, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_FP, store_offset - 4, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S1, store_offset - 8, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S2, store_offset - 12, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S3, store_offset - 16, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S4, store_offset - 20, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S5, store_offset - 24, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S6, store_offset - 28, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S7, store_offset - 32, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_RA, stack_adjust - 4, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_FP, stack_adjust - 8, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S1, stack_adjust - 12, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S2, stack_adjust - 16, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S3, stack_adjust - 20, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S4, stack_adjust - 24, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S5, stack_adjust - 28, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S6, stack_adjust - 32, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S7, stack_adjust - 36, RV_REG_SP), ctx); emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); @@ -770,12 +799,13 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); /* - * if ((temp_tcc = tcc - 1) < 0) + * temp_tcc = tcc - 1; + * if (tcc < 0) * goto out; */ emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - emit_bcc(BPF_JSLT, RV_REG_T1, RV_REG_ZERO, off, ctx); + emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); /* * prog = array->ptrs[index]; @@ -1259,17 +1289,20 @@ notsupported: void bpf_jit_build_prologue(struct rv_jit_context *ctx) { - /* Make space to save 9 registers: ra, fp, s1--s7. */ - int stack_adjust = 9 * sizeof(u32), store_offset, bpf_stack_adjust; const s8 *fp = bpf2rv32[BPF_REG_FP]; const s8 *r1 = bpf2rv32[BPF_REG_1]; - - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + int stack_adjust = 0; + int bpf_stack_adjust = + round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + + /* Make space for callee-saved registers. */ + stack_adjust += NR_SAVED_REGISTERS * sizeof(u32); + /* Make space for BPF registers on stack. */ + stack_adjust += BPF_JIT_SCRATCH_REGS * sizeof(u32); + /* Make space for BPF stack. */ stack_adjust += bpf_stack_adjust; - - store_offset = stack_adjust - 4; - - stack_adjust += 4 * BPF_JIT_SCRATCH_REGS; + /* Round up for stack alignment. */ + stack_adjust = round_up(stack_adjust, STACK_ALIGN); /* * The first instruction sets the tail-call-counter (TCC) register. @@ -1280,24 +1313,24 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); /* Save callee-save registers. */ - emit(rv_sw(RV_REG_SP, store_offset - 0, RV_REG_RA), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 4, RV_REG_FP), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 8, RV_REG_S1), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 12, RV_REG_S2), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 16, RV_REG_S3), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 20, RV_REG_S4), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 24, RV_REG_S5), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 28, RV_REG_S6), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 32, RV_REG_S7), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 4, RV_REG_RA), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 8, RV_REG_FP), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 12, RV_REG_S1), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 16, RV_REG_S2), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 20, RV_REG_S3), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 24, RV_REG_S4), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 28, RV_REG_S5), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 32, RV_REG_S6), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 36, RV_REG_S7), ctx); /* Set fp: used as the base address for stacked BPF registers. */ emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); - /* Set up BPF stack pointer. */ + /* Set up BPF frame pointer. */ emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx); emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx); - /* Set up context pointer. */ + /* Set up BPF context pointer. */ emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx); emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index d208a9fd6c52..6cfd164cbe88 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -515,7 +515,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_LSH | BPF_X: emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); - if (!is64) + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_X: @@ -542,13 +542,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* dst = BSWAP##imm(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: - { - int shift = 64 - imm; - - emit(rv_slli(rd, rd, shift), ctx); - emit(rv_srli(rd, rd, shift), ctx); + switch (imm) { + case 16: + emit(rv_slli(rd, rd, 48), ctx); + emit(rv_srli(rd, rd, 48), ctx); + break; + case 32: + if (!aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case 64: + /* Do nothing */ + break; + } break; - } + case BPF_ALU | BPF_END | BPF_FROM_BE: emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); @@ -692,19 +700,19 @@ out_be: case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_LSH | BPF_K: emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); - if (!is64) + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); - if (!is64) + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_ARSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); - if (!is64) + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -784,11 +792,15 @@ out_be: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP | BPF_JSLE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - case BPF_JMP | BPF_JSET | BPF_K: - case BPF_JMP32 | BPF_JSET | BPF_K: rvoff = rv_offset(i, off, ctx); s = ctx->ninsns; - emit_imm(RV_REG_T1, imm, ctx); + if (imm) { + emit_imm(RV_REG_T1, imm, ctx); + rs = RV_REG_T1; + } else { + /* If imm is 0, simply use zero register. */ + rs = RV_REG_ZERO; + } if (!is64) { if (is_signed_bpf_cond(BPF_OP(code))) emit_sext_32_rd(&rd, ctx); @@ -799,16 +811,28 @@ out_be: /* Adjust for extra insns */ rvoff -= (e - s) << 2; + emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); + break; - if (BPF_OP(code) == BPF_JSET) { - /* Adjust for and */ - rvoff -= 4; - emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); - emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, - ctx); + case BPF_JMP | BPF_JSET | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_K: + rvoff = rv_offset(i, off, ctx); + s = ctx->ninsns; + if (is_12b_int(imm)) { + emit(rv_andi(RV_REG_T1, rd, imm), ctx); } else { - emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); + emit_imm(RV_REG_T1, imm, ctx); + emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); } + /* For jset32, we should clear the upper 32 bits of t1, but + * sign-extension is sufficient here and saves one instruction, + * as t1 is used only in comparison against zero. + */ + if (!is64 && imm < 0) + emit(rv_addiw(RV_REG_T1, RV_REG_T1, 0), ctx); + e = ctx->ninsns; + rvoff -= (e - s) << 2; + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); break; /* function call */ |