diff options
856 files changed, 45131 insertions, 13203 deletions
diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst index 609b71f5747d..de27e1620821 100644 --- a/Documentation/bpf/bpf_devel_QA.rst +++ b/Documentation/bpf/bpf_devel_QA.rst @@ -635,12 +635,12 @@ test coverage. Q: clang flag for target bpf? ----------------------------- -Q: In some cases clang flag ``-target bpf`` is used but in other cases the +Q: In some cases clang flag ``--target=bpf`` is used but in other cases the default clang target, which matches the underlying architecture, is used. What is the difference and when I should use which? A: Although LLVM IR generation and optimization try to stay architecture -independent, ``-target <arch>`` still has some impact on generated code: +independent, ``--target=<arch>`` still has some impact on generated code: - BPF program may recursively include header file(s) with file scope inline assembly codes. The default target can handle this well, @@ -658,7 +658,7 @@ independent, ``-target <arch>`` still has some impact on generated code: The clang option ``-fno-jump-tables`` can be used to disable switch table generation. -- For clang ``-target bpf``, it is guaranteed that pointer or long / +- For clang ``--target=bpf``, it is guaranteed that pointer or long / unsigned long types will always have a width of 64 bit, no matter whether underlying clang binary or default target (or kernel) is 32 bit. However, when native clang target is used, then it will @@ -668,7 +668,7 @@ independent, ``-target <arch>`` still has some impact on generated code: while the BPF LLVM back end still operates in 64 bit. The native target is mostly needed in tracing for the case of walking ``pt_regs`` or other kernel structures where CPU's register width matters. - Otherwise, ``clang -target bpf`` is generally recommended. + Otherwise, ``clang --target=bpf`` is generally recommended. You should use default target when: @@ -685,7 +685,7 @@ when: into these structures is verified by the BPF verifier and may result in verification failures if the native architecture is not aligned with the BPF architecture, e.g. 64-bit. An example of this is - BPF_PROG_TYPE_SK_MSG require ``-target bpf`` + BPF_PROG_TYPE_SK_MSG require ``--target=bpf`` .. Links diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 7cd7c5415a99..f32db1f44ae9 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -990,7 +990,7 @@ format.:: } g2; int main() { return 0; } int test() { return 0; } - -bash-4.4$ clang -c -g -O2 -target bpf t2.c + -bash-4.4$ clang -c -g -O2 --target=bpf t2.c -bash-4.4$ readelf -S t2.o ...... [ 8] .BTF PROGBITS 0000000000000000 00000247 @@ -1000,7 +1000,7 @@ format.:: [10] .rel.BTF.ext REL 0000000000000000 000007e0 0000000000000040 0000000000000010 16 9 8 ...... - -bash-4.4$ clang -S -g -O2 -target bpf t2.c + -bash-4.4$ clang -S -g -O2 --target=bpf t2.c -bash-4.4$ cat t2.s ...... .section .BTF,"",@progbits diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst index dbb39e8f9889..1ff177b89d66 100644 --- a/Documentation/bpf/index.rst +++ b/Documentation/bpf/index.rst @@ -12,9 +12,9 @@ that goes into great technical depth about the BPF Architecture. .. toctree:: :maxdepth: 1 - instruction-set verifier libbpf/index + standardization/index btf faq syscall_api @@ -29,7 +29,6 @@ that goes into great technical depth about the BPF Architecture. bpf_licensing test_debug clang-notes - linux-notes other redirect diff --git a/Documentation/bpf/llvm_reloc.rst b/Documentation/bpf/llvm_reloc.rst index e4a777a6a3a2..450e6403fe3d 100644 --- a/Documentation/bpf/llvm_reloc.rst +++ b/Documentation/bpf/llvm_reloc.rst @@ -28,7 +28,7 @@ For example, for the following code:: return g1 + g2 + l1 + l2; } -Compiled with ``clang -target bpf -O2 -c test.c``, the following is +Compiled with ``clang --target=bpf -O2 -c test.c``, the following is the code with ``llvm-objdump -dr test.o``:: 0: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0 ll @@ -157,7 +157,7 @@ and ``call`` instructions. For example:: return gfunc(a, b) + lfunc(a, b) + global; } -Compiled with ``clang -target bpf -O2 -c test.c``, we will have +Compiled with ``clang --target=bpf -O2 -c test.c``, we will have following code with `llvm-objdump -dr test.o``:: Disassembly of section .text: @@ -203,7 +203,7 @@ The following is an example to show how R_BPF_64_ABS64 could be generated:: int global() { return 0; } struct t { void *g; } gbl = { global }; -Compiled with ``clang -target bpf -O2 -g -c test.c``, we will see a +Compiled with ``clang --target=bpf -O2 -g -c test.c``, we will see a relocation below in ``.data`` section with command ``llvm-readelf -r test.o``:: diff --git a/Documentation/bpf/standardization/index.rst b/Documentation/bpf/standardization/index.rst new file mode 100644 index 000000000000..09c6ba055fd7 --- /dev/null +++ b/Documentation/bpf/standardization/index.rst @@ -0,0 +1,18 @@ +.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +=================== +BPF Standardization +=================== + +This directory contains documents that are being iterated on as part of the BPF +standardization effort with the IETF. See the `IETF BPF Working Group`_ page +for the working group charter, documents, and more. + +.. toctree:: + :maxdepth: 1 + + instruction-set + linux-notes + +.. Links: +.. _IETF BPF Working Group: https://datatracker.ietf.org/wg/bpf/about/ diff --git a/Documentation/bpf/instruction-set.rst b/Documentation/bpf/standardization/instruction-set.rst index 6644842cd3ea..751e657973f0 100644 --- a/Documentation/bpf/instruction-set.rst +++ b/Documentation/bpf/standardization/instruction-set.rst @@ -165,7 +165,7 @@ BPF_OR 0x40 dst \|= src BPF_AND 0x50 dst &= src BPF_LSH 0x60 dst <<= (src & mask) BPF_RSH 0x70 dst >>= (src & mask) -BPF_NEG 0x80 dst = ~src +BPF_NEG 0x80 dst = -src BPF_MOD 0x90 dst = (src != 0) ? (dst % src) : dst BPF_XOR 0xa0 dst ^= src BPF_MOV 0xb0 dst = src diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/standardization/linux-notes.rst index 508d009d3bed..00d2693de025 100644 --- a/Documentation/bpf/linux-notes.rst +++ b/Documentation/bpf/standardization/linux-notes.rst @@ -45,7 +45,8 @@ On Linux, this integer is a BTF ID. Legacy BPF Packet access instructions ===================================== -As mentioned in the `ISA standard documentation <instruction-set.rst#legacy-bpf-packet-access-instructions>`_, +As mentioned in the `ISA standard documentation +<instruction-set.html#legacy-bpf-packet-access-instructions>`_, Linux has special eBPF instructions for access to packet data that have been carried over from classic BPF to retain the performance of legacy socket filters running in the eBPF interpreter. diff --git a/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml b/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml new file mode 100644 index 000000000000..aa3162c74833 --- /dev/null +++ b/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/brcm,asp-v2.0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom ASP 2.0 Ethernet controller + +maintainers: + - Justin Chen <justin.chen@broadcom.com> + - Florian Fainelli <florian.fainelli@broadcom.com> + +description: Broadcom Ethernet controller first introduced with 72165 + +properties: + compatible: + oneOf: + - items: + - enum: + - brcm,bcm74165-asp + - const: brcm,asp-v2.1 + - items: + - enum: + - brcm,bcm72165-asp + - const: brcm,asp-v2.0 + + "#address-cells": + const: 1 + "#size-cells": + const: 1 + + reg: + maxItems: 1 + + ranges: true + + interrupts: + minItems: 1 + items: + - description: RX/TX interrupt + - description: Port 0 Wake-on-LAN + - description: Port 1 Wake-on-LAN + + clocks: + maxItems: 1 + + ethernet-ports: + type: object + properties: + "#address-cells": + const: 1 + "#size-cells": + const: 0 + + patternProperties: + "^port@[0-9]+$": + type: object + + $ref: ethernet-controller.yaml# + + unevaluatedProperties: false + + properties: + reg: + maxItems: 1 + description: Port number + + brcm,channel: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + ASP Channel Number + + The depacketizer channel that consumes packets from + the unimac/port. + + required: + - reg + - brcm,channel + + additionalProperties: false + +patternProperties: + "^mdio@[0-9a-f]+$": + type: object + $ref: brcm,unimac-mdio.yaml + + description: + ASP internal UniMAC MDIO bus + +required: + - compatible + - reg + - interrupts + - clocks + - ranges + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + ethernet@9c00000 { + compatible = "brcm,bcm72165-asp", "brcm,asp-v2.0"; + reg = <0x9c00000 0x1fff14>; + interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>; + ranges = <0x0 0x9c00000 0x1fff14>; + clocks = <&scmi 14>; + #address-cells = <1>; + #size-cells = <1>; + + mdio@c614 { + compatible = "brcm,asp-v2.0-mdio"; + reg = <0xc614 0x8>; + reg-names = "mdio"; + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@1 { + reg = <1>; + }; + }; + + mdio@ce14 { + compatible = "brcm,asp-v2.0-mdio"; + reg = <0xce14 0x8>; + reg-names = "mdio"; + #address-cells = <1>; + #size-cells = <0>; + + phy1: ethernet-phy@1 { + reg = <1>; + }; + }; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + brcm,channel = <8>; + phy-mode = "rgmii"; + phy-handle = <&phy0>; + }; + + port@1 { + reg = <1>; + brcm,channel = <9>; + phy-mode = "rgmii"; + phy-handle = <&phy1>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/brcm,unimac-mdio.yaml b/Documentation/devicetree/bindings/net/brcm,unimac-mdio.yaml index 0be426ee1e44..6684810fcbf0 100644 --- a/Documentation/devicetree/bindings/net/brcm,unimac-mdio.yaml +++ b/Documentation/devicetree/bindings/net/brcm,unimac-mdio.yaml @@ -22,6 +22,8 @@ properties: - brcm,genet-mdio-v3 - brcm,genet-mdio-v4 - brcm,genet-mdio-v5 + - brcm,asp-v2.0-mdio + - brcm,asp-v2.1-mdio - brcm,unimac-mdio reg: diff --git a/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml b/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml index 9c494957a07a..e42ea28d6ab4 100644 --- a/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml +++ b/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml @@ -21,6 +21,7 @@ properties: - const: allwinner,sun4i-a10-can - const: allwinner,sun4i-a10-can - const: allwinner,sun8i-r40-can + - const: allwinner,sun20i-d1-can reg: maxItems: 1 @@ -37,8 +38,9 @@ properties: if: properties: compatible: - contains: - const: allwinner,sun8i-r40-can + enum: + - allwinner,sun8i-r40-can + - allwinner,sun20i-d1-can then: required: diff --git a/Documentation/devicetree/bindings/net/can/bosch,m_can.yaml b/Documentation/devicetree/bindings/net/can/bosch,m_can.yaml index 67879aab623b..bb518c831f7b 100644 --- a/Documentation/devicetree/bindings/net/can/bosch,m_can.yaml +++ b/Documentation/devicetree/bindings/net/can/bosch,m_can.yaml @@ -122,8 +122,6 @@ required: - compatible - reg - reg-names - - interrupts - - interrupt-names - clocks - clock-names - bosch,mram-cfg @@ -132,6 +130,7 @@ additionalProperties: false examples: - | + // Example with interrupts #include <dt-bindings/clock/imx6sx-clock.h> can@20e8000 { compatible = "bosch,m_can"; @@ -149,4 +148,21 @@ examples: }; }; + - | + // Example with timer polling + #include <dt-bindings/clock/imx6sx-clock.h> + can@20e8000 { + compatible = "bosch,m_can"; + reg = <0x020e8000 0x4000>, <0x02298000 0x4000>; + reg-names = "m_can", "message_ram"; + clocks = <&clks IMX6SX_CLK_CANFD>, + <&clks IMX6SX_CLK_CANFD>; + clock-names = "hclk", "cclk"; + bosch,mram-cfg = <0x0 0 0 32 0 0 0 1>; + + can-transceiver { + max-bitrate = <5000000>; + }; + }; + ... diff --git a/Documentation/devicetree/bindings/net/can/xilinx,can.yaml b/Documentation/devicetree/bindings/net/can/xilinx,can.yaml index 897d2cbda45b..64d57c343e6f 100644 --- a/Documentation/devicetree/bindings/net/can/xilinx,can.yaml +++ b/Documentation/devicetree/bindings/net/can/xilinx,can.yaml @@ -46,6 +46,9 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: CAN Tx mailbox buffer count (CAN FD) + resets: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.yaml b/Documentation/devicetree/bindings/net/dsa/dsa.yaml index 8d971813bab6..ec74a660beda 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa.yaml +++ b/Documentation/devicetree/bindings/net/dsa/dsa.yaml @@ -36,7 +36,7 @@ additionalProperties: true $defs: ethernet-ports: description: A DSA switch without any extra port properties - $ref: '#/' + $ref: '#' patternProperties: "^(ethernet-)?ports$": diff --git a/Documentation/devicetree/bindings/net/mediatek,net.yaml b/Documentation/devicetree/bindings/net/mediatek,net.yaml index 31cc0c412805..e74502a0afe8 100644 --- a/Documentation/devicetree/bindings/net/mediatek,net.yaml +++ b/Documentation/devicetree/bindings/net/mediatek,net.yaml @@ -19,10 +19,12 @@ properties: enum: - mediatek,mt2701-eth - mediatek,mt7623-eth + - mediatek,mt7621-eth - mediatek,mt7622-eth - mediatek,mt7629-eth - mediatek,mt7981-eth - mediatek,mt7986-eth + - mediatek,mt7988-eth - ralink,rt5350-eth reg: @@ -32,7 +34,7 @@ properties: clock-names: true interrupts: - minItems: 3 + minItems: 1 maxItems: 4 power-domains: @@ -60,6 +62,12 @@ properties: Phandle to the mediatek hifsys controller used to provide various clocks and reset to the system. + mediatek,infracfg: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the syscon node that handles the path from GMAC to + PHY variants. + mediatek,sgmiisys: $ref: /schemas/types.yaml#/definitions/phandle-array minItems: 1 @@ -121,6 +129,8 @@ allOf: - const: gp1 - const: gp2 + mediatek,infracfg: false + mediatek,pctl: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -135,6 +145,32 @@ allOf: properties: compatible: contains: + enum: + - mediatek,mt7621-eth + then: + properties: + interrupts: + maxItems: 1 + + clocks: + minItems: 2 + maxItems: 2 + + clock-names: + items: + - const: ethif + - const: fe + + mediatek,infracfg: false + + mediatek,wed: false + + mediatek,wed-pcie: false + + - if: + properties: + compatible: + contains: const: mediatek,mt7622-eth then: properties: @@ -159,6 +195,8 @@ allOf: - const: sgmii_ck - const: eth2pll + mediatek,infracfg: false + mediatek,sgmiisys: minItems: 1 maxItems: 1 @@ -204,12 +242,6 @@ allOf: - const: sgmii_ck - const: eth2pll - mediatek,infracfg: - $ref: /schemas/types.yaml#/definitions/phandle - description: - Phandle to the syscon node that handles the path from GMAC to - PHY variants. - mediatek,sgmiisys: minItems: 2 maxItems: 2 @@ -250,6 +282,8 @@ allOf: - const: netsys0 - const: netsys1 + mediatek,infracfg: false + mediatek,sgmiisys: minItems: 2 maxItems: 2 @@ -286,6 +320,67 @@ allOf: - const: netsys0 - const: netsys1 + mediatek,infracfg: false + + mediatek,sgmiisys: + minItems: 2 + maxItems: 2 + + - if: + properties: + compatible: + contains: + const: mediatek,mt7988-eth + then: + properties: + interrupts: + minItems: 4 + + clocks: + minItems: 34 + maxItems: 34 + + clock-names: + items: + - const: crypto + - const: fe + - const: gp2 + - const: gp1 + - const: gp3 + - const: ethwarp_wocpu2 + - const: ethwarp_wocpu1 + - const: ethwarp_wocpu0 + - const: esw + - const: netsys0 + - const: netsys1 + - const: sgmii_tx250m + - const: sgmii_rx250m + - const: sgmii2_tx250m + - const: sgmii2_rx250m + - const: top_usxgmii0_sel + - const: top_usxgmii1_sel + - const: top_sgm0_sel + - const: top_sgm1_sel + - const: top_xfi_phy0_xtal_sel + - const: top_xfi_phy1_xtal_sel + - const: top_eth_gmii_sel + - const: top_eth_refck_50m_sel + - const: top_eth_sys_200m_sel + - const: top_eth_sys_sel + - const: top_eth_xgmii_sel + - const: top_eth_mii_sel + - const: top_netsys_sel + - const: top_netsys_500m_sel + - const: top_netsys_pao_2x_sel + - const: top_netsys_sync_250m_sel + - const: top_netsys_ppefb_250m_sel + - const: top_netsys_warp_sel + - const: wocpu1 + - const: wocpu0 + - const: xgp1 + - const: xgp2 + - const: xgp3 + mediatek,sgmiisys: minItems: 2 maxItems: 2 diff --git a/Documentation/devicetree/bindings/net/motorcomm,yt8xxx.yaml b/Documentation/devicetree/bindings/net/motorcomm,yt8xxx.yaml index 157e3bbcaf6f..26688e2302ea 100644 --- a/Documentation/devicetree/bindings/net/motorcomm,yt8xxx.yaml +++ b/Documentation/devicetree/bindings/net/motorcomm,yt8xxx.yaml @@ -52,6 +52,40 @@ properties: for a timer. type: boolean + motorcomm,rx-clk-drv-microamp: + description: | + drive strength of rx_clk rgmii pad. + The YT8531 RGMII LDO voltage supports 1.8V/3.3V, and the LDO voltage can + be configured with hardware pull-up resistors to match the SOC voltage + (usually 1.8V). + The software can read the registers to obtain the LDO voltage and configure + the legal drive strength(curren). + ===================================================== + | voltage | current Available (uA) | + | 1.8v | 1200 2100 2700 2910 3110 3600 3970 4350 | + | 3.3v | 3070 4080 4370 4680 5020 5450 5740 6140 | + ===================================================== + enum: [ 1200, 2100, 2700, 2910, 3070, 3110, 3600, 3970, + 4080, 4350, 4370, 4680, 5020, 5450, 5740, 6140 ] + default: 2910 + + motorcomm,rx-data-drv-microamp: + description: | + drive strength of rx_data/rx_ctl rgmii pad. + The YT8531 RGMII LDO voltage supports 1.8V/3.3V, and the LDO voltage can + be configured with hardware pull-up resistors to match the SOC voltage + (usually 1.8V). + The software can read the registers to obtain the LDO voltage and configure + the legal drive strength(curren). + ===================================================== + | voltage | current Available (uA) | + | 1.8v | 1200 2100 2700 2910 3110 3600 3970 4350 | + | 3.3v | 3070 4080 4370 4680 5020 5450 5740 6140 | + ===================================================== + enum: [ 1200, 2100, 2700, 2910, 3070, 3110, 3600, 3970, + 4080, 4350, 4370, 4680, 5020, 5450, 5740, 6140 ] + default: 2910 + motorcomm,tx-clk-adj-enabled: description: | This configuration is mainly to adapt to VF2 with JH7110 SoC. diff --git a/Documentation/devicetree/bindings/net/oxnas-dwmac.txt b/Documentation/devicetree/bindings/net/oxnas-dwmac.txt deleted file mode 100644 index 27db496f1ce8..000000000000 --- a/Documentation/devicetree/bindings/net/oxnas-dwmac.txt +++ /dev/null @@ -1,41 +0,0 @@ -* Oxford Semiconductor OXNAS DWMAC Ethernet controller - -The device inherits all the properties of the dwmac/stmmac devices -described in the file stmmac.txt in the current directory with the -following changes. - -Required properties on all platforms: - -- compatible: For the OX820 SoC, it should be : - - "oxsemi,ox820-dwmac" to select glue - - "snps,dwmac-3.512" to select IP version. - For the OX810SE SoC, it should be : - - "oxsemi,ox810se-dwmac" to select glue - - "snps,dwmac-3.512" to select IP version. - -- clocks: Should contain phandles to the following clocks -- clock-names: Should contain the following: - - "stmmaceth" for the host clock - see stmmac.txt - - "gmac" for the peripheral gate clock - -- oxsemi,sys-ctrl: a phandle to the system controller syscon node - -Example : - -etha: ethernet@40400000 { - compatible = "oxsemi,ox820-dwmac", "snps,dwmac-3.512"; - reg = <0x40400000 0x2000>; - interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "macirq", "eth_wake_irq"; - mac-address = [000000000000]; /* Filled in by U-Boot */ - phy-mode = "rgmii"; - - clocks = <&stdclk CLK_820_ETHA>, <&gmacclk>; - clock-names = "gmac", "stmmaceth"; - resets = <&reset RESET_MAC>; - - /* Regmap for sys registers */ - oxsemi,sys-ctrl = <&sys>; - -}; diff --git a/Documentation/devicetree/bindings/net/qca,ar803x.yaml b/Documentation/devicetree/bindings/net/qca,ar803x.yaml index 161d28919316..3acd09f0da86 100644 --- a/Documentation/devicetree/bindings/net/qca,ar803x.yaml +++ b/Documentation/devicetree/bindings/net/qca,ar803x.yaml @@ -75,6 +75,7 @@ properties: description: Initial data for the VDDIO regulator. Set this to 1.5V or 1.8V. $ref: /schemas/regulator/regulator.yaml + unevaluatedProperties: false vddh-regulator: type: object @@ -82,6 +83,7 @@ properties: Dummy subnode to model the external connection of the PHY VDDH regulator to VDDIO. $ref: /schemas/regulator/regulator.yaml + unevaluatedProperties: false unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml index 7f324c6da915..70bbc4220e2a 100644 --- a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml +++ b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml @@ -80,6 +80,7 @@ properties: "output" means GMAC provides the reference clock. $ref: /schemas/types.yaml#/definitions/string enum: [input, output] + default: input rockchip,grf: description: The phandle of the syscon node for the general register file. diff --git a/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml b/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml new file mode 100644 index 000000000000..8ec30b3eb760 --- /dev/null +++ b/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/ti,icssg-prueth.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments ICSSG PRUSS Ethernet + +maintainers: + - Md Danish Anwar <danishanwar@ti.com> + +description: + Ethernet based on the Programmable Real-Time Unit and Industrial + Communication Subsystem. + +allOf: + - $ref: /schemas/remoteproc/ti,pru-consumer.yaml# + +properties: + compatible: + enum: + - ti,am654-icssg-prueth # for AM65x SoC family + + sram: + $ref: /schemas/types.yaml#/definitions/phandle + description: + phandle to MSMC SRAM node + + dmas: + maxItems: 10 + + dma-names: + items: + - const: tx0-0 + - const: tx0-1 + - const: tx0-2 + - const: tx0-3 + - const: tx1-0 + - const: tx1-1 + - const: tx1-2 + - const: tx1-3 + - const: rx0 + - const: rx1 + + ti,mii-g-rt: + $ref: /schemas/types.yaml#/definitions/phandle + description: + phandle to MII_G_RT module's syscon regmap. + + ti,mii-rt: + $ref: /schemas/types.yaml#/definitions/phandle + description: + phandle to MII_RT module's syscon regmap + + interrupts: + maxItems: 2 + description: + Interrupt specifiers to TX timestamp IRQ. + + interrupt-names: + items: + - const: tx_ts0 + - const: tx_ts1 + + ethernet-ports: + type: object + additionalProperties: false + + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + + patternProperties: + ^port@[0-1]$: + type: object + description: ICSSG PRUETH external ports + $ref: ethernet-controller.yaml# + unevaluatedProperties: false + + properties: + reg: + items: + - enum: [0, 1] + description: ICSSG PRUETH port number + + interrupts: + maxItems: 1 + + ti,syscon-rgmii-delay: + items: + - items: + - description: phandle to system controller node + - description: The offset to ICSSG control register + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + phandle to system controller node and register offset + to ICSSG control register for RGMII transmit delay + + required: + - reg + anyOf: + - required: + - port@0 + - required: + - port@1 + +required: + - compatible + - sram + - dmas + - dma-names + - ethernet-ports + - ti,mii-g-rt + - interrupts + - interrupt-names + +unevaluatedProperties: false + +examples: + - | + /* Example k3-am654 base board SR2.0, dual-emac */ + pruss2_eth: ethernet { + compatible = "ti,am654-icssg-prueth"; + pinctrl-names = "default"; + pinctrl-0 = <&icssg2_rgmii_pins_default>; + sram = <&msmc_ram>; + + ti,prus = <&pru2_0>, <&rtu2_0>, <&tx_pru2_0>, + <&pru2_1>, <&rtu2_1>, <&tx_pru2_1>; + firmware-name = "ti-pruss/am65x-pru0-prueth-fw.elf", + "ti-pruss/am65x-rtu0-prueth-fw.elf", + "ti-pruss/am65x-txpru0-prueth-fw.elf", + "ti-pruss/am65x-pru1-prueth-fw.elf", + "ti-pruss/am65x-rtu1-prueth-fw.elf", + "ti-pruss/am65x-txpru1-prueth-fw.elf"; + ti,pruss-gp-mux-sel = <2>, /* MII mode */ + <2>, + <2>, + <2>, /* MII mode */ + <2>, + <2>; + dmas = <&main_udmap 0xc300>, /* egress slice 0 */ + <&main_udmap 0xc301>, /* egress slice 0 */ + <&main_udmap 0xc302>, /* egress slice 0 */ + <&main_udmap 0xc303>, /* egress slice 0 */ + <&main_udmap 0xc304>, /* egress slice 1 */ + <&main_udmap 0xc305>, /* egress slice 1 */ + <&main_udmap 0xc306>, /* egress slice 1 */ + <&main_udmap 0xc307>, /* egress slice 1 */ + <&main_udmap 0x4300>, /* ingress slice 0 */ + <&main_udmap 0x4301>; /* ingress slice 1 */ + dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3", + "tx1-0", "tx1-1", "tx1-2", "tx1-3", + "rx0", "rx1"; + ti,mii-g-rt = <&icssg2_mii_g_rt>; + interrupt-parent = <&icssg2_intc>; + interrupts = <24 0 2>, <25 1 3>; + interrupt-names = "tx_ts0", "tx_ts1"; + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + pruss2_emac0: port@0 { + reg = <0>; + phy-handle = <&pruss2_eth0_phy>; + phy-mode = "rgmii-id"; + interrupts-extended = <&icssg2_intc 24>; + ti,syscon-rgmii-delay = <&scm_conf 0x4120>; + /* Filled in by bootloader */ + local-mac-address = [00 00 00 00 00 00]; + }; + + pruss2_emac1: port@1 { + reg = <1>; + phy-handle = <&pruss2_eth1_phy>; + phy-mode = "rgmii-id"; + interrupts-extended = <&icssg2_intc 25>; + ti,syscon-rgmii-delay = <&scm_conf 0x4124>; + /* Filled in by bootloader */ + local-mac-address = [00 00 00 00 00 00]; + }; + }; + }; diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index b99e7ffef7a1..1c7284fd535b 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -14,7 +14,7 @@ definitions: - name: basic doc: - XDP feautues set supported by all drivers + XDP features set supported by all drivers (XDP_ABORTED, XDP_DROP, XDP_PASS, XDP_TX) - name: redirect @@ -62,6 +62,12 @@ attribute-sets: type: u64 enum: xdp-act enum-as-flags: true + - + name: xdp-zc-max-segs + doc: max fragment count supported by ZC driver + type: u32 + checks: + min: 1 operations: list: @@ -77,6 +83,7 @@ operations: attributes: - ifindex - xdp-features + - xdp-zc-max-segs dump: reply: *dev-all - diff --git a/Documentation/networking/af_xdp.rst b/Documentation/networking/af_xdp.rst index 1cc35de336a4..dceeb0d763aa 100644 --- a/Documentation/networking/af_xdp.rst +++ b/Documentation/networking/af_xdp.rst @@ -462,8 +462,92 @@ XDP_OPTIONS getsockopt Gets options from an XDP socket. The only one supported so far is XDP_OPTIONS_ZEROCOPY which tells you if zero-copy is on or not. +Multi-Buffer Support +==================== + +With multi-buffer support, programs using AF_XDP sockets can receive +and transmit packets consisting of multiple buffers both in copy and +zero-copy mode. For example, a packet can consist of two +frames/buffers, one with the header and the other one with the data, +or a 9K Ethernet jumbo frame can be constructed by chaining together +three 4K frames. + +Some definitions: + +* A packet consists of one or more frames + +* A descriptor in one of the AF_XDP rings always refers to a single + frame. In the case the packet consists of a single frame, the + descriptor refers to the whole packet. + +To enable multi-buffer support for an AF_XDP socket, use the new bind +flag XDP_USE_SG. If this is not provided, all multi-buffer packets +will be dropped just as before. Note that the XDP program loaded also +needs to be in multi-buffer mode. This can be accomplished by using +"xdp.frags" as the section name of the XDP program used. + +To represent a packet consisting of multiple frames, a new flag called +XDP_PKT_CONTD is introduced in the options field of the Rx and Tx +descriptors. If it is true (1) the packet continues with the next +descriptor and if it is false (0) it means this is the last descriptor +of the packet. Why the reverse logic of end-of-packet (eop) flag found +in many NICs? Just to preserve compatibility with non-multi-buffer +applications that have this bit set to false for all packets on Rx, +and the apps set the options field to zero for Tx, as anything else +will be treated as an invalid descriptor. + +These are the semantics for producing packets onto AF_XDP Tx ring +consisting of multiple frames: + +* When an invalid descriptor is found, all the other + descriptors/frames of this packet are marked as invalid and not + completed. The next descriptor is treated as the start of a new + packet, even if this was not the intent (because we cannot guess + the intent). As before, if your program is producing invalid + descriptors you have a bug that must be fixed. + +* Zero length descriptors are treated as invalid descriptors. + +* For copy mode, the maximum supported number of frames in a packet is + equal to CONFIG_MAX_SKB_FRAGS + 1. If it is exceeded, all + descriptors accumulated so far are dropped and treated as + invalid. To produce an application that will work on any system + regardless of this config setting, limit the number of frags to 18, + as the minimum value of the config is 17. + +* For zero-copy mode, the limit is up to what the NIC HW + supports. Usually at least five on the NICs we have checked. We + consciously chose to not enforce a rigid limit (such as + CONFIG_MAX_SKB_FRAGS + 1) for zero-copy mode, as it would have + resulted in copy actions under the hood to fit into what limit the + NIC supports. Kind of defeats the purpose of zero-copy mode. How to + probe for this limit is explained in the "probe for multi-buffer + support" section. + +On the Rx path in copy-mode, the xsk core copies the XDP data into +multiple descriptors, if needed, and sets the XDP_PKT_CONTD flag as +detailed before. Zero-copy mode works the same, though the data is not +copied. When the application gets a descriptor with the XDP_PKT_CONTD +flag set to one, it means that the packet consists of multiple buffers +and it continues with the next buffer in the following +descriptor. When a descriptor with XDP_PKT_CONTD == 0 is received, it +means that this is the last buffer of the packet. AF_XDP guarantees +that only a complete packet (all frames in the packet) is sent to the +application. If there is not enough space in the AF_XDP Rx ring, all +frames of the packet will be dropped. + +If application reads a batch of descriptors, using for example the libxdp +interfaces, it is not guaranteed that the batch will end with a full +packet. It might end in the middle of a packet and the rest of the +buffers of that packet will arrive at the beginning of the next batch, +since the libxdp interface does not read the whole ring (unless you +have an enormous batch size or a very small ring size). + +An example program each for Rx and Tx multi-buffer support can be found +later in this document. + Usage -===== +----- In order to use AF_XDP sockets two parts are needed. The user-space application and the XDP program. For a complete setup and @@ -541,6 +625,131 @@ like this: But please use the libbpf functions as they are optimized and ready to use. Will make your life easier. +Usage Multi-Buffer Rx +--------------------- + +Here is a simple Rx path pseudo-code example (using libxdp interfaces +for simplicity). Error paths have been excluded to keep it short: + +.. code-block:: c + + void rx_packets(struct xsk_socket_info *xsk) + { + static bool new_packet = true; + u32 idx_rx = 0, idx_fq = 0; + static char *pkt; + + int rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); + + xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); + + for (int i = 0; i < rcvd; i++) { + struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); + char *frag = xsk_umem__get_data(xsk->umem->buffer, desc->addr); + bool eop = !(desc->options & XDP_PKT_CONTD); + + if (new_packet) + pkt = frag; + else + add_frag_to_pkt(pkt, frag); + + if (eop) + process_pkt(pkt); + + new_packet = eop; + + *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = desc->addr; + } + + xsk_ring_prod__submit(&xsk->umem->fq, rcvd); + xsk_ring_cons__release(&xsk->rx, rcvd); + } + +Usage Multi-Buffer Tx +--------------------- + +Here is an example Tx path pseudo-code (using libxdp interfaces for +simplicity) ignoring that the umem is finite in size, and that we +eventually will run out of packets to send. Also assumes pkts.addr +points to a valid location in the umem. + +.. code-block:: c + + void tx_packets(struct xsk_socket_info *xsk, struct pkt *pkts, + int batch_size) + { + u32 idx, i, pkt_nb = 0; + + xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx); + + for (i = 0; i < batch_size;) { + u64 addr = pkts[pkt_nb].addr; + u32 len = pkts[pkt_nb].size; + + do { + struct xdp_desc *tx_desc; + + tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i++); + tx_desc->addr = addr; + + if (len > xsk_frame_size) { + tx_desc->len = xsk_frame_size; + tx_desc->options = XDP_PKT_CONTD; + } else { + tx_desc->len = len; + tx_desc->options = 0; + pkt_nb++; + } + len -= tx_desc->len; + addr += xsk_frame_size; + + if (i == batch_size) { + /* Remember len, addr, pkt_nb for next iteration. + * Skipped for simplicity. + */ + break; + } + } while (len); + } + + xsk_ring_prod__submit(&xsk->tx, i); + } + +Probing for Multi-Buffer Support +-------------------------------- + +To discover if a driver supports multi-buffer AF_XDP in SKB or DRV +mode, use the XDP_FEATURES feature of netlink in linux/netdev.h to +query for NETDEV_XDP_ACT_RX_SG support. This is the same flag as for +querying for XDP multi-buffer support. If XDP supports multi-buffer in +a driver, then AF_XDP will also support that in SKB and DRV mode. + +To discover if a driver supports multi-buffer AF_XDP in zero-copy +mode, use XDP_FEATURES and first check the NETDEV_XDP_ACT_XSK_ZEROCOPY +flag. If it is set, it means that at least zero-copy is supported and +you should go and check the netlink attribute +NETDEV_A_DEV_XDP_ZC_MAX_SEGS in linux/netdev.h. An unsigned integer +value will be returned stating the max number of frags that are +supported by this device in zero-copy mode. These are the possible +return values: + +1: Multi-buffer for zero-copy is not supported by this device, as max + one fragment supported means that multi-buffer is not possible. + +>=2: Multi-buffer is supported in zero-copy mode for this device. The + returned number signifies the max number of frags supported. + +For an example on how these are used through libbpf, please take a +look at tools/testing/selftests/bpf/xskxceiver.c. + +Multi-Buffer Support for Zero-Copy Drivers +------------------------------------------ + +Zero-copy drivers usually use the batched APIs for Rx and Tx +processing. Note that the Tx batch API guarantees that it will provide +a batch of Tx descriptors that ends with full packet at the end. This +to facilitate extending a zero-copy driver with multi-buffer support. + Sample application ================== diff --git a/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst b/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst index bfd233cfac35..1e196cb9ce25 100644 --- a/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst +++ b/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst @@ -332,3 +332,11 @@ Setup HTB offload # tc class add dev <interface> parent 1: classid 1:1 htb rate 10Gbit prio 1 # tc class add dev <interface> parent 1: classid 1:2 htb rate 10Gbit prio 7 + +4. Create tc classes with same priorities and different quantum:: + + # tc class add dev <interface> parent 1: classid 1:1 htb rate 10Gbit prio 2 quantum 409600 + + # tc class add dev <interface> parent 1: classid 1:2 htb rate 10Gbit prio 2 quantum 188416 + + # tc class add dev <interface> parent 1: classid 1:3 htb rate 10Gbit prio 2 quantum 32768 diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 4a010a7cde7f..a66054d0763a 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -321,6 +321,7 @@ tcp_abort_on_overflow - BOOLEAN option can harm clients of your server. tcp_adv_win_scale - INTEGER + Obsolete since linux-6.6 Count buffering overhead as bytes/2^tcp_adv_win_scale (if tcp_adv_win_scale > 0) or bytes-bytes/2^(-tcp_adv_win_scale), if it is <= 0. @@ -2287,6 +2288,14 @@ accept_ra_min_hop_limit - INTEGER Default: 1 +accept_ra_min_lft - INTEGER + Minimum acceptable lifetime value in Router Advertisement. + + RA sections with a lifetime less than this value shall be + ignored. Zero lifetimes stay unaffected. + + Default: 0 + accept_ra_pinfo - BOOLEAN Learn Prefix Information in Router Advertisement. diff --git a/Documentation/networking/netconsole.rst b/Documentation/networking/netconsole.rst index dd0518e002f6..7a9de0568e84 100644 --- a/Documentation/networking/netconsole.rst +++ b/Documentation/networking/netconsole.rst @@ -13,6 +13,8 @@ IPv6 support by Cong Wang <xiyou.wangcong@gmail.com>, Jan 1 2013 Extended console support by Tejun Heo <tj@kernel.org>, May 1 2015 +Release prepend support by Breno Leitao <leitao@debian.org>, Jul 7 2023 + Please send bug reports to Matt Mackall <mpm@selenic.com> Satyam Sharma <satyam.sharma@gmail.com>, and Cong Wang <xiyou.wangcong@gmail.com> @@ -34,10 +36,11 @@ Sender and receiver configuration: It takes a string configuration parameter "netconsole" in the following format:: - netconsole=[+][src-port]@[src-ip]/[<dev>],[tgt-port]@<tgt-ip>/[tgt-macaddr] + netconsole=[+][r][src-port]@[src-ip]/[<dev>],[tgt-port]@<tgt-ip>/[tgt-macaddr] where + if present, enable extended console support + r if present, prepend kernel version (release) to the message src-port source for UDP packets (defaults to 6665) src-ip source IP to use (interface address) dev network interface (eth0) @@ -125,6 +128,7 @@ The interface exposes these parameters of a netconsole target to userspace: ============== ================================= ============ enabled Is this target currently enabled? (read-write) extended Extended mode enabled (read-write) + release Prepend kernel release to message (read-write) dev_name Local network interface name (read-write) local_port Source UDP port to use (read-write) remote_port Remote agent's UDP port (read-write) @@ -165,6 +169,11 @@ following format which is the same as /dev/kmsg:: <level>,<sequnum>,<timestamp>,<contflag>;<message text> +If 'r' (release) feature is enabled, the kernel release version is +prepended to the start of the message. Example:: + + 6.4.0,6,444,501151268,-;netconsole: network logging started + Non printable characters in <message text> are escaped using "\xff" notation. If the message contains optional dictionary, verbatim newline is used as the delimiter. diff --git a/Documentation/networking/page_pool.rst b/Documentation/networking/page_pool.rst index 873efd97f822..53b5448cc0f1 100644 --- a/Documentation/networking/page_pool.rst +++ b/Documentation/networking/page_pool.rst @@ -13,9 +13,9 @@ replacing dev_alloc_pages(). API keeps track of in-flight pages, in order to let API user know when it is safe to free a page_pool object. Thus, API users -must run page_pool_release_page() when a page is leaving the page_pool or -call page_pool_put_page() where appropriate in order to maintain correct -accounting. +must call page_pool_put_page() to free the page, or attach +the page to a page_pool-aware objects like skbs marked with +skb_mark_for_recycle(). API user must call page_pool_put_page() once on a page, as it will either recycle the page, or in case of refcnt > 1, it will @@ -64,87 +64,68 @@ This lockless guarantee naturally comes from running under a NAPI softirq. The protection doesn't strictly have to be NAPI, any guarantee that allocating a page will cause no race conditions is enough. -* page_pool_create(): Create a pool. - * flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV - * order: 2^order pages on allocation - * pool_size: size of the ptr_ring - * nid: preferred NUMA node for allocation - * dev: struct device. Used on DMA operations - * dma_dir: DMA direction - * max_len: max DMA sync memory size - * offset: DMA address offset - -* page_pool_put_page(): The outcome of this depends on the page refcnt. If the - driver bumps the refcnt > 1 this will unmap the page. If the page refcnt is 1 - the allocator owns the page and will try to recycle it in one of the pool - caches. If PP_FLAG_DMA_SYNC_DEV is set, the page will be synced for_device - using dma_sync_single_range_for_device(). - -* page_pool_put_full_page(): Similar to page_pool_put_page(), but will DMA sync - for the entire memory area configured in area pool->max_len. - -* page_pool_recycle_direct(): Similar to page_pool_put_full_page() but caller - must guarantee safe context (e.g NAPI), since it will recycle the page - directly into the pool fast cache. - -* page_pool_release_page(): Unmap the page (if mapped) and account for it on - in-flight counters. - -* page_pool_dev_alloc_pages(): Get a page from the page allocator or page_pool - caches. - -* page_pool_get_dma_addr(): Retrieve the stored DMA address. - -* page_pool_get_dma_dir(): Retrieve the stored DMA direction. - -* page_pool_put_page_bulk(): Tries to refill a number of pages into the - ptr_ring cache holding ptr_ring producer lock. If the ptr_ring is full, - page_pool_put_page_bulk() will release leftover pages to the page allocator. - page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx - completion loop for the XDP_REDIRECT use case. - Please note the caller must not use data area after running - page_pool_put_page_bulk(), as this function overwrites it. - -* page_pool_get_stats(): Retrieve statistics about the page_pool. This API - is only available if the kernel has been configured with - ``CONFIG_PAGE_POOL_STATS=y``. A pointer to a caller allocated ``struct - page_pool_stats`` structure is passed to this API which is filled in. The - caller can then report those stats to the user (perhaps via ethtool, - debugfs, etc.). See below for an example usage of this API. +.. kernel-doc:: net/core/page_pool.c + :identifiers: page_pool_create + +.. kernel-doc:: include/net/page_pool.h + :identifiers: struct page_pool_params + +.. kernel-doc:: include/net/page_pool.h + :identifiers: page_pool_put_page page_pool_put_full_page + page_pool_recycle_direct page_pool_dev_alloc_pages + page_pool_get_dma_addr page_pool_get_dma_dir + +.. kernel-doc:: net/core/page_pool.c + :identifiers: page_pool_put_page_bulk page_pool_get_stats + +DMA sync +-------- +Driver is always responsible for syncing the pages for the CPU. +Drivers may choose to take care of syncing for the device as well +or set the ``PP_FLAG_DMA_SYNC_DEV`` flag to request that pages +allocated from the page pool are already synced for the device. + +If ``PP_FLAG_DMA_SYNC_DEV`` is set, the driver must inform the core what portion +of the buffer has to be synced. This allows the core to avoid syncing the entire +page when the drivers knows that the device only accessed a portion of the page. + +Most drivers will reserve headroom in front of the frame. This part +of the buffer is not touched by the device, so to avoid syncing +it drivers can set the ``offset`` field in struct page_pool_params +appropriately. + +For pages recycled on the XDP xmit and skb paths the page pool will +use the ``max_len`` member of struct page_pool_params to decide how +much of the page needs to be synced (starting at ``offset``). +When directly freeing pages in the driver (page_pool_put_page()) +the ``dma_sync_size`` argument specifies how much of the buffer needs +to be synced. + +If in doubt set ``offset`` to 0, ``max_len`` to ``PAGE_SIZE`` and +pass -1 as ``dma_sync_size``. That combination of arguments is always +correct. + +Note that the syncing parameters are for the entire page. +This is important to remember when using fragments (``PP_FLAG_PAGE_FRAG``), +where allocated buffers may be smaller than a full page. +Unless the driver author really understands page pool internals +it's recommended to always use ``offset = 0``, ``max_len = PAGE_SIZE`` +with fragmented page pools. Stats API and structures ------------------------ If the kernel is configured with ``CONFIG_PAGE_POOL_STATS=y``, the API -``page_pool_get_stats()`` and structures described below are available. It -takes a pointer to a ``struct page_pool`` and a pointer to a ``struct -page_pool_stats`` allocated by the caller. +page_pool_get_stats() and structures described below are available. +It takes a pointer to a ``struct page_pool`` and a pointer to a struct +page_pool_stats allocated by the caller. -The API will fill in the provided ``struct page_pool_stats`` with +The API will fill in the provided struct page_pool_stats with statistics about the page_pool. -The stats structure has the following fields:: - - struct page_pool_stats { - struct page_pool_alloc_stats alloc_stats; - struct page_pool_recycle_stats recycle_stats; - }; - - -The ``struct page_pool_alloc_stats`` has the following fields: - * ``fast``: successful fast path allocations - * ``slow``: slow path order-0 allocations - * ``slow_high_order``: slow path high order allocations - * ``empty``: ptr ring is empty, so a slow path allocation was forced. - * ``refill``: an allocation which triggered a refill of the cache - * ``waive``: pages obtained from the ptr ring that cannot be added to - the cache due to a NUMA mismatch. - -The ``struct page_pool_recycle_stats`` has the following fields: - * ``cached``: recycling placed page in the page pool cache - * ``cache_full``: page pool cache was full - * ``ring``: page placed into the ptr ring - * ``ring_full``: page released from page pool because the ptr ring was full - * ``released_refcnt``: page released (and not recycled) because refcnt > 1 +.. kernel-doc:: include/net/page_pool.h + :identifiers: struct page_pool_recycle_stats + struct page_pool_alloc_stats + struct page_pool_stats Coding examples =============== @@ -194,7 +175,7 @@ NAPI poller if XDP_DROP: page_pool_recycle_direct(page_pool, page); } else (packet_is_skb) { - page_pool_release_page(page_pool, page); + skb_mark_for_recycle(skb); new_page = page_pool_dev_alloc_pages(page_pool); } } diff --git a/MAINTAINERS b/MAINTAINERS index 92d23da63638..3f32da783b31 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3685,6 +3685,7 @@ F: include/linux/filter.h F: include/linux/tnum.h F: kernel/bpf/core.c F: kernel/bpf/dispatcher.c +F: kernel/bpf/mprog.c F: kernel/bpf/syscall.c F: kernel/bpf/tnum.c F: kernel/bpf/trampoline.c @@ -3695,7 +3696,7 @@ R: David Vernet <void@manifault.com> L: bpf@vger.kernel.org L: bpf@ietf.org S: Maintained -F: Documentation/bpf/instruction-set.rst +F: Documentation/bpf/standardization/ BPF [GENERAL] (Safe Dynamic Programs and Tools) M: Alexei Starovoitov <ast@kernel.org> @@ -3778,13 +3779,15 @@ L: netdev@vger.kernel.org S: Maintained F: kernel/bpf/bpf_struct* -BPF [NETWORKING] (tc BPF, sock_addr) +BPF [NETWORKING] (tcx & tc BPF, sock_addr) M: Martin KaFai Lau <martin.lau@linux.dev> M: Daniel Borkmann <daniel@iogearbox.net> R: John Fastabend <john.fastabend@gmail.com> L: bpf@vger.kernel.org L: netdev@vger.kernel.org S: Maintained +F: include/net/tcx.h +F: kernel/bpf/tcx.c F: net/core/filter.c F: net/sched/act_bpf.c F: net/sched/cls_bpf.c @@ -3836,6 +3839,15 @@ S: Maintained F: kernel/bpf/stackmap.c F: kernel/trace/bpf_trace.c +BROADCOM ASP 2.0 ETHERNET DRIVER +M: Justin Chen <justin.chen@broadcom.com> +M: Florian Fainelli <florian.fainelli@broadcom.com> +L: bcm-kernel-feedback-list@broadcom.com +L: netdev@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml +F: drivers/net/ethernet/broadcom/asp2/ + BROADCOM B44 10/100 ETHERNET DRIVER M: Michael Chan <michael.chan@broadcom.com> L: netdev@vger.kernel.org @@ -7593,6 +7605,13 @@ L: linux-mmc@vger.kernel.org S: Supported F: drivers/mmc/host/cqhci* +EMS CPC-PCI CAN DRIVER +M: Gerhard Uttenthaler <uttenthaler@ems-wuensche.com> +M: support@ems-wuensche.com +L: linux-can@vger.kernel.org +S: Maintained +F: drivers/net/can/sja1000/ems_pci.c + EMULEX 10Gbps iSCSI - OneConnect DRIVER M: Ketan Mukadam <ketan.mukadam@broadcom.com> L: linux-scsi@vger.kernel.org @@ -7732,6 +7751,7 @@ F: include/linux/mii.h F: include/linux/of_net.h F: include/linux/phy.h F: include/linux/phy_fixed.h +F: include/linux/phylib_stubs.h F: include/linux/platform_data/mdio-bcm-unimac.h F: include/linux/platform_data/mdio-gpio.h F: include/trace/events/mdio.h diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi index 1bb1e5cae602..4086c0cc0f9d 100644 --- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi +++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi @@ -131,6 +131,18 @@ pins = "PB6", "PB7"; function = "uart3"; }; + + /omit-if-no-ref/ + can0_pins: can0-pins { + pins = "PB2", "PB3"; + function = "can0"; + }; + + /omit-if-no-ref/ + can1_pins: can1-pins { + pins = "PB4", "PB5"; + function = "can1"; + }; }; ccu: clock-controller@2001000 { @@ -879,5 +891,23 @@ clock-names = "bus", "hosc", "ahb"; #clock-cells = <1>; }; + + can0: can@2504000 { + compatible = "allwinner,sun20i-d1-can"; + reg = <0x02504000 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(21) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_CAN0>; + resets = <&ccu RST_BUS_CAN0>; + status = "disabled"; + }; + + can1: can@2504400 { + compatible = "allwinner,sun20i-d1-can"; + reg = <0x02504400 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(22) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_CAN1>; + resets = <&ccu RST_BUS_CAN1>; + status = "disabled"; + }; }; }; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 438adb695daa..83c4b45dc65f 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1857,59 +1857,177 @@ emit_jmp: return proglen; } -static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_regs, - int stack_size) +static void clean_stack_garbage(const struct btf_func_model *m, + u8 **pprog, int nr_stack_slots, + int stack_size) { - int i, j, arg_size; - bool next_same_struct = false; + int arg_size, off; + u8 *prog; + + /* Generally speaking, the compiler will pass the arguments + * on-stack with "push" instruction, which will take 8-byte + * on the stack. In this case, there won't be garbage values + * while we copy the arguments from origin stack frame to current + * in BPF_DW. + * + * However, sometimes the compiler will only allocate 4-byte on + * the stack for the arguments. For now, this case will only + * happen if there is only one argument on-stack and its size + * not more than 4 byte. In this case, there will be garbage + * values on the upper 4-byte where we store the argument on + * current stack frame. + * + * arguments on origin stack: + * + * stack_arg_1(4-byte) xxx(4-byte) + * + * what we copy: + * + * stack_arg_1(8-byte): stack_arg_1(origin) xxx + * + * and the xxx is the garbage values which we should clean here. + */ + if (nr_stack_slots != 1) + return; + + /* the size of the last argument */ + arg_size = m->arg_size[m->nr_args - 1]; + if (arg_size <= 4) { + off = -(stack_size - 4); + prog = *pprog; + /* mov DWORD PTR [rbp + off], 0 */ + if (!is_imm8(off)) + EMIT2_off32(0xC7, 0x85, off); + else + EMIT3(0xC7, 0x45, off); + EMIT(0, 4); + *pprog = prog; + } +} + +/* get the count of the regs that are used to pass arguments */ +static int get_nr_used_regs(const struct btf_func_model *m) +{ + int i, arg_regs, nr_used_regs = 0; + + for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + arg_regs = (m->arg_size[i] + 7) / 8; + if (nr_used_regs + arg_regs <= 6) + nr_used_regs += arg_regs; + + if (nr_used_regs >= 6) + break; + } + + return nr_used_regs; +} + +static void save_args(const struct btf_func_model *m, u8 **prog, + int stack_size, bool for_call_origin) +{ + int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; + int i, j; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0, j = 0; i < min(nr_regs, 6); i++) { - /* The arg_size is at most 16 bytes, enforced by the verifier. */ - arg_size = m->arg_size[j]; - if (arg_size > 8) { - arg_size = 8; - next_same_struct = !next_same_struct; - } + for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + arg_regs = (m->arg_size[i] + 7) / 8; - emit_stx(prog, bytes_to_bpf_size(arg_size), - BPF_REG_FP, - i == 5 ? X86_REG_R9 : BPF_REG_1 + i, - -(stack_size - i * 8)); + /* According to the research of Yonghong, struct members + * should be all in register or all on the stack. + * Meanwhile, the compiler will pass the argument on regs + * if the remaining regs can hold the argument. + * + * Disorder of the args can happen. For example: + * + * struct foo_struct { + * long a; + * int b; + * }; + * int foo(char, char, char, char, char, struct foo_struct, + * char); + * + * the arg1-5,arg7 will be passed by regs, and arg6 will + * by stack. + */ + if (nr_regs + arg_regs > 6) { + /* copy function arguments from origin stack frame + * into current stack frame. + * + * The starting address of the arguments on-stack + * is: + * rbp + 8(push rbp) + + * 8(return addr of origin call) + + * 8(return addr of the caller) + * which means: rbp + 24 + */ + for (j = 0; j < arg_regs; j++) { + emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, + nr_stack_slots * 8 + 0x18); + emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, + -stack_size); + + if (!nr_stack_slots) + first_off = stack_size; + stack_size -= 8; + nr_stack_slots++; + } + } else { + /* Only copy the arguments on-stack to current + * 'stack_size' and ignore the regs, used to + * prepare the arguments on-stack for orign call. + */ + if (for_call_origin) { + nr_regs += arg_regs; + continue; + } - j = next_same_struct ? j : j + 1; + /* copy the arguments from regs into stack */ + for (j = 0; j < arg_regs; j++) { + emit_stx(prog, BPF_DW, BPF_REG_FP, + nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, + -stack_size); + stack_size -= 8; + nr_regs++; + } + } } + + clean_stack_garbage(m, prog, nr_stack_slots, first_off); } -static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_regs, +static void restore_regs(const struct btf_func_model *m, u8 **prog, int stack_size) { - int i, j, arg_size; - bool next_same_struct = false; + int i, j, arg_regs, nr_regs = 0; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10] * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8] + * + * The logic here is similar to what we do in save_args() */ - for (i = 0, j = 0; i < min(nr_regs, 6); i++) { - /* The arg_size is at most 16 bytes, enforced by the verifier. */ - arg_size = m->arg_size[j]; - if (arg_size > 8) { - arg_size = 8; - next_same_struct = !next_same_struct; + for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + arg_regs = (m->arg_size[i] + 7) / 8; + if (nr_regs + arg_regs <= 6) { + for (j = 0; j < arg_regs; j++) { + emit_ldx(prog, BPF_DW, + nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, + BPF_REG_FP, + -stack_size); + stack_size -= 8; + nr_regs++; + } + } else { + stack_size -= 8 * arg_regs; } - emit_ldx(prog, bytes_to_bpf_size(arg_size), - i == 5 ? X86_REG_R9 : BPF_REG_1 + i, - BPF_REG_FP, - -(stack_size - i * 8)); - - j = next_same_struct ? j : j + 1; + if (nr_regs >= 6) + break; } } @@ -1938,7 +2056,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); /* arg2: lea rsi, [rbp - ctx_cookie_off] */ - EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); + if (!is_imm8(-run_ctx_off)) + EMIT3_off32(0x48, 0x8D, 0xB5, -run_ctx_off); + else + EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); if (emit_rsb_call(&prog, bpf_trampoline_enter(p), prog)) return -EINVAL; @@ -1954,7 +2075,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_nops(&prog, 2); /* arg1: lea rdi, [rbp - stack_size] */ - EMIT4(0x48, 0x8D, 0x7D, -stack_size); + if (!is_imm8(-stack_size)) + EMIT3_off32(0x48, 0x8D, 0xBD, -stack_size); + else + EMIT4(0x48, 0x8D, 0x7D, -stack_size); /* arg2: progs[i]->insnsi for interpreter */ if (!p->jited) emit_mov_imm64(&prog, BPF_REG_2, @@ -1984,7 +2108,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, /* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); /* arg3: lea rdx, [rbp - run_ctx_off] */ - EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); + if (!is_imm8(-run_ctx_off)) + EMIT3_off32(0x48, 0x8D, 0x95, -run_ctx_off); + else + EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); if (emit_rsb_call(&prog, bpf_trampoline_exit(p), prog)) return -EINVAL; @@ -2136,7 +2263,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *func_addr) { int i, ret, nr_regs = m->nr_args, stack_size = 0; - int regs_off, nregs_off, ip_off, run_ctx_off; + int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; @@ -2150,8 +2277,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) nr_regs += (m->arg_size[i] + 7) / 8 - 1; - /* x86-64 supports up to 6 arguments. 7+ can be added in the future */ - if (nr_regs > 6) + /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 + * are passed through regs, the remains are through stack. + */ + if (nr_regs > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; /* Generated trampoline stack layout: @@ -2170,7 +2299,14 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * + * RBP - rbx_off [ rbx value ] always + * * RBP - run_ctx_off [ bpf_tramp_run_ctx ] + * + * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG + * [ ... ] + * [ stack_arg2 ] + * RBP - arg_stack_off [ stack_arg1 ] */ /* room for return value of orig_call or fentry prog */ @@ -2190,9 +2326,26 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ip_off = stack_size; + stack_size += 8; + rbx_off = stack_size; + stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; run_ctx_off = stack_size; + if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { + /* the space that used to pass arguments on-stack */ + stack_size += (nr_regs - get_nr_used_regs(m)) * 8; + /* make sure the stack pointer is 16-byte aligned if we + * need pass arguments on stack, which means + * [stack_size + 8(rbp) + 8(rip) + 8(origin rip)] + * should be 16-byte aligned. Following code depend on + * that stack_size is already 8-byte aligned. + */ + stack_size += (stack_size % 16) ? 0 : 8; + } + + arg_stack_off = stack_size; + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip patched call instruction and point orig_call to actual * body of the kernel function. @@ -2212,8 +2365,14 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i x86_call_depth_emit_accounting(&prog, NULL); EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ - EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ - EMIT1(0x53); /* push rbx */ + if (!is_imm8(stack_size)) + /* sub rsp, stack_size */ + EMIT3_off32(0x48, 0x81, 0xEC, stack_size); + else + /* sub rsp, stack_size */ + EMIT4(0x48, 0x83, 0xEC, stack_size); + /* mov QWORD PTR [rbp - rbx_off], rbx */ + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); /* Store number of argument registers of the traced function: * mov rax, nr_regs @@ -2231,7 +2390,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off); } - save_regs(m, &prog, nr_regs, regs_off); + save_args(m, &prog, regs_off, false); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -2261,7 +2420,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, nr_regs, regs_off); + restore_regs(m, &prog, regs_off); + save_args(m, &prog, arg_stack_off, true); if (flags & BPF_TRAMP_F_ORIG_STACK) { emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); @@ -2302,7 +2462,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, nr_regs, regs_off); + restore_regs(m, &prog, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be @@ -2321,7 +2481,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (save_ret) emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); - EMIT1(0x5B); /* pop rbx */ + emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); EMIT1(0xC9); /* leave */ if (flags & BPF_TRAMP_F_SKIP_FRAME) /* skip our return address and return to parent */ diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c index ccac1c453080..05d562e9c8b1 100644 --- a/drivers/connector/cn_proc.c +++ b/drivers/connector/cn_proc.c @@ -48,8 +48,47 @@ static DEFINE_PER_CPU(struct local_event, local_event) = { .lock = INIT_LOCAL_LOCK(lock), }; +static int cn_filter(struct sock *dsk, struct sk_buff *skb, void *data) +{ + __u32 what, exit_code, *ptr; + enum proc_cn_mcast_op mc_op; + uintptr_t val; + + if (!dsk || !data) + return 0; + + ptr = (__u32 *)data; + what = *ptr++; + exit_code = *ptr; + val = ((struct proc_input *)(dsk->sk_user_data))->event_type; + mc_op = ((struct proc_input *)(dsk->sk_user_data))->mcast_op; + + if (mc_op == PROC_CN_MCAST_IGNORE) + return 1; + + if ((__u32)val == PROC_EVENT_ALL) + return 0; + + /* + * Drop packet if we have to report only non-zero exit status + * (PROC_EVENT_NONZERO_EXIT) and exit status is 0 + */ + if (((__u32)val & PROC_EVENT_NONZERO_EXIT) && + (what == PROC_EVENT_EXIT)) { + if (exit_code) + return 0; + } + + if ((__u32)val & what) + return 0; + + return 1; +} + static inline void send_msg(struct cn_msg *msg) { + __u32 filter_data[2]; + local_lock(&local_event.lock); msg->seq = __this_cpu_inc_return(local_event.count) - 1; @@ -61,7 +100,16 @@ static inline void send_msg(struct cn_msg *msg) * * If cn_netlink_send() fails, the data is not sent. */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_NOWAIT); + filter_data[0] = ((struct proc_event *)msg->data)->what; + if (filter_data[0] == PROC_EVENT_EXIT) { + filter_data[1] = + ((struct proc_event *)msg->data)->event_data.exit.exit_code; + } else { + filter_data[1] = 0; + } + + cn_netlink_send_mult(msg, msg->len, 0, CN_IDX_PROC, GFP_NOWAIT, + cn_filter, (void *)filter_data); local_unlock(&local_event.lock); } @@ -341,16 +389,17 @@ static void cn_proc_ack(int err, int rcvd_seq, int rcvd_ack) /** * cn_proc_mcast_ctl - * @data: message sent from userspace via the connector + * @msg: message sent from userspace via the connector + * @nsp: NETLINK_CB of the client's socket buffer */ static void cn_proc_mcast_ctl(struct cn_msg *msg, struct netlink_skb_parms *nsp) { - enum proc_cn_mcast_op *mc_op = NULL; - int err = 0; - - if (msg->len != sizeof(*mc_op)) - return; + enum proc_cn_mcast_op mc_op = 0, prev_mc_op = 0; + struct proc_input *pinput = NULL; + enum proc_cn_event ev_type = 0; + int err = 0, initial = 0; + struct sock *sk = NULL; /* * Events are reported with respect to the initial pid @@ -361,19 +410,51 @@ static void cn_proc_mcast_ctl(struct cn_msg *msg, !task_is_in_init_pid_ns(current)) return; - /* Can only change if privileged. */ - if (!__netlink_ns_capable(nsp, &init_user_ns, CAP_NET_ADMIN)) { - err = EPERM; - goto out; + if (msg->len == sizeof(*pinput)) { + pinput = (struct proc_input *)msg->data; + mc_op = pinput->mcast_op; + ev_type = pinput->event_type; + } else if (msg->len == sizeof(mc_op)) { + mc_op = *((enum proc_cn_mcast_op *)msg->data); + ev_type = PROC_EVENT_ALL; + } else { + return; + } + + ev_type = valid_event((enum proc_cn_event)ev_type); + + if (ev_type == PROC_EVENT_NONE) + ev_type = PROC_EVENT_ALL; + + if (nsp->sk) { + sk = nsp->sk; + if (sk->sk_user_data == NULL) { + sk->sk_user_data = kzalloc(sizeof(struct proc_input), + GFP_KERNEL); + if (sk->sk_user_data == NULL) { + err = ENOMEM; + goto out; + } + initial = 1; + } else { + prev_mc_op = + ((struct proc_input *)(sk->sk_user_data))->mcast_op; + } + ((struct proc_input *)(sk->sk_user_data))->event_type = + ev_type; + ((struct proc_input *)(sk->sk_user_data))->mcast_op = mc_op; } - mc_op = (enum proc_cn_mcast_op *)msg->data; - switch (*mc_op) { + switch (mc_op) { case PROC_CN_MCAST_LISTEN: - atomic_inc(&proc_event_num_listeners); + if (initial || (prev_mc_op != PROC_CN_MCAST_LISTEN)) + atomic_inc(&proc_event_num_listeners); break; case PROC_CN_MCAST_IGNORE: - atomic_dec(&proc_event_num_listeners); + if (!initial && (prev_mc_op != PROC_CN_MCAST_IGNORE)) + atomic_dec(&proc_event_num_listeners); + ((struct proc_input *)(sk->sk_user_data))->event_type = + PROC_EVENT_NONE; break; default: err = EINVAL; diff --git a/drivers/connector/connector.c b/drivers/connector/connector.c index 48ec7ce6ecac..7f7b94f616a6 100644 --- a/drivers/connector/connector.c +++ b/drivers/connector/connector.c @@ -59,7 +59,9 @@ static int cn_already_initialized; * both, or if both are zero then the group is looked up and sent there. */ int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 __group, - gfp_t gfp_mask) + gfp_t gfp_mask, + int (*filter)(struct sock *dsk, struct sk_buff *skb, void *data), + void *filter_data) { struct cn_callback_entry *__cbq; unsigned int size; @@ -110,8 +112,9 @@ int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 __group, NETLINK_CB(skb).dst_group = group; if (group) - return netlink_broadcast(dev->nls, skb, portid, group, - gfp_mask); + return netlink_broadcast_filtered(dev->nls, skb, portid, group, + gfp_mask, filter, + (void *)filter_data); return netlink_unicast(dev->nls, skb, portid, !gfpflags_allow_blocking(gfp_mask)); } @@ -121,7 +124,8 @@ EXPORT_SYMBOL_GPL(cn_netlink_send_mult); int cn_netlink_send(struct cn_msg *msg, u32 portid, u32 __group, gfp_t gfp_mask) { - return cn_netlink_send_mult(msg, msg->len, portid, __group, gfp_mask); + return cn_netlink_send_mult(msg, msg->len, portid, __group, gfp_mask, + NULL, NULL); } EXPORT_SYMBOL_GPL(cn_netlink_send); @@ -163,6 +167,31 @@ static int cn_call_callback(struct sk_buff *skb) } /* + * Allow non-root access for NETLINK_CONNECTOR family having CN_IDX_PROC + * multicast group. + */ +static int cn_bind(struct net *net, int group) +{ + unsigned long groups = (unsigned long) group; + + if (ns_capable(net->user_ns, CAP_NET_ADMIN)) + return 0; + + if (test_bit(CN_IDX_PROC - 1, &groups)) + return 0; + + return -EPERM; +} + +static void cn_release(struct sock *sk, unsigned long *groups) +{ + if (groups && test_bit(CN_IDX_PROC - 1, groups)) { + kfree(sk->sk_user_data); + sk->sk_user_data = NULL; + } +} + +/* * Main netlink receiving function. * * It checks skb, netlink header and msg sizes, and calls callback helper. @@ -249,6 +278,9 @@ static int cn_init(void) struct netlink_kernel_cfg cfg = { .groups = CN_NETLINK_USERS + 0xf, .input = cn_rx_skb, + .flags = NL_CFG_F_NONROOT_RECV, + .bind = cn_bind, + .release = cn_release, }; dev->nls = netlink_kernel_create(&init_net, NETLINK_CONNECTOR, &cfg); diff --git a/drivers/hid/bpf/entrypoints/Makefile b/drivers/hid/bpf/entrypoints/Makefile index a12edcfa4fe3..43b99b5575cf 100644 --- a/drivers/hid/bpf/entrypoints/Makefile +++ b/drivers/hid/bpf/entrypoints/Makefile @@ -58,7 +58,7 @@ entrypoints.lskel.h: $(OUTPUT)/entrypoints.bpf.o | $(BPFTOOL) $(OUTPUT)/entrypoints.bpf.o: entrypoints.bpf.c $(OUTPUT)/vmlinux.h $(BPFOBJ) | $(OUTPUT) $(call msg,BPF,$@) - $(Q)$(CLANG) -g -O2 -target bpf $(INCLUDES) \ + $(Q)$(CLANG) -g -O2 --target=bpf $(INCLUDES) \ -c $(filter %.c,$^) -o $@ && \ $(LLVM_STRIP) -g $@ diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c index b9dbad3a8af8..cc5049eb25f8 100644 --- a/drivers/net/bonding/bond_alb.c +++ b/drivers/net/bonding/bond_alb.c @@ -668,7 +668,7 @@ static struct slave *rlb_arp_xmit(struct sk_buff *skb, struct bonding *bond) dev = ip_dev_find(dev_net(bond->dev), arp->ip_src); if (dev) { - if (netif_is_bridge_master(dev)) { + if (netif_is_any_bridge_master(dev)) { dev_put(dev); return NULL; } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 484c9e3e5e82..f04d4f28eb1a 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -4446,11 +4446,6 @@ static int bond_eth_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cm { struct bonding *bond = netdev_priv(bond_dev); struct mii_ioctl_data *mii = NULL; - const struct net_device_ops *ops; - struct net_device *real_dev; - struct hwtstamp_config cfg; - struct ifreq ifrr; - int res = 0; netdev_dbg(bond_dev, "bond_eth_ioctl: cmd=%d\n", cmd); @@ -4477,44 +4472,11 @@ static int bond_eth_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cm } break; - case SIOCSHWTSTAMP: - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - if (!(cfg.flags & HWTSTAMP_FLAG_BONDED_PHC_INDEX)) - return -EOPNOTSUPP; - - fallthrough; - case SIOCGHWTSTAMP: - real_dev = bond_option_active_slave_get_rcu(bond); - if (!real_dev) - return -EOPNOTSUPP; - - strscpy_pad(ifrr.ifr_name, real_dev->name, IFNAMSIZ); - ifrr.ifr_ifru = ifr->ifr_ifru; - - ops = real_dev->netdev_ops; - if (netif_device_present(real_dev) && ops->ndo_eth_ioctl) { - res = ops->ndo_eth_ioctl(real_dev, &ifrr, cmd); - if (res) - return res; - - ifr->ifr_ifru = ifrr.ifr_ifru; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - /* Set the BOND_PHC_INDEX flag to notify user space */ - cfg.flags |= HWTSTAMP_FLAG_BONDED_PHC_INDEX; - - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? - -EFAULT : 0; - } - fallthrough; default: - res = -EOPNOTSUPP; + return -EOPNOTSUPP; } - return res; + return 0; } static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmd) @@ -5688,6 +5650,67 @@ static u32 bond_mode_bcast_speed(struct slave *slave, u32 speed) return speed; } +/* Set the BOND_PHC_INDEX flag to notify user space */ +static int bond_set_phc_index_flag(struct kernel_hwtstamp_config *kernel_cfg) +{ + struct ifreq *ifr = kernel_cfg->ifr; + struct hwtstamp_config cfg; + + if (kernel_cfg->copied_to_user) { + /* Lower device has a legacy implementation */ + if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) + return -EFAULT; + + cfg.flags |= HWTSTAMP_FLAG_BONDED_PHC_INDEX; + if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg))) + return -EFAULT; + } else { + kernel_cfg->flags |= HWTSTAMP_FLAG_BONDED_PHC_INDEX; + } + + return 0; +} + +static int bond_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) +{ + struct bonding *bond = netdev_priv(dev); + struct net_device *real_dev; + int err; + + real_dev = bond_option_active_slave_get_rcu(bond); + if (!real_dev) + return -EOPNOTSUPP; + + err = generic_hwtstamp_get_lower(real_dev, cfg); + if (err) + return err; + + return bond_set_phc_index_flag(cfg); +} + +static int bond_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + struct bonding *bond = netdev_priv(dev); + struct net_device *real_dev; + int err; + + if (!(cfg->flags & HWTSTAMP_FLAG_BONDED_PHC_INDEX)) + return -EOPNOTSUPP; + + real_dev = bond_option_active_slave_get_rcu(bond); + if (!real_dev) + return -EOPNOTSUPP; + + err = generic_hwtstamp_set_lower(real_dev, cfg, extack); + if (err) + return err; + + return bond_set_phc_index_flag(cfg); +} + static int bond_ethtool_get_link_ksettings(struct net_device *bond_dev, struct ethtool_link_ksettings *cmd) { @@ -5836,6 +5859,8 @@ static const struct net_device_ops bond_netdev_ops = { .ndo_bpf = bond_xdp, .ndo_xdp_xmit = bond_xdp_xmit, .ndo_xdp_get_xmit_slave = bond_xdp_get_xmit_slave, + .ndo_hwtstamp_get = bond_hwtstamp_get, + .ndo_hwtstamp_set = bond_hwtstamp_set, }; static const struct device_type bond_type = { diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig index a5c5036dfb94..649453a3c858 100644 --- a/drivers/net/can/Kconfig +++ b/drivers/net/can/Kconfig @@ -160,8 +160,13 @@ config CAN_KVASER_PCIEFD Kvaser PCIEcan 4xHS Kvaser PCIEcan 2xHS v2 Kvaser PCIEcan HS v2 + Kvaser PCIEcan 1xCAN v3 + Kvaser PCIEcan 2xCAN v3 + Kvaser PCIEcan 4xCAN v2 Kvaser Mini PCI Express HS v2 Kvaser Mini PCI Express 2xHS v2 + Kvaser Mini PCI Express 1xCAN v3 + Kvaser Mini PCI Express 2xCAN v3 config CAN_SLCAN tristate "Serial / USB serial CAN Adaptors (slcan)" @@ -185,10 +190,10 @@ config CAN_SLCAN config CAN_SUN4I tristate "Allwinner A10 CAN controller" - depends on MACH_SUN4I || MACH_SUN7I || COMPILE_TEST + depends on MACH_SUN4I || MACH_SUN7I || RISCV || COMPILE_TEST help Say Y here if you want to use CAN controller found on Allwinner - A10/A20 SoCs. + A10/A20/D1 SoCs. To compile this driver as a module, choose M here: the module will be called sun4i_can. diff --git a/drivers/net/can/bxcan.c b/drivers/net/can/bxcan.c index 39de7164bc4e..49cf9682b925 100644 --- a/drivers/net/can/bxcan.c +++ b/drivers/net/can/bxcan.c @@ -23,7 +23,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/regmap.h> diff --git a/drivers/net/can/dev/rx-offload.c b/drivers/net/can/dev/rx-offload.c index 161e45a7e8c1..77091f7d1fa7 100644 --- a/drivers/net/can/dev/rx-offload.c +++ b/drivers/net/can/dev/rx-offload.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2014 Protonic Holland, * David Jander - * Copyright (C) 2014-2021 Pengutronix, + * Copyright (C) 2014-2021, 2023 Pengutronix, * Marc Kleine-Budde <kernel@pengutronix.de> */ @@ -240,9 +240,10 @@ int can_rx_offload_queue_timestamp(struct can_rx_offload *offload, } EXPORT_SYMBOL_GPL(can_rx_offload_queue_timestamp); -unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, - unsigned int idx, u32 timestamp, - unsigned int *frame_len_ptr) +unsigned int +can_rx_offload_get_echo_skb_queue_timestamp(struct can_rx_offload *offload, + unsigned int idx, u32 timestamp, + unsigned int *frame_len_ptr) { struct net_device *dev = offload->dev; struct net_device_stats *stats = &dev->stats; @@ -262,7 +263,7 @@ unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, return len; } -EXPORT_SYMBOL_GPL(can_rx_offload_get_echo_skb); +EXPORT_SYMBOL_GPL(can_rx_offload_get_echo_skb_queue_timestamp); int can_rx_offload_queue_tail(struct can_rx_offload *offload, struct sk_buff *skb) @@ -279,6 +280,31 @@ int can_rx_offload_queue_tail(struct can_rx_offload *offload, } EXPORT_SYMBOL_GPL(can_rx_offload_queue_tail); +unsigned int +can_rx_offload_get_echo_skb_queue_tail(struct can_rx_offload *offload, + unsigned int idx, + unsigned int *frame_len_ptr) +{ + struct net_device *dev = offload->dev; + struct net_device_stats *stats = &dev->stats; + struct sk_buff *skb; + unsigned int len; + int err; + + skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr); + if (!skb) + return 0; + + err = can_rx_offload_queue_tail(offload, skb); + if (err) { + stats->rx_errors++; + stats->tx_fifo_errors++; + } + + return len; +} +EXPORT_SYMBOL_GPL(can_rx_offload_get_echo_skb_queue_tail); + void can_rx_offload_irq_finish(struct can_rx_offload *offload) { unsigned long flags; diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c index ff0fc18baf13..48273d51f271 100644 --- a/drivers/net/can/flexcan/flexcan-core.c +++ b/drivers/net/can/flexcan/flexcan-core.c @@ -1097,8 +1097,8 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id) handled = IRQ_HANDLED; stats->tx_bytes += - can_rx_offload_get_echo_skb(&priv->offload, 0, - reg_ctrl << 16, NULL); + can_rx_offload_get_echo_skb_queue_timestamp(&priv->offload, 0, + reg_ctrl << 16, NULL); stats->tx_packets++; /* after sending a RTR frame MB is in RX mode */ diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index 3174efdae271..6d3ba71a6a73 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -30,8 +30,9 @@ #include <linux/ethtool.h> #include <linux/io.h> #include <linux/can/dev.h> +#include <linux/platform_device.h> #include <linux/spinlock.h> -#include <linux/of_platform.h> +#include <linux/of.h> #include <linux/of_irq.h> #include <linux/dma-mapping.h> diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index 1d6642c94f2f..72307297d75e 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -20,7 +20,6 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/can/dev.h> diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index db6256f2b1b3..a57005faa04f 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (C) 2018 KVASER AB, Sweden. All rights reserved. * Parts of this driver are based on the following: - * - Kvaser linux pciefd driver (version 5.25) + * - Kvaser linux pciefd driver (version 5.42) * - PEAK linux canfd driver */ @@ -33,37 +33,27 @@ MODULE_DESCRIPTION("CAN driver for Kvaser CAN/PCIe devices"); #define KVASER_PCIEFD_DMA_SIZE (4U * 1024U) #define KVASER_PCIEFD_VENDOR 0x1a07 +/* Altera based devices */ #define KVASER_PCIEFD_4HS_DEVICE_ID 0x000d #define KVASER_PCIEFD_2HS_V2_DEVICE_ID 0x000e #define KVASER_PCIEFD_HS_V2_DEVICE_ID 0x000f #define KVASER_PCIEFD_MINIPCIE_HS_V2_DEVICE_ID 0x0010 #define KVASER_PCIEFD_MINIPCIE_2HS_V2_DEVICE_ID 0x0011 -/* PCIe IRQ registers */ -#define KVASER_PCIEFD_IRQ_REG 0x40 -#define KVASER_PCIEFD_IEN_REG 0x50 -/* DMA address translation map register base */ -#define KVASER_PCIEFD_DMA_MAP_BASE 0x1000 -/* Loopback control register */ -#define KVASER_PCIEFD_LOOP_REG 0x1f000 -/* System identification and information registers */ -#define KVASER_PCIEFD_SYSID_BASE 0x1f020 -#define KVASER_PCIEFD_SYSID_VERSION_REG (KVASER_PCIEFD_SYSID_BASE + 0x8) -#define KVASER_PCIEFD_SYSID_CANFREQ_REG (KVASER_PCIEFD_SYSID_BASE + 0xc) -#define KVASER_PCIEFD_SYSID_BUSFREQ_REG (KVASER_PCIEFD_SYSID_BASE + 0x10) -#define KVASER_PCIEFD_SYSID_BUILD_REG (KVASER_PCIEFD_SYSID_BASE + 0x14) -/* Shared receive buffer registers */ -#define KVASER_PCIEFD_SRB_BASE 0x1f200 -#define KVASER_PCIEFD_SRB_FIFO_LAST_REG (KVASER_PCIEFD_SRB_BASE + 0x1f4) -#define KVASER_PCIEFD_SRB_CMD_REG (KVASER_PCIEFD_SRB_BASE + 0x200) -#define KVASER_PCIEFD_SRB_IEN_REG (KVASER_PCIEFD_SRB_BASE + 0x204) -#define KVASER_PCIEFD_SRB_IRQ_REG (KVASER_PCIEFD_SRB_BASE + 0x20c) -#define KVASER_PCIEFD_SRB_STAT_REG (KVASER_PCIEFD_SRB_BASE + 0x210) -#define KVASER_PCIEFD_SRB_RX_NR_PACKETS_REG (KVASER_PCIEFD_SRB_BASE + 0x214) -#define KVASER_PCIEFD_SRB_CTRL_REG (KVASER_PCIEFD_SRB_BASE + 0x218) +/* SmartFusion2 based devices */ +#define KVASER_PCIEFD_2CAN_V3_DEVICE_ID 0x0012 +#define KVASER_PCIEFD_1CAN_V3_DEVICE_ID 0x0013 +#define KVASER_PCIEFD_4CAN_V2_DEVICE_ID 0x0014 +#define KVASER_PCIEFD_MINIPCIE_2CAN_V3_DEVICE_ID 0x0015 +#define KVASER_PCIEFD_MINIPCIE_1CAN_V3_DEVICE_ID 0x0016 + +/* Altera SerDes Enable 64-bit DMA address translation */ +#define KVASER_PCIEFD_ALTERA_DMA_64BIT BIT(0) + +/* SmartFusion2 SerDes LSB address translation mask */ +#define KVASER_PCIEFD_SF2_DMA_LSB_MASK GENMASK(31, 12) + /* Kvaser KCAN CAN controller registers */ -#define KVASER_PCIEFD_KCAN0_BASE 0x10000 -#define KVASER_PCIEFD_KCAN_BASE_OFFSET 0x1000 #define KVASER_PCIEFD_KCAN_FIFO_REG 0x100 #define KVASER_PCIEFD_KCAN_FIFO_LAST_REG 0x180 #define KVASER_PCIEFD_KCAN_CTRL_REG 0x2c0 @@ -77,13 +67,20 @@ MODULE_DESCRIPTION("CAN driver for Kvaser CAN/PCIe devices"); #define KVASER_PCIEFD_KCAN_BUS_LOAD_REG 0x424 #define KVASER_PCIEFD_KCAN_BTRD_REG 0x428 #define KVASER_PCIEFD_KCAN_PWM_REG 0x430 - -/* PCI interrupt fields */ -#define KVASER_PCIEFD_IRQ_SRB BIT(4) -#define KVASER_PCIEFD_IRQ_ALL_MASK GENMASK(4, 0) - -/* Enable 64-bit DMA address translation */ -#define KVASER_PCIEFD_64BIT_DMA_BIT BIT(0) +/* System identification and information registers */ +#define KVASER_PCIEFD_SYSID_VERSION_REG 0x8 +#define KVASER_PCIEFD_SYSID_CANFREQ_REG 0xc +#define KVASER_PCIEFD_SYSID_BUSFREQ_REG 0x10 +#define KVASER_PCIEFD_SYSID_BUILD_REG 0x14 +/* Shared receive buffer FIFO registers */ +#define KVASER_PCIEFD_SRB_FIFO_LAST_REG 0x1f4 +/* Shared receive buffer registers */ +#define KVASER_PCIEFD_SRB_CMD_REG 0x0 +#define KVASER_PCIEFD_SRB_IEN_REG 0x04 +#define KVASER_PCIEFD_SRB_IRQ_REG 0x0c +#define KVASER_PCIEFD_SRB_STAT_REG 0x10 +#define KVASER_PCIEFD_SRB_RX_NR_PACKETS_REG 0x14 +#define KVASER_PCIEFD_SRB_CTRL_REG 0x18 /* System build information fields */ #define KVASER_PCIEFD_SYSID_VERSION_NR_CHAN_MASK GENMASK(31, 24) @@ -253,7 +250,122 @@ MODULE_DESCRIPTION("CAN driver for Kvaser CAN/PCIe devices"); /* KCAN Error detected packet, second word */ #define KVASER_PCIEFD_EPACK_DIR_TX BIT(0) +/* Macros for calculating addresses of registers */ +#define KVASER_PCIEFD_GET_BLOCK_ADDR(pcie, block) \ + ((pcie)->reg_base + (pcie)->driver_data->address_offset->block) +#define KVASER_PCIEFD_PCI_IEN_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), pci_ien)) +#define KVASER_PCIEFD_PCI_IRQ_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), pci_irq)) +#define KVASER_PCIEFD_SERDES_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), serdes)) +#define KVASER_PCIEFD_SYSID_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), sysid)) +#define KVASER_PCIEFD_LOOPBACK_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), loopback)) +#define KVASER_PCIEFD_SRB_FIFO_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), kcan_srb_fifo)) +#define KVASER_PCIEFD_SRB_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), kcan_srb)) +#define KVASER_PCIEFD_KCAN_CH0_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), kcan_ch0)) +#define KVASER_PCIEFD_KCAN_CH1_ADDR(pcie) \ + (KVASER_PCIEFD_GET_BLOCK_ADDR((pcie), kcan_ch1)) +#define KVASER_PCIEFD_KCAN_CHANNEL_SPAN(pcie) \ + (KVASER_PCIEFD_KCAN_CH1_ADDR((pcie)) - KVASER_PCIEFD_KCAN_CH0_ADDR((pcie))) +#define KVASER_PCIEFD_KCAN_CHX_ADDR(pcie, i) \ + (KVASER_PCIEFD_KCAN_CH0_ADDR((pcie)) + (i) * KVASER_PCIEFD_KCAN_CHANNEL_SPAN((pcie))) + struct kvaser_pciefd; +static void kvaser_pciefd_write_dma_map_altera(struct kvaser_pciefd *pcie, + dma_addr_t addr, int index); +static void kvaser_pciefd_write_dma_map_sf2(struct kvaser_pciefd *pcie, + dma_addr_t addr, int index); + +struct kvaser_pciefd_address_offset { + u32 serdes; + u32 pci_ien; + u32 pci_irq; + u32 sysid; + u32 loopback; + u32 kcan_srb_fifo; + u32 kcan_srb; + u32 kcan_ch0; + u32 kcan_ch1; +}; + +struct kvaser_pciefd_dev_ops { + void (*kvaser_pciefd_write_dma_map)(struct kvaser_pciefd *pcie, + dma_addr_t addr, int index); +}; + +struct kvaser_pciefd_irq_mask { + u32 kcan_rx0; + u32 kcan_tx[KVASER_PCIEFD_MAX_CAN_CHANNELS]; + u32 all; +}; + +struct kvaser_pciefd_driver_data { + const struct kvaser_pciefd_address_offset *address_offset; + const struct kvaser_pciefd_irq_mask *irq_mask; + const struct kvaser_pciefd_dev_ops *ops; +}; + +static const struct kvaser_pciefd_address_offset kvaser_pciefd_altera_address_offset = { + .serdes = 0x1000, + .pci_ien = 0x50, + .pci_irq = 0x40, + .sysid = 0x1f020, + .loopback = 0x1f000, + .kcan_srb_fifo = 0x1f200, + .kcan_srb = 0x1f400, + .kcan_ch0 = 0x10000, + .kcan_ch1 = 0x11000, +}; + +static const struct kvaser_pciefd_address_offset kvaser_pciefd_sf2_address_offset = { + .serdes = 0x280c8, + .pci_ien = 0x102004, + .pci_irq = 0x102008, + .sysid = 0x100000, + .loopback = 0x103000, + .kcan_srb_fifo = 0x120000, + .kcan_srb = 0x121000, + .kcan_ch0 = 0x140000, + .kcan_ch1 = 0x142000, +}; + +static const struct kvaser_pciefd_irq_mask kvaser_pciefd_altera_irq_mask = { + .kcan_rx0 = BIT(4), + .kcan_tx = { BIT(0), BIT(1), BIT(2), BIT(3) }, + .all = GENMASK(4, 0), +}; + +static const struct kvaser_pciefd_irq_mask kvaser_pciefd_sf2_irq_mask = { + .kcan_rx0 = BIT(4), + .kcan_tx = { BIT(16), BIT(17), BIT(18), BIT(19) }, + .all = GENMASK(19, 16) | BIT(4), +}; + +static const struct kvaser_pciefd_dev_ops kvaser_pciefd_altera_dev_ops = { + .kvaser_pciefd_write_dma_map = kvaser_pciefd_write_dma_map_altera, +}; + +static const struct kvaser_pciefd_dev_ops kvaser_pciefd_sf2_dev_ops = { + .kvaser_pciefd_write_dma_map = kvaser_pciefd_write_dma_map_sf2, +}; + +static const struct kvaser_pciefd_driver_data kvaser_pciefd_altera_driver_data = { + .address_offset = &kvaser_pciefd_altera_address_offset, + .irq_mask = &kvaser_pciefd_altera_irq_mask, + .ops = &kvaser_pciefd_altera_dev_ops, +}; + +static const struct kvaser_pciefd_driver_data kvaser_pciefd_sf2_driver_data = { + .address_offset = &kvaser_pciefd_sf2_address_offset, + .irq_mask = &kvaser_pciefd_sf2_irq_mask, + .ops = &kvaser_pciefd_sf2_dev_ops, +}; struct kvaser_pciefd_can { struct can_priv can; @@ -273,6 +385,7 @@ struct kvaser_pciefd { struct pci_dev *pci; void __iomem *reg_base; struct kvaser_pciefd_can *can[KVASER_PCIEFD_MAX_CAN_CHANNELS]; + const struct kvaser_pciefd_driver_data *driver_data; void *dma_data[KVASER_PCIEFD_DMA_COUNT]; u8 nr_channels; u32 bus_freq; @@ -305,18 +418,43 @@ static const struct can_bittiming_const kvaser_pciefd_bittiming_const = { static struct pci_device_id kvaser_pciefd_id_table[] = { { PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_4HS_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_altera_driver_data, }, { PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_2HS_V2_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_altera_driver_data, }, { PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_HS_V2_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_altera_driver_data, }, { PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_MINIPCIE_HS_V2_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_altera_driver_data, }, { PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_MINIPCIE_2HS_V2_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_altera_driver_data, + }, + { + PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_2CAN_V3_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_sf2_driver_data, + }, + { + PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_1CAN_V3_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_sf2_driver_data, + }, + { + PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_4CAN_V2_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_sf2_driver_data, + }, + { + PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_MINIPCIE_2CAN_V3_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_sf2_driver_data, + }, + { + PCI_DEVICE(KVASER_PCIEFD_VENDOR, KVASER_PCIEFD_MINIPCIE_1CAN_V3_DEVICE_ID), + .driver_data = (kernel_ulong_t)&kvaser_pciefd_sf2_driver_data, }, { 0, @@ -783,8 +921,7 @@ static int kvaser_pciefd_setup_can_ctrls(struct kvaser_pciefd *pcie) can = netdev_priv(netdev); netdev->netdev_ops = &kvaser_pciefd_netdev_ops; netdev->ethtool_ops = &kvaser_pciefd_ethtool_ops; - can->reg_base = pcie->reg_base + KVASER_PCIEFD_KCAN0_BASE + - i * KVASER_PCIEFD_KCAN_BASE_OFFSET; + can->reg_base = KVASER_PCIEFD_KCAN_CHX_ADDR(pcie, i); can->kv_pcie = pcie; can->cmd_seq = 0; can->err_rep_cnt = 0; @@ -865,20 +1002,37 @@ static int kvaser_pciefd_reg_candev(struct kvaser_pciefd *pcie) return 0; } -static void kvaser_pciefd_write_dma_map(struct kvaser_pciefd *pcie, - dma_addr_t addr, int offset) +static void kvaser_pciefd_write_dma_map_altera(struct kvaser_pciefd *pcie, + dma_addr_t addr, int index) { + void __iomem *serdes_base; u32 word1, word2; #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - word1 = addr | KVASER_PCIEFD_64BIT_DMA_BIT; + word1 = addr | KVASER_PCIEFD_ALTERA_DMA_64BIT; word2 = addr >> 32; #else word1 = addr; word2 = 0; #endif - iowrite32(word1, pcie->reg_base + offset); - iowrite32(word2, pcie->reg_base + offset + 4); + serdes_base = KVASER_PCIEFD_SERDES_ADDR(pcie) + 0x8 * index; + iowrite32(word1, serdes_base); + iowrite32(word2, serdes_base + 0x4); +} + +static void kvaser_pciefd_write_dma_map_sf2(struct kvaser_pciefd *pcie, + dma_addr_t addr, int index) +{ + void __iomem *serdes_base; + u32 lsb = addr & KVASER_PCIEFD_SF2_DMA_LSB_MASK; + u32 msb = 0x0; + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + msb = addr >> 32; +#endif + serdes_base = KVASER_PCIEFD_SERDES_ADDR(pcie) + 0x10 * index; + iowrite32(lsb, serdes_base); + iowrite32(msb, serdes_base + 0x4); } static int kvaser_pciefd_setup_dma(struct kvaser_pciefd *pcie) @@ -889,10 +1043,8 @@ static int kvaser_pciefd_setup_dma(struct kvaser_pciefd *pcie) dma_addr_t dma_addr[KVASER_PCIEFD_DMA_COUNT]; /* Disable the DMA */ - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_SRB_CTRL_REG); + iowrite32(0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CTRL_REG); for (i = 0; i < KVASER_PCIEFD_DMA_COUNT; i++) { - unsigned int offset = KVASER_PCIEFD_DMA_MAP_BASE + 8 * i; - pcie->dma_data[i] = dmam_alloc_coherent(&pcie->pci->dev, KVASER_PCIEFD_DMA_SIZE, &dma_addr[i], @@ -903,24 +1055,25 @@ static int kvaser_pciefd_setup_dma(struct kvaser_pciefd *pcie) KVASER_PCIEFD_DMA_SIZE); return -ENOMEM; } - kvaser_pciefd_write_dma_map(pcie, dma_addr[i], offset); + pcie->driver_data->ops->kvaser_pciefd_write_dma_map(pcie, dma_addr[i], i); } /* Reset Rx FIFO, and both DMA buffers */ iowrite32(KVASER_PCIEFD_SRB_CMD_FOR | KVASER_PCIEFD_SRB_CMD_RDB0 | KVASER_PCIEFD_SRB_CMD_RDB1, - pcie->reg_base + KVASER_PCIEFD_SRB_CMD_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); /* Empty Rx FIFO */ srb_packet_count = FIELD_GET(KVASER_PCIEFD_SRB_RX_NR_PACKETS_MASK, - ioread32(pcie->reg_base + KVASER_PCIEFD_SRB_RX_NR_PACKETS_REG)); + ioread32(KVASER_PCIEFD_SRB_ADDR(pcie) + + KVASER_PCIEFD_SRB_RX_NR_PACKETS_REG)); while (srb_packet_count) { /* Drop current packet in FIFO */ - ioread32(pcie->reg_base + KVASER_PCIEFD_SRB_FIFO_LAST_REG); + ioread32(KVASER_PCIEFD_SRB_FIFO_ADDR(pcie) + KVASER_PCIEFD_SRB_FIFO_LAST_REG); srb_packet_count--; } - srb_status = ioread32(pcie->reg_base + KVASER_PCIEFD_SRB_STAT_REG); + srb_status = ioread32(KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_STAT_REG); if (!(srb_status & KVASER_PCIEFD_SRB_STAT_DI)) { dev_err(&pcie->pci->dev, "DMA not idle before enabling\n"); return -EIO; @@ -928,7 +1081,7 @@ static int kvaser_pciefd_setup_dma(struct kvaser_pciefd *pcie) /* Enable the DMA */ iowrite32(KVASER_PCIEFD_SRB_CTRL_DMA_ENABLE, - pcie->reg_base + KVASER_PCIEFD_SRB_CTRL_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CTRL_REG); return 0; } @@ -937,30 +1090,29 @@ static int kvaser_pciefd_setup_board(struct kvaser_pciefd *pcie) { u32 version, srb_status, build; - version = ioread32(pcie->reg_base + KVASER_PCIEFD_SYSID_VERSION_REG); + version = ioread32(KVASER_PCIEFD_SYSID_ADDR(pcie) + KVASER_PCIEFD_SYSID_VERSION_REG); pcie->nr_channels = min(KVASER_PCIEFD_MAX_CAN_CHANNELS, FIELD_GET(KVASER_PCIEFD_SYSID_VERSION_NR_CHAN_MASK, version)); - build = ioread32(pcie->reg_base + KVASER_PCIEFD_SYSID_BUILD_REG); + build = ioread32(KVASER_PCIEFD_SYSID_ADDR(pcie) + KVASER_PCIEFD_SYSID_BUILD_REG); dev_dbg(&pcie->pci->dev, "Version %lu.%lu.%lu\n", FIELD_GET(KVASER_PCIEFD_SYSID_VERSION_MAJOR_MASK, version), FIELD_GET(KVASER_PCIEFD_SYSID_VERSION_MINOR_MASK, version), FIELD_GET(KVASER_PCIEFD_SYSID_BUILD_SEQ_MASK, build)); - srb_status = ioread32(pcie->reg_base + KVASER_PCIEFD_SRB_STAT_REG); + srb_status = ioread32(KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_STAT_REG); if (!(srb_status & KVASER_PCIEFD_SRB_STAT_DMA)) { dev_err(&pcie->pci->dev, "Hardware without DMA is not supported\n"); return -ENODEV; } - pcie->bus_freq = ioread32(pcie->reg_base + - KVASER_PCIEFD_SYSID_BUSFREQ_REG); - pcie->freq = ioread32(pcie->reg_base + KVASER_PCIEFD_SYSID_CANFREQ_REG); + pcie->bus_freq = ioread32(KVASER_PCIEFD_SYSID_ADDR(pcie) + KVASER_PCIEFD_SYSID_BUSFREQ_REG); + pcie->freq = ioread32(KVASER_PCIEFD_SYSID_ADDR(pcie) + KVASER_PCIEFD_SYSID_CANFREQ_REG); pcie->freq_to_ticks_div = pcie->freq / 1000000; if (pcie->freq_to_ticks_div == 0) pcie->freq_to_ticks_div = 1; /* Turn off all loopback functionality */ - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_LOOP_REG); + iowrite32(0, KVASER_PCIEFD_LOOPBACK_ADDR(pcie)); return 0; } @@ -1430,21 +1582,20 @@ static int kvaser_pciefd_read_buffer(struct kvaser_pciefd *pcie, int dma_buf) static void kvaser_pciefd_receive_irq(struct kvaser_pciefd *pcie) { - u32 irq; + u32 irq = ioread32(KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); - irq = ioread32(pcie->reg_base + KVASER_PCIEFD_SRB_IRQ_REG); if (irq & KVASER_PCIEFD_SRB_IRQ_DPD0) { kvaser_pciefd_read_buffer(pcie, 0); /* Reset DMA buffer 0 */ iowrite32(KVASER_PCIEFD_SRB_CMD_RDB0, - pcie->reg_base + KVASER_PCIEFD_SRB_CMD_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); } if (irq & KVASER_PCIEFD_SRB_IRQ_DPD1) { kvaser_pciefd_read_buffer(pcie, 1); /* Reset DMA buffer 1 */ iowrite32(KVASER_PCIEFD_SRB_CMD_RDB1, - pcie->reg_base + KVASER_PCIEFD_SRB_CMD_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); } if (irq & KVASER_PCIEFD_SRB_IRQ_DOF0 || @@ -1453,7 +1604,7 @@ static void kvaser_pciefd_receive_irq(struct kvaser_pciefd *pcie) irq & KVASER_PCIEFD_SRB_IRQ_DUF1) dev_err(&pcie->pci->dev, "DMA IRQ error 0x%08X\n", irq); - iowrite32(irq, pcie->reg_base + KVASER_PCIEFD_SRB_IRQ_REG); + iowrite32(irq, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); } static void kvaser_pciefd_transmit_irq(struct kvaser_pciefd_can *can) @@ -1479,15 +1630,14 @@ static void kvaser_pciefd_transmit_irq(struct kvaser_pciefd_can *can) static irqreturn_t kvaser_pciefd_irq_handler(int irq, void *dev) { struct kvaser_pciefd *pcie = (struct kvaser_pciefd *)dev; - u32 board_irq; + const struct kvaser_pciefd_irq_mask *irq_mask = pcie->driver_data->irq_mask; + u32 board_irq = ioread32(KVASER_PCIEFD_PCI_IRQ_ADDR(pcie)); int i; - board_irq = ioread32(pcie->reg_base + KVASER_PCIEFD_IRQ_REG); - - if (!(board_irq & KVASER_PCIEFD_IRQ_ALL_MASK)) + if (!(board_irq & irq_mask->all)) return IRQ_NONE; - if (board_irq & KVASER_PCIEFD_IRQ_SRB) + if (board_irq & irq_mask->kcan_rx0) kvaser_pciefd_receive_irq(pcie); for (i = 0; i < pcie->nr_channels; i++) { @@ -1498,7 +1648,7 @@ static irqreturn_t kvaser_pciefd_irq_handler(int irq, void *dev) } /* Check that mask matches channel (i) IRQ mask */ - if (board_irq & (1 << i)) + if (board_irq & irq_mask->kcan_tx[i]) kvaser_pciefd_transmit_irq(pcie->can[i]); } @@ -1525,6 +1675,8 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, { int err; struct kvaser_pciefd *pcie; + const struct kvaser_pciefd_irq_mask *irq_mask; + void __iomem *irq_en_base; pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL); if (!pcie) @@ -1532,6 +1684,8 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, pcie); pcie->pci = pdev; + pcie->driver_data = (const struct kvaser_pciefd_driver_data *)id->driver_data; + irq_mask = pcie->driver_data->irq_mask; err = pci_enable_device(pdev); if (err) @@ -1567,22 +1721,21 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, goto err_teardown_can_ctrls; iowrite32(KVASER_PCIEFD_SRB_IRQ_DPD0 | KVASER_PCIEFD_SRB_IRQ_DPD1, - pcie->reg_base + KVASER_PCIEFD_SRB_IRQ_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); iowrite32(KVASER_PCIEFD_SRB_IRQ_DPD0 | KVASER_PCIEFD_SRB_IRQ_DPD1 | KVASER_PCIEFD_SRB_IRQ_DOF0 | KVASER_PCIEFD_SRB_IRQ_DOF1 | KVASER_PCIEFD_SRB_IRQ_DUF0 | KVASER_PCIEFD_SRB_IRQ_DUF1, - pcie->reg_base + KVASER_PCIEFD_SRB_IEN_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IEN_REG); /* Enable PCI interrupts */ - iowrite32(KVASER_PCIEFD_IRQ_ALL_MASK, - pcie->reg_base + KVASER_PCIEFD_IEN_REG); - + irq_en_base = KVASER_PCIEFD_PCI_IEN_ADDR(pcie); + iowrite32(irq_mask->all, irq_en_base); /* Ready the DMA buffers */ iowrite32(KVASER_PCIEFD_SRB_CMD_RDB0, - pcie->reg_base + KVASER_PCIEFD_SRB_CMD_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); iowrite32(KVASER_PCIEFD_SRB_CMD_RDB1, - pcie->reg_base + KVASER_PCIEFD_SRB_CMD_REG); + KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); err = kvaser_pciefd_reg_candev(pcie); if (err) @@ -1592,12 +1745,12 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, err_free_irq: /* Disable PCI interrupts */ - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_IEN_REG); + iowrite32(0, irq_en_base); free_irq(pcie->pci->irq, pcie); err_teardown_can_ctrls: kvaser_pciefd_teardown_can_ctrls(pcie); - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_SRB_CTRL_REG); + iowrite32(0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CTRL_REG); pci_clear_master(pdev); err_pci_iounmap: @@ -1636,8 +1789,8 @@ static void kvaser_pciefd_remove(struct pci_dev *pdev) kvaser_pciefd_remove_all_ctrls(pcie); /* Disable interrupts */ - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_SRB_CTRL_REG); - iowrite32(0, pcie->reg_base + KVASER_PCIEFD_IEN_REG); + iowrite32(0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CTRL_REG); + iowrite32(0, KVASER_PCIEFD_PCI_IEN_ADDR(pcie)); free_irq(pcie->pci->irq, pcie); diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index c5af92bcc9c9..d204703521d2 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -11,6 +11,7 @@ #include <linux/bitfield.h> #include <linux/can/dev.h> #include <linux/ethtool.h> +#include <linux/hrtimer.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> @@ -18,7 +19,6 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/phy/phy.h> #include <linux/pinctrl/consumer.h> #include <linux/platform_device.h> @@ -308,6 +308,9 @@ enum m_can_reg { #define TX_EVENT_MM_MASK GENMASK(31, 24) #define TX_EVENT_TXTS_MASK GENMASK(15, 0) +/* Hrtimer polling interval */ +#define HRTIMER_POLL_INTERVAL_MS 1 + /* The ID and DLC registers are adjacent in M_CAN FIFO memory, * and we can save a (potentially slow) bus round trip by combining * reads and writes to them. @@ -1013,10 +1016,10 @@ static void m_can_tx_update_stats(struct m_can_classdev *cdev, if (cdev->is_peripheral) stats->tx_bytes += - can_rx_offload_get_echo_skb(&cdev->offload, - msg_mark, - timestamp, - NULL); + can_rx_offload_get_echo_skb_queue_timestamp(&cdev->offload, + msg_mark, + timestamp, + NULL); else stats->tx_bytes += can_get_echo_skb(dev, msg_mark, NULL); @@ -1414,6 +1417,12 @@ static int m_can_start(struct net_device *dev) m_can_enable_all_interrupts(cdev); + if (!dev->irq) { + dev_dbg(cdev->dev, "Start hrtimer\n"); + hrtimer_start(&cdev->hrtimer, ms_to_ktime(HRTIMER_POLL_INTERVAL_MS), + HRTIMER_MODE_REL_PINNED); + } + return 0; } @@ -1568,6 +1577,11 @@ static void m_can_stop(struct net_device *dev) { struct m_can_classdev *cdev = netdev_priv(dev); + if (!dev->irq) { + dev_dbg(cdev->dev, "Stop hrtimer\n"); + hrtimer_cancel(&cdev->hrtimer); + } + /* disable all interrupts */ m_can_disable_all_interrupts(cdev); @@ -1793,6 +1807,18 @@ static netdev_tx_t m_can_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; } +static enum hrtimer_restart hrtimer_callback(struct hrtimer *timer) +{ + struct m_can_classdev *cdev = container_of(timer, struct + m_can_classdev, hrtimer); + + m_can_isr(0, cdev->net); + + hrtimer_forward_now(timer, ms_to_ktime(HRTIMER_POLL_INTERVAL_MS)); + + return HRTIMER_RESTART; +} + static int m_can_open(struct net_device *dev) { struct m_can_classdev *cdev = netdev_priv(dev); @@ -1831,7 +1857,7 @@ static int m_can_open(struct net_device *dev) err = request_threaded_irq(dev->irq, NULL, m_can_isr, IRQF_ONESHOT, dev->name, dev); - } else { + } else if (dev->irq) { err = request_irq(dev->irq, m_can_isr, IRQF_SHARED, dev->name, dev); } @@ -2027,6 +2053,9 @@ int m_can_class_register(struct m_can_classdev *cdev) goto clk_disable; } + if (!cdev->net->irq) + cdev->hrtimer.function = &hrtimer_callback; + ret = m_can_dev_setup(cdev); if (ret) goto rx_offload_del; diff --git a/drivers/net/can/m_can/m_can.h b/drivers/net/can/m_can/m_can.h index a839dc71dc9b..c543928c756f 100644 --- a/drivers/net/can/m_can/m_can.h +++ b/drivers/net/can/m_can/m_can.h @@ -15,6 +15,7 @@ #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/freezer.h> +#include <linux/hrtimer.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> @@ -22,7 +23,6 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/phy/phy.h> #include <linux/pinctrl/consumer.h> #include <linux/pm_runtime.h> @@ -93,6 +93,8 @@ struct m_can_classdev { int is_peripheral; struct mram_cfg mcfg[MRAM_CFG_NUM]; + + struct hrtimer hrtimer; }; struct m_can_classdev *m_can_class_allocate_dev(struct device *dev, int sizeof_priv); diff --git a/drivers/net/can/m_can/m_can_platform.c b/drivers/net/can/m_can/m_can_platform.c index 94dc82644113..cdb28d6a092c 100644 --- a/drivers/net/can/m_can/m_can_platform.c +++ b/drivers/net/can/m_can/m_can_platform.c @@ -5,6 +5,7 @@ // // Copyright (C) 2018-19 Texas Instruments Incorporated - http://www.ti.com/ +#include <linux/hrtimer.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> @@ -82,7 +83,7 @@ static int m_can_plat_probe(struct platform_device *pdev) void __iomem *addr; void __iomem *mram_addr; struct phy *transceiver; - int irq, ret = 0; + int irq = 0, ret = 0; mcan_class = m_can_class_allocate_dev(&pdev->dev, sizeof(struct m_can_plat_priv)); @@ -96,12 +97,24 @@ static int m_can_plat_probe(struct platform_device *pdev) goto probe_fail; addr = devm_platform_ioremap_resource_byname(pdev, "m_can"); - irq = platform_get_irq_byname(pdev, "int0"); - if (IS_ERR(addr) || irq < 0) { - ret = -EINVAL; + if (IS_ERR(addr)) { + ret = PTR_ERR(addr); goto probe_fail; } + if (device_property_present(mcan_class->dev, "interrupts") || + device_property_present(mcan_class->dev, "interrupt-names")) { + irq = platform_get_irq_byname(pdev, "int0"); + if (irq < 0) { + ret = irq; + goto probe_fail; + } + } else { + dev_dbg(mcan_class->dev, "Polling enabled, initialize hrtimer"); + hrtimer_init(&mcan_class->hrtimer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_PINNED); + } + /* message ram could be shared */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "message_ram"); if (!res) { diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index e4d748913439..b82842718735 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -34,7 +34,6 @@ #include <linux/moduleparam.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/reset.h> diff --git a/drivers/net/can/sja1000/ems_pci.c b/drivers/net/can/sja1000/ems_pci.c index c56e27223e5f..ac86640998a8 100644 --- a/drivers/net/can/sja1000/ems_pci.c +++ b/drivers/net/can/sja1000/ems_pci.c @@ -148,7 +148,7 @@ static void ems_pci_v1_write_reg(const struct sja1000_priv *priv, static void ems_pci_v1_post_irq(const struct sja1000_priv *priv) { - struct ems_pci_card *card = (struct ems_pci_card *)priv->priv; + struct ems_pci_card *card = priv->priv; /* reset int flag of pita */ writel(PITA2_ICR_INT0_EN | PITA2_ICR_INT0, @@ -168,7 +168,7 @@ static void ems_pci_v2_write_reg(const struct sja1000_priv *priv, static void ems_pci_v2_post_irq(const struct sja1000_priv *priv) { - struct ems_pci_card *card = (struct ems_pci_card *)priv->priv; + struct ems_pci_card *card = priv->priv; writel(PLX_ICSR_ENA_CLR, card->conf_addr + PLX_ICSR); } @@ -186,7 +186,7 @@ static void ems_pci_v3_write_reg(const struct sja1000_priv *priv, static void ems_pci_v3_post_irq(const struct sja1000_priv *priv) { - struct ems_pci_card *card = (struct ems_pci_card *)priv->priv; + struct ems_pci_card *card = priv->priv; writel(ASIX_LINTSR_INT0AC, card->conf_addr + ASIX_LINTSR); } diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c index 4e59952c66d4..33f0e46ab1c2 100644 --- a/drivers/net/can/sja1000/sja1000_platform.c +++ b/drivers/net/can/sja1000/sja1000_platform.c @@ -17,7 +17,6 @@ #include <linux/clk.h> #include <linux/io.h> #include <linux/of.h> -#include <linux/of_device.h> #include "sja1000.h" diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-tef.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-tef.c index 237617b0c125..e5bd57b65aaf 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-tef.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-tef.c @@ -111,9 +111,9 @@ mcp251xfd_handle_tefif_one(struct mcp251xfd_priv *priv, if (skb) mcp251xfd_skb_set_timestamp(priv, skb, hw_tef_obj->ts); stats->tx_bytes += - can_rx_offload_get_echo_skb(&priv->offload, - tef_tail, hw_tef_obj->ts, - frame_len_ptr); + can_rx_offload_get_echo_skb_queue_timestamp(&priv->offload, + tef_tail, hw_tef_obj->ts, + frame_len_ptr); stats->tx_packets++; priv->tef->tail++; diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 0827830bbf28..ab8d01784686 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -59,7 +59,6 @@ #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset.h> @@ -91,6 +90,8 @@ #define SUN4I_REG_BUF12_ADDR 0x0070 /* CAN Tx/Rx Buffer 12 */ #define SUN4I_REG_ACPC_ADDR 0x0040 /* CAN Acceptance Code 0 */ #define SUN4I_REG_ACPM_ADDR 0x0044 /* CAN Acceptance Mask 0 */ +#define SUN4I_REG_ACPC_ADDR_D1 0x0028 /* CAN Acceptance Code 0 on the D1 */ +#define SUN4I_REG_ACPM_ADDR_D1 0x002C /* CAN Acceptance Mask 0 on the D1 */ #define SUN4I_REG_RBUF_RBACK_START_ADDR 0x0180 /* CAN transmit buffer start */ #define SUN4I_REG_RBUF_RBACK_END_ADDR 0x01b0 /* CAN transmit buffer end */ @@ -205,9 +206,11 @@ * struct sun4ican_quirks - Differences between SoC variants. * * @has_reset: SoC needs reset deasserted. + * @acp_offset: Offset of ACPC and ACPM registers */ struct sun4ican_quirks { bool has_reset; + int acp_offset; }; struct sun4ican_priv { @@ -216,6 +219,7 @@ struct sun4ican_priv { struct clk *clk; struct reset_control *reset; spinlock_t cmdreg_lock; /* lock for concurrent cmd register writes */ + int acp_offset; }; static const struct can_bittiming_const sun4ican_bittiming_const = { @@ -338,8 +342,8 @@ static int sun4i_can_start(struct net_device *dev) } /* set filters - we accept all */ - writel(0x00000000, priv->base + SUN4I_REG_ACPC_ADDR); - writel(0xFFFFFFFF, priv->base + SUN4I_REG_ACPM_ADDR); + writel(0x00000000, priv->base + SUN4I_REG_ACPC_ADDR + priv->acp_offset); + writel(0xFFFFFFFF, priv->base + SUN4I_REG_ACPM_ADDR + priv->acp_offset); /* clear error counters and error code capture */ writel(0, priv->base + SUN4I_REG_ERRC_ADDR); @@ -768,10 +772,17 @@ static const struct ethtool_ops sun4ican_ethtool_ops = { static const struct sun4ican_quirks sun4ican_quirks_a10 = { .has_reset = false, + .acp_offset = 0, }; static const struct sun4ican_quirks sun4ican_quirks_r40 = { .has_reset = true, + .acp_offset = 0, +}; + +static const struct sun4ican_quirks sun4ican_quirks_d1 = { + .has_reset = true, + .acp_offset = (SUN4I_REG_ACPC_ADDR_D1 - SUN4I_REG_ACPC_ADDR), }; static const struct of_device_id sun4ican_of_match[] = { @@ -785,6 +796,9 @@ static const struct of_device_id sun4ican_of_match[] = { .compatible = "allwinner,sun8i-r40-can", .data = &sun4ican_quirks_r40 }, { + .compatible = "allwinner,sun20i-d1-can", + .data = &sun4ican_quirks_d1 + }, { /* sentinel */ }, }; @@ -870,6 +884,7 @@ static int sun4ican_probe(struct platform_device *pdev) priv->base = addr; priv->clk = clk; priv->reset = reset; + priv->acp_offset = quirks->acp_offset; spin_lock_init(&priv->cmdreg_lock); platform_set_drvdata(pdev, dev); @@ -907,4 +922,4 @@ module_platform_driver(sun4i_can_driver); MODULE_AUTHOR("Peter Chen <xingkongcp@gmail.com>"); MODULE_AUTHOR("Gerhard Bertelsmann <info@gerhard-bertelsmann.de>"); MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("CAN driver for Allwinner SoCs (A10/A20)"); +MODULE_DESCRIPTION("CAN driver for Allwinner SoCs (A10/A20/D1)"); diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index 54284661992e..5aab440074c6 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -21,7 +21,6 @@ #include <linux/clk.h> #include <linux/io.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/regulator/consumer.h> #include <linux/can/dev.h> @@ -748,8 +747,8 @@ static irqreturn_t ti_hecc_interrupt(int irq, void *dev_id) spin_unlock_irqrestore(&priv->mbx_lock, flags); stamp = hecc_read_stamp(priv, mbxno); stats->tx_bytes += - can_rx_offload_get_echo_skb(&priv->offload, - mbxno, stamp, NULL); + can_rx_offload_get_echo_skb_queue_timestamp(&priv->offload, + mbxno, stamp, NULL); stats->tx_packets++; --priv->tx_tail; } diff --git a/drivers/net/can/usb/Kconfig b/drivers/net/can/usb/Kconfig index 58fcd2b34820..d1450722cb3c 100644 --- a/drivers/net/can/usb/Kconfig +++ b/drivers/net/can/usb/Kconfig @@ -52,6 +52,7 @@ config CAN_F81604 config CAN_GS_USB tristate "Geschwister Schneider UG and candleLight compatible interfaces" + select CAN_RX_OFFLOAD help This driver supports the Geschwister Schneider and bytewerk.org candleLight compatible diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index bd9eb066ecf1..95b0fdb602c8 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -5,6 +5,7 @@ * Copyright (C) 2013-2016 Geschwister Schneider Technologie-, * Entwicklungs- und Vertriebs UG (Haftungsbeschränkt). * Copyright (C) 2016 Hubert Denkmair + * Copyright (c) 2023 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de> * * Many thanks to all socketcan devs! */ @@ -24,6 +25,7 @@ #include <linux/can.h> #include <linux/can/dev.h> #include <linux/can/error.h> +#include <linux/can/rx-offload.h> /* Device specific constants */ #define USB_GS_USB_1_VENDOR_ID 0x1d50 @@ -282,6 +284,8 @@ struct gs_host_frame { #define GS_MAX_TX_URBS 10 /* Only launch a max of GS_MAX_RX_URBS usb requests at a time. */ #define GS_MAX_RX_URBS 30 +#define GS_NAPI_WEIGHT 32 + /* Maximum number of interfaces the driver supports per device. * Current hardware only supports 3 interfaces. The future may vary. */ @@ -295,6 +299,7 @@ struct gs_tx_context { struct gs_can { struct can_priv can; /* must be the first member */ + struct can_rx_offload offload; struct gs_usb *parent; struct net_device *netdev; @@ -506,27 +511,64 @@ static void gs_update_state(struct gs_can *dev, struct can_frame *cf) } } -static void gs_usb_set_timestamp(struct gs_can *dev, struct sk_buff *skb, - const struct gs_host_frame *hf) +static u32 gs_usb_set_timestamp(struct gs_can *dev, struct sk_buff *skb, + const struct gs_host_frame *hf) { u32 timestamp; - if (!(dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)) - return; - if (hf->flags & GS_CAN_FLAG_FD) timestamp = le32_to_cpu(hf->canfd_ts->timestamp_us); else timestamp = le32_to_cpu(hf->classic_can_ts->timestamp_us); - gs_usb_skb_set_timestamp(dev, skb, timestamp); + if (skb) + gs_usb_skb_set_timestamp(dev, skb, timestamp); + + return timestamp; +} + +static void gs_usb_rx_offload(struct gs_can *dev, struct sk_buff *skb, + const struct gs_host_frame *hf) +{ + struct can_rx_offload *offload = &dev->offload; + int rc; + + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) { + const u32 ts = gs_usb_set_timestamp(dev, skb, hf); + + rc = can_rx_offload_queue_timestamp(offload, skb, ts); + } else { + rc = can_rx_offload_queue_tail(offload, skb); + } + + if (rc) + dev->netdev->stats.rx_fifo_errors++; +} + +static unsigned int +gs_usb_get_echo_skb(struct gs_can *dev, struct sk_buff *skb, + const struct gs_host_frame *hf) +{ + struct can_rx_offload *offload = &dev->offload; + const u32 echo_id = hf->echo_id; + unsigned int len; + + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) { + const u32 ts = gs_usb_set_timestamp(dev, skb, hf); + + len = can_rx_offload_get_echo_skb_queue_timestamp(offload, echo_id, + ts, NULL); + } else { + len = can_rx_offload_get_echo_skb_queue_tail(offload, echo_id, + NULL); + } - return; + return len; } static void gs_usb_receive_bulk_callback(struct urb *urb) { - struct gs_usb *usbcan = urb->context; + struct gs_usb *parent = urb->context; struct gs_can *dev; struct net_device *netdev; int rc; @@ -537,7 +579,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) struct canfd_frame *cfd; struct sk_buff *skb; - BUG_ON(!usbcan); + BUG_ON(!parent); switch (urb->status) { case 0: /* success */ @@ -554,7 +596,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) if (hf->channel >= GS_MAX_INTF) goto device_detach; - dev = usbcan->canch[hf->channel]; + dev = parent->canch[hf->channel]; netdev = dev->netdev; stats = &netdev->stats; @@ -567,7 +609,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) if (hf->echo_id == -1) { /* normal rx */ if (hf->flags & GS_CAN_FLAG_FD) { - skb = alloc_canfd_skb(dev->netdev, &cfd); + skb = alloc_canfd_skb(netdev, &cfd); if (!skb) return; @@ -580,7 +622,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) memcpy(cfd->data, hf->canfd->data, cfd->len); } else { - skb = alloc_can_skb(dev->netdev, &cf); + skb = alloc_can_skb(netdev, &cf); if (!skb) return; @@ -594,12 +636,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) gs_update_state(dev, cf); } - gs_usb_set_timestamp(dev, skb, hf); - - netdev->stats.rx_packets++; - netdev->stats.rx_bytes += hf->can_dlc; - - netif_rx(skb); + gs_usb_rx_offload(dev, skb, hf); } else { /* echo_id == hf->echo_id */ if (hf->echo_id >= GS_MAX_TX_URBS) { netdev_err(netdev, @@ -619,12 +656,8 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) } skb = dev->can.echo_skb[hf->echo_id]; - gs_usb_set_timestamp(dev, skb, hf); - - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += can_get_echo_skb(netdev, hf->echo_id, - NULL); - + stats->tx_packets++; + stats->tx_bytes += gs_usb_get_echo_skb(dev, skb, hf); gs_free_tx_context(txc); atomic_dec(&dev->active_tx_urbs); @@ -633,6 +666,9 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) } if (hf->flags & GS_CAN_FLAG_OVERFLOW) { + stats->rx_over_errors++; + stats->rx_errors++; + skb = alloc_can_err_skb(netdev, &cf); if (!skb) goto resubmit_urb; @@ -640,25 +676,26 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) cf->can_id |= CAN_ERR_CRTL; cf->len = CAN_ERR_DLC; cf->data[1] = CAN_ERR_CRTL_RX_OVERFLOW; - stats->rx_over_errors++; - stats->rx_errors++; - netif_rx(skb); + + gs_usb_rx_offload(dev, skb, hf); } - resubmit_urb: - usb_fill_bulk_urb(urb, usbcan->udev, - usb_rcvbulkpipe(usbcan->udev, GS_USB_ENDPOINT_IN), + can_rx_offload_irq_finish(&dev->offload); + +resubmit_urb: + usb_fill_bulk_urb(urb, parent->udev, + usb_rcvbulkpipe(parent->udev, GS_USB_ENDPOINT_IN), hf, dev->parent->hf_size_rx, - gs_usb_receive_bulk_callback, usbcan); + gs_usb_receive_bulk_callback, parent); rc = usb_submit_urb(urb, GFP_ATOMIC); /* USB failure take down all interfaces */ if (rc == -ENODEV) { - device_detach: +device_detach: for (rc = 0; rc < GS_MAX_INTF; rc++) { - if (usbcan->canch[rc]) - netif_device_detach(usbcan->canch[rc]->netdev); + if (parent->canch[rc]) + netif_device_detach(parent->canch[rc]->netdev); } } } @@ -742,10 +779,8 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, goto nomem_urb; hf = kmalloc(dev->hf_size_tx, GFP_ATOMIC); - if (!hf) { - netdev_err(netdev, "No memory left for USB buffer\n"); + if (!hf) goto nomem_hf; - } idx = txc->echo_id; @@ -818,12 +853,12 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; - badidx: +badidx: kfree(hf); - nomem_hf: +nomem_hf: usb_free_urb(urb); - nomem_urb: +nomem_urb: gs_free_tx_context(txc); dev_kfree_skb(skb); stats->tx_dropped++; @@ -860,6 +895,8 @@ static int gs_can_open(struct net_device *netdev) dev->hf_size_tx = struct_size(hf, classic_can, 1); } + can_rx_offload_enable(&dev->offload); + if (!parent->active_channels) { if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) gs_usb_timestamp_init(parent); @@ -878,8 +915,6 @@ static int gs_can_open(struct net_device *netdev) buf = kmalloc(dev->parent->hf_size_rx, GFP_KERNEL); if (!buf) { - netdev_err(netdev, - "No memory left for USB buffer\n"); rc = -ENOMEM; goto out_usb_free_urb; } @@ -902,7 +937,8 @@ static int gs_can_open(struct net_device *netdev) netif_device_detach(dev->netdev); netdev_err(netdev, - "usb_submit failed (err=%d)\n", rc); + "usb_submit_urb() failed, error %pe\n", + ERR_PTR(rc)); goto out_usb_unanchor_urb; } @@ -969,6 +1005,7 @@ out_usb_kill_anchored_urbs: gs_usb_timestamp_stop(parent); } + can_rx_offload_disable(&dev->offload); close_candev(netdev); return rc; @@ -1033,9 +1070,7 @@ static int gs_can_close(struct net_device *netdev) dev->can.state = CAN_STATE_STOPPED; /* reset the device */ - rc = gs_cmd_reset(dev); - if (rc < 0) - netdev_warn(netdev, "Couldn't shutdown device (err=%d)", rc); + gs_cmd_reset(dev); /* reset tx contexts */ for (rc = 0; rc < GS_MAX_TX_URBS; rc++) { @@ -1043,6 +1078,8 @@ static int gs_can_close(struct net_device *netdev) dev->tx_context[rc].echo_id = GS_MAX_TX_URBS; } + can_rx_offload_disable(&dev->offload); + /* close the netdev */ close_candev(netdev); @@ -1342,6 +1379,7 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->can.data_bittiming_const = &dev->data_bt_const; } + can_rx_offload_add_manual(netdev, &dev->offload, GS_NAPI_WEIGHT); SET_NETDEV_DEV(netdev, &intf->dev); rc = register_candev(dev->netdev); @@ -1349,12 +1387,14 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev_err(&intf->dev, "Couldn't register candev for channel %d (%pe)\n", channel, ERR_PTR(rc)); - goto out_free_candev; + goto out_can_rx_offload_del; } return dev; - out_free_candev: +out_can_rx_offload_del: + can_rx_offload_del(&dev->offload); +out_free_candev: free_candev(dev->netdev); return ERR_PTR(rc); } @@ -1362,7 +1402,7 @@ static struct gs_can *gs_make_candev(unsigned int channel, static void gs_destroy_candev(struct gs_can *dev) { unregister_candev(dev->netdev); - usb_kill_anchored_urbs(&dev->tx_submitted); + can_rx_offload_del(&dev->offload); free_candev(dev->netdev); } @@ -1371,7 +1411,7 @@ static int gs_usb_probe(struct usb_interface *intf, { struct usb_device *udev = interface_to_usbdev(intf); struct gs_host_frame *hf; - struct gs_usb *dev; + struct gs_usb *parent; struct gs_host_config hconf = { .byte_order = cpu_to_le32(0x0000beef), }; @@ -1414,49 +1454,49 @@ static int gs_usb_probe(struct usb_interface *intf, return -EINVAL; } - dev = kzalloc(sizeof(*dev), GFP_KERNEL); - if (!dev) + parent = kzalloc(sizeof(*parent), GFP_KERNEL); + if (!parent) return -ENOMEM; - init_usb_anchor(&dev->rx_submitted); + init_usb_anchor(&parent->rx_submitted); - usb_set_intfdata(intf, dev); - dev->udev = udev; + usb_set_intfdata(intf, parent); + parent->udev = udev; for (i = 0; i < icount; i++) { unsigned int hf_size_rx = 0; - dev->canch[i] = gs_make_candev(i, intf, &dconf); - if (IS_ERR_OR_NULL(dev->canch[i])) { + parent->canch[i] = gs_make_candev(i, intf, &dconf); + if (IS_ERR_OR_NULL(parent->canch[i])) { /* save error code to return later */ - rc = PTR_ERR(dev->canch[i]); + rc = PTR_ERR(parent->canch[i]); /* on failure destroy previously created candevs */ icount = i; for (i = 0; i < icount; i++) - gs_destroy_candev(dev->canch[i]); + gs_destroy_candev(parent->canch[i]); - usb_kill_anchored_urbs(&dev->rx_submitted); - kfree(dev); + usb_kill_anchored_urbs(&parent->rx_submitted); + kfree(parent); return rc; } - dev->canch[i]->parent = dev; + parent->canch[i]->parent = parent; /* set RX packet size based on FD and if hardware - * timestamps are supported. - */ - if (dev->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD) { - if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + * timestamps are supported. + */ + if (parent->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD) { + if (parent->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) hf_size_rx = struct_size(hf, canfd_ts, 1); else hf_size_rx = struct_size(hf, canfd, 1); } else { - if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + if (parent->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) hf_size_rx = struct_size(hf, classic_can_ts, 1); else hf_size_rx = struct_size(hf, classic_can, 1); } - dev->hf_size_rx = max(dev->hf_size_rx, hf_size_rx); + parent->hf_size_rx = max(parent->hf_size_rx, hf_size_rx); } return 0; @@ -1464,22 +1504,21 @@ static int gs_usb_probe(struct usb_interface *intf, static void gs_usb_disconnect(struct usb_interface *intf) { - struct gs_usb *dev = usb_get_intfdata(intf); + struct gs_usb *parent = usb_get_intfdata(intf); unsigned int i; usb_set_intfdata(intf, NULL); - if (!dev) { + if (!parent) { dev_err(&intf->dev, "Disconnect (nodata)\n"); return; } for (i = 0; i < GS_MAX_INTF; i++) - if (dev->canch[i]) - gs_destroy_candev(dev->canch[i]); + if (parent->canch[i]) + gs_destroy_candev(parent->canch[i]); - usb_kill_anchored_urbs(&dev->rx_submitted); - kfree(dev); + kfree(parent); } static const struct usb_device_id gs_usb_table[] = { diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index d881e1d30183..24ad9f593a77 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -214,19 +214,6 @@ void peak_usb_get_ts_time(struct peak_time_ref *time_ref, u32 ts, ktime_t *time) } } -/* - * post received skb after having set any hw timestamp - */ -int peak_usb_netif_rx(struct sk_buff *skb, - struct peak_time_ref *time_ref, u32 ts_low) -{ - struct skb_shared_hwtstamps *hwts = skb_hwtstamps(skb); - - peak_usb_get_ts_time(time_ref, ts_low, &hwts->hwtstamp); - - return netif_rx(skb); -} - /* post received skb with native 64-bit hw timestamp */ int peak_usb_netif_rx_64(struct sk_buff *skb, u32 ts_low, u32 ts_high) { diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.h b/drivers/net/can/usb/peak_usb/pcan_usb_core.h index 980e315186cf..f6cf84bb718f 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.h +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.h @@ -142,8 +142,6 @@ void peak_usb_init_time_ref(struct peak_time_ref *time_ref, void peak_usb_update_ts_now(struct peak_time_ref *time_ref, u32 ts_now); void peak_usb_set_ts_now(struct peak_time_ref *time_ref, u32 ts_now); void peak_usb_get_ts_time(struct peak_time_ref *time_ref, u32 ts, ktime_t *tv); -int peak_usb_netif_rx(struct sk_buff *skb, - struct peak_time_ref *time_ref, u32 ts_low); int peak_usb_netif_rx_64(struct sk_buff *skb, u32 ts_low, u32 ts_high); void peak_usb_async_complete(struct urb *urb); void peak_usb_restart_complete(struct peak_usb_device *dev); diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index a0f7bcec719c..39a63b7313a4 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -284,7 +284,7 @@ struct ucan_priv { */ spinlock_t echo_skb_lock; - /* usb device information information */ + /* usb device information */ u8 intf_index; u8 in_ep_addr; u8 out_ep_addr; diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index 4d3283db3a13..abe58f103043 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -30,6 +30,7 @@ #include <linux/can/error.h> #include <linux/phy/phy.h> #include <linux/pm_runtime.h> +#include <linux/reset.h> #define DRIVER_NAME "xilinx_can" @@ -200,6 +201,7 @@ struct xcan_devtype_data { * @can_clk: Pointer to struct clk * @devtype: Device type specific constants * @transceiver: Optional pointer to associated CAN transceiver + * @rstc: Pointer to reset control */ struct xcan_priv { struct can_priv can; @@ -218,6 +220,7 @@ struct xcan_priv { struct clk *can_clk; struct xcan_devtype_data devtype; struct phy *transceiver; + struct reset_control *rstc; }; /* CAN Bittiming constants as per Xilinx CAN specs */ @@ -1799,6 +1802,16 @@ static int xcan_probe(struct platform_device *pdev) priv->can.do_get_berr_counter = xcan_get_berr_counter; priv->can.ctrlmode_supported = CAN_CTRLMODE_LOOPBACK | CAN_CTRLMODE_BERR_REPORTING; + priv->rstc = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + if (IS_ERR(priv->rstc)) { + dev_err(&pdev->dev, "Cannot get CAN reset.\n"); + ret = PTR_ERR(priv->rstc); + goto err_free; + } + + ret = reset_control_reset(priv->rstc); + if (ret) + goto err_free; if (devtype->cantype == XAXI_CANFD) { priv->can.data_bittiming_const = @@ -1827,7 +1840,7 @@ static int xcan_probe(struct platform_device *pdev) /* Get IRQ for the device */ ret = platform_get_irq(pdev, 0); if (ret < 0) - goto err_free; + goto err_reset; ndev->irq = ret; @@ -1843,21 +1856,21 @@ static int xcan_probe(struct platform_device *pdev) if (IS_ERR(priv->can_clk)) { ret = dev_err_probe(&pdev->dev, PTR_ERR(priv->can_clk), "device clock not found\n"); - goto err_free; + goto err_reset; } priv->bus_clk = devm_clk_get(&pdev->dev, devtype->bus_clk_name); if (IS_ERR(priv->bus_clk)) { ret = dev_err_probe(&pdev->dev, PTR_ERR(priv->bus_clk), "bus clock not found\n"); - goto err_free; + goto err_reset; } transceiver = devm_phy_optional_get(&pdev->dev, NULL); if (IS_ERR(transceiver)) { ret = PTR_ERR(transceiver); dev_err_probe(&pdev->dev, ret, "failed to get phy\n"); - goto err_free; + goto err_reset; } priv->transceiver = transceiver; @@ -1904,6 +1917,8 @@ static int xcan_probe(struct platform_device *pdev) err_disableclks: pm_runtime_put(priv->dev); pm_runtime_disable(&pdev->dev); +err_reset: + reset_control_assert(priv->rstc); err_free: free_candev(ndev); err: @@ -1920,9 +1935,11 @@ err: static void xcan_remove(struct platform_device *pdev) { struct net_device *ndev = platform_get_drvdata(pdev); + struct xcan_priv *priv = netdev_priv(ndev); unregister_candev(ndev); pm_runtime_disable(&pdev->dev); + reset_control_assert(priv->rstc); free_candev(ndev); } diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 3464ce5e7470..4e27dc913cf7 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -1393,12 +1393,6 @@ static void b53_phylink_get_caps(struct dsa_switch *ds, int port, /* Get the implementation specific capabilities */ if (dev->ops->phylink_get_caps) dev->ops->phylink_get_caps(dev, port, config); - - /* This driver does not make use of the speed, duplex, pause or the - * advertisement in its mac_config, so it is safe to mark this driver - * as non-legacy. - */ - config->legacy_pre_march2020 = false; } static struct phylink_pcs *b53_phylink_mac_select_pcs(struct dsa_switch *ds, diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c index 8b422b298cd5..4d55d8d18376 100644 --- a/drivers/net/dsa/b53/b53_mdio.c +++ b/drivers/net/dsa/b53/b53_mdio.c @@ -19,6 +19,7 @@ #include <linux/kernel.h> #include <linux/phy.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/delay.h> #include <linux/brcmphy.h> #include <linux/rtnetlink.h> diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c index 5db1ed26f03a..5e39641ea887 100644 --- a/drivers/net/dsa/b53/b53_mmap.c +++ b/drivers/net/dsa/b53/b53_mmap.c @@ -19,6 +19,7 @@ #include <linux/bits.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/io.h> #include <linux/platform_device.h> #include <linux/platform_data/b53.h> diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c index af50001ccdd4..720f4e4ed0b0 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.c +++ b/drivers/net/dsa/hirschmann/hellcreek.c @@ -11,7 +11,6 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/platform_device.h> #include <linux/bitops.h> diff --git a/drivers/net/dsa/hirschmann/hellcreek_hwtstamp.c b/drivers/net/dsa/hirschmann/hellcreek_hwtstamp.c index ffd06cf8c44f..bd7aacc71a63 100644 --- a/drivers/net/dsa/hirschmann/hellcreek_hwtstamp.c +++ b/drivers/net/dsa/hirschmann/hellcreek_hwtstamp.c @@ -298,17 +298,10 @@ static void hellcreek_get_rxts(struct hellcreek *hellcreek, struct sk_buff_head received; unsigned long flags; - /* The latched timestamp belongs to one of the received frames. */ + /* Construct Rx timestamps for all received PTP packets. */ __skb_queue_head_init(&received); - - /* Lock & disable interrupts */ spin_lock_irqsave(&rxq->lock, flags); - - /* Add the reception queue "rxq" to the "received" queue an reintialize - * "rxq". From now on, we deal with "received" not with "rxq" - */ skb_queue_splice_tail_init(rxq, &received); - spin_unlock_irqrestore(&rxq->lock, flags); for (; skb; skb = __skb_dequeue(&received)) { diff --git a/drivers/net/dsa/hirschmann/hellcreek_ptp.c b/drivers/net/dsa/hirschmann/hellcreek_ptp.c index 3e44ccb7db84..5249a1c2a80b 100644 --- a/drivers/net/dsa/hirschmann/hellcreek_ptp.c +++ b/drivers/net/dsa/hirschmann/hellcreek_ptp.c @@ -9,6 +9,7 @@ * Kurt Kanzenbach <kurt@linutronix.de> */ +#include <linux/of.h> #include <linux/ptp_clock_kernel.h> #include "hellcreek.h" #include "hellcreek_ptp.h" diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c index ff76444057d2..ee67adeb2cdb 100644 --- a/drivers/net/dsa/lan9303-core.c +++ b/drivers/net/dsa/lan9303-core.c @@ -8,6 +8,7 @@ #include <linux/regmap.h> #include <linux/mutex.h> #include <linux/mii.h> +#include <linux/of.h> #include <linux/phy.h> #include <linux/if_bridge.h> #include <linux/if_vlan.h> @@ -1290,12 +1291,6 @@ static void lan9303_phylink_get_caps(struct dsa_switch *ds, int port, __set_bit(PHY_INTERFACE_MODE_GMII, config->supported_interfaces); } - - /* This driver does not make use of the speed, duplex, pause or the - * advertisement in its mac_config, so it is safe to mark this driver - * as non-legacy. - */ - config->legacy_pre_march2020 = false; } static void lan9303_phylink_mac_link_up(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/microchip/ksz8863_smi.c b/drivers/net/dsa/microchip/ksz8863_smi.c index fd6e2e69a42a..5711a59e2ac9 100644 --- a/drivers/net/dsa/microchip/ksz8863_smi.c +++ b/drivers/net/dsa/microchip/ksz8863_smi.c @@ -5,6 +5,9 @@ * Copyright (C) 2019 Pengutronix, Michael Grzeschik <kernel@pengutronix.de> */ +#include <linux/mod_devicetable.h> +#include <linux/property.h> + #include "ksz8.h" #include "ksz_common.h" diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 6c0623f88654..6673122266b7 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -18,8 +18,8 @@ #include <linux/if_vlan.h> #include <linux/irq.h> #include <linux/irqdomain.h> +#include <linux/of.h> #include <linux/of_mdio.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/micrel_phy.h> #include <net/dsa.h> @@ -1619,8 +1619,6 @@ static void ksz_phylink_get_caps(struct dsa_switch *ds, int port, { struct ksz_device *dev = ds->priv; - config->legacy_pre_march2020 = false; - if (dev->info->supports_mii[port]) __set_bit(PHY_INTERFACE_MODE_MII, config->supported_interfaces); diff --git a/drivers/net/dsa/mt7530-mmio.c b/drivers/net/dsa/mt7530-mmio.c index 1a3d4b692f34..0a6a2fe34e64 100644 --- a/drivers/net/dsa/mt7530-mmio.c +++ b/drivers/net/dsa/mt7530-mmio.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/regulator/consumer.h> #include <linux/reset.h> diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 38b3c6dda386..8fbda739c1b3 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -2949,12 +2949,6 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port, config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD; - /* This driver does not make use of the speed, duplex, pause or the - * advertisement in its mac_config, so it is safe to mark this driver - * as non-legacy. - */ - config->legacy_pre_march2020 = false; - priv->info->mac_port_get_caps(ds, port, config); } diff --git a/drivers/net/dsa/mv88e6xxx/Makefile b/drivers/net/dsa/mv88e6xxx/Makefile index 1409e691ab77..a9a9651187db 100644 --- a/drivers/net/dsa/mv88e6xxx/Makefile +++ b/drivers/net/dsa/mv88e6xxx/Makefile @@ -9,6 +9,9 @@ mv88e6xxx-objs += global2.o mv88e6xxx-objs += global2_avb.o mv88e6xxx-objs += global2_scratch.o mv88e6xxx-$(CONFIG_NET_DSA_MV88E6XXX_PTP) += hwtstamp.o +mv88e6xxx-objs += pcs-6185.o +mv88e6xxx-objs += pcs-6352.o +mv88e6xxx-objs += pcs-639x.o mv88e6xxx-objs += phy.o mv88e6xxx-objs += port.o mv88e6xxx-objs += port_hidden.o diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index c7d51a539451..bce9c9e43752 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -23,7 +23,7 @@ #include <linux/list.h> #include <linux/mdio.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/platform_data/mv88e6xxx.h> @@ -499,81 +499,6 @@ static int mv88e6xxx_port_ppu_updates(struct mv88e6xxx_chip *chip, int port) return !!(reg & MV88E6XXX_PORT_STS_PHY_DETECT); } -static int mv88e6xxx_serdes_pcs_get_state(struct dsa_switch *ds, int port, - struct phylink_link_state *state) -{ - struct mv88e6xxx_chip *chip = ds->priv; - int lane; - int err; - - mv88e6xxx_reg_lock(chip); - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0 && chip->info->ops->serdes_pcs_get_state) - err = chip->info->ops->serdes_pcs_get_state(chip, port, lane, - state); - else - err = -EOPNOTSUPP; - mv88e6xxx_reg_unlock(chip); - - return err; -} - -static int mv88e6xxx_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port, - unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise) -{ - const struct mv88e6xxx_ops *ops = chip->info->ops; - int lane; - - if (ops->serdes_pcs_config) { - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0) - return ops->serdes_pcs_config(chip, port, lane, mode, - interface, advertise); - } - - return 0; -} - -static void mv88e6xxx_serdes_pcs_an_restart(struct dsa_switch *ds, int port) -{ - struct mv88e6xxx_chip *chip = ds->priv; - const struct mv88e6xxx_ops *ops; - int err = 0; - int lane; - - ops = chip->info->ops; - - if (ops->serdes_pcs_an_restart) { - mv88e6xxx_reg_lock(chip); - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0) - err = ops->serdes_pcs_an_restart(chip, port, lane); - mv88e6xxx_reg_unlock(chip); - - if (err) - dev_err(ds->dev, "p%d: failed to restart AN\n", port); - } -} - -static int mv88e6xxx_serdes_pcs_link_up(struct mv88e6xxx_chip *chip, int port, - unsigned int mode, - int speed, int duplex) -{ - const struct mv88e6xxx_ops *ops = chip->info->ops; - int lane; - - if (!phylink_autoneg_inband(mode) && ops->serdes_pcs_link_up) { - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0) - return ops->serdes_pcs_link_up(chip, port, lane, - speed, duplex); - } - - return 0; -} - static const u8 mv88e6185_phy_interface_modes[] = { [MV88E6185_PORT_STS_CMODE_GMII_FD] = PHY_INTERFACE_MODE_GMII, [MV88E6185_PORT_STS_CMODE_MII_100_FD_PS] = PHY_INTERFACE_MODE_MII, @@ -853,6 +778,20 @@ static void mv88e6xxx_get_caps(struct dsa_switch *ds, int port, } } +static struct phylink_pcs *mv88e6xxx_mac_select_pcs(struct dsa_switch *ds, + int port, + phy_interface_t interface) +{ + struct mv88e6xxx_chip *chip = ds->priv; + struct phylink_pcs *pcs = ERR_PTR(-EOPNOTSUPP); + + if (chip->info->ops->pcs_ops) + pcs = chip->info->ops->pcs_ops->pcs_select(chip, port, + interface); + + return pcs; +} + static int mv88e6xxx_mac_prepare(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface) { @@ -889,16 +828,6 @@ static void mv88e6xxx_mac_config(struct dsa_switch *ds, int port, state->interface); if (err && err != -EOPNOTSUPP) goto err_unlock; - - err = mv88e6xxx_serdes_pcs_config(chip, port, mode, - state->interface, - state->advertising); - /* FIXME: we should restart negotiation if something changed - - * which is something we get if we convert to using phylinks - * PCS operations. - */ - if (err > 0) - err = 0; } err_unlock: @@ -982,17 +911,6 @@ static void mv88e6xxx_mac_link_up(struct dsa_switch *ds, int port, */ if (!mv88e6xxx_port_ppu_updates(chip, port) || mode == MLO_AN_FIXED) { - /* FIXME: for an automedia port, should we force the link - * down here - what if the link comes up due to "other" media - * while we're bringing the port up, how is the exclusivity - * handled in the Marvell hardware? E.g. port 2 on 88E6390 - * shared between internal PHY and Serdes. - */ - err = mv88e6xxx_serdes_pcs_link_up(chip, port, mode, speed, - duplex); - if (err) - goto error; - if (ops->port_set_speed_duplex) { err = ops->port_set_speed_duplex(chip, port, speed, duplex); @@ -3163,102 +3081,6 @@ static int mv88e6xxx_setup_egress_floods(struct mv88e6xxx_chip *chip, int port) return 0; } -static irqreturn_t mv88e6xxx_serdes_irq_thread_fn(int irq, void *dev_id) -{ - struct mv88e6xxx_port *mvp = dev_id; - struct mv88e6xxx_chip *chip = mvp->chip; - irqreturn_t ret = IRQ_NONE; - int port = mvp->port; - int lane; - - mv88e6xxx_reg_lock(chip); - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0) - ret = mv88e6xxx_serdes_irq_status(chip, port, lane); - mv88e6xxx_reg_unlock(chip); - - return ret; -} - -static int mv88e6xxx_serdes_irq_request(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - struct mv88e6xxx_port *dev_id = &chip->ports[port]; - unsigned int irq; - int err; - - /* Nothing to request if this SERDES port has no IRQ */ - irq = mv88e6xxx_serdes_irq_mapping(chip, port); - if (!irq) - return 0; - - snprintf(dev_id->serdes_irq_name, sizeof(dev_id->serdes_irq_name), - "mv88e6xxx-%s-serdes-%d", dev_name(chip->dev), port); - - /* Requesting the IRQ will trigger IRQ callbacks, so release the lock */ - mv88e6xxx_reg_unlock(chip); - err = request_threaded_irq(irq, NULL, mv88e6xxx_serdes_irq_thread_fn, - IRQF_ONESHOT, dev_id->serdes_irq_name, - dev_id); - mv88e6xxx_reg_lock(chip); - if (err) - return err; - - dev_id->serdes_irq = irq; - - return mv88e6xxx_serdes_irq_enable(chip, port, lane); -} - -static int mv88e6xxx_serdes_irq_free(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - struct mv88e6xxx_port *dev_id = &chip->ports[port]; - unsigned int irq = dev_id->serdes_irq; - int err; - - /* Nothing to free if no IRQ has been requested */ - if (!irq) - return 0; - - err = mv88e6xxx_serdes_irq_disable(chip, port, lane); - - /* Freeing the IRQ will trigger IRQ callbacks, so release the lock */ - mv88e6xxx_reg_unlock(chip); - free_irq(irq, dev_id); - mv88e6xxx_reg_lock(chip); - - dev_id->serdes_irq = 0; - - return err; -} - -static int mv88e6xxx_serdes_power(struct mv88e6xxx_chip *chip, int port, - bool on) -{ - int lane; - int err; - - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane < 0) - return 0; - - if (on) { - err = mv88e6xxx_serdes_power_up(chip, port, lane); - if (err) - return err; - - err = mv88e6xxx_serdes_irq_request(chip, port, lane); - } else { - err = mv88e6xxx_serdes_irq_free(chip, port, lane); - if (err) - return err; - - err = mv88e6xxx_serdes_power_down(chip, port, lane); - } - - return err; -} - static int mv88e6xxx_set_egress_port(struct mv88e6xxx_chip *chip, enum mv88e6xxx_egress_direction direction, int port) @@ -3322,56 +3144,17 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port) { struct device_node *phy_handle = NULL; struct dsa_switch *ds = chip->ds; - phy_interface_t mode; struct dsa_port *dp; - int tx_amp, speed; + int tx_amp; int err; u16 reg; chip->ports[port].chip = chip; chip->ports[port].port = port; - dp = dsa_to_port(ds, port); - - /* MAC Forcing register: don't force link, speed, duplex or flow control - * state to any particular values on physical ports, but force the CPU - * port and all DSA ports to their maximum bandwidth and full duplex. - */ - if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) { - struct phylink_config pl_config = {}; - unsigned long caps; - - chip->info->ops->phylink_get_caps(chip, port, &pl_config); - - caps = pl_config.mac_capabilities; - - if (chip->info->ops->port_max_speed_mode) - mode = chip->info->ops->port_max_speed_mode(chip, port); - else - mode = PHY_INTERFACE_MODE_NA; - - if (caps & MAC_10000FD) - speed = SPEED_10000; - else if (caps & MAC_5000FD) - speed = SPEED_5000; - else if (caps & MAC_2500FD) - speed = SPEED_2500; - else if (caps & MAC_1000) - speed = SPEED_1000; - else if (caps & MAC_100) - speed = SPEED_100; - else - speed = SPEED_10; - - err = mv88e6xxx_port_setup_mac(chip, port, LINK_FORCED_UP, - speed, DUPLEX_FULL, - PAUSE_OFF, mode); - } else { - err = mv88e6xxx_port_setup_mac(chip, port, LINK_UNFORCED, - SPEED_UNFORCED, DUPLEX_UNFORCED, - PAUSE_ON, - PHY_INTERFACE_MODE_NA); - } + err = mv88e6xxx_port_setup_mac(chip, port, LINK_UNFORCED, + SPEED_UNFORCED, DUPLEX_UNFORCED, + PAUSE_ON, PHY_INTERFACE_MODE_NA); if (err) return err; @@ -3548,6 +3331,7 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port) } if (chip->info->ops->serdes_set_tx_amplitude) { + dp = dsa_to_port(ds, port); if (dp) phy_handle = of_parse_phandle(dp->dn, "phy-handle", 0); @@ -3621,29 +3405,6 @@ static int mv88e6xxx_change_mtu(struct dsa_switch *ds, int port, int new_mtu) return ret; } -static int mv88e6xxx_port_enable(struct dsa_switch *ds, int port, - struct phy_device *phydev) -{ - struct mv88e6xxx_chip *chip = ds->priv; - int err; - - mv88e6xxx_reg_lock(chip); - err = mv88e6xxx_serdes_power(chip, port, true); - mv88e6xxx_reg_unlock(chip); - - return err; -} - -static void mv88e6xxx_port_disable(struct dsa_switch *ds, int port) -{ - struct mv88e6xxx_chip *chip = ds->priv; - - mv88e6xxx_reg_lock(chip); - if (mv88e6xxx_serdes_power(chip, port, false)) - dev_err(chip->dev, "failed to power off SERDES\n"); - mv88e6xxx_reg_unlock(chip); -} - static int mv88e6xxx_set_ageing_time(struct dsa_switch *ds, unsigned int ageing_time) { @@ -4106,12 +3867,26 @@ out_mdios: static int mv88e6xxx_port_setup(struct dsa_switch *ds, int port) { + struct mv88e6xxx_chip *chip = ds->priv; + int err; + + if (chip->info->ops->pcs_ops->pcs_init) { + err = chip->info->ops->pcs_ops->pcs_init(chip, port); + if (err) + return err; + } + return mv88e6xxx_setup_devlink_regions_port(ds, port); } static void mv88e6xxx_port_teardown(struct dsa_switch *ds, int port) { + struct mv88e6xxx_chip *chip = ds->priv; + mv88e6xxx_teardown_devlink_regions_port(ds, port); + + if (chip->info->ops->pcs_ops->pcs_teardown) + chip->info->ops->pcs_ops->pcs_teardown(chip, port); } static int mv88e6xxx_get_eeprom_len(struct dsa_switch *ds) @@ -4228,15 +4003,13 @@ static const struct mv88e6xxx_ops mv88e6095_ops = { .stats_get_strings = mv88e6095_stats_get_strings, .stats_get_stats = mv88e6095_stats_get_stats, .mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu, - .serdes_power = mv88e6185_serdes_power, - .serdes_get_lane = mv88e6185_serdes_get_lane, - .serdes_pcs_get_state = mv88e6185_serdes_pcs_get_state, .ppu_enable = mv88e6185_g1_ppu_enable, .ppu_disable = mv88e6185_g1_ppu_disable, .reset = mv88e6185_g1_reset, .vtu_getnext = mv88e6185_g1_vtu_getnext, .vtu_loadpurge = mv88e6185_g1_vtu_loadpurge, .phylink_get_caps = mv88e6095_phylink_get_caps, + .pcs_ops = &mv88e6185_pcs_ops, .set_max_frame_size = mv88e6185_g1_set_max_frame_size, }; @@ -4274,18 +4047,14 @@ static const struct mv88e6xxx_ops mv88e6097_ops = { .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, .mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu, - .serdes_power = mv88e6185_serdes_power, - .serdes_get_lane = mv88e6185_serdes_get_lane, - .serdes_pcs_get_state = mv88e6185_serdes_pcs_get_state, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6097_serdes_irq_enable, - .serdes_irq_status = mv88e6097_serdes_irq_status, .pot_clear = mv88e6xxx_g2_pot_clear, .reset = mv88e6352_g1_reset, .rmu_disable = mv88e6085_g1_rmu_disable, .vtu_getnext = mv88e6352_g1_vtu_getnext, .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .phylink_get_caps = mv88e6095_phylink_get_caps, + .pcs_ops = &mv88e6185_pcs_ops, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, .set_max_frame_size = mv88e6185_g1_set_max_frame_size, @@ -4421,16 +4190,8 @@ static const struct mv88e6xxx_ops mv88e6141_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6341_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .gpio_ops = &mv88e6352_gpio_ops, .serdes_get_sset_count = mv88e6390_serdes_get_sset_count, .serdes_get_strings = mv88e6390_serdes_get_strings, @@ -4438,6 +4199,7 @@ static const struct mv88e6xxx_ops mv88e6141_ops = { .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, .serdes_get_regs = mv88e6390_serdes_get_regs, .phylink_get_caps = mv88e6341_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6161_ops = { @@ -4618,16 +4380,11 @@ static const struct mv88e6xxx_ops mv88e6172_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_get_lane = mv88e6352_serdes_get_lane, - .serdes_pcs_get_state = mv88e6352_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6352_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6352_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6352_serdes_pcs_link_up, - .serdes_power = mv88e6352_serdes_power, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, .gpio_ops = &mv88e6352_gpio_ops, .phylink_get_caps = mv88e6352_phylink_get_caps, + .pcs_ops = &mv88e6352_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6175_ops = { @@ -4723,20 +4480,13 @@ static const struct mv88e6xxx_ops mv88e6176_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_get_lane = mv88e6352_serdes_get_lane, - .serdes_pcs_get_state = mv88e6352_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6352_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6352_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6352_serdes_pcs_link_up, - .serdes_power = mv88e6352_serdes_power, .serdes_irq_mapping = mv88e6352_serdes_irq_mapping, - .serdes_irq_enable = mv88e6352_serdes_irq_enable, - .serdes_irq_status = mv88e6352_serdes_irq_status, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, .gpio_ops = &mv88e6352_gpio_ops, .phylink_get_caps = mv88e6352_phylink_get_caps, + .pcs_ops = &mv88e6352_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6185_ops = { @@ -4766,9 +4516,6 @@ static const struct mv88e6xxx_ops mv88e6185_ops = { .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, .mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu, - .serdes_power = mv88e6185_serdes_power, - .serdes_get_lane = mv88e6185_serdes_get_lane, - .serdes_pcs_get_state = mv88e6185_serdes_pcs_get_state, .set_cascade_port = mv88e6185_g1_set_cascade_port, .ppu_enable = mv88e6185_g1_ppu_enable, .ppu_disable = mv88e6185_g1_ppu_disable, @@ -4776,6 +4523,7 @@ static const struct mv88e6xxx_ops mv88e6185_ops = { .vtu_getnext = mv88e6185_g1_vtu_getnext, .vtu_loadpurge = mv88e6185_g1_vtu_loadpurge, .phylink_get_caps = mv88e6185_phylink_get_caps, + .pcs_ops = &mv88e6185_pcs_ops, .set_max_frame_size = mv88e6185_g1_set_max_frame_size, }; @@ -4826,22 +4574,15 @@ static const struct mv88e6xxx_ops mv88e6190_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .serdes_get_strings = mv88e6390_serdes_get_strings, .serdes_get_stats = mv88e6390_serdes_get_stats, .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, .serdes_get_regs = mv88e6390_serdes_get_regs, .gpio_ops = &mv88e6352_gpio_ops, .phylink_get_caps = mv88e6390_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6190x_ops = { @@ -4891,22 +4632,15 @@ static const struct mv88e6xxx_ops mv88e6190x_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390x_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .serdes_get_strings = mv88e6390_serdes_get_strings, .serdes_get_stats = mv88e6390_serdes_get_stats, .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, .serdes_get_regs = mv88e6390_serdes_get_regs, .gpio_ops = &mv88e6352_gpio_ops, .phylink_get_caps = mv88e6390x_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6191_ops = { @@ -4954,16 +4688,8 @@ static const struct mv88e6xxx_ops mv88e6191_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .serdes_get_strings = mv88e6390_serdes_get_strings, .serdes_get_stats = mv88e6390_serdes_get_stats, .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, @@ -4971,6 +4697,7 @@ static const struct mv88e6xxx_ops mv88e6191_ops = { .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, .phylink_get_caps = mv88e6390_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6240_ops = { @@ -5020,15 +4747,7 @@ static const struct mv88e6xxx_ops mv88e6240_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_get_lane = mv88e6352_serdes_get_lane, - .serdes_pcs_get_state = mv88e6352_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6352_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6352_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6352_serdes_pcs_link_up, - .serdes_power = mv88e6352_serdes_power, .serdes_irq_mapping = mv88e6352_serdes_irq_mapping, - .serdes_irq_enable = mv88e6352_serdes_irq_enable, - .serdes_irq_status = mv88e6352_serdes_irq_status, .serdes_get_regs_len = mv88e6352_serdes_get_regs_len, .serdes_get_regs = mv88e6352_serdes_get_regs, .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, @@ -5036,6 +4755,7 @@ static const struct mv88e6xxx_ops mv88e6240_ops = { .avb_ops = &mv88e6352_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, .phylink_get_caps = mv88e6352_phylink_get_caps, + .pcs_ops = &mv88e6352_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6250_ops = { @@ -5127,16 +4847,8 @@ static const struct mv88e6xxx_ops mv88e6290_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .serdes_get_strings = mv88e6390_serdes_get_strings, .serdes_get_stats = mv88e6390_serdes_get_stats, .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, @@ -5145,6 +4857,7 @@ static const struct mv88e6xxx_ops mv88e6290_ops = { .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6390_ptp_ops, .phylink_get_caps = mv88e6390_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6320_ops = { @@ -5289,16 +5002,8 @@ static const struct mv88e6xxx_ops mv88e6341_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6341_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, @@ -5308,6 +5013,7 @@ static const struct mv88e6xxx_ops mv88e6341_ops = { .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, .serdes_get_regs = mv88e6390_serdes_get_regs, .phylink_get_caps = mv88e6341_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6350_ops = { @@ -5451,15 +5157,7 @@ static const struct mv88e6xxx_ops mv88e6352_ops = { .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, .stu_loadpurge = mv88e6352_g1_stu_loadpurge, - .serdes_get_lane = mv88e6352_serdes_get_lane, - .serdes_pcs_get_state = mv88e6352_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6352_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6352_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6352_serdes_pcs_link_up, - .serdes_power = mv88e6352_serdes_power, .serdes_irq_mapping = mv88e6352_serdes_irq_mapping, - .serdes_irq_enable = mv88e6352_serdes_irq_enable, - .serdes_irq_status = mv88e6352_serdes_irq_status, .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6352_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, @@ -5470,6 +5168,7 @@ static const struct mv88e6xxx_ops mv88e6352_ops = { .serdes_get_regs = mv88e6352_serdes_get_regs, .serdes_set_tx_amplitude = mv88e6352_serdes_set_tx_amplitude, .phylink_get_caps = mv88e6352_phylink_get_caps, + .pcs_ops = &mv88e6352_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6390_ops = { @@ -5520,16 +5219,8 @@ static const struct mv88e6xxx_ops mv88e6390_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390_serdes_get_lane, - /* Check status register pause & lpa register */ - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6390_ptp_ops, @@ -5539,6 +5230,7 @@ static const struct mv88e6xxx_ops mv88e6390_ops = { .serdes_get_regs_len = mv88e6390_serdes_get_regs_len, .serdes_get_regs = mv88e6390_serdes_get_regs, .phylink_get_caps = mv88e6390_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6390x_ops = { @@ -5589,15 +5281,8 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6390_serdes_power, .serdes_get_lane = mv88e6390x_serdes_get_lane, - .serdes_pcs_get_state = mv88e6390_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6390_serdes_irq_enable, - .serdes_irq_status = mv88e6390_serdes_irq_status, .serdes_get_sset_count = mv88e6390_serdes_get_sset_count, .serdes_get_strings = mv88e6390_serdes_get_strings, .serdes_get_stats = mv88e6390_serdes_get_stats, @@ -5607,11 +5292,11 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = { .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6390_ptp_ops, .phylink_get_caps = mv88e6390x_phylink_get_caps, + .pcs_ops = &mv88e6390_pcs_ops, }; static const struct mv88e6xxx_ops mv88e6393x_ops = { /* MV88E6XXX_FAMILY_6393 */ - .setup_errata = mv88e6393x_serdes_setup_errata, .irl_init_all = mv88e6390_g2_irl_init_all, .get_eeprom = mv88e6xxx_g2_get_eeprom8, .set_eeprom = mv88e6xxx_g2_set_eeprom8, @@ -5661,20 +5346,14 @@ static const struct mv88e6xxx_ops mv88e6393x_ops = { .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, .stu_getnext = mv88e6390_g1_stu_getnext, .stu_loadpurge = mv88e6390_g1_stu_loadpurge, - .serdes_power = mv88e6393x_serdes_power, .serdes_get_lane = mv88e6393x_serdes_get_lane, - .serdes_pcs_get_state = mv88e6393x_serdes_pcs_get_state, - .serdes_pcs_config = mv88e6390_serdes_pcs_config, - .serdes_pcs_an_restart = mv88e6390_serdes_pcs_an_restart, - .serdes_pcs_link_up = mv88e6390_serdes_pcs_link_up, .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, - .serdes_irq_enable = mv88e6393x_serdes_irq_enable, - .serdes_irq_status = mv88e6393x_serdes_irq_status, /* TODO: serdes stats */ .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6390_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, .phylink_get_caps = mv88e6393x_phylink_get_caps, + .pcs_ops = &mv88e6393x_pcs_ops, }; static const struct mv88e6xxx_info mv88e6xxx_table[] = { @@ -7106,18 +6785,15 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = { .port_setup = mv88e6xxx_port_setup, .port_teardown = mv88e6xxx_port_teardown, .phylink_get_caps = mv88e6xxx_get_caps, - .phylink_mac_link_state = mv88e6xxx_serdes_pcs_get_state, + .phylink_mac_select_pcs = mv88e6xxx_mac_select_pcs, .phylink_mac_prepare = mv88e6xxx_mac_prepare, .phylink_mac_config = mv88e6xxx_mac_config, .phylink_mac_finish = mv88e6xxx_mac_finish, - .phylink_mac_an_restart = mv88e6xxx_serdes_pcs_an_restart, .phylink_mac_link_down = mv88e6xxx_mac_link_down, .phylink_mac_link_up = mv88e6xxx_mac_link_up, .get_strings = mv88e6xxx_get_strings, .get_ethtool_stats = mv88e6xxx_get_ethtool_stats, .get_sset_count = mv88e6xxx_get_sset_count, - .port_enable = mv88e6xxx_port_enable, - .port_disable = mv88e6xxx_port_disable, .port_max_mtu = mv88e6xxx_get_max_mtu, .port_change_mtu = mv88e6xxx_change_mtu, .get_mac_eee = mv88e6xxx_get_mac_eee, diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h index 0ad34b2d8913..44383a03ef2f 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.h +++ b/drivers/net/dsa/mv88e6xxx/chip.h @@ -205,6 +205,7 @@ struct mv88e6xxx_irq_ops; struct mv88e6xxx_gpio_ops; struct mv88e6xxx_avb_ops; struct mv88e6xxx_ptp_ops; +struct mv88e6xxx_pcs_ops; struct mv88e6xxx_irq { u16 masked; @@ -285,9 +286,8 @@ struct mv88e6xxx_port { u8 cmode; bool mirror_ingress; bool mirror_egress; - unsigned int serdes_irq; - char serdes_irq_name[64]; struct devlink_region *region; + void *pcs_private; /* MacAuth Bypass control flag */ bool mab; @@ -590,31 +590,12 @@ struct mv88e6xxx_ops { int (*mgmt_rsvd2cpu)(struct mv88e6xxx_chip *chip); - /* Power on/off a SERDES interface */ - int (*serdes_power)(struct mv88e6xxx_chip *chip, int port, int lane, - bool up); - /* SERDES lane mapping */ int (*serdes_get_lane)(struct mv88e6xxx_chip *chip, int port); - int (*serdes_pcs_get_state)(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state); - int (*serdes_pcs_config)(struct mv88e6xxx_chip *chip, int port, - int lane, unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise); - int (*serdes_pcs_an_restart)(struct mv88e6xxx_chip *chip, int port, - int lane); - int (*serdes_pcs_link_up)(struct mv88e6xxx_chip *chip, int port, - int lane, int speed, int duplex); - /* SERDES interrupt handling */ unsigned int (*serdes_irq_mapping)(struct mv88e6xxx_chip *chip, int port); - int (*serdes_irq_enable)(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable); - irqreturn_t (*serdes_irq_status)(struct mv88e6xxx_chip *chip, int port, - int lane); /* Statistics from the SERDES interface */ int (*serdes_get_sset_count)(struct mv88e6xxx_chip *chip, int port); @@ -664,6 +645,8 @@ struct mv88e6xxx_ops { void (*phylink_get_caps)(struct mv88e6xxx_chip *chip, int port, struct phylink_config *config); + const struct mv88e6xxx_pcs_ops *pcs_ops; + /* Max Frame Size */ int (*set_max_frame_size)(struct mv88e6xxx_chip *chip, int mtu); }; @@ -736,6 +719,14 @@ struct mv88e6xxx_ptp_ops { u32 cc_mult_dem; }; +struct mv88e6xxx_pcs_ops { + int (*pcs_init)(struct mv88e6xxx_chip *chip, int port); + void (*pcs_teardown)(struct mv88e6xxx_chip *chip, int port); + struct phylink_pcs *(*pcs_select)(struct mv88e6xxx_chip *chip, int port, + phy_interface_t mode); + +}; + #define STATS_TYPE_PORT BIT(0) #define STATS_TYPE_BANK0 BIT(1) #define STATS_TYPE_BANK1 BIT(2) diff --git a/drivers/net/dsa/mv88e6xxx/pcs-6185.c b/drivers/net/dsa/mv88e6xxx/pcs-6185.c new file mode 100644 index 000000000000..4d677f836807 --- /dev/null +++ b/drivers/net/dsa/mv88e6xxx/pcs-6185.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Marvell 88E6185 family SERDES PCS support + * + * Copyright (c) 2008 Marvell Semiconductor + * + * Copyright (c) 2017 Andrew Lunn <andrew@lunn.ch> + */ +#include <linux/phylink.h> + +#include "global2.h" +#include "port.h" +#include "serdes.h" + +struct mv88e6185_pcs { + struct phylink_pcs phylink_pcs; + unsigned int irq; + char name[64]; + + struct mv88e6xxx_chip *chip; + int port; +}; + +static struct mv88e6185_pcs *pcs_to_mv88e6185_pcs(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct mv88e6185_pcs, phylink_pcs); +} + +static irqreturn_t mv88e6185_pcs_handle_irq(int irq, void *dev_id) +{ + struct mv88e6185_pcs *mpcs = dev_id; + struct mv88e6xxx_chip *chip; + irqreturn_t ret = IRQ_NONE; + bool link_up; + u16 status; + int port; + int err; + + chip = mpcs->chip; + port = mpcs->port; + + mv88e6xxx_reg_lock(chip); + err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &status); + mv88e6xxx_reg_unlock(chip); + + if (!err) { + link_up = !!(status & MV88E6XXX_PORT_STS_LINK); + + phylink_pcs_change(&mpcs->phylink_pcs, link_up); + + ret = IRQ_HANDLED; + } + + return ret; +} + +static void mv88e6185_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct mv88e6185_pcs *mpcs = pcs_to_mv88e6185_pcs(pcs); + struct mv88e6xxx_chip *chip = mpcs->chip; + int port = mpcs->port; + u16 status; + int err; + + mv88e6xxx_reg_lock(chip); + err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &status); + mv88e6xxx_reg_unlock(chip); + + if (err) + status = 0; + + state->link = !!(status & MV88E6XXX_PORT_STS_LINK); + if (state->link) { + state->duplex = status & MV88E6XXX_PORT_STS_DUPLEX ? + DUPLEX_FULL : DUPLEX_HALF; + + switch (status & MV88E6XXX_PORT_STS_SPEED_MASK) { + case MV88E6XXX_PORT_STS_SPEED_1000: + state->speed = SPEED_1000; + break; + + case MV88E6XXX_PORT_STS_SPEED_100: + state->speed = SPEED_100; + break; + + case MV88E6XXX_PORT_STS_SPEED_10: + state->speed = SPEED_10; + break; + + default: + state->link = false; + break; + } + } +} + +static int mv88e6185_pcs_config(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + return 0; +} + +static void mv88e6185_pcs_an_restart(struct phylink_pcs *pcs) +{ +} + +static const struct phylink_pcs_ops mv88e6185_phylink_pcs_ops = { + .pcs_get_state = mv88e6185_pcs_get_state, + .pcs_config = mv88e6185_pcs_config, + .pcs_an_restart = mv88e6185_pcs_an_restart, +}; + +static int mv88e6185_pcs_init(struct mv88e6xxx_chip *chip, int port) +{ + struct mv88e6185_pcs *mpcs; + struct device *dev; + unsigned int irq; + int err; + + /* There are no configurable serdes lanes on this switch chip, so + * we use the static cmode configuration to determine whether we + * have a PCS or not. + */ + if (chip->ports[port].cmode != MV88E6185_PORT_STS_CMODE_SERDES && + chip->ports[port].cmode != MV88E6185_PORT_STS_CMODE_1000BASE_X) + return 0; + + dev = chip->dev; + + mpcs = kzalloc(sizeof(*mpcs), GFP_KERNEL); + if (!mpcs) + return -ENOMEM; + + mpcs->chip = chip; + mpcs->port = port; + mpcs->phylink_pcs.ops = &mv88e6185_phylink_pcs_ops; + + irq = mv88e6xxx_serdes_irq_mapping(chip, port); + if (irq) { + snprintf(mpcs->name, sizeof(mpcs->name), + "mv88e6xxx-%s-serdes-%d", dev_name(dev), port); + + err = request_threaded_irq(irq, NULL, mv88e6185_pcs_handle_irq, + IRQF_ONESHOT, mpcs->name, mpcs); + if (err) { + kfree(mpcs); + return err; + } + + mpcs->irq = irq; + } else { + mpcs->phylink_pcs.poll = true; + } + + chip->ports[port].pcs_private = &mpcs->phylink_pcs; + + return 0; +} + +static void mv88e6185_pcs_teardown(struct mv88e6xxx_chip *chip, int port) +{ + struct mv88e6185_pcs *mpcs; + + mpcs = chip->ports[port].pcs_private; + if (!mpcs) + return; + + if (mpcs->irq) + free_irq(mpcs->irq, mpcs); + + kfree(mpcs); + + chip->ports[port].pcs_private = NULL; +} + +static struct phylink_pcs *mv88e6185_pcs_select(struct mv88e6xxx_chip *chip, + int port, + phy_interface_t interface) +{ + return chip->ports[port].pcs_private; +} + +const struct mv88e6xxx_pcs_ops mv88e6185_pcs_ops = { + .pcs_init = mv88e6185_pcs_init, + .pcs_teardown = mv88e6185_pcs_teardown, + .pcs_select = mv88e6185_pcs_select, +}; diff --git a/drivers/net/dsa/mv88e6xxx/pcs-6352.c b/drivers/net/dsa/mv88e6xxx/pcs-6352.c new file mode 100644 index 000000000000..88f624b65470 --- /dev/null +++ b/drivers/net/dsa/mv88e6xxx/pcs-6352.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Marvell 88E6352 family SERDES PCS support + * + * Copyright (c) 2008 Marvell Semiconductor + * + * Copyright (c) 2017 Andrew Lunn <andrew@lunn.ch> + */ +#include <linux/phylink.h> + +#include "global2.h" +#include "port.h" +#include "serdes.h" + +/* Definitions from drivers/net/phy/marvell.c, which would be good to reuse. */ +#define MII_M1011_PHY_STATUS 17 +#define MII_M1011_IMASK 18 +#define MII_M1011_IMASK_LINK_CHANGE BIT(10) +#define MII_M1011_IEVENT 19 +#define MII_M1011_IEVENT_LINK_CHANGE BIT(10) +#define MII_MARVELL_PHY_PAGE 22 +#define MII_MARVELL_FIBER_PAGE 1 + +struct marvell_c22_pcs { + struct mdio_device mdio; + struct phylink_pcs phylink_pcs; + unsigned int irq; + char name[64]; + bool (*link_check)(struct marvell_c22_pcs *mpcs); + struct mv88e6xxx_port *port; +}; + +static struct marvell_c22_pcs *pcs_to_marvell_c22_pcs(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct marvell_c22_pcs, phylink_pcs); +} + +static int marvell_c22_pcs_set_fiber_page(struct marvell_c22_pcs *mpcs) +{ + u16 page; + int err; + + mutex_lock(&mpcs->mdio.bus->mdio_lock); + + err = __mdiodev_read(&mpcs->mdio, MII_MARVELL_PHY_PAGE); + if (err < 0) { + dev_err(mpcs->mdio.dev.parent, + "%s: can't read Serdes page register: %pe\n", + mpcs->name, ERR_PTR(err)); + return err; + } + + page = err; + + err = __mdiodev_write(&mpcs->mdio, MII_MARVELL_PHY_PAGE, + MII_MARVELL_FIBER_PAGE); + if (err) { + dev_err(mpcs->mdio.dev.parent, + "%s: can't set Serdes page register: %pe\n", + mpcs->name, ERR_PTR(err)); + return err; + } + + return page; +} + +static int marvell_c22_pcs_restore_page(struct marvell_c22_pcs *mpcs, + int oldpage, int ret) +{ + int err; + + if (oldpage >= 0) { + err = __mdiodev_write(&mpcs->mdio, MII_MARVELL_PHY_PAGE, + oldpage); + if (err) + dev_err(mpcs->mdio.dev.parent, + "%s: can't restore Serdes page register: %pe\n", + mpcs->name, ERR_PTR(err)); + if (!err || ret < 0) + err = ret; + } else { + err = oldpage; + } + mutex_unlock(&mpcs->mdio.bus->mdio_lock); + + return err; +} + +static irqreturn_t marvell_c22_pcs_handle_irq(int irq, void *dev_id) +{ + struct marvell_c22_pcs *mpcs = dev_id; + irqreturn_t status = IRQ_NONE; + int err, oldpage; + + oldpage = marvell_c22_pcs_set_fiber_page(mpcs); + if (oldpage < 0) + goto fail; + + err = __mdiodev_read(&mpcs->mdio, MII_M1011_IEVENT); + if (err >= 0 && err & MII_M1011_IEVENT_LINK_CHANGE) { + phylink_pcs_change(&mpcs->phylink_pcs, true); + status = IRQ_HANDLED; + } + +fail: + marvell_c22_pcs_restore_page(mpcs, oldpage, 0); + + return status; +} + +static int marvell_c22_pcs_modify(struct marvell_c22_pcs *mpcs, u8 reg, + u16 mask, u16 val) +{ + int oldpage, err = 0; + + oldpage = marvell_c22_pcs_set_fiber_page(mpcs); + if (oldpage >= 0) + err = __mdiodev_modify(&mpcs->mdio, reg, mask, val); + + return marvell_c22_pcs_restore_page(mpcs, oldpage, err); +} + +static int marvell_c22_pcs_power(struct marvell_c22_pcs *mpcs, + bool on) +{ + u16 val = on ? 0 : BMCR_PDOWN; + + return marvell_c22_pcs_modify(mpcs, MII_BMCR, BMCR_PDOWN, val); +} + +static int marvell_c22_pcs_control_irq(struct marvell_c22_pcs *mpcs, + bool enable) +{ + u16 val = enable ? MII_M1011_IMASK_LINK_CHANGE : 0; + + return marvell_c22_pcs_modify(mpcs, MII_M1011_IMASK, + MII_M1011_IMASK_LINK_CHANGE, val); +} + +static int marvell_c22_pcs_enable(struct phylink_pcs *pcs) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + int err; + + err = marvell_c22_pcs_power(mpcs, true); + if (err) + return err; + + return marvell_c22_pcs_control_irq(mpcs, !!mpcs->irq); +} + +static void marvell_c22_pcs_disable(struct phylink_pcs *pcs) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + + marvell_c22_pcs_control_irq(mpcs, false); + marvell_c22_pcs_power(mpcs, false); +} + +static void marvell_c22_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + int oldpage, bmsr, lpa, status; + + state->link = false; + + if (mpcs->link_check && !mpcs->link_check(mpcs)) + return; + + oldpage = marvell_c22_pcs_set_fiber_page(mpcs); + if (oldpage >= 0) { + bmsr = __mdiodev_read(&mpcs->mdio, MII_BMSR); + lpa = __mdiodev_read(&mpcs->mdio, MII_LPA); + status = __mdiodev_read(&mpcs->mdio, MII_M1011_PHY_STATUS); + } + + if (marvell_c22_pcs_restore_page(mpcs, oldpage, 0) >= 0 && + bmsr >= 0 && lpa >= 0 && status >= 0) + mv88e6xxx_pcs_decode_state(mpcs->mdio.dev.parent, bmsr, lpa, + status, state); +} + +static int marvell_c22_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + int oldpage, adv, err, ret = 0; + u16 bmcr; + + adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (adv < 0) + return 0; + + bmcr = neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED ? BMCR_ANENABLE : 0; + + oldpage = marvell_c22_pcs_set_fiber_page(mpcs); + if (oldpage < 0) + goto restore; + + err = __mdiodev_modify_changed(&mpcs->mdio, MII_ADVERTISE, 0xffff, adv); + ret = err; + if (err < 0) + goto restore; + + err = __mdiodev_modify_changed(&mpcs->mdio, MII_BMCR, BMCR_ANENABLE, + bmcr); + if (err < 0) { + ret = err; + goto restore; + } + + /* If the ANENABLE bit was changed, the PHY will restart negotiation, + * so we don't need to flag a change to trigger its own restart. + */ + if (err) + ret = 0; + +restore: + return marvell_c22_pcs_restore_page(mpcs, oldpage, ret); +} + +static void marvell_c22_pcs_an_restart(struct phylink_pcs *pcs) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + + marvell_c22_pcs_modify(mpcs, MII_BMCR, BMCR_ANRESTART, BMCR_ANRESTART); +} + +static void marvell_c22_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, int speed, + int duplex) +{ + struct marvell_c22_pcs *mpcs = pcs_to_marvell_c22_pcs(pcs); + u16 bmcr; + int err; + + if (phylink_autoneg_inband(mode)) + return; + + bmcr = mii_bmcr_encode_fixed(speed, duplex); + + err = marvell_c22_pcs_modify(mpcs, MII_BMCR, BMCR_SPEED100 | + BMCR_FULLDPLX | BMCR_SPEED1000, bmcr); + if (err) + dev_err(mpcs->mdio.dev.parent, + "%s: failed to configure mpcs: %pe\n", mpcs->name, + ERR_PTR(err)); +} + +static const struct phylink_pcs_ops marvell_c22_pcs_ops = { + .pcs_enable = marvell_c22_pcs_enable, + .pcs_disable = marvell_c22_pcs_disable, + .pcs_get_state = marvell_c22_pcs_get_state, + .pcs_config = marvell_c22_pcs_config, + .pcs_an_restart = marvell_c22_pcs_an_restart, + .pcs_link_up = marvell_c22_pcs_link_up, +}; + +static struct marvell_c22_pcs *marvell_c22_pcs_alloc(struct device *dev, + struct mii_bus *bus, + unsigned int addr) +{ + struct marvell_c22_pcs *mpcs; + + mpcs = kzalloc(sizeof(*mpcs), GFP_KERNEL); + if (!mpcs) + return NULL; + + mpcs->mdio.dev.parent = dev; + mpcs->mdio.bus = bus; + mpcs->mdio.addr = addr; + mpcs->phylink_pcs.ops = &marvell_c22_pcs_ops; + mpcs->phylink_pcs.neg_mode = true; + + return mpcs; +} + +static int marvell_c22_pcs_setup_irq(struct marvell_c22_pcs *mpcs, + unsigned int irq) +{ + int err; + + mpcs->phylink_pcs.poll = !irq; + mpcs->irq = irq; + + if (irq) { + err = request_threaded_irq(irq, NULL, + marvell_c22_pcs_handle_irq, + IRQF_ONESHOT, mpcs->name, mpcs); + if (err) + return err; + } + + return 0; +} + +/* mv88e6352 specifics */ + +static bool mv88e6352_pcs_link_check(struct marvell_c22_pcs *mpcs) +{ + struct mv88e6xxx_port *port = mpcs->port; + struct mv88e6xxx_chip *chip = port->chip; + u8 cmode; + + /* Port 4 can be in auto-media mode. Check that the port is + * associated with the mpcs. + */ + mv88e6xxx_reg_lock(chip); + chip->info->ops->port_get_cmode(chip, port->port, &cmode); + mv88e6xxx_reg_unlock(chip); + + return cmode == MV88E6XXX_PORT_STS_CMODE_100BASEX || + cmode == MV88E6XXX_PORT_STS_CMODE_1000BASEX || + cmode == MV88E6XXX_PORT_STS_CMODE_SGMII; +} + +static int mv88e6352_pcs_init(struct mv88e6xxx_chip *chip, int port) +{ + struct marvell_c22_pcs *mpcs; + struct mii_bus *bus; + struct device *dev; + unsigned int irq; + int err; + + mv88e6xxx_reg_lock(chip); + err = mv88e6352_g2_scratch_port_has_serdes(chip, port); + mv88e6xxx_reg_unlock(chip); + if (err <= 0) + return err; + + irq = mv88e6xxx_serdes_irq_mapping(chip, port); + bus = mv88e6xxx_default_mdio_bus(chip); + dev = chip->dev; + + mpcs = marvell_c22_pcs_alloc(dev, bus, MV88E6352_ADDR_SERDES); + if (!mpcs) + return -ENOMEM; + + snprintf(mpcs->name, sizeof(mpcs->name), + "mv88e6xxx-%s-serdes-%d", dev_name(dev), port); + + mpcs->link_check = mv88e6352_pcs_link_check; + mpcs->port = &chip->ports[port]; + + err = marvell_c22_pcs_setup_irq(mpcs, irq); + if (err) { + kfree(mpcs); + return err; + } + + chip->ports[port].pcs_private = &mpcs->phylink_pcs; + + return 0; +} + +static void mv88e6352_pcs_teardown(struct mv88e6xxx_chip *chip, int port) +{ + struct marvell_c22_pcs *mpcs; + struct phylink_pcs *pcs; + + pcs = chip->ports[port].pcs_private; + if (!pcs) + return; + + mpcs = pcs_to_marvell_c22_pcs(pcs); + + if (mpcs->irq) + free_irq(mpcs->irq, mpcs); + + kfree(mpcs); + + chip->ports[port].pcs_private = NULL; +} + +static struct phylink_pcs *mv88e6352_pcs_select(struct mv88e6xxx_chip *chip, + int port, + phy_interface_t interface) +{ + return chip->ports[port].pcs_private; +} + +const struct mv88e6xxx_pcs_ops mv88e6352_pcs_ops = { + .pcs_init = mv88e6352_pcs_init, + .pcs_teardown = mv88e6352_pcs_teardown, + .pcs_select = mv88e6352_pcs_select, +}; diff --git a/drivers/net/dsa/mv88e6xxx/pcs-639x.c b/drivers/net/dsa/mv88e6xxx/pcs-639x.c new file mode 100644 index 000000000000..ba373656bfe1 --- /dev/null +++ b/drivers/net/dsa/mv88e6xxx/pcs-639x.c @@ -0,0 +1,943 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Marvell 88E6352 family SERDES PCS support + * + * Copyright (c) 2008 Marvell Semiconductor + * + * Copyright (c) 2017 Andrew Lunn <andrew@lunn.ch> + */ +#include <linux/interrupt.h> +#include <linux/irqdomain.h> +#include <linux/mii.h> + +#include "chip.h" +#include "global2.h" +#include "phy.h" +#include "port.h" +#include "serdes.h" + +struct mv88e639x_pcs { + struct mdio_device mdio; + struct phylink_pcs sgmii_pcs; + struct phylink_pcs xg_pcs; + bool erratum_3_14; + bool supports_5g; + phy_interface_t interface; + unsigned int irq; + char name[64]; + irqreturn_t (*handle_irq)(struct mv88e639x_pcs *mpcs); +}; + +static int mv88e639x_read(struct mv88e639x_pcs *mpcs, u16 regnum, u16 *val) +{ + int err; + + err = mdiodev_c45_read(&mpcs->mdio, MDIO_MMD_PHYXS, regnum); + if (err < 0) + return err; + + *val = err; + + return 0; +} + +static int mv88e639x_write(struct mv88e639x_pcs *mpcs, u16 regnum, u16 val) +{ + return mdiodev_c45_write(&mpcs->mdio, MDIO_MMD_PHYXS, regnum, val); +} + +static int mv88e639x_modify(struct mv88e639x_pcs *mpcs, u16 regnum, u16 mask, + u16 val) +{ + return mdiodev_c45_modify(&mpcs->mdio, MDIO_MMD_PHYXS, regnum, mask, + val); +} + +static int mv88e639x_modify_changed(struct mv88e639x_pcs *mpcs, u16 regnum, + u16 mask, u16 set) +{ + return mdiodev_c45_modify_changed(&mpcs->mdio, MDIO_MMD_PHYXS, regnum, + mask, set); +} + +static struct mv88e639x_pcs * +mv88e639x_pcs_alloc(struct device *dev, struct mii_bus *bus, unsigned int addr, + int port) +{ + struct mv88e639x_pcs *mpcs; + + mpcs = kzalloc(sizeof(*mpcs), GFP_KERNEL); + if (!mpcs) + return NULL; + + mpcs->mdio.dev.parent = dev; + mpcs->mdio.bus = bus; + mpcs->mdio.addr = addr; + + snprintf(mpcs->name, sizeof(mpcs->name), + "mv88e6xxx-%s-serdes-%d", dev_name(dev), port); + + return mpcs; +} + +static irqreturn_t mv88e639x_pcs_handle_irq(int irq, void *dev_id) +{ + struct mv88e639x_pcs *mpcs = dev_id; + irqreturn_t (*handler)(struct mv88e639x_pcs *); + + handler = READ_ONCE(mpcs->handle_irq); + if (!handler) + return IRQ_NONE; + + return handler(mpcs); +} + +static int mv88e639x_pcs_setup_irq(struct mv88e639x_pcs *mpcs, + struct mv88e6xxx_chip *chip, int port) +{ + unsigned int irq; + + irq = mv88e6xxx_serdes_irq_mapping(chip, port); + if (!irq) { + /* Use polling mode */ + mpcs->sgmii_pcs.poll = true; + mpcs->xg_pcs.poll = true; + return 0; + } + + mpcs->irq = irq; + + return request_threaded_irq(irq, NULL, mv88e639x_pcs_handle_irq, + IRQF_ONESHOT, mpcs->name, mpcs); +} + +static void mv88e639x_pcs_teardown(struct mv88e6xxx_chip *chip, int port) +{ + struct mv88e639x_pcs *mpcs = chip->ports[port].pcs_private; + + if (!mpcs) + return; + + if (mpcs->irq) + free_irq(mpcs->irq, mpcs); + + kfree(mpcs); + + chip->ports[port].pcs_private = NULL; +} + +static struct mv88e639x_pcs *sgmii_pcs_to_mv88e639x_pcs(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct mv88e639x_pcs, sgmii_pcs); +} + +static irqreturn_t mv88e639x_sgmii_handle_irq(struct mv88e639x_pcs *mpcs) +{ + u16 int_status; + int err; + + err = mv88e639x_read(mpcs, MV88E6390_SGMII_INT_STATUS, &int_status); + if (err) + return IRQ_NONE; + + if (int_status & (MV88E6390_SGMII_INT_LINK_DOWN | + MV88E6390_SGMII_INT_LINK_UP)) { + phylink_pcs_change(&mpcs->sgmii_pcs, + int_status & MV88E6390_SGMII_INT_LINK_UP); + + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int mv88e639x_sgmii_pcs_control_irq(struct mv88e639x_pcs *mpcs, + bool enable) +{ + u16 val = 0; + + if (enable) + val |= MV88E6390_SGMII_INT_LINK_DOWN | + MV88E6390_SGMII_INT_LINK_UP; + + return mv88e639x_modify(mpcs, MV88E6390_SGMII_INT_ENABLE, + MV88E6390_SGMII_INT_LINK_DOWN | + MV88E6390_SGMII_INT_LINK_UP, val); +} + +static int mv88e639x_sgmii_pcs_control_pwr(struct mv88e639x_pcs *mpcs, + bool enable) +{ + u16 mask, val; + + if (enable) { + mask = BMCR_RESET | BMCR_LOOPBACK | BMCR_PDOWN; + val = 0; + } else { + mask = val = BMCR_PDOWN; + } + + return mv88e639x_modify(mpcs, MV88E6390_SGMII_BMCR, mask, val); +} + +static int mv88e639x_sgmii_pcs_enable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + /* power enable done in post_config */ + mpcs->handle_irq = mv88e639x_sgmii_handle_irq; + + return mv88e639x_sgmii_pcs_control_irq(mpcs, !!mpcs->irq); +} + +static void mv88e639x_sgmii_pcs_disable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_sgmii_pcs_control_irq(mpcs, false); + mv88e639x_sgmii_pcs_control_pwr(mpcs, false); +} + +static void mv88e639x_sgmii_pcs_pre_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_sgmii_pcs_control_pwr(mpcs, false); +} + +static int mv88e6390_erratum_3_14(struct mv88e639x_pcs *mpcs) +{ + const int lanes[] = { MV88E6390_PORT9_LANE0, MV88E6390_PORT9_LANE1, + MV88E6390_PORT9_LANE2, MV88E6390_PORT9_LANE3, + MV88E6390_PORT10_LANE0, MV88E6390_PORT10_LANE1, + MV88E6390_PORT10_LANE2, MV88E6390_PORT10_LANE3 }; + int err, i; + + /* 88e6190x and 88e6390x errata 3.14: + * After chip reset, SERDES reconfiguration or SERDES core + * Software Reset, the SERDES lanes may not be properly aligned + * resulting in CRC errors + */ + + for (i = 0; i < ARRAY_SIZE(lanes); i++) { + err = mdiobus_c45_write(mpcs->mdio.bus, lanes[i], + MDIO_MMD_PHYXS, + 0xf054, 0x400C); + if (err) + return err; + + err = mdiobus_c45_write(mpcs->mdio.bus, lanes[i], + MDIO_MMD_PHYXS, + 0xf054, 0x4000); + if (err) + return err; + } + + return 0; +} + +static int mv88e639x_sgmii_pcs_post_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + int err; + + mv88e639x_sgmii_pcs_control_pwr(mpcs, true); + + if (mpcs->erratum_3_14) { + err = mv88e6390_erratum_3_14(mpcs); + if (err) + dev_err(mpcs->mdio.dev.parent, + "failed to apply erratum 3.14: %pe\n", + ERR_PTR(err)); + } + + return 0; +} + +static void mv88e639x_sgmii_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + u16 bmsr, lpa, status; + int err; + + err = mv88e639x_read(mpcs, MV88E6390_SGMII_BMSR, &bmsr); + if (err) { + dev_err(mpcs->mdio.dev.parent, + "can't read Serdes PHY %s: %pe\n", + "BMSR", ERR_PTR(err)); + state->link = false; + return; + } + + err = mv88e639x_read(mpcs, MV88E6390_SGMII_LPA, &lpa); + if (err) { + dev_err(mpcs->mdio.dev.parent, + "can't read Serdes PHY %s: %pe\n", + "LPA", ERR_PTR(err)); + state->link = false; + return; + } + + err = mv88e639x_read(mpcs, MV88E6390_SGMII_PHY_STATUS, &status); + if (err) { + dev_err(mpcs->mdio.dev.parent, + "can't read Serdes PHY %s: %pe\n", + "status", ERR_PTR(err)); + state->link = false; + return; + } + + mv88e6xxx_pcs_decode_state(mpcs->mdio.dev.parent, bmsr, lpa, status, + state); +} + +static int mv88e639x_sgmii_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + u16 val, bmcr; + bool changed; + int adv, err; + + adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (adv < 0) + return 0; + + mpcs->interface = interface; + + err = mv88e639x_modify_changed(mpcs, MV88E6390_SGMII_ADVERTISE, + 0xffff, adv); + if (err < 0) + return err; + + changed = err > 0; + + err = mv88e639x_read(mpcs, MV88E6390_SGMII_BMCR, &val); + if (err) + return err; + + if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) + bmcr = val | BMCR_ANENABLE; + else + bmcr = val & ~BMCR_ANENABLE; + + /* setting ANENABLE triggers a restart of negotiation */ + if (bmcr == val) + return changed; + + return mv88e639x_write(mpcs, MV88E6390_SGMII_BMCR, bmcr); +} + +static void mv88e639x_sgmii_pcs_an_restart(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_modify(mpcs, MV88E6390_SGMII_BMCR, + BMCR_ANRESTART, BMCR_ANRESTART); +} + +static void mv88e639x_sgmii_pcs_link_up(struct phylink_pcs *pcs, + unsigned int mode, + phy_interface_t interface, + int speed, int duplex) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + u16 bmcr; + int err; + + if (phylink_autoneg_inband(mode)) + return; + + bmcr = mii_bmcr_encode_fixed(speed, duplex); + + err = mv88e639x_modify(mpcs, MV88E6390_SGMII_BMCR, + BMCR_SPEED1000 | BMCR_SPEED100 | BMCR_FULLDPLX, + bmcr); + if (err) + dev_err(mpcs->mdio.dev.parent, + "can't access Serdes PHY %s: %pe\n", + "BMCR", ERR_PTR(err)); +} + +static const struct phylink_pcs_ops mv88e639x_sgmii_pcs_ops = { + .pcs_enable = mv88e639x_sgmii_pcs_enable, + .pcs_disable = mv88e639x_sgmii_pcs_disable, + .pcs_pre_config = mv88e639x_sgmii_pcs_pre_config, + .pcs_post_config = mv88e639x_sgmii_pcs_post_config, + .pcs_get_state = mv88e639x_sgmii_pcs_get_state, + .pcs_an_restart = mv88e639x_sgmii_pcs_an_restart, + .pcs_config = mv88e639x_sgmii_pcs_config, + .pcs_link_up = mv88e639x_sgmii_pcs_link_up, +}; + +static struct mv88e639x_pcs *xg_pcs_to_mv88e639x_pcs(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct mv88e639x_pcs, xg_pcs); +} + +static int mv88e639x_xg_pcs_enable(struct mv88e639x_pcs *mpcs) +{ + return mv88e639x_modify(mpcs, MV88E6390_10G_CTRL1, + MDIO_CTRL1_RESET | MDIO_PCS_CTRL1_LOOPBACK | + MDIO_CTRL1_LPOWER, 0); +} + +static void mv88e639x_xg_pcs_disable(struct mv88e639x_pcs *mpcs) +{ + mv88e639x_modify(mpcs, MV88E6390_10G_CTRL1, MDIO_CTRL1_LPOWER, + MDIO_CTRL1_LPOWER); +} + +static void mv88e639x_xg_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + u16 status; + int err; + + state->link = false; + + err = mv88e639x_read(mpcs, MV88E6390_10G_STAT1, &status); + if (err) { + dev_err(mpcs->mdio.dev.parent, + "can't read Serdes PHY %s: %pe\n", + "STAT1", ERR_PTR(err)); + return; + } + + state->link = !!(status & MDIO_STAT1_LSTATUS); + if (state->link) { + switch (state->interface) { + case PHY_INTERFACE_MODE_5GBASER: + state->speed = SPEED_5000; + break; + + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_RXAUI: + case PHY_INTERFACE_MODE_XAUI: + state->speed = SPEED_10000; + break; + + default: + state->link = false; + return; + } + + state->duplex = DUPLEX_FULL; + } +} + +static int mv88e639x_xg_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + return 0; +} + +static struct phylink_pcs * +mv88e639x_pcs_select(struct mv88e6xxx_chip *chip, int port, + phy_interface_t mode) +{ + struct mv88e639x_pcs *mpcs; + + mpcs = chip->ports[port].pcs_private; + if (!mpcs) + return NULL; + + switch (mode) { + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + return &mpcs->sgmii_pcs; + + case PHY_INTERFACE_MODE_5GBASER: + if (!mpcs->supports_5g) + return NULL; + fallthrough; + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_XAUI: + case PHY_INTERFACE_MODE_RXAUI: + return &mpcs->xg_pcs; + + default: + return NULL; + } +} + +/* Marvell 88E6390 Specific support */ + +static irqreturn_t mv88e6390_xg_handle_irq(struct mv88e639x_pcs *mpcs) +{ + u16 int_status; + int err; + + err = mv88e639x_read(mpcs, MV88E6390_10G_INT_STATUS, &int_status); + if (err) + return IRQ_NONE; + + if (int_status & (MV88E6390_10G_INT_LINK_DOWN | + MV88E6390_10G_INT_LINK_UP)) { + phylink_pcs_change(&mpcs->xg_pcs, + int_status & MV88E6390_10G_INT_LINK_UP); + + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int mv88e6390_xg_control_irq(struct mv88e639x_pcs *mpcs, bool enable) +{ + u16 val = 0; + + if (enable) + val = MV88E6390_10G_INT_LINK_DOWN | MV88E6390_10G_INT_LINK_UP; + + return mv88e639x_modify(mpcs, MV88E6390_10G_INT_ENABLE, + MV88E6390_10G_INT_LINK_DOWN | + MV88E6390_10G_INT_LINK_UP, val); +} + +static int mv88e6390_xg_pcs_enable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + int err; + + err = mv88e639x_xg_pcs_enable(mpcs); + if (err) + return err; + + mpcs->handle_irq = mv88e6390_xg_handle_irq; + + return mv88e6390_xg_control_irq(mpcs, !!mpcs->irq); +} + +static void mv88e6390_xg_pcs_disable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + + mv88e6390_xg_control_irq(mpcs, false); + mv88e639x_xg_pcs_disable(mpcs); +} + +static const struct phylink_pcs_ops mv88e6390_xg_pcs_ops = { + .pcs_enable = mv88e6390_xg_pcs_enable, + .pcs_disable = mv88e6390_xg_pcs_disable, + .pcs_get_state = mv88e639x_xg_pcs_get_state, + .pcs_config = mv88e639x_xg_pcs_config, +}; + +static int mv88e6390_pcs_enable_checker(struct mv88e639x_pcs *mpcs) +{ + return mv88e639x_modify(mpcs, MV88E6390_PG_CONTROL, + MV88E6390_PG_CONTROL_ENABLE_PC, + MV88E6390_PG_CONTROL_ENABLE_PC); +} + +static int mv88e6390_pcs_init(struct mv88e6xxx_chip *chip, int port) +{ + struct mv88e639x_pcs *mpcs; + struct mii_bus *bus; + struct device *dev; + int lane, err; + + lane = mv88e6xxx_serdes_get_lane(chip, port); + if (lane < 0) + return 0; + + bus = mv88e6xxx_default_mdio_bus(chip); + dev = chip->dev; + + mpcs = mv88e639x_pcs_alloc(dev, bus, lane, port); + if (!mpcs) + return -ENOMEM; + + mpcs->sgmii_pcs.ops = &mv88e639x_sgmii_pcs_ops; + mpcs->sgmii_pcs.neg_mode = true; + mpcs->xg_pcs.ops = &mv88e6390_xg_pcs_ops; + mpcs->xg_pcs.neg_mode = true; + + if (chip->info->prod_num == MV88E6XXX_PORT_SWITCH_ID_PROD_6190X || + chip->info->prod_num == MV88E6XXX_PORT_SWITCH_ID_PROD_6390X) + mpcs->erratum_3_14 = true; + + err = mv88e639x_pcs_setup_irq(mpcs, chip, port); + if (err) + goto err_free; + + /* 6390 and 6390x has the checker, 6393x doesn't appear to? */ + /* This is to enable gathering the statistics. Maybe this + * should call out to a helper? Or we could do this at init time. + */ + err = mv88e6390_pcs_enable_checker(mpcs); + if (err) + goto err_free; + + chip->ports[port].pcs_private = mpcs; + + return 0; + +err_free: + kfree(mpcs); + return err; +} + +const struct mv88e6xxx_pcs_ops mv88e6390_pcs_ops = { + .pcs_init = mv88e6390_pcs_init, + .pcs_teardown = mv88e639x_pcs_teardown, + .pcs_select = mv88e639x_pcs_select, +}; + +/* Marvell 88E6393X Specific support */ + +static int mv88e6393x_power_lane(struct mv88e639x_pcs *mpcs, bool enable) +{ + u16 val = MV88E6393X_SERDES_CTRL1_TX_PDOWN | + MV88E6393X_SERDES_CTRL1_RX_PDOWN; + + return mv88e639x_modify(mpcs, MV88E6393X_SERDES_CTRL1, val, + enable ? 0 : val); +} + +/* mv88e6393x family errata 4.6: + * Cannot clear PwrDn bit on SERDES if device is configured CPU_MGD mode or + * P0_mode is configured for [x]MII. + * Workaround: Set SERDES register 4.F002 bit 5=0 and bit 15=1. + * + * It seems that after this workaround the SERDES is automatically powered up + * (the bit is cleared), so power it down. + */ +static int mv88e6393x_erratum_4_6(struct mv88e639x_pcs *mpcs) +{ + int err; + + err = mv88e639x_modify(mpcs, MV88E6393X_SERDES_POC, + MV88E6393X_SERDES_POC_PDOWN | + MV88E6393X_SERDES_POC_RESET, + MV88E6393X_SERDES_POC_RESET); + if (err) + return err; + + err = mv88e639x_modify(mpcs, MV88E6390_SGMII_BMCR, + BMCR_PDOWN, BMCR_PDOWN); + if (err) + return err; + + err = mv88e639x_sgmii_pcs_control_pwr(mpcs, false); + if (err) + return err; + + return mv88e6393x_power_lane(mpcs, false); +} + +/* mv88e6393x family errata 4.8: + * When a SERDES port is operating in 1000BASE-X or SGMII mode link may not + * come up after hardware reset or software reset of SERDES core. Workaround + * is to write SERDES register 4.F074.14=1 for only those modes and 0 in all + * other modes. + */ +static int mv88e6393x_erratum_4_8(struct mv88e639x_pcs *mpcs) +{ + u16 reg, poc; + int err; + + err = mv88e639x_read(mpcs, MV88E6393X_SERDES_POC, &poc); + if (err) + return err; + + poc &= MV88E6393X_SERDES_POC_PCS_MASK; + if (poc == MV88E6393X_SERDES_POC_PCS_1000BASEX || + poc == MV88E6393X_SERDES_POC_PCS_SGMII_PHY || + poc == MV88E6393X_SERDES_POC_PCS_SGMII_MAC) + reg = MV88E6393X_ERRATA_4_8_BIT; + else + reg = 0; + + return mv88e639x_modify(mpcs, MV88E6393X_ERRATA_4_8_REG, + MV88E6393X_ERRATA_4_8_BIT, reg); +} + +/* mv88e6393x family errata 5.2: + * For optimal signal integrity the following sequence should be applied to + * SERDES operating in 10G mode. These registers only apply to 10G operation + * and have no effect on other speeds. + */ +static int mv88e6393x_erratum_5_2(struct mv88e639x_pcs *mpcs) +{ + static const struct { + u16 dev, reg, val, mask; + } fixes[] = { + { MDIO_MMD_VEND1, 0x8093, 0xcb5a, 0xffff }, + { MDIO_MMD_VEND1, 0x8171, 0x7088, 0xffff }, + { MDIO_MMD_VEND1, 0x80c9, 0x311a, 0xffff }, + { MDIO_MMD_VEND1, 0x80a2, 0x8000, 0xff7f }, + { MDIO_MMD_VEND1, 0x80a9, 0x0000, 0xfff0 }, + { MDIO_MMD_VEND1, 0x80a3, 0x0000, 0xf8ff }, + { MDIO_MMD_PHYXS, MV88E6393X_SERDES_POC, + MV88E6393X_SERDES_POC_RESET, MV88E6393X_SERDES_POC_RESET }, + }; + int err, i; + + for (i = 0; i < ARRAY_SIZE(fixes); ++i) { + err = mdiodev_c45_modify(&mpcs->mdio, fixes[i].dev, + fixes[i].reg, fixes[i].mask, + fixes[i].val); + if (err) + return err; + } + + return 0; +} + +/* Inband AN is broken on Amethyst in 2500base-x mode when set by standard + * mechanism (via cmode). + * We can get around this by configuring the PCS mode to 1000base-x and then + * writing value 0x58 to register 1e.8000. (This must be done while SerDes + * receiver and transmitter are disabled, which is, when this function is + * called.) + * It seem that when we do this configuration to 2500base-x mode (by changing + * PCS mode to 1000base-x and frequency to 3.125 GHz from 1.25 GHz) and then + * configure to sgmii or 1000base-x, the device thinks that it already has + * SerDes at 1.25 GHz and does not change the 1e.8000 register, leaving SerDes + * at 3.125 GHz. + * To avoid this, change PCS mode back to 2500base-x when disabling SerDes from + * 2500base-x mode. + */ +static int mv88e6393x_fix_2500basex_an(struct mv88e639x_pcs *mpcs, bool on) +{ + u16 reg; + int err; + + if (on) + reg = MV88E6393X_SERDES_POC_PCS_1000BASEX | + MV88E6393X_SERDES_POC_AN; + else + reg = MV88E6393X_SERDES_POC_PCS_2500BASEX; + + reg |= MV88E6393X_SERDES_POC_RESET; + + err = mv88e639x_modify(mpcs, MV88E6393X_SERDES_POC, + MV88E6393X_SERDES_POC_PCS_MASK | + MV88E6393X_SERDES_POC_AN | + MV88E6393X_SERDES_POC_RESET, reg); + if (err) + return err; + + return mdiodev_c45_write(&mpcs->mdio, MDIO_MMD_VEND1, 0x8000, 0x58); +} + +static int mv88e6393x_sgmii_apply_2500basex_an(struct mv88e639x_pcs *mpcs, + phy_interface_t interface, + bool enable) +{ + int err; + + if (interface != PHY_INTERFACE_MODE_2500BASEX) + return 0; + + err = mv88e6393x_fix_2500basex_an(mpcs, enable); + if (err) + dev_err(mpcs->mdio.dev.parent, + "failed to %s 2500basex fix: %pe\n", + enable ? "enable" : "disable", ERR_PTR(err)); + + return err; +} + +static void mv88e6393x_sgmii_pcs_disable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_sgmii_pcs_disable(pcs); + mv88e6393x_power_lane(mpcs, false); + mv88e6393x_sgmii_apply_2500basex_an(mpcs, mpcs->interface, false); +} + +static void mv88e6393x_sgmii_pcs_pre_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_sgmii_pcs_pre_config(pcs, interface); + mv88e6393x_power_lane(mpcs, false); + mv88e6393x_sgmii_apply_2500basex_an(mpcs, mpcs->interface, false); +} + +static int mv88e6393x_sgmii_pcs_post_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = sgmii_pcs_to_mv88e639x_pcs(pcs); + int err; + + err = mv88e6393x_erratum_4_8(mpcs); + if (err) + return err; + + err = mv88e6393x_sgmii_apply_2500basex_an(mpcs, interface, true); + if (err) + return err; + + err = mv88e6393x_power_lane(mpcs, true); + if (err) + return err; + + return mv88e639x_sgmii_pcs_post_config(pcs, interface); +} + +static const struct phylink_pcs_ops mv88e6393x_sgmii_pcs_ops = { + .pcs_enable = mv88e639x_sgmii_pcs_enable, + .pcs_disable = mv88e6393x_sgmii_pcs_disable, + .pcs_pre_config = mv88e6393x_sgmii_pcs_pre_config, + .pcs_post_config = mv88e6393x_sgmii_pcs_post_config, + .pcs_get_state = mv88e639x_sgmii_pcs_get_state, + .pcs_an_restart = mv88e639x_sgmii_pcs_an_restart, + .pcs_config = mv88e639x_sgmii_pcs_config, + .pcs_link_up = mv88e639x_sgmii_pcs_link_up, +}; + +static irqreturn_t mv88e6393x_xg_handle_irq(struct mv88e639x_pcs *mpcs) +{ + u16 int_status, stat1; + bool link_down; + int err; + + err = mv88e639x_read(mpcs, MV88E6393X_10G_INT_STATUS, &int_status); + if (err) + return IRQ_NONE; + + if (int_status & MV88E6393X_10G_INT_LINK_CHANGE) { + err = mv88e639x_read(mpcs, MV88E6390_10G_STAT1, &stat1); + if (err) + return IRQ_NONE; + + link_down = !(stat1 & MDIO_STAT1_LSTATUS); + + phylink_pcs_change(&mpcs->xg_pcs, !link_down); + + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int mv88e6393x_xg_control_irq(struct mv88e639x_pcs *mpcs, bool enable) +{ + u16 val = 0; + + if (enable) + val = MV88E6393X_10G_INT_LINK_CHANGE; + + return mv88e639x_modify(mpcs, MV88E6393X_10G_INT_ENABLE, + MV88E6393X_10G_INT_LINK_CHANGE, val); +} + +static int mv88e6393x_xg_pcs_enable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + + mpcs->handle_irq = mv88e6393x_xg_handle_irq; + + return mv88e6393x_xg_control_irq(mpcs, !!mpcs->irq); +} + +static void mv88e6393x_xg_pcs_disable(struct phylink_pcs *pcs) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + + mv88e6393x_xg_control_irq(mpcs, false); + mv88e639x_xg_pcs_disable(mpcs); + mv88e6393x_power_lane(mpcs, false); +} + +/* The PCS has to be powered down while CMODE is changed */ +static void mv88e6393x_xg_pcs_pre_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + + mv88e639x_xg_pcs_disable(mpcs); + mv88e6393x_power_lane(mpcs, false); +} + +static int mv88e6393x_xg_pcs_post_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct mv88e639x_pcs *mpcs = xg_pcs_to_mv88e639x_pcs(pcs); + int err; + + if (interface == PHY_INTERFACE_MODE_10GBASER) { + err = mv88e6393x_erratum_5_2(mpcs); + if (err) + return err; + } + + err = mv88e6393x_power_lane(mpcs, true); + if (err) + return err; + + return mv88e639x_xg_pcs_enable(mpcs); +} + +static const struct phylink_pcs_ops mv88e6393x_xg_pcs_ops = { + .pcs_enable = mv88e6393x_xg_pcs_enable, + .pcs_disable = mv88e6393x_xg_pcs_disable, + .pcs_pre_config = mv88e6393x_xg_pcs_pre_config, + .pcs_post_config = mv88e6393x_xg_pcs_post_config, + .pcs_get_state = mv88e639x_xg_pcs_get_state, + .pcs_config = mv88e639x_xg_pcs_config, +}; + +static int mv88e6393x_pcs_init(struct mv88e6xxx_chip *chip, int port) +{ + struct mv88e639x_pcs *mpcs; + struct mii_bus *bus; + struct device *dev; + int lane, err; + + lane = mv88e6xxx_serdes_get_lane(chip, port); + if (lane < 0) + return 0; + + bus = mv88e6xxx_default_mdio_bus(chip); + dev = chip->dev; + + mpcs = mv88e639x_pcs_alloc(dev, bus, lane, port); + if (!mpcs) + return -ENOMEM; + + mpcs->sgmii_pcs.ops = &mv88e6393x_sgmii_pcs_ops; + mpcs->sgmii_pcs.neg_mode = true; + mpcs->xg_pcs.ops = &mv88e6393x_xg_pcs_ops; + mpcs->xg_pcs.neg_mode = true; + mpcs->supports_5g = true; + + err = mv88e6393x_erratum_4_6(mpcs); + if (err) + goto err_free; + + err = mv88e639x_pcs_setup_irq(mpcs, chip, port); + if (err) + goto err_free; + + chip->ports[port].pcs_private = mpcs; + + return 0; + +err_free: + kfree(mpcs); + return err; +} + +const struct mv88e6xxx_pcs_ops mv88e6393x_pcs_ops = { + .pcs_init = mv88e6393x_pcs_init, + .pcs_teardown = mv88e639x_pcs_teardown, + .pcs_select = mv88e639x_pcs_select, +}; diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c index dd66ec902d4c..5394a8cf7bf1 100644 --- a/drivers/net/dsa/mv88e6xxx/port.c +++ b/drivers/net/dsa/mv88e6xxx/port.c @@ -524,7 +524,6 @@ static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port, phy_interface_t mode, bool force) { u16 cmode; - int lane; u16 reg; int err; @@ -577,19 +576,6 @@ static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port, if (cmode == chip->ports[port].cmode && !force) return 0; - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane >= 0) { - if (chip->ports[port].serdes_irq) { - err = mv88e6xxx_serdes_irq_disable(chip, port, lane); - if (err) - return err; - } - - err = mv88e6xxx_serdes_power_down(chip, port, lane); - if (err) - return err; - } - chip->ports[port].cmode = 0; if (cmode) { @@ -605,22 +591,6 @@ static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port, return err; chip->ports[port].cmode = cmode; - - lane = mv88e6xxx_serdes_get_lane(chip, port); - if (lane == -ENODEV) - return 0; - if (lane < 0) - return lane; - - err = mv88e6xxx_serdes_power_up(chip, port, lane); - if (err) - return err; - - if (chip->ports[port].serdes_irq) { - err = mv88e6xxx_serdes_irq_enable(chip, port, lane); - if (err) - return err; - } } return 0; diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c index 80167d53212f..3b4b42651fa3 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.c +++ b/drivers/net/dsa/mv88e6xxx/serdes.c @@ -39,15 +39,8 @@ static int mv88e6390_serdes_read(struct mv88e6xxx_chip *chip, return mv88e6xxx_phy_read_c45(chip, lane, device, reg, val); } -static int mv88e6390_serdes_write(struct mv88e6xxx_chip *chip, - int lane, int device, int reg, u16 val) -{ - return mv88e6xxx_phy_write_c45(chip, lane, device, reg, val); -} - -static int mv88e6xxx_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, - u16 bmsr, u16 lpa, u16 status, - struct phylink_link_state *state) +int mv88e6xxx_pcs_decode_state(struct device *dev, u16 bmsr, u16 lpa, + u16 status, struct phylink_link_state *state) { state->link = false; @@ -88,7 +81,7 @@ static int mv88e6xxx_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, state->speed = SPEED_10; break; default: - dev_err(chip->dev, "invalid PHY speed\n"); + dev_err(dev, "invalid PHY speed\n"); return -EINVAL; } } else if (state->link && @@ -117,160 +110,6 @@ static int mv88e6xxx_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, return 0; } -int mv88e6352_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool up) -{ - u16 val, new_val; - int err; - - err = mv88e6352_serdes_read(chip, MII_BMCR, &val); - if (err) - return err; - - if (up) - new_val = val & ~BMCR_PDOWN; - else - new_val = val | BMCR_PDOWN; - - if (val != new_val) - err = mv88e6352_serdes_write(chip, MII_BMCR, new_val); - - return err; -} - -int mv88e6352_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port, - int lane, unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise) -{ - u16 adv, bmcr, val; - bool changed; - int err; - - switch (interface) { - case PHY_INTERFACE_MODE_SGMII: - adv = 0x0001; - break; - - case PHY_INTERFACE_MODE_1000BASEX: - adv = linkmode_adv_to_mii_adv_x(advertise, - ETHTOOL_LINK_MODE_1000baseX_Full_BIT); - break; - - default: - return 0; - } - - err = mv88e6352_serdes_read(chip, MII_ADVERTISE, &val); - if (err) - return err; - - changed = val != adv; - if (changed) { - err = mv88e6352_serdes_write(chip, MII_ADVERTISE, adv); - if (err) - return err; - } - - err = mv88e6352_serdes_read(chip, MII_BMCR, &val); - if (err) - return err; - - if (phylink_autoneg_inband(mode)) - bmcr = val | BMCR_ANENABLE; - else - bmcr = val & ~BMCR_ANENABLE; - - if (bmcr == val) - return changed; - - return mv88e6352_serdes_write(chip, MII_BMCR, bmcr); -} - -int mv88e6352_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state) -{ - u16 bmsr, lpa, status; - int err; - - err = mv88e6352_serdes_read(chip, MII_BMSR, &bmsr); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY BMSR: %d\n", err); - return err; - } - - err = mv88e6352_serdes_read(chip, 0x11, &status); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY status: %d\n", err); - return err; - } - - err = mv88e6352_serdes_read(chip, MII_LPA, &lpa); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY LPA: %d\n", err); - return err; - } - - return mv88e6xxx_serdes_pcs_get_state(chip, bmsr, lpa, status, state); -} - -int mv88e6352_serdes_pcs_an_restart(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - u16 bmcr; - int err; - - err = mv88e6352_serdes_read(chip, MII_BMCR, &bmcr); - if (err) - return err; - - return mv88e6352_serdes_write(chip, MII_BMCR, bmcr | BMCR_ANRESTART); -} - -int mv88e6352_serdes_pcs_link_up(struct mv88e6xxx_chip *chip, int port, - int lane, int speed, int duplex) -{ - u16 val, bmcr; - int err; - - err = mv88e6352_serdes_read(chip, MII_BMCR, &val); - if (err) - return err; - - bmcr = val & ~(BMCR_SPEED100 | BMCR_FULLDPLX | BMCR_SPEED1000); - switch (speed) { - case SPEED_1000: - bmcr |= BMCR_SPEED1000; - break; - case SPEED_100: - bmcr |= BMCR_SPEED100; - break; - case SPEED_10: - break; - } - - if (duplex == DUPLEX_FULL) - bmcr |= BMCR_FULLDPLX; - - if (bmcr == val) - return 0; - - return mv88e6352_serdes_write(chip, MII_BMCR, bmcr); -} - -int mv88e6352_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) -{ - u8 cmode = chip->ports[port].cmode; - int lane = -ENODEV; - - if ((cmode == MV88E6XXX_PORT_STS_CMODE_100BASEX) || - (cmode == MV88E6XXX_PORT_STS_CMODE_1000BASEX) || - (cmode == MV88E6XXX_PORT_STS_CMODE_SGMII)) - lane = 0xff; /* Unused */ - - return lane; -} - struct mv88e6352_serdes_hw_stat { char string[ETH_GSTRING_LEN]; int sizeof_stat; @@ -363,51 +202,6 @@ int mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, return ARRAY_SIZE(mv88e6352_serdes_hw_stats); } -static void mv88e6352_serdes_irq_link(struct mv88e6xxx_chip *chip, int port) -{ - u16 bmsr; - int err; - - /* If the link has dropped, we want to know about it. */ - err = mv88e6352_serdes_read(chip, MII_BMSR, &bmsr); - if (err) { - dev_err(chip->dev, "can't read Serdes BMSR: %d\n", err); - return; - } - - dsa_port_phylink_mac_change(chip->ds, port, !!(bmsr & BMSR_LSTATUS)); -} - -irqreturn_t mv88e6352_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - irqreturn_t ret = IRQ_NONE; - u16 status; - int err; - - err = mv88e6352_serdes_read(chip, MV88E6352_SERDES_INT_STATUS, &status); - if (err) - return ret; - - if (status & MV88E6352_SERDES_INT_LINK_CHANGE) { - ret = IRQ_HANDLED; - mv88e6352_serdes_irq_link(chip, port); - } - - return ret; -} - -int mv88e6352_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable) -{ - u16 val = 0; - - if (enable) - val |= MV88E6352_SERDES_INT_LINK_CHANGE; - - return mv88e6352_serdes_write(chip, MV88E6352_SERDES_INT_ENABLE, val); -} - unsigned int mv88e6352_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port) { return irq_find_mapping(chip->g2_irq.domain, MV88E6352_SERDES_IRQ); @@ -461,115 +255,6 @@ int mv88e6341_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) return lane; } -int mv88e6185_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool up) -{ - /* The serdes power can't be controlled on this switch chip but we need - * to supply this function to avoid returning -EOPNOTSUPP in - * mv88e6xxx_serdes_power_up/mv88e6xxx_serdes_power_down - */ - return 0; -} - -int mv88e6185_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) -{ - /* There are no configurable serdes lanes on this switch chip but we - * need to return a non-negative lane number so that callers of - * mv88e6xxx_serdes_get_lane() know this is a serdes port. - */ - switch (chip->ports[port].cmode) { - case MV88E6185_PORT_STS_CMODE_SERDES: - case MV88E6185_PORT_STS_CMODE_1000BASE_X: - return 0; - default: - return -ENODEV; - } -} - -int mv88e6185_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state) -{ - int err; - u16 status; - - err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &status); - if (err) - return err; - - state->link = !!(status & MV88E6XXX_PORT_STS_LINK); - - if (state->link) { - state->duplex = status & MV88E6XXX_PORT_STS_DUPLEX ? DUPLEX_FULL : DUPLEX_HALF; - - switch (status & MV88E6XXX_PORT_STS_SPEED_MASK) { - case MV88E6XXX_PORT_STS_SPEED_1000: - state->speed = SPEED_1000; - break; - case MV88E6XXX_PORT_STS_SPEED_100: - state->speed = SPEED_100; - break; - case MV88E6XXX_PORT_STS_SPEED_10: - state->speed = SPEED_10; - break; - default: - dev_err(chip->dev, "invalid PHY speed\n"); - return -EINVAL; - } - } else { - state->duplex = DUPLEX_UNKNOWN; - state->speed = SPEED_UNKNOWN; - } - - return 0; -} - -int mv88e6097_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable) -{ - u8 cmode = chip->ports[port].cmode; - - /* The serdes interrupts are enabled in the G2_INT_MASK register. We - * need to return 0 to avoid returning -EOPNOTSUPP in - * mv88e6xxx_serdes_irq_enable/mv88e6xxx_serdes_irq_disable - */ - switch (cmode) { - case MV88E6185_PORT_STS_CMODE_SERDES: - case MV88E6185_PORT_STS_CMODE_1000BASE_X: - return 0; - } - - return -EOPNOTSUPP; -} - -static void mv88e6097_serdes_irq_link(struct mv88e6xxx_chip *chip, int port) -{ - u16 status; - int err; - - err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &status); - if (err) { - dev_err(chip->dev, "can't read port status: %d\n", err); - return; - } - - dsa_port_phylink_mac_change(chip->ds, port, !!(status & MV88E6XXX_PORT_STS_LINK)); -} - -irqreturn_t mv88e6097_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - u8 cmode = chip->ports[port].cmode; - - switch (cmode) { - case MV88E6185_PORT_STS_CMODE_SERDES: - case MV88E6185_PORT_STS_CMODE_1000BASE_X: - mv88e6097_serdes_irq_link(chip, port); - return IRQ_HANDLED; - } - - return IRQ_NONE; -} - int mv88e6390_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) { u8 cmode = chip->ports[port].cmode; @@ -690,57 +375,6 @@ int mv88e6393x_serdes_get_lane(struct mv88e6xxx_chip *chip, int port) return lane; } -/* Set power up/down for 10GBASE-R and 10GBASE-X4/X2 */ -static int mv88e6390_serdes_power_10g(struct mv88e6xxx_chip *chip, int lane, - bool up) -{ - u16 val, new_val; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_10G_CTRL1, &val); - - if (err) - return err; - - if (up) - new_val = val & ~(MDIO_CTRL1_RESET | - MDIO_PCS_CTRL1_LOOPBACK | - MDIO_CTRL1_LPOWER); - else - new_val = val | MDIO_CTRL1_LPOWER; - - if (val != new_val) - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_10G_CTRL1, new_val); - - return err; -} - -/* Set power up/down for SGMII and 1000Base-X */ -static int mv88e6390_serdes_power_sgmii(struct mv88e6xxx_chip *chip, int lane, - bool up) -{ - u16 val, new_val; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, &val); - if (err) - return err; - - if (up) - new_val = val & ~(BMCR_RESET | BMCR_LOOPBACK | BMCR_PDOWN); - else - new_val = val | BMCR_PDOWN; - - if (val != new_val) - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, new_val); - - return err; -} - struct mv88e6390_serdes_hw_stat { char string[ETH_GSTRING_LEN]; int reg; @@ -814,484 +448,6 @@ int mv88e6390_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, return ARRAY_SIZE(mv88e6390_serdes_hw_stats); } -static int mv88e6390_serdes_enable_checker(struct mv88e6xxx_chip *chip, int lane) -{ - u16 reg; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_PG_CONTROL, ®); - if (err) - return err; - - reg |= MV88E6390_PG_CONTROL_ENABLE_PC; - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_PG_CONTROL, reg); -} - -int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool up) -{ - u8 cmode = chip->ports[port].cmode; - int err; - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - err = mv88e6390_serdes_power_sgmii(chip, lane, up); - break; - case MV88E6XXX_PORT_STS_CMODE_XAUI: - case MV88E6XXX_PORT_STS_CMODE_RXAUI: - err = mv88e6390_serdes_power_10g(chip, lane, up); - break; - default: - err = -EINVAL; - break; - } - - if (!err && up) - err = mv88e6390_serdes_enable_checker(chip, lane); - - return err; -} - -int mv88e6390_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port, - int lane, unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise) -{ - u16 val, bmcr, adv; - bool changed; - int err; - - switch (interface) { - case PHY_INTERFACE_MODE_SGMII: - adv = 0x0001; - break; - - case PHY_INTERFACE_MODE_1000BASEX: - adv = linkmode_adv_to_mii_adv_x(advertise, - ETHTOOL_LINK_MODE_1000baseX_Full_BIT); - break; - - case PHY_INTERFACE_MODE_2500BASEX: - adv = linkmode_adv_to_mii_adv_x(advertise, - ETHTOOL_LINK_MODE_2500baseX_Full_BIT); - break; - - default: - return 0; - } - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_ADVERTISE, &val); - if (err) - return err; - - changed = val != adv; - if (changed) { - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_ADVERTISE, adv); - if (err) - return err; - } - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, &val); - if (err) - return err; - - if (phylink_autoneg_inband(mode)) - bmcr = val | BMCR_ANENABLE; - else - bmcr = val & ~BMCR_ANENABLE; - - /* setting ANENABLE triggers a restart of negotiation */ - if (bmcr == val) - return changed; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, bmcr); -} - -static int mv88e6390_serdes_pcs_get_state_sgmii(struct mv88e6xxx_chip *chip, - int port, int lane, struct phylink_link_state *state) -{ - u16 bmsr, lpa, status; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMSR, &bmsr); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY BMSR: %d\n", err); - return err; - } - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_PHY_STATUS, &status); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY status: %d\n", err); - return err; - } - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_LPA, &lpa); - if (err) { - dev_err(chip->dev, "can't read Serdes PHY LPA: %d\n", err); - return err; - } - - return mv88e6xxx_serdes_pcs_get_state(chip, bmsr, lpa, status, state); -} - -static int mv88e6390_serdes_pcs_get_state_10g(struct mv88e6xxx_chip *chip, - int port, int lane, struct phylink_link_state *state) -{ - u16 status; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_10G_STAT1, &status); - if (err) - return err; - - state->link = !!(status & MDIO_STAT1_LSTATUS); - if (state->link) { - state->speed = SPEED_10000; - state->duplex = DUPLEX_FULL; - } - - return 0; -} - -static int mv88e6393x_serdes_pcs_get_state_10g(struct mv88e6xxx_chip *chip, - int port, int lane, - struct phylink_link_state *state) -{ - u16 status; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_10G_STAT1, &status); - if (err) - return err; - - state->link = !!(status & MDIO_STAT1_LSTATUS); - if (state->link) { - if (state->interface == PHY_INTERFACE_MODE_5GBASER) - state->speed = SPEED_5000; - else - state->speed = SPEED_10000; - state->duplex = DUPLEX_FULL; - } - return 0; -} - -/* USXGMII registers for Marvell switch 88e639x are undocumented and this function is based - * on some educated guesses. It appears that there are no status bits related to - * autonegotiation complete or flow control. - */ -static int mv88e639x_serdes_pcs_get_state_usxgmii(struct mv88e6xxx_chip *chip, - int port, int lane, - struct phylink_link_state *state) -{ - u16 status, lp_status; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_USXGMII_PHY_STATUS, &status); - if (err) { - dev_err(chip->dev, "can't read Serdes USXGMII PHY status: %d\n", err); - return err; - } - dev_dbg(chip->dev, "USXGMII PHY status: 0x%x\n", status); - - state->link = !!(status & MDIO_USXGMII_LINK); - state->an_complete = state->link; - - if (state->link) { - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_USXGMII_LP_STATUS, &lp_status); - if (err) { - dev_err(chip->dev, "can't read Serdes USXGMII LP status: %d\n", err); - return err; - } - dev_dbg(chip->dev, "USXGMII LP status: 0x%x\n", lp_status); - /* lp_status appears to include the "link" bit as per USXGMII spec. */ - phylink_decode_usxgmii_word(state, lp_status); - } - return 0; -} - -int mv88e6390_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state) -{ - switch (state->interface) { - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_2500BASEX: - return mv88e6390_serdes_pcs_get_state_sgmii(chip, port, lane, - state); - case PHY_INTERFACE_MODE_XAUI: - case PHY_INTERFACE_MODE_RXAUI: - return mv88e6390_serdes_pcs_get_state_10g(chip, port, lane, - state); - - default: - return -EOPNOTSUPP; - } -} - -int mv88e6393x_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state) -{ - switch (state->interface) { - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_2500BASEX: - return mv88e6390_serdes_pcs_get_state_sgmii(chip, port, lane, - state); - case PHY_INTERFACE_MODE_5GBASER: - case PHY_INTERFACE_MODE_10GBASER: - return mv88e6393x_serdes_pcs_get_state_10g(chip, port, lane, - state); - case PHY_INTERFACE_MODE_USXGMII: - return mv88e639x_serdes_pcs_get_state_usxgmii(chip, port, lane, - state); - - default: - return -EOPNOTSUPP; - } -} - -int mv88e6390_serdes_pcs_an_restart(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - u16 bmcr; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, &bmcr); - if (err) - return err; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, - bmcr | BMCR_ANRESTART); -} - -int mv88e6390_serdes_pcs_link_up(struct mv88e6xxx_chip *chip, int port, - int lane, int speed, int duplex) -{ - u16 val, bmcr; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, &val); - if (err) - return err; - - bmcr = val & ~(BMCR_SPEED100 | BMCR_FULLDPLX | BMCR_SPEED1000); - switch (speed) { - case SPEED_2500: - case SPEED_1000: - bmcr |= BMCR_SPEED1000; - break; - case SPEED_100: - bmcr |= BMCR_SPEED100; - break; - case SPEED_10: - break; - } - - if (duplex == DUPLEX_FULL) - bmcr |= BMCR_FULLDPLX; - - if (bmcr == val) - return 0; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMCR, bmcr); -} - -static void mv88e6390_serdes_irq_link_sgmii(struct mv88e6xxx_chip *chip, - int port, int lane) -{ - u16 bmsr; - int err; - - /* If the link has dropped, we want to know about it. */ - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_BMSR, &bmsr); - if (err) { - dev_err(chip->dev, "can't read Serdes BMSR: %d\n", err); - return; - } - - dsa_port_phylink_mac_change(chip->ds, port, !!(bmsr & BMSR_LSTATUS)); -} - -static void mv88e6393x_serdes_irq_link_10g(struct mv88e6xxx_chip *chip, - int port, u8 lane) -{ - u16 status; - int err; - - /* If the link has dropped, we want to know about it. */ - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_10G_STAT1, &status); - if (err) { - dev_err(chip->dev, "can't read Serdes STAT1: %d\n", err); - return; - } - - dsa_port_phylink_mac_change(chip->ds, port, !!(status & MDIO_STAT1_LSTATUS)); -} - -static int mv88e6390_serdes_irq_enable_sgmii(struct mv88e6xxx_chip *chip, - int lane, bool enable) -{ - u16 val = 0; - - if (enable) - val |= MV88E6390_SGMII_INT_LINK_DOWN | - MV88E6390_SGMII_INT_LINK_UP; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_INT_ENABLE, val); -} - -int mv88e6390_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable) -{ - u8 cmode = chip->ports[port].cmode; - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - return mv88e6390_serdes_irq_enable_sgmii(chip, lane, enable); - } - - return 0; -} - -static int mv88e6390_serdes_irq_status_sgmii(struct mv88e6xxx_chip *chip, - int lane, u16 *status) -{ - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6390_SGMII_INT_STATUS, status); - - return err; -} - -static int mv88e6393x_serdes_irq_enable_10g(struct mv88e6xxx_chip *chip, - u8 lane, bool enable) -{ - u16 val = 0; - - if (enable) - val |= MV88E6393X_10G_INT_LINK_CHANGE; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_10G_INT_ENABLE, val); -} - -int mv88e6393x_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, - int lane, bool enable) -{ - u8 cmode = chip->ports[port].cmode; - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - return mv88e6390_serdes_irq_enable_sgmii(chip, lane, enable); - case MV88E6393X_PORT_STS_CMODE_5GBASER: - case MV88E6393X_PORT_STS_CMODE_10GBASER: - case MV88E6393X_PORT_STS_CMODE_USXGMII: - return mv88e6393x_serdes_irq_enable_10g(chip, lane, enable); - } - - return 0; -} - -static int mv88e6393x_serdes_irq_status_10g(struct mv88e6xxx_chip *chip, - u8 lane, u16 *status) -{ - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_10G_INT_STATUS, status); - - return err; -} - -irqreturn_t mv88e6393x_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - u8 cmode = chip->ports[port].cmode; - irqreturn_t ret = IRQ_NONE; - u16 status; - int err; - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - err = mv88e6390_serdes_irq_status_sgmii(chip, lane, &status); - if (err) - return ret; - if (status & (MV88E6390_SGMII_INT_LINK_DOWN | - MV88E6390_SGMII_INT_LINK_UP)) { - ret = IRQ_HANDLED; - mv88e6390_serdes_irq_link_sgmii(chip, port, lane); - } - break; - case MV88E6393X_PORT_STS_CMODE_5GBASER: - case MV88E6393X_PORT_STS_CMODE_10GBASER: - case MV88E6393X_PORT_STS_CMODE_USXGMII: - err = mv88e6393x_serdes_irq_status_10g(chip, lane, &status); - if (err) - return err; - if (status & MV88E6393X_10G_INT_LINK_CHANGE) { - ret = IRQ_HANDLED; - mv88e6393x_serdes_irq_link_10g(chip, port, lane); - } - break; - } - - return ret; -} - -irqreturn_t mv88e6390_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane) -{ - u8 cmode = chip->ports[port].cmode; - irqreturn_t ret = IRQ_NONE; - u16 status; - int err; - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - err = mv88e6390_serdes_irq_status_sgmii(chip, lane, &status); - if (err) - return ret; - if (status & (MV88E6390_SGMII_INT_LINK_DOWN | - MV88E6390_SGMII_INT_LINK_UP)) { - ret = IRQ_HANDLED; - mv88e6390_serdes_irq_link_sgmii(chip, port, lane); - } - } - - return ret; -} - unsigned int mv88e6390_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port) { return irq_find_mapping(chip->g2_irq.domain, port); @@ -1390,259 +546,3 @@ int mv88e6352_serdes_set_tx_amplitude(struct mv88e6xxx_chip *chip, int port, return mv88e6352_serdes_write(chip, MV88E6352_SERDES_SPEC_CTRL2, ctrl); } - -static int mv88e6393x_serdes_power_lane(struct mv88e6xxx_chip *chip, int lane, - bool on) -{ - u16 reg; - int err; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_CTRL1, ®); - if (err) - return err; - - if (on) - reg &= ~(MV88E6393X_SERDES_CTRL1_TX_PDOWN | - MV88E6393X_SERDES_CTRL1_RX_PDOWN); - else - reg |= MV88E6393X_SERDES_CTRL1_TX_PDOWN | - MV88E6393X_SERDES_CTRL1_RX_PDOWN; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_CTRL1, reg); -} - -static int mv88e6393x_serdes_erratum_4_6(struct mv88e6xxx_chip *chip, int lane) -{ - u16 reg; - int err; - - /* mv88e6393x family errata 4.6: - * Cannot clear PwrDn bit on SERDES if device is configured CPU_MGD - * mode or P0_mode is configured for [x]MII. - * Workaround: Set SERDES register 4.F002 bit 5=0 and bit 15=1. - * - * It seems that after this workaround the SERDES is automatically - * powered up (the bit is cleared), so power it down. - */ - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_POC, ®); - if (err) - return err; - - reg &= ~MV88E6393X_SERDES_POC_PDOWN; - reg |= MV88E6393X_SERDES_POC_RESET; - - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_POC, reg); - if (err) - return err; - - err = mv88e6390_serdes_power_sgmii(chip, lane, false); - if (err) - return err; - - return mv88e6393x_serdes_power_lane(chip, lane, false); -} - -int mv88e6393x_serdes_setup_errata(struct mv88e6xxx_chip *chip) -{ - int err; - - err = mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT0_LANE); - if (err) - return err; - - err = mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT9_LANE); - if (err) - return err; - - return mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT10_LANE); -} - -static int mv88e6393x_serdes_erratum_4_8(struct mv88e6xxx_chip *chip, int lane) -{ - u16 reg, pcs; - int err; - - /* mv88e6393x family errata 4.8: - * When a SERDES port is operating in 1000BASE-X or SGMII mode link may - * not come up after hardware reset or software reset of SERDES core. - * Workaround is to write SERDES register 4.F074.14=1 for only those - * modes and 0 in all other modes. - */ - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_POC, &pcs); - if (err) - return err; - - pcs &= MV88E6393X_SERDES_POC_PCS_MASK; - - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_ERRATA_4_8_REG, ®); - if (err) - return err; - - if (pcs == MV88E6393X_SERDES_POC_PCS_1000BASEX || - pcs == MV88E6393X_SERDES_POC_PCS_SGMII_PHY || - pcs == MV88E6393X_SERDES_POC_PCS_SGMII_MAC) - reg |= MV88E6393X_ERRATA_4_8_BIT; - else - reg &= ~MV88E6393X_ERRATA_4_8_BIT; - - return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_ERRATA_4_8_REG, reg); -} - -static int mv88e6393x_serdes_erratum_5_2(struct mv88e6xxx_chip *chip, int lane, - u8 cmode) -{ - static const struct { - u16 dev, reg, val, mask; - } fixes[] = { - { MDIO_MMD_VEND1, 0x8093, 0xcb5a, 0xffff }, - { MDIO_MMD_VEND1, 0x8171, 0x7088, 0xffff }, - { MDIO_MMD_VEND1, 0x80c9, 0x311a, 0xffff }, - { MDIO_MMD_VEND1, 0x80a2, 0x8000, 0xff7f }, - { MDIO_MMD_VEND1, 0x80a9, 0x0000, 0xfff0 }, - { MDIO_MMD_VEND1, 0x80a3, 0x0000, 0xf8ff }, - { MDIO_MMD_PHYXS, MV88E6393X_SERDES_POC, - MV88E6393X_SERDES_POC_RESET, MV88E6393X_SERDES_POC_RESET }, - }; - int err, i; - u16 reg; - - /* mv88e6393x family errata 5.2: - * For optimal signal integrity the following sequence should be applied - * to SERDES operating in 10G mode. These registers only apply to 10G - * operation and have no effect on other speeds. - */ - if (cmode != MV88E6393X_PORT_STS_CMODE_10GBASER && - cmode != MV88E6393X_PORT_STS_CMODE_USXGMII) - return 0; - - for (i = 0; i < ARRAY_SIZE(fixes); ++i) { - err = mv88e6390_serdes_read(chip, lane, fixes[i].dev, - fixes[i].reg, ®); - if (err) - return err; - - reg &= ~fixes[i].mask; - reg |= fixes[i].val; - - err = mv88e6390_serdes_write(chip, lane, fixes[i].dev, - fixes[i].reg, reg); - if (err) - return err; - } - - return 0; -} - -static int mv88e6393x_serdes_fix_2500basex_an(struct mv88e6xxx_chip *chip, - int lane, u8 cmode, bool on) -{ - u16 reg; - int err; - - if (cmode != MV88E6XXX_PORT_STS_CMODE_2500BASEX) - return 0; - - /* Inband AN is broken on Amethyst in 2500base-x mode when set by - * standard mechanism (via cmode). - * We can get around this by configuring the PCS mode to 1000base-x - * and then writing value 0x58 to register 1e.8000. (This must be done - * while SerDes receiver and transmitter are disabled, which is, when - * this function is called.) - * It seem that when we do this configuration to 2500base-x mode (by - * changing PCS mode to 1000base-x and frequency to 3.125 GHz from - * 1.25 GHz) and then configure to sgmii or 1000base-x, the device - * thinks that it already has SerDes at 1.25 GHz and does not change - * the 1e.8000 register, leaving SerDes at 3.125 GHz. - * To avoid this, change PCS mode back to 2500base-x when disabling - * SerDes from 2500base-x mode. - */ - err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_POC, ®); - if (err) - return err; - - reg &= ~(MV88E6393X_SERDES_POC_PCS_MASK | MV88E6393X_SERDES_POC_AN); - if (on) - reg |= MV88E6393X_SERDES_POC_PCS_1000BASEX | - MV88E6393X_SERDES_POC_AN; - else - reg |= MV88E6393X_SERDES_POC_PCS_2500BASEX; - reg |= MV88E6393X_SERDES_POC_RESET; - - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS, - MV88E6393X_SERDES_POC, reg); - if (err) - return err; - - err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_VEND1, 0x8000, 0x58); - if (err) - return err; - - return 0; -} - -int mv88e6393x_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool on) -{ - u8 cmode = chip->ports[port].cmode; - int err; - - if (port != 0 && port != 9 && port != 10) - return -EOPNOTSUPP; - - if (on) { - err = mv88e6393x_serdes_erratum_4_8(chip, lane); - if (err) - return err; - - err = mv88e6393x_serdes_erratum_5_2(chip, lane, cmode); - if (err) - return err; - - err = mv88e6393x_serdes_fix_2500basex_an(chip, lane, cmode, - true); - if (err) - return err; - - err = mv88e6393x_serdes_power_lane(chip, lane, true); - if (err) - return err; - } - - switch (cmode) { - case MV88E6XXX_PORT_STS_CMODE_SGMII: - case MV88E6XXX_PORT_STS_CMODE_1000BASEX: - case MV88E6XXX_PORT_STS_CMODE_2500BASEX: - err = mv88e6390_serdes_power_sgmii(chip, lane, on); - break; - case MV88E6393X_PORT_STS_CMODE_5GBASER: - case MV88E6393X_PORT_STS_CMODE_10GBASER: - case MV88E6393X_PORT_STS_CMODE_USXGMII: - err = mv88e6390_serdes_power_10g(chip, lane, on); - break; - default: - err = -EINVAL; - break; - } - - if (err) - return err; - - if (!on) { - err = mv88e6393x_serdes_power_lane(chip, lane, false); - if (err) - return err; - - err = mv88e6393x_serdes_fix_2500basex_an(chip, lane, cmode, - false); - } - - return err; -} diff --git a/drivers/net/dsa/mv88e6xxx/serdes.h b/drivers/net/dsa/mv88e6xxx/serdes.h index e245687ddb1d..aac95cab46e3 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.h +++ b/drivers/net/dsa/mv88e6xxx/serdes.h @@ -12,6 +12,8 @@ #include "chip.h" +struct phylink_link_state; + #define MV88E6352_ADDR_SERDES 0x0f #define MV88E6352_SERDES_PAGE_FIBER 0x01 #define MV88E6352_SERDES_IRQ 0x0b @@ -44,6 +46,10 @@ /* 10GBASE-R and 10GBASE-X4/X2 */ #define MV88E6390_10G_CTRL1 (0x1000 + MDIO_CTRL1) #define MV88E6390_10G_STAT1 (0x1000 + MDIO_STAT1) +#define MV88E6390_10G_INT_ENABLE 0x9001 +#define MV88E6390_10G_INT_LINK_DOWN BIT(3) +#define MV88E6390_10G_INT_LINK_UP BIT(2) +#define MV88E6390_10G_INT_STATUS 0x9003 #define MV88E6393X_10G_INT_ENABLE 0x9000 #define MV88E6393X_10G_INT_LINK_CHANGE BIT(2) #define MV88E6393X_10G_INT_STATUS 0x9001 @@ -107,65 +113,17 @@ #define MV88E6393X_ERRATA_4_8_REG 0xF074 #define MV88E6393X_ERRATA_4_8_BIT BIT(14) -int mv88e6185_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); +int mv88e6xxx_pcs_decode_state(struct device *dev, u16 bmsr, u16 lpa, + u16 status, struct phylink_link_state *state); + int mv88e6341_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); -int mv88e6352_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); int mv88e6390_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); int mv88e6390x_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); int mv88e6393x_serdes_get_lane(struct mv88e6xxx_chip *chip, int port); -int mv88e6352_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port, - int lane, unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise); -int mv88e6390_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port, - int lane, unsigned int mode, - phy_interface_t interface, - const unsigned long *advertise); -int mv88e6185_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state); -int mv88e6352_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state); -int mv88e6390_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state); -int mv88e6393x_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port, - int lane, struct phylink_link_state *state); -int mv88e6352_serdes_pcs_an_restart(struct mv88e6xxx_chip *chip, int port, - int lane); -int mv88e6390_serdes_pcs_an_restart(struct mv88e6xxx_chip *chip, int port, - int lane); -int mv88e6352_serdes_pcs_link_up(struct mv88e6xxx_chip *chip, int port, - int lane, int speed, int duplex); -int mv88e6390_serdes_pcs_link_up(struct mv88e6xxx_chip *chip, int port, - int lane, int speed, int duplex); unsigned int mv88e6352_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port); unsigned int mv88e6390_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port); -int mv88e6185_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool up); -int mv88e6352_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool on); -int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool on); -int mv88e6393x_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane, - bool on); -int mv88e6393x_serdes_setup_errata(struct mv88e6xxx_chip *chip); -int mv88e6097_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable); -int mv88e6352_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable); -int mv88e6390_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, int lane, - bool enable); -int mv88e6393x_serdes_irq_enable(struct mv88e6xxx_chip *chip, int port, - int lane, bool enable); -irqreturn_t mv88e6097_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane); -irqreturn_t mv88e6352_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane); -irqreturn_t mv88e6390_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane); -irqreturn_t mv88e6393x_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, - int lane); int mv88e6352_serdes_get_sset_count(struct mv88e6xxx_chip *chip, int port); int mv88e6352_serdes_get_strings(struct mv88e6xxx_chip *chip, int port, uint8_t *data); @@ -195,24 +153,6 @@ static inline int mv88e6xxx_serdes_get_lane(struct mv88e6xxx_chip *chip, return chip->info->ops->serdes_get_lane(chip, port); } -static inline int mv88e6xxx_serdes_power_up(struct mv88e6xxx_chip *chip, - int port, int lane) -{ - if (!chip->info->ops->serdes_power) - return -EOPNOTSUPP; - - return chip->info->ops->serdes_power(chip, port, lane, true); -} - -static inline int mv88e6xxx_serdes_power_down(struct mv88e6xxx_chip *chip, - int port, int lane) -{ - if (!chip->info->ops->serdes_power) - return -EOPNOTSUPP; - - return chip->info->ops->serdes_power(chip, port, lane, false); -} - static inline unsigned int mv88e6xxx_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port) { @@ -222,31 +162,9 @@ mv88e6xxx_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int port) return chip->info->ops->serdes_irq_mapping(chip, port); } -static inline int mv88e6xxx_serdes_irq_enable(struct mv88e6xxx_chip *chip, - int port, int lane) -{ - if (!chip->info->ops->serdes_irq_enable) - return -EOPNOTSUPP; - - return chip->info->ops->serdes_irq_enable(chip, port, lane, true); -} - -static inline int mv88e6xxx_serdes_irq_disable(struct mv88e6xxx_chip *chip, - int port, int lane) -{ - if (!chip->info->ops->serdes_irq_enable) - return -EOPNOTSUPP; - - return chip->info->ops->serdes_irq_enable(chip, port, lane, false); -} - -static inline irqreturn_t -mv88e6xxx_serdes_irq_status(struct mv88e6xxx_chip *chip, int port, int lane) -{ - if (!chip->info->ops->serdes_irq_status) - return IRQ_NONE; - - return chip->info->ops->serdes_irq_status(chip, port, lane); -} +extern const struct mv88e6xxx_pcs_ops mv88e6185_pcs_ops; +extern const struct mv88e6xxx_pcs_ops mv88e6352_pcs_ops; +extern const struct mv88e6xxx_pcs_ops mv88e6390_pcs_ops; +extern const struct mv88e6xxx_pcs_ops mv88e6393x_pcs_ops; #endif diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index 8da46d284e35..fd7eb4a52918 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -1042,12 +1042,6 @@ static void felix_phylink_get_caps(struct dsa_switch *ds, int port, { struct ocelot *ocelot = ds->priv; - /* This driver does not make use of the speed, duplex, pause or the - * advertisement in its mac_config, so it is safe to mark this driver - * as non-legacy. - */ - config->legacy_pre_march2020 = false; - config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD | MAC_2500FD; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 1c113957fcf4..4a6e52929d25 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -16,6 +16,7 @@ #include <net/pkt_sched.h> #include <linux/iopoll.h> #include <linux/mdio.h> +#include <linux/of.h> #include <linux/pci.h> #include <linux/time.h> #include "felix.h" @@ -1745,10 +1746,10 @@ static int vsc9959_stream_identify(struct flow_cls_offload *f, struct flow_dissector *dissector = rule->match.dissector; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS))) return -EOPNOTSUPP; if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index 15003b2af264..8f912bda120b 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -2,13 +2,14 @@ /* Distributed Switch Architecture VSC9953 driver * Copyright (C) 2020, Maxim Kochetkov <fido_max@inbox.ru> */ +#include <linux/platform_device.h> #include <linux/types.h> #include <soc/mscc/ocelot_vcap.h> #include <soc/mscc/ocelot_sys.h> #include <soc/mscc/ocelot.h> #include <linux/mdio/mdio-mscc-miim.h> +#include <linux/mod_devicetable.h> #include <linux/of_mdio.h> -#include <linux/of_platform.h> #include <linux/pcs-lynx.h> #include <linux/dsa/ocelot.h> #include <linux/iopoll.h> diff --git a/drivers/net/dsa/qca/ar9331.c b/drivers/net/dsa/qca/ar9331.c index 3b0937031499..8d9d271ac3af 100644 --- a/drivers/net/dsa/qca/ar9331.c +++ b/drivers/net/dsa/qca/ar9331.c @@ -1012,7 +1012,7 @@ static const struct regmap_config ar9331_mdio_regmap_config = { .wr_table = &ar9331_register_set, .rd_table = &ar9331_register_set, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static struct regmap_bus ar9331_sw_bus = { diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c index efe9380d4a15..de1dc22cf683 100644 --- a/drivers/net/dsa/qca/qca8k-8xxx.c +++ b/drivers/net/dsa/qca/qca8k-8xxx.c @@ -1400,8 +1400,6 @@ static void qca8k_phylink_get_caps(struct dsa_switch *ds, int port, config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD; - - config->legacy_pre_march2020 = false; } static void @@ -1758,11 +1756,52 @@ static int qca8k_connect_tag_protocol(struct dsa_switch *ds, return 0; } +static void qca8k_setup_hol_fixup(struct qca8k_priv *priv, int port) +{ + u32 mask; + + switch (port) { + /* The 2 CPU port and port 5 requires some different + * priority than any other ports. + */ + case 0: + case 5: + case 6: + mask = QCA8K_PORT_HOL_CTRL0_EG_PRI0(0x3) | + QCA8K_PORT_HOL_CTRL0_EG_PRI1(0x4) | + QCA8K_PORT_HOL_CTRL0_EG_PRI2(0x4) | + QCA8K_PORT_HOL_CTRL0_EG_PRI3(0x4) | + QCA8K_PORT_HOL_CTRL0_EG_PRI4(0x6) | + QCA8K_PORT_HOL_CTRL0_EG_PRI5(0x8) | + QCA8K_PORT_HOL_CTRL0_EG_PORT(0x1e); + break; + default: + mask = QCA8K_PORT_HOL_CTRL0_EG_PRI0(0x3) | + QCA8K_PORT_HOL_CTRL0_EG_PRI1(0x4) | + QCA8K_PORT_HOL_CTRL0_EG_PRI2(0x6) | + QCA8K_PORT_HOL_CTRL0_EG_PRI3(0x8) | + QCA8K_PORT_HOL_CTRL0_EG_PORT(0x19); + } + regmap_write(priv->regmap, QCA8K_REG_PORT_HOL_CTRL0(port), mask); + + mask = QCA8K_PORT_HOL_CTRL1_ING(0x6) | + QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN | + QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | + QCA8K_PORT_HOL_CTRL1_WRED_EN; + regmap_update_bits(priv->regmap, QCA8K_REG_PORT_HOL_CTRL1(port), + QCA8K_PORT_HOL_CTRL1_ING_BUF_MASK | + QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN | + QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | + QCA8K_PORT_HOL_CTRL1_WRED_EN, + mask); +} + static int qca8k_setup(struct dsa_switch *ds) { struct qca8k_priv *priv = ds->priv; - int cpu_port, ret, i; + struct dsa_port *dp; + int cpu_port, ret; u32 mask; cpu_port = qca8k_find_cpu_port(ds); @@ -1817,27 +1856,27 @@ qca8k_setup(struct dsa_switch *ds) dev_warn(priv->dev, "mib init failed"); /* Initial setup of all ports */ - for (i = 0; i < QCA8K_NUM_PORTS; i++) { + dsa_switch_for_each_port(dp, ds) { /* Disable forwarding by default on all ports */ - ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i), + ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(dp->index), QCA8K_PORT_LOOKUP_MEMBER, 0); if (ret) return ret; + } - /* Enable QCA header mode on all cpu ports */ - if (dsa_is_cpu_port(ds, i)) { - ret = qca8k_write(priv, QCA8K_REG_PORT_HDR_CTRL(i), - FIELD_PREP(QCA8K_PORT_HDR_CTRL_TX_MASK, QCA8K_PORT_HDR_CTRL_ALL) | - FIELD_PREP(QCA8K_PORT_HDR_CTRL_RX_MASK, QCA8K_PORT_HDR_CTRL_ALL)); - if (ret) { - dev_err(priv->dev, "failed enabling QCA header mode"); - return ret; - } + /* Disable MAC by default on all user ports */ + dsa_switch_for_each_user_port(dp, ds) + qca8k_port_set_status(priv, dp->index, 0); + + /* Enable QCA header mode on all cpu ports */ + dsa_switch_for_each_cpu_port(dp, ds) { + ret = qca8k_write(priv, QCA8K_REG_PORT_HDR_CTRL(dp->index), + FIELD_PREP(QCA8K_PORT_HDR_CTRL_TX_MASK, QCA8K_PORT_HDR_CTRL_ALL) | + FIELD_PREP(QCA8K_PORT_HDR_CTRL_RX_MASK, QCA8K_PORT_HDR_CTRL_ALL)); + if (ret) { + dev_err(priv->dev, "failed enabling QCA header mode on port %d", dp->index); + return ret; } - - /* Disable MAC by default on all user ports */ - if (dsa_is_user_port(ds, i)) - qca8k_port_set_status(priv, i, 0); } /* Forward all unknown frames to CPU port for Linux processing @@ -1852,92 +1891,55 @@ qca8k_setup(struct dsa_switch *ds) if (ret) return ret; + /* CPU port gets connected to all user ports of the switch */ + ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(cpu_port), + QCA8K_PORT_LOOKUP_MEMBER, dsa_user_ports(ds)); + if (ret) + return ret; + /* Setup connection between CPU port & user ports - * Configure specific switch configuration for ports + * Individual user ports get connected to CPU port only */ - for (i = 0; i < QCA8K_NUM_PORTS; i++) { - /* CPU port gets connected to all user ports of the switch */ - if (dsa_is_cpu_port(ds, i)) { - ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i), - QCA8K_PORT_LOOKUP_MEMBER, dsa_user_ports(ds)); - if (ret) - return ret; - } + dsa_switch_for_each_user_port(dp, ds) { + u8 port = dp->index; - /* Individual user ports get connected to CPU port only */ - if (dsa_is_user_port(ds, i)) { - ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i), - QCA8K_PORT_LOOKUP_MEMBER, - BIT(cpu_port)); - if (ret) - return ret; - - /* Enable ARP Auto-learning by default */ - ret = regmap_set_bits(priv->regmap, QCA8K_PORT_LOOKUP_CTRL(i), - QCA8K_PORT_LOOKUP_LEARN); - if (ret) - return ret; - - /* For port based vlans to work we need to set the - * default egress vid - */ - ret = qca8k_rmw(priv, QCA8K_EGRESS_VLAN(i), - QCA8K_EGREES_VLAN_PORT_MASK(i), - QCA8K_EGREES_VLAN_PORT(i, QCA8K_PORT_VID_DEF)); - if (ret) - return ret; - - ret = qca8k_write(priv, QCA8K_REG_PORT_VLAN_CTRL0(i), - QCA8K_PORT_VLAN_CVID(QCA8K_PORT_VID_DEF) | - QCA8K_PORT_VLAN_SVID(QCA8K_PORT_VID_DEF)); - if (ret) - return ret; - } + ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port), + QCA8K_PORT_LOOKUP_MEMBER, + BIT(cpu_port)); + if (ret) + return ret; - /* The port 5 of the qca8337 have some problem in flood condition. The - * original legacy driver had some specific buffer and priority settings - * for the different port suggested by the QCA switch team. Add this - * missing settings to improve switch stability under load condition. - * This problem is limited to qca8337 and other qca8k switch are not affected. + ret = regmap_clear_bits(priv->regmap, QCA8K_PORT_LOOKUP_CTRL(port), + QCA8K_PORT_LOOKUP_LEARN); + if (ret) + return ret; + + /* For port based vlans to work we need to set the + * default egress vid */ - if (priv->switch_id == QCA8K_ID_QCA8337) { - switch (i) { - /* The 2 CPU port and port 5 requires some different - * priority than any other ports. - */ - case 0: - case 5: - case 6: - mask = QCA8K_PORT_HOL_CTRL0_EG_PRI0(0x3) | - QCA8K_PORT_HOL_CTRL0_EG_PRI1(0x4) | - QCA8K_PORT_HOL_CTRL0_EG_PRI2(0x4) | - QCA8K_PORT_HOL_CTRL0_EG_PRI3(0x4) | - QCA8K_PORT_HOL_CTRL0_EG_PRI4(0x6) | - QCA8K_PORT_HOL_CTRL0_EG_PRI5(0x8) | - QCA8K_PORT_HOL_CTRL0_EG_PORT(0x1e); - break; - default: - mask = QCA8K_PORT_HOL_CTRL0_EG_PRI0(0x3) | - QCA8K_PORT_HOL_CTRL0_EG_PRI1(0x4) | - QCA8K_PORT_HOL_CTRL0_EG_PRI2(0x6) | - QCA8K_PORT_HOL_CTRL0_EG_PRI3(0x8) | - QCA8K_PORT_HOL_CTRL0_EG_PORT(0x19); - } - qca8k_write(priv, QCA8K_REG_PORT_HOL_CTRL0(i), mask); - - mask = QCA8K_PORT_HOL_CTRL1_ING(0x6) | - QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN | - QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | - QCA8K_PORT_HOL_CTRL1_WRED_EN; - qca8k_rmw(priv, QCA8K_REG_PORT_HOL_CTRL1(i), - QCA8K_PORT_HOL_CTRL1_ING_BUF_MASK | - QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN | - QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | - QCA8K_PORT_HOL_CTRL1_WRED_EN, - mask); - } + ret = qca8k_rmw(priv, QCA8K_EGRESS_VLAN(port), + QCA8K_EGREES_VLAN_PORT_MASK(port), + QCA8K_EGREES_VLAN_PORT(port, QCA8K_PORT_VID_DEF)); + if (ret) + return ret; + + ret = qca8k_write(priv, QCA8K_REG_PORT_VLAN_CTRL0(port), + QCA8K_PORT_VLAN_CVID(QCA8K_PORT_VID_DEF) | + QCA8K_PORT_VLAN_SVID(QCA8K_PORT_VID_DEF)); + if (ret) + return ret; } + /* The port 5 of the qca8337 have some problem in flood condition. The + * original legacy driver had some specific buffer and priority settings + * for the different port suggested by the QCA switch team. Add this + * missing settings to improve switch stability under load condition. + * This problem is limited to qca8337 and other qca8k switch are not affected. + */ + if (priv->switch_id == QCA8K_ID_QCA8337) + dsa_switch_for_each_available_port(dp, ds) + qca8k_setup_hol_fixup(priv, dp->index); + /* Special GLOBAL_FC_THRESH value are needed for ar8327 switch */ if (priv->switch_id == QCA8K_ID_QCA8327) { mask = QCA8K_GLOBAL_FC_GOL_XON_THRES(288) | @@ -1980,6 +1982,8 @@ static const struct dsa_switch_ops qca8k_switch_ops = { .port_change_mtu = qca8k_port_change_mtu, .port_max_mtu = qca8k_port_max_mtu, .port_stp_state_set = qca8k_port_stp_state_set, + .port_pre_bridge_flags = qca8k_port_pre_bridge_flags, + .port_bridge_flags = qca8k_port_bridge_flags, .port_bridge_join = qca8k_port_bridge_join, .port_bridge_leave = qca8k_port_bridge_leave, .port_fast_age = qca8k_port_fast_age, diff --git a/drivers/net/dsa/qca/qca8k-common.c b/drivers/net/dsa/qca/qca8k-common.c index 13b8452ce5b2..fce04ce12cf9 100644 --- a/drivers/net/dsa/qca/qca8k-common.c +++ b/drivers/net/dsa/qca/qca8k-common.c @@ -565,9 +565,26 @@ int qca8k_get_mac_eee(struct dsa_switch *ds, int port, return 0; } +static int qca8k_port_configure_learning(struct dsa_switch *ds, int port, + bool learning) +{ + struct qca8k_priv *priv = ds->priv; + + if (learning) + return regmap_set_bits(priv->regmap, + QCA8K_PORT_LOOKUP_CTRL(port), + QCA8K_PORT_LOOKUP_LEARN); + else + return regmap_clear_bits(priv->regmap, + QCA8K_PORT_LOOKUP_CTRL(port), + QCA8K_PORT_LOOKUP_LEARN); +} + void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) { + struct dsa_port *dp = dsa_to_port(ds, port); struct qca8k_priv *priv = ds->priv; + bool learning = false; u32 stp_state; switch (state) { @@ -582,8 +599,11 @@ void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) break; case BR_STATE_LEARNING: stp_state = QCA8K_PORT_LOOKUP_STATE_LEARNING; + learning = dp->learning; break; case BR_STATE_FORWARDING: + learning = dp->learning; + fallthrough; default: stp_state = QCA8K_PORT_LOOKUP_STATE_FORWARD; break; @@ -591,6 +611,34 @@ void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port), QCA8K_PORT_LOOKUP_STATE_MASK, stp_state); + + qca8k_port_configure_learning(ds, port, learning); +} + +int qca8k_port_pre_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) +{ + if (flags.mask & ~BR_LEARNING) + return -EINVAL; + + return 0; +} + +int qca8k_port_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) +{ + int ret; + + if (flags.mask & BR_LEARNING) { + ret = qca8k_port_configure_learning(ds, port, + flags.val & BR_LEARNING); + if (ret) + return ret; + } + + return 0; } int qca8k_port_bridge_join(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/qca/qca8k-leds.c b/drivers/net/dsa/qca/qca8k-leds.c index 1261e0bb21ef..e8c16e76e34b 100644 --- a/drivers/net/dsa/qca/qca8k-leds.c +++ b/drivers/net/dsa/qca/qca8k-leds.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/property.h> #include <linux/regmap.h> #include <net/dsa.h> diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h index c5cc8a172d65..8f88b7db384d 100644 --- a/drivers/net/dsa/qca/qca8k.h +++ b/drivers/net/dsa/qca/qca8k.h @@ -522,6 +522,12 @@ int qca8k_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e); /* Common bridge function */ void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state); +int qca8k_port_pre_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack); +int qca8k_port_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack); int qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct dsa_bridge bridge, bool *tx_fwd_offload, diff --git a/drivers/net/dsa/realtek/realtek-mdio.c b/drivers/net/dsa/realtek/realtek-mdio.c index 5a8fe707ca25..4310e7793e58 100644 --- a/drivers/net/dsa/realtek/realtek-mdio.c +++ b/drivers/net/dsa/realtek/realtek-mdio.c @@ -20,7 +20,7 @@ */ #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/overflow.h> #include <linux/regmap.h> diff --git a/drivers/net/dsa/realtek/realtek-smi.c b/drivers/net/dsa/realtek/realtek-smi.c index 1b447d96b9c4..c2bd8bb6c9c2 100644 --- a/drivers/net/dsa/realtek/realtek-smi.c +++ b/drivers/net/dsa/realtek/realtek-smi.c @@ -31,7 +31,6 @@ #include <linux/spinlock.h> #include <linux/skbuff.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/delay.h> #include <linux/gpio/consumer.h> diff --git a/drivers/net/dsa/sja1105/sja1105_flower.c b/drivers/net/dsa/sja1105/sja1105_flower.c index fad5afe3819c..9e8ca182c722 100644 --- a/drivers/net/dsa/sja1105/sja1105_flower.c +++ b/drivers/net/dsa/sja1105/sja1105_flower.c @@ -205,10 +205,10 @@ static int sja1105_flower_parse_key(struct sja1105_private *priv, u16 pcp = U16_MAX; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used"); return -EOPNOTSUPP; diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 3529a565b4aa..331bb1c6676a 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -15,7 +15,6 @@ #include <linux/of.h> #include <linux/of_net.h> #include <linux/of_mdio.h> -#include <linux/of_device.h> #include <linux/pcs/pcs-xpcs.h> #include <linux/netdev_features.h> #include <linux/netdevice.h> @@ -1396,12 +1395,6 @@ static void sja1105_phylink_get_caps(struct dsa_switch *ds, int port, struct sja1105_xmii_params_entry *mii; phy_interface_t phy_mode; - /* This driver does not make use of the speed, duplex, pause or the - * advertisement in its mac_config, so it is safe to mark this driver - * as non-legacy. - */ - config->legacy_pre_march2020 = false; - phy_mode = priv->phy_mode[port]; if (phy_mode == PHY_INTERFACE_MODE_SGMII || phy_mode == PHY_INTERFACE_MODE_2500BASEX) { diff --git a/drivers/net/dsa/vitesse-vsc73xx-core.c b/drivers/net/dsa/vitesse-vsc73xx-core.c index ef1a4a7c47b2..4f09e7438f3b 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-core.c +++ b/drivers/net/dsa/vitesse-vsc73xx-core.c @@ -18,7 +18,6 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/bitops.h> #include <linux/if_bridge.h> diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index fa622639d640..753fef757f11 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -7,7 +7,7 @@ #include <net/dsa.h> #include <linux/etherdevice.h> #include <linux/if_bridge.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/netdev_features.h> #include <linux/if_hsr.h> #include "xrs700x.h" diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c index aa0d2f3aaeaa..597a02c75d52 100644 --- a/drivers/net/ethernet/aeroflex/greth.c +++ b/drivers/net/ethernet/aeroflex/greth.c @@ -29,9 +29,9 @@ #include <linux/io.h> #include <linux/crc32.h> #include <linux/mii.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_net.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <asm/cacheflush.h> #include <asm/byteorder.h> diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c index a30d0f172986..78231c85234d 100644 --- a/drivers/net/ethernet/alacritech/slicoss.c +++ b/drivers/net/ethernet/alacritech/slicoss.c @@ -1520,10 +1520,8 @@ static void slic_get_ethtool_stats(struct net_device *dev, static void slic_get_strings(struct net_device *dev, u32 stringset, u8 *data) { - if (stringset == ETH_SS_STATS) { + if (stringset == ETH_SS_STATS) memcpy(data, slic_stats_strings, sizeof(slic_stats_strings)); - data += sizeof(slic_stats_strings); - } } static void slic_get_drvinfo(struct net_device *dev, diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c index 561af8e5b3ea..6787a5fae908 100644 --- a/drivers/net/ethernet/amd/pds_core/auxbus.c +++ b/drivers/net/ethernet/amd/pds_core/auxbus.c @@ -11,7 +11,7 @@ * @pf_pdev: ptr to the PF driver struct * @devname: name that includes service into, e.g. pds_core.vDPA * - * Return: 0 on success, or + * Return: positive client ID (ci) on success, or * negative for error */ int pds_client_register(struct pci_dev *pf_pdev, char *devname) diff --git a/drivers/net/ethernet/amd/sunlance.c b/drivers/net/ethernet/amd/sunlance.c index 68ca1225eedc..33bb539ad70a 100644 --- a/drivers/net/ethernet/amd/sunlance.c +++ b/drivers/net/ethernet/amd/sunlance.c @@ -92,7 +92,7 @@ static char lancestr[] = "LANCE"; #include <linux/bitops.h> #include <linux/dma-mapping.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/gfp.h> #include <linux/pgtable.h> diff --git a/drivers/net/ethernet/apm/xgene-v2/main.h b/drivers/net/ethernet/apm/xgene-v2/main.h index b3985a7be59d..7be6f83e22fe 100644 --- a/drivers/net/ethernet/apm/xgene-v2/main.h +++ b/drivers/net/ethernet/apm/xgene-v2/main.h @@ -22,6 +22,7 @@ #include <linux/of_mdio.h> #include <linux/prefetch.h> #include <linux/phy.h> +#include <linux/platform_device.h> #include <net/ip.h> #include "mac.h" #include "enet.h" diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c index 390671640388..41d96f4b23d8 100644 --- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c +++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c @@ -1632,7 +1632,7 @@ static int xgene_enet_get_irqs(struct xgene_enet_pdata *pdata) for (i = 0; i < max_irqs; i++) { ret = platform_get_irq(pdev, i); - if (ret <= 0) { + if (ret < 0) { if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) { max_irqs = i; pdata->rxq_cnt = max_irqs / 2; @@ -1640,7 +1640,7 @@ static int xgene_enet_get_irqs(struct xgene_enet_pdata *pdata) pdata->cq_cnt = max_irqs / 2; break; } - return ret ? : -ENXIO; + return ret; } pdata->irqs[i] = ret; } diff --git a/drivers/net/ethernet/apple/macmace.c b/drivers/net/ethernet/apple/macmace.c index 8fcaf1639920..8775c3234e91 100644 --- a/drivers/net/ethernet/apple/macmace.c +++ b/drivers/net/ethernet/apple/macmace.c @@ -77,7 +77,7 @@ struct mace_frame { u8 pad4; u32 pad5; u32 pad6; - u8 data[1]; + DECLARE_FLEX_ARRAY(u8, data); /* And frame continues.. */ }; diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c index 5dfc751572ed..220400a633f5 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c @@ -93,7 +93,7 @@ static u32 hw_atl2_sem_act_rslvr_get(struct aq_hw_s *self) static int hw_atl2_hw_reset(struct aq_hw_s *self) { - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; + struct hw_atl2_priv *priv = self->priv; int err; err = hw_atl2_utils_soft_reset(self); @@ -378,8 +378,8 @@ static int hw_atl2_hw_init_tx_path(struct aq_hw_s *self) static void hw_atl2_hw_init_new_rx_filters(struct aq_hw_s *self) { - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; u8 *prio_tc_map = self->aq_nic_cfg->prio_tc_map; + struct hw_atl2_priv *priv = self->priv; u16 action; u8 index; int i; @@ -433,7 +433,7 @@ static void hw_atl2_hw_new_rx_filter_vlan_promisc(struct aq_hw_s *self, u16 off_action = (!promisc && !hw_atl_rpfl2promiscuous_mode_en_get(self)) ? HW_ATL2_ACTION_DROP : HW_ATL2_ACTION_DISABLE; - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; + struct hw_atl2_priv *priv = self->priv; u8 index; index = priv->art_base_index + HW_ATL2_RPF_VLAN_PROMISC_OFF_INDEX; @@ -445,7 +445,7 @@ static void hw_atl2_hw_new_rx_filter_vlan_promisc(struct aq_hw_s *self, static void hw_atl2_hw_new_rx_filter_promisc(struct aq_hw_s *self, bool promisc) { u16 off_action = promisc ? HW_ATL2_ACTION_DISABLE : HW_ATL2_ACTION_DROP; - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; + struct hw_atl2_priv *priv = self->priv; bool vlan_promisc_enable; u8 index; @@ -539,8 +539,8 @@ static int hw_atl2_hw_init(struct aq_hw_s *self, const u8 *mac_addr) [AQ_HW_IRQ_MSIX] = { 0x20000022U, 0x20000026U }, }; - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; struct aq_nic_cfg_s *aq_nic_cfg = self->aq_nic_cfg; + struct hw_atl2_priv *priv = self->priv; u8 base_index, count; int err; @@ -770,7 +770,7 @@ static struct aq_stats_s *hw_atl2_utils_get_hw_stats(struct aq_hw_s *self) static int hw_atl2_hw_vlan_set(struct aq_hw_s *self, struct aq_rx_filter_vlan *aq_vlans) { - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; + struct hw_atl2_priv *priv = self->priv; u32 queue; u8 index; int i; diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c index 674683b54304..52e2070a4a2f 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c @@ -413,8 +413,8 @@ do { \ static int aq_a2_fw_update_stats(struct aq_hw_s *self) { - struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv; struct aq_stats_s *cs = &self->curr_stats; + struct hw_atl2_priv *priv = self->priv; struct statistics_s stats; struct version_s version; int err; diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c index 2b427d8a1831..31ee477dd131 100644 --- a/drivers/net/ethernet/arc/emac_main.c +++ b/drivers/net/ethernet/arc/emac_main.c @@ -15,11 +15,11 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> #include "emac.h" diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index ff1a5edf8df1..009e0b3066fa 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -29,9 +29,10 @@ #include <linux/if_vlan.h> #include <linux/mfd/syscon.h> +#include <linux/of.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/phylink.h> #include <linux/regmap.h> #include <linux/reset.h> diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index 948586bf1b5b..75ca3ddda1f5 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -255,4 +255,16 @@ config BNXT_HWMON Say Y if you want to expose the thermal sensor data on NetXtreme-C/E devices, via the hwmon sysfs interface. +config BCMASP + tristate "Broadcom ASP 2.0 Ethernet support" + depends on ARCH_BRCMSTB || COMPILE_TEST + default ARCH_BRCMSTB + depends on OF + select MII + select PHYLIB + select MDIO_BCM_UNIMAC + help + This configuration enables the Broadcom ASP 2.0 Ethernet controller + driver which is present in Broadcom STB SoCs such as 72165. + endif # NET_VENDOR_BROADCOM diff --git a/drivers/net/ethernet/broadcom/Makefile b/drivers/net/ethernet/broadcom/Makefile index 0ddfb5b5d53c..bac5cb6ad0cd 100644 --- a/drivers/net/ethernet/broadcom/Makefile +++ b/drivers/net/ethernet/broadcom/Makefile @@ -17,3 +17,4 @@ obj-$(CONFIG_BGMAC_BCMA) += bgmac-bcma.o bgmac-bcma-mdio.o obj-$(CONFIG_BGMAC_PLATFORM) += bgmac-platform.o obj-$(CONFIG_SYSTEMPORT) += bcmsysport.o obj-$(CONFIG_BNXT) += bnxt/ +obj-$(CONFIG_BCMASP) += asp2/ diff --git a/drivers/net/ethernet/broadcom/asp2/Makefile b/drivers/net/ethernet/broadcom/asp2/Makefile new file mode 100644 index 000000000000..e07550315f83 --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_BCMASP) += bcm-asp.o +bcm-asp-objs := bcmasp.o bcmasp_intf.o bcmasp_ethtool.o diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.c b/drivers/net/ethernet/broadcom/asp2/bcmasp.c new file mode 100644 index 000000000000..eb35ced1c8ba --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.c @@ -0,0 +1,1437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Broadcom STB ASP 2.0 Driver + * + * Copyright (c) 2023 Broadcom + */ +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/clk.h> + +#include "bcmasp.h" +#include "bcmasp_intf_defs.h" + +static void _intr2_mask_clear(struct bcmasp_priv *priv, u32 mask) +{ + intr2_core_wl(priv, mask, ASP_INTR2_MASK_CLEAR); + priv->irq_mask &= ~mask; +} + +static void _intr2_mask_set(struct bcmasp_priv *priv, u32 mask) +{ + intr2_core_wl(priv, mask, ASP_INTR2_MASK_SET); + priv->irq_mask |= mask; +} + +void bcmasp_enable_tx_irq(struct bcmasp_intf *intf, int en) +{ + struct bcmasp_priv *priv = intf->parent; + + if (en) + _intr2_mask_clear(priv, ASP_INTR2_TX_DESC(intf->channel)); + else + _intr2_mask_set(priv, ASP_INTR2_TX_DESC(intf->channel)); +} +EXPORT_SYMBOL_GPL(bcmasp_enable_tx_irq); + +void bcmasp_enable_rx_irq(struct bcmasp_intf *intf, int en) +{ + struct bcmasp_priv *priv = intf->parent; + + if (en) + _intr2_mask_clear(priv, ASP_INTR2_RX_ECH(intf->channel)); + else + _intr2_mask_set(priv, ASP_INTR2_RX_ECH(intf->channel)); +} +EXPORT_SYMBOL_GPL(bcmasp_enable_rx_irq); + +static void bcmasp_intr2_mask_set_all(struct bcmasp_priv *priv) +{ + _intr2_mask_set(priv, 0xffffffff); + priv->irq_mask = 0xffffffff; +} + +static void bcmasp_intr2_clear_all(struct bcmasp_priv *priv) +{ + intr2_core_wl(priv, 0xffffffff, ASP_INTR2_CLEAR); +} + +static void bcmasp_intr2_handling(struct bcmasp_intf *intf, u32 status) +{ + if (status & ASP_INTR2_RX_ECH(intf->channel)) { + if (likely(napi_schedule_prep(&intf->rx_napi))) { + bcmasp_enable_rx_irq(intf, 0); + __napi_schedule_irqoff(&intf->rx_napi); + } + } + + if (status & ASP_INTR2_TX_DESC(intf->channel)) { + if (likely(napi_schedule_prep(&intf->tx_napi))) { + bcmasp_enable_tx_irq(intf, 0); + __napi_schedule_irqoff(&intf->tx_napi); + } + } +} + +static irqreturn_t bcmasp_isr(int irq, void *data) +{ + struct bcmasp_priv *priv = data; + struct bcmasp_intf *intf; + u32 status; + + status = intr2_core_rl(priv, ASP_INTR2_STATUS) & + ~intr2_core_rl(priv, ASP_INTR2_MASK_STATUS); + + intr2_core_wl(priv, status, ASP_INTR2_CLEAR); + + if (unlikely(status == 0)) { + dev_warn(&priv->pdev->dev, "l2 spurious interrupt\n"); + return IRQ_NONE; + } + + /* Handle intferfaces */ + list_for_each_entry(intf, &priv->intfs, list) + bcmasp_intr2_handling(intf, status); + + return IRQ_HANDLED; +} + +void bcmasp_flush_rx_port(struct bcmasp_intf *intf) +{ + struct bcmasp_priv *priv = intf->parent; + u32 mask; + + switch (intf->port) { + case 0: + mask = ASP_CTRL_UMAC0_FLUSH_MASK; + break; + case 1: + mask = ASP_CTRL_UMAC1_FLUSH_MASK; + break; + case 2: + mask = ASP_CTRL_SPB_FLUSH_MASK; + break; + default: + /* Not valid port */ + return; + } + + rx_ctrl_core_wl(priv, mask, priv->hw_info->rx_ctrl_flush); +} + +static void bcmasp_netfilt_hw_en_wake(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt) +{ + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_OFFSET_L3_1(64), + ASP_RX_FILTER_NET_OFFSET(nfilt->hw_index)); + + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_OFFSET_L2(32) | + ASP_RX_FILTER_NET_OFFSET_L3_0(32) | + ASP_RX_FILTER_NET_OFFSET_L3_1(96) | + ASP_RX_FILTER_NET_OFFSET_L4(32), + ASP_RX_FILTER_NET_OFFSET(nfilt->hw_index + 1)); + + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->port + 8) | + ASP_RX_FILTER_NET_CFG_EN | + ASP_RX_FILTER_NET_CFG_L2_EN | + ASP_RX_FILTER_NET_CFG_L3_EN | + ASP_RX_FILTER_NET_CFG_L4_EN | + ASP_RX_FILTER_NET_CFG_L3_FRM(2) | + ASP_RX_FILTER_NET_CFG_L4_FRM(2) | + ASP_RX_FILTER_NET_CFG_UMC(nfilt->port), + ASP_RX_FILTER_NET_CFG(nfilt->hw_index)); + + rx_filter_core_wl(priv, ASP_RX_FILTER_NET_CFG_CH(nfilt->port + 8) | + ASP_RX_FILTER_NET_CFG_EN | + ASP_RX_FILTER_NET_CFG_L2_EN | + ASP_RX_FILTER_NET_CFG_L3_EN | + ASP_RX_FILTER_NET_CFG_L4_EN | + ASP_RX_FILTER_NET_CFG_L3_FRM(2) | + ASP_RX_FILTER_NET_CFG_L4_FRM(2) | + ASP_RX_FILTER_NET_CFG_UMC(nfilt->port), + ASP_RX_FILTER_NET_CFG(nfilt->hw_index + 1)); +} + +#define MAX_WAKE_FILTER_SIZE 256 +enum asp_netfilt_reg_type { + ASP_NETFILT_MATCH = 0, + ASP_NETFILT_MASK, + ASP_NETFILT_MAX +}; + +static int bcmasp_netfilt_get_reg_offset(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + enum asp_netfilt_reg_type reg_type, + u32 offset) +{ + u32 block_index, filter_sel; + + if (offset < 32) { + block_index = ASP_RX_FILTER_NET_L2; + filter_sel = nfilt->hw_index; + } else if (offset < 64) { + block_index = ASP_RX_FILTER_NET_L2; + filter_sel = nfilt->hw_index + 1; + } else if (offset < 96) { + block_index = ASP_RX_FILTER_NET_L3_0; + filter_sel = nfilt->hw_index; + } else if (offset < 128) { + block_index = ASP_RX_FILTER_NET_L3_0; + filter_sel = nfilt->hw_index + 1; + } else if (offset < 160) { + block_index = ASP_RX_FILTER_NET_L3_1; + filter_sel = nfilt->hw_index; + } else if (offset < 192) { + block_index = ASP_RX_FILTER_NET_L3_1; + filter_sel = nfilt->hw_index + 1; + } else if (offset < 224) { + block_index = ASP_RX_FILTER_NET_L4; + filter_sel = nfilt->hw_index; + } else if (offset < 256) { + block_index = ASP_RX_FILTER_NET_L4; + filter_sel = nfilt->hw_index + 1; + } else { + return -EINVAL; + } + + switch (reg_type) { + case ASP_NETFILT_MATCH: + return ASP_RX_FILTER_NET_PAT(filter_sel, block_index, + (offset % 32)); + case ASP_NETFILT_MASK: + return ASP_RX_FILTER_NET_MASK(filter_sel, block_index, + (offset % 32)); + default: + return -EINVAL; + } +} + +static void bcmasp_netfilt_wr(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + enum asp_netfilt_reg_type reg_type, + u32 val, u32 offset) +{ + int reg_offset; + + /* HW only accepts 4 byte aligned writes */ + if (!IS_ALIGNED(offset, 4) || offset > MAX_WAKE_FILTER_SIZE) + return; + + reg_offset = bcmasp_netfilt_get_reg_offset(priv, nfilt, reg_type, + offset); + + rx_filter_core_wl(priv, val, reg_offset); +} + +static u32 bcmasp_netfilt_rd(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + enum asp_netfilt_reg_type reg_type, + u32 offset) +{ + int reg_offset; + + /* HW only accepts 4 byte aligned writes */ + if (!IS_ALIGNED(offset, 4) || offset > MAX_WAKE_FILTER_SIZE) + return 0; + + reg_offset = bcmasp_netfilt_get_reg_offset(priv, nfilt, reg_type, + offset); + + return rx_filter_core_rl(priv, reg_offset); +} + +static int bcmasp_netfilt_wr_m_wake(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + u32 offset, void *match, void *mask, + size_t size) +{ + u32 shift, mask_val = 0, match_val = 0; + bool first_byte = true; + + if ((offset + size) > MAX_WAKE_FILTER_SIZE) + return -EINVAL; + + while (size--) { + /* The HW only accepts 4 byte aligned writes, so if we + * begin unaligned or if remaining bytes less than 4, + * we need to read then write to avoid losing current + * register state + */ + if (first_byte && (!IS_ALIGNED(offset, 4) || size < 3)) { + match_val = bcmasp_netfilt_rd(priv, nfilt, + ASP_NETFILT_MATCH, + ALIGN_DOWN(offset, 4)); + mask_val = bcmasp_netfilt_rd(priv, nfilt, + ASP_NETFILT_MASK, + ALIGN_DOWN(offset, 4)); + } + + shift = (3 - (offset % 4)) * 8; + match_val &= ~GENMASK(shift + 7, shift); + mask_val &= ~GENMASK(shift + 7, shift); + match_val |= (u32)(*((u8 *)match) << shift); + mask_val |= (u32)(*((u8 *)mask) << shift); + + /* If last byte or last byte of word, write to reg */ + if (!size || ((offset % 4) == 3)) { + bcmasp_netfilt_wr(priv, nfilt, ASP_NETFILT_MATCH, + match_val, ALIGN_DOWN(offset, 4)); + bcmasp_netfilt_wr(priv, nfilt, ASP_NETFILT_MASK, + mask_val, ALIGN_DOWN(offset, 4)); + first_byte = true; + } else { + first_byte = false; + } + + offset++; + match++; + mask++; + } + + return 0; +} + +static void bcmasp_netfilt_reset_hw(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt) +{ + int i; + + for (i = 0; i < MAX_WAKE_FILTER_SIZE; i += 4) { + bcmasp_netfilt_wr(priv, nfilt, ASP_NETFILT_MATCH, 0, i); + bcmasp_netfilt_wr(priv, nfilt, ASP_NETFILT_MASK, 0, i); + } +} + +static void bcmasp_netfilt_tcpip4_wr(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + struct ethtool_tcpip4_spec *match, + struct ethtool_tcpip4_spec *mask, + u32 offset) +{ + __be16 val_16, mask_16; + + val_16 = htons(ETH_P_IP); + mask_16 = htons(0xFFFF); + bcmasp_netfilt_wr_m_wake(priv, nfilt, (ETH_ALEN * 2) + offset, + &val_16, &mask_16, sizeof(val_16)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 1, + &match->tos, &mask->tos, + sizeof(match->tos)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 12, + &match->ip4src, &mask->ip4src, + sizeof(match->ip4src)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 16, + &match->ip4dst, &mask->ip4dst, + sizeof(match->ip4dst)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 20, + &match->psrc, &mask->psrc, + sizeof(match->psrc)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 22, + &match->pdst, &mask->pdst, + sizeof(match->pdst)); +} + +static void bcmasp_netfilt_tcpip6_wr(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt, + struct ethtool_tcpip6_spec *match, + struct ethtool_tcpip6_spec *mask, + u32 offset) +{ + __be16 val_16, mask_16; + + val_16 = htons(ETH_P_IPV6); + mask_16 = htons(0xFFFF); + bcmasp_netfilt_wr_m_wake(priv, nfilt, (ETH_ALEN * 2) + offset, + &val_16, &mask_16, sizeof(val_16)); + val_16 = htons(match->tclass << 4); + mask_16 = htons(mask->tclass << 4); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset, + &val_16, &mask_16, sizeof(val_16)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 8, + &match->ip6src, &mask->ip6src, + sizeof(match->ip6src)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 24, + &match->ip6dst, &mask->ip6dst, + sizeof(match->ip6dst)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 40, + &match->psrc, &mask->psrc, + sizeof(match->psrc)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 42, + &match->pdst, &mask->pdst, + sizeof(match->pdst)); +} + +static int bcmasp_netfilt_wr_to_hw(struct bcmasp_priv *priv, + struct bcmasp_net_filter *nfilt) +{ + struct ethtool_rx_flow_spec *fs = &nfilt->fs; + unsigned int offset = 0; + __be16 val_16, mask_16; + u8 val_8, mask_8; + + /* Currently only supports wake filters */ + if (!nfilt->wake_filter) + return -EINVAL; + + bcmasp_netfilt_reset_hw(priv, nfilt); + + if (fs->flow_type & FLOW_MAC_EXT) { + bcmasp_netfilt_wr_m_wake(priv, nfilt, 0, &fs->h_ext.h_dest, + &fs->m_ext.h_dest, + sizeof(fs->h_ext.h_dest)); + } + + if ((fs->flow_type & FLOW_EXT) && + (fs->m_ext.vlan_etype || fs->m_ext.vlan_tci)) { + bcmasp_netfilt_wr_m_wake(priv, nfilt, (ETH_ALEN * 2), + &fs->h_ext.vlan_etype, + &fs->m_ext.vlan_etype, + sizeof(fs->h_ext.vlan_etype)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ((ETH_ALEN * 2) + 2), + &fs->h_ext.vlan_tci, + &fs->m_ext.vlan_tci, + sizeof(fs->h_ext.vlan_tci)); + offset += VLAN_HLEN; + } + + switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case ETHER_FLOW: + bcmasp_netfilt_wr_m_wake(priv, nfilt, 0, + &fs->h_u.ether_spec.h_dest, + &fs->m_u.ether_spec.h_dest, + sizeof(fs->h_u.ether_spec.h_dest)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_ALEN, + &fs->h_u.ether_spec.h_source, + &fs->m_u.ether_spec.h_source, + sizeof(fs->h_u.ether_spec.h_source)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, (ETH_ALEN * 2) + offset, + &fs->h_u.ether_spec.h_proto, + &fs->m_u.ether_spec.h_proto, + sizeof(fs->h_u.ether_spec.h_proto)); + + break; + case IP_USER_FLOW: + val_16 = htons(ETH_P_IP); + mask_16 = htons(0xFFFF); + bcmasp_netfilt_wr_m_wake(priv, nfilt, (ETH_ALEN * 2) + offset, + &val_16, &mask_16, sizeof(val_16)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 1, + &fs->h_u.usr_ip4_spec.tos, + &fs->m_u.usr_ip4_spec.tos, + sizeof(fs->h_u.usr_ip4_spec.tos)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 9, + &fs->h_u.usr_ip4_spec.proto, + &fs->m_u.usr_ip4_spec.proto, + sizeof(fs->h_u.usr_ip4_spec.proto)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 12, + &fs->h_u.usr_ip4_spec.ip4src, + &fs->m_u.usr_ip4_spec.ip4src, + sizeof(fs->h_u.usr_ip4_spec.ip4src)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 16, + &fs->h_u.usr_ip4_spec.ip4dst, + &fs->m_u.usr_ip4_spec.ip4dst, + sizeof(fs->h_u.usr_ip4_spec.ip4dst)); + if (!fs->m_u.usr_ip4_spec.l4_4_bytes) + break; + + /* Only supports 20 byte IPv4 header */ + val_8 = 0x45; + mask_8 = 0xFF; + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset, + &val_8, &mask_8, sizeof(val_8)); + bcmasp_netfilt_wr_m_wake(priv, nfilt, + ETH_HLEN + 20 + offset, + &fs->h_u.usr_ip4_spec.l4_4_bytes, + &fs->m_u.usr_ip4_spec.l4_4_bytes, + sizeof(fs->h_u.usr_ip4_spec.l4_4_bytes) + ); + break; + case TCP_V4_FLOW: + val_8 = IPPROTO_TCP; + mask_8 = 0xFF; + bcmasp_netfilt_tcpip4_wr(priv, nfilt, &fs->h_u.tcp_ip4_spec, + &fs->m_u.tcp_ip4_spec, offset); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 9, + &val_8, &mask_8, sizeof(val_8)); + break; + case UDP_V4_FLOW: + val_8 = IPPROTO_UDP; + mask_8 = 0xFF; + bcmasp_netfilt_tcpip4_wr(priv, nfilt, &fs->h_u.udp_ip4_spec, + &fs->m_u.udp_ip4_spec, offset); + + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 9, + &val_8, &mask_8, sizeof(val_8)); + break; + case TCP_V6_FLOW: + val_8 = IPPROTO_TCP; + mask_8 = 0xFF; + bcmasp_netfilt_tcpip6_wr(priv, nfilt, &fs->h_u.tcp_ip6_spec, + &fs->m_u.tcp_ip6_spec, offset); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 6, + &val_8, &mask_8, sizeof(val_8)); + break; + case UDP_V6_FLOW: + val_8 = IPPROTO_UDP; + mask_8 = 0xFF; + bcmasp_netfilt_tcpip6_wr(priv, nfilt, &fs->h_u.udp_ip6_spec, + &fs->m_u.udp_ip6_spec, offset); + bcmasp_netfilt_wr_m_wake(priv, nfilt, ETH_HLEN + offset + 6, + &val_8, &mask_8, sizeof(val_8)); + break; + } + + bcmasp_netfilt_hw_en_wake(priv, nfilt); + + return 0; +} + +void bcmasp_netfilt_suspend(struct bcmasp_intf *intf) +{ + struct bcmasp_priv *priv = intf->parent; + bool write = false; + int ret, i; + + /* Write all filters to HW */ + for (i = 0; i < NUM_NET_FILTERS; i++) { + /* If the filter does not match the port, skip programming. */ + if (!priv->net_filters[i].claimed || + priv->net_filters[i].port != intf->port) + continue; + + if (i > 0 && (i % 2) && + priv->net_filters[i].wake_filter && + priv->net_filters[i - 1].wake_filter) + continue; + + ret = bcmasp_netfilt_wr_to_hw(priv, &priv->net_filters[i]); + if (!ret) + write = true; + } + + /* Successfully programmed at least one wake filter + * so enable top level wake config + */ + if (write) + rx_filter_core_wl(priv, (ASP_RX_FILTER_OPUT_EN | + ASP_RX_FILTER_LNR_MD | + ASP_RX_FILTER_GEN_WK_EN | + ASP_RX_FILTER_NT_FLT_EN), + ASP_RX_FILTER_BLK_CTRL); +} + +void bcmasp_netfilt_get_all_active(struct bcmasp_intf *intf, u32 *rule_locs, + u32 *rule_cnt) +{ + struct bcmasp_priv *priv = intf->parent; + int j = 0, i; + + for (i = 0; i < NUM_NET_FILTERS; i++) { + if (!priv->net_filters[i].claimed || + priv->net_filters[i].port != intf->port) + continue; + + if (i > 0 && (i % 2) && + priv->net_filters[i].wake_filter && + priv->net_filters[i - 1].wake_filter) + continue; + + rule_locs[j++] = priv->net_filters[i].fs.location; + } + + *rule_cnt = j; +} + +int bcmasp_netfilt_get_active(struct bcmasp_intf *intf) +{ + struct bcmasp_priv *priv = intf->parent; + int cnt = 0, i; + + for (i = 0; i < NUM_NET_FILTERS; i++) { + if (!priv->net_filters[i].claimed || + priv->net_filters[i].port != intf->port) + continue; + + /* Skip over a wake filter pair */ + if (i > 0 && (i % 2) && + priv->net_filters[i].wake_filter && + priv->net_filters[i - 1].wake_filter) + continue; + + cnt++; + } + + return cnt; +} + +bool bcmasp_netfilt_check_dup(struct bcmasp_intf *intf, + struct ethtool_rx_flow_spec *fs) +{ + struct bcmasp_priv *priv = intf->parent; + struct ethtool_rx_flow_spec *cur; + size_t fs_size = 0; + int i; + + for (i = 0; i < NUM_NET_FILTERS; i++) { + if (!priv->net_filters[i].claimed || + priv->net_filters[i].port != intf->port) + continue; + + cur = &priv->net_filters[i].fs; + + if (cur->flow_type != fs->flow_type || + cur->ring_cookie != fs->ring_cookie) + continue; + + switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case ETHER_FLOW: + fs_size = sizeof(struct ethhdr); + break; + case IP_USER_FLOW: + fs_size = sizeof(struct ethtool_usrip4_spec); + break; + case TCP_V6_FLOW: + case UDP_V6_FLOW: + fs_size = sizeof(struct ethtool_tcpip6_spec); + break; + case TCP_V4_FLOW: + case UDP_V4_FLOW: + fs_size = sizeof(struct ethtool_tcpip4_spec); + break; + default: + continue; + } + + if (memcmp(&cur->h_u, &fs->h_u, fs_size) || + memcmp(&cur->m_u, &fs->m_u, fs_size)) + continue; + + if (cur->flow_type & FLOW_EXT) { + if (cur->h_ext.vlan_etype != fs->h_ext.vlan_etype || + cur->m_ext.vlan_etype != fs->m_ext.vlan_etype || + cur->h_ext.vlan_tci != fs->h_ext.vlan_tci || + cur->m_ext.vlan_tci != fs->m_ext.vlan_tci || + cur->h_ext.data[0] != fs->h_ext.data[0]) + continue; + } + if (cur->flow_type & FLOW_MAC_EXT) { + if (memcmp(&cur->h_ext.h_dest, + &fs->h_ext.h_dest, ETH_ALEN) || + memcmp(&cur->m_ext.h_dest, + &fs->m_ext.h_dest, ETH_ALEN)) + continue; + } + + return true; + } + + return false; +} + +/* If no network filter found, return open filter. + * If no more open filters return NULL + */ +struct bcmasp_net_filter *bcmasp_netfilt_get_init(struct bcmasp_intf *intf, + int loc, bool wake_filter, + bool init) +{ + struct bcmasp_net_filter *nfilter = NULL; + struct bcmasp_priv *priv = intf->parent; + int i, open_index = -1; + + /* Check whether we exceed the filter table capacity */ + if (loc != RX_CLS_LOC_ANY && loc >= NUM_NET_FILTERS) + return ERR_PTR(-EINVAL); + + /* If the filter location is busy (already claimed) and we are initializing + * the filter (insertion), return a busy error code. + */ + if (loc != RX_CLS_LOC_ANY && init && priv->net_filters[loc].claimed) + return ERR_PTR(-EBUSY); + + /* We need two filters for wake-up, so we cannot use an odd filter */ + if (wake_filter && loc != RX_CLS_LOC_ANY && (loc % 2)) + return ERR_PTR(-EINVAL); + + /* Initialize the loop index based on the desired location or from 0 */ + i = loc == RX_CLS_LOC_ANY ? 0 : loc; + + for ( ; i < NUM_NET_FILTERS; i++) { + /* Found matching network filter */ + if (!init && + priv->net_filters[i].claimed && + priv->net_filters[i].hw_index == i && + priv->net_filters[i].port == intf->port) + return &priv->net_filters[i]; + + /* If we don't need a new filter or new filter already found */ + if (!init || open_index >= 0) + continue; + + /* Wake filter conslidates two filters to cover more bytes + * Wake filter is open if... + * 1. It is an even filter + * 2. The current and next filter is not claimed + */ + if (wake_filter && !(i % 2) && !priv->net_filters[i].claimed && + !priv->net_filters[i + 1].claimed) + open_index = i; + else if (!priv->net_filters[i].claimed) + open_index = i; + } + + if (open_index >= 0) { + nfilter = &priv->net_filters[open_index]; + nfilter->claimed = true; + nfilter->port = intf->port; + nfilter->hw_index = open_index; + } + + if (wake_filter && open_index >= 0) { + /* Claim next filter */ + priv->net_filters[open_index + 1].claimed = true; + priv->net_filters[open_index + 1].wake_filter = true; + nfilter->wake_filter = true; + } + + return nfilter ? nfilter : ERR_PTR(-EINVAL); +} + +void bcmasp_netfilt_release(struct bcmasp_intf *intf, + struct bcmasp_net_filter *nfilt) +{ + struct bcmasp_priv *priv = intf->parent; + + if (nfilt->wake_filter) { + memset(&priv->net_filters[nfilt->hw_index + 1], 0, + sizeof(struct bcmasp_net_filter)); + } + + memset(nfilt, 0, sizeof(struct bcmasp_net_filter)); +} + +static void bcmasp_addr_to_uint(unsigned char *addr, u32 *high, u32 *low) +{ + *high = (u32)(addr[0] << 8 | addr[1]); + *low = (u32)(addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | + addr[5]); +} + +static void bcmasp_set_mda_filter(struct bcmasp_intf *intf, + const unsigned char *addr, + unsigned char *mask, + unsigned int i) +{ + struct bcmasp_priv *priv = intf->parent; + u32 addr_h, addr_l, mask_h, mask_l; + + /* Set local copy */ + ether_addr_copy(priv->mda_filters[i].mask, mask); + ether_addr_copy(priv->mda_filters[i].addr, addr); + + /* Write to HW */ + bcmasp_addr_to_uint(priv->mda_filters[i].mask, &mask_h, &mask_l); + bcmasp_addr_to_uint(priv->mda_filters[i].addr, &addr_h, &addr_l); + rx_filter_core_wl(priv, addr_h, ASP_RX_FILTER_MDA_PAT_H(i)); + rx_filter_core_wl(priv, addr_l, ASP_RX_FILTER_MDA_PAT_L(i)); + rx_filter_core_wl(priv, mask_h, ASP_RX_FILTER_MDA_MSK_H(i)); + rx_filter_core_wl(priv, mask_l, ASP_RX_FILTER_MDA_MSK_L(i)); +} + +static void bcmasp_en_mda_filter(struct bcmasp_intf *intf, bool en, + unsigned int i) +{ + struct bcmasp_priv *priv = intf->parent; + + if (priv->mda_filters[i].en == en) + return; + + priv->mda_filters[i].en = en; + priv->mda_filters[i].port = intf->port; + + rx_filter_core_wl(priv, ((intf->channel + 8) | + (en << ASP_RX_FILTER_MDA_CFG_EN_SHIFT) | + ASP_RX_FILTER_MDA_CFG_UMC_SEL(intf->port)), + ASP_RX_FILTER_MDA_CFG(i)); +} + +/* There are 32 MDA filters shared between all ports, we reserve 4 filters per + * port for the following. + * - Promisc: Filter to allow all packets when promisc is enabled + * - All Multicast + * - Broadcast + * - Own address + * + * The reserved filters are identified as so. + * - Promisc: (index * 4) + 0 + * - All Multicast: (index * 4) + 1 + * - Broadcast: (index * 4) + 2 + * - Own address: (index * 4) + 3 + */ +enum asp_rx_filter_id { + ASP_RX_FILTER_MDA_PROMISC = 0, + ASP_RX_FILTER_MDA_ALLMULTI, + ASP_RX_FILTER_MDA_BROADCAST, + ASP_RX_FILTER_MDA_OWN_ADDR, + ASP_RX_FILTER_MDA_RES_MAX, +}; + +#define ASP_RX_FILT_MDA(intf, name) (((intf)->index * \ + ASP_RX_FILTER_MDA_RES_MAX) \ + + ASP_RX_FILTER_MDA_##name) + +static int bcmasp_total_res_mda_cnt(struct bcmasp_priv *priv) +{ + return list_count_nodes(&priv->intfs) * ASP_RX_FILTER_MDA_RES_MAX; +} + +void bcmasp_set_promisc(struct bcmasp_intf *intf, bool en) +{ + unsigned int i = ASP_RX_FILT_MDA(intf, PROMISC); + unsigned char promisc[ETH_ALEN]; + + eth_zero_addr(promisc); + /* Set mask to 00:00:00:00:00:00 to match all packets */ + bcmasp_set_mda_filter(intf, promisc, promisc, i); + bcmasp_en_mda_filter(intf, en, i); +} + +void bcmasp_set_allmulti(struct bcmasp_intf *intf, bool en) +{ + unsigned char allmulti[] = {0x01, 0x00, 0x00, 0x00, 0x00, 0x00}; + unsigned int i = ASP_RX_FILT_MDA(intf, ALLMULTI); + + /* Set mask to 01:00:00:00:00:00 to match all multicast */ + bcmasp_set_mda_filter(intf, allmulti, allmulti, i); + bcmasp_en_mda_filter(intf, en, i); +} + +void bcmasp_set_broad(struct bcmasp_intf *intf, bool en) +{ + unsigned int i = ASP_RX_FILT_MDA(intf, BROADCAST); + unsigned char addr[ETH_ALEN]; + + eth_broadcast_addr(addr); + bcmasp_set_mda_filter(intf, addr, addr, i); + bcmasp_en_mda_filter(intf, en, i); +} + +void bcmasp_set_oaddr(struct bcmasp_intf *intf, const unsigned char *addr, + bool en) +{ + unsigned char mask[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + unsigned int i = ASP_RX_FILT_MDA(intf, OWN_ADDR); + + bcmasp_set_mda_filter(intf, addr, mask, i); + bcmasp_en_mda_filter(intf, en, i); +} + +void bcmasp_disable_all_filters(struct bcmasp_intf *intf) +{ + struct bcmasp_priv *priv = intf->parent; + unsigned int i; + int res_count; + + res_count = bcmasp_total_res_mda_cnt(intf->parent); + + /* Disable all filters held by this port */ + for (i = res_count; i < NUM_MDA_FILTERS; i++) { + if (priv->mda_filters[i].en && + priv->mda_filters[i].port == intf->port) + bcmasp_en_mda_filter(intf, 0, i); + } +} + +static int bcmasp_combine_set_filter(struct bcmasp_intf *intf, + unsigned char *addr, unsigned char *mask, + int i) +{ + struct bcmasp_priv *priv = intf->parent; + u64 addr1, addr2, mask1, mask2, mask3; + + /* Switch to u64 to help with the calculations */ + addr1 = ether_addr_to_u64(priv->mda_filters[i].addr); + mask1 = ether_addr_to_u64(priv->mda_filters[i].mask); + addr2 = ether_addr_to_u64(addr); + mask2 = ether_addr_to_u64(mask); + + /* Check if one filter resides within the other */ + mask3 = mask1 & mask2; + if (mask3 == mask1 && ((addr1 & mask1) == (addr2 & mask1))) { + /* Filter 2 resides within filter 1, so everything is good */ + return 0; + } else if (mask3 == mask2 && ((addr1 & mask2) == (addr2 & mask2))) { + /* Filter 1 resides within filter 2, so swap filters */ + bcmasp_set_mda_filter(intf, addr, mask, i); + return 0; + } + + /* Unable to combine */ + return -EINVAL; +} + +int bcmasp_set_en_mda_filter(struct bcmasp_intf *intf, unsigned char *addr, + unsigned char *mask) +{ + struct bcmasp_priv *priv = intf->parent; + int ret, res_count; + unsigned int i; + + res_count = bcmasp_total_res_mda_cnt(intf->parent); + + for (i = res_count; i < NUM_MDA_FILTERS; i++) { + /* If filter not enabled or belongs to another port skip */ + if (!priv->mda_filters[i].en || + priv->mda_filters[i].port != intf->port) + continue; + + /* Attempt to combine filters */ + ret = bcmasp_combine_set_filter(intf, addr, mask, i); + if (!ret) { + intf->mib.filters_combine_cnt++; + return 0; + } + } + + /* Create new filter if possible */ + for (i = res_count; i < NUM_MDA_FILTERS; i++) { + if (priv->mda_filters[i].en) + continue; + + bcmasp_set_mda_filter(intf, addr, mask, i); + bcmasp_en_mda_filter(intf, 1, i); + return 0; + } + + /* No room for new filter */ + return -EINVAL; +} + +static void bcmasp_core_init_filters(struct bcmasp_priv *priv) +{ + unsigned int i; + + /* Disable all filters and reset software view since the HW + * can lose context while in deep sleep suspend states + */ + for (i = 0; i < NUM_MDA_FILTERS; i++) { + rx_filter_core_wl(priv, 0x0, ASP_RX_FILTER_MDA_CFG(i)); + priv->mda_filters[i].en = 0; + } + + for (i = 0; i < NUM_NET_FILTERS; i++) + rx_filter_core_wl(priv, 0x0, ASP_RX_FILTER_NET_CFG(i)); + + /* Top level filter enable bit should be enabled at all times, set + * GEN_WAKE_CLEAR to clear the network filter wake-up which would + * otherwise be sticky + */ + rx_filter_core_wl(priv, (ASP_RX_FILTER_OPUT_EN | + ASP_RX_FILTER_MDA_EN | + ASP_RX_FILTER_GEN_WK_CLR | + ASP_RX_FILTER_NT_FLT_EN), + ASP_RX_FILTER_BLK_CTRL); +} + +/* ASP core initialization */ +static void bcmasp_core_init(struct bcmasp_priv *priv) +{ + tx_analytics_core_wl(priv, 0x0, ASP_TX_ANALYTICS_CTRL); + rx_analytics_core_wl(priv, 0x4, ASP_RX_ANALYTICS_CTRL); + + rx_edpkt_core_wl(priv, (ASP_EDPKT_HDR_SZ_128 << ASP_EDPKT_HDR_SZ_SHIFT), + ASP_EDPKT_HDR_CFG); + rx_edpkt_core_wl(priv, + (ASP_EDPKT_ENDI_BT_SWP_WD << ASP_EDPKT_ENDI_DESC_SHIFT), + ASP_EDPKT_ENDI); + + rx_edpkt_core_wl(priv, 0x1b, ASP_EDPKT_BURST_BUF_PSCAL_TOUT); + rx_edpkt_core_wl(priv, 0x3e8, ASP_EDPKT_BURST_BUF_WRITE_TOUT); + rx_edpkt_core_wl(priv, 0x3e8, ASP_EDPKT_BURST_BUF_READ_TOUT); + + rx_edpkt_core_wl(priv, ASP_EDPKT_ENABLE_EN, ASP_EDPKT_ENABLE); + + /* Disable and clear both UniMAC's wake-up interrupts to avoid + * sticky interrupts. + */ + _intr2_mask_set(priv, ASP_INTR2_UMC0_WAKE | ASP_INTR2_UMC1_WAKE); + intr2_core_wl(priv, ASP_INTR2_UMC0_WAKE | ASP_INTR2_UMC1_WAKE, + ASP_INTR2_CLEAR); +} + +static void bcmasp_core_clock_select(struct bcmasp_priv *priv, bool slow) +{ + u32 reg; + + reg = ctrl_core_rl(priv, ASP_CTRL_CORE_CLOCK_SELECT); + if (slow) + reg &= ~ASP_CTRL_CORE_CLOCK_SELECT_MAIN; + else + reg |= ASP_CTRL_CORE_CLOCK_SELECT_MAIN; + ctrl_core_wl(priv, reg, ASP_CTRL_CORE_CLOCK_SELECT); +} + +static void bcmasp_core_clock_set_ll(struct bcmasp_priv *priv, u32 clr, u32 set) +{ + u32 reg; + + reg = ctrl_core_rl(priv, ASP_CTRL_CLOCK_CTRL); + reg &= ~clr; + reg |= set; + ctrl_core_wl(priv, reg, ASP_CTRL_CLOCK_CTRL); + + reg = ctrl_core_rl(priv, ASP_CTRL_SCRATCH_0); + reg &= ~clr; + reg |= set; + ctrl_core_wl(priv, reg, ASP_CTRL_SCRATCH_0); +} + +static void bcmasp_core_clock_set(struct bcmasp_priv *priv, u32 clr, u32 set) +{ + unsigned long flags; + + spin_lock_irqsave(&priv->clk_lock, flags); + bcmasp_core_clock_set_ll(priv, clr, set); + spin_unlock_irqrestore(&priv->clk_lock, flags); +} + +void bcmasp_core_clock_set_intf(struct bcmasp_intf *intf, bool en) +{ + u32 intf_mask = ASP_CTRL_CLOCK_CTRL_ASP_RGMII_DIS(intf->port); + struct bcmasp_priv *priv = intf->parent; + unsigned long flags; + u32 reg; + + /* When enabling an interface, if the RX or TX clocks were not enabled, + * enable them. Conversely, while disabling an interface, if this is + * the last one enabled, we can turn off the shared RX and TX clocks as + * well. We control enable bits which is why we test for equality on + * the RGMII clock bit mask. + */ + spin_lock_irqsave(&priv->clk_lock, flags); + if (en) { + intf_mask |= ASP_CTRL_CLOCK_CTRL_ASP_TX_DISABLE | + ASP_CTRL_CLOCK_CTRL_ASP_RX_DISABLE; + bcmasp_core_clock_set_ll(priv, intf_mask, 0); + } else { + reg = ctrl_core_rl(priv, ASP_CTRL_SCRATCH_0) | intf_mask; + if ((reg & ASP_CTRL_CLOCK_CTRL_ASP_RGMII_MASK) == + ASP_CTRL_CLOCK_CTRL_ASP_RGMII_MASK) + intf_mask |= ASP_CTRL_CLOCK_CTRL_ASP_TX_DISABLE | + ASP_CTRL_CLOCK_CTRL_ASP_RX_DISABLE; + bcmasp_core_clock_set_ll(priv, 0, intf_mask); + } + spin_unlock_irqrestore(&priv->clk_lock, flags); +} + +static irqreturn_t bcmasp_isr_wol(int irq, void *data) +{ + struct bcmasp_priv *priv = data; + u32 status; + + /* No L3 IRQ, so we good */ + if (priv->wol_irq <= 0) + goto irq_handled; + + status = wakeup_intr2_core_rl(priv, ASP_WAKEUP_INTR2_STATUS) & + ~wakeup_intr2_core_rl(priv, ASP_WAKEUP_INTR2_MASK_STATUS); + wakeup_intr2_core_wl(priv, status, ASP_WAKEUP_INTR2_CLEAR); + +irq_handled: + pm_wakeup_event(&priv->pdev->dev, 0); + return IRQ_HANDLED; +} + +static int bcmasp_get_and_request_irq(struct bcmasp_priv *priv, int i) +{ + struct platform_device *pdev = priv->pdev; + int irq, ret; + + irq = platform_get_irq_optional(pdev, i); + if (irq < 0) + return irq; + + ret = devm_request_irq(&pdev->dev, irq, bcmasp_isr_wol, 0, + pdev->name, priv); + if (ret) + return ret; + + return irq; +} + +static void bcmasp_init_wol_shared(struct bcmasp_priv *priv) +{ + struct platform_device *pdev = priv->pdev; + struct device *dev = &pdev->dev; + int irq; + + irq = bcmasp_get_and_request_irq(priv, 1); + if (irq < 0) { + dev_warn(dev, "Failed to init WoL irq: %d\n", irq); + return; + } + + priv->wol_irq = irq; + priv->wol_irq_enabled_mask = 0; + device_set_wakeup_capable(&pdev->dev, 1); +} + +static void bcmasp_enable_wol_shared(struct bcmasp_intf *intf, bool en) +{ + struct bcmasp_priv *priv = intf->parent; + struct device *dev = &priv->pdev->dev; + + if (en) { + if (priv->wol_irq_enabled_mask) { + set_bit(intf->port, &priv->wol_irq_enabled_mask); + return; + } + + /* First enable */ + set_bit(intf->port, &priv->wol_irq_enabled_mask); + enable_irq_wake(priv->wol_irq); + device_set_wakeup_enable(dev, 1); + } else { + if (!priv->wol_irq_enabled_mask) + return; + + clear_bit(intf->port, &priv->wol_irq_enabled_mask); + if (priv->wol_irq_enabled_mask) + return; + + /* Last disable */ + disable_irq_wake(priv->wol_irq); + device_set_wakeup_enable(dev, 0); + } +} + +static void bcmasp_wol_irq_destroy_shared(struct bcmasp_priv *priv) +{ + if (priv->wol_irq > 0) + free_irq(priv->wol_irq, priv); +} + +static void bcmasp_init_wol_per_intf(struct bcmasp_priv *priv) +{ + struct platform_device *pdev = priv->pdev; + struct device *dev = &pdev->dev; + struct bcmasp_intf *intf; + int irq; + + list_for_each_entry(intf, &priv->intfs, list) { + irq = bcmasp_get_and_request_irq(priv, intf->port + 1); + if (irq < 0) { + dev_warn(dev, "Failed to init WoL irq(port %d): %d\n", + intf->port, irq); + continue; + } + + intf->wol_irq = irq; + intf->wol_irq_enabled = false; + device_set_wakeup_capable(&pdev->dev, 1); + } +} + +static void bcmasp_enable_wol_per_intf(struct bcmasp_intf *intf, bool en) +{ + struct device *dev = &intf->parent->pdev->dev; + + if (en ^ intf->wol_irq_enabled) + irq_set_irq_wake(intf->wol_irq, en); + + intf->wol_irq_enabled = en; + device_set_wakeup_enable(dev, en); +} + +static void bcmasp_wol_irq_destroy_per_intf(struct bcmasp_priv *priv) +{ + struct bcmasp_intf *intf; + + list_for_each_entry(intf, &priv->intfs, list) { + if (intf->wol_irq > 0) + free_irq(intf->wol_irq, priv); + } +} + +static struct bcmasp_hw_info v20_hw_info = { + .rx_ctrl_flush = ASP_RX_CTRL_FLUSH, + .umac2fb = UMAC2FB_OFFSET, + .rx_ctrl_fb_out_frame_count = ASP_RX_CTRL_FB_OUT_FRAME_COUNT, + .rx_ctrl_fb_filt_out_frame_count = ASP_RX_CTRL_FB_FILT_OUT_FRAME_COUNT, + .rx_ctrl_fb_rx_fifo_depth = ASP_RX_CTRL_FB_RX_FIFO_DEPTH, +}; + +static const struct bcmasp_plat_data v20_plat_data = { + .init_wol = bcmasp_init_wol_per_intf, + .enable_wol = bcmasp_enable_wol_per_intf, + .destroy_wol = bcmasp_wol_irq_destroy_per_intf, + .hw_info = &v20_hw_info, +}; + +static struct bcmasp_hw_info v21_hw_info = { + .rx_ctrl_flush = ASP_RX_CTRL_FLUSH_2_1, + .umac2fb = UMAC2FB_OFFSET_2_1, + .rx_ctrl_fb_out_frame_count = ASP_RX_CTRL_FB_OUT_FRAME_COUNT_2_1, + .rx_ctrl_fb_filt_out_frame_count = + ASP_RX_CTRL_FB_FILT_OUT_FRAME_COUNT_2_1, + .rx_ctrl_fb_rx_fifo_depth = ASP_RX_CTRL_FB_RX_FIFO_DEPTH_2_1, +}; + +static const struct bcmasp_plat_data v21_plat_data = { + .init_wol = bcmasp_init_wol_shared, + .enable_wol = bcmasp_enable_wol_shared, + .destroy_wol = bcmasp_wol_irq_destroy_shared, + .hw_info = &v21_hw_info, +}; + +static const struct of_device_id bcmasp_of_match[] = { + { .compatible = "brcm,asp-v2.0", .data = &v20_plat_data }, + { .compatible = "brcm,asp-v2.1", .data = &v21_plat_data }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, bcmasp_of_match); + +static const struct of_device_id bcmasp_mdio_of_match[] = { + { .compatible = "brcm,asp-v2.1-mdio", }, + { .compatible = "brcm,asp-v2.0-mdio", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, bcmasp_mdio_of_match); + +static void bcmasp_remove_intfs(struct bcmasp_priv *priv) +{ + struct bcmasp_intf *intf, *n; + + list_for_each_entry_safe(intf, n, &priv->intfs, list) { + list_del(&intf->list); + bcmasp_interface_destroy(intf); + } +} + +static int bcmasp_probe(struct platform_device *pdev) +{ + struct device_node *ports_node, *intf_node; + const struct bcmasp_plat_data *pdata; + struct device *dev = &pdev->dev; + struct bcmasp_priv *priv; + struct bcmasp_intf *intf; + int ret = 0, count = 0; + unsigned int i; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->irq = platform_get_irq(pdev, 0); + if (priv->irq <= 0) + return -EINVAL; + + priv->clk = devm_clk_get_optional_enabled(dev, "sw_asp"); + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), + "failed to request clock\n"); + + /* Base from parent node */ + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return dev_err_probe(dev, PTR_ERR(priv->base), "failed to iomap\n"); + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(dev, ret, "unable to set DMA mask: %d\n", ret); + + dev_set_drvdata(&pdev->dev, priv); + priv->pdev = pdev; + spin_lock_init(&priv->mda_lock); + spin_lock_init(&priv->clk_lock); + mutex_init(&priv->wol_lock); + mutex_init(&priv->net_lock); + INIT_LIST_HEAD(&priv->intfs); + + pdata = device_get_match_data(&pdev->dev); + if (!pdata) + return dev_err_probe(dev, -EINVAL, "unable to find platform data\n"); + + priv->init_wol = pdata->init_wol; + priv->enable_wol = pdata->enable_wol; + priv->destroy_wol = pdata->destroy_wol; + priv->hw_info = pdata->hw_info; + + /* Enable all clocks to ensure successful probing */ + bcmasp_core_clock_set(priv, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE, 0); + + /* Switch to the main clock */ + bcmasp_core_clock_select(priv, false); + + bcmasp_intr2_mask_set_all(priv); + bcmasp_intr2_clear_all(priv); + + ret = devm_request_irq(&pdev->dev, priv->irq, bcmasp_isr, 0, + pdev->name, priv); + if (ret) + return dev_err_probe(dev, ret, "failed to request ASP interrupt: %d", ret); + + /* Register mdio child nodes */ + of_platform_populate(dev->of_node, bcmasp_mdio_of_match, NULL, dev); + + /* ASP specific initialization, Needs to be done regardless of + * how many interfaces come up. + */ + bcmasp_core_init(priv); + bcmasp_core_init_filters(priv); + + ports_node = of_find_node_by_name(dev->of_node, "ethernet-ports"); + if (!ports_node) { + dev_warn(dev, "No ports found\n"); + return -EINVAL; + } + + i = 0; + for_each_available_child_of_node(ports_node, intf_node) { + intf = bcmasp_interface_create(priv, intf_node, i); + if (!intf) { + dev_err(dev, "Cannot create eth interface %d\n", i); + bcmasp_remove_intfs(priv); + goto of_put_exit; + } + list_add_tail(&intf->list, &priv->intfs); + i++; + } + + /* Check and enable WoL */ + priv->init_wol(priv); + + /* Drop the clock reference count now and let ndo_open()/ndo_close() + * manage it for us from now on. + */ + bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE); + + clk_disable_unprepare(priv->clk); + + /* Now do the registration of the network ports which will take care + * of managing the clock properly. + */ + list_for_each_entry(intf, &priv->intfs, list) { + ret = register_netdev(intf->ndev); + if (ret) { + netdev_err(intf->ndev, + "failed to register net_device: %d\n", ret); + priv->destroy_wol(priv); + bcmasp_remove_intfs(priv); + goto of_put_exit; + } + count++; + } + + dev_info(dev, "Initialized %d port(s)\n", count); + +of_put_exit: + of_node_put(ports_node); + return ret; +} + +static int bcmasp_remove(struct platform_device *pdev) +{ + struct bcmasp_priv *priv = dev_get_drvdata(&pdev->dev); + + if (!priv) + return 0; + + priv->destroy_wol(priv); + bcmasp_remove_intfs(priv); + + return 0; +} + +static void bcmasp_shutdown(struct platform_device *pdev) +{ + bcmasp_remove(pdev); +} + +static int __maybe_unused bcmasp_suspend(struct device *d) +{ + struct bcmasp_priv *priv = dev_get_drvdata(d); + struct bcmasp_intf *intf; + int ret; + + list_for_each_entry(intf, &priv->intfs, list) { + ret = bcmasp_interface_suspend(intf); + if (ret) + break; + } + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + /* Whether Wake-on-LAN is enabled or not, we can always disable + * the shared TX clock + */ + bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_TX_DISABLE); + + bcmasp_core_clock_select(priv, true); + + clk_disable_unprepare(priv->clk); + + return ret; +} + +static int __maybe_unused bcmasp_resume(struct device *d) +{ + struct bcmasp_priv *priv = dev_get_drvdata(d); + struct bcmasp_intf *intf; + int ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + /* Switch to the main clock domain */ + bcmasp_core_clock_select(priv, false); + + /* Re-enable all clocks for re-initialization */ + bcmasp_core_clock_set(priv, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE, 0); + + bcmasp_core_init(priv); + bcmasp_core_init_filters(priv); + + /* And disable them to let the network devices take care of them */ + bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE); + + clk_disable_unprepare(priv->clk); + + list_for_each_entry(intf, &priv->intfs, list) { + ret = bcmasp_interface_resume(intf); + if (ret) + break; + } + + return ret; +} + +static SIMPLE_DEV_PM_OPS(bcmasp_pm_ops, + bcmasp_suspend, bcmasp_resume); + +static struct platform_driver bcmasp_driver = { + .probe = bcmasp_probe, + .remove = bcmasp_remove, + .shutdown = bcmasp_shutdown, + .driver = { + .name = "brcm,asp-v2", + .of_match_table = bcmasp_of_match, + .pm = &bcmasp_pm_ops, + }, +}; +module_platform_driver(bcmasp_driver); + +MODULE_DESCRIPTION("Broadcom ASP 2.0 Ethernet controller driver"); +MODULE_ALIAS("platform:brcm,asp-v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.h b/drivers/net/ethernet/broadcom/asp2/bcmasp.h new file mode 100644 index 000000000000..6bfcaa7f95a8 --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.h @@ -0,0 +1,586 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __BCMASP_H +#define __BCMASP_H + +#include <linux/netdevice.h> +#include <linux/phy.h> +#include <linux/io-64-nonatomic-hi-lo.h> +#include <uapi/linux/ethtool.h> + +#define ASP_INTR2_OFFSET 0x1000 +#define ASP_INTR2_STATUS 0x0 +#define ASP_INTR2_SET 0x4 +#define ASP_INTR2_CLEAR 0x8 +#define ASP_INTR2_MASK_STATUS 0xc +#define ASP_INTR2_MASK_SET 0x10 +#define ASP_INTR2_MASK_CLEAR 0x14 + +#define ASP_INTR2_RX_ECH(intr) BIT(intr) +#define ASP_INTR2_TX_DESC(intr) BIT((intr) + 14) +#define ASP_INTR2_UMC0_WAKE BIT(22) +#define ASP_INTR2_UMC1_WAKE BIT(28) + +#define ASP_WAKEUP_INTR2_OFFSET 0x1200 +#define ASP_WAKEUP_INTR2_STATUS 0x0 +#define ASP_WAKEUP_INTR2_SET 0x4 +#define ASP_WAKEUP_INTR2_CLEAR 0x8 +#define ASP_WAKEUP_INTR2_MASK_STATUS 0xc +#define ASP_WAKEUP_INTR2_MASK_SET 0x10 +#define ASP_WAKEUP_INTR2_MASK_CLEAR 0x14 +#define ASP_WAKEUP_INTR2_MPD_0 BIT(0) +#define ASP_WAKEUP_INTR2_MPD_1 BIT(1) +#define ASP_WAKEUP_INTR2_FILT_0 BIT(2) +#define ASP_WAKEUP_INTR2_FILT_1 BIT(3) +#define ASP_WAKEUP_INTR2_FW BIT(4) + +#define ASP_TX_ANALYTICS_OFFSET 0x4c000 +#define ASP_TX_ANALYTICS_CTRL 0x0 + +#define ASP_RX_ANALYTICS_OFFSET 0x98000 +#define ASP_RX_ANALYTICS_CTRL 0x0 + +#define ASP_RX_CTRL_OFFSET 0x9f000 +#define ASP_RX_CTRL_UMAC_0_FRAME_COUNT 0x8 +#define ASP_RX_CTRL_UMAC_1_FRAME_COUNT 0xc +#define ASP_RX_CTRL_FB_0_FRAME_COUNT 0x14 +#define ASP_RX_CTRL_FB_1_FRAME_COUNT 0x18 +#define ASP_RX_CTRL_FB_8_FRAME_COUNT 0x1c +/* asp2.1 diverges offsets here */ +/* ASP2.0 */ +#define ASP_RX_CTRL_FB_OUT_FRAME_COUNT 0x20 +#define ASP_RX_CTRL_FB_FILT_OUT_FRAME_COUNT 0x24 +#define ASP_RX_CTRL_FLUSH 0x28 +#define ASP_CTRL_UMAC0_FLUSH_MASK (BIT(0) | BIT(12)) +#define ASP_CTRL_UMAC1_FLUSH_MASK (BIT(1) | BIT(13)) +#define ASP_CTRL_SPB_FLUSH_MASK (BIT(8) | BIT(20)) +#define ASP_RX_CTRL_FB_RX_FIFO_DEPTH 0x30 +/* ASP2.1 */ +#define ASP_RX_CTRL_FB_9_FRAME_COUNT_2_1 0x20 +#define ASP_RX_CTRL_FB_10_FRAME_COUNT_2_1 0x24 +#define ASP_RX_CTRL_FB_OUT_FRAME_COUNT_2_1 0x28 +#define ASP_RX_CTRL_FB_FILT_OUT_FRAME_COUNT_2_1 0x2c +#define ASP_RX_CTRL_FLUSH_2_1 0x30 +#define ASP_RX_CTRL_FB_RX_FIFO_DEPTH_2_1 0x38 + +#define ASP_RX_FILTER_OFFSET 0x80000 +#define ASP_RX_FILTER_BLK_CTRL 0x0 +#define ASP_RX_FILTER_OPUT_EN BIT(0) +#define ASP_RX_FILTER_MDA_EN BIT(1) +#define ASP_RX_FILTER_LNR_MD BIT(2) +#define ASP_RX_FILTER_GEN_WK_EN BIT(3) +#define ASP_RX_FILTER_GEN_WK_CLR BIT(4) +#define ASP_RX_FILTER_NT_FLT_EN BIT(5) +#define ASP_RX_FILTER_MDA_CFG(sel) (((sel) * 0x14) + 0x100) +#define ASP_RX_FILTER_MDA_CFG_EN_SHIFT 8 +#define ASP_RX_FILTER_MDA_CFG_UMC_SEL(sel) ((sel) > 1 ? BIT(17) : \ + BIT((sel) + 9)) +#define ASP_RX_FILTER_MDA_PAT_H(sel) (((sel) * 0x14) + 0x104) +#define ASP_RX_FILTER_MDA_PAT_L(sel) (((sel) * 0x14) + 0x108) +#define ASP_RX_FILTER_MDA_MSK_H(sel) (((sel) * 0x14) + 0x10c) +#define ASP_RX_FILTER_MDA_MSK_L(sel) (((sel) * 0x14) + 0x110) +#define ASP_RX_FILTER_MDA_CFG(sel) (((sel) * 0x14) + 0x100) +#define ASP_RX_FILTER_MDA_PAT_H(sel) (((sel) * 0x14) + 0x104) +#define ASP_RX_FILTER_MDA_PAT_L(sel) (((sel) * 0x14) + 0x108) +#define ASP_RX_FILTER_MDA_MSK_H(sel) (((sel) * 0x14) + 0x10c) +#define ASP_RX_FILTER_MDA_MSK_L(sel) (((sel) * 0x14) + 0x110) +#define ASP_RX_FILTER_NET_CFG(sel) (((sel) * 0xa04) + 0x400) +#define ASP_RX_FILTER_NET_CFG_CH(sel) ((sel) << 0) +#define ASP_RX_FILTER_NET_CFG_EN BIT(9) +#define ASP_RX_FILTER_NET_CFG_L2_EN BIT(10) +#define ASP_RX_FILTER_NET_CFG_L3_EN BIT(11) +#define ASP_RX_FILTER_NET_CFG_L4_EN BIT(12) +#define ASP_RX_FILTER_NET_CFG_L3_FRM(sel) ((sel) << 13) +#define ASP_RX_FILTER_NET_CFG_L4_FRM(sel) ((sel) << 15) +#define ASP_RX_FILTER_NET_CFG_UMC(sel) BIT((sel) + 19) +#define ASP_RX_FILTER_NET_CFG_DMA_EN BIT(27) + +#define ASP_RX_FILTER_NET_OFFSET_MAX 32 +#define ASP_RX_FILTER_NET_PAT(sel, block, off) \ + (((sel) * 0xa04) + ((block) * 0x200) + (off) + 0x600) +#define ASP_RX_FILTER_NET_MASK(sel, block, off) \ + (((sel) * 0xa04) + ((block) * 0x200) + (off) + 0x700) + +#define ASP_RX_FILTER_NET_OFFSET(sel) (((sel) * 0xa04) + 0xe00) +#define ASP_RX_FILTER_NET_OFFSET_L2(val) ((val) << 0) +#define ASP_RX_FILTER_NET_OFFSET_L3_0(val) ((val) << 8) +#define ASP_RX_FILTER_NET_OFFSET_L3_1(val) ((val) << 16) +#define ASP_RX_FILTER_NET_OFFSET_L4(val) ((val) << 24) + +enum asp_rx_net_filter_block { + ASP_RX_FILTER_NET_L2 = 0, + ASP_RX_FILTER_NET_L3_0, + ASP_RX_FILTER_NET_L3_1, + ASP_RX_FILTER_NET_L4, + ASP_RX_FILTER_NET_BLOCK_MAX +}; + +#define ASP_EDPKT_OFFSET 0x9c000 +#define ASP_EDPKT_ENABLE 0x4 +#define ASP_EDPKT_ENABLE_EN BIT(0) +#define ASP_EDPKT_HDR_CFG 0xc +#define ASP_EDPKT_HDR_SZ_SHIFT 2 +#define ASP_EDPKT_HDR_SZ_32 0 +#define ASP_EDPKT_HDR_SZ_64 1 +#define ASP_EDPKT_HDR_SZ_96 2 +#define ASP_EDPKT_HDR_SZ_128 3 +#define ASP_EDPKT_BURST_BUF_PSCAL_TOUT 0x10 +#define ASP_EDPKT_BURST_BUF_WRITE_TOUT 0x14 +#define ASP_EDPKT_BURST_BUF_READ_TOUT 0x18 +#define ASP_EDPKT_RX_TS_COUNTER 0x38 +#define ASP_EDPKT_ENDI 0x48 +#define ASP_EDPKT_ENDI_DESC_SHIFT 8 +#define ASP_EDPKT_ENDI_NO_BT_SWP 0 +#define ASP_EDPKT_ENDI_BT_SWP_WD 1 +#define ASP_EDPKT_RX_PKT_CNT 0x138 +#define ASP_EDPKT_HDR_EXTR_CNT 0x13c +#define ASP_EDPKT_HDR_OUT_CNT 0x140 + +#define ASP_CTRL 0x101000 +#define ASP_CTRL_ASP_SW_INIT 0x04 +#define ASP_CTRL_ASP_SW_INIT_ACPUSS_CORE BIT(0) +#define ASP_CTRL_ASP_SW_INIT_ASP_TX BIT(1) +#define ASP_CTRL_ASP_SW_INIT_AS_RX BIT(2) +#define ASP_CTRL_ASP_SW_INIT_ASP_RGMII_UMAC0 BIT(3) +#define ASP_CTRL_ASP_SW_INIT_ASP_RGMII_UMAC1 BIT(4) +#define ASP_CTRL_ASP_SW_INIT_ASP_XMEMIF BIT(5) +#define ASP_CTRL_CLOCK_CTRL 0x04 +#define ASP_CTRL_CLOCK_CTRL_ASP_TX_DISABLE BIT(0) +#define ASP_CTRL_CLOCK_CTRL_ASP_RX_DISABLE BIT(1) +#define ASP_CTRL_CLOCK_CTRL_ASP_RGMII_SHIFT 2 +#define ASP_CTRL_CLOCK_CTRL_ASP_RGMII_MASK (0x7 << ASP_CTRL_CLOCK_CTRL_ASP_RGMII_SHIFT) +#define ASP_CTRL_CLOCK_CTRL_ASP_RGMII_DIS(x) BIT(ASP_CTRL_CLOCK_CTRL_ASP_RGMII_SHIFT + (x)) +#define ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE GENMASK(4, 0) +#define ASP_CTRL_CORE_CLOCK_SELECT 0x08 +#define ASP_CTRL_CORE_CLOCK_SELECT_MAIN BIT(0) +#define ASP_CTRL_SCRATCH_0 0x0c + +struct bcmasp_tx_cb { + struct sk_buff *skb; + unsigned int bytes_sent; + bool last; + + DEFINE_DMA_UNMAP_ADDR(dma_addr); + DEFINE_DMA_UNMAP_LEN(dma_len); +}; + +struct bcmasp_res { + /* Per interface resources */ + /* Port */ + void __iomem *umac; + void __iomem *umac2fb; + void __iomem *rgmii; + + /* TX slowpath/configuration */ + void __iomem *tx_spb_ctrl; + void __iomem *tx_spb_top; + void __iomem *tx_epkt_core; + void __iomem *tx_pause_ctrl; +}; + +#define DESC_ADDR(x) ((x) & GENMASK_ULL(39, 0)) +#define DESC_FLAGS(x) ((x) & GENMASK_ULL(63, 40)) + +struct bcmasp_desc { + u64 buf; + #define DESC_CHKSUM BIT_ULL(40) + #define DESC_CRC_ERR BIT_ULL(41) + #define DESC_RX_SYM_ERR BIT_ULL(42) + #define DESC_NO_OCT_ALN BIT_ULL(43) + #define DESC_PKT_TRUC BIT_ULL(44) + /* 39:0 (TX/RX) bits 0-39 of buf addr + * 40 (RX) checksum + * 41 (RX) crc_error + * 42 (RX) rx_symbol_error + * 43 (RX) non_octet_aligned + * 44 (RX) pkt_truncated + * 45 Reserved + * 56:46 (RX) mac_filter_id + * 60:57 (RX) rx_port_num (0-unicmac0, 1-unimac1) + * 61 Reserved + * 63:62 (TX) forward CRC, overwrite CRC + */ + u32 size; + u32 flags; + #define DESC_INT_EN BIT(0) + #define DESC_SOF BIT(1) + #define DESC_EOF BIT(2) + #define DESC_EPKT_CMD BIT(3) + #define DESC_SCRAM_ST BIT(8) + #define DESC_SCRAM_END BIT(9) + #define DESC_PCPP BIT(10) + #define DESC_PPPP BIT(11) + /* 0 (TX) tx_int_en + * 1 (TX/RX) SOF + * 2 (TX/RX) EOF + * 3 (TX) epkt_command + * 6:4 (TX) PA + * 7 (TX) pause at desc end + * 8 (TX) scram_start + * 9 (TX) scram_end + * 10 (TX) PCPP + * 11 (TX) PPPP + * 14:12 Reserved + * 15 (TX) pid ch Valid + * 19:16 (TX) data_pkt_type + * 32:20 (TX) pid_channel (RX) nw_filter_id + */ +}; + +struct bcmasp_intf; + +struct bcmasp_intf_stats64 { + /* Rx Stats */ + u64_stats_t rx_packets; + u64_stats_t rx_bytes; + u64_stats_t rx_errors; + u64_stats_t rx_dropped; + u64_stats_t rx_crc_errs; + u64_stats_t rx_sym_errs; + + /* Tx Stats*/ + u64_stats_t tx_packets; + u64_stats_t tx_bytes; + + struct u64_stats_sync syncp; +}; + +struct bcmasp_mib_counters { + u32 edpkt_ts; + u32 edpkt_rx_pkt_cnt; + u32 edpkt_hdr_ext_cnt; + u32 edpkt_hdr_out_cnt; + u32 umac_frm_cnt; + u32 fb_frm_cnt; + u32 fb_rx_fifo_depth; + u32 fb_out_frm_cnt; + u32 fb_filt_out_frm_cnt; + u32 alloc_rx_skb_failed; + u32 tx_dma_failed; + u32 mc_filters_full_cnt; + u32 uc_filters_full_cnt; + u32 filters_combine_cnt; + u32 promisc_filters_cnt; + u32 tx_realloc_offload_failed; + u32 tx_timeout_cnt; +}; + +struct bcmasp_intf_ops { + unsigned long (*rx_desc_read)(struct bcmasp_intf *intf); + void (*rx_buffer_write)(struct bcmasp_intf *intf, dma_addr_t addr); + void (*rx_desc_write)(struct bcmasp_intf *intf, dma_addr_t addr); + unsigned long (*tx_read)(struct bcmasp_intf *intf); + void (*tx_write)(struct bcmasp_intf *intf, dma_addr_t addr); +}; + +struct bcmasp_priv; + +struct bcmasp_intf { + struct list_head list; + struct net_device *ndev; + struct bcmasp_priv *parent; + + /* ASP Ch */ + int channel; + int port; + const struct bcmasp_intf_ops *ops; + + /* Used for splitting shared resources */ + int index; + + struct napi_struct tx_napi; + /* TX ring, starts on a new cacheline boundary */ + void __iomem *tx_spb_dma; + int tx_spb_index; + int tx_spb_clean_index; + struct bcmasp_desc *tx_spb_cpu; + dma_addr_t tx_spb_dma_addr; + dma_addr_t tx_spb_dma_valid; + dma_addr_t tx_spb_dma_read; + struct bcmasp_tx_cb *tx_cbs; + + /* RX ring, starts on a new cacheline boundary */ + void __iomem *rx_edpkt_cfg; + void __iomem *rx_edpkt_dma; + int rx_edpkt_index; + int rx_buf_order; + struct bcmasp_desc *rx_edpkt_cpu; + dma_addr_t rx_edpkt_dma_addr; + dma_addr_t rx_edpkt_dma_read; + + /* RX buffer prefetcher ring*/ + void *rx_ring_cpu; + dma_addr_t rx_ring_dma; + dma_addr_t rx_ring_dma_valid; + struct napi_struct rx_napi; + + struct bcmasp_res res; + unsigned int crc_fwd; + + /* PHY device */ + struct device_node *phy_dn; + struct device_node *ndev_dn; + phy_interface_t phy_interface; + bool internal_phy; + int old_pause; + int old_link; + int old_duplex; + + u32 msg_enable; + + /* Statistics */ + struct bcmasp_intf_stats64 stats64; + struct bcmasp_mib_counters mib; + + u32 wolopts; + u8 sopass[SOPASS_MAX]; + /* Used if per intf wol irq */ + int wol_irq; + unsigned int wol_irq_enabled:1; + + struct ethtool_eee eee; +}; + +#define NUM_NET_FILTERS 32 +struct bcmasp_net_filter { + struct ethtool_rx_flow_spec fs; + + bool claimed; + bool wake_filter; + + int port; + unsigned int hw_index; +}; + +#define NUM_MDA_FILTERS 32 +struct bcmasp_mda_filter { + /* Current owner of this filter */ + int port; + bool en; + u8 addr[ETH_ALEN]; + u8 mask[ETH_ALEN]; +}; + +struct bcmasp_hw_info { + u32 rx_ctrl_flush; + u32 umac2fb; + u32 rx_ctrl_fb_out_frame_count; + u32 rx_ctrl_fb_filt_out_frame_count; + u32 rx_ctrl_fb_rx_fifo_depth; +}; + +struct bcmasp_plat_data { + void (*init_wol)(struct bcmasp_priv *priv); + void (*enable_wol)(struct bcmasp_intf *intf, bool en); + void (*destroy_wol)(struct bcmasp_priv *priv); + struct bcmasp_hw_info *hw_info; +}; + +struct bcmasp_priv { + struct platform_device *pdev; + struct clk *clk; + + int irq; + u32 irq_mask; + + /* Used if shared wol irq */ + struct mutex wol_lock; + int wol_irq; + unsigned long wol_irq_enabled_mask; + + void (*init_wol)(struct bcmasp_priv *priv); + void (*enable_wol)(struct bcmasp_intf *intf, bool en); + void (*destroy_wol)(struct bcmasp_priv *priv); + + void __iomem *base; + struct bcmasp_hw_info *hw_info; + + struct list_head intfs; + + struct bcmasp_mda_filter mda_filters[NUM_MDA_FILTERS]; + + /* MAC destination address filters lock */ + spinlock_t mda_lock; + + /* Protects accesses to ASP_CTRL_CLOCK_CTRL */ + spinlock_t clk_lock; + + struct bcmasp_net_filter net_filters[NUM_NET_FILTERS]; + + /* Network filter lock */ + struct mutex net_lock; +}; + +static inline unsigned long bcmasp_intf_rx_desc_read(struct bcmasp_intf *intf) +{ + return intf->ops->rx_desc_read(intf); +} + +static inline void bcmasp_intf_rx_buffer_write(struct bcmasp_intf *intf, + dma_addr_t addr) +{ + intf->ops->rx_buffer_write(intf, addr); +} + +static inline void bcmasp_intf_rx_desc_write(struct bcmasp_intf *intf, + dma_addr_t addr) +{ + intf->ops->rx_desc_write(intf, addr); +} + +static inline unsigned long bcmasp_intf_tx_read(struct bcmasp_intf *intf) +{ + return intf->ops->tx_read(intf); +} + +static inline void bcmasp_intf_tx_write(struct bcmasp_intf *intf, + dma_addr_t addr) +{ + intf->ops->tx_write(intf, addr); +} + +#define __BCMASP_IO_MACRO(name, m) \ +static inline u32 name##_rl(struct bcmasp_intf *intf, u32 off) \ +{ \ + u32 reg = readl_relaxed(intf->m + off); \ + return reg; \ +} \ +static inline void name##_wl(struct bcmasp_intf *intf, u32 val, u32 off)\ +{ \ + writel_relaxed(val, intf->m + off); \ +} + +#define BCMASP_IO_MACRO(name) __BCMASP_IO_MACRO(name, res.name) +#define BCMASP_FP_IO_MACRO(name) __BCMASP_IO_MACRO(name, name) + +BCMASP_IO_MACRO(umac); +BCMASP_IO_MACRO(umac2fb); +BCMASP_IO_MACRO(rgmii); +BCMASP_FP_IO_MACRO(tx_spb_dma); +BCMASP_IO_MACRO(tx_spb_ctrl); +BCMASP_IO_MACRO(tx_spb_top); +BCMASP_IO_MACRO(tx_epkt_core); +BCMASP_IO_MACRO(tx_pause_ctrl); +BCMASP_FP_IO_MACRO(rx_edpkt_dma); +BCMASP_FP_IO_MACRO(rx_edpkt_cfg); + +#define __BCMASP_FP_IO_MACRO_Q(name, m) \ +static inline u64 name##_rq(struct bcmasp_intf *intf, u32 off) \ +{ \ + u64 reg = readq_relaxed(intf->m + off); \ + return reg; \ +} \ +static inline void name##_wq(struct bcmasp_intf *intf, u64 val, u32 off)\ +{ \ + writeq_relaxed(val, intf->m + off); \ +} + +#define BCMASP_FP_IO_MACRO_Q(name) __BCMASP_FP_IO_MACRO_Q(name, name) + +BCMASP_FP_IO_MACRO_Q(tx_spb_dma); +BCMASP_FP_IO_MACRO_Q(rx_edpkt_dma); +BCMASP_FP_IO_MACRO_Q(rx_edpkt_cfg); + +#define PKT_OFFLOAD_NOP (0 << 28) +#define PKT_OFFLOAD_HDR_OP (1 << 28) +#define PKT_OFFLOAD_HDR_WRBACK BIT(19) +#define PKT_OFFLOAD_HDR_COUNT(x) ((x) << 16) +#define PKT_OFFLOAD_HDR_SIZE_1(x) ((x) << 4) +#define PKT_OFFLOAD_HDR_SIZE_2(x) (x) +#define PKT_OFFLOAD_HDR2_SIZE_2(x) ((x) << 24) +#define PKT_OFFLOAD_HDR2_SIZE_3(x) ((x) << 12) +#define PKT_OFFLOAD_HDR2_SIZE_4(x) (x) +#define PKT_OFFLOAD_EPKT_OP (2 << 28) +#define PKT_OFFLOAD_EPKT_WRBACK BIT(23) +#define PKT_OFFLOAD_EPKT_IP(x) ((x) << 21) +#define PKT_OFFLOAD_EPKT_TP(x) ((x) << 19) +#define PKT_OFFLOAD_EPKT_LEN(x) ((x) << 16) +#define PKT_OFFLOAD_EPKT_CSUM_L3 BIT(15) +#define PKT_OFFLOAD_EPKT_CSUM_L2 BIT(14) +#define PKT_OFFLOAD_EPKT_ID(x) ((x) << 12) +#define PKT_OFFLOAD_EPKT_SEQ(x) ((x) << 10) +#define PKT_OFFLOAD_EPKT_TS(x) ((x) << 8) +#define PKT_OFFLOAD_EPKT_BLOC(x) (x) +#define PKT_OFFLOAD_END_OP (7 << 28) + +struct bcmasp_pkt_offload { + __be32 nop; + __be32 header; + __be32 header2; + __be32 epkt; + __be32 end; +}; + +#define BCMASP_CORE_IO_MACRO(name, offset) \ +static inline u32 name##_core_rl(struct bcmasp_priv *priv, \ + u32 off) \ +{ \ + u32 reg = readl_relaxed(priv->base + (offset) + off); \ + return reg; \ +} \ +static inline void name##_core_wl(struct bcmasp_priv *priv, \ + u32 val, u32 off) \ +{ \ + writel_relaxed(val, priv->base + (offset) + off); \ +} + +BCMASP_CORE_IO_MACRO(intr2, ASP_INTR2_OFFSET); +BCMASP_CORE_IO_MACRO(wakeup_intr2, ASP_WAKEUP_INTR2_OFFSET); +BCMASP_CORE_IO_MACRO(tx_analytics, ASP_TX_ANALYTICS_OFFSET); +BCMASP_CORE_IO_MACRO(rx_analytics, ASP_RX_ANALYTICS_OFFSET); +BCMASP_CORE_IO_MACRO(rx_ctrl, ASP_RX_CTRL_OFFSET); +BCMASP_CORE_IO_MACRO(rx_filter, ASP_RX_FILTER_OFFSET); +BCMASP_CORE_IO_MACRO(rx_edpkt, ASP_EDPKT_OFFSET); +BCMASP_CORE_IO_MACRO(ctrl, ASP_CTRL); + +struct bcmasp_intf *bcmasp_interface_create(struct bcmasp_priv *priv, + struct device_node *ndev_dn, int i); + +void bcmasp_interface_destroy(struct bcmasp_intf *intf); + +void bcmasp_enable_tx_irq(struct bcmasp_intf *intf, int en); + +void bcmasp_enable_rx_irq(struct bcmasp_intf *intf, int en); + +void bcmasp_flush_rx_port(struct bcmasp_intf *intf); + +extern const struct ethtool_ops bcmasp_ethtool_ops; + +int bcmasp_interface_suspend(struct bcmasp_intf *intf); + +int bcmasp_interface_resume(struct bcmasp_intf *intf); + +void bcmasp_set_promisc(struct bcmasp_intf *intf, bool en); + +void bcmasp_set_allmulti(struct bcmasp_intf *intf, bool en); + +void bcmasp_set_broad(struct bcmasp_intf *intf, bool en); + +void bcmasp_set_oaddr(struct bcmasp_intf *intf, const unsigned char *addr, + bool en); + +int bcmasp_set_en_mda_filter(struct bcmasp_intf *intf, unsigned char *addr, + unsigned char *mask); + +void bcmasp_disable_all_filters(struct bcmasp_intf *intf); + +void bcmasp_core_clock_set_intf(struct bcmasp_intf *intf, bool en); + +struct bcmasp_net_filter *bcmasp_netfilt_get_init(struct bcmasp_intf *intf, + int loc, bool wake_filter, + bool init); + +bool bcmasp_netfilt_check_dup(struct bcmasp_intf *intf, + struct ethtool_rx_flow_spec *fs); + +void bcmasp_netfilt_release(struct bcmasp_intf *intf, + struct bcmasp_net_filter *nfilt); + +int bcmasp_netfilt_get_active(struct bcmasp_intf *intf); + +void bcmasp_netfilt_get_all_active(struct bcmasp_intf *intf, u32 *rule_locs, + u32 *rule_cnt); + +void bcmasp_netfilt_suspend(struct bcmasp_intf *intf); + +void bcmasp_eee_enable_set(struct bcmasp_intf *intf, bool enable); +#endif diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c new file mode 100644 index 000000000000..c4f1604d5ab3 --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "bcmasp_ethtool: " fmt + +#include <asm-generic/unaligned.h> +#include <linux/ethtool.h> +#include <linux/netdevice.h> +#include <linux/platform_device.h> + +#include "bcmasp.h" +#include "bcmasp_intf_defs.h" + +enum bcmasp_stat_type { + BCMASP_STAT_RX_EDPKT, + BCMASP_STAT_RX_CTRL, + BCMASP_STAT_RX_CTRL_PER_INTF, + BCMASP_STAT_SOFT, +}; + +struct bcmasp_stats { + char stat_string[ETH_GSTRING_LEN]; + enum bcmasp_stat_type type; + u32 reg_offset; +}; + +#define STAT_BCMASP_SOFT_MIB(str) { \ + .stat_string = str, \ + .type = BCMASP_STAT_SOFT, \ +} + +#define STAT_BCMASP_OFFSET(str, _type, offset) { \ + .stat_string = str, \ + .type = _type, \ + .reg_offset = offset, \ +} + +#define STAT_BCMASP_RX_EDPKT(str, offset) \ + STAT_BCMASP_OFFSET(str, BCMASP_STAT_RX_EDPKT, offset) +#define STAT_BCMASP_RX_CTRL(str, offset) \ + STAT_BCMASP_OFFSET(str, BCMASP_STAT_RX_CTRL, offset) +#define STAT_BCMASP_RX_CTRL_PER_INTF(str, offset) \ + STAT_BCMASP_OFFSET(str, BCMASP_STAT_RX_CTRL_PER_INTF, offset) + +/* Must match the order of struct bcmasp_mib_counters */ +static const struct bcmasp_stats bcmasp_gstrings_stats[] = { + /* EDPKT counters */ + STAT_BCMASP_RX_EDPKT("RX Time Stamp", ASP_EDPKT_RX_TS_COUNTER), + STAT_BCMASP_RX_EDPKT("RX PKT Count", ASP_EDPKT_RX_PKT_CNT), + STAT_BCMASP_RX_EDPKT("RX PKT Buffered", ASP_EDPKT_HDR_EXTR_CNT), + STAT_BCMASP_RX_EDPKT("RX PKT Pushed to DRAM", ASP_EDPKT_HDR_OUT_CNT), + /* ASP RX control */ + STAT_BCMASP_RX_CTRL_PER_INTF("Frames From Unimac", + ASP_RX_CTRL_UMAC_0_FRAME_COUNT), + STAT_BCMASP_RX_CTRL_PER_INTF("Frames From Port", + ASP_RX_CTRL_FB_0_FRAME_COUNT), + STAT_BCMASP_RX_CTRL_PER_INTF("RX Buffer FIFO Depth", + ASP_RX_CTRL_FB_RX_FIFO_DEPTH), + STAT_BCMASP_RX_CTRL("Frames Out(Buffer)", + ASP_RX_CTRL_FB_OUT_FRAME_COUNT), + STAT_BCMASP_RX_CTRL("Frames Out(Filters)", + ASP_RX_CTRL_FB_FILT_OUT_FRAME_COUNT), + /* Software maintained statistics */ + STAT_BCMASP_SOFT_MIB("RX SKB Alloc Failed"), + STAT_BCMASP_SOFT_MIB("TX DMA Failed"), + STAT_BCMASP_SOFT_MIB("Multicast Filters Full"), + STAT_BCMASP_SOFT_MIB("Unicast Filters Full"), + STAT_BCMASP_SOFT_MIB("MDA Filters Combined"), + STAT_BCMASP_SOFT_MIB("Promisc Filter Set"), + STAT_BCMASP_SOFT_MIB("TX Realloc For Offload Failed"), + STAT_BCMASP_SOFT_MIB("Tx Timeout Count"), +}; + +#define BCMASP_STATS_LEN ARRAY_SIZE(bcmasp_gstrings_stats) + +static u16 bcmasp_stat_fixup_offset(struct bcmasp_intf *intf, + const struct bcmasp_stats *s) +{ + struct bcmasp_priv *priv = intf->parent; + + if (!strcmp("Frames Out(Buffer)", s->stat_string)) + return priv->hw_info->rx_ctrl_fb_out_frame_count; + + if (!strcmp("Frames Out(Filters)", s->stat_string)) + return priv->hw_info->rx_ctrl_fb_filt_out_frame_count; + + if (!strcmp("RX Buffer FIFO Depth", s->stat_string)) + return priv->hw_info->rx_ctrl_fb_rx_fifo_depth; + + return s->reg_offset; +} + +static int bcmasp_get_sset_count(struct net_device *dev, int string_set) +{ + switch (string_set) { + case ETH_SS_STATS: + return BCMASP_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void bcmasp_get_strings(struct net_device *dev, u32 stringset, + u8 *data) +{ + unsigned int i; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < BCMASP_STATS_LEN; i++) { + memcpy(data + i * ETH_GSTRING_LEN, + bcmasp_gstrings_stats[i].stat_string, + ETH_GSTRING_LEN); + } + break; + default: + return; + } +} + +static void bcmasp_update_mib_counters(struct bcmasp_intf *intf) +{ + unsigned int i; + + for (i = 0; i < BCMASP_STATS_LEN; i++) { + const struct bcmasp_stats *s; + u32 offset, val; + char *p; + + s = &bcmasp_gstrings_stats[i]; + offset = bcmasp_stat_fixup_offset(intf, s); + switch (s->type) { + case BCMASP_STAT_SOFT: + continue; + case BCMASP_STAT_RX_EDPKT: + val = rx_edpkt_core_rl(intf->parent, offset); + break; + case BCMASP_STAT_RX_CTRL: + val = rx_ctrl_core_rl(intf->parent, offset); + break; + case BCMASP_STAT_RX_CTRL_PER_INTF: + offset += sizeof(u32) * intf->port; + val = rx_ctrl_core_rl(intf->parent, offset); + break; + default: + continue; + } + p = (char *)(&intf->mib) + (i * sizeof(u32)); + put_unaligned(val, (u32 *)p); + } +} + +static void bcmasp_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + unsigned int i; + char *p; + + if (netif_running(dev)) + bcmasp_update_mib_counters(intf); + + for (i = 0; i < BCMASP_STATS_LEN; i++) { + p = (char *)(&intf->mib) + (i * sizeof(u32)); + data[i] = *(u32 *)p; + } +} + +static void bcmasp_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + strscpy(info->driver, "bcmasp", sizeof(info->driver)); + strscpy(info->bus_info, dev_name(dev->dev.parent), + sizeof(info->bus_info)); +} + +static u32 bcmasp_get_msglevel(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + return intf->msg_enable; +} + +static void bcmasp_set_msglevel(struct net_device *dev, u32 level) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + intf->msg_enable = level; +} + +#define BCMASP_SUPPORTED_WAKE (WAKE_MAGIC | WAKE_MAGICSECURE | WAKE_FILTER) +static void bcmasp_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + wol->supported = BCMASP_SUPPORTED_WAKE; + wol->wolopts = intf->wolopts; + memset(wol->sopass, 0, sizeof(wol->sopass)); + + if (wol->wolopts & WAKE_MAGICSECURE) + memcpy(wol->sopass, intf->sopass, sizeof(intf->sopass)); +} + +static int bcmasp_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct bcmasp_priv *priv = intf->parent; + struct device *kdev = &priv->pdev->dev; + + if (!device_can_wakeup(kdev)) + return -EOPNOTSUPP; + + /* Interface Specific */ + intf->wolopts = wol->wolopts; + if (intf->wolopts & WAKE_MAGICSECURE) + memcpy(intf->sopass, wol->sopass, sizeof(wol->sopass)); + + mutex_lock(&priv->wol_lock); + priv->enable_wol(intf, !!intf->wolopts); + mutex_unlock(&priv->wol_lock); + + return 0; +} + +static int bcmasp_flow_insert(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct bcmasp_net_filter *nfilter; + u32 loc = cmd->fs.location; + bool wake = false; + + if (cmd->fs.ring_cookie == RX_CLS_FLOW_WAKE) + wake = true; + + /* Currently only supports WAKE filters */ + if (!wake) + return -EOPNOTSUPP; + + switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case ETHER_FLOW: + case IP_USER_FLOW: + case TCP_V4_FLOW: + case UDP_V4_FLOW: + case TCP_V6_FLOW: + case UDP_V6_FLOW: + break; + default: + return -EOPNOTSUPP; + } + + /* Check if filter already exists */ + if (bcmasp_netfilt_check_dup(intf, &cmd->fs)) + return -EINVAL; + + nfilter = bcmasp_netfilt_get_init(intf, loc, wake, true); + if (IS_ERR(nfilter)) + return PTR_ERR(nfilter); + + /* Return the location where we did insert the filter */ + cmd->fs.location = nfilter->hw_index; + memcpy(&nfilter->fs, &cmd->fs, sizeof(struct ethtool_rx_flow_spec)); + + /* Since we only support wake filters, defer register programming till + * suspend time. + */ + return 0; +} + +static int bcmasp_flow_delete(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct bcmasp_net_filter *nfilter; + + nfilter = bcmasp_netfilt_get_init(intf, cmd->fs.location, false, false); + if (IS_ERR(nfilter)) + return PTR_ERR(nfilter); + + bcmasp_netfilt_release(intf, nfilter); + + return 0; +} + +static int bcmasp_flow_get(struct bcmasp_intf *intf, struct ethtool_rxnfc *cmd) +{ + struct bcmasp_net_filter *nfilter; + + nfilter = bcmasp_netfilt_get_init(intf, cmd->fs.location, false, false); + if (IS_ERR(nfilter)) + return PTR_ERR(nfilter); + + memcpy(&cmd->fs, &nfilter->fs, sizeof(nfilter->fs)); + + cmd->data = NUM_NET_FILTERS; + + return 0; +} + +static int bcmasp_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + int ret = -EOPNOTSUPP; + + mutex_lock(&intf->parent->net_lock); + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + ret = bcmasp_flow_insert(dev, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + ret = bcmasp_flow_delete(dev, cmd); + break; + default: + break; + } + + mutex_unlock(&intf->parent->net_lock); + + return ret; +} + +static int bcmasp_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + int err = 0; + + mutex_lock(&intf->parent->net_lock); + + switch (cmd->cmd) { + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = bcmasp_netfilt_get_active(intf); + /* We support specifying rule locations */ + cmd->data |= RX_CLS_LOC_SPECIAL; + break; + case ETHTOOL_GRXCLSRULE: + err = bcmasp_flow_get(intf, cmd); + break; + case ETHTOOL_GRXCLSRLALL: + bcmasp_netfilt_get_all_active(intf, rule_locs, &cmd->rule_cnt); + cmd->data = NUM_NET_FILTERS; + break; + default: + err = -EOPNOTSUPP; + break; + } + + mutex_unlock(&intf->parent->net_lock); + + return err; +} + +void bcmasp_eee_enable_set(struct bcmasp_intf *intf, bool enable) +{ + u32 reg; + + reg = umac_rl(intf, UMC_EEE_CTRL); + if (enable) + reg |= EEE_EN; + else + reg &= ~EEE_EN; + umac_wl(intf, reg, UMC_EEE_CTRL); + + intf->eee.eee_enabled = enable; + intf->eee.eee_active = enable; +} + +static int bcmasp_get_eee(struct net_device *dev, struct ethtool_eee *e) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct ethtool_eee *p = &intf->eee; + + if (!dev->phydev) + return -ENODEV; + + e->eee_enabled = p->eee_enabled; + e->eee_active = p->eee_active; + e->tx_lpi_enabled = p->tx_lpi_enabled; + e->tx_lpi_timer = umac_rl(intf, UMC_EEE_LPI_TIMER); + + return phy_ethtool_get_eee(dev->phydev, e); +} + +static int bcmasp_set_eee(struct net_device *dev, struct ethtool_eee *e) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct ethtool_eee *p = &intf->eee; + int ret; + + if (!dev->phydev) + return -ENODEV; + + if (!p->eee_enabled) { + bcmasp_eee_enable_set(intf, false); + } else { + ret = phy_init_eee(dev->phydev, 0); + if (ret) { + netif_err(intf, hw, dev, + "EEE initialization failed: %d\n", ret); + return ret; + } + + umac_wl(intf, e->tx_lpi_timer, UMC_EEE_LPI_TIMER); + intf->eee.eee_active = ret >= 0; + intf->eee.tx_lpi_enabled = e->tx_lpi_enabled; + bcmasp_eee_enable_set(intf, true); + } + + return phy_ethtool_set_eee(dev->phydev, e); +} + +static void bcmasp_get_eth_mac_stats(struct net_device *dev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + mac_stats->FramesTransmittedOK = umac_rl(intf, UMC_GTPOK); + mac_stats->SingleCollisionFrames = umac_rl(intf, UMC_GTSCL); + mac_stats->MultipleCollisionFrames = umac_rl(intf, UMC_GTMCL); + mac_stats->FramesReceivedOK = umac_rl(intf, UMC_GRPOK); + mac_stats->FrameCheckSequenceErrors = umac_rl(intf, UMC_GRFCS); + mac_stats->AlignmentErrors = umac_rl(intf, UMC_GRALN); + mac_stats->OctetsTransmittedOK = umac_rl(intf, UMC_GTBYT); + mac_stats->FramesWithDeferredXmissions = umac_rl(intf, UMC_GTDRF); + mac_stats->LateCollisions = umac_rl(intf, UMC_GTLCL); + mac_stats->FramesAbortedDueToXSColls = umac_rl(intf, UMC_GTXCL); + mac_stats->OctetsReceivedOK = umac_rl(intf, UMC_GRBYT); + mac_stats->MulticastFramesXmittedOK = umac_rl(intf, UMC_GTMCA); + mac_stats->BroadcastFramesXmittedOK = umac_rl(intf, UMC_GTBCA); + mac_stats->FramesWithExcessiveDeferral = umac_rl(intf, UMC_GTEDF); + mac_stats->MulticastFramesReceivedOK = umac_rl(intf, UMC_GRMCA); + mac_stats->BroadcastFramesReceivedOK = umac_rl(intf, UMC_GRBCA); +} + +static const struct ethtool_rmon_hist_range bcmasp_rmon_ranges[] = { + { 0, 64}, + { 65, 127}, + { 128, 255}, + { 256, 511}, + { 512, 1023}, + { 1024, 1518}, + { 1519, 1522}, + {} +}; + +static void bcmasp_get_rmon_stats(struct net_device *dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + *ranges = bcmasp_rmon_ranges; + + rmon_stats->undersize_pkts = umac_rl(intf, UMC_RRUND); + rmon_stats->oversize_pkts = umac_rl(intf, UMC_GROVR); + rmon_stats->fragments = umac_rl(intf, UMC_RRFRG); + rmon_stats->jabbers = umac_rl(intf, UMC_GRJBR); + + rmon_stats->hist[0] = umac_rl(intf, UMC_GR64); + rmon_stats->hist[1] = umac_rl(intf, UMC_GR127); + rmon_stats->hist[2] = umac_rl(intf, UMC_GR255); + rmon_stats->hist[3] = umac_rl(intf, UMC_GR511); + rmon_stats->hist[4] = umac_rl(intf, UMC_GR1023); + rmon_stats->hist[5] = umac_rl(intf, UMC_GR1518); + rmon_stats->hist[6] = umac_rl(intf, UMC_GRMGV); + + rmon_stats->hist_tx[0] = umac_rl(intf, UMC_TR64); + rmon_stats->hist_tx[1] = umac_rl(intf, UMC_TR127); + rmon_stats->hist_tx[2] = umac_rl(intf, UMC_TR255); + rmon_stats->hist_tx[3] = umac_rl(intf, UMC_TR511); + rmon_stats->hist_tx[4] = umac_rl(intf, UMC_TR1023); + rmon_stats->hist_tx[5] = umac_rl(intf, UMC_TR1518); + rmon_stats->hist_tx[6] = umac_rl(intf, UMC_TRMGV); +} + +static void bcmasp_get_eth_ctrl_stats(struct net_device *dev, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + ctrl_stats->MACControlFramesTransmitted = umac_rl(intf, UMC_GTXCF); + ctrl_stats->MACControlFramesReceived = umac_rl(intf, UMC_GRXCF); + ctrl_stats->UnsupportedOpcodesReceived = umac_rl(intf, UMC_GRXUO); +} + +const struct ethtool_ops bcmasp_ethtool_ops = { + .get_drvinfo = bcmasp_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_link_ksettings = phy_ethtool_get_link_ksettings, + .set_link_ksettings = phy_ethtool_set_link_ksettings, + .get_msglevel = bcmasp_get_msglevel, + .set_msglevel = bcmasp_set_msglevel, + .get_wol = bcmasp_get_wol, + .set_wol = bcmasp_set_wol, + .get_rxnfc = bcmasp_get_rxnfc, + .set_rxnfc = bcmasp_set_rxnfc, + .set_eee = bcmasp_set_eee, + .get_eee = bcmasp_get_eee, + .get_eth_mac_stats = bcmasp_get_eth_mac_stats, + .get_rmon_stats = bcmasp_get_rmon_stats, + .get_eth_ctrl_stats = bcmasp_get_eth_ctrl_stats, + .get_strings = bcmasp_get_strings, + .get_ethtool_stats = bcmasp_get_ethtool_stats, + .get_sset_count = bcmasp_get_sset_count, +}; diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c new file mode 100644 index 000000000000..53e542881255 --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "bcmasp_intf: " fmt + +#include <asm/byteorder.h> +#include <linux/brcmphy.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/etherdevice.h> +#include <linux/netdevice.h> +#include <linux/of_net.h> +#include <linux/of_mdio.h> +#include <linux/phy.h> +#include <linux/phy_fixed.h> +#include <linux/ptp_classify.h> +#include <linux/platform_device.h> +#include <net/ip.h> +#include <net/ipv6.h> + +#include "bcmasp.h" +#include "bcmasp_intf_defs.h" + +static int incr_ring(int index, int ring_count) +{ + index++; + if (index == ring_count) + return 0; + + return index; +} + +/* Points to last byte of descriptor */ +static dma_addr_t incr_last_byte(dma_addr_t addr, dma_addr_t beg, + int ring_count) +{ + dma_addr_t end = beg + (ring_count * DESC_SIZE); + + addr += DESC_SIZE; + if (addr > end) + return beg + DESC_SIZE - 1; + + return addr; +} + +/* Points to first byte of descriptor */ +static dma_addr_t incr_first_byte(dma_addr_t addr, dma_addr_t beg, + int ring_count) +{ + dma_addr_t end = beg + (ring_count * DESC_SIZE); + + addr += DESC_SIZE; + if (addr >= end) + return beg; + + return addr; +} + +static void bcmasp_enable_tx(struct bcmasp_intf *intf, int en) +{ + if (en) { + tx_spb_ctrl_wl(intf, TX_SPB_CTRL_ENABLE_EN, TX_SPB_CTRL_ENABLE); + tx_epkt_core_wl(intf, (TX_EPKT_C_CFG_MISC_EN | + TX_EPKT_C_CFG_MISC_PT | + (intf->port << TX_EPKT_C_CFG_MISC_PS_SHIFT)), + TX_EPKT_C_CFG_MISC); + } else { + tx_spb_ctrl_wl(intf, 0x0, TX_SPB_CTRL_ENABLE); + tx_epkt_core_wl(intf, 0x0, TX_EPKT_C_CFG_MISC); + } +} + +static void bcmasp_enable_rx(struct bcmasp_intf *intf, int en) +{ + if (en) + rx_edpkt_cfg_wl(intf, RX_EDPKT_CFG_ENABLE_EN, + RX_EDPKT_CFG_ENABLE); + else + rx_edpkt_cfg_wl(intf, 0x0, RX_EDPKT_CFG_ENABLE); +} + +static void bcmasp_set_rx_mode(struct net_device *dev) +{ + unsigned char mask[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + struct bcmasp_intf *intf = netdev_priv(dev); + struct netdev_hw_addr *ha; + int ret; + + spin_lock_bh(&intf->parent->mda_lock); + + bcmasp_disable_all_filters(intf); + + if (dev->flags & IFF_PROMISC) + goto set_promisc; + + bcmasp_set_promisc(intf, 0); + + bcmasp_set_broad(intf, 1); + + bcmasp_set_oaddr(intf, dev->dev_addr, 1); + + if (dev->flags & IFF_ALLMULTI) { + bcmasp_set_allmulti(intf, 1); + } else { + bcmasp_set_allmulti(intf, 0); + + netdev_for_each_mc_addr(ha, dev) { + ret = bcmasp_set_en_mda_filter(intf, ha->addr, mask); + if (ret) { + intf->mib.mc_filters_full_cnt++; + goto set_promisc; + } + } + } + + netdev_for_each_uc_addr(ha, dev) { + ret = bcmasp_set_en_mda_filter(intf, ha->addr, mask); + if (ret) { + intf->mib.uc_filters_full_cnt++; + goto set_promisc; + } + } + + spin_unlock_bh(&intf->parent->mda_lock); + return; + +set_promisc: + bcmasp_set_promisc(intf, 1); + intf->mib.promisc_filters_cnt++; + + /* disable all filters used by this port */ + bcmasp_disable_all_filters(intf); + + spin_unlock_bh(&intf->parent->mda_lock); +} + +static void bcmasp_clean_txcb(struct bcmasp_intf *intf, int index) +{ + struct bcmasp_tx_cb *txcb = &intf->tx_cbs[index]; + + txcb->skb = NULL; + dma_unmap_addr_set(txcb, dma_addr, 0); + dma_unmap_len_set(txcb, dma_len, 0); + txcb->last = false; +} + +static int tx_spb_ring_full(struct bcmasp_intf *intf, int cnt) +{ + int next_index, i; + + /* Check if we have enough room for cnt descriptors */ + for (i = 0; i < cnt; i++) { + next_index = incr_ring(intf->tx_spb_index, DESC_RING_COUNT); + if (next_index == intf->tx_spb_clean_index) + return 1; + } + + return 0; +} + +static struct sk_buff *bcmasp_csum_offload(struct net_device *dev, + struct sk_buff *skb, + bool *csum_hw) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + u32 header = 0, header2 = 0, epkt = 0; + struct bcmasp_pkt_offload *offload; + unsigned int header_cnt = 0; + u8 ip_proto; + int ret; + + if (skb->ip_summed != CHECKSUM_PARTIAL) + return skb; + + ret = skb_cow_head(skb, sizeof(*offload)); + if (ret < 0) { + intf->mib.tx_realloc_offload_failed++; + goto help; + } + + switch (skb->protocol) { + case htons(ETH_P_IP): + header |= PKT_OFFLOAD_HDR_SIZE_2((ip_hdrlen(skb) >> 8) & 0xf); + header2 |= PKT_OFFLOAD_HDR2_SIZE_2(ip_hdrlen(skb) & 0xff); + epkt |= PKT_OFFLOAD_EPKT_IP(0) | PKT_OFFLOAD_EPKT_CSUM_L2; + ip_proto = ip_hdr(skb)->protocol; + header_cnt += 2; + break; + case htons(ETH_P_IPV6): + header |= PKT_OFFLOAD_HDR_SIZE_2((IP6_HLEN >> 8) & 0xf); + header2 |= PKT_OFFLOAD_HDR2_SIZE_2(IP6_HLEN & 0xff); + epkt |= PKT_OFFLOAD_EPKT_IP(1) | PKT_OFFLOAD_EPKT_CSUM_L2; + ip_proto = ipv6_hdr(skb)->nexthdr; + header_cnt += 2; + break; + default: + goto help; + } + + switch (ip_proto) { + case IPPROTO_TCP: + header2 |= PKT_OFFLOAD_HDR2_SIZE_3(tcp_hdrlen(skb)); + epkt |= PKT_OFFLOAD_EPKT_TP(0) | PKT_OFFLOAD_EPKT_CSUM_L3; + header_cnt++; + break; + case IPPROTO_UDP: + header2 |= PKT_OFFLOAD_HDR2_SIZE_3(UDP_HLEN); + epkt |= PKT_OFFLOAD_EPKT_TP(1) | PKT_OFFLOAD_EPKT_CSUM_L3; + header_cnt++; + break; + default: + goto help; + } + + offload = (struct bcmasp_pkt_offload *)skb_push(skb, sizeof(*offload)); + + header |= PKT_OFFLOAD_HDR_OP | PKT_OFFLOAD_HDR_COUNT(header_cnt) | + PKT_OFFLOAD_HDR_SIZE_1(ETH_HLEN); + epkt |= PKT_OFFLOAD_EPKT_OP; + + offload->nop = htonl(PKT_OFFLOAD_NOP); + offload->header = htonl(header); + offload->header2 = htonl(header2); + offload->epkt = htonl(epkt); + offload->end = htonl(PKT_OFFLOAD_END_OP); + *csum_hw = true; + + return skb; + +help: + skb_checksum_help(skb); + + return skb; +} + +static unsigned long bcmasp_rx_edpkt_dma_rq(struct bcmasp_intf *intf) +{ + return rx_edpkt_dma_rq(intf, RX_EDPKT_DMA_VALID); +} + +static void bcmasp_rx_edpkt_cfg_wq(struct bcmasp_intf *intf, dma_addr_t addr) +{ + rx_edpkt_cfg_wq(intf, addr, RX_EDPKT_RING_BUFFER_READ); +} + +static void bcmasp_rx_edpkt_dma_wq(struct bcmasp_intf *intf, dma_addr_t addr) +{ + rx_edpkt_dma_wq(intf, addr, RX_EDPKT_DMA_READ); +} + +static unsigned long bcmasp_tx_spb_dma_rq(struct bcmasp_intf *intf) +{ + return tx_spb_dma_rq(intf, TX_SPB_DMA_READ); +} + +static void bcmasp_tx_spb_dma_wq(struct bcmasp_intf *intf, dma_addr_t addr) +{ + tx_spb_dma_wq(intf, addr, TX_SPB_DMA_VALID); +} + +static const struct bcmasp_intf_ops bcmasp_intf_ops = { + .rx_desc_read = bcmasp_rx_edpkt_dma_rq, + .rx_buffer_write = bcmasp_rx_edpkt_cfg_wq, + .rx_desc_write = bcmasp_rx_edpkt_dma_wq, + .tx_read = bcmasp_tx_spb_dma_rq, + .tx_write = bcmasp_tx_spb_dma_wq, +}; + +static netdev_tx_t bcmasp_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + unsigned int total_bytes, size; + int spb_index, nr_frags, i, j; + struct bcmasp_tx_cb *txcb; + dma_addr_t mapping, valid; + struct bcmasp_desc *desc; + bool csum_hw = false; + struct device *kdev; + skb_frag_t *frag; + + kdev = &intf->parent->pdev->dev; + + nr_frags = skb_shinfo(skb)->nr_frags; + + if (tx_spb_ring_full(intf, nr_frags + 1)) { + netif_stop_queue(dev); + if (net_ratelimit()) + netdev_err(dev, "Tx Ring Full!\n"); + return NETDEV_TX_BUSY; + } + + /* Save skb len before adding csum offload header */ + total_bytes = skb->len; + skb = bcmasp_csum_offload(dev, skb, &csum_hw); + if (!skb) + return NETDEV_TX_OK; + + spb_index = intf->tx_spb_index; + valid = intf->tx_spb_dma_valid; + for (i = 0; i <= nr_frags; i++) { + if (!i) { + size = skb_headlen(skb); + if (!nr_frags && size < (ETH_ZLEN + ETH_FCS_LEN)) { + if (skb_put_padto(skb, ETH_ZLEN + ETH_FCS_LEN)) + return NETDEV_TX_OK; + size = skb->len; + } + mapping = dma_map_single(kdev, skb->data, size, + DMA_TO_DEVICE); + } else { + frag = &skb_shinfo(skb)->frags[i - 1]; + size = skb_frag_size(frag); + mapping = skb_frag_dma_map(kdev, frag, 0, size, + DMA_TO_DEVICE); + } + + if (dma_mapping_error(kdev, mapping)) { + intf->mib.tx_dma_failed++; + spb_index = intf->tx_spb_index; + for (j = 0; j < i; j++) { + bcmasp_clean_txcb(intf, spb_index); + spb_index = incr_ring(spb_index, + DESC_RING_COUNT); + } + /* Rewind so we do not have a hole */ + spb_index = intf->tx_spb_index; + return NETDEV_TX_OK; + } + + txcb = &intf->tx_cbs[spb_index]; + desc = &intf->tx_spb_cpu[spb_index]; + memset(desc, 0, sizeof(*desc)); + txcb->skb = skb; + txcb->bytes_sent = total_bytes; + dma_unmap_addr_set(txcb, dma_addr, mapping); + dma_unmap_len_set(txcb, dma_len, size); + if (!i) { + desc->flags |= DESC_SOF; + if (csum_hw) + desc->flags |= DESC_EPKT_CMD; + } + + if (i == nr_frags) { + desc->flags |= DESC_EOF; + txcb->last = true; + } + + desc->buf = mapping; + desc->size = size; + desc->flags |= DESC_INT_EN; + + netif_dbg(intf, tx_queued, dev, + "%s dma_buf=%pad dma_len=0x%x flags=0x%x index=0x%x\n", + __func__, &mapping, desc->size, desc->flags, + spb_index); + + spb_index = incr_ring(spb_index, DESC_RING_COUNT); + valid = incr_last_byte(valid, intf->tx_spb_dma_addr, + DESC_RING_COUNT); + } + + /* Ensure all descriptors have been written to DRAM for the + * hardware to see up-to-date contents. + */ + wmb(); + + intf->tx_spb_index = spb_index; + intf->tx_spb_dma_valid = valid; + bcmasp_intf_tx_write(intf, intf->tx_spb_dma_valid); + + if (tx_spb_ring_full(intf, MAX_SKB_FRAGS + 1)) + netif_stop_queue(dev); + + return NETDEV_TX_OK; +} + +static void bcmasp_netif_start(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + bcmasp_set_rx_mode(dev); + napi_enable(&intf->tx_napi); + napi_enable(&intf->rx_napi); + + bcmasp_enable_rx_irq(intf, 1); + bcmasp_enable_tx_irq(intf, 1); + + phy_start(dev->phydev); +} + +static void umac_reset(struct bcmasp_intf *intf) +{ + umac_wl(intf, 0x0, UMC_CMD); + umac_wl(intf, UMC_CMD_SW_RESET, UMC_CMD); + usleep_range(10, 100); + umac_wl(intf, 0x0, UMC_CMD); +} + +static void umac_set_hw_addr(struct bcmasp_intf *intf, + const unsigned char *addr) +{ + u32 mac0 = (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | + addr[3]; + u32 mac1 = (addr[4] << 8) | addr[5]; + + umac_wl(intf, mac0, UMC_MAC0); + umac_wl(intf, mac1, UMC_MAC1); +} + +static void umac_enable_set(struct bcmasp_intf *intf, u32 mask, + unsigned int enable) +{ + u32 reg; + + reg = umac_rl(intf, UMC_CMD); + if (enable) + reg |= mask; + else + reg &= ~mask; + umac_wl(intf, reg, UMC_CMD); + + /* UniMAC stops on a packet boundary, wait for a full-sized packet + * to be processed (1 msec). + */ + if (enable == 0) + usleep_range(1000, 2000); +} + +static void umac_init(struct bcmasp_intf *intf) +{ + umac_wl(intf, 0x800, UMC_FRM_LEN); + umac_wl(intf, 0xffff, UMC_PAUSE_CNTRL); + umac_wl(intf, 0x800, UMC_RX_MAX_PKT_SZ); + umac_enable_set(intf, UMC_CMD_PROMISC, 1); +} + +static int bcmasp_tx_poll(struct napi_struct *napi, int budget) +{ + struct bcmasp_intf *intf = + container_of(napi, struct bcmasp_intf, tx_napi); + struct bcmasp_intf_stats64 *stats = &intf->stats64; + struct device *kdev = &intf->parent->pdev->dev; + unsigned long read, released = 0; + struct bcmasp_tx_cb *txcb; + struct bcmasp_desc *desc; + dma_addr_t mapping; + + read = bcmasp_intf_tx_read(intf); + while (intf->tx_spb_dma_read != read) { + txcb = &intf->tx_cbs[intf->tx_spb_clean_index]; + mapping = dma_unmap_addr(txcb, dma_addr); + + dma_unmap_single(kdev, mapping, + dma_unmap_len(txcb, dma_len), + DMA_TO_DEVICE); + + if (txcb->last) { + dev_consume_skb_any(txcb->skb); + + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->tx_packets); + u64_stats_add(&stats->tx_bytes, txcb->bytes_sent); + u64_stats_update_end(&stats->syncp); + } + + desc = &intf->tx_spb_cpu[intf->tx_spb_clean_index]; + + netif_dbg(intf, tx_done, intf->ndev, + "%s dma_buf=%pad dma_len=0x%x flags=0x%x c_index=0x%x\n", + __func__, &mapping, desc->size, desc->flags, + intf->tx_spb_clean_index); + + bcmasp_clean_txcb(intf, intf->tx_spb_clean_index); + released++; + + intf->tx_spb_clean_index = incr_ring(intf->tx_spb_clean_index, + DESC_RING_COUNT); + intf->tx_spb_dma_read = incr_first_byte(intf->tx_spb_dma_read, + intf->tx_spb_dma_addr, + DESC_RING_COUNT); + } + + /* Ensure all descriptors have been written to DRAM for the hardware + * to see updated contents. + */ + wmb(); + + napi_complete(&intf->tx_napi); + + bcmasp_enable_tx_irq(intf, 1); + + if (released) + netif_wake_queue(intf->ndev); + + return 0; +} + +static int bcmasp_rx_poll(struct napi_struct *napi, int budget) +{ + struct bcmasp_intf *intf = + container_of(napi, struct bcmasp_intf, rx_napi); + struct bcmasp_intf_stats64 *stats = &intf->stats64; + struct device *kdev = &intf->parent->pdev->dev; + unsigned long processed = 0; + struct bcmasp_desc *desc; + struct sk_buff *skb; + dma_addr_t valid; + void *data; + u64 flags; + u32 len; + + valid = bcmasp_intf_rx_desc_read(intf) + 1; + if (valid == intf->rx_edpkt_dma_addr + DESC_RING_SIZE) + valid = intf->rx_edpkt_dma_addr; + + while ((processed < budget) && (valid != intf->rx_edpkt_dma_read)) { + desc = &intf->rx_edpkt_cpu[intf->rx_edpkt_index]; + + /* Ensure that descriptor has been fully written to DRAM by + * hardware before reading by the CPU + */ + rmb(); + + /* Calculate virt addr by offsetting from physical addr */ + data = intf->rx_ring_cpu + + (DESC_ADDR(desc->buf) - intf->rx_ring_dma); + + flags = DESC_FLAGS(desc->buf); + if (unlikely(flags & (DESC_CRC_ERR | DESC_RX_SYM_ERR))) { + if (net_ratelimit()) { + netif_err(intf, rx_status, intf->ndev, + "flags=0x%llx\n", flags); + } + + u64_stats_update_begin(&stats->syncp); + if (flags & DESC_CRC_ERR) + u64_stats_inc(&stats->rx_crc_errs); + if (flags & DESC_RX_SYM_ERR) + u64_stats_inc(&stats->rx_sym_errs); + u64_stats_update_end(&stats->syncp); + + goto next; + } + + dma_sync_single_for_cpu(kdev, DESC_ADDR(desc->buf), desc->size, + DMA_FROM_DEVICE); + + len = desc->size; + + skb = napi_alloc_skb(napi, len); + if (!skb) { + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->rx_dropped); + u64_stats_update_end(&stats->syncp); + intf->mib.alloc_rx_skb_failed++; + + goto next; + } + + skb_put(skb, len); + memcpy(skb->data, data, len); + + skb_pull(skb, 2); + len -= 2; + if (likely(intf->crc_fwd)) { + skb_trim(skb, len - ETH_FCS_LEN); + len -= ETH_FCS_LEN; + } + + if ((intf->ndev->features & NETIF_F_RXCSUM) && + (desc->buf & DESC_CHKSUM)) + skb->ip_summed = CHECKSUM_UNNECESSARY; + + skb->protocol = eth_type_trans(skb, intf->ndev); + + napi_gro_receive(napi, skb); + + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->rx_packets); + u64_stats_add(&stats->rx_bytes, len); + u64_stats_update_end(&stats->syncp); + +next: + bcmasp_intf_rx_buffer_write(intf, (DESC_ADDR(desc->buf) + + desc->size)); + + processed++; + intf->rx_edpkt_dma_read = + incr_first_byte(intf->rx_edpkt_dma_read, + intf->rx_edpkt_dma_addr, + DESC_RING_COUNT); + intf->rx_edpkt_index = incr_ring(intf->rx_edpkt_index, + DESC_RING_COUNT); + } + + bcmasp_intf_rx_desc_write(intf, intf->rx_edpkt_dma_read); + + if (processed < budget) { + napi_complete_done(&intf->rx_napi, processed); + bcmasp_enable_rx_irq(intf, 1); + } + + return processed; +} + +static void bcmasp_adj_link(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct phy_device *phydev = dev->phydev; + u32 cmd_bits = 0, reg; + int changed = 0; + + if (intf->old_link != phydev->link) { + changed = 1; + intf->old_link = phydev->link; + } + + if (intf->old_duplex != phydev->duplex) { + changed = 1; + intf->old_duplex = phydev->duplex; + } + + switch (phydev->speed) { + case SPEED_2500: + cmd_bits = UMC_CMD_SPEED_2500; + break; + case SPEED_1000: + cmd_bits = UMC_CMD_SPEED_1000; + break; + case SPEED_100: + cmd_bits = UMC_CMD_SPEED_100; + break; + case SPEED_10: + cmd_bits = UMC_CMD_SPEED_10; + break; + default: + break; + } + cmd_bits <<= UMC_CMD_SPEED_SHIFT; + + if (phydev->duplex == DUPLEX_HALF) + cmd_bits |= UMC_CMD_HD_EN; + + if (intf->old_pause != phydev->pause) { + changed = 1; + intf->old_pause = phydev->pause; + } + + if (!phydev->pause) + cmd_bits |= UMC_CMD_RX_PAUSE_IGNORE | UMC_CMD_TX_PAUSE_IGNORE; + + if (!changed) + return; + + if (phydev->link) { + reg = umac_rl(intf, UMC_CMD); + reg &= ~((UMC_CMD_SPEED_MASK << UMC_CMD_SPEED_SHIFT) | + UMC_CMD_HD_EN | UMC_CMD_RX_PAUSE_IGNORE | + UMC_CMD_TX_PAUSE_IGNORE); + reg |= cmd_bits; + umac_wl(intf, reg, UMC_CMD); + + intf->eee.eee_active = phy_init_eee(phydev, 0) >= 0; + bcmasp_eee_enable_set(intf, intf->eee.eee_active); + } + + reg = rgmii_rl(intf, RGMII_OOB_CNTRL); + if (phydev->link) + reg |= RGMII_LINK; + else + reg &= ~RGMII_LINK; + rgmii_wl(intf, reg, RGMII_OOB_CNTRL); + + if (changed) + phy_print_status(phydev); +} + +static int bcmasp_init_rx(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + struct page *buffer_pg; + dma_addr_t dma; + void *p; + u32 reg; + int ret; + + intf->rx_buf_order = get_order(RING_BUFFER_SIZE); + buffer_pg = alloc_pages(GFP_KERNEL, intf->rx_buf_order); + + dma = dma_map_page(kdev, buffer_pg, 0, RING_BUFFER_SIZE, + DMA_FROM_DEVICE); + if (dma_mapping_error(kdev, dma)) { + __free_pages(buffer_pg, intf->rx_buf_order); + return -ENOMEM; + } + intf->rx_ring_cpu = page_to_virt(buffer_pg); + intf->rx_ring_dma = dma; + intf->rx_ring_dma_valid = intf->rx_ring_dma + RING_BUFFER_SIZE - 1; + + p = dma_alloc_coherent(kdev, DESC_RING_SIZE, &intf->rx_edpkt_dma_addr, + GFP_KERNEL); + if (!p) { + ret = -ENOMEM; + goto free_rx_ring; + } + intf->rx_edpkt_cpu = p; + + netif_napi_add(intf->ndev, &intf->rx_napi, bcmasp_rx_poll); + + intf->rx_edpkt_dma_read = intf->rx_edpkt_dma_addr; + intf->rx_edpkt_index = 0; + + /* Make sure channels are disabled */ + rx_edpkt_cfg_wl(intf, 0x0, RX_EDPKT_CFG_ENABLE); + + /* Rx SPB */ + rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_READ); + rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_WRITE); + rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_BASE); + rx_edpkt_cfg_wq(intf, intf->rx_ring_dma_valid, + RX_EDPKT_RING_BUFFER_END); + rx_edpkt_cfg_wq(intf, intf->rx_ring_dma_valid, + RX_EDPKT_RING_BUFFER_VALID); + + /* EDPKT */ + rx_edpkt_cfg_wl(intf, (RX_EDPKT_CFG_CFG0_RBUF_4K << + RX_EDPKT_CFG_CFG0_DBUF_SHIFT) | + (RX_EDPKT_CFG_CFG0_64_ALN << + RX_EDPKT_CFG_CFG0_BALN_SHIFT) | + (RX_EDPKT_CFG_CFG0_EFRM_STUF), + RX_EDPKT_CFG_CFG0); + rx_edpkt_dma_wq(intf, intf->rx_edpkt_dma_addr, RX_EDPKT_DMA_WRITE); + rx_edpkt_dma_wq(intf, intf->rx_edpkt_dma_addr, RX_EDPKT_DMA_READ); + rx_edpkt_dma_wq(intf, intf->rx_edpkt_dma_addr, RX_EDPKT_DMA_BASE); + rx_edpkt_dma_wq(intf, intf->rx_edpkt_dma_addr + (DESC_RING_SIZE - 1), + RX_EDPKT_DMA_END); + rx_edpkt_dma_wq(intf, intf->rx_edpkt_dma_addr + (DESC_RING_SIZE - 1), + RX_EDPKT_DMA_VALID); + + reg = UMAC2FB_CFG_DEFAULT_EN | + ((intf->channel + 11) << UMAC2FB_CFG_CHID_SHIFT); + reg |= (0xd << UMAC2FB_CFG_OK_SEND_SHIFT); + umac2fb_wl(intf, reg, UMAC2FB_CFG); + + return 0; + +free_rx_ring: + dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, + DMA_FROM_DEVICE); + __free_pages(virt_to_page(intf->rx_ring_cpu), intf->rx_buf_order); + + return ret; +} + +static void bcmasp_reclaim_free_all_rx(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + + dma_free_coherent(kdev, DESC_RING_SIZE, intf->rx_edpkt_cpu, + intf->rx_edpkt_dma_addr); + dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, + DMA_FROM_DEVICE); + __free_pages(virt_to_page(intf->rx_ring_cpu), intf->rx_buf_order); +} + +static int bcmasp_init_tx(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + void *p; + int ret; + + p = dma_alloc_coherent(kdev, DESC_RING_SIZE, &intf->tx_spb_dma_addr, + GFP_KERNEL); + if (!p) + return -ENOMEM; + + intf->tx_spb_cpu = p; + intf->tx_spb_dma_valid = intf->tx_spb_dma_addr + DESC_RING_SIZE - 1; + intf->tx_spb_dma_read = intf->tx_spb_dma_addr; + + intf->tx_cbs = kcalloc(DESC_RING_COUNT, sizeof(struct bcmasp_tx_cb), + GFP_KERNEL); + if (!intf->tx_cbs) { + ret = -ENOMEM; + goto free_tx_spb; + } + + intf->tx_spb_index = 0; + intf->tx_spb_clean_index = 0; + + netif_napi_add_tx(intf->ndev, &intf->tx_napi, bcmasp_tx_poll); + + /* Make sure channels are disabled */ + tx_spb_ctrl_wl(intf, 0x0, TX_SPB_CTRL_ENABLE); + tx_epkt_core_wl(intf, 0x0, TX_EPKT_C_CFG_MISC); + + /* Tx SPB */ + tx_spb_ctrl_wl(intf, ((intf->channel + 8) << TX_SPB_CTRL_XF_BID_SHIFT), + TX_SPB_CTRL_XF_CTRL2); + tx_pause_ctrl_wl(intf, (1 << (intf->channel + 8)), TX_PAUSE_MAP_VECTOR); + tx_spb_top_wl(intf, 0x1e, TX_SPB_TOP_BLKOUT); + tx_spb_top_wl(intf, 0x0, TX_SPB_TOP_SPRE_BW_CTRL); + + tx_spb_dma_wq(intf, intf->tx_spb_dma_addr, TX_SPB_DMA_READ); + tx_spb_dma_wq(intf, intf->tx_spb_dma_addr, TX_SPB_DMA_BASE); + tx_spb_dma_wq(intf, intf->tx_spb_dma_valid, TX_SPB_DMA_END); + tx_spb_dma_wq(intf, intf->tx_spb_dma_valid, TX_SPB_DMA_VALID); + + return 0; + +free_tx_spb: + dma_free_coherent(kdev, DESC_RING_SIZE, intf->tx_spb_cpu, + intf->tx_spb_dma_addr); + + return ret; +} + +static void bcmasp_reclaim_free_all_tx(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + + /* Free descriptors */ + dma_free_coherent(kdev, DESC_RING_SIZE, intf->tx_spb_cpu, + intf->tx_spb_dma_addr); + + /* Free cbs */ + kfree(intf->tx_cbs); +} + +static void bcmasp_ephy_enable_set(struct bcmasp_intf *intf, bool enable) +{ + u32 mask = RGMII_EPHY_CFG_IDDQ_BIAS | RGMII_EPHY_CFG_EXT_PWRDOWN | + RGMII_EPHY_CFG_IDDQ_GLOBAL; + u32 reg; + + reg = rgmii_rl(intf, RGMII_EPHY_CNTRL); + if (enable) { + reg &= ~RGMII_EPHY_CK25_DIS; + rgmii_wl(intf, reg, RGMII_EPHY_CNTRL); + mdelay(1); + + reg &= ~mask; + reg |= RGMII_EPHY_RESET; + rgmii_wl(intf, reg, RGMII_EPHY_CNTRL); + mdelay(1); + + reg &= ~RGMII_EPHY_RESET; + } else { + reg |= mask | RGMII_EPHY_RESET; + rgmii_wl(intf, reg, RGMII_EPHY_CNTRL); + mdelay(1); + reg |= RGMII_EPHY_CK25_DIS; + } + rgmii_wl(intf, reg, RGMII_EPHY_CNTRL); + mdelay(1); + + /* Set or clear the LED control override to avoid lighting up LEDs + * while the EPHY is powered off and drawing unnecessary current. + */ + reg = rgmii_rl(intf, RGMII_SYS_LED_CNTRL); + if (enable) + reg &= ~RGMII_SYS_LED_CNTRL_LINK_OVRD; + else + reg |= RGMII_SYS_LED_CNTRL_LINK_OVRD; + rgmii_wl(intf, reg, RGMII_SYS_LED_CNTRL); +} + +static void bcmasp_rgmii_mode_en_set(struct bcmasp_intf *intf, bool enable) +{ + u32 reg; + + reg = rgmii_rl(intf, RGMII_OOB_CNTRL); + reg &= ~RGMII_OOB_DIS; + if (enable) + reg |= RGMII_MODE_EN; + else + reg &= ~RGMII_MODE_EN; + rgmii_wl(intf, reg, RGMII_OOB_CNTRL); +} + +static void bcmasp_netif_deinit(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + u32 reg, timeout = 1000; + + napi_disable(&intf->tx_napi); + + bcmasp_enable_tx(intf, 0); + + /* Flush any TX packets in the pipe */ + tx_spb_dma_wl(intf, TX_SPB_DMA_FIFO_FLUSH, TX_SPB_DMA_FIFO_CTRL); + do { + reg = tx_spb_dma_rl(intf, TX_SPB_DMA_FIFO_STATUS); + if (!(reg & TX_SPB_DMA_FIFO_FLUSH)) + break; + usleep_range(1000, 2000); + } while (timeout-- > 0); + tx_spb_dma_wl(intf, 0x0, TX_SPB_DMA_FIFO_CTRL); + + umac_enable_set(intf, UMC_CMD_TX_EN, 0); + + phy_stop(dev->phydev); + + umac_enable_set(intf, UMC_CMD_RX_EN, 0); + + bcmasp_flush_rx_port(intf); + usleep_range(1000, 2000); + bcmasp_enable_rx(intf, 0); + + napi_disable(&intf->rx_napi); + + /* Disable interrupts */ + bcmasp_enable_tx_irq(intf, 0); + bcmasp_enable_rx_irq(intf, 0); + + netif_napi_del(&intf->tx_napi); + bcmasp_reclaim_free_all_tx(intf); + + netif_napi_del(&intf->rx_napi); + bcmasp_reclaim_free_all_rx(intf); +} + +static int bcmasp_stop(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + netif_dbg(intf, ifdown, dev, "bcmasp stop\n"); + + /* Stop tx from updating HW */ + netif_tx_disable(dev); + + bcmasp_netif_deinit(dev); + + phy_disconnect(dev->phydev); + + /* Disable internal EPHY or external PHY */ + if (intf->internal_phy) + bcmasp_ephy_enable_set(intf, false); + else + bcmasp_rgmii_mode_en_set(intf, false); + + /* Disable the interface clocks */ + bcmasp_core_clock_set_intf(intf, false); + + clk_disable_unprepare(intf->parent->clk); + + return 0; +} + +static void bcmasp_configure_port(struct bcmasp_intf *intf) +{ + u32 reg, id_mode_dis = 0; + + reg = rgmii_rl(intf, RGMII_PORT_CNTRL); + reg &= ~RGMII_PORT_MODE_MASK; + + switch (intf->phy_interface) { + case PHY_INTERFACE_MODE_RGMII: + /* RGMII_NO_ID: TXC transitions at the same time as TXD + * (requires PCB or receiver-side delay) + * RGMII: Add 2ns delay on TXC (90 degree shift) + * + * ID is implicitly disabled for 100Mbps (RG)MII operation. + */ + id_mode_dis = RGMII_ID_MODE_DIS; + fallthrough; + case PHY_INTERFACE_MODE_RGMII_TXID: + reg |= RGMII_PORT_MODE_EXT_GPHY; + break; + case PHY_INTERFACE_MODE_MII: + reg |= RGMII_PORT_MODE_EXT_EPHY; + break; + default: + break; + } + + if (intf->internal_phy) + reg |= RGMII_PORT_MODE_EPHY; + + rgmii_wl(intf, reg, RGMII_PORT_CNTRL); + + reg = rgmii_rl(intf, RGMII_OOB_CNTRL); + reg &= ~RGMII_ID_MODE_DIS; + reg |= id_mode_dis; + rgmii_wl(intf, reg, RGMII_OOB_CNTRL); +} + +static int bcmasp_netif_init(struct net_device *dev, bool phy_connect) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + phy_interface_t phy_iface = intf->phy_interface; + u32 phy_flags = PHY_BRCM_AUTO_PWRDWN_ENABLE | + PHY_BRCM_DIS_TXCRXC_NOENRGY | + PHY_BRCM_IDDQ_SUSPEND; + struct phy_device *phydev = NULL; + int ret; + + /* Always enable interface clocks */ + bcmasp_core_clock_set_intf(intf, true); + + /* Enable internal PHY or external PHY before any MAC activity */ + if (intf->internal_phy) + bcmasp_ephy_enable_set(intf, true); + else + bcmasp_rgmii_mode_en_set(intf, true); + bcmasp_configure_port(intf); + + /* This is an ugly quirk but we have not been correctly + * interpreting the phy_interface values and we have done that + * across different drivers, so at least we are consistent in + * our mistakes. + * + * When the Generic PHY driver is in use either the PHY has + * been strapped or programmed correctly by the boot loader so + * we should stick to our incorrect interpretation since we + * have validated it. + * + * Now when a dedicated PHY driver is in use, we need to + * reverse the meaning of the phy_interface_mode values to + * something that the PHY driver will interpret and act on such + * that we have two mistakes canceling themselves so to speak. + * We only do this for the two modes that GENET driver + * officially supports on Broadcom STB chips: + * PHY_INTERFACE_MODE_RGMII and PHY_INTERFACE_MODE_RGMII_TXID. + * Other modes are not *officially* supported with the boot + * loader and the scripted environment generating Device Tree + * blobs for those platforms. + * + * Note that internal PHY and fixed-link configurations are not + * affected because they use different phy_interface_t values + * or the Generic PHY driver. + */ + switch (phy_iface) { + case PHY_INTERFACE_MODE_RGMII: + phy_iface = PHY_INTERFACE_MODE_RGMII_ID; + break; + case PHY_INTERFACE_MODE_RGMII_TXID: + phy_iface = PHY_INTERFACE_MODE_RGMII_RXID; + break; + default: + break; + } + + if (phy_connect) { + phydev = of_phy_connect(dev, intf->phy_dn, + bcmasp_adj_link, phy_flags, + phy_iface); + if (!phydev) { + ret = -ENODEV; + netdev_err(dev, "could not attach to PHY\n"); + goto err_phy_disable; + } + } else if (!intf->wolopts) { + ret = phy_resume(dev->phydev); + if (ret) + goto err_phy_disable; + } + + umac_reset(intf); + + umac_init(intf); + + /* Disable the UniMAC RX/TX */ + umac_enable_set(intf, (UMC_CMD_RX_EN | UMC_CMD_TX_EN), 0); + + umac_set_hw_addr(intf, dev->dev_addr); + + intf->old_duplex = -1; + intf->old_link = -1; + intf->old_pause = -1; + + ret = bcmasp_init_tx(intf); + if (ret) + goto err_phy_disconnect; + + /* Turn on asp */ + bcmasp_enable_tx(intf, 1); + + ret = bcmasp_init_rx(intf); + if (ret) + goto err_reclaim_tx; + + bcmasp_enable_rx(intf, 1); + + /* Turn on UniMAC TX/RX */ + umac_enable_set(intf, (UMC_CMD_RX_EN | UMC_CMD_TX_EN), 1); + + intf->crc_fwd = !!(umac_rl(intf, UMC_CMD) & UMC_CMD_CRC_FWD); + + bcmasp_netif_start(dev); + + netif_start_queue(dev); + + return 0; + +err_reclaim_tx: + bcmasp_reclaim_free_all_tx(intf); +err_phy_disconnect: + if (phydev) + phy_disconnect(phydev); +err_phy_disable: + if (intf->internal_phy) + bcmasp_ephy_enable_set(intf, false); + else + bcmasp_rgmii_mode_en_set(intf, false); + return ret; +} + +static int bcmasp_open(struct net_device *dev) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + int ret; + + netif_dbg(intf, ifup, dev, "bcmasp open\n"); + + ret = clk_prepare_enable(intf->parent->clk); + if (ret) + return ret; + + ret = bcmasp_netif_init(dev, true); + if (ret) + clk_disable_unprepare(intf->parent->clk); + + return ret; +} + +static void bcmasp_tx_timeout(struct net_device *dev, unsigned int txqueue) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + netif_dbg(intf, tx_err, dev, "transmit timeout!\n"); + intf->mib.tx_timeout_cnt++; +} + +static int bcmasp_get_phys_port_name(struct net_device *dev, + char *name, size_t len) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + + if (snprintf(name, len, "p%d", intf->port) >= len) + return -EINVAL; + + return 0; +} + +static void bcmasp_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *stats) +{ + struct bcmasp_intf *intf = netdev_priv(dev); + struct bcmasp_intf_stats64 *lstats; + unsigned int start; + + lstats = &intf->stats64; + + do { + start = u64_stats_fetch_begin(&lstats->syncp); + stats->rx_packets = u64_stats_read(&lstats->rx_packets); + stats->rx_bytes = u64_stats_read(&lstats->rx_bytes); + stats->rx_dropped = u64_stats_read(&lstats->rx_dropped); + stats->rx_crc_errors = u64_stats_read(&lstats->rx_crc_errs); + stats->rx_frame_errors = u64_stats_read(&lstats->rx_sym_errs); + stats->rx_errors = stats->rx_crc_errors + stats->rx_frame_errors; + + stats->tx_packets = u64_stats_read(&lstats->tx_packets); + stats->tx_bytes = u64_stats_read(&lstats->tx_bytes); + } while (u64_stats_fetch_retry(&lstats->syncp, start)); +} + +static const struct net_device_ops bcmasp_netdev_ops = { + .ndo_open = bcmasp_open, + .ndo_stop = bcmasp_stop, + .ndo_start_xmit = bcmasp_xmit, + .ndo_tx_timeout = bcmasp_tx_timeout, + .ndo_set_rx_mode = bcmasp_set_rx_mode, + .ndo_get_phys_port_name = bcmasp_get_phys_port_name, + .ndo_eth_ioctl = phy_do_ioctl_running, + .ndo_set_mac_address = eth_mac_addr, + .ndo_get_stats64 = bcmasp_get_stats64, +}; + +static void bcmasp_map_res(struct bcmasp_priv *priv, struct bcmasp_intf *intf) +{ + /* Per port */ + intf->res.umac = priv->base + UMC_OFFSET(intf); + intf->res.umac2fb = priv->base + (priv->hw_info->umac2fb + + (intf->port * 0x4)); + intf->res.rgmii = priv->base + RGMII_OFFSET(intf); + + /* Per ch */ + intf->tx_spb_dma = priv->base + TX_SPB_DMA_OFFSET(intf); + intf->res.tx_spb_ctrl = priv->base + TX_SPB_CTRL_OFFSET(intf); + intf->res.tx_spb_top = priv->base + TX_SPB_TOP_OFFSET(intf); + intf->res.tx_epkt_core = priv->base + TX_EPKT_C_OFFSET(intf); + intf->res.tx_pause_ctrl = priv->base + TX_PAUSE_CTRL_OFFSET(intf); + + intf->rx_edpkt_dma = priv->base + RX_EDPKT_DMA_OFFSET(intf); + intf->rx_edpkt_cfg = priv->base + RX_EDPKT_CFG_OFFSET(intf); +} + +#define MAX_IRQ_STR_LEN 64 +struct bcmasp_intf *bcmasp_interface_create(struct bcmasp_priv *priv, + struct device_node *ndev_dn, int i) +{ + struct device *dev = &priv->pdev->dev; + struct bcmasp_intf *intf; + struct net_device *ndev; + int ch, port, ret; + + if (of_property_read_u32(ndev_dn, "reg", &port)) { + dev_warn(dev, "%s: invalid port number\n", ndev_dn->name); + goto err; + } + + if (of_property_read_u32(ndev_dn, "brcm,channel", &ch)) { + dev_warn(dev, "%s: invalid ch number\n", ndev_dn->name); + goto err; + } + + ndev = alloc_etherdev(sizeof(struct bcmasp_intf)); + if (!ndev) { + dev_warn(dev, "%s: unable to alloc ndev\n", ndev_dn->name); + goto err; + } + intf = netdev_priv(ndev); + + intf->parent = priv; + intf->ndev = ndev; + intf->channel = ch; + intf->port = port; + intf->ndev_dn = ndev_dn; + intf->index = i; + + ret = of_get_phy_mode(ndev_dn, &intf->phy_interface); + if (ret < 0) { + dev_err(dev, "invalid PHY mode property\n"); + goto err_free_netdev; + } + + if (intf->phy_interface == PHY_INTERFACE_MODE_INTERNAL) + intf->internal_phy = true; + + intf->phy_dn = of_parse_phandle(ndev_dn, "phy-handle", 0); + if (!intf->phy_dn && of_phy_is_fixed_link(ndev_dn)) { + ret = of_phy_register_fixed_link(ndev_dn); + if (ret) { + dev_warn(dev, "%s: failed to register fixed PHY\n", + ndev_dn->name); + goto err_free_netdev; + } + intf->phy_dn = ndev_dn; + } + + /* Map resource */ + bcmasp_map_res(priv, intf); + + if ((!phy_interface_mode_is_rgmii(intf->phy_interface) && + intf->phy_interface != PHY_INTERFACE_MODE_MII && + intf->phy_interface != PHY_INTERFACE_MODE_INTERNAL) || + (intf->port != 1 && intf->internal_phy)) { + netdev_err(intf->ndev, "invalid PHY mode: %s for port %d\n", + phy_modes(intf->phy_interface), intf->port); + ret = -EINVAL; + goto err_free_netdev; + } + + ret = of_get_ethdev_address(ndev_dn, ndev); + if (ret) { + netdev_warn(ndev, "using random Ethernet MAC\n"); + eth_hw_addr_random(ndev); + } + + SET_NETDEV_DEV(ndev, dev); + intf->ops = &bcmasp_intf_ops; + ndev->netdev_ops = &bcmasp_netdev_ops; + ndev->ethtool_ops = &bcmasp_ethtool_ops; + intf->msg_enable = netif_msg_init(-1, NETIF_MSG_DRV | + NETIF_MSG_PROBE | + NETIF_MSG_LINK); + ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | + NETIF_F_RXCSUM; + ndev->hw_features |= ndev->features; + ndev->needed_headroom += sizeof(struct bcmasp_pkt_offload); + + return intf; + +err_free_netdev: + free_netdev(ndev); +err: + return NULL; +} + +void bcmasp_interface_destroy(struct bcmasp_intf *intf) +{ + if (intf->ndev->reg_state == NETREG_REGISTERED) + unregister_netdev(intf->ndev); + if (of_phy_is_fixed_link(intf->ndev_dn)) + of_phy_deregister_fixed_link(intf->ndev_dn); + free_netdev(intf->ndev); +} + +static void bcmasp_suspend_to_wol(struct bcmasp_intf *intf) +{ + struct net_device *ndev = intf->ndev; + u32 reg; + + reg = umac_rl(intf, UMC_MPD_CTRL); + if (intf->wolopts & (WAKE_MAGIC | WAKE_MAGICSECURE)) + reg |= UMC_MPD_CTRL_MPD_EN; + reg &= ~UMC_MPD_CTRL_PSW_EN; + if (intf->wolopts & WAKE_MAGICSECURE) { + /* Program the SecureOn password */ + umac_wl(intf, get_unaligned_be16(&intf->sopass[0]), + UMC_PSW_MS); + umac_wl(intf, get_unaligned_be32(&intf->sopass[2]), + UMC_PSW_LS); + reg |= UMC_MPD_CTRL_PSW_EN; + } + umac_wl(intf, reg, UMC_MPD_CTRL); + + if (intf->wolopts & WAKE_FILTER) + bcmasp_netfilt_suspend(intf); + + /* UniMAC receive needs to be turned on */ + umac_enable_set(intf, UMC_CMD_RX_EN, 1); + + if (intf->parent->wol_irq > 0) { + wakeup_intr2_core_wl(intf->parent, 0xffffffff, + ASP_WAKEUP_INTR2_MASK_CLEAR); + } + + netif_dbg(intf, wol, ndev, "entered WOL mode\n"); +} + +int bcmasp_interface_suspend(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + struct net_device *dev = intf->ndev; + int ret = 0; + + if (!netif_running(dev)) + return 0; + + netif_device_detach(dev); + + bcmasp_netif_deinit(dev); + + if (!intf->wolopts) { + ret = phy_suspend(dev->phydev); + if (ret) + goto out; + + if (intf->internal_phy) + bcmasp_ephy_enable_set(intf, false); + else + bcmasp_rgmii_mode_en_set(intf, false); + + /* If Wake-on-LAN is disabled, we can safely + * disable the network interface clocks. + */ + bcmasp_core_clock_set_intf(intf, false); + } + + if (device_may_wakeup(kdev) && intf->wolopts) + bcmasp_suspend_to_wol(intf); + + clk_disable_unprepare(intf->parent->clk); + + return ret; + +out: + bcmasp_netif_init(dev, false); + return ret; +} + +static void bcmasp_resume_from_wol(struct bcmasp_intf *intf) +{ + u32 reg; + + reg = umac_rl(intf, UMC_MPD_CTRL); + reg &= ~UMC_MPD_CTRL_MPD_EN; + umac_wl(intf, reg, UMC_MPD_CTRL); + + if (intf->parent->wol_irq > 0) { + wakeup_intr2_core_wl(intf->parent, 0xffffffff, + ASP_WAKEUP_INTR2_MASK_SET); + } +} + +int bcmasp_interface_resume(struct bcmasp_intf *intf) +{ + struct net_device *dev = intf->ndev; + int ret; + + if (!netif_running(dev)) + return 0; + + ret = clk_prepare_enable(intf->parent->clk); + if (ret) + return ret; + + ret = bcmasp_netif_init(dev, false); + if (ret) + goto out; + + bcmasp_resume_from_wol(intf); + + if (intf->eee.eee_enabled) + bcmasp_eee_enable_set(intf, true); + + netif_device_attach(dev); + + return 0; + +out: + clk_disable_unprepare(intf->parent->clk); + return ret; +} diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h new file mode 100644 index 000000000000..ad742612895f --- /dev/null +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __BCMASP_INTF_DEFS_H +#define __BCMASP_INTF_DEFS_H + +#define UMC_OFFSET(intf) \ + ((((intf)->port) * 0x800) + 0xc000) +#define UMC_CMD 0x008 +#define UMC_CMD_TX_EN BIT(0) +#define UMC_CMD_RX_EN BIT(1) +#define UMC_CMD_SPEED_SHIFT 0x2 +#define UMC_CMD_SPEED_MASK 0x3 +#define UMC_CMD_SPEED_10 0x0 +#define UMC_CMD_SPEED_100 0x1 +#define UMC_CMD_SPEED_1000 0x2 +#define UMC_CMD_SPEED_2500 0x3 +#define UMC_CMD_PROMISC BIT(4) +#define UMC_CMD_PAD_EN BIT(5) +#define UMC_CMD_CRC_FWD BIT(6) +#define UMC_CMD_PAUSE_FWD BIT(7) +#define UMC_CMD_RX_PAUSE_IGNORE BIT(8) +#define UMC_CMD_TX_ADDR_INS BIT(9) +#define UMC_CMD_HD_EN BIT(10) +#define UMC_CMD_SW_RESET BIT(13) +#define UMC_CMD_LCL_LOOP_EN BIT(15) +#define UMC_CMD_AUTO_CONFIG BIT(22) +#define UMC_CMD_CNTL_FRM_EN BIT(23) +#define UMC_CMD_NO_LEN_CHK BIT(24) +#define UMC_CMD_RMT_LOOP_EN BIT(25) +#define UMC_CMD_PRBL_EN BIT(27) +#define UMC_CMD_TX_PAUSE_IGNORE BIT(28) +#define UMC_CMD_TX_RX_EN BIT(29) +#define UMC_CMD_RUNT_FILTER_DIS BIT(30) +#define UMC_MAC0 0x0c +#define UMC_MAC1 0x10 +#define UMC_FRM_LEN 0x14 +#define UMC_EEE_CTRL 0x64 +#define EN_LPI_RX_PAUSE BIT(0) +#define EN_LPI_TX_PFC BIT(1) +#define EN_LPI_TX_PAUSE BIT(2) +#define EEE_EN BIT(3) +#define RX_FIFO_CHECK BIT(4) +#define EEE_TX_CLK_DIS BIT(5) +#define DIS_EEE_10M BIT(6) +#define LP_IDLE_PREDICTION_MODE BIT(7) +#define UMC_EEE_LPI_TIMER 0x68 +#define UMC_PAUSE_CNTRL 0x330 +#define UMC_TX_FLUSH 0x334 +#define UMC_GR64 0x400 +#define UMC_GR127 0x404 +#define UMC_GR255 0x408 +#define UMC_GR511 0x40c +#define UMC_GR1023 0x410 +#define UMC_GR1518 0x414 +#define UMC_GRMGV 0x418 +#define UMC_GR2047 0x41c +#define UMC_GR4095 0x420 +#define UMC_GR9216 0x424 +#define UMC_GRPKT 0x428 +#define UMC_GRBYT 0x42c +#define UMC_GRMCA 0x430 +#define UMC_GRBCA 0x434 +#define UMC_GRFCS 0x438 +#define UMC_GRXCF 0x43c +#define UMC_GRXPF 0x440 +#define UMC_GRXUO 0x444 +#define UMC_GRALN 0x448 +#define UMC_GRFLR 0x44c +#define UMC_GRCDE 0x450 +#define UMC_GRFCR 0x454 +#define UMC_GROVR 0x458 +#define UMC_GRJBR 0x45c +#define UMC_GRMTUE 0x460 +#define UMC_GRPOK 0x464 +#define UMC_GRUC 0x468 +#define UMC_GRPPP 0x46c +#define UMC_GRMCRC 0x470 +#define UMC_TR64 0x480 +#define UMC_TR127 0x484 +#define UMC_TR255 0x488 +#define UMC_TR511 0x48c +#define UMC_TR1023 0x490 +#define UMC_TR1518 0x494 +#define UMC_TRMGV 0x498 +#define UMC_TR2047 0x49c +#define UMC_TR4095 0x4a0 +#define UMC_TR9216 0x4a4 +#define UMC_GTPKT 0x4a8 +#define UMC_GTMCA 0x4ac +#define UMC_GTBCA 0x4b0 +#define UMC_GTXPF 0x4b4 +#define UMC_GTXCF 0x4b8 +#define UMC_GTFCS 0x4bc +#define UMC_GTOVR 0x4c0 +#define UMC_GTDRF 0x4c4 +#define UMC_GTEDF 0x4c8 +#define UMC_GTSCL 0x4cc +#define UMC_GTMCL 0x4d0 +#define UMC_GTLCL 0x4d4 +#define UMC_GTXCL 0x4d8 +#define UMC_GTFRG 0x4dc +#define UMC_GTNCL 0x4e0 +#define UMC_GTJBR 0x4e4 +#define UMC_GTBYT 0x4e8 +#define UMC_GTPOK 0x4ec +#define UMC_GTUC 0x4f0 +#define UMC_RRPKT 0x500 +#define UMC_RRUND 0x504 +#define UMC_RRFRG 0x508 +#define UMC_RRBYT 0x50c +#define UMC_MIB_CNTRL 0x580 +#define UMC_MIB_CNTRL_RX_CNT_RST BIT(0) +#define UMC_MIB_CNTRL_RUNT_CNT_RST BIT(1) +#define UMC_MIB_CNTRL_TX_CNT_RST BIT(2) +#define UMC_RX_MAX_PKT_SZ 0x608 +#define UMC_MPD_CTRL 0x620 +#define UMC_MPD_CTRL_MPD_EN BIT(0) +#define UMC_MPD_CTRL_PSW_EN BIT(27) +#define UMC_PSW_MS 0x624 +#define UMC_PSW_LS 0x628 + +#define UMAC2FB_OFFSET_2_1 0x9f044 +#define UMAC2FB_OFFSET 0x9f03c +#define UMAC2FB_CFG 0x0 +#define UMAC2FB_CFG_OPUT_EN BIT(0) +#define UMAC2FB_CFG_VLAN_EN BIT(1) +#define UMAC2FB_CFG_SNAP_EN BIT(2) +#define UMAC2FB_CFG_BCM_TG_EN BIT(3) +#define UMAC2FB_CFG_IPUT_EN BIT(4) +#define UMAC2FB_CFG_CHID_SHIFT 8 +#define UMAC2FB_CFG_OK_SEND_SHIFT 24 +#define UMAC2FB_CFG_DEFAULT_EN \ + (UMAC2FB_CFG_OPUT_EN | UMAC2FB_CFG_VLAN_EN \ + | UMAC2FB_CFG_SNAP_EN | UMAC2FB_CFG_IPUT_EN) + +#define RGMII_OFFSET(intf) \ + ((((intf)->port) * 0x100) + 0xd000) +#define RGMII_EPHY_CNTRL 0x00 +#define RGMII_EPHY_CFG_IDDQ_BIAS BIT(0) +#define RGMII_EPHY_CFG_EXT_PWRDOWN BIT(1) +#define RGMII_EPHY_CFG_FORCE_DLL_EN BIT(2) +#define RGMII_EPHY_CFG_IDDQ_GLOBAL BIT(3) +#define RGMII_EPHY_CK25_DIS BIT(4) +#define RGMII_EPHY_RESET BIT(7) +#define RGMII_OOB_CNTRL 0x0c +#define RGMII_LINK BIT(4) +#define RGMII_OOB_DIS BIT(5) +#define RGMII_MODE_EN BIT(6) +#define RGMII_ID_MODE_DIS BIT(16) + +#define RGMII_PORT_CNTRL 0x60 +#define RGMII_PORT_MODE_EPHY 0 +#define RGMII_PORT_MODE_GPHY 1 +#define RGMII_PORT_MODE_EXT_EPHY 2 +#define RGMII_PORT_MODE_EXT_GPHY 3 +#define RGMII_PORT_MODE_EXT_RVMII 4 +#define RGMII_PORT_MODE_MASK GENMASK(2, 0) + +#define RGMII_SYS_LED_CNTRL 0x74 +#define RGMII_SYS_LED_CNTRL_LINK_OVRD BIT(15) + +#define TX_SPB_DMA_OFFSET(intf) \ + ((((intf)->channel) * 0x30) + 0x48180) +#define TX_SPB_DMA_READ 0x00 +#define TX_SPB_DMA_BASE 0x08 +#define TX_SPB_DMA_END 0x10 +#define TX_SPB_DMA_VALID 0x18 +#define TX_SPB_DMA_FIFO_CTRL 0x20 +#define TX_SPB_DMA_FIFO_FLUSH BIT(0) +#define TX_SPB_DMA_FIFO_STATUS 0x24 + +#define TX_SPB_CTRL_OFFSET(intf) \ + ((((intf)->channel) * 0x68) + 0x49340) +#define TX_SPB_CTRL_ENABLE 0x0 +#define TX_SPB_CTRL_ENABLE_EN BIT(0) +#define TX_SPB_CTRL_XF_CTRL2 0x20 +#define TX_SPB_CTRL_XF_BID_SHIFT 16 + +#define TX_SPB_TOP_OFFSET(intf) \ + ((((intf)->channel) * 0x1c) + 0x4a0e0) +#define TX_SPB_TOP_BLKOUT 0x0 +#define TX_SPB_TOP_SPRE_BW_CTRL 0x4 + +#define TX_EPKT_C_OFFSET(intf) \ + ((((intf)->channel) * 0x120) + 0x40900) +#define TX_EPKT_C_CFG_MISC 0x0 +#define TX_EPKT_C_CFG_MISC_EN BIT(0) +#define TX_EPKT_C_CFG_MISC_PT BIT(1) +#define TX_EPKT_C_CFG_MISC_PS_SHIFT 14 +#define TX_EPKT_C_CFG_MISC_FD_SHIFT 20 + +#define TX_PAUSE_CTRL_OFFSET(intf) \ + ((((intf)->channel * 0xc) + 0x49a20)) +#define TX_PAUSE_MAP_VECTOR 0x8 + +#define RX_EDPKT_DMA_OFFSET(intf) \ + ((((intf)->channel) * 0x38) + 0x9ca00) +#define RX_EDPKT_DMA_WRITE 0x00 +#define RX_EDPKT_DMA_READ 0x08 +#define RX_EDPKT_DMA_BASE 0x10 +#define RX_EDPKT_DMA_END 0x18 +#define RX_EDPKT_DMA_VALID 0x20 +#define RX_EDPKT_DMA_FULLNESS 0x28 +#define RX_EDPKT_DMA_MIN_THRES 0x2c +#define RX_EDPKT_DMA_CH_XONOFF 0x30 + +#define RX_EDPKT_CFG_OFFSET(intf) \ + ((((intf)->channel) * 0x70) + 0x9c600) +#define RX_EDPKT_CFG_CFG0 0x0 +#define RX_EDPKT_CFG_CFG0_DBUF_SHIFT 9 +#define RX_EDPKT_CFG_CFG0_RBUF 0x0 +#define RX_EDPKT_CFG_CFG0_RBUF_4K 0x1 +#define RX_EDPKT_CFG_CFG0_BUF_4K 0x2 +/* EFRM STUFF, 0 = no byte stuff, 1 = two byte stuff */ +#define RX_EDPKT_CFG_CFG0_EFRM_STUF BIT(11) +#define RX_EDPKT_CFG_CFG0_BALN_SHIFT 12 +#define RX_EDPKT_CFG_CFG0_NO_ALN 0 +#define RX_EDPKT_CFG_CFG0_4_ALN 2 +#define RX_EDPKT_CFG_CFG0_64_ALN 6 +#define RX_EDPKT_RING_BUFFER_WRITE 0x38 +#define RX_EDPKT_RING_BUFFER_READ 0x40 +#define RX_EDPKT_RING_BUFFER_BASE 0x48 +#define RX_EDPKT_RING_BUFFER_END 0x50 +#define RX_EDPKT_RING_BUFFER_VALID 0x58 +#define RX_EDPKT_CFG_ENABLE 0x6c +#define RX_EDPKT_CFG_ENABLE_EN BIT(0) + +#define RX_SPB_DMA_OFFSET(intf) \ + ((((intf)->channel) * 0x30) + 0xa0000) +#define RX_SPB_DMA_READ 0x00 +#define RX_SPB_DMA_BASE 0x08 +#define RX_SPB_DMA_END 0x10 +#define RX_SPB_DMA_VALID 0x18 +#define RX_SPB_DMA_FIFO_CTRL 0x20 +#define RX_SPB_DMA_FIFO_FLUSH BIT(0) +#define RX_SPB_DMA_FIFO_STATUS 0x24 + +#define RX_SPB_CTRL_OFFSET(intf) \ + ((((intf)->channel - 6) * 0x68) + 0xa1000) +#define RX_SPB_CTRL_ENABLE 0x00 +#define RX_SPB_CTRL_ENABLE_EN BIT(0) + +#define RX_PAUSE_CTRL_OFFSET(intf) \ + ((((intf)->channel - 6) * 0x4) + 0xa1138) +#define RX_PAUSE_MAP_VECTOR 0x00 + +#define RX_SPB_TOP_CTRL_OFFSET(intf) \ + ((((intf)->channel - 6) * 0x14) + 0xa2000) +#define RX_SPB_TOP_BLKOUT 0x00 + +#define NUM_4K_BUFFERS 32 +#define RING_BUFFER_SIZE (PAGE_SIZE * NUM_4K_BUFFERS) + +#define DESC_RING_COUNT (64 * NUM_4K_BUFFERS) +#define DESC_SIZE 16 +#define DESC_RING_SIZE (DESC_RING_COUNT * DESC_SIZE) + +#endif diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c index 542c69822649..8e04552d2216 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c @@ -890,7 +890,7 @@ static void bnx2x_set_one_mac_e2(struct bnx2x *bp, (struct eth_classify_rules_ramrod_data *)(raw->rdata); int rule_cnt = rule_idx + 1, cmd = elem->cmd_data.vlan_mac.cmd; union eth_classify_rule_cmd *rule_entry = &data->rules[rule_idx]; - bool add = (cmd == BNX2X_VLAN_MAC_ADD) ? true : false; + bool add = cmd == BNX2X_VLAN_MAC_ADD; unsigned long *vlan_mac_flags = &elem->cmd_data.vlan_mac.vlan_mac_flags; u8 *mac = elem->cmd_data.vlan_mac.u.mac.mac; @@ -1075,7 +1075,7 @@ static void bnx2x_set_one_vlan_e2(struct bnx2x *bp, int rule_cnt = rule_idx + 1; union eth_classify_rule_cmd *rule_entry = &data->rules[rule_idx]; enum bnx2x_vlan_mac_cmd cmd = elem->cmd_data.vlan_mac.cmd; - bool add = (cmd == BNX2X_VLAN_MAC_ADD) ? true : false; + bool add = cmd == BNX2X_VLAN_MAC_ADD; u16 vlan = elem->cmd_data.vlan_mac.u.vlan.vlan; /* Reset the ramrod data buffer for the first rule */ @@ -1125,7 +1125,7 @@ static void bnx2x_set_one_vlan_mac_e2(struct bnx2x *bp, int rule_cnt = rule_idx + 1; union eth_classify_rule_cmd *rule_entry = &data->rules[rule_idx]; enum bnx2x_vlan_mac_cmd cmd = elem->cmd_data.vlan_mac.cmd; - bool add = (cmd == BNX2X_VLAN_MAC_ADD) ? true : false; + bool add = cmd == BNX2X_VLAN_MAC_ADD; u16 vlan = elem->cmd_data.vlan_mac.u.vlan_mac.vlan; u8 *mac = elem->cmd_data.vlan_mac.u.vlan_mac.mac; u16 inner_mac; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 1eb490c48c52..6a643aae7802 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -293,6 +293,60 @@ static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) BNXT_DB_CQ(db, idx); } +static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) +{ + if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) + return; + + if (BNXT_PF(bp)) + queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); + else + schedule_delayed_work(&bp->fw_reset_task, delay); +} + +static void __bnxt_queue_sp_work(struct bnxt *bp) +{ + if (BNXT_PF(bp)) + queue_work(bnxt_pf_wq, &bp->sp_task); + else + schedule_work(&bp->sp_task); +} + +static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event) +{ + set_bit(event, &bp->sp_event); + __bnxt_queue_sp_work(bp); +} + +static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) +{ + if (!rxr->bnapi->in_reset) { + rxr->bnapi->in_reset = true; + if (bp->flags & BNXT_FLAG_CHIP_P5) + set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); + else + set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); + __bnxt_queue_sp_work(bp); + } + rxr->rx_next_cons = 0xffff; +} + +void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + int idx) +{ + struct bnxt_napi *bnapi = txr->bnapi; + + if (bnapi->tx_fault) + return; + + netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)", + txr->txq_index, bnapi->tx_pkts, + txr->tx_cons, txr->tx_prod, idx); + WARN_ON_ONCE(1); + bnapi->tx_fault = 1; + bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); +} + const u16 bnxt_lhint_arr[] = { TX_BD_FLAGS_LHINT_512_AND_SMALLER, TX_BD_FLAGS_LHINT_512_TO_1023, @@ -653,6 +707,11 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) skb = tx_buf->skb; tx_buf->skb = NULL; + if (unlikely(!skb)) { + bnxt_sched_reset_txr(bp, txr, i); + return; + } + tx_bytes += skb->len; if (tx_buf->is_push) { @@ -686,7 +745,7 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) next_tx_int: cons = NEXT_TX(cons); - dev_kfree_skb_any(skb); + dev_consume_skb_any(skb); } bnapi->tx_pkts = 0; @@ -1244,38 +1303,6 @@ static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, return 0; } -static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) -{ - if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) - return; - - if (BNXT_PF(bp)) - queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); - else - schedule_delayed_work(&bp->fw_reset_task, delay); -} - -static void bnxt_queue_sp_work(struct bnxt *bp) -{ - if (BNXT_PF(bp)) - queue_work(bnxt_pf_wq, &bp->sp_task); - else - schedule_work(&bp->sp_task); -} - -static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) -{ - if (!rxr->bnapi->in_reset) { - rxr->bnapi->in_reset = true; - if (bp->flags & BNXT_FLAG_CHIP_P5) - set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); - else - set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } - rxr->rx_next_cons = 0xffff; -} - static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) { struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; @@ -1330,7 +1357,7 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", cons, rxr->rx_next_cons, TPA_START_ERROR_CODE(tpa_start1)); - bnxt_sched_reset(bp, rxr); + bnxt_sched_reset_rxr(bp, rxr); return; } /* Store cfa_code in tpa_info to use in tpa_end @@ -1854,7 +1881,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (rxr->rx_next_cons != 0xffff) netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", cons, rxr->rx_next_cons); - bnxt_sched_reset(bp, rxr); + bnxt_sched_reset_rxr(bp, rxr); if (rc1) return rc1; goto next_rx_no_prod_no_len; @@ -1892,7 +1919,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { netdev_warn_once(bp->dev, "RX buffer error %x\n", rx_err); - bnxt_sched_reset(bp, rxr); + bnxt_sched_reset_rxr(bp, rxr); } } goto next_rx_no_len; @@ -2339,7 +2366,7 @@ static int bnxt_async_event_process(struct bnxt *bp, goto async_event_process_exit; } rxr = bp->bnapi[grp_idx]->rx_ring; - bnxt_sched_reset(bp, rxr); + bnxt_sched_reset_rxr(bp, rxr); goto async_event_process_exit; } case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { @@ -2394,7 +2421,7 @@ static int bnxt_async_event_process(struct bnxt *bp, default: goto async_event_process_exit; } - bnxt_queue_sp_work(bp); + __bnxt_queue_sp_work(bp); async_event_process_exit: return 0; } @@ -2423,8 +2450,7 @@ static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) } set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); - set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT); break; case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: @@ -2582,7 +2608,7 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) { - if (bnapi->tx_pkts) + if (bnapi->tx_pkts && !bnapi->tx_fault) bnapi->tx_int(bp, bnapi, budget); if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { @@ -9439,6 +9465,8 @@ static void bnxt_enable_napi(struct bnxt *bp) struct bnxt_napi *bnapi = bp->bnapi[i]; struct bnxt_cp_ring_info *cpr; + bnapi->tx_fault = 0; + cpr = &bnapi->cp_ring; if (bnapi->in_reset) cpr->sw_stats.rx.rx_resets++; @@ -11048,8 +11076,7 @@ static void bnxt_set_rx_mode(struct net_device *dev) if (mask != vnic->rx_mask || uc_update || mc_update) { vnic->rx_mask = mask; - set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); } } @@ -11614,8 +11641,7 @@ static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) struct bnxt *bp = netdev_priv(dev); netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); - set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); } static void bnxt_fw_health_check(struct bnxt *bp) @@ -11652,8 +11678,7 @@ static void bnxt_fw_health_check(struct bnxt *bp) return; fw_reset: - set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT); } static void bnxt_timer(struct timer_list *t) @@ -11670,21 +11695,15 @@ static void bnxt_timer(struct timer_list *t) if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) bnxt_fw_health_check(bp); - if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) { - set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } + if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) + bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT); - if (bnxt_tc_flower_enabled(bp)) { - set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } + if (bnxt_tc_flower_enabled(bp)) + bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT); #ifdef CONFIG_RFS_ACCEL - if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { - set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } + if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) + bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); #endif /*CONFIG_RFS_ACCEL*/ if (bp->link_info.phy_retry) { @@ -11692,21 +11711,17 @@ static void bnxt_timer(struct timer_list *t) bp->link_info.phy_retry = false; netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); } else { - set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT); } } - if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) { - set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } + if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) + bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && - netif_carrier_ok(dev)) { - set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); - } + netif_carrier_ok(dev)) + bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT); + bnxt_restart_timer: mod_timer(&bp->timer, jiffies + bp->current_interval); } @@ -12985,8 +13000,7 @@ static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, bp->ntp_fltr_count++; spin_unlock_bh(&bp->ntp_fltr_lock); - set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); - bnxt_queue_sp_work(bp); + bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); return new_fltr->sw_id; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index bb95c3dc5270..362918876d3c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -1008,6 +1008,7 @@ struct bnxt_napi { int budget); int tx_pkts; u8 events; + u8 tx_fault:1; u32 flags; #define BNXT_NAPI_FLAG_XDP 0x1 @@ -2329,6 +2330,8 @@ int bnxt_get_avail_msix(struct bnxt *bp, int num); int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init); void bnxt_tx_disable(struct bnxt *bp); void bnxt_tx_enable(struct bnxt *bp); +void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + int idx); void bnxt_report_link(struct bnxt *bp); int bnxt_update_link(struct bnxt *bp, bool chng_link_state); int bnxt_hwrm_set_pause(struct bnxt *); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c index caab3d626a2a..31f85f3e2364 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c @@ -130,7 +130,7 @@ static int bnxt_hwrm_queue_cos2bw_cfg(struct bnxt *bp, struct ieee_ets *ets, BW_VALUE_UNIT_PERCENT1_100); } data = &req->unused_0 + qidx * (sizeof(cos2bw) - 4); - memcpy(data, &cos2bw.queue_id, sizeof(cos2bw) - 4); + memcpy(data, &cos2bw.cfg, sizeof(cos2bw) - 4); if (qidx == 0) { req->queue_id0 = cos2bw.queue_id; req->unused_0 = 0; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h index 716742522161..5b2a6f678244 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h @@ -27,11 +27,12 @@ struct bnxt_cos2bw_cfg { u8 queue_id; __le32 min_bw; __le32 max_bw; -#define BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) u8 tsa; u8 pri_lvl; u8 bw_weight; ); +/* for min_bw / max_bw */ +#define BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) u8 unused; }; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c index d8afcf8d6b30..38d89d80b4a9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c @@ -373,9 +373,9 @@ static int bnxt_tc_parse_flow(struct bnxt *bp, struct flow_dissector *dissector = rule->match.dissector; /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ - if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || - (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) { - netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x\n", + if ((dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || + (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) == 0) { + netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index fb43232310b2..2ce46d7affe4 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -153,6 +153,7 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) tx_buf->action = 0; tx_buf->xdpf = NULL; } else if (tx_buf->action == XDP_TX) { + tx_buf->action = 0; rx_doorbell_needed = true; last_tx_cons = tx_cons; @@ -162,6 +163,9 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) tx_buf = &txr->tx_buf_ring[tx_cons]; page_pool_recycle_direct(rxr->page_pool, tx_buf->page); } + } else { + bnxt_sched_reset_txr(bp, txr, i); + return; } tx_cons = NEXT_TX(tx_cons); } diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 2b5761ad2f92..24bade875ca6 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -2077,12 +2077,8 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) spin_lock(&ring->lock); if (ring->free_bds <= (nr_frags + 1)) { - if (!netif_tx_queue_stopped(txq)) { + if (!netif_tx_queue_stopped(txq)) netif_tx_stop_queue(txq); - netdev_err(dev, - "%s: tx ring %d full when queue %d awake\n", - __func__, index, ring->queue); - } ret = NETDEV_TX_BUSY; goto out; } diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index d6d90f9722a7..31191b520b58 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -1037,8 +1037,7 @@ bnad_cb_ccb_destroy(struct bnad *bnad, struct bna_ccb *ccb) static void bnad_cb_tx_stall(struct bnad *bnad, struct bna_tx *tx) { - struct bnad_tx_info *tx_info = - (struct bnad_tx_info *)tx->priv; + struct bnad_tx_info *tx_info = tx->priv; struct bna_tcb *tcb; u32 txq_id; int i; @@ -1056,7 +1055,7 @@ bnad_cb_tx_stall(struct bnad *bnad, struct bna_tx *tx) static void bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) { - struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv; + struct bnad_tx_info *tx_info = tx->priv; struct bna_tcb *tcb; u32 txq_id; int i; @@ -1133,7 +1132,7 @@ bnad_tx_cleanup(struct delayed_work *work) static void bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) { - struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv; + struct bnad_tx_info *tx_info = tx->priv; struct bna_tcb *tcb; int i; @@ -1149,7 +1148,7 @@ bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) static void bnad_cb_rx_stall(struct bnad *bnad, struct bna_rx *rx) { - struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; + struct bnad_rx_info *rx_info = rx->priv; struct bna_ccb *ccb; struct bnad_rx_ctrl *rx_ctrl; int i; @@ -1208,7 +1207,7 @@ bnad_rx_cleanup(void *work) static void bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) { - struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; + struct bnad_rx_info *rx_info = rx->priv; struct bna_ccb *ccb; struct bnad_rx_ctrl *rx_ctrl; int i; @@ -1231,7 +1230,7 @@ bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) static void bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx) { - struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; + struct bnad_rx_info *rx_info = rx->priv; struct bna_ccb *ccb; struct bna_rcb *rcb; struct bnad_rx_ctrl *rx_ctrl; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index f6a0f12a6d52..b61566afb2f4 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -26,7 +26,6 @@ #include <linux/platform_device.h> #include <linux/phylink.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_mdio.h> #include <linux/of_net.h> diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c index d3541159487d..72ac4a34424b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c @@ -313,15 +313,15 @@ static int cxgb4_validate_flow_match(struct net_device *dev, u16 ethtype_key = 0; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IP))) { - netdev_warn(dev, "Unsupported key used: 0x%x\n", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP))) { + netdev_warn(dev, "Unsupported key used: 0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 68562a82d036..62f62bff74a5 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -22,6 +22,7 @@ #include <crypto/internal/hash.h> #include <linux/tls.h> #include <net/tls.h> +#include <net/tls_prot.h> #include <net/tls_toe.h> #include "t4fw_api.h" diff --git a/drivers/net/ethernet/cirrus/cs89x0.c b/drivers/net/ethernet/cirrus/cs89x0.c index 276c32c3926a..d323c5c23521 100644 --- a/drivers/net/ethernet/cirrus/cs89x0.c +++ b/drivers/net/ethernet/cirrus/cs89x0.c @@ -54,7 +54,6 @@ #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/kernel.h> #include <linux/types.h> @@ -1855,9 +1854,8 @@ static int __init cs89x0_platform_probe(struct platform_device *pdev) return -ENOMEM; dev->irq = platform_get_irq(pdev, 0); - if (dev->irq <= 0) { - dev_warn(&dev->dev, "interrupt resource missing\n"); - err = -ENXIO; + if (dev->irq < 0) { + err = dev->irq; goto free; } diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 5715b9ab2712..692cb2d04c1c 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -2415,8 +2415,8 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev) /* Interrupt */ irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return irq ? irq : -ENODEV; + if (irq < 0) + return irq; port->irq = irq; /* Clock the port */ diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 84751bb303a6..079f9f6ae21a 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -1333,7 +1333,7 @@ static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, skb = tsnep_build_skb(rx, page, length); if (skb) { - page_pool_release_page(rx->page_pool, page); + skb_mark_for_recycle(skb); rx->packets++; rx->bytes += length; diff --git a/drivers/net/ethernet/ezchip/nps_enet.c b/drivers/net/ethernet/ezchip/nps_enet.c index f1eb660aaee2..edf000e7bab4 100644 --- a/drivers/net/ethernet/ezchip/nps_enet.c +++ b/drivers/net/ethernet/ezchip/nps_enet.c @@ -6,10 +6,9 @@ #include <linux/module.h> #include <linux/etherdevice.h> #include <linux/interrupt.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> +#include <linux/mod_devicetable.h> #include <linux/of_net.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include "nps_enet.h" #define DRV_NAME "nps_mgt_enet" diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index a03879a27b04..9135b918dd49 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -177,16 +177,20 @@ static void ftgmac100_write_mac_addr(struct ftgmac100 *priv, const u8 *mac) iowrite32(laddr, priv->base + FTGMAC100_OFFSET_MAC_LADR); } -static void ftgmac100_initial_mac(struct ftgmac100 *priv) +static int ftgmac100_initial_mac(struct ftgmac100 *priv) { u8 mac[ETH_ALEN]; unsigned int m; unsigned int l; + int err; - if (!device_get_ethdev_address(priv->dev, priv->netdev)) { + err = of_get_ethdev_address(priv->dev->of_node, priv->netdev); + if (err == -EPROBE_DEFER) + return err; + if (!err) { dev_info(priv->dev, "Read MAC address %pM from device tree\n", priv->netdev->dev_addr); - return; + return 0; } m = ioread32(priv->base + FTGMAC100_OFFSET_MAC_MADR); @@ -207,6 +211,8 @@ static void ftgmac100_initial_mac(struct ftgmac100 *priv) dev_info(priv->dev, "Generated random MAC address %pM\n", priv->netdev->dev_addr); } + + return 0; } static int ftgmac100_set_mac_addr(struct net_device *dev, void *p) @@ -1843,7 +1849,9 @@ static int ftgmac100_probe(struct platform_device *pdev) priv->aneg_pause = true; /* MAC address from chip or random one */ - ftgmac100_initial_mac(priv); + err = ftgmac100_initial_mac(priv); + if (err) + goto err_phy_connect; np = pdev->dev.of_node; if (np && (of_device_is_compatible(np, "aspeed,ast2400-mac") || diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 431f8917dc39..dcbc598b11c6 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -7,8 +7,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/init.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_platform.h> #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/io.h> @@ -17,6 +17,7 @@ #include <linux/icmp.h> #include <linux/ip.h> #include <linux/ipv6.h> +#include <linux/platform_device.h> #include <linux/udp.h> #include <linux/tcp.h> #include <linux/net.h> @@ -3497,7 +3498,7 @@ free_netdev: return err; } -static int dpaa_remove(struct platform_device *pdev) +static void dpaa_remove(struct platform_device *pdev) { struct net_device *net_dev; struct dpaa_priv *priv; @@ -3516,6 +3517,9 @@ static int dpaa_remove(struct platform_device *pdev) phylink_destroy(priv->mac_dev->phylink); err = dpaa_fq_free(dev, &priv->dpaa_fq_list); + if (err) + dev_err(dev, "Failed to free FQs on remove (%pE)\n", + ERR_PTR(err)); qman_delete_cgr_safe(&priv->ingress_cgr); qman_release_cgrid(priv->ingress_cgr.cgrid); @@ -3527,8 +3531,6 @@ static int dpaa_remove(struct platform_device *pdev) dpaa_bps_free(priv); free_netdev(net_dev); - - return err; } static const struct platform_device_id dpaa_devtype[] = { @@ -3546,7 +3548,7 @@ static struct platform_driver dpaa_driver = { }, .id_table = dpaa_devtype, .probe = dpaa_eth_probe, - .remove = dpaa_remove + .remove_new = dpaa_remove }; static int __init dpaa_load(void) diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c index 9c71cbbb13d8..5bd0b36d1feb 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c @@ -6,7 +6,9 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/string.h> +#include <linux/of.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/net_tstamp.h> #include <linux/fsl/ptp_qoriq.h> diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c index c39b866e2582..4798fb7fe35d 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c @@ -17,14 +17,14 @@ static int dpaa2_switch_flower_parse_key(struct flow_cls_offload *cls, struct dpsw_acl_fields *acl_h, *acl_m; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_IP) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used"); return -EOPNOTSUPP; @@ -539,9 +539,9 @@ static int dpaa2_switch_flower_parse_mirror_key(struct flow_cls_offload *cls, int ret = -EOPNOTSUPP; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_VLAN))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { NL_SET_ERR_MSG_MOD(extack, "Mirroring is supported only per VLAN"); return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ierb.c b/drivers/net/ethernet/freescale/enetc/enetc_ierb.c index b307bef4dc29..d39617ab9306 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ierb.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ierb.c @@ -18,8 +18,8 @@ */ #include <linux/io.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_device.h> #include <linux/pci.h> #include <linux/platform_device.h> #include "enetc.h" diff --git a/drivers/net/ethernet/freescale/enetc/enetc_qos.c b/drivers/net/ethernet/freescale/enetc/enetc_qos.c index 270cbd5e8684..2513b44056c1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_qos.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_qos.c @@ -483,13 +483,13 @@ struct enetc_psfp { static struct actions_fwd enetc_act_fwd[] = { { BIT(FLOW_ACTION_GATE), - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS), + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS), FILTER_ACTION_TYPE_PSFP }, { BIT(FLOW_ACTION_POLICE) | BIT(FLOW_ACTION_GATE), - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS), + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS), FILTER_ACTION_TYPE_PSFP }, /* example for ACL actions */ @@ -1069,8 +1069,8 @@ revert_sid: return err; } -static struct actions_fwd *enetc_check_flow_actions(u64 acts, - unsigned int inputkeys) +static struct actions_fwd * +enetc_check_flow_actions(u64 acts, unsigned long long inputkeys) { int i; diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index 63a053dea819..110f2e9f3e3f 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -651,12 +651,9 @@ struct fec_enet_private { struct ptp_clock *ptp_clock; struct ptp_clock_info ptp_caps; - unsigned long last_overflow_check; spinlock_t tmreg_lock; struct cyclecounter cc; struct timecounter tc; - int rx_hwtstamp_filter; - u32 base_incval; u32 cycle_speed; int hwts_rx_en; int hwts_tx_en; @@ -679,8 +676,6 @@ struct fec_enet_private { struct ethtool_eee eee; unsigned int clk_ref_rate; - u32 rx_copybreak; - /* ptp clock period in ns*/ unsigned int ptp_inc; @@ -703,9 +698,9 @@ struct fec_enet_private { void fec_ptp_init(struct platform_device *pdev, int irq_idx); void fec_ptp_stop(struct platform_device *pdev); void fec_ptp_start_cyclecounter(struct net_device *ndev); -void fec_ptp_disable_hwts(struct net_device *ndev); -int fec_ptp_set(struct net_device *ndev, struct ifreq *ifr); -int fec_ptp_get(struct net_device *ndev, struct ifreq *ifr); +int fec_ptp_set(struct net_device *ndev, struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack); +void fec_ptp_get(struct net_device *ndev, struct kernel_hwtstamp_config *config); /****************************************************************************/ #endif /* FEC_H */ diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 66b5cbdb43b9..43f14cec91e9 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -325,8 +325,6 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); #define FEC_WOL_FLAG_ENABLE (0x1 << 1) #define FEC_WOL_FLAG_SLEEP_ON (0x1 << 2) -#define COPYBREAK_DEFAULT 256 - /* Max number of allowed TCP segments for software TSO */ #define FEC_MAX_TSO_SEGS 100 #define FEC_MAX_SKB_DESCS (FEC_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) @@ -3059,44 +3057,6 @@ static int fec_enet_set_coalesce(struct net_device *ndev, return 0; } -static int fec_enet_get_tunable(struct net_device *netdev, - const struct ethtool_tunable *tuna, - void *data) -{ - struct fec_enet_private *fep = netdev_priv(netdev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - *(u32 *)data = fep->rx_copybreak; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static int fec_enet_set_tunable(struct net_device *netdev, - const struct ethtool_tunable *tuna, - const void *data) -{ - struct fec_enet_private *fep = netdev_priv(netdev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - fep->rx_copybreak = *(u32 *)data; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - /* LPI Sleep Ts count base on tx clk (clk_ref). * The lpi sleep cnt value = X us / (cycle_ns). */ @@ -3234,8 +3194,6 @@ static const struct ethtool_ops fec_enet_ethtool_ops = { .get_sset_count = fec_enet_get_sset_count, #endif .get_ts_info = fec_enet_get_ts_info, - .get_tunable = fec_enet_get_tunable, - .set_tunable = fec_enet_set_tunable, .get_wol = fec_enet_get_wol, .set_wol = fec_enet_set_wol, .get_eee = fec_enet_get_eee, @@ -3245,33 +3203,6 @@ static const struct ethtool_ops fec_enet_ethtool_ops = { .self_test = net_selftest, }; -static int fec_enet_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) -{ - struct fec_enet_private *fep = netdev_priv(ndev); - struct phy_device *phydev = ndev->phydev; - - if (!netif_running(ndev)) - return -EINVAL; - - if (!phydev) - return -ENODEV; - - if (fep->bufdesc_ex) { - bool use_fec_hwts = !phy_has_hwtstamp(phydev); - - if (cmd == SIOCSHWTSTAMP) { - if (use_fec_hwts) - return fec_ptp_set(ndev, rq); - fec_ptp_disable_hwts(ndev); - } else if (cmd == SIOCGHWTSTAMP) { - if (use_fec_hwts) - return fec_ptp_get(ndev, rq); - } - } - - return phy_mii_ioctl(phydev, rq, cmd); -} - static void fec_enet_free_buffers(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); @@ -3937,6 +3868,37 @@ static int fec_enet_xdp_xmit(struct net_device *dev, return sent_frames; } +static int fec_hwtstamp_get(struct net_device *ndev, + struct kernel_hwtstamp_config *config) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (!netif_running(ndev)) + return -EINVAL; + + if (!fep->bufdesc_ex) + return -EOPNOTSUPP; + + fec_ptp_get(ndev, config); + + return 0; +} + +static int fec_hwtstamp_set(struct net_device *ndev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (!netif_running(ndev)) + return -EINVAL; + + if (!fep->bufdesc_ex) + return -EOPNOTSUPP; + + return fec_ptp_set(ndev, config, extack); +} + static const struct net_device_ops fec_netdev_ops = { .ndo_open = fec_enet_open, .ndo_stop = fec_enet_close, @@ -3946,13 +3908,15 @@ static const struct net_device_ops fec_netdev_ops = { .ndo_validate_addr = eth_validate_addr, .ndo_tx_timeout = fec_timeout, .ndo_set_mac_address = fec_set_mac_address, - .ndo_eth_ioctl = fec_enet_ioctl, + .ndo_eth_ioctl = phy_do_ioctl_running, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = fec_poll_controller, #endif .ndo_set_features = fec_set_features, .ndo_bpf = fec_enet_bpf, .ndo_xdp_xmit = fec_enet_xdp_xmit, + .ndo_hwtstamp_get = fec_hwtstamp_get, + .ndo_hwtstamp_set = fec_hwtstamp_set, }; static const unsigned short offset_des_active_rxq[] = { @@ -4018,9 +3982,6 @@ static int fec_enet_init(struct net_device *ndev) if (ret) goto free_queue_mem; - /* make sure MAC we just acquired is programmed into the hw */ - fec_set_mac_address(ndev, NULL); - /* Set receive and transmit descriptor base. */ for (i = 0; i < fep->num_rx_queues; i++) { struct fec_enet_priv_rx_q *rxq = fep->rx_queue[i]; @@ -4486,7 +4447,6 @@ fec_probe(struct platform_device *pdev) if (fep->bufdesc_ex && fep->ptp_clock) netdev_info(ndev, "registered PHC device %d\n", fep->dev_id); - fep->rx_copybreak = COPYBREAK_DEFAULT; INIT_WORK(&fep->tx_timeout_work, fec_enet_timeout_work); pm_runtime_mark_last_busy(&pdev->dev); @@ -4526,7 +4486,7 @@ failed_ioremap: return ret; } -static int +static void fec_drv_remove(struct platform_device *pdev) { struct net_device *ndev = platform_get_drvdata(pdev); @@ -4562,7 +4522,6 @@ fec_drv_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); free_netdev(ndev); - return 0; } static int __maybe_unused fec_suspend(struct device *dev) @@ -4718,7 +4677,7 @@ static struct platform_driver fec_driver = { }, .id_table = fec_devtype, .probe = fec_probe, - .remove = fec_drv_remove, + .remove_new = fec_drv_remove, }; module_platform_driver(fec_driver); diff --git a/drivers/net/ethernet/freescale/fec_mpc52xx.c b/drivers/net/ethernet/freescale/fec_mpc52xx.c index b88816b71ddf..ebae71ec26c6 100644 --- a/drivers/net/ethernet/freescale/fec_mpc52xx.c +++ b/drivers/net/ethernet/freescale/fec_mpc52xx.c @@ -29,12 +29,12 @@ #include <linux/crc32.h> #include <linux/hardirq.h> #include <linux/delay.h> +#include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> @@ -974,7 +974,7 @@ err_netdev: return rv; } -static int +static void mpc52xx_fec_remove(struct platform_device *op) { struct net_device *ndev; @@ -998,8 +998,6 @@ mpc52xx_fec_remove(struct platform_device *op) release_mem_region(ndev->base_addr, sizeof(struct mpc52xx_fec)); free_netdev(ndev); - - return 0; } #ifdef CONFIG_PM @@ -1042,7 +1040,7 @@ static struct platform_driver mpc52xx_fec_driver = { .of_match_table = mpc52xx_fec_match, }, .probe = mpc52xx_fec_probe, - .remove = mpc52xx_fec_remove, + .remove_new = mpc52xx_fec_remove, #ifdef CONFIG_PM .suspend = mpc52xx_fec_of_suspend, .resume = mpc52xx_fec_of_resume, diff --git a/drivers/net/ethernet/freescale/fec_mpc52xx_phy.c b/drivers/net/ethernet/freescale/fec_mpc52xx_phy.c index 95f778cce98c..39689826cc8f 100644 --- a/drivers/net/ethernet/freescale/fec_mpc52xx_phy.c +++ b/drivers/net/ethernet/freescale/fec_mpc52xx_phy.c @@ -13,10 +13,11 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/phy.h> -#include <linux/of_platform.h> #include <linux/slab.h> +#include <linux/of.h> #include <linux/of_address.h> #include <linux/of_mdio.h> +#include <linux/platform_device.h> #include <asm/io.h> #include <asm/mpc52xx.h> #include "fec_mpc52xx.h" @@ -117,7 +118,7 @@ static int mpc52xx_fec_mdio_probe(struct platform_device *of) return err; } -static int mpc52xx_fec_mdio_remove(struct platform_device *of) +static void mpc52xx_fec_mdio_remove(struct platform_device *of) { struct mii_bus *bus = platform_get_drvdata(of); struct mpc52xx_fec_mdio_priv *priv = bus->priv; @@ -126,8 +127,6 @@ static int mpc52xx_fec_mdio_remove(struct platform_device *of) iounmap(priv->regs); kfree(priv); mdiobus_free(bus); - - return 0; } static const struct of_device_id mpc52xx_fec_mdio_match[] = { @@ -145,7 +144,7 @@ struct platform_driver mpc52xx_fec_mdio_driver = { .of_match_table = mpc52xx_fec_mdio_match, }, .probe = mpc52xx_fec_mdio_probe, - .remove = mpc52xx_fec_mdio_remove, + .remove_new = mpc52xx_fec_mdio_remove, }; /* let fec driver call it, since this has to be registered before it */ diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index ab86bb8562ef..181d9bfbee22 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -30,7 +30,6 @@ #include <linux/phy.h> #include <linux/fec.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_net.h> @@ -443,21 +442,21 @@ static int fec_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) */ static int fec_ptp_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) { - struct fec_enet_private *adapter = + struct fec_enet_private *fep = container_of(ptp, struct fec_enet_private, ptp_caps); u64 ns; unsigned long flags; - mutex_lock(&adapter->ptp_clk_mutex); + mutex_lock(&fep->ptp_clk_mutex); /* Check the ptp clock */ - if (!adapter->ptp_clk_on) { - mutex_unlock(&adapter->ptp_clk_mutex); + if (!fep->ptp_clk_on) { + mutex_unlock(&fep->ptp_clk_mutex); return -EINVAL; } - spin_lock_irqsave(&adapter->tmreg_lock, flags); - ns = timecounter_read(&adapter->tc); - spin_unlock_irqrestore(&adapter->tmreg_lock, flags); - mutex_unlock(&adapter->ptp_clk_mutex); + spin_lock_irqsave(&fep->tmreg_lock, flags); + ns = timecounter_read(&fep->tc); + spin_unlock_irqrestore(&fep->tmreg_lock, flags); + mutex_unlock(&fep->ptp_clk_mutex); *ts = ns_to_timespec64(ns); @@ -606,28 +605,12 @@ static int fec_ptp_enable(struct ptp_clock_info *ptp, } } -/** - * fec_ptp_disable_hwts - disable hardware time stamping - * @ndev: pointer to net_device - */ -void fec_ptp_disable_hwts(struct net_device *ndev) -{ - struct fec_enet_private *fep = netdev_priv(ndev); - - fep->hwts_tx_en = 0; - fep->hwts_rx_en = 0; -} - -int fec_ptp_set(struct net_device *ndev, struct ifreq *ifr) +int fec_ptp_set(struct net_device *ndev, struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) { struct fec_enet_private *fep = netdev_priv(ndev); - struct hwtstamp_config config; - - if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) - return -EFAULT; - - switch (config.tx_type) { + switch (config->tx_type) { case HWTSTAMP_TX_OFF: fep->hwts_tx_en = 0; break; @@ -638,33 +621,28 @@ int fec_ptp_set(struct net_device *ndev, struct ifreq *ifr) return -ERANGE; } - switch (config.rx_filter) { + switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: fep->hwts_rx_en = 0; break; default: fep->hwts_rx_en = 1; - config.rx_filter = HWTSTAMP_FILTER_ALL; + config->rx_filter = HWTSTAMP_FILTER_ALL; break; } - return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? - -EFAULT : 0; + return 0; } -int fec_ptp_get(struct net_device *ndev, struct ifreq *ifr) +void fec_ptp_get(struct net_device *ndev, struct kernel_hwtstamp_config *config) { struct fec_enet_private *fep = netdev_priv(ndev); - struct hwtstamp_config config; - - config.flags = 0; - config.tx_type = fep->hwts_tx_en ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; - config.rx_filter = (fep->hwts_rx_en ? - HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE); - return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? - -EFAULT : 0; + config->flags = 0; + config->tx_type = fep->hwts_tx_en ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; + config->rx_filter = (fep->hwts_rx_en ? + HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE); } /* diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index 9d85fb136e34..d96028f01770 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/fsl/guts.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/delay.h> #include <linux/module.h> diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c index ab90fe2bee5e..406e75e9e5ea 100644 --- a/drivers/net/ethernet/freescale/fman/fman_port.c +++ b/drivers/net/ethernet/freescale/fman/fman_port.c @@ -6,6 +6,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/io.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/interrupt.h> diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c index 43665806c590..9767586b4eb3 100644 --- a/drivers/net/ethernet/freescale/fman/mac.c +++ b/drivers/net/ethernet/freescale/fman/mac.c @@ -18,6 +18,7 @@ #include <linux/phylink.h> #include <linux/etherdevice.h> #include <linux/libfdt_env.h> +#include <linux/platform_device.h> #include "mac.h" #include "fman_mac.h" @@ -331,12 +332,11 @@ _return_of_node_put: return err; } -static int mac_remove(struct platform_device *pdev) +static void mac_remove(struct platform_device *pdev) { struct mac_device *mac_dev = platform_get_drvdata(pdev); platform_device_unregister(mac_dev->priv->eth_dev); - return 0; } static struct platform_driver mac_driver = { @@ -345,7 +345,7 @@ static struct platform_driver mac_driver = { .of_match_table = mac_match, }, .probe = mac_probe, - .remove = mac_remove, + .remove_new = mac_remove, }; builtin_platform_driver(mac_driver); diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c index 8844a9a04fcf..f9f5b28cc72e 100644 --- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c +++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c @@ -1051,7 +1051,7 @@ out_free_fpi: return ret; } -static int fs_enet_remove(struct platform_device *ofdev) +static void fs_enet_remove(struct platform_device *ofdev) { struct net_device *ndev = platform_get_drvdata(ofdev); struct fs_enet_private *fep = netdev_priv(ndev); @@ -1066,7 +1066,6 @@ static int fs_enet_remove(struct platform_device *ofdev) if (of_phy_is_fixed_link(ofdev->dev.of_node)) of_phy_deregister_fixed_link(ofdev->dev.of_node); free_netdev(ndev); - return 0; } static const struct of_device_id fs_enet_match[] = { @@ -1113,7 +1112,7 @@ static struct platform_driver fs_enet_driver = { .of_match_table = fs_enet_match, }, .probe = fs_enet_probe, - .remove = fs_enet_remove, + .remove_new = fs_enet_remove, }; #ifdef CONFIG_NET_POLL_CONTROLLER diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c index b47490be872c..925428f1b0c8 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c +++ b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c @@ -32,7 +32,6 @@ #include <linux/platform_device.h> #include <linux/phy.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/gfp.h> #include <linux/pgtable.h> diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c index 61f4b6e50d29..f609dc112458 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c +++ b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c @@ -32,7 +32,6 @@ #include <linux/fs.h> #include <linux/platform_device.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/gfp.h> diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c index 64300ac13e02..66d40da5cde0 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c +++ b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c @@ -32,7 +32,6 @@ #include <linux/platform_device.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> #include <asm/irq.h> #include <linux/uaccess.h> diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c index 21de56345503..91a69fc2f7c2 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c +++ b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c @@ -192,7 +192,7 @@ out: return ret; } -static int fs_enet_mdio_remove(struct platform_device *ofdev) +static void fs_enet_mdio_remove(struct platform_device *ofdev) { struct mii_bus *bus = platform_get_drvdata(ofdev); struct bb_info *bitbang = bus->priv; @@ -201,8 +201,6 @@ static int fs_enet_mdio_remove(struct platform_device *ofdev) free_mdio_bitbang(bus); iounmap(bitbang->dir); kfree(bitbang); - - return 0; } static const struct of_device_id fs_enet_mdio_bb_match[] = { @@ -219,7 +217,7 @@ static struct platform_driver fs_enet_bb_mdio_driver = { .of_match_table = fs_enet_mdio_bb_match, }, .probe = fs_enet_mdio_probe, - .remove = fs_enet_mdio_remove, + .remove_new = fs_enet_mdio_remove, }; module_platform_driver(fs_enet_bb_mdio_driver); diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c index 59a8f0bd0f5c..1910df250c33 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c +++ b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c @@ -187,7 +187,7 @@ out: return ret; } -static int fs_enet_mdio_remove(struct platform_device *ofdev) +static void fs_enet_mdio_remove(struct platform_device *ofdev) { struct mii_bus *bus = platform_get_drvdata(ofdev); struct fec_info *fec = bus->priv; @@ -196,8 +196,6 @@ static int fs_enet_mdio_remove(struct platform_device *ofdev) iounmap(fec->fecp); kfree(fec); mdiobus_free(bus); - - return 0; } static const struct of_device_id fs_enet_mdio_fec_match[] = { @@ -220,7 +218,7 @@ static struct platform_driver fs_enet_fec_mdio_driver = { .of_match_table = fs_enet_mdio_fec_match, }, .probe = fs_enet_mdio_probe, - .remove = fs_enet_mdio_remove, + .remove_new = fs_enet_mdio_remove, }; module_platform_driver(fs_enet_fec_mdio_driver); diff --git a/drivers/net/ethernet/freescale/fsl_pq_mdio.c b/drivers/net/ethernet/freescale/fsl_pq_mdio.c index 9d58d8334467..eee675a25b2c 100644 --- a/drivers/net/ethernet/freescale/fsl_pq_mdio.c +++ b/drivers/net/ethernet/freescale/fsl_pq_mdio.c @@ -12,6 +12,7 @@ */ #include <linux/kernel.h> +#include <linux/platform_device.h> #include <linux/string.h> #include <linux/errno.h> #include <linux/slab.h> @@ -511,7 +512,7 @@ error: } -static int fsl_pq_mdio_remove(struct platform_device *pdev) +static void fsl_pq_mdio_remove(struct platform_device *pdev) { struct device *device = &pdev->dev; struct mii_bus *bus = dev_get_drvdata(device); @@ -521,8 +522,6 @@ static int fsl_pq_mdio_remove(struct platform_device *pdev) iounmap(priv->map); mdiobus_free(bus); - - return 0; } static struct platform_driver fsl_pq_mdio_driver = { @@ -531,7 +530,7 @@ static struct platform_driver fsl_pq_mdio_driver = { .of_match_table = fsl_pq_mdio_match, }, .probe = fsl_pq_mdio_probe, - .remove = fsl_pq_mdio_remove, + .remove_new = fsl_pq_mdio_remove, }; module_platform_driver(fsl_pq_mdio_driver); diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index 38d5013c6fed..e3dfbd7a4236 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -60,6 +60,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/kernel.h> +#include <linux/platform_device.h> #include <linux/string.h> #include <linux/errno.h> #include <linux/unistd.h> @@ -75,7 +76,6 @@ #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> -#include <linux/of_platform.h> #include <linux/ip.h> #include <linux/tcp.h> #include <linux/udp.h> @@ -3364,7 +3364,7 @@ register_fail: return err; } -static int gfar_remove(struct platform_device *ofdev) +static void gfar_remove(struct platform_device *ofdev) { struct gfar_private *priv = platform_get_drvdata(ofdev); struct device_node *np = ofdev->dev.of_node; @@ -3381,8 +3381,6 @@ static int gfar_remove(struct platform_device *ofdev) gfar_free_rx_queues(priv); gfar_free_tx_queues(priv); free_gfar_dev(priv); - - return 0; } #ifdef CONFIG_PM @@ -3642,7 +3640,7 @@ static struct platform_driver gfar_driver = { .of_match_table = gfar_match, }, .probe = gfar_probe, - .remove = gfar_remove, + .remove_new = gfar_remove, }; module_platform_driver(gfar_driver); diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index b2b0d3c26fcc..7a15b9245698 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -38,7 +38,9 @@ #include <linux/phy.h> #include <linux/sort.h> #include <linux/if_vlan.h> +#include <linux/of.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/fsl/ptp_qoriq.h> #include "gianfar.h" diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c index 7a4cb4f07c32..ab421243a419 100644 --- a/drivers/net/ethernet/freescale/ucc_geth.c +++ b/drivers/net/ethernet/freescale/ucc_geth.c @@ -28,11 +28,12 @@ #include <linux/phy.h> #include <linux/phy_fixed.h> #include <linux/workqueue.h> +#include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/uaccess.h> #include <asm/irq.h> @@ -3753,7 +3754,7 @@ err_free_info: return err; } -static int ucc_geth_remove(struct platform_device* ofdev) +static void ucc_geth_remove(struct platform_device* ofdev) { struct net_device *dev = platform_get_drvdata(ofdev); struct ucc_geth_private *ugeth = netdev_priv(dev); @@ -3767,8 +3768,6 @@ static int ucc_geth_remove(struct platform_device* ofdev) of_node_put(ugeth->ug_info->phy_node); kfree(ugeth->ug_info); free_netdev(dev); - - return 0; } static const struct of_device_id ucc_geth_match[] = { @@ -3787,7 +3786,7 @@ static struct platform_driver ucc_geth_driver = { .of_match_table = ucc_geth_match, }, .probe = ucc_geth_probe, - .remove = ucc_geth_remove, + .remove_new = ucc_geth_remove, .suspend = ucc_geth_suspend, .resume = ucc_geth_resume, }; diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c index a13b4ba4d6e1..65dc07d0df0f 100644 --- a/drivers/net/ethernet/freescale/xgmac_mdio.c +++ b/drivers/net/ethernet/freescale/xgmac_mdio.c @@ -19,10 +19,10 @@ #include <linux/kernel.h> #include <linux/mdio.h> #include <linux/module.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_mdio.h> -#include <linux/of_platform.h> #include <linux/phy.h> +#include <linux/platform_device.h> #include <linux/slab.h> /* Number of microseconds to wait for a register to respond */ diff --git a/drivers/net/ethernet/google/gve/gve_desc.h b/drivers/net/ethernet/google/gve/gve_desc.h index f4ae9e19b844..c2874cdcf40c 100644 --- a/drivers/net/ethernet/google/gve/gve_desc.h +++ b/drivers/net/ethernet/google/gve/gve_desc.h @@ -105,10 +105,10 @@ union gve_rx_data_slot { __be64 addr; }; -/* GVE Recive Packet Descriptor Seq No */ +/* GVE Receive Packet Descriptor Seq No */ #define GVE_SEQNO(x) (be16_to_cpu(x) & 0x7) -/* GVE Recive Packet Descriptor Flags */ +/* GVE Receive Packet Descriptor Flags */ #define GVE_RXFLG(x) cpu_to_be16(1 << (3 + (x))) #define GVE_RXF_FRAG GVE_RXFLG(3) /* IP Fragment */ #define GVE_RXF_IPV4 GVE_RXFLG(4) /* IPv4 */ diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c index 50c3f5d6611f..ecf92a5d56bb 100644 --- a/drivers/net/ethernet/hisilicon/hip04_eth.c +++ b/drivers/net/ethernet/hisilicon/hip04_eth.c @@ -960,8 +960,8 @@ static int hip04_mac_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (irq <= 0) { - ret = -EINVAL; + if (irq < 0) { + ret = irq; goto init_fail; } diff --git a/drivers/net/ethernet/hisilicon/hisi_femac.c b/drivers/net/ethernet/hisilicon/hisi_femac.c index ce2571c16e43..cb7b0293fe85 100644 --- a/drivers/net/ethernet/hisilicon/hisi_femac.c +++ b/drivers/net/ethernet/hisilicon/hisi_femac.c @@ -862,8 +862,8 @@ static int hisi_femac_drv_probe(struct platform_device *pdev) goto out_disconnect_phy; ndev->irq = platform_get_irq(pdev, 0); - if (ndev->irq <= 0) { - ret = -ENODEV; + if (ndev->irq < 0) { + ret = ndev->irq; goto out_disconnect_phy; } diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c index f867e9531117..26d22bb04b87 100644 --- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c +++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c @@ -1206,9 +1206,8 @@ static int hix5hd2_dev_probe(struct platform_device *pdev) } ndev->irq = platform_get_irq(pdev, 0); - if (ndev->irq <= 0) { - netdev_err(ndev, "No irq resource\n"); - ret = -EINVAL; + if (ndev->irq < 0) { + ret = ndev->irq; goto out_phy_node; } diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c index a7eb87da4e70..a08d1f0a5a16 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c @@ -9,9 +9,6 @@ #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/platform_device.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/of_platform.h> #include "hns_dsaf_ppe.h" diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c index e2ff3ca198d1..93344563a259 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c @@ -11,10 +11,6 @@ #include <linux/etherdevice.h> #include <asm/cacheflush.h> #include <linux/platform_device.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/of_platform.h> -#include <linux/of_irq.h> #include <linux/spinlock.h> #include "hns_dsaf_main.h" diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 407d30ee55d2..36858a72d771 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -569,8 +569,8 @@ static void hns3_get_strings(struct net_device *netdev, u32 stringset, u8 *data) static u64 *hns3_get_stats_tqps(struct hnae3_handle *handle, u64 *data) { - struct hns3_nic_priv *nic_priv = (struct hns3_nic_priv *)handle->priv; struct hnae3_knic_private_info *kinfo = &handle->kinfo; + struct hns3_nic_priv *nic_priv = handle->priv; struct hns3_enet_ring *ring; u8 *stat; int i, j; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index bf675c15fbb9..83ab89f44250 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -7315,14 +7315,14 @@ static int hclge_parse_cls_flower(struct hclge_dev *hdev, struct flow_dissector *dissector = flow->match.dissector; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS))) { - dev_err(&hdev->pdev->dev, "unsupported key set: %#x\n", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS))) { + dev_err(&hdev->pdev->dev, "unsupported key set: %#llx\n", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/hisilicon/hns_mdio.c b/drivers/net/ethernet/hisilicon/hns_mdio.c index 9232caaf0bdc..409a89d80220 100644 --- a/drivers/net/ethernet/hisilicon/hns_mdio.c +++ b/drivers/net/ethernet/hisilicon/hns_mdio.c @@ -217,7 +217,7 @@ static void hns_mdio_cmd_write(struct hns_mdio_device *mdio_dev, static int hns_mdio_write_c22(struct mii_bus *bus, int phy_id, int regnum, u16 data) { - struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv; + struct hns_mdio_device *mdio_dev = bus->priv; u16 reg = (u16)(regnum & 0xffff); u16 cmd_reg_cfg; int ret; @@ -259,7 +259,7 @@ static int hns_mdio_write_c22(struct mii_bus *bus, static int hns_mdio_write_c45(struct mii_bus *bus, int phy_id, int devad, int regnum, u16 data) { - struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv; + struct hns_mdio_device *mdio_dev = bus->priv; u16 reg = (u16)(regnum & 0xffff); u16 cmd_reg_cfg; int ret; @@ -312,7 +312,7 @@ static int hns_mdio_write_c45(struct mii_bus *bus, int phy_id, int devad, */ static int hns_mdio_read_c22(struct mii_bus *bus, int phy_id, int regnum) { - struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv; + struct hns_mdio_device *mdio_dev = bus->priv; u16 reg = (u16)(regnum & 0xffff); u16 reg_val; int ret; @@ -363,7 +363,7 @@ static int hns_mdio_read_c22(struct mii_bus *bus, int phy_id, int regnum) static int hns_mdio_read_c45(struct mii_bus *bus, int phy_id, int devad, int regnum) { - struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv; + struct hns_mdio_device *mdio_dev = bus->priv; u16 reg = (u16)(regnum & 0xffff); u16 reg_val; int ret; @@ -424,7 +424,7 @@ static int hns_mdio_read_c45(struct mii_bus *bus, int phy_id, int devad, */ static int hns_mdio_reset(struct mii_bus *bus) { - struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv; + struct hns_mdio_device *mdio_dev = bus->priv; const struct hns_mdio_sc_reg *sc_reg; int ret; diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c index b4aff59b3eb4..0a56e9752464 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_main.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c @@ -31,6 +31,7 @@ #include <linux/prefetch.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/platform_device.h> #include <net/ip.h> diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index c97095abd26a..0c314bf97480 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -38,6 +38,7 @@ #include <linux/of_irq.h> #include <linux/of_net.h> #include <linux/of_mdio.h> +#include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/slab.h> diff --git a/drivers/net/ethernet/ibm/emac/core.h b/drivers/net/ethernet/ibm/emac/core.h index 89a1b0fea158..295516b07662 100644 --- a/drivers/net/ethernet/ibm/emac/core.h +++ b/drivers/net/ethernet/ibm/emac/core.h @@ -27,7 +27,6 @@ #include <linux/netdevice.h> #include <linux/dma-mapping.h> #include <linux/spinlock.h> -#include <linux/of_platform.h> #include <linux/slab.h> #include <asm/io.h> diff --git a/drivers/net/ethernet/ibm/emac/mal.c b/drivers/net/ethernet/ibm/emac/mal.c index ff5487bbebe3..c3236b59e7e9 100644 --- a/drivers/net/ethernet/ibm/emac/mal.c +++ b/drivers/net/ethernet/ibm/emac/mal.c @@ -22,7 +22,9 @@ #include <linux/delay.h> #include <linux/slab.h> +#include <linux/of.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include "core.h" #include <asm/dcr-regs.h> diff --git a/drivers/net/ethernet/ibm/emac/rgmii.c b/drivers/net/ethernet/ibm/emac/rgmii.c index 50358cf00130..fd437f986edf 100644 --- a/drivers/net/ethernet/ibm/emac/rgmii.c +++ b/drivers/net/ethernet/ibm/emac/rgmii.c @@ -19,7 +19,9 @@ #include <linux/slab.h> #include <linux/kernel.h> #include <linux/ethtool.h> +#include <linux/of.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <asm/io.h> #include "emac.h" diff --git a/drivers/net/ethernet/ibm/emac/tah.c b/drivers/net/ethernet/ibm/emac/tah.c index 008bbdaf1204..aae9a88d95d7 100644 --- a/drivers/net/ethernet/ibm/emac/tah.c +++ b/drivers/net/ethernet/ibm/emac/tah.c @@ -14,7 +14,9 @@ * * Copyright (c) 2005 Eugene Surovegin <ebs@ebshome.net> */ +#include <linux/mod_devicetable.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <asm/io.h> #include "emac.h" diff --git a/drivers/net/ethernet/ibm/emac/zmii.c b/drivers/net/ethernet/ibm/emac/zmii.c index 57a25c7a9e70..6337388ee5f4 100644 --- a/drivers/net/ethernet/ibm/emac/zmii.c +++ b/drivers/net/ethernet/ibm/emac/zmii.c @@ -19,7 +19,9 @@ #include <linux/slab.h> #include <linux/kernel.h> #include <linux/ethtool.h> +#include <linux/mod_devicetable.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <asm/io.h> #include "emac.h" diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.c b/drivers/net/ethernet/intel/i40e/i40e_adminq.c index 86fac8f959bb..100eb77b8dfe 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_adminq.c +++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ -#include "i40e_status.h" #include "i40e_type.h" #include "i40e_register.h" #include "i40e_adminq.h" @@ -284,7 +283,7 @@ static int i40e_config_asq_regs(struct i40e_hw *hw) /* Check one register to verify that config was applied */ reg = rd32(hw, hw->aq.asq.bal); if (reg != lower_32_bits(hw->aq.asq.desc_buf.pa)) - ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; + ret_code = -EIO; return ret_code; } @@ -316,7 +315,7 @@ static int i40e_config_arq_regs(struct i40e_hw *hw) /* Check one register to verify that config was applied */ reg = rd32(hw, hw->aq.arq.bal); if (reg != lower_32_bits(hw->aq.arq.desc_buf.pa)) - ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; + ret_code = -EIO; return ret_code; } @@ -340,14 +339,14 @@ static int i40e_init_asq(struct i40e_hw *hw) if (hw->aq.asq.count > 0) { /* queue already initialized */ - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto init_adminq_exit; } /* verify input for valid configuration */ if ((hw->aq.num_asq_entries == 0) || (hw->aq.asq_buf_size == 0)) { - ret_code = I40E_ERR_CONFIG; + ret_code = -EIO; goto init_adminq_exit; } @@ -399,14 +398,14 @@ static int i40e_init_arq(struct i40e_hw *hw) if (hw->aq.arq.count > 0) { /* queue already initialized */ - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto init_adminq_exit; } /* verify input for valid configuration */ if ((hw->aq.num_arq_entries == 0) || (hw->aq.arq_buf_size == 0)) { - ret_code = I40E_ERR_CONFIG; + ret_code = -EIO; goto init_adminq_exit; } @@ -452,7 +451,7 @@ static int i40e_shutdown_asq(struct i40e_hw *hw) mutex_lock(&hw->aq.asq_mutex); if (hw->aq.asq.count == 0) { - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto shutdown_asq_out; } @@ -486,7 +485,7 @@ static int i40e_shutdown_arq(struct i40e_hw *hw) mutex_lock(&hw->aq.arq_mutex); if (hw->aq.arq.count == 0) { - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto shutdown_arq_out; } @@ -594,7 +593,7 @@ int i40e_init_adminq(struct i40e_hw *hw) (hw->aq.num_asq_entries == 0) || (hw->aq.arq_buf_size == 0) || (hw->aq.asq_buf_size == 0)) { - ret_code = I40E_ERR_CONFIG; + ret_code = -EIO; goto init_adminq_exit; } @@ -626,13 +625,13 @@ int i40e_init_adminq(struct i40e_hw *hw) &hw->aq.api_maj_ver, &hw->aq.api_min_ver, NULL); - if (ret_code != I40E_ERR_ADMIN_QUEUE_TIMEOUT) + if (ret_code != -EIO) break; retry++; msleep(100); i40e_resume_aq(hw); } while (retry < 10); - if (ret_code != I40E_SUCCESS) + if (ret_code != 0) goto init_adminq_free_arq; /* Some features were introduced in different FW API version @@ -672,7 +671,7 @@ int i40e_init_adminq(struct i40e_hw *hw) hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE; if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) { - ret_code = I40E_ERR_FIRMWARE_API_VERSION; + ret_code = -EIO; goto init_adminq_free_arq; } @@ -799,7 +798,7 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, if (hw->aq.asq.count == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Admin queue not initialized.\n"); - status = I40E_ERR_QUEUE_EMPTY; + status = -EIO; goto asq_send_command_error; } @@ -809,7 +808,7 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, if (val >= hw->aq.num_asq_entries) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: head overrun at %d\n", val); - status = I40E_ERR_ADMIN_QUEUE_FULL; + status = -ENOSPC; goto asq_send_command_error; } @@ -840,7 +839,7 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Invalid buffer size: %d.\n", buff_size); - status = I40E_ERR_INVALID_SIZE; + status = -EINVAL; goto asq_send_command_error; } @@ -848,7 +847,7 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Async flag not set along with postpone flag"); - status = I40E_ERR_PARAM; + status = -EINVAL; goto asq_send_command_error; } @@ -863,7 +862,7 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Error queue is full.\n"); - status = I40E_ERR_ADMIN_QUEUE_FULL; + status = -ENOSPC; goto asq_send_command_error; } @@ -940,9 +939,9 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK) status = 0; else if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_EBUSY) - status = I40E_ERR_NOT_READY; + status = -EBUSY; else - status = I40E_ERR_ADMIN_QUEUE_ERROR; + status = -EIO; hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval; } @@ -960,11 +959,11 @@ i40e_asq_send_command_atomic_exec(struct i40e_hw *hw, if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: AQ Critical error.\n"); - status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR; + status = -EIO; } else { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: Writeback timeout.\n"); - status = I40E_ERR_ADMIN_QUEUE_TIMEOUT; + status = -EIO; } } @@ -1106,7 +1105,7 @@ int i40e_clean_arq_element(struct i40e_hw *hw, if (hw->aq.arq.count == 0) { i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Admin queue not initialized.\n"); - ret_code = I40E_ERR_QUEUE_EMPTY; + ret_code = -EIO; goto clean_arq_element_err; } @@ -1114,7 +1113,7 @@ int i40e_clean_arq_element(struct i40e_hw *hw, ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; if (ntu == ntc) { /* nothing to do - shouldn't need to update ring's values */ - ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK; + ret_code = -EALREADY; goto clean_arq_element_out; } @@ -1126,7 +1125,7 @@ int i40e_clean_arq_element(struct i40e_hw *hw, (enum i40e_admin_queue_err)le16_to_cpu(desc->retval); flags = le16_to_cpu(desc->flags); if (flags & I40E_AQ_FLAG_ERR) { - ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; + ret_code = -EIO; i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Event received with error 0x%X.\n", diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.h b/drivers/net/ethernet/intel/i40e/i40e_adminq.h index ee394aacef4d..267f2e0a21ce 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_adminq.h +++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.h @@ -5,7 +5,6 @@ #define _I40E_ADMINQ_H_ #include "i40e_osdep.h" -#include "i40e_status.h" #include "i40e_adminq_cmd.h" #define I40E_ADMINQ_DESC(R, i) \ @@ -117,7 +116,7 @@ static inline int i40e_aq_rc_to_posix(int aq_ret, int aq_rc) }; /* aq_rc is invalid if AQ timed out */ - if (aq_ret == I40E_ERR_ADMIN_QUEUE_TIMEOUT) + if (aq_ret == -EIO) return -EAGAIN; if (!((u32)aq_rc < (sizeof(aq_to_posix) / sizeof((aq_to_posix)[0])))) diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index ed88e38d488b..eeef20f77106 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -56,7 +56,7 @@ int i40e_set_mac_type(struct i40e_hw *hw) break; } } else { - status = I40E_ERR_DEVICE_NOT_SUPPORTED; + status = -ENODEV; } hw_dbg(hw, "i40e_set_mac_type found mac: %d, returns: %d\n", @@ -660,7 +660,7 @@ int i40e_init_shared_code(struct i40e_hw *hw) case I40E_MAC_X722: break; default: - return I40E_ERR_DEVICE_NOT_SUPPORTED; + return -ENODEV; } hw->phy.get_link_info = true; @@ -780,7 +780,7 @@ int i40e_get_port_mac_addr(struct i40e_hw *hw, u8 *mac_addr) if (flags & I40E_AQC_PORT_ADDR_VALID) ether_addr_copy(mac_addr, addrs.port_mac); else - status = I40E_ERR_INVALID_MAC_ADDR; + status = -EINVAL; return status; } @@ -858,7 +858,7 @@ int i40e_read_pba_string(struct i40e_hw *hw, u8 *pba_num, pba_size--; if (pba_num_size < (((u32)pba_size * 2) + 1)) { hw_dbg(hw, "Buffer too small for PBA data.\n"); - return I40E_ERR_PARAM; + return -EINVAL; } for (i = 0; i < pba_size; i++) { @@ -955,7 +955,7 @@ static int i40e_poll_globr(struct i40e_hw *hw, hw_dbg(hw, "Global reset failed.\n"); hw_dbg(hw, "I40E_GLGEN_RSTAT = 0x%x\n", reg); - return I40E_ERR_RESET_FAILED; + return -EIO; } #define I40E_PF_RESET_WAIT_COUNT_A0 200 @@ -995,7 +995,7 @@ int i40e_pf_reset(struct i40e_hw *hw) } if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { hw_dbg(hw, "Global reset polling failed to complete.\n"); - return I40E_ERR_RESET_FAILED; + return -EIO; } /* Now Wait for the FW to be ready */ @@ -1014,7 +1014,7 @@ int i40e_pf_reset(struct i40e_hw *hw) I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { hw_dbg(hw, "wait for FW Reset complete timedout\n"); hw_dbg(hw, "I40E_GLNVM_ULD = 0x%x\n", reg); - return I40E_ERR_RESET_FAILED; + return -EIO; } /* If there was a Global Reset in progress when we got here, @@ -1040,10 +1040,10 @@ int i40e_pf_reset(struct i40e_hw *hw) } if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { if (i40e_poll_globr(hw, grst_del)) - return I40E_ERR_RESET_FAILED; + return -EIO; } else if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { hw_dbg(hw, "PF reset polling failed to complete.\n"); - return I40E_ERR_RESET_FAILED; + return -EIO; } } @@ -1318,7 +1318,7 @@ i40e_aq_get_phy_capabilities(struct i40e_hw *hw, int status; if (!abilities) - return I40E_ERR_PARAM; + return -EINVAL; do { i40e_fill_default_direct_cmd_desc(&desc, @@ -1341,12 +1341,12 @@ i40e_aq_get_phy_capabilities(struct i40e_hw *hw, switch (hw->aq.asq_last_status) { case I40E_AQ_RC_EIO: - status = I40E_ERR_UNKNOWN_PHY; + status = -EIO; break; case I40E_AQ_RC_EAGAIN: usleep_range(1000, 2000); total_delay++; - status = I40E_ERR_TIMEOUT; + status = -EIO; break; /* also covers I40E_AQ_RC_OK */ default: @@ -1396,7 +1396,7 @@ int i40e_aq_set_phy_config(struct i40e_hw *hw, int status; if (!config) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_phy_config); @@ -2312,7 +2312,7 @@ int i40e_aq_send_driver_version(struct i40e_hw *hw, u16 len; if (dv == NULL) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_driver_version); @@ -2430,7 +2430,7 @@ int i40e_aq_add_veb(struct i40e_hw *hw, u16 uplink_seid, /* SEIDs need to either both be set or both be 0 for floating VEB */ if (!!uplink_seid != !!downlink_seid) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_add_veb); @@ -2485,7 +2485,7 @@ int i40e_aq_get_veb_parameters(struct i40e_hw *hw, int status; if (veb_seid == 0) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_get_veb_parameters); @@ -2575,7 +2575,7 @@ i40e_aq_add_macvlan(struct i40e_hw *hw, u16 seid, u16 buf_size; if (count == 0 || !mv_list || !hw) - return I40E_ERR_PARAM; + return -EINVAL; buf_size = i40e_prepare_add_macvlan(mv_list, &desc, count, seid); @@ -2608,7 +2608,7 @@ i40e_aq_add_macvlan_v2(struct i40e_hw *hw, u16 seid, u16 buf_size; if (count == 0 || !mv_list || !hw) - return I40E_ERR_PARAM; + return -EINVAL; buf_size = i40e_prepare_add_macvlan(mv_list, &desc, count, seid); @@ -2638,7 +2638,7 @@ i40e_aq_remove_macvlan(struct i40e_hw *hw, u16 seid, int status; if (count == 0 || !mv_list || !hw) - return I40E_ERR_PARAM; + return -EINVAL; buf_size = count * sizeof(*mv_list); @@ -2685,7 +2685,7 @@ i40e_aq_remove_macvlan_v2(struct i40e_hw *hw, u16 seid, u16 buf_size; if (count == 0 || !mv_list || !hw) - return I40E_ERR_PARAM; + return -EINVAL; buf_size = count * sizeof(*mv_list); @@ -2791,7 +2791,7 @@ int i40e_aq_add_mirrorrule(struct i40e_hw *hw, u16 sw_seid, if (!(rule_type == I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS || rule_type == I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS)) { if (count == 0 || !mr_list) - return I40E_ERR_PARAM; + return -EINVAL; } return i40e_mirrorrule_op(hw, i40e_aqc_opc_add_mirror_rule, sw_seid, @@ -2827,7 +2827,7 @@ int i40e_aq_delete_mirrorrule(struct i40e_hw *hw, u16 sw_seid, * not matter. */ if (count == 0 || !mr_list) - return I40E_ERR_PARAM; + return -EINVAL; } return i40e_mirrorrule_op(hw, i40e_aqc_opc_delete_mirror_rule, sw_seid, @@ -2892,7 +2892,7 @@ int i40e_aq_debug_read_register(struct i40e_hw *hw, int status; if (reg_val == NULL) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_debug_read_reg); @@ -3031,7 +3031,7 @@ int i40e_aq_read_nvm(struct i40e_hw *hw, u8 module_pointer, /* In offset the highest byte must be zeroed. */ if (offset & 0xFF000000) { - status = I40E_ERR_PARAM; + status = -EINVAL; goto i40e_aq_read_nvm_exit; } @@ -3076,7 +3076,7 @@ int i40e_aq_erase_nvm(struct i40e_hw *hw, u8 module_pointer, /* In offset the highest byte must be zeroed. */ if (offset & 0xFF000000) { - status = I40E_ERR_PARAM; + status = -EINVAL; goto i40e_aq_erase_nvm_exit; } @@ -3368,7 +3368,7 @@ int i40e_aq_discover_capabilities(struct i40e_hw *hw, if (list_type_opc != i40e_aqc_opc_list_func_capabilities && list_type_opc != i40e_aqc_opc_list_dev_capabilities) { - status = I40E_ERR_PARAM; + status = -EINVAL; goto exit; } @@ -3416,7 +3416,7 @@ int i40e_aq_update_nvm(struct i40e_hw *hw, u8 module_pointer, /* In offset the highest byte must be zeroed. */ if (offset & 0xFF000000) { - status = I40E_ERR_PARAM; + status = -EINVAL; goto i40e_aq_update_nvm_exit; } @@ -3473,7 +3473,7 @@ int i40e_aq_rearrange_nvm(struct i40e_hw *hw, I40E_AQ_NVM_REARRANGE_TO_STRUCT); if (!rearrange_nvm) { - status = I40E_ERR_PARAM; + status = -EINVAL; goto i40e_aq_rearrange_nvm_exit; } @@ -3510,7 +3510,7 @@ int i40e_aq_get_lldp_mib(struct i40e_hw *hw, u8 bridge_type, int status; if (buff_size == 0 || !buff) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_lldp_get_mib); /* Indirect Command */ @@ -3558,7 +3558,7 @@ i40e_aq_set_lldp_mib(struct i40e_hw *hw, cmd = (struct i40e_aqc_lldp_set_local_mib *)&desc.params.raw; if (buff_size == 0 || !buff) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_lldp_set_local_mib); @@ -3627,7 +3627,7 @@ i40e_aq_restore_lldp(struct i40e_hw *hw, u8 *setting, bool restore, if (!(hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT)) { i40e_debug(hw, I40E_DEBUG_ALL, "Restore LLDP not supported by current FW version.\n"); - return I40E_ERR_DEVICE_NOT_SUPPORTED; + return -ENODEV; } i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_lldp_restore); @@ -3729,7 +3729,7 @@ i40e_aq_set_dcb_parameters(struct i40e_hw *hw, bool dcb_enable, int status; if (!(hw->flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) - return I40E_ERR_DEVICE_NOT_SUPPORTED; + return -ENODEV; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_dcb_parameters); @@ -3760,7 +3760,7 @@ int i40e_aq_get_cee_dcb_config(struct i40e_hw *hw, int status; if (buff_size == 0 || !buff) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_get_cee_dcb_cfg); @@ -3848,7 +3848,7 @@ int i40e_aq_delete_element(struct i40e_hw *hw, u16 seid, int status; if (seid == 0) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_delete_element); @@ -3922,7 +3922,7 @@ static int i40e_aq_tx_sched_cmd(struct i40e_hw *hw, u16 seid, cmd_param_flag = false; break; default: - return I40E_ERR_PARAM; + return -EINVAL; } i40e_fill_default_direct_cmd_desc(&desc, opcode); @@ -4148,7 +4148,7 @@ i40e_validate_filter_settings(struct i40e_hw *hw, fcoe_filt_size <<= (u32)settings->fcoe_filt_num; break; default: - return I40E_ERR_PARAM; + return -EINVAL; } switch (settings->fcoe_cntx_num) { @@ -4160,7 +4160,7 @@ i40e_validate_filter_settings(struct i40e_hw *hw, fcoe_cntx_size <<= (u32)settings->fcoe_cntx_num; break; default: - return I40E_ERR_PARAM; + return -EINVAL; } /* Validate PE settings passed */ @@ -4178,7 +4178,7 @@ i40e_validate_filter_settings(struct i40e_hw *hw, case I40E_HASH_FILTER_SIZE_1M: break; default: - return I40E_ERR_PARAM; + return -EINVAL; } switch (settings->pe_cntx_num) { @@ -4194,7 +4194,7 @@ i40e_validate_filter_settings(struct i40e_hw *hw, case I40E_DMA_CNTX_SIZE_256K: break; default: - return I40E_ERR_PARAM; + return -EINVAL; } /* FCHSIZE + FCDSIZE should not be greater than PMFCOEFMAX */ @@ -4202,7 +4202,7 @@ i40e_validate_filter_settings(struct i40e_hw *hw, fcoe_fmax = (val & I40E_GLHMC_FCOEFMAX_PMFCOEFMAX_MASK) >> I40E_GLHMC_FCOEFMAX_PMFCOEFMAX_SHIFT; if (fcoe_filt_size + fcoe_cntx_size > fcoe_fmax) - return I40E_ERR_INVALID_SIZE; + return -EINVAL; return 0; } @@ -4224,7 +4224,7 @@ int i40e_set_filter_control(struct i40e_hw *hw, u32 val; if (!settings) - return I40E_ERR_PARAM; + return -EINVAL; /* Validate the input settings */ ret = i40e_validate_filter_settings(hw, settings); @@ -4306,7 +4306,7 @@ int i40e_aq_add_rem_control_packet_filter(struct i40e_hw *hw, int status; if (vsi_seid == 0) - return I40E_ERR_PARAM; + return -EINVAL; if (is_add) { i40e_fill_default_direct_cmd_desc(&desc, @@ -4381,7 +4381,7 @@ static int i40e_aq_alternate_read(struct i40e_hw *hw, int status; if (!reg_val0) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_alternate_read); cmd_resp->address0 = cpu_to_le32(reg_addr0); @@ -4517,7 +4517,7 @@ int i40e_aq_debug_dump(struct i40e_hw *hw, u8 cluster_id, int status; if (buff_size == 0 || !buff) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_debug_dump_internals); @@ -4635,7 +4635,7 @@ int i40e_read_phy_register_clause22(struct i40e_hw *hw, u16 reg, u8 phy_addr, u16 *value) { u8 port_num = (u8)hw->func_caps.mdio_port_num; - int status = I40E_ERR_TIMEOUT; + int status = -EIO; u32 command = 0; u16 retry = 1000; @@ -4680,7 +4680,7 @@ int i40e_write_phy_register_clause22(struct i40e_hw *hw, u16 reg, u8 phy_addr, u16 value) { u8 port_num = (u8)hw->func_caps.mdio_port_num; - int status = I40E_ERR_TIMEOUT; + int status = -EIO; u32 command = 0; u16 retry = 1000; @@ -4721,7 +4721,7 @@ int i40e_read_phy_register_clause45(struct i40e_hw *hw, u8 page, u16 reg, u8 phy_addr, u16 *value) { u8 port_num = hw->func_caps.mdio_port_num; - int status = I40E_ERR_TIMEOUT; + int status = -EIO; u32 command = 0; u16 retry = 1000; @@ -4755,7 +4755,7 @@ int i40e_read_phy_register_clause45(struct i40e_hw *hw, (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); - status = I40E_ERR_TIMEOUT; + status = -EIO; retry = 1000; wr32(hw, I40E_GLGEN_MSCA(port_num), command); do { @@ -4795,7 +4795,7 @@ int i40e_write_phy_register_clause45(struct i40e_hw *hw, u8 page, u16 reg, u8 phy_addr, u16 value) { u8 port_num = hw->func_caps.mdio_port_num; - int status = I40E_ERR_TIMEOUT; + int status = -EIO; u16 retry = 1000; u32 command = 0; @@ -4831,7 +4831,7 @@ int i40e_write_phy_register_clause45(struct i40e_hw *hw, (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); - status = I40E_ERR_TIMEOUT; + status = -EIO; retry = 1000; wr32(hw, I40E_GLGEN_MSCA(port_num), command); do { @@ -4880,7 +4880,7 @@ int i40e_write_phy_register(struct i40e_hw *hw, phy_addr, value); break; default: - status = I40E_ERR_UNKNOWN_PHY; + status = -EIO; break; } @@ -4919,7 +4919,7 @@ int i40e_read_phy_register(struct i40e_hw *hw, phy_addr, value); break; default: - status = I40E_ERR_UNKNOWN_PHY; + status = -EIO; break; } @@ -5109,7 +5109,7 @@ int i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr, I40E_PHY_COM_REG_PAGE, true, I40E_PHY_LED_PROV_REG_1, ®_val_aq, NULL); - if (status == I40E_SUCCESS) + if (status == 0) *val = (u16)reg_val_aq; return status; } @@ -5204,7 +5204,7 @@ int i40e_aq_rx_ctl_read_register(struct i40e_hw *hw, int status; if (!reg_val) - return I40E_ERR_PARAM; + return -EINVAL; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_rx_ctl_reg_read); @@ -5644,7 +5644,7 @@ i40e_validate_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, if (track_id == I40E_DDP_TRACKID_INVALID) { i40e_debug(hw, I40E_DEBUG_PACKAGE, "Invalid track_id\n"); - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; } dev_cnt = profile->device_table_count; @@ -5657,7 +5657,7 @@ i40e_validate_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, if (dev_cnt && i == dev_cnt) { i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support DDP\n"); - return I40E_ERR_DEVICE_NOT_SUPPORTED; + return -ENODEV; } I40E_SECTION_TABLE(profile, sec_tbl); @@ -5672,14 +5672,14 @@ i40e_validate_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, sec->section.type == SECTION_TYPE_RB_AQ) { i40e_debug(hw, I40E_DEBUG_PACKAGE, "Not a roll-back package\n"); - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; } } else { if (sec->section.type == SECTION_TYPE_RB_AQ || sec->section.type == SECTION_TYPE_RB_MMIO) { i40e_debug(hw, I40E_DEBUG_PACKAGE, "Not an original package\n"); - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; } } } diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb.c b/drivers/net/ethernet/intel/i40e/i40e_dcb.c index 90638b67f8dc..f81e744c0fb3 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_dcb.c +++ b/drivers/net/ethernet/intel/i40e/i40e_dcb.c @@ -17,7 +17,7 @@ int i40e_get_dcbx_status(struct i40e_hw *hw, u16 *status) u32 reg; if (!status) - return I40E_ERR_PARAM; + return -EINVAL; reg = rd32(hw, I40E_PRTDCB_GENS); *status = (u16)((reg & I40E_PRTDCB_GENS_DCBX_STATUS_MASK) >> @@ -508,7 +508,7 @@ int i40e_lldp_to_dcb_config(u8 *lldpmib, u16 type; if (!lldpmib || !dcbcfg) - return I40E_ERR_PARAM; + return -EINVAL; /* set to the start of LLDPDU */ lldpmib += ETH_HLEN; @@ -874,7 +874,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change) int ret = 0; if (!hw->func_caps.dcb) - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; /* Read LLDP NVM area */ if (hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT) { @@ -885,7 +885,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change) else if (hw->mac.type == I40E_MAC_X722) offset = I40E_LLDP_CURRENT_STATUS_X722_OFFSET; else - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; ret = i40e_read_nvm_module_data(hw, I40E_SR_EMP_SR_SETTINGS_PTR, @@ -897,7 +897,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change) ret = i40e_read_lldp_cfg(hw, &lldp_cfg); } if (ret) - return I40E_ERR_NOT_READY; + return -EBUSY; /* Get the LLDP AdminStatus for the current port */ adminstatus = lldp_cfg.adminstatus >> (hw->port * 4); @@ -906,7 +906,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change) /* LLDP agent disabled */ if (!adminstatus) { hw->dcbx_status = I40E_DCBX_STATUS_DISABLED; - return I40E_ERR_NOT_READY; + return -EBUSY; } /* Get DCBX status */ @@ -922,7 +922,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change) if (ret) return ret; } else if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) { - return I40E_ERR_NOT_READY; + return -EBUSY; } /* Configure the LLDP MIB change event */ @@ -949,7 +949,7 @@ i40e_get_fw_lldp_status(struct i40e_hw *hw, int ret; if (!lldp_status) - return I40E_ERR_PARAM; + return -EINVAL; /* Allocate buffer for the LLDPDU */ ret = i40e_allocate_virt_mem(hw, &mem, I40E_LLDPDU_SIZE); @@ -1299,7 +1299,7 @@ int i40e_dcb_config_to_lldp(u8 *lldpmib, u16 *miblen, sizeof(tlv->typelength) + length); } while (tlvid < I40E_TLV_ID_END_OF_LLDPPDU); *miblen = offset; - return I40E_SUCCESS; + return 0; } /** @@ -1957,7 +1957,7 @@ int i40e_read_lldp_cfg(struct i40e_hw *hw, u32 mem; if (!lldp_cfg) - return I40E_ERR_PARAM; + return -EINVAL; ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); if (ret) diff --git a/drivers/net/ethernet/intel/i40e/i40e_ddp.c b/drivers/net/ethernet/intel/i40e/i40e_ddp.c index 7e8183762fd9..969120587cad 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ddp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ddp.c @@ -344,7 +344,7 @@ int i40e_ddp_load(struct net_device *netdev, const u8 *data, size_t size, if (is_add) { status = i40e_write_profile(&pf->hw, profile_hdr, track_id); if (status) { - if (status == I40E_ERR_DEVICE_NOT_SUPPORTED) { + if (status == -ENODEV) { netdev_err(netdev, "Profile is not supported by the device."); return -EPERM; diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c index 62497f5565c5..1a497cb07710 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c +++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c @@ -1309,7 +1309,7 @@ static ssize_t i40e_dbg_command_write(struct file *filp, ret = i40e_asq_send_command(&pf->hw, desc, NULL, 0, NULL); if (!ret) { dev_info(&pf->pdev->dev, "AQ command sent Status : Success\n"); - } else if (ret == I40E_ERR_ADMIN_QUEUE_ERROR) { + } else if (ret == -EIO) { dev_info(&pf->pdev->dev, "AQ command send failed Opcode %x AQ Error: %d\n", desc->opcode, pf->hw.aq.asq_last_status); @@ -1370,7 +1370,7 @@ static ssize_t i40e_dbg_command_write(struct file *filp, buffer_len, NULL); if (!ret) { dev_info(&pf->pdev->dev, "AQ command sent Status : Success\n"); - } else if (ret == I40E_ERR_ADMIN_QUEUE_ERROR) { + } else if (ret == -EIO) { dev_info(&pf->pdev->dev, "AQ command send failed Opcode %x AQ Error: %d\n", desc->opcode, pf->hw.aq.asq_last_status); diff --git a/drivers/net/ethernet/intel/i40e/i40e_diag.c b/drivers/net/ethernet/intel/i40e/i40e_diag.c index 97fe1787a8f4..b1ad7c4259b9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_diag.c +++ b/drivers/net/ethernet/intel/i40e/i40e_diag.c @@ -28,7 +28,7 @@ static int i40e_diag_reg_pattern_test(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_DIAG, "%s: reg pattern test failed - reg 0x%08x pat 0x%08x val 0x%08x\n", __func__, reg, pat, val); - return I40E_ERR_DIAG_TEST_FAILED; + return -EIO; } } @@ -38,7 +38,7 @@ static int i40e_diag_reg_pattern_test(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_DIAG, "%s: reg restore test failed - reg 0x%08x orig_val 0x%08x val 0x%08x\n", __func__, reg, orig_val, val); - return I40E_ERR_DIAG_TEST_FAILED; + return -EIO; } return 0; @@ -127,5 +127,5 @@ int i40e_diag_eeprom_test(struct i40e_hw *hw) BIT(I40E_SR_CONTROL_WORD_1_SHIFT))) return i40e_validate_nvm_checksum(hw, NULL); else - return I40E_ERR_DIAG_TEST_FAILED; + return -EIO; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index afc4fa8c66af..bd1321bf7e26 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5699,8 +5699,8 @@ static int i40e_set_eee(struct net_device *netdev, struct ethtool_eee *edata) struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; - int status = I40E_SUCCESS; __le16 eee_capability; + int status = 0; /* Deny parameters we don't support */ if (i40e_is_eee_param_supported(netdev, edata)) diff --git a/drivers/net/ethernet/intel/i40e/i40e_hmc.c b/drivers/net/ethernet/intel/i40e/i40e_hmc.c index 46f7950a0049..96ee63aca7a1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_hmc.c +++ b/drivers/net/ethernet/intel/i40e/i40e_hmc.c @@ -4,7 +4,6 @@ #include "i40e.h" #include "i40e_osdep.h" #include "i40e_register.h" -#include "i40e_status.h" #include "i40e_alloc.h" #include "i40e_hmc.h" #include "i40e_type.h" @@ -26,18 +25,18 @@ int i40e_add_sd_table_entry(struct i40e_hw *hw, enum i40e_memory_type mem_type __attribute__((unused)); struct i40e_hmc_sd_entry *sd_entry; bool dma_mem_alloc_done = false; - int ret_code = I40E_SUCCESS; struct i40e_dma_mem mem; + int ret_code = 0; u64 alloc_len; if (NULL == hmc_info->sd_table.sd_entry) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_entry\n"); goto exit; } if (sd_index >= hmc_info->sd_table.sd_cnt) { - ret_code = I40E_ERR_INVALID_SD_INDEX; + ret_code = -EINVAL; hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_index\n"); goto exit; } @@ -121,7 +120,7 @@ int i40e_add_pd_table_entry(struct i40e_hw *hw, u64 *pd_addr; if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) { - ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX; + ret_code = -EINVAL; hw_dbg(hw, "i40e_add_pd_table_entry: bad pd_index\n"); goto exit; } @@ -200,13 +199,13 @@ int i40e_remove_pd_bp(struct i40e_hw *hw, sd_idx = idx / I40E_HMC_PD_CNT_IN_SD; rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD; if (sd_idx >= hmc_info->sd_table.sd_cnt) { - ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX; + ret_code = -EINVAL; hw_dbg(hw, "i40e_remove_pd_bp: bad idx\n"); goto exit; } sd_entry = &hmc_info->sd_table.sd_entry[sd_idx]; if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) { - ret_code = I40E_ERR_INVALID_SD_TYPE; + ret_code = -EINVAL; hw_dbg(hw, "i40e_remove_pd_bp: wrong sd_entry type\n"); goto exit; } @@ -251,7 +250,7 @@ int i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info, sd_entry = &hmc_info->sd_table.sd_entry[idx]; I40E_DEC_BP_REFCNT(&sd_entry->u.bp); if (sd_entry->u.bp.ref_cnt) { - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto exit; } I40E_DEC_SD_REFCNT(&hmc_info->sd_table); @@ -276,7 +275,7 @@ int i40e_remove_sd_bp_new(struct i40e_hw *hw, struct i40e_hmc_sd_entry *sd_entry; if (!is_pf) - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; /* get the entry and decrease its ref counter */ sd_entry = &hmc_info->sd_table.sd_entry[idx]; @@ -299,7 +298,7 @@ int i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info, sd_entry = &hmc_info->sd_table.sd_entry[idx]; if (sd_entry->u.pd_table.ref_cnt) { - ret_code = I40E_ERR_NOT_READY; + ret_code = -EBUSY; goto exit; } @@ -325,7 +324,7 @@ int i40e_remove_pd_page_new(struct i40e_hw *hw, struct i40e_hmc_sd_entry *sd_entry; if (!is_pf) - return I40E_NOT_SUPPORTED; + return -EOPNOTSUPP; sd_entry = &hmc_info->sd_table.sd_entry[idx]; I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED); diff --git a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c index 40c101f286d1..474365bf0648 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c +++ b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c @@ -111,7 +111,7 @@ int i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num, /* validate values requested by driver don't exceed HMC capacity */ if (txq_num > obj->max_cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_init_lan_hmc: Tx context: asks for 0x%x but max allowed is 0x%x, returns error %d\n", txq_num, obj->max_cnt, ret_code); goto init_lan_hmc_out; @@ -134,7 +134,7 @@ int i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num, /* validate values requested by driver don't exceed HMC capacity */ if (rxq_num > obj->max_cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_init_lan_hmc: Rx context: asks for 0x%x but max allowed is 0x%x, returns error %d\n", rxq_num, obj->max_cnt, ret_code); goto init_lan_hmc_out; @@ -157,7 +157,7 @@ int i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num, /* validate values requested by driver don't exceed HMC capacity */ if (fcoe_cntx_num > obj->max_cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_init_lan_hmc: FCoE context: asks for 0x%x but max allowed is 0x%x, returns error %d\n", fcoe_cntx_num, obj->max_cnt, ret_code); goto init_lan_hmc_out; @@ -180,7 +180,7 @@ int i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num, /* validate values requested by driver don't exceed HMC capacity */ if (fcoe_filt_num > obj->max_cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_init_lan_hmc: FCoE filter: asks for 0x%x but max allowed is 0x%x, returns error %d\n", fcoe_filt_num, obj->max_cnt, ret_code); goto init_lan_hmc_out; @@ -289,30 +289,30 @@ static int i40e_create_lan_hmc_object(struct i40e_hw *hw, u32 i, j; if (NULL == info) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_create_lan_hmc_object: bad info ptr\n"); goto exit; } if (NULL == info->hmc_info) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_create_lan_hmc_object: bad hmc_info ptr\n"); goto exit; } if (I40E_HMC_INFO_SIGNATURE != info->hmc_info->signature) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_create_lan_hmc_object: bad signature\n"); goto exit; } if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_INDEX; + ret_code = -EINVAL; hw_dbg(hw, "i40e_create_lan_hmc_object: returns error %d\n", ret_code); goto exit; } if ((info->start_idx + info->count) > info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_create_lan_hmc_object: returns error %d\n", ret_code); goto exit; @@ -324,8 +324,8 @@ static int i40e_create_lan_hmc_object(struct i40e_hw *hw, &sd_idx, &sd_lmt); if (sd_idx >= info->hmc_info->sd_table.sd_cnt || sd_lmt > info->hmc_info->sd_table.sd_cnt) { - ret_code = I40E_ERR_INVALID_SD_INDEX; - goto exit; + ret_code = -EINVAL; + goto exit; } /* find pd index */ I40E_FIND_PD_INDEX_LIMIT(info->hmc_info, info->rsrc_type, @@ -393,7 +393,7 @@ static int i40e_create_lan_hmc_object(struct i40e_hw *hw, j, sd_entry->entry_type); break; default: - ret_code = I40E_ERR_INVALID_SD_TYPE; + ret_code = -EINVAL; goto exit; } } @@ -417,7 +417,7 @@ exit_sd_error: i40e_remove_sd_bp(hw, info->hmc_info, (j - 1)); break; default: - ret_code = I40E_ERR_INVALID_SD_TYPE; + ret_code = -EINVAL; break; } j--; @@ -474,7 +474,7 @@ try_type_paged: break; default: /* unsupported type */ - ret_code = I40E_ERR_INVALID_SD_TYPE; + ret_code = -EINVAL; hw_dbg(hw, "i40e_configure_lan_hmc: Unknown SD type: %d\n", ret_code); goto configure_lan_hmc_out; @@ -530,34 +530,34 @@ static int i40e_delete_lan_hmc_object(struct i40e_hw *hw, u32 i, j; if (NULL == info) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: bad info ptr\n"); goto exit; } if (NULL == info->hmc_info) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: bad info->hmc_info ptr\n"); goto exit; } if (I40E_HMC_INFO_SIGNATURE != info->hmc_info->signature) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: bad hmc_info->signature\n"); goto exit; } if (NULL == info->hmc_info->sd_table.sd_entry) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: bad sd_entry\n"); goto exit; } if (NULL == info->hmc_info->hmc_obj) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: bad hmc_info->hmc_obj\n"); goto exit; } if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_INDEX; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: returns error %d\n", ret_code); goto exit; @@ -565,7 +565,7 @@ static int i40e_delete_lan_hmc_object(struct i40e_hw *hw, if ((info->start_idx + info->count) > info->hmc_info->hmc_obj[info->rsrc_type].cnt) { - ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT; + ret_code = -EINVAL; hw_dbg(hw, "i40e_delete_hmc_object: returns error %d\n", ret_code); goto exit; @@ -599,7 +599,7 @@ static int i40e_delete_lan_hmc_object(struct i40e_hw *hw, &sd_idx, &sd_lmt); if (sd_idx >= info->hmc_info->sd_table.sd_cnt || sd_lmt > info->hmc_info->sd_table.sd_cnt) { - ret_code = I40E_ERR_INVALID_SD_INDEX; + ret_code = -EINVAL; goto exit; } @@ -987,29 +987,29 @@ int i40e_hmc_get_object_va(struct i40e_hw *hw, u8 **object_base, int ret_code = 0; if (NULL == hmc_info) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_hmc_get_object_va: bad hmc_info ptr\n"); goto exit; } if (NULL == hmc_info->hmc_obj) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_hmc_get_object_va: bad hmc_info->hmc_obj ptr\n"); goto exit; } if (NULL == object_base) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_hmc_get_object_va: bad object_base ptr\n"); goto exit; } if (I40E_HMC_INFO_SIGNATURE != hmc_info->signature) { - ret_code = I40E_ERR_BAD_PTR; + ret_code = -EINVAL; hw_dbg(hw, "i40e_hmc_get_object_va: bad hmc_info->signature\n"); goto exit; } if (obj_idx >= hmc_info->hmc_obj[rsrc_type].cnt) { hw_dbg(hw, "i40e_hmc_get_object_va: returns error %d\n", ret_code); - ret_code = I40E_ERR_INVALID_HMC_OBJ_INDEX; + ret_code = -EINVAL; goto exit; } /* find sd index and limit */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 29ad1797adce..a5ba873c3b24 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -3585,11 +3585,6 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) if (ring->xsk_pool) { ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); - /* For AF_XDP ZC, we disallow packets to span on - * multiple buffers, thus letting us skip that - * handling in the fast-path. - */ - chain_len = 1; ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); @@ -5714,7 +5709,7 @@ int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset) int ret; if (!vsi) - return I40E_ERR_PARAM; + return -EINVAL; pf = vsi->back; hw = &pf->hw; @@ -7158,7 +7153,7 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf) */ if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) { dev_info(&pf->pdev->dev, "DCB is not supported.\n"); - err = I40E_NOT_SUPPORTED; + err = -EOPNOTSUPP; goto out; } if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) { @@ -7468,7 +7463,7 @@ static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) non_zero_phy_type = true; else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) - return I40E_SUCCESS; + return 0; /* To force link we need to set bits for all supported PHY types, * but there are now more than 32, so we need to split the bitmap @@ -7519,7 +7514,7 @@ static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) i40e_aq_set_link_restart_an(hw, is_up, NULL); - return I40E_SUCCESS; + return 0; } /** @@ -8366,7 +8361,7 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, }; if (filter->flags >= ARRAY_SIZE(flag_table)) - return I40E_ERR_CONFIG; + return -EIO; memset(&cld_filter, 0, sizeof(cld_filter)); @@ -8530,15 +8525,15 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, u8 field_flags = 0; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { - dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { + dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } @@ -8580,7 +8575,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", match.mask->dst); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -8590,7 +8585,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", match.mask->src); - return I40E_ERR_CONFIG; + return -EIO; } } ether_addr_copy(filter->dst_mac, match.key->dst); @@ -8608,7 +8603,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", match.mask->vlan_id); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -8632,7 +8627,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", &match.mask->dst); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -8642,13 +8637,13 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", &match.mask->src); - return I40E_ERR_CONFIG; + return -EIO; } } if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); - return I40E_ERR_CONFIG; + return -EIO; } filter->dst_ipv4 = match.key->dst; filter->src_ipv4 = match.key->src; @@ -8666,7 +8661,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, ipv6_addr_loopback(&match.key->src)) { dev_err(&pf->pdev->dev, "Bad ipv6, addr is LOOPBACK\n"); - return I40E_ERR_CONFIG; + return -EIO; } if (!ipv6_addr_any(&match.mask->dst) || !ipv6_addr_any(&match.mask->src)) @@ -8688,7 +8683,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", be16_to_cpu(match.mask->src)); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -8698,7 +8693,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi, } else { dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", be16_to_cpu(match.mask->dst)); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -9906,11 +9901,11 @@ static void i40e_link_event(struct i40e_pf *pf) status = i40e_get_link_status(&pf->hw, &new_link); /* On success, disable temp link polling */ - if (status == I40E_SUCCESS) { + if (status == 0) { clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); } else { /* Enable link polling temporarily until i40e_get_link_status - * returns I40E_SUCCESS + * returns 0 */ set_bit(__I40E_TEMP_LINK_POLLING, pf->state); dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", @@ -10164,7 +10159,7 @@ static void i40e_clean_adminq_subtask(struct i40e_pf *pf) do { ret = i40e_clean_arq_element(hw, &event, &pending); - if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) + if (ret == -EALREADY) break; else if (ret) { dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); @@ -12574,7 +12569,7 @@ int i40e_commit_partition_bw_setting(struct i40e_pf *pf) dev_info(&pf->pdev->dev, "Commit BW only works on partition 1! This is partition %d", pf->hw.partition_id); - ret = I40E_NOT_SUPPORTED; + ret = -EOPNOTSUPP; goto bw_commit_out; } @@ -12656,10 +12651,10 @@ static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 - int read_status = I40E_SUCCESS; u16 sr_emp_sr_settings_ptr = 0; u16 features_enable = 0; u16 link_behavior = 0; + int read_status = 0; bool ret = false; read_status = i40e_read_nvm_word(&pf->hw, @@ -13822,6 +13817,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi) NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_XSK_ZEROCOPY | NETDEV_XDP_ACT_RX_SG; + netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD; } else { /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to @@ -15466,12 +15462,12 @@ static int i40e_pf_loop_reset(struct i40e_pf *pf) int ret; ret = i40e_pf_reset(hw); - while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) { + while (ret != 0 && time_before(jiffies, time_end)) { usleep_range(10000, 20000); ret = i40e_pf_reset(hw); } - if (ret == I40E_SUCCESS) + if (ret == 0) pf->pfr_count++; else dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); @@ -15514,10 +15510,10 @@ static int i40e_handle_resets(struct i40e_pf *pf) const int pfr = i40e_pf_loop_reset(pf); const bool is_empr = i40e_check_fw_empr(pf); - if (is_empr || pfr != I40E_SUCCESS) + if (is_empr || pfr != 0) dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); - return is_empr ? I40E_ERR_RESET_FAILED : pfr; + return is_empr ? -EIO : pfr; } /** @@ -15810,7 +15806,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = i40e_init_adminq(hw); if (err) { - if (err == I40E_ERR_FIRMWARE_API_VERSION) + if (err == -EIO) dev_info(&pdev->dev, "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", hw->aq.api_maj_ver, diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c index 9da0c87f0328..4bb300fbb777 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c +++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c @@ -37,7 +37,7 @@ int i40e_init_nvm(struct i40e_hw *hw) nvm->blank_nvm_mode = false; } else { /* Blank programming mode */ nvm->blank_nvm_mode = true; - ret_code = I40E_ERR_NVM_BLANK_MODE; + ret_code = -EIO; i40e_debug(hw, I40E_DEBUG_NVM, "NVM init error: unsupported blank mode.\n"); } @@ -111,8 +111,8 @@ i40e_i40e_acquire_nvm_exit: **/ void i40e_release_nvm(struct i40e_hw *hw) { - int ret_code = I40E_SUCCESS; u32 total_delay = 0; + int ret_code = 0; if (hw->nvm.blank_nvm_mode) return; @@ -122,7 +122,7 @@ void i40e_release_nvm(struct i40e_hw *hw) /* there are some rare cases when trying to release the resource * results in an admin Q timeout, so handle them correctly */ - while ((ret_code == I40E_ERR_ADMIN_QUEUE_TIMEOUT) && + while ((ret_code == -EIO) && (total_delay < hw->aq.asq_cmd_timeout)) { usleep_range(1000, 2000); ret_code = i40e_aq_release_resource(hw, @@ -140,7 +140,7 @@ void i40e_release_nvm(struct i40e_hw *hw) **/ static int i40e_poll_sr_srctl_done_bit(struct i40e_hw *hw) { - int ret_code = I40E_ERR_TIMEOUT; + int ret_code = -EIO; u32 srctl, wait_cnt; /* Poll the I40E_GLNVM_SRCTL until the done bit is set */ @@ -152,7 +152,7 @@ static int i40e_poll_sr_srctl_done_bit(struct i40e_hw *hw) } udelay(5); } - if (ret_code == I40E_ERR_TIMEOUT) + if (ret_code == -EIO) i40e_debug(hw, I40E_DEBUG_NVM, "Done bit in GLNVM_SRCTL not set"); return ret_code; } @@ -168,14 +168,14 @@ static int i40e_poll_sr_srctl_done_bit(struct i40e_hw *hw) static int i40e_read_nvm_word_srctl(struct i40e_hw *hw, u16 offset, u16 *data) { - int ret_code = I40E_ERR_TIMEOUT; + int ret_code = -EIO; u32 sr_reg; if (offset >= hw->nvm.sr_size) { i40e_debug(hw, I40E_DEBUG_NVM, "NVM read error: offset %d beyond Shadow RAM limit %d\n", offset, hw->nvm.sr_size); - ret_code = I40E_ERR_PARAM; + ret_code = -EINVAL; goto read_nvm_exit; } @@ -222,7 +222,7 @@ static int i40e_read_nvm_aq(struct i40e_hw *hw, bool last_command) { struct i40e_asq_cmd_details cmd_details; - int ret_code = I40E_ERR_NVM; + int ret_code = -EIO; memset(&cmd_details, 0, sizeof(cmd_details)); cmd_details.wb_desc = &hw->nvm_wb_desc; @@ -267,7 +267,7 @@ static int i40e_read_nvm_aq(struct i40e_hw *hw, static int i40e_read_nvm_word_aq(struct i40e_hw *hw, u16 offset, u16 *data) { - int ret_code = I40E_ERR_TIMEOUT; + int ret_code = -EIO; ret_code = i40e_read_nvm_aq(hw, 0x0, offset, 1, data, true); *data = le16_to_cpu(*(__le16 *)data); @@ -348,7 +348,7 @@ int i40e_read_nvm_module_data(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_ALL, "Reading nvm word failed.Error code: %d.\n", status); - return I40E_ERR_NVM; + return -EIO; } } #define I40E_NVM_INVALID_PTR_VAL 0x7FFF @@ -358,7 +358,7 @@ int i40e_read_nvm_module_data(struct i40e_hw *hw, if (ptr_value == I40E_NVM_INVALID_PTR_VAL || ptr_value == I40E_NVM_INVALID_VAL) { i40e_debug(hw, I40E_DEBUG_ALL, "Pointer not initialized.\n"); - return I40E_ERR_BAD_PTR; + return -EINVAL; } /* Check whether the module is in SR mapped area or outside */ @@ -367,7 +367,7 @@ int i40e_read_nvm_module_data(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_ALL, "Reading nvm data failed. Pointer points outside of the Shared RAM mapped area.\n"); - return I40E_ERR_PARAM; + return -EINVAL; } else { /* Read from the Shadow RAM */ @@ -377,7 +377,7 @@ int i40e_read_nvm_module_data(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_ALL, "Reading nvm word failed.Error code: %d.\n", status); - return I40E_ERR_NVM; + return -EIO; } offset = ptr_value + module_offset + specific_ptr + @@ -549,7 +549,7 @@ static int i40e_write_nvm_aq(struct i40e_hw *hw, u8 module_pointer, bool last_command) { struct i40e_asq_cmd_details cmd_details; - int ret_code = I40E_ERR_NVM; + int ret_code = -EIO; memset(&cmd_details, 0, sizeof(cmd_details)); cmd_details.wb_desc = &hw->nvm_wb_desc; @@ -614,7 +614,7 @@ static int i40e_calc_nvm_checksum(struct i40e_hw *hw, /* read pointer to VPD area */ ret_code = __i40e_read_nvm_word(hw, I40E_SR_VPD_PTR, &vpd_module); if (ret_code) { - ret_code = I40E_ERR_NVM_CHECKSUM; + ret_code = -EIO; goto i40e_calc_nvm_checksum_exit; } @@ -622,7 +622,7 @@ static int i40e_calc_nvm_checksum(struct i40e_hw *hw, ret_code = __i40e_read_nvm_word(hw, I40E_SR_PCIE_ALT_AUTO_LOAD_PTR, &pcie_alt_module); if (ret_code) { - ret_code = I40E_ERR_NVM_CHECKSUM; + ret_code = -EIO; goto i40e_calc_nvm_checksum_exit; } @@ -636,7 +636,7 @@ static int i40e_calc_nvm_checksum(struct i40e_hw *hw, ret_code = __i40e_read_nvm_buffer(hw, i, &words, data); if (ret_code) { - ret_code = I40E_ERR_NVM_CHECKSUM; + ret_code = -EIO; goto i40e_calc_nvm_checksum_exit; } } @@ -724,7 +724,7 @@ int i40e_validate_nvm_checksum(struct i40e_hw *hw, * calculated checksum */ if (checksum_local != checksum_sr) - ret_code = I40E_ERR_NVM_CHECKSUM; + ret_code = -EIO; /* If the user cares, return the calculated checksum */ if (checksum) @@ -839,7 +839,7 @@ int i40e_nvmupd_command(struct i40e_hw *hw, if (upd_cmd == I40E_NVMUPD_STATUS) { if (!cmd->data_size) { *perrno = -EFAULT; - return I40E_ERR_BUF_TOO_SHORT; + return -EINVAL; } bytes[0] = hw->nvmupd_state; @@ -896,7 +896,7 @@ int i40e_nvmupd_command(struct i40e_hw *hw, break; } - status = I40E_ERR_NOT_READY; + status = -EBUSY; *perrno = -EBUSY; break; @@ -904,7 +904,7 @@ int i40e_nvmupd_command(struct i40e_hw *hw, /* invalid state, should never happen */ i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: no such state %d\n", hw->nvmupd_state); - status = I40E_NOT_SUPPORTED; + status = -EOPNOTSUPP; *perrno = -ESRCH; break; } @@ -1045,7 +1045,7 @@ static int i40e_nvmupd_state_init(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: bad cmd %s in init state\n", i40e_nvm_update_state_str[upd_cmd]); - status = I40E_ERR_NVM; + status = -EIO; *perrno = -ESRCH; break; } @@ -1087,7 +1087,7 @@ static int i40e_nvmupd_state_reading(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: bad cmd %s in reading state.\n", i40e_nvm_update_state_str[upd_cmd]); - status = I40E_NOT_SUPPORTED; + status = -EOPNOTSUPP; *perrno = -ESRCH; break; } @@ -1174,7 +1174,7 @@ retry: i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: bad cmd %s in writing state.\n", i40e_nvm_update_state_str[upd_cmd]); - status = I40E_NOT_SUPPORTED; + status = -EOPNOTSUPP; *perrno = -ESRCH; break; } @@ -1398,7 +1398,7 @@ static int i40e_nvmupd_exec_aq(struct i40e_hw *hw, "NVMUPD: not enough aq desc bytes for exec, size %d < %d\n", cmd->data_size, aq_desc_len); *perrno = -EINVAL; - return I40E_ERR_PARAM; + return -EINVAL; } aq_desc = (struct i40e_aq_desc *)bytes; @@ -1473,7 +1473,7 @@ static int i40e_nvmupd_get_aq_result(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_NVM, "%s: offset too big %d > %d\n", __func__, cmd->offset, aq_total_len); *perrno = -EINVAL; - return I40E_ERR_PARAM; + return -EINVAL; } /* check copylength range */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c index c37abbb3cd06..8a26811140b4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c @@ -1132,7 +1132,7 @@ int i40e_ptp_alloc_pins(struct i40e_pf *pf) if (!pf->ptp_pins) { dev_warn(&pf->pdev->dev, "Cannot allocate memory for PTP pins structure.\n"); - return -I40E_ERR_NO_MEMORY; + return -ENOMEM; } pf->ptp_pins->sdp3_2 = off; diff --git a/drivers/net/ethernet/intel/i40e/i40e_status.h b/drivers/net/ethernet/intel/i40e/i40e_status.h deleted file mode 100644 index 4d2782e76038..000000000000 --- a/drivers/net/ethernet/intel/i40e/i40e_status.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright(c) 2013 - 2018 Intel Corporation. */ - -#ifndef _I40E_STATUS_H_ -#define _I40E_STATUS_H_ - -/* Error Codes */ -enum i40e_status_code { - I40E_SUCCESS = 0, - I40E_ERR_NVM = -1, - I40E_ERR_NVM_CHECKSUM = -2, - I40E_ERR_CONFIG = -4, - I40E_ERR_PARAM = -5, - I40E_ERR_UNKNOWN_PHY = -7, - I40E_ERR_INVALID_MAC_ADDR = -10, - I40E_ERR_DEVICE_NOT_SUPPORTED = -11, - I40E_ERR_RESET_FAILED = -15, - I40E_ERR_NO_AVAILABLE_VSI = -17, - I40E_ERR_NO_MEMORY = -18, - I40E_ERR_BAD_PTR = -19, - I40E_ERR_INVALID_SIZE = -26, - I40E_ERR_QUEUE_EMPTY = -32, - I40E_ERR_TIMEOUT = -37, - I40E_ERR_INVALID_SD_INDEX = -45, - I40E_ERR_INVALID_PAGE_DESC_INDEX = -46, - I40E_ERR_INVALID_SD_TYPE = -47, - I40E_ERR_INVALID_HMC_OBJ_INDEX = -49, - I40E_ERR_INVALID_HMC_OBJ_COUNT = -50, - I40E_ERR_ADMIN_QUEUE_ERROR = -53, - I40E_ERR_ADMIN_QUEUE_TIMEOUT = -54, - I40E_ERR_BUF_TOO_SHORT = -55, - I40E_ERR_ADMIN_QUEUE_FULL = -56, - I40E_ERR_ADMIN_QUEUE_NO_WORK = -57, - I40E_ERR_NVM_BLANK_MODE = -59, - I40E_ERR_NOT_IMPLEMENTED = -60, - I40E_ERR_DIAG_TEST_FAILED = -62, - I40E_ERR_NOT_READY = -63, - I40E_NOT_SUPPORTED = -64, - I40E_ERR_FIRMWARE_API_VERSION = -65, - I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR = -66, -}; - -#endif /* _I40E_STATUS_H_ */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index 8b8bf4880faa..0b3a27f118fb 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -2284,8 +2284,8 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, * If the buffer is an EOP buffer, this function exits returning false, * otherwise return true indicating that this is in fact a non-EOP buffer. */ -static bool i40e_is_non_eop(struct i40e_ring *rx_ring, - union i40e_rx_desc *rx_desc) +bool i40e_is_non_eop(struct i40e_ring *rx_ring, + union i40e_rx_desc *rx_desc) { /* if we are the last buffer then there is nothing else to do */ #define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT) diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h index 8c3d24012c54..900b0d9ede9f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h @@ -473,6 +473,8 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size); bool __i40e_chk_linearize(struct sk_buff *skb); int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags); +bool i40e_is_non_eop(struct i40e_ring *rx_ring, + union i40e_rx_desc *rx_desc); /** * i40e_get_head - Retrieve head from head writeback diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index 388c3d36d96a..5f61546f50d8 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -4,7 +4,6 @@ #ifndef _I40E_TYPE_H_ #define _I40E_TYPE_H_ -#include "i40e_status.h" #include "i40e_osdep.h" #include "i40e_register.h" #include "i40e_adminq.h" diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index be59ba3774e1..98aca9f8b602 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -1346,14 +1346,14 @@ static int i40e_config_vf_promiscuous_mode(struct i40e_vf *vf, bool alluni) { struct i40e_pf *pf = vf->pf; - int aq_ret = I40E_SUCCESS; struct i40e_vsi *vsi; + int aq_ret = 0; u16 num_vlans; s16 *vl; vsi = i40e_find_vsi_from_id(pf, vsi_id); if (!i40e_vc_isvalid_vsi_id(vf, vsi_id) || !vsi) - return I40E_ERR_PARAM; + return -EINVAL; if (vf->port_vlan_id) { aq_ret = i40e_set_vsi_promisc(vf, vsi->seid, allmulti, @@ -1363,7 +1363,7 @@ static int i40e_config_vf_promiscuous_mode(struct i40e_vf *vf, i40e_get_vlan_list_sync(vsi, &num_vlans, &vl); if (!vl) - return I40E_ERR_NO_MEMORY; + return -ENOMEM; aq_ret = i40e_set_vsi_promisc(vf, vsi->seid, allmulti, alluni, vl, num_vlans); @@ -2037,7 +2037,7 @@ static int i40e_vc_get_version_msg(struct i40e_vf *vf, u8 *msg) if (VF_IS_V10(&vf->vf_ver)) info.minor = VIRTCHNL_VERSION_MINOR_NO_VF_CAPS; return i40e_vc_send_msg_to_vf(vf, VIRTCHNL_OP_VERSION, - I40E_SUCCESS, (u8 *)&info, + 0, (u8 *)&info, sizeof(struct virtchnl_version_info)); } @@ -2099,14 +2099,14 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) int ret; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_INIT)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } len = struct_size(vfres, vsi_res, num_vsis); vfres = kzalloc(len, GFP_KERNEL); if (!vfres) { - aq_ret = I40E_ERR_NO_MEMORY; + aq_ret = -ENOMEM; len = 0; goto err; } @@ -2159,7 +2159,7 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) dev_err(&pf->pdev->dev, "VF %d requested polling mode: this feature is supported only when the device is running in single function per port (SFP) mode\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RX_POLLING; @@ -2227,7 +2227,7 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } if (!test_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps)) { @@ -2243,12 +2243,12 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf, u8 *msg) } if (info->flags > I40E_MAX_VF_PROMISC_FLAGS) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } if (!i40e_vc_isvalid_vsi_id(vf, info->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } @@ -2315,17 +2315,17 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_isvalid_vsi_id(vf, qci->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (qci->num_queue_pairs > I40E_MAX_VF_QUEUES) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -2333,7 +2333,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) for (i = 0; i < vf->num_tc; i++) num_qps_all += vf->ch[i].num_qps; if (num_qps_all != qci->num_queue_pairs) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } } @@ -2346,7 +2346,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) if (!vf->adq_enabled) { if (!i40e_vc_isvalid_queue_id(vf, vsi_id, qpi->txq.queue_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -2355,14 +2355,14 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) if (qpi->txq.vsi_id != qci->vsi_id || qpi->rxq.vsi_id != qci->vsi_id || qpi->rxq.queue_id != vsi_queue_id) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } } if (vf->adq_enabled) { if (idx >= ARRAY_SIZE(vf->ch)) { - aq_ret = I40E_ERR_NO_AVAILABLE_VSI; + aq_ret = -ENODEV; goto error_param; } vsi_id = vf->ch[idx].vsi_id; @@ -2372,7 +2372,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) &qpi->rxq) || i40e_config_vsi_tx_queue(vf, vsi_id, vsi_queue_id, &qpi->txq)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -2383,7 +2383,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) */ if (vf->adq_enabled) { if (idx >= ARRAY_SIZE(vf->ch)) { - aq_ret = I40E_ERR_NO_AVAILABLE_VSI; + aq_ret = -ENODEV; goto error_param; } if (j == (vf->ch[idx].num_qps - 1)) { @@ -2406,7 +2406,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) vsi->num_queue_pairs = vf->ch[i].num_qps; if (i40e_update_adq_vsi_queues(vsi, i)) { - aq_ret = I40E_ERR_CONFIG; + aq_ret = -EIO; goto error_param; } } @@ -2464,13 +2464,13 @@ static int i40e_vc_config_irq_map_msg(struct i40e_vf *vf, u8 *msg) int i; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (irqmap_info->num_vectors > vf->pf->hw.func_caps.num_msix_vectors_vf) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -2479,18 +2479,18 @@ static int i40e_vc_config_irq_map_msg(struct i40e_vf *vf, u8 *msg) /* validate msg params */ if (!i40e_vc_isvalid_vector_id(vf, map->vector_id) || !i40e_vc_isvalid_vsi_id(vf, map->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } vsi_id = map->vsi_id; if (i40e_validate_queue_map(vf, vsi_id, map->rxq_map)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (i40e_validate_queue_map(vf, vsi_id, map->txq_map)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -2579,29 +2579,29 @@ static int i40e_vc_enable_queues_msg(struct i40e_vf *vf, u8 *msg) int i; if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_validate_vqs_bitmaps(vqs)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } /* Use the queue bit map sent by the VF */ if (i40e_ctrl_vf_rx_rings(pf->vsi[vf->lan_vsi_idx], vqs->rx_queues, true)) { - aq_ret = I40E_ERR_TIMEOUT; + aq_ret = -EIO; goto error_param; } if (i40e_ctrl_vf_tx_rings(pf->vsi[vf->lan_vsi_idx], vqs->tx_queues, true)) { - aq_ret = I40E_ERR_TIMEOUT; + aq_ret = -EIO; goto error_param; } @@ -2610,7 +2610,7 @@ static int i40e_vc_enable_queues_msg(struct i40e_vf *vf, u8 *msg) /* zero belongs to LAN VSI */ for (i = 1; i < vf->num_tc; i++) { if (i40e_vsi_start_rings(pf->vsi[vf->ch[i].vsi_idx])) - aq_ret = I40E_ERR_TIMEOUT; + aq_ret = -EIO; } } @@ -2636,29 +2636,29 @@ static int i40e_vc_disable_queues_msg(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_validate_vqs_bitmaps(vqs)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } /* Use the queue bit map sent by the VF */ if (i40e_ctrl_vf_tx_rings(pf->vsi[vf->lan_vsi_idx], vqs->tx_queues, false)) { - aq_ret = I40E_ERR_TIMEOUT; + aq_ret = -EIO; goto error_param; } if (i40e_ctrl_vf_rx_rings(pf->vsi[vf->lan_vsi_idx], vqs->rx_queues, false)) { - aq_ret = I40E_ERR_TIMEOUT; + aq_ret = -EIO; goto error_param; } error_param: @@ -2790,18 +2790,18 @@ static int i40e_vc_get_stats_msg(struct i40e_vf *vf, u8 *msg) memset(&stats, 0, sizeof(struct i40e_eth_stats)); if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (!i40e_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } vsi = pf->vsi[vf->lan_vsi_idx]; if (!vsi) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } i40e_update_eth_stats(vsi); @@ -2862,7 +2862,7 @@ static inline int i40e_check_vf_permission(struct i40e_vf *vf, is_zero_ether_addr(addr)) { dev_err(&pf->pdev->dev, "invalid VF MAC addr %pM\n", addr); - return I40E_ERR_INVALID_MAC_ADDR; + return -EINVAL; } /* If the host VMM administrator has set the VF MAC address @@ -2998,7 +2998,7 @@ static int i40e_vc_add_mac_addr_msg(struct i40e_vf *vf, u8 *msg) if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) || !i40e_vc_isvalid_vsi_id(vf, al->vsi_id)) { - ret = I40E_ERR_PARAM; + ret = -EINVAL; goto error_param; } @@ -3027,7 +3027,7 @@ static int i40e_vc_add_mac_addr_msg(struct i40e_vf *vf, u8 *msg) dev_err(&pf->pdev->dev, "Unable to add MAC filter %pM for VF %d\n", al->list[i].addr, vf->vf_id); - ret = I40E_ERR_PARAM; + ret = -EINVAL; spin_unlock_bh(&vsi->mac_filter_hash_lock); goto error_param; } @@ -3067,7 +3067,7 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg) if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) || !i40e_vc_isvalid_vsi_id(vf, al->vsi_id)) { - ret = I40E_ERR_PARAM; + ret = -EINVAL; goto error_param; } @@ -3076,7 +3076,7 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg) is_zero_ether_addr(al->list[i].addr)) { dev_err(&pf->pdev->dev, "Invalid MAC addr %pM for VF %d\n", al->list[i].addr, vf->vf_id); - ret = I40E_ERR_INVALID_MAC_ADDR; + ret = -EINVAL; goto error_param; } if (ether_addr_equal(al->list[i].addr, vf->default_lan_addr.addr)) @@ -3088,7 +3088,7 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg) /* delete addresses from the list */ for (i = 0; i < al->num_elements; i++) if (i40e_del_mac_filter(vsi, al->list[i].addr)) { - ret = I40E_ERR_INVALID_MAC_ADDR; + ret = -EINVAL; spin_unlock_bh(&vsi->mac_filter_hash_lock); goto error_param; } @@ -3149,13 +3149,13 @@ static int i40e_vc_add_vlan_msg(struct i40e_vf *vf, u8 *msg) } if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) || !i40e_vc_isvalid_vsi_id(vf, vfl->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } for (i = 0; i < vfl->num_elements; i++) { if (vfl->vlan_id[i] > I40E_MAX_VLANID) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; dev_err(&pf->pdev->dev, "invalid VF VLAN id %d\n", vfl->vlan_id[i]); goto error_param; @@ -3163,7 +3163,7 @@ static int i40e_vc_add_vlan_msg(struct i40e_vf *vf, u8 *msg) } vsi = pf->vsi[vf->lan_vsi_idx]; if (vsi->info.pvid) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -3214,13 +3214,13 @@ static int i40e_vc_remove_vlan_msg(struct i40e_vf *vf, u8 *msg) if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) || !i40e_vc_isvalid_vsi_id(vf, vfl->vsi_id)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } for (i = 0; i < vfl->num_elements; i++) { if (vfl->vlan_id[i] > I40E_MAX_VLANID) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } } @@ -3228,7 +3228,7 @@ static int i40e_vc_remove_vlan_msg(struct i40e_vf *vf, u8 *msg) vsi = pf->vsi[vf->lan_vsi_idx]; if (vsi->info.pvid) { if (vfl->num_elements > 1 || vfl->vlan_id[0]) - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -3269,7 +3269,7 @@ static int i40e_vc_rdma_msg(struct i40e_vf *vf, u8 *msg, u16 msglen) if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) || !test_bit(I40E_VF_STATE_RDMAENA, &vf->vf_states)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } @@ -3298,13 +3298,13 @@ static int i40e_vc_rdma_qvmap_msg(struct i40e_vf *vf, u8 *msg, bool config) if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) || !test_bit(I40E_VF_STATE_RDMAENA, &vf->vf_states)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto error_param; } if (config) { if (i40e_config_rdma_qvlist(vf, qvlist_info)) - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; } else { i40e_release_rdma_qvlist(vf); } @@ -3335,7 +3335,7 @@ static int i40e_vc_config_rss_key(struct i40e_vf *vf, u8 *msg) if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) || !i40e_vc_isvalid_vsi_id(vf, vrk->vsi_id) || vrk->key_len != I40E_HKEY_ARRAY_SIZE) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3366,13 +3366,13 @@ static int i40e_vc_config_rss_lut(struct i40e_vf *vf, u8 *msg) if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) || !i40e_vc_isvalid_vsi_id(vf, vrl->vsi_id) || vrl->lut_entries != I40E_VF_HLUT_ARRAY_SIZE) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } for (i = 0; i < vrl->lut_entries; i++) if (vrl->lut[i] >= vf->num_queue_pairs) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3399,14 +3399,14 @@ static int i40e_vc_get_rss_hena(struct i40e_vf *vf, u8 *msg) int len = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } len = sizeof(struct virtchnl_rss_hena); vrh = kzalloc(len, GFP_KERNEL); if (!vrh) { - aq_ret = I40E_ERR_NO_MEMORY; + aq_ret = -ENOMEM; len = 0; goto err; } @@ -3435,7 +3435,7 @@ static int i40e_vc_set_rss_hena(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } i40e_write_rx_ctl(hw, I40E_VFQF_HENA1(0, vf->vf_id), (u32)vrh->hena); @@ -3460,7 +3460,7 @@ static int i40e_vc_enable_vlan_stripping(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3486,7 +3486,7 @@ static int i40e_vc_disable_vlan_stripping(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3574,7 +3574,7 @@ static int i40e_validate_cloud_filter(struct i40e_vf *vf, dev_err(&pf->pdev->dev, "VF %d not trusted, make VF trusted to add advanced mode ADq cloud filters\n", vf->vf_id); - return I40E_ERR_CONFIG; + return -EIO; } } @@ -3627,9 +3627,9 @@ static int i40e_validate_cloud_filter(struct i40e_vf *vf, } } - return I40E_SUCCESS; + return 0; err: - return I40E_ERR_CONFIG; + return -EIO; } /** @@ -3713,7 +3713,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) int i, ret; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3721,7 +3721,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) dev_info(&pf->pdev->dev, "VF %d: ADq not enabled, can't apply cloud filter\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3729,7 +3729,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) dev_info(&pf->pdev->dev, "VF %d: Invalid input, can't apply cloud filter\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3844,7 +3844,7 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) int i, ret; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } @@ -3852,7 +3852,7 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) dev_info(&pf->pdev->dev, "VF %d: ADq is not enabled, can't apply cloud filter\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } @@ -3860,7 +3860,7 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) dev_info(&pf->pdev->dev, "VF %d: Invalid input/s, can't apply cloud filter\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err_out; } @@ -3953,7 +3953,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) u64 speed = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3961,7 +3961,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) if (vf->spoofchk) { dev_err(&pf->pdev->dev, "Spoof check is ON, turn it OFF to enable ADq\n"); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3969,7 +3969,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) dev_err(&pf->pdev->dev, "VF %d attempting to enable ADq, but hasn't properly negotiated that capability\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3978,7 +3978,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) dev_err(&pf->pdev->dev, "VF %d trying to set %u TCs, valid range 1-%u TCs per VF\n", vf->vf_id, tci->num_tc, I40E_MAX_VF_VSI); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -3990,7 +3990,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) "VF %d: TC %d trying to set %u queues, valid range 1-%u queues per TC\n", vf->vf_id, i, tci->list[i].count, I40E_DEFAULT_QUEUES_PER_VF); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -4001,7 +4001,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) dev_err(&pf->pdev->dev, "No queues left to allocate to VF %d\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } else { /* we need to allocate max VF queues to enable ADq so as to @@ -4016,7 +4016,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) if (speed == SPEED_UNKNOWN) { dev_err(&pf->pdev->dev, "Cannot detect link speed\n"); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -4029,7 +4029,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) "Invalid max tx rate %llu specified for VF %d.", tci->list[i].max_tx_rate, vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } else { vf->ch[i].max_tx_rate = @@ -4045,7 +4045,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg) /* reset the VF in order to allocate resources */ i40e_vc_reset_vf(vf, true); - return I40E_SUCCESS; + return 0; /* send the response to the VF */ err: @@ -4064,7 +4064,7 @@ static int i40e_vc_del_qch_msg(struct i40e_vf *vf, u8 *msg) int aq_ret = 0; if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) { - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; goto err; } @@ -4079,13 +4079,13 @@ static int i40e_vc_del_qch_msg(struct i40e_vf *vf, u8 *msg) } else { dev_info(&pf->pdev->dev, "VF %d trying to delete queue channels but ADq isn't enabled\n", vf->vf_id); - aq_ret = I40E_ERR_PARAM; + aq_ret = -EINVAL; } /* reset the VF in order to allocate resources */ i40e_vc_reset_vf(vf, true); - return I40E_SUCCESS; + return 0; err: return i40e_vc_send_resp_to_vf(vf, VIRTCHNL_OP_DISABLE_CHANNELS, @@ -4119,21 +4119,16 @@ int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode, /* Check if VF is disabled. */ if (test_bit(I40E_VF_STATE_DISABLED, &vf->vf_states)) - return I40E_ERR_PARAM; + return -EINVAL; /* perform basic checks on the msg */ ret = virtchnl_vc_validate_vf_msg(&vf->vf_ver, v_opcode, msg, msglen); if (ret) { - i40e_vc_send_resp_to_vf(vf, v_opcode, I40E_ERR_PARAM); + i40e_vc_send_resp_to_vf(vf, v_opcode, -EINVAL); dev_err(&pf->pdev->dev, "Invalid message from VF %d, opcode %d, len %d\n", local_vf_id, v_opcode, msglen); - switch (ret) { - case VIRTCHNL_STATUS_ERR_PARAM: - return -EPERM; - default: - return -EINVAL; - } + return ret; } switch (v_opcode) { @@ -4226,7 +4221,7 @@ int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode, dev_err(&pf->pdev->dev, "Unsupported opcode %d from VF %d\n", v_opcode, local_vf_id); ret = i40e_vc_send_resp_to_vf(vf, v_opcode, - I40E_ERR_NOT_IMPLEMENTED); + -EOPNOTSUPP); break; } @@ -4305,6 +4300,38 @@ err_out: } /** + * i40e_check_vf_init_timeout + * @vf: the virtual function + * + * Check that the VF's initialization was successfully done and if not + * wait up to 300ms for its finish. + * + * Returns true when VF is initialized, false on timeout + **/ +static bool i40e_check_vf_init_timeout(struct i40e_vf *vf) +{ + int i; + + /* When the VF is resetting wait until it is done. + * It can take up to 200 milliseconds, but wait for + * up to 300 milliseconds to be safe. + */ + for (i = 0; i < 15; i++) { + if (test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) + return true; + msleep(20); + } + + if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) { + dev_err(&vf->pf->pdev->dev, + "VF %d still in reset. Try again.\n", vf->vf_id); + return false; + } + + return true; +} + +/** * i40e_ndo_set_vf_mac * @netdev: network interface device structure * @vf_id: VF identifier @@ -4322,7 +4349,6 @@ int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) int ret = 0; struct hlist_node *h; int bkt; - u8 i; if (test_and_set_bit(__I40E_VIRTCHNL_OP_PENDING, pf->state)) { dev_warn(&pf->pdev->dev, "Unable to configure VFs, other operation is pending.\n"); @@ -4335,21 +4361,7 @@ int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) goto error_param; vf = &pf->vf[vf_id]; - - /* When the VF is resetting wait until it is done. - * It can take up to 200 milliseconds, - * but wait for up to 300 milliseconds to be safe. - * Acquire the VSI pointer only after the VF has been - * properly initialized. - */ - for (i = 0; i < 15; i++) { - if (test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) - break; - msleep(20); - } - if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) { - dev_err(&pf->pdev->dev, "VF %d still in reset. Try again.\n", - vf_id); + if (!i40e_check_vf_init_timeout(vf)) { ret = -EAGAIN; goto error_param; } @@ -4451,13 +4463,11 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id, } vf = &pf->vf[vf_id]; - vsi = pf->vsi[vf->lan_vsi_idx]; - if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) { - dev_err(&pf->pdev->dev, "VF %d still in reset. Try again.\n", - vf_id); + if (!i40e_check_vf_init_timeout(vf)) { ret = -EAGAIN; goto error_pvid; } + vsi = pf->vsi[vf->lan_vsi_idx]; if (le16_to_cpu(vsi->info.pvid) == vlanprio) /* duplicate request, so just return success */ @@ -4601,13 +4611,11 @@ int i40e_ndo_set_vf_bw(struct net_device *netdev, int vf_id, int min_tx_rate, } vf = &pf->vf[vf_id]; - vsi = pf->vsi[vf->lan_vsi_idx]; - if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) { - dev_err(&pf->pdev->dev, "VF %d still in reset. Try again.\n", - vf_id); + if (!i40e_check_vf_init_timeout(vf)) { ret = -EAGAIN; goto error; } + vsi = pf->vsi[vf->lan_vsi_idx]; ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); if (ret) @@ -4774,9 +4782,7 @@ int i40e_ndo_set_vf_spoofchk(struct net_device *netdev, int vf_id, bool enable) } vf = &(pf->vf[vf_id]); - if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) { - dev_err(&pf->pdev->dev, "VF %d still in reset. Try again.\n", - vf_id); + if (!i40e_check_vf_init_timeout(vf)) { ret = -EAGAIN; goto out; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 05ec1181471e..37f41c8a682f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -294,8 +294,14 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, { unsigned int totalsize = xdp->data_end - xdp->data_meta; unsigned int metasize = xdp->data - xdp->data_meta; + struct skb_shared_info *sinfo = NULL; struct sk_buff *skb; + u32 nr_frags = 0; + if (unlikely(xdp_buff_has_frags(xdp))) { + sinfo = xdp_get_shared_info_from_buff(xdp); + nr_frags = sinfo->nr_frags; + } net_prefetch(xdp->data_meta); /* allocate a skb to store the frags */ @@ -312,6 +318,28 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, __skb_pull(skb, metasize); } + if (likely(!xdp_buff_has_frags(xdp))) + goto out; + + for (int i = 0; i < nr_frags; i++) { + struct skb_shared_info *skinfo = skb_shinfo(skb); + skb_frag_t *frag = &sinfo->frags[i]; + struct page *page; + void *addr; + + page = dev_alloc_page(); + if (!page) { + dev_kfree_skb(skb); + return NULL; + } + addr = page_to_virt(page); + + memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); + + __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++, + addr, 0, skb_frag_size(frag)); + } + out: xsk_buff_free(xdp); return skb; @@ -322,14 +350,13 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, union i40e_rx_desc *rx_desc, unsigned int *rx_packets, unsigned int *rx_bytes, - unsigned int size, unsigned int xdp_res, bool *failure) { struct sk_buff *skb; *rx_packets = 1; - *rx_bytes = size; + *rx_bytes = xdp_get_buff_len(xdp_buff); if (likely(xdp_res == I40E_XDP_REDIR) || xdp_res == I40E_XDP_TX) return; @@ -363,7 +390,6 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, return; } - *rx_bytes = skb->len; i40e_process_skb_fields(rx_ring, rx_desc, skb); napi_gro_receive(&rx_ring->q_vector->napi, skb); return; @@ -374,6 +400,31 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, WARN_ON_ONCE(1); } +static int +i40e_add_xsk_frag(struct i40e_ring *rx_ring, struct xdp_buff *first, + struct xdp_buff *xdp, const unsigned int size) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); + + if (!xdp_buff_has_frags(first)) { + sinfo->nr_frags = 0; + sinfo->xdp_frags_size = 0; + xdp_buff_set_frags_flag(first); + } + + if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { + xsk_buff_free(first); + return -ENOMEM; + } + + __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, + virt_to_page(xdp->data_hard_start), 0, size); + sinfo->xdp_frags_size += size; + xsk_buff_add_frag(xdp); + + return 0; +} + /** * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring * @rx_ring: Rx ring @@ -384,13 +435,18 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; + u16 next_to_process = rx_ring->next_to_process; u16 next_to_clean = rx_ring->next_to_clean; u16 count_mask = rx_ring->count - 1; unsigned int xdp_res, xdp_xmit = 0; + struct xdp_buff *first = NULL; struct bpf_prog *xdp_prog; bool failure = false; u16 cleaned_count; + if (next_to_process != next_to_clean) + first = *i40e_rx_bi(rx_ring, next_to_clean); + /* NB! xdp_prog will always be !NULL, due to the fact that * this path is enabled by setting an XDP program. */ @@ -404,7 +460,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) unsigned int size; u64 qword; - rx_desc = I40E_RX_DESC(rx_ring, next_to_clean); + rx_desc = I40E_RX_DESC(rx_ring, next_to_process); qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); /* This memory barrier is needed to keep us from reading @@ -417,9 +473,9 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) i40e_clean_programming_status(rx_ring, rx_desc->raw.qword[0], qword); - bi = *i40e_rx_bi(rx_ring, next_to_clean); + bi = *i40e_rx_bi(rx_ring, next_to_process); xsk_buff_free(bi); - next_to_clean = (next_to_clean + 1) & count_mask; + next_to_process = (next_to_process + 1) & count_mask; continue; } @@ -428,22 +484,35 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) if (!size) break; - bi = *i40e_rx_bi(rx_ring, next_to_clean); + bi = *i40e_rx_bi(rx_ring, next_to_process); xsk_buff_set_size(bi, size); xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); - xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog); - i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets, - &rx_bytes, size, xdp_res, &failure); + if (!first) + first = bi; + else if (i40e_add_xsk_frag(rx_ring, first, bi, size)) + break; + + next_to_process = (next_to_process + 1) & count_mask; + + if (i40e_is_non_eop(rx_ring, rx_desc)) + continue; + + xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog); + i40e_handle_xdp_result_zc(rx_ring, first, rx_desc, &rx_packets, + &rx_bytes, xdp_res, &failure); + first->flags = 0; + next_to_clean = next_to_process; if (failure) break; total_rx_packets += rx_packets; total_rx_bytes += rx_bytes; xdp_xmit |= xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR); - next_to_clean = (next_to_clean + 1) & count_mask; + first = NULL; } rx_ring->next_to_clean = next_to_clean; + rx_ring->next_to_process = next_to_process; cleaned_count = (next_to_clean - rx_ring->next_to_use - 1) & count_mask; if (cleaned_count >= I40E_RX_BUFFER_WRITE) @@ -466,6 +535,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) static void i40e_xmit_pkt(struct i40e_ring *xdp_ring, struct xdp_desc *desc, unsigned int *total_bytes) { + u32 cmd = I40E_TX_DESC_CMD_ICRC | xsk_is_eop_desc(desc); struct i40e_tx_desc *tx_desc; dma_addr_t dma; @@ -474,8 +544,7 @@ static void i40e_xmit_pkt(struct i40e_ring *xdp_ring, struct xdp_desc *desc, tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use++); tx_desc->buffer_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC | I40E_TX_DESC_CMD_EOP, - 0, desc->len, 0); + tx_desc->cmd_type_offset_bsz = build_ctob(cmd, 0, desc->len, 0); *total_bytes += desc->len; } @@ -489,14 +558,14 @@ static void i40e_xmit_pkt_batch(struct i40e_ring *xdp_ring, struct xdp_desc *des u32 i; loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { + u32 cmd = I40E_TX_DESC_CMD_ICRC | xsk_is_eop_desc(&desc[i]); + dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc[i].addr); xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc[i].len); tx_desc = I40E_TX_DESC(xdp_ring, ntu++); tx_desc->buffer_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC | - I40E_TX_DESC_CMD_EOP, - 0, desc[i].len, 0); + tx_desc->cmd_type_offset_bsz = build_ctob(cmd, 0, desc[i].len, 0); *total_bytes += desc[i].len; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 9610ca770349..7b300c86ceda 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -3744,15 +3744,15 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, struct virtchnl_filter *vf = &filter->f; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { - dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { + dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index 817977e3039d..960277d78e09 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -47,5 +47,5 @@ ice-$(CONFIG_PTP_1588_CLOCK) += ice_ptp.o ice_ptp_hw.o ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_nl.o ice_dcb_lib.o ice-$(CONFIG_RFS_ACCEL) += ice_arfs.o ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o -ice-$(CONFIG_ICE_SWITCHDEV) += ice_eswitch.o +ice-$(CONFIG_ICE_SWITCHDEV) += ice_eswitch.o ice_eswitch_br.o ice-$(CONFIG_GNSS) += ice_gnss.o diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 4ba3d99439a0..5ac0ad12f9f1 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -200,6 +200,8 @@ enum ice_feature { ICE_F_PTP_EXTTS, ICE_F_SMA_CTRL, ICE_F_GNSS, + ICE_F_ROCE_LAG, + ICE_F_SRIOV_LAG, ICE_F_MAX }; @@ -370,6 +372,7 @@ struct ice_vsi { u16 rx_buf_len; struct ice_aqc_vsi_props info; /* VSI properties */ + struct ice_vsi_vlan_info vlan_info; /* vlan config to be restored */ /* VSI stats */ struct rtnl_link_stats64 net_stats; @@ -517,6 +520,7 @@ enum ice_misc_thread_tasks { struct ice_switchdev_info { struct ice_vsi *control_vsi; struct ice_vsi *uplink_vsi; + struct ice_esw_br_offloads *br_offloads; bool is_running; }; @@ -567,6 +571,7 @@ struct ice_pf { struct mutex sw_mutex; /* lock for protecting VSI alloc flow */ struct mutex tc_mutex; /* lock to protect TC changes */ struct mutex adev_mutex; /* lock to protect aux device access */ + struct mutex lag_mutex; /* protect ice_lag struct in PF */ u32 msg_enable; struct ice_ptp ptp; struct gnss_serial *gnss_serial; @@ -626,6 +631,7 @@ struct ice_pf { struct ice_lag *lag; /* Link Aggregation information */ struct ice_switchdev_info switchdev; + struct ice_esw_br_port *br_port; #define ICE_INVALID_AGG_NODE_ID 0 #define ICE_PF_AGG_NODE_ID_START 1 @@ -636,6 +642,8 @@ struct ice_pf { struct ice_agg_node vf_agg_node[ICE_MAX_VF_AGG_NODES]; }; +extern struct workqueue_struct *ice_lag_wq; + struct ice_netdev_priv { struct ice_vsi *vsi; struct ice_repr *repr; @@ -853,7 +861,7 @@ static inline bool ice_is_adq_active(struct ice_pf *pf) return false; } -bool netif_is_ice(struct net_device *dev); +bool netif_is_ice(const struct net_device *dev); int ice_vsi_setup_tx_rings(struct ice_vsi *vsi); int ice_vsi_setup_rx_rings(struct ice_vsi *vsi); int ice_vsi_open_ctrl(struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 63d3e1dcbba5..c0ad34b42531 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -120,6 +120,9 @@ struct ice_aqc_list_caps_elem { #define ICE_AQC_CAPS_PCIE_RESET_AVOIDANCE 0x0076 #define ICE_AQC_CAPS_POST_UPDATE_RESET_RESTRICT 0x0077 #define ICE_AQC_CAPS_NVM_MGMT 0x0080 +#define ICE_AQC_CAPS_FW_LAG_SUPPORT 0x0092 +#define ICE_AQC_BIT_ROCEV2_LAG 0x01 +#define ICE_AQC_BIT_SRIOV_LAG 0x02 u8 major_ver; u8 minor_ver; @@ -232,6 +235,8 @@ struct ice_aqc_set_port_params { #define ICE_AQC_SET_P_PARAMS_DOUBLE_VLAN_ENA BIT(2) __le16 bad_frame_vsi; __le16 swid; +#define ICE_AQC_PORT_SWID_VALID BIT(15) +#define ICE_AQC_PORT_SWID_M 0xFF u8 reserved[10]; }; @@ -241,10 +246,12 @@ struct ice_aqc_set_port_params { * Allocate Resources command (indirect 0x0208) * Free Resources command (indirect 0x0209) * Get Allocated Resource Descriptors Command (indirect 0x020A) + * Share Resource command (indirect 0x020B) */ #define ICE_AQC_RES_TYPE_VSI_LIST_REP 0x03 #define ICE_AQC_RES_TYPE_VSI_LIST_PRUNE 0x04 #define ICE_AQC_RES_TYPE_RECIPE 0x05 +#define ICE_AQC_RES_TYPE_SWID 0x07 #define ICE_AQC_RES_TYPE_FDIR_COUNTER_BLOCK 0x21 #define ICE_AQC_RES_TYPE_FDIR_GUARANTEED_ENTRIES 0x22 #define ICE_AQC_RES_TYPE_FDIR_SHARED_ENTRIES 0x23 @@ -264,6 +271,7 @@ struct ice_aqc_set_port_params { /* Allocate Resources command (indirect 0x0208) * Free Resources command (indirect 0x0209) + * Share Resource command (indirect 0x020B) */ struct ice_aqc_alloc_free_res_cmd { __le16 num_entries; /* Number of Resource entries */ @@ -818,7 +826,11 @@ struct ice_aqc_txsched_move_grp_info_hdr { __le32 src_parent_teid; __le32 dest_parent_teid; __le16 num_elems; - __le16 reserved; + u8 mode; +#define ICE_AQC_MOVE_ELEM_MODE_SAME_PF 0x0 +#define ICE_AQC_MOVE_ELEM_MODE_GIVE_OWN 0x1 +#define ICE_AQC_MOVE_ELEM_MODE_KEEP_OWN 0x2 + u8 reserved; }; struct ice_aqc_move_elem { @@ -1923,6 +1935,42 @@ struct ice_aqc_dis_txq_item { __le16 q_id[]; } __packed; +/* Move/Reconfigure Tx queue (indirect 0x0C32) */ +struct ice_aqc_cfg_txqs { + u8 cmd_type; +#define ICE_AQC_Q_CFG_MOVE_NODE 0x1 +#define ICE_AQC_Q_CFG_TC_CHNG 0x2 +#define ICE_AQC_Q_CFG_MOVE_TC_CHNG 0x3 +#define ICE_AQC_Q_CFG_SUBSEQ_CALL BIT(2) +#define ICE_AQC_Q_CFG_FLUSH BIT(3) + u8 num_qs; + u8 port_num_chng; +#define ICE_AQC_Q_CFG_SRC_PRT_M 0x7 +#define ICE_AQC_Q_CFG_DST_PRT_S 3 +#define ICE_AQC_Q_CFG_DST_PRT_M (0x7 << ICE_AQC_Q_CFG_DST_PRT_S) + u8 time_out; +#define ICE_AQC_Q_CFG_TIMEOUT_S 2 +#define ICE_AQC_Q_CFG_TIMEOUT_M (0x1F << ICE_AQC_Q_CFG_TIMEOUT_S) + __le32 blocked_cgds; + __le32 addr_high; + __le32 addr_low; +}; + +/* Per Q struct for Move/Reconfigure Tx LAN Queues (indirect 0x0C32) */ +struct ice_aqc_cfg_txq_perq { + __le16 q_handle; + u8 tc; + u8 rsvd; + __le32 q_teid; +}; + +/* The buffer for Move/Reconfigure Tx LAN Queues (indirect 0x0C32) */ +struct ice_aqc_cfg_txqs_buf { + __le32 src_parent_teid; + __le32 dst_parent_teid; + struct ice_aqc_cfg_txq_perq queue_info[]; +}; + /* Add Tx RDMA Queue Set (indirect 0x0C33) */ struct ice_aqc_add_rdma_qset { u8 num_qset_grps; @@ -2181,6 +2229,7 @@ struct ice_aq_desc { struct ice_aqc_neigh_dev_req neigh_dev; struct ice_aqc_add_txqs add_txqs; struct ice_aqc_dis_txqs dis_txqs; + struct ice_aqc_cfg_txqs cfg_txqs; struct ice_aqc_add_rdma_qset add_rdma_qset; struct ice_aqc_add_get_update_free_vsi vsi_cmd; struct ice_aqc_add_update_free_vsi_resp add_update_free_vsi_res; @@ -2263,6 +2312,7 @@ enum ice_adminq_opc { /* Alloc/Free/Get Resources */ ice_aqc_opc_alloc_res = 0x0208, ice_aqc_opc_free_res = 0x0209, + ice_aqc_opc_share_res = 0x020B, ice_aqc_opc_set_vlan_mode_parameters = 0x020C, ice_aqc_opc_get_vlan_mode_parameters = 0x020D, @@ -2356,6 +2406,7 @@ enum ice_adminq_opc { /* Tx queue handling commands/events */ ice_aqc_opc_add_txqs = 0x0C30, ice_aqc_opc_dis_txqs = 0x0C31, + ice_aqc_opc_cfg_txqs = 0x0C32, ice_aqc_opc_add_rdma_qset = 0x0C33, /* package commands */ diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index b678bdf96f3a..9ab9fb558b5e 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -408,7 +408,6 @@ static unsigned int ice_rx_offset(struct ice_rx_ring *rx_ring) */ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) { - int chain_len = ICE_MAX_CHAINED_RX_BUFS; struct ice_vsi *vsi = ring->vsi; u32 rxdid = ICE_RXDID_FLEX_NIC; struct ice_rlan_ctx rlan_ctx; @@ -472,17 +471,11 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) */ rlan_ctx.showiv = 0; - /* For AF_XDP ZC, we disallow packets to span on - * multiple buffers, thus letting us skip that - * handling in the fast-path. - */ - if (ring->xsk_pool) - chain_len = 1; /* Max packet size for this queue - must not be set to a larger value * than 5 x DBUF */ rlan_ctx.rxmax = min_t(u32, vsi->max_frame, - chain_len * ring->rx_buf_len); + ICE_MAX_CHAINED_RX_BUFS * ring->rx_buf_len); /* Rx queue threshold in units of 64 */ rlan_ctx.lrxqthresh = 1; diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index e16d4c83ed5f..dade0a50299c 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -2241,6 +2241,14 @@ ice_parse_common_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps, "%s: reset_restrict_support = %d\n", prefix, caps->reset_restrict_support); break; + case ICE_AQC_CAPS_FW_LAG_SUPPORT: + caps->roce_lag = !!(number & ICE_AQC_BIT_ROCEV2_LAG); + ice_debug(hw, ICE_DBG_INIT, "%s: roce_lag = %u\n", + prefix, caps->roce_lag); + caps->sriov_lag = !!(number & ICE_AQC_BIT_SRIOV_LAG); + ice_debug(hw, ICE_DBG_INIT, "%s: sriov_lag = %u\n", + prefix, caps->sriov_lag); + break; default: /* Not one of the recognized common capabilities */ found = false; @@ -4222,6 +4230,53 @@ do_aq: } /** + * ice_aq_cfg_lan_txq + * @hw: pointer to the hardware structure + * @buf: buffer for command + * @buf_size: size of buffer in bytes + * @num_qs: number of queues being configured + * @oldport: origination lport + * @newport: destination lport + * @cd: pointer to command details structure or NULL + * + * Move/Configure LAN Tx queue (0x0C32) + * + * There is a better AQ command to use for moving nodes, so only coding + * this one for configuring the node. + */ +int +ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, + u16 buf_size, u16 num_qs, u8 oldport, u8 newport, + struct ice_sq_cd *cd) +{ + struct ice_aqc_cfg_txqs *cmd; + struct ice_aq_desc desc; + int status; + + cmd = &desc.params.cfg_txqs; + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_cfg_txqs); + desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); + + if (!buf) + return -EINVAL; + + cmd->cmd_type = ICE_AQC_Q_CFG_TC_CHNG; + cmd->num_qs = num_qs; + cmd->port_num_chng = (oldport & ICE_AQC_Q_CFG_SRC_PRT_M); + cmd->port_num_chng |= (newport << ICE_AQC_Q_CFG_DST_PRT_S) & + ICE_AQC_Q_CFG_DST_PRT_M; + cmd->time_out = (5 << ICE_AQC_Q_CFG_TIMEOUT_S) & + ICE_AQC_Q_CFG_TIMEOUT_M; + cmd->blocked_cgds = 0; + + status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd); + if (status) + ice_debug(hw, ICE_DBG_SCHED, "Failed to reconfigure nodes %d\n", + hw->adminq.sq_last_status); + return status; +} + +/** * ice_aq_add_rdma_qsets * @hw: pointer to the hardware structure * @num_qset_grps: Number of RDMA Qset groups @@ -4700,6 +4755,7 @@ ice_dis_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u8 num_queues, break; ice_free_sched_node(pi, node); q_ctx->q_handle = ICE_INVAL_Q_HANDLE; + q_ctx->q_teid = ICE_INVAL_TEID; } mutex_unlock(&pi->sched_lock); kfree(qg_list); diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 81961a7d6598..df12a9d8d28c 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -186,6 +186,10 @@ int ice_ena_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 q_handle, u8 num_qgrps, struct ice_aqc_add_tx_qgrp *buf, u16 buf_size, struct ice_sq_cd *cd); +int +ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, + u16 buf_size, u16 num_qs, u8 oldport, u8 newport, + struct ice_sq_cd *cd); int ice_replay_vsi(struct ice_hw *hw, u16 vsi_handle); void ice_replay_post(struct ice_hw *hw); void ice_output_fw_log(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf); diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c index 3eb01731e496..e1fbc6de452d 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c @@ -70,6 +70,11 @@ static int ice_dcbnl_setets(struct net_device *netdev, struct ieee_ets *ets) !(pf->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)) return -EINVAL; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return -EINVAL; + } + new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; mutex_lock(&pf->tc_mutex); @@ -170,6 +175,11 @@ static u8 ice_dcbnl_setdcbx(struct net_device *netdev, u8 mode) if (mode == pf->dcbx_cap) return ICE_DCB_NO_HW_CHG; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return ICE_DCB_NO_HW_CHG; + } + qos_cfg = &pf->hw.port_info->qos_cfg; /* DSCP configuration is not DCBx negotiated */ @@ -261,6 +271,11 @@ static int ice_dcbnl_setpfc(struct net_device *netdev, struct ieee_pfc *pfc) !(pf->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)) return -EINVAL; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return -EINVAL; + } + mutex_lock(&pf->tc_mutex); new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; @@ -323,6 +338,11 @@ static void ice_dcbnl_set_pfc_cfg(struct net_device *netdev, int prio, u8 set) if (prio >= ICE_MAX_USER_PRIORITY) return; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return; + } + new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; new_cfg->pfc.pfccap = pf->hw.func_caps.common_cap.maxtc; @@ -379,6 +399,11 @@ static u8 ice_dcbnl_setstate(struct net_device *netdev, u8 state) !(pf->dcbx_cap & DCB_CAP_DCBX_VER_CEE)) return ICE_DCB_NO_HW_CHG; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return ICE_DCB_NO_HW_CHG; + } + /* Nothing to do */ if (!!state == test_bit(ICE_FLAG_DCB_ENA, pf->flags)) return ICE_DCB_NO_HW_CHG; @@ -451,6 +476,11 @@ ice_dcbnl_set_pg_tc_cfg_tx(struct net_device *netdev, int tc, if (tc >= ICE_MAX_TRAFFIC_CLASS) return; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return; + } + new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; /* prio_type, bwg_id and bw_pct per UP are not supported */ @@ -505,6 +535,11 @@ ice_dcbnl_set_pg_bwg_cfg_tx(struct net_device *netdev, int pgid, u8 bw_pct) if (pgid >= ICE_MAX_TRAFFIC_CLASS) return; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return; + } + new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; new_cfg->etscfg.tcbwtable[pgid] = bw_pct; @@ -725,6 +760,11 @@ static int ice_dcbnl_setapp(struct net_device *netdev, struct dcb_app *app) return -EINVAL; } + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return -EINVAL; + } + max_tc = pf->hw.func_caps.common_cap.maxtc; if (app->priority >= max_tc) { netdev_err(netdev, "TC %d out of range, max TC %d\n", @@ -836,6 +876,11 @@ static int ice_dcbnl_delapp(struct net_device *netdev, struct dcb_app *app) return -EINVAL; } + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return -EINVAL; + } + mutex_lock(&pf->tc_mutex); old_cfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg; @@ -937,6 +982,11 @@ static u8 ice_dcbnl_cee_set_all(struct net_device *netdev) !(pf->dcbx_cap & DCB_CAP_DCBX_VER_CEE)) return ICE_DCB_NO_HW_CHG; + if (pf->lag && pf->lag->bonded) { + netdev_err(netdev, "DCB changes not allowed when in a bond\n"); + return ICE_DCB_NO_HW_CHG; + } + new_cfg = &pf->hw.port_info->qos_cfg.desired_dcbx_cfg; mutex_lock(&pf->tc_mutex); diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index ad0a007b7398..9a53a5e5d73e 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -4,6 +4,7 @@ #include "ice.h" #include "ice_lib.h" #include "ice_eswitch.h" +#include "ice_eswitch_br.h" #include "ice_fltr.h" #include "ice_repr.h" #include "ice_devlink.h" @@ -103,17 +104,28 @@ static int ice_eswitch_setup_env(struct ice_pf *pf) rule_added = true; } + vlan_ops = ice_get_compat_vsi_vlan_ops(uplink_vsi); + if (vlan_ops->dis_rx_filtering(uplink_vsi)) + goto err_dis_rx; + if (ice_vsi_update_security(uplink_vsi, ice_vsi_ctx_set_allow_override)) goto err_override_uplink; if (ice_vsi_update_security(ctrl_vsi, ice_vsi_ctx_set_allow_override)) goto err_override_control; + if (ice_vsi_update_local_lb(uplink_vsi, true)) + goto err_override_local_lb; + return 0; +err_override_local_lb: + ice_vsi_update_security(ctrl_vsi, ice_vsi_ctx_clear_allow_override); err_override_control: ice_vsi_update_security(uplink_vsi, ice_vsi_ctx_clear_allow_override); err_override_uplink: + vlan_ops->ena_rx_filtering(uplink_vsi); +err_dis_rx: if (rule_added) ice_clear_dflt_vsi(uplink_vsi); err_def_rx: @@ -306,6 +318,9 @@ void ice_eswitch_update_repr(struct ice_vsi *vsi) repr->src_vsi = vsi; repr->dst->u.port_info.port_id = vsi->vsi_num; + if (repr->br_port) + repr->br_port->vsi = vsi; + ret = ice_vsi_update_security(vsi, ice_vsi_ctx_clear_antispoof); if (ret) { ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr, ICE_FWD_TO_VSI); @@ -331,6 +346,9 @@ ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev) np = netdev_priv(netdev); vsi = np->vsi; + if (!vsi || !ice_is_switchdev_running(vsi->back)) + return NETDEV_TX_BUSY; + if (ice_is_reset_in_progress(vsi->back->state) || test_bit(ICE_VF_DIS, vsi->back->state)) return NETDEV_TX_BUSY; @@ -378,9 +396,14 @@ static void ice_eswitch_release_env(struct ice_pf *pf) { struct ice_vsi *uplink_vsi = pf->switchdev.uplink_vsi; struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi; + struct ice_vsi_vlan_ops *vlan_ops; + + vlan_ops = ice_get_compat_vsi_vlan_ops(uplink_vsi); + ice_vsi_update_local_lb(uplink_vsi, false); ice_vsi_update_security(ctrl_vsi, ice_vsi_ctx_clear_allow_override); ice_vsi_update_security(uplink_vsi, ice_vsi_ctx_clear_allow_override); + vlan_ops->ena_rx_filtering(uplink_vsi); ice_clear_dflt_vsi(uplink_vsi); ice_fltr_add_mac_and_broadcast(uplink_vsi, uplink_vsi->port_info->mac.perm_addr, @@ -455,16 +478,24 @@ static void ice_eswitch_napi_disable(struct ice_pf *pf) */ static int ice_eswitch_enable_switchdev(struct ice_pf *pf) { - struct ice_vsi *ctrl_vsi; + struct ice_vsi *ctrl_vsi, *uplink_vsi; + + uplink_vsi = ice_get_main_vsi(pf); + if (!uplink_vsi) + return -ENODEV; + + if (netif_is_any_bridge_port(uplink_vsi->netdev)) { + dev_err(ice_pf_to_dev(pf), + "Uplink port cannot be a bridge port\n"); + return -EINVAL; + } pf->switchdev.control_vsi = ice_eswitch_vsi_setup(pf, pf->hw.port_info); if (!pf->switchdev.control_vsi) return -ENODEV; ctrl_vsi = pf->switchdev.control_vsi; - pf->switchdev.uplink_vsi = ice_get_main_vsi(pf); - if (!pf->switchdev.uplink_vsi) - goto err_vsi; + pf->switchdev.uplink_vsi = uplink_vsi; if (ice_eswitch_setup_env(pf)) goto err_vsi; @@ -480,10 +511,15 @@ static int ice_eswitch_enable_switchdev(struct ice_pf *pf) if (ice_vsi_open(ctrl_vsi)) goto err_setup_reprs; + if (ice_eswitch_br_offloads_init(pf)) + goto err_br_offloads; + ice_eswitch_napi_enable(pf); return 0; +err_br_offloads: + ice_vsi_close(ctrl_vsi); err_setup_reprs: ice_repr_rem_from_all_vfs(pf); err_repr_add: @@ -502,8 +538,8 @@ static void ice_eswitch_disable_switchdev(struct ice_pf *pf) struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi; ice_eswitch_napi_disable(pf); + ice_eswitch_br_offloads_deinit(pf); ice_eswitch_release_env(pf); - ice_rem_adv_rule_for_vsi(&pf->hw, ctrl_vsi->idx); ice_eswitch_release_reprs(pf, ctrl_vsi); ice_vsi_release(ctrl_vsi); ice_repr_rem_from_all_vfs(pf); diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch_br.c b/drivers/net/ethernet/intel/ice/ice_eswitch_br.c new file mode 100644 index 000000000000..cc7357ed6e5f --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_eswitch_br.c @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023, Intel Corporation. */ + +#include "ice.h" +#include "ice_eswitch_br.h" +#include "ice_repr.h" +#include "ice_switch.h" +#include "ice_vlan.h" +#include "ice_vf_vsi_vlan_ops.h" +#include "ice_trace.h" + +#define ICE_ESW_BRIDGE_UPDATE_INTERVAL msecs_to_jiffies(1000) + +static const struct rhashtable_params ice_fdb_ht_params = { + .key_offset = offsetof(struct ice_esw_br_fdb_entry, data), + .key_len = sizeof(struct ice_esw_br_fdb_data), + .head_offset = offsetof(struct ice_esw_br_fdb_entry, ht_node), + .automatic_shrinking = true, +}; + +static bool ice_eswitch_br_is_dev_valid(const struct net_device *dev) +{ + /* Accept only PF netdev and PRs */ + return ice_is_port_repr_netdev(dev) || netif_is_ice(dev); +} + +static struct ice_esw_br_port * +ice_eswitch_br_netdev_to_port(struct net_device *dev) +{ + if (ice_is_port_repr_netdev(dev)) { + struct ice_repr *repr = ice_netdev_to_repr(dev); + + return repr->br_port; + } else if (netif_is_ice(dev)) { + struct ice_pf *pf = ice_netdev_to_pf(dev); + + return pf->br_port; + } + + return NULL; +} + +static void +ice_eswitch_br_ingress_rule_setup(struct ice_adv_rule_info *rule_info, + u8 pf_id, u16 vf_vsi_idx) +{ + rule_info->sw_act.vsi_handle = vf_vsi_idx; + rule_info->sw_act.flag |= ICE_FLTR_RX; + rule_info->sw_act.src = pf_id; + rule_info->priority = 5; +} + +static void +ice_eswitch_br_egress_rule_setup(struct ice_adv_rule_info *rule_info, + u16 pf_vsi_idx) +{ + rule_info->sw_act.vsi_handle = pf_vsi_idx; + rule_info->sw_act.flag |= ICE_FLTR_TX; + rule_info->flags_info.act = ICE_SINGLE_ACT_LAN_ENABLE; + rule_info->flags_info.act_valid = true; + rule_info->priority = 5; +} + +static int +ice_eswitch_br_rule_delete(struct ice_hw *hw, struct ice_rule_query_data *rule) +{ + int err; + + if (!rule) + return -EINVAL; + + err = ice_rem_adv_rule_by_id(hw, rule); + kfree(rule); + + return err; +} + +static u16 +ice_eswitch_br_get_lkups_cnt(u16 vid) +{ + return ice_eswitch_br_is_vid_valid(vid) ? 2 : 1; +} + +static void +ice_eswitch_br_add_vlan_lkup(struct ice_adv_lkup_elem *list, u16 vid) +{ + if (ice_eswitch_br_is_vid_valid(vid)) { + list[1].type = ICE_VLAN_OFOS; + list[1].h_u.vlan_hdr.vlan = cpu_to_be16(vid & VLAN_VID_MASK); + list[1].m_u.vlan_hdr.vlan = cpu_to_be16(0xFFFF); + } +} + +static struct ice_rule_query_data * +ice_eswitch_br_fwd_rule_create(struct ice_hw *hw, int vsi_idx, int port_type, + const unsigned char *mac, u16 vid) +{ + struct ice_adv_rule_info rule_info = { 0 }; + struct ice_rule_query_data *rule; + struct ice_adv_lkup_elem *list; + u16 lkups_cnt; + int err; + + lkups_cnt = ice_eswitch_br_get_lkups_cnt(vid); + + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return ERR_PTR(-ENOMEM); + + list = kcalloc(lkups_cnt, sizeof(*list), GFP_ATOMIC); + if (!list) { + err = -ENOMEM; + goto err_list_alloc; + } + + switch (port_type) { + case ICE_ESWITCH_BR_UPLINK_PORT: + ice_eswitch_br_egress_rule_setup(&rule_info, vsi_idx); + break; + case ICE_ESWITCH_BR_VF_REPR_PORT: + ice_eswitch_br_ingress_rule_setup(&rule_info, hw->pf_id, + vsi_idx); + break; + default: + err = -EINVAL; + goto err_add_rule; + } + + list[0].type = ICE_MAC_OFOS; + ether_addr_copy(list[0].h_u.eth_hdr.dst_addr, mac); + eth_broadcast_addr(list[0].m_u.eth_hdr.dst_addr); + + ice_eswitch_br_add_vlan_lkup(list, vid); + + rule_info.need_pass_l2 = true; + + rule_info.sw_act.fltr_act = ICE_FWD_TO_VSI; + + err = ice_add_adv_rule(hw, list, lkups_cnt, &rule_info, rule); + if (err) + goto err_add_rule; + + kfree(list); + + return rule; + +err_add_rule: + kfree(list); +err_list_alloc: + kfree(rule); + + return ERR_PTR(err); +} + +static struct ice_rule_query_data * +ice_eswitch_br_guard_rule_create(struct ice_hw *hw, u16 vsi_idx, + const unsigned char *mac, u16 vid) +{ + struct ice_adv_rule_info rule_info = { 0 }; + struct ice_rule_query_data *rule; + struct ice_adv_lkup_elem *list; + int err = -ENOMEM; + u16 lkups_cnt; + + lkups_cnt = ice_eswitch_br_get_lkups_cnt(vid); + + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + goto err_exit; + + list = kcalloc(lkups_cnt, sizeof(*list), GFP_ATOMIC); + if (!list) + goto err_list_alloc; + + list[0].type = ICE_MAC_OFOS; + ether_addr_copy(list[0].h_u.eth_hdr.src_addr, mac); + eth_broadcast_addr(list[0].m_u.eth_hdr.src_addr); + + ice_eswitch_br_add_vlan_lkup(list, vid); + + rule_info.allow_pass_l2 = true; + rule_info.sw_act.vsi_handle = vsi_idx; + rule_info.sw_act.fltr_act = ICE_NOP; + rule_info.priority = 5; + + err = ice_add_adv_rule(hw, list, lkups_cnt, &rule_info, rule); + if (err) + goto err_add_rule; + + kfree(list); + + return rule; + +err_add_rule: + kfree(list); +err_list_alloc: + kfree(rule); +err_exit: + return ERR_PTR(err); +} + +static struct ice_esw_br_flow * +ice_eswitch_br_flow_create(struct device *dev, struct ice_hw *hw, int vsi_idx, + int port_type, const unsigned char *mac, u16 vid) +{ + struct ice_rule_query_data *fwd_rule, *guard_rule; + struct ice_esw_br_flow *flow; + int err; + + flow = kzalloc(sizeof(*flow), GFP_KERNEL); + if (!flow) + return ERR_PTR(-ENOMEM); + + fwd_rule = ice_eswitch_br_fwd_rule_create(hw, vsi_idx, port_type, mac, + vid); + err = PTR_ERR_OR_ZERO(fwd_rule); + if (err) { + dev_err(dev, "Failed to create eswitch bridge %sgress forward rule, err: %d\n", + port_type == ICE_ESWITCH_BR_UPLINK_PORT ? "e" : "in", + err); + goto err_fwd_rule; + } + + guard_rule = ice_eswitch_br_guard_rule_create(hw, vsi_idx, mac, vid); + err = PTR_ERR_OR_ZERO(guard_rule); + if (err) { + dev_err(dev, "Failed to create eswitch bridge %sgress guard rule, err: %d\n", + port_type == ICE_ESWITCH_BR_UPLINK_PORT ? "e" : "in", + err); + goto err_guard_rule; + } + + flow->fwd_rule = fwd_rule; + flow->guard_rule = guard_rule; + + return flow; + +err_guard_rule: + ice_eswitch_br_rule_delete(hw, fwd_rule); +err_fwd_rule: + kfree(flow); + + return ERR_PTR(err); +} + +static struct ice_esw_br_fdb_entry * +ice_eswitch_br_fdb_find(struct ice_esw_br *bridge, const unsigned char *mac, + u16 vid) +{ + struct ice_esw_br_fdb_data data = { + .vid = vid, + }; + + ether_addr_copy(data.addr, mac); + return rhashtable_lookup_fast(&bridge->fdb_ht, &data, + ice_fdb_ht_params); +} + +static void +ice_eswitch_br_flow_delete(struct ice_pf *pf, struct ice_esw_br_flow *flow) +{ + struct device *dev = ice_pf_to_dev(pf); + int err; + + err = ice_eswitch_br_rule_delete(&pf->hw, flow->fwd_rule); + if (err) + dev_err(dev, "Failed to delete FDB forward rule, err: %d\n", + err); + + err = ice_eswitch_br_rule_delete(&pf->hw, flow->guard_rule); + if (err) + dev_err(dev, "Failed to delete FDB guard rule, err: %d\n", + err); + + kfree(flow); +} + +static struct ice_esw_br_vlan * +ice_esw_br_port_vlan_lookup(struct ice_esw_br *bridge, u16 vsi_idx, u16 vid) +{ + struct ice_pf *pf = bridge->br_offloads->pf; + struct device *dev = ice_pf_to_dev(pf); + struct ice_esw_br_port *port; + struct ice_esw_br_vlan *vlan; + + port = xa_load(&bridge->ports, vsi_idx); + if (!port) { + dev_info(dev, "Bridge port lookup failed (vsi=%u)\n", vsi_idx); + return ERR_PTR(-EINVAL); + } + + vlan = xa_load(&port->vlans, vid); + if (!vlan) { + dev_info(dev, "Bridge port vlan metadata lookup failed (vsi=%u)\n", + vsi_idx); + return ERR_PTR(-EINVAL); + } + + return vlan; +} + +static void +ice_eswitch_br_fdb_entry_delete(struct ice_esw_br *bridge, + struct ice_esw_br_fdb_entry *fdb_entry) +{ + struct ice_pf *pf = bridge->br_offloads->pf; + + rhashtable_remove_fast(&bridge->fdb_ht, &fdb_entry->ht_node, + ice_fdb_ht_params); + list_del(&fdb_entry->list); + + ice_eswitch_br_flow_delete(pf, fdb_entry->flow); + + kfree(fdb_entry); +} + +static void +ice_eswitch_br_fdb_offload_notify(struct net_device *dev, + const unsigned char *mac, u16 vid, + unsigned long val) +{ + struct switchdev_notifier_fdb_info fdb_info = { + .addr = mac, + .vid = vid, + .offloaded = true, + }; + + call_switchdev_notifiers(val, dev, &fdb_info.info, NULL); +} + +static void +ice_eswitch_br_fdb_entry_notify_and_cleanup(struct ice_esw_br *bridge, + struct ice_esw_br_fdb_entry *entry) +{ + if (!(entry->flags & ICE_ESWITCH_BR_FDB_ADDED_BY_USER)) + ice_eswitch_br_fdb_offload_notify(entry->dev, entry->data.addr, + entry->data.vid, + SWITCHDEV_FDB_DEL_TO_BRIDGE); + ice_eswitch_br_fdb_entry_delete(bridge, entry); +} + +static void +ice_eswitch_br_fdb_entry_find_and_delete(struct ice_esw_br *bridge, + const unsigned char *mac, u16 vid) +{ + struct ice_pf *pf = bridge->br_offloads->pf; + struct ice_esw_br_fdb_entry *fdb_entry; + struct device *dev = ice_pf_to_dev(pf); + + fdb_entry = ice_eswitch_br_fdb_find(bridge, mac, vid); + if (!fdb_entry) { + dev_err(dev, "FDB entry with mac: %pM and vid: %u not found\n", + mac, vid); + return; + } + + trace_ice_eswitch_br_fdb_entry_find_and_delete(fdb_entry); + ice_eswitch_br_fdb_entry_notify_and_cleanup(bridge, fdb_entry); +} + +static void +ice_eswitch_br_fdb_entry_create(struct net_device *netdev, + struct ice_esw_br_port *br_port, + bool added_by_user, + const unsigned char *mac, u16 vid) +{ + struct ice_esw_br *bridge = br_port->bridge; + struct ice_pf *pf = bridge->br_offloads->pf; + struct device *dev = ice_pf_to_dev(pf); + struct ice_esw_br_fdb_entry *fdb_entry; + struct ice_esw_br_flow *flow; + struct ice_esw_br_vlan *vlan; + struct ice_hw *hw = &pf->hw; + unsigned long event; + int err; + + /* untagged filtering is not yet supported */ + if (!(bridge->flags & ICE_ESWITCH_BR_VLAN_FILTERING) && vid) + return; + + if ((bridge->flags & ICE_ESWITCH_BR_VLAN_FILTERING)) { + vlan = ice_esw_br_port_vlan_lookup(bridge, br_port->vsi_idx, + vid); + if (IS_ERR(vlan)) { + dev_err(dev, "Failed to find vlan lookup, err: %ld\n", + PTR_ERR(vlan)); + return; + } + } + + fdb_entry = ice_eswitch_br_fdb_find(bridge, mac, vid); + if (fdb_entry) + ice_eswitch_br_fdb_entry_notify_and_cleanup(bridge, fdb_entry); + + fdb_entry = kzalloc(sizeof(*fdb_entry), GFP_KERNEL); + if (!fdb_entry) { + err = -ENOMEM; + goto err_exit; + } + + flow = ice_eswitch_br_flow_create(dev, hw, br_port->vsi_idx, + br_port->type, mac, vid); + if (IS_ERR(flow)) { + err = PTR_ERR(flow); + goto err_add_flow; + } + + ether_addr_copy(fdb_entry->data.addr, mac); + fdb_entry->data.vid = vid; + fdb_entry->br_port = br_port; + fdb_entry->flow = flow; + fdb_entry->dev = netdev; + fdb_entry->last_use = jiffies; + event = SWITCHDEV_FDB_ADD_TO_BRIDGE; + + if (added_by_user) { + fdb_entry->flags |= ICE_ESWITCH_BR_FDB_ADDED_BY_USER; + event = SWITCHDEV_FDB_OFFLOADED; + } + + err = rhashtable_insert_fast(&bridge->fdb_ht, &fdb_entry->ht_node, + ice_fdb_ht_params); + if (err) + goto err_fdb_insert; + + list_add(&fdb_entry->list, &bridge->fdb_list); + trace_ice_eswitch_br_fdb_entry_create(fdb_entry); + + ice_eswitch_br_fdb_offload_notify(netdev, mac, vid, event); + + return; + +err_fdb_insert: + ice_eswitch_br_flow_delete(pf, flow); +err_add_flow: + kfree(fdb_entry); +err_exit: + dev_err(dev, "Failed to create fdb entry, err: %d\n", err); +} + +static void +ice_eswitch_br_fdb_work_dealloc(struct ice_esw_br_fdb_work *fdb_work) +{ + kfree(fdb_work->fdb_info.addr); + kfree(fdb_work); +} + +static void +ice_eswitch_br_fdb_event_work(struct work_struct *work) +{ + struct ice_esw_br_fdb_work *fdb_work = ice_work_to_fdb_work(work); + bool added_by_user = fdb_work->fdb_info.added_by_user; + const unsigned char *mac = fdb_work->fdb_info.addr; + u16 vid = fdb_work->fdb_info.vid; + struct ice_esw_br_port *br_port; + + rtnl_lock(); + + br_port = ice_eswitch_br_netdev_to_port(fdb_work->dev); + if (!br_port) + goto err_exit; + + switch (fdb_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + ice_eswitch_br_fdb_entry_create(fdb_work->dev, br_port, + added_by_user, mac, vid); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + ice_eswitch_br_fdb_entry_find_and_delete(br_port->bridge, + mac, vid); + break; + default: + goto err_exit; + } + +err_exit: + rtnl_unlock(); + dev_put(fdb_work->dev); + ice_eswitch_br_fdb_work_dealloc(fdb_work); +} + +static struct ice_esw_br_fdb_work * +ice_eswitch_br_fdb_work_alloc(struct switchdev_notifier_fdb_info *fdb_info, + struct net_device *dev, + unsigned long event) +{ + struct ice_esw_br_fdb_work *work; + unsigned char *mac; + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return ERR_PTR(-ENOMEM); + + INIT_WORK(&work->work, ice_eswitch_br_fdb_event_work); + memcpy(&work->fdb_info, fdb_info, sizeof(work->fdb_info)); + + mac = kzalloc(ETH_ALEN, GFP_ATOMIC); + if (!mac) { + kfree(work); + return ERR_PTR(-ENOMEM); + } + + ether_addr_copy(mac, fdb_info->addr); + work->fdb_info.addr = mac; + work->event = event; + work->dev = dev; + + return work; +} + +static int +ice_eswitch_br_switchdev_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + struct switchdev_notifier_fdb_info *fdb_info; + struct switchdev_notifier_info *info = ptr; + struct ice_esw_br_offloads *br_offloads; + struct ice_esw_br_fdb_work *work; + struct netlink_ext_ack *extack; + struct net_device *upper; + + br_offloads = ice_nb_to_br_offloads(nb, switchdev_nb); + extack = switchdev_notifier_info_to_extack(ptr); + + upper = netdev_master_upper_dev_get_rcu(dev); + if (!upper) + return NOTIFY_DONE; + + if (!netif_is_bridge_master(upper)) + return NOTIFY_DONE; + + if (!ice_eswitch_br_is_dev_valid(dev)) + return NOTIFY_DONE; + + if (!ice_eswitch_br_netdev_to_port(dev)) + return NOTIFY_DONE; + + switch (event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + case SWITCHDEV_FDB_DEL_TO_DEVICE: + fdb_info = container_of(info, typeof(*fdb_info), info); + + work = ice_eswitch_br_fdb_work_alloc(fdb_info, dev, event); + if (IS_ERR(work)) { + NL_SET_ERR_MSG_MOD(extack, "Failed to init switchdev fdb work"); + return notifier_from_errno(PTR_ERR(work)); + } + dev_hold(dev); + + queue_work(br_offloads->wq, &work->work); + break; + default: + break; + } + return NOTIFY_DONE; +} + +static void ice_eswitch_br_fdb_flush(struct ice_esw_br *bridge) +{ + struct ice_esw_br_fdb_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) + ice_eswitch_br_fdb_entry_notify_and_cleanup(bridge, entry); +} + +static void +ice_eswitch_br_vlan_filtering_set(struct ice_esw_br *bridge, bool enable) +{ + if (enable == !!(bridge->flags & ICE_ESWITCH_BR_VLAN_FILTERING)) + return; + + ice_eswitch_br_fdb_flush(bridge); + if (enable) + bridge->flags |= ICE_ESWITCH_BR_VLAN_FILTERING; + else + bridge->flags &= ~ICE_ESWITCH_BR_VLAN_FILTERING; +} + +static void +ice_eswitch_br_clear_pvid(struct ice_esw_br_port *port) +{ + struct ice_vlan port_vlan = ICE_VLAN(ETH_P_8021Q, port->pvid, 0); + struct ice_vsi_vlan_ops *vlan_ops; + + vlan_ops = ice_get_compat_vsi_vlan_ops(port->vsi); + + vlan_ops->del_vlan(port->vsi, &port_vlan); + vlan_ops->clear_port_vlan(port->vsi); + + ice_vf_vsi_disable_port_vlan(port->vsi); + + port->pvid = 0; +} + +static void +ice_eswitch_br_vlan_cleanup(struct ice_esw_br_port *port, + struct ice_esw_br_vlan *vlan) +{ + struct ice_esw_br_fdb_entry *fdb_entry, *tmp; + struct ice_esw_br *bridge = port->bridge; + + trace_ice_eswitch_br_vlan_cleanup(vlan); + + list_for_each_entry_safe(fdb_entry, tmp, &bridge->fdb_list, list) { + if (vlan->vid == fdb_entry->data.vid) + ice_eswitch_br_fdb_entry_delete(bridge, fdb_entry); + } + + xa_erase(&port->vlans, vlan->vid); + if (port->pvid == vlan->vid) + ice_eswitch_br_clear_pvid(port); + kfree(vlan); +} + +static void ice_eswitch_br_port_vlans_flush(struct ice_esw_br_port *port) +{ + struct ice_esw_br_vlan *vlan; + unsigned long index; + + xa_for_each(&port->vlans, index, vlan) + ice_eswitch_br_vlan_cleanup(port, vlan); +} + +static int +ice_eswitch_br_set_pvid(struct ice_esw_br_port *port, + struct ice_esw_br_vlan *vlan) +{ + struct ice_vlan port_vlan = ICE_VLAN(ETH_P_8021Q, vlan->vid, 0); + struct device *dev = ice_pf_to_dev(port->vsi->back); + struct ice_vsi_vlan_ops *vlan_ops; + int err; + + if (port->pvid == vlan->vid || vlan->vid == 1) + return 0; + + /* Setting port vlan on uplink isn't supported by hw */ + if (port->type == ICE_ESWITCH_BR_UPLINK_PORT) + return -EOPNOTSUPP; + + if (port->pvid) { + dev_info(dev, + "Port VLAN (vsi=%u, vid=%u) already exists on the port, remove it before adding new one\n", + port->vsi_idx, port->pvid); + return -EEXIST; + } + + ice_vf_vsi_enable_port_vlan(port->vsi); + + vlan_ops = ice_get_compat_vsi_vlan_ops(port->vsi); + err = vlan_ops->set_port_vlan(port->vsi, &port_vlan); + if (err) + return err; + + err = vlan_ops->add_vlan(port->vsi, &port_vlan); + if (err) + return err; + + ice_eswitch_br_port_vlans_flush(port); + port->pvid = vlan->vid; + + return 0; +} + +static struct ice_esw_br_vlan * +ice_eswitch_br_vlan_create(u16 vid, u16 flags, struct ice_esw_br_port *port) +{ + struct device *dev = ice_pf_to_dev(port->vsi->back); + struct ice_esw_br_vlan *vlan; + int err; + + vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); + if (!vlan) + return ERR_PTR(-ENOMEM); + + vlan->vid = vid; + vlan->flags = flags; + if ((flags & BRIDGE_VLAN_INFO_PVID) && + (flags & BRIDGE_VLAN_INFO_UNTAGGED)) { + err = ice_eswitch_br_set_pvid(port, vlan); + if (err) + goto err_set_pvid; + } else if ((flags & BRIDGE_VLAN_INFO_PVID) || + (flags & BRIDGE_VLAN_INFO_UNTAGGED)) { + dev_info(dev, "VLAN push and pop are supported only simultaneously\n"); + err = -EOPNOTSUPP; + goto err_set_pvid; + } + + err = xa_insert(&port->vlans, vlan->vid, vlan, GFP_KERNEL); + if (err) + goto err_insert; + + trace_ice_eswitch_br_vlan_create(vlan); + + return vlan; + +err_insert: + if (port->pvid) + ice_eswitch_br_clear_pvid(port); +err_set_pvid: + kfree(vlan); + return ERR_PTR(err); +} + +static int +ice_eswitch_br_port_vlan_add(struct ice_esw_br *bridge, u16 vsi_idx, u16 vid, + u16 flags, struct netlink_ext_ack *extack) +{ + struct ice_esw_br_port *port; + struct ice_esw_br_vlan *vlan; + + port = xa_load(&bridge->ports, vsi_idx); + if (!port) + return -EINVAL; + + if (port->pvid) { + dev_info(ice_pf_to_dev(port->vsi->back), + "Port VLAN (vsi=%u, vid=%d) exists on the port, remove it to add trunk VLANs\n", + port->vsi_idx, port->pvid); + return -EEXIST; + } + + vlan = xa_load(&port->vlans, vid); + if (vlan) { + if (vlan->flags == flags) + return 0; + + ice_eswitch_br_vlan_cleanup(port, vlan); + } + + vlan = ice_eswitch_br_vlan_create(vid, flags, port); + if (IS_ERR(vlan)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Failed to create VLAN entry, vid: %u, vsi: %u", + vid, vsi_idx); + return PTR_ERR(vlan); + } + + return 0; +} + +static void +ice_eswitch_br_port_vlan_del(struct ice_esw_br *bridge, u16 vsi_idx, u16 vid) +{ + struct ice_esw_br_port *port; + struct ice_esw_br_vlan *vlan; + + port = xa_load(&bridge->ports, vsi_idx); + if (!port) + return; + + vlan = xa_load(&port->vlans, vid); + if (!vlan) + return; + + ice_eswitch_br_vlan_cleanup(port, vlan); +} + +static int +ice_eswitch_br_port_obj_add(struct net_device *netdev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack) +{ + struct ice_esw_br_port *br_port = ice_eswitch_br_netdev_to_port(netdev); + struct switchdev_obj_port_vlan *vlan; + int err; + + if (!br_port) + return -EINVAL; + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); + err = ice_eswitch_br_port_vlan_add(br_port->bridge, + br_port->vsi_idx, vlan->vid, + vlan->flags, extack); + return err; + default: + return -EOPNOTSUPP; + } +} + +static int +ice_eswitch_br_port_obj_del(struct net_device *netdev, const void *ctx, + const struct switchdev_obj *obj) +{ + struct ice_esw_br_port *br_port = ice_eswitch_br_netdev_to_port(netdev); + struct switchdev_obj_port_vlan *vlan; + + if (!br_port) + return -EINVAL; + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); + ice_eswitch_br_port_vlan_del(br_port->bridge, br_port->vsi_idx, + vlan->vid); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int +ice_eswitch_br_port_obj_attr_set(struct net_device *netdev, const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack) +{ + struct ice_esw_br_port *br_port = ice_eswitch_br_netdev_to_port(netdev); + + if (!br_port) + return -EINVAL; + + switch (attr->id) { + case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING: + ice_eswitch_br_vlan_filtering_set(br_port->bridge, + attr->u.vlan_filtering); + return 0; + case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME: + br_port->bridge->ageing_time = + clock_t_to_jiffies(attr->u.ageing_time); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int +ice_eswitch_br_event_blocking(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + int err; + + switch (event) { + case SWITCHDEV_PORT_OBJ_ADD: + err = switchdev_handle_port_obj_add(dev, ptr, + ice_eswitch_br_is_dev_valid, + ice_eswitch_br_port_obj_add); + break; + case SWITCHDEV_PORT_OBJ_DEL: + err = switchdev_handle_port_obj_del(dev, ptr, + ice_eswitch_br_is_dev_valid, + ice_eswitch_br_port_obj_del); + break; + case SWITCHDEV_PORT_ATTR_SET: + err = switchdev_handle_port_attr_set(dev, ptr, + ice_eswitch_br_is_dev_valid, + ice_eswitch_br_port_obj_attr_set); + break; + default: + err = 0; + } + + return notifier_from_errno(err); +} + +static void +ice_eswitch_br_port_deinit(struct ice_esw_br *bridge, + struct ice_esw_br_port *br_port) +{ + struct ice_esw_br_fdb_entry *fdb_entry, *tmp; + struct ice_vsi *vsi = br_port->vsi; + + list_for_each_entry_safe(fdb_entry, tmp, &bridge->fdb_list, list) { + if (br_port == fdb_entry->br_port) + ice_eswitch_br_fdb_entry_delete(bridge, fdb_entry); + } + + if (br_port->type == ICE_ESWITCH_BR_UPLINK_PORT && vsi->back) + vsi->back->br_port = NULL; + else if (vsi->vf && vsi->vf->repr) + vsi->vf->repr->br_port = NULL; + + xa_erase(&bridge->ports, br_port->vsi_idx); + ice_eswitch_br_port_vlans_flush(br_port); + kfree(br_port); +} + +static struct ice_esw_br_port * +ice_eswitch_br_port_init(struct ice_esw_br *bridge) +{ + struct ice_esw_br_port *br_port; + + br_port = kzalloc(sizeof(*br_port), GFP_KERNEL); + if (!br_port) + return ERR_PTR(-ENOMEM); + + xa_init(&br_port->vlans); + + br_port->bridge = bridge; + + return br_port; +} + +static int +ice_eswitch_br_vf_repr_port_init(struct ice_esw_br *bridge, + struct ice_repr *repr) +{ + struct ice_esw_br_port *br_port; + int err; + + br_port = ice_eswitch_br_port_init(bridge); + if (IS_ERR(br_port)) + return PTR_ERR(br_port); + + br_port->vsi = repr->src_vsi; + br_port->vsi_idx = br_port->vsi->idx; + br_port->type = ICE_ESWITCH_BR_VF_REPR_PORT; + repr->br_port = br_port; + + err = xa_insert(&bridge->ports, br_port->vsi_idx, br_port, GFP_KERNEL); + if (err) { + ice_eswitch_br_port_deinit(bridge, br_port); + return err; + } + + return 0; +} + +static int +ice_eswitch_br_uplink_port_init(struct ice_esw_br *bridge, struct ice_pf *pf) +{ + struct ice_vsi *vsi = pf->switchdev.uplink_vsi; + struct ice_esw_br_port *br_port; + int err; + + br_port = ice_eswitch_br_port_init(bridge); + if (IS_ERR(br_port)) + return PTR_ERR(br_port); + + br_port->vsi = vsi; + br_port->vsi_idx = br_port->vsi->idx; + br_port->type = ICE_ESWITCH_BR_UPLINK_PORT; + pf->br_port = br_port; + + err = xa_insert(&bridge->ports, br_port->vsi_idx, br_port, GFP_KERNEL); + if (err) { + ice_eswitch_br_port_deinit(bridge, br_port); + return err; + } + + return 0; +} + +static void +ice_eswitch_br_ports_flush(struct ice_esw_br *bridge) +{ + struct ice_esw_br_port *port; + unsigned long i; + + xa_for_each(&bridge->ports, i, port) + ice_eswitch_br_port_deinit(bridge, port); +} + +static void +ice_eswitch_br_deinit(struct ice_esw_br_offloads *br_offloads, + struct ice_esw_br *bridge) +{ + if (!bridge) + return; + + /* Cleanup all the ports that were added asynchronously + * through NETDEV_CHANGEUPPER event. + */ + ice_eswitch_br_ports_flush(bridge); + WARN_ON(!xa_empty(&bridge->ports)); + xa_destroy(&bridge->ports); + rhashtable_destroy(&bridge->fdb_ht); + + br_offloads->bridge = NULL; + kfree(bridge); +} + +static struct ice_esw_br * +ice_eswitch_br_init(struct ice_esw_br_offloads *br_offloads, int ifindex) +{ + struct ice_esw_br *bridge; + int err; + + bridge = kzalloc(sizeof(*bridge), GFP_KERNEL); + if (!bridge) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(&bridge->fdb_ht, &ice_fdb_ht_params); + if (err) { + kfree(bridge); + return ERR_PTR(err); + } + + INIT_LIST_HEAD(&bridge->fdb_list); + bridge->br_offloads = br_offloads; + bridge->ifindex = ifindex; + bridge->ageing_time = clock_t_to_jiffies(BR_DEFAULT_AGEING_TIME); + xa_init(&bridge->ports); + br_offloads->bridge = bridge; + + return bridge; +} + +static struct ice_esw_br * +ice_eswitch_br_get(struct ice_esw_br_offloads *br_offloads, int ifindex, + struct netlink_ext_ack *extack) +{ + struct ice_esw_br *bridge = br_offloads->bridge; + + if (bridge) { + if (bridge->ifindex != ifindex) { + NL_SET_ERR_MSG_MOD(extack, + "Only one bridge is supported per eswitch"); + return ERR_PTR(-EOPNOTSUPP); + } + return bridge; + } + + /* Create the bridge if it doesn't exist yet */ + bridge = ice_eswitch_br_init(br_offloads, ifindex); + if (IS_ERR(bridge)) + NL_SET_ERR_MSG_MOD(extack, "Failed to init the bridge"); + + return bridge; +} + +static void +ice_eswitch_br_verify_deinit(struct ice_esw_br_offloads *br_offloads, + struct ice_esw_br *bridge) +{ + /* Remove the bridge if it exists and there are no ports left */ + if (!bridge || !xa_empty(&bridge->ports)) + return; + + ice_eswitch_br_deinit(br_offloads, bridge); +} + +static int +ice_eswitch_br_port_unlink(struct ice_esw_br_offloads *br_offloads, + struct net_device *dev, int ifindex, + struct netlink_ext_ack *extack) +{ + struct ice_esw_br_port *br_port = ice_eswitch_br_netdev_to_port(dev); + struct ice_esw_br *bridge; + + if (!br_port) { + NL_SET_ERR_MSG_MOD(extack, + "Port representor is not attached to any bridge"); + return -EINVAL; + } + + if (br_port->bridge->ifindex != ifindex) { + NL_SET_ERR_MSG_MOD(extack, + "Port representor is attached to another bridge"); + return -EINVAL; + } + + bridge = br_port->bridge; + + trace_ice_eswitch_br_port_unlink(br_port); + ice_eswitch_br_port_deinit(br_port->bridge, br_port); + ice_eswitch_br_verify_deinit(br_offloads, bridge); + + return 0; +} + +static int +ice_eswitch_br_port_link(struct ice_esw_br_offloads *br_offloads, + struct net_device *dev, int ifindex, + struct netlink_ext_ack *extack) +{ + struct ice_esw_br *bridge; + int err; + + if (ice_eswitch_br_netdev_to_port(dev)) { + NL_SET_ERR_MSG_MOD(extack, + "Port is already attached to the bridge"); + return -EINVAL; + } + + bridge = ice_eswitch_br_get(br_offloads, ifindex, extack); + if (IS_ERR(bridge)) + return PTR_ERR(bridge); + + if (ice_is_port_repr_netdev(dev)) { + struct ice_repr *repr = ice_netdev_to_repr(dev); + + err = ice_eswitch_br_vf_repr_port_init(bridge, repr); + trace_ice_eswitch_br_port_link(repr->br_port); + } else { + struct ice_pf *pf = ice_netdev_to_pf(dev); + + err = ice_eswitch_br_uplink_port_init(bridge, pf); + trace_ice_eswitch_br_port_link(pf->br_port); + } + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to init bridge port"); + goto err_port_init; + } + + return 0; + +err_port_init: + ice_eswitch_br_verify_deinit(br_offloads, bridge); + return err; +} + +static int +ice_eswitch_br_port_changeupper(struct notifier_block *nb, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct netdev_notifier_changeupper_info *info = ptr; + struct ice_esw_br_offloads *br_offloads; + struct netlink_ext_ack *extack; + struct net_device *upper; + + br_offloads = ice_nb_to_br_offloads(nb, netdev_nb); + + if (!ice_eswitch_br_is_dev_valid(dev)) + return 0; + + upper = info->upper_dev; + if (!netif_is_bridge_master(upper)) + return 0; + + extack = netdev_notifier_info_to_extack(&info->info); + + if (info->linking) + return ice_eswitch_br_port_link(br_offloads, dev, + upper->ifindex, extack); + else + return ice_eswitch_br_port_unlink(br_offloads, dev, + upper->ifindex, extack); +} + +static int +ice_eswitch_br_port_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + int err = 0; + + switch (event) { + case NETDEV_CHANGEUPPER: + err = ice_eswitch_br_port_changeupper(nb, ptr); + break; + } + + return notifier_from_errno(err); +} + +static void +ice_eswitch_br_offloads_dealloc(struct ice_pf *pf) +{ + struct ice_esw_br_offloads *br_offloads = pf->switchdev.br_offloads; + + ASSERT_RTNL(); + + if (!br_offloads) + return; + + ice_eswitch_br_deinit(br_offloads, br_offloads->bridge); + + pf->switchdev.br_offloads = NULL; + kfree(br_offloads); +} + +static struct ice_esw_br_offloads * +ice_eswitch_br_offloads_alloc(struct ice_pf *pf) +{ + struct ice_esw_br_offloads *br_offloads; + + ASSERT_RTNL(); + + if (pf->switchdev.br_offloads) + return ERR_PTR(-EEXIST); + + br_offloads = kzalloc(sizeof(*br_offloads), GFP_KERNEL); + if (!br_offloads) + return ERR_PTR(-ENOMEM); + + pf->switchdev.br_offloads = br_offloads; + br_offloads->pf = pf; + + return br_offloads; +} + +void +ice_eswitch_br_offloads_deinit(struct ice_pf *pf) +{ + struct ice_esw_br_offloads *br_offloads; + + br_offloads = pf->switchdev.br_offloads; + if (!br_offloads) + return; + + cancel_delayed_work_sync(&br_offloads->update_work); + unregister_netdevice_notifier(&br_offloads->netdev_nb); + unregister_switchdev_blocking_notifier(&br_offloads->switchdev_blk); + unregister_switchdev_notifier(&br_offloads->switchdev_nb); + destroy_workqueue(br_offloads->wq); + /* Although notifier block is unregistered just before, + * so we don't get any new events, some events might be + * already in progress. Hold the rtnl lock and wait for + * them to finished. + */ + rtnl_lock(); + ice_eswitch_br_offloads_dealloc(pf); + rtnl_unlock(); +} + +static void ice_eswitch_br_update(struct ice_esw_br_offloads *br_offloads) +{ + struct ice_esw_br *bridge = br_offloads->bridge; + struct ice_esw_br_fdb_entry *entry, *tmp; + + if (!bridge) + return; + + rtnl_lock(); + list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) { + if (entry->flags & ICE_ESWITCH_BR_FDB_ADDED_BY_USER) + continue; + + if (time_is_after_eq_jiffies(entry->last_use + + bridge->ageing_time)) + continue; + + ice_eswitch_br_fdb_entry_notify_and_cleanup(bridge, entry); + } + rtnl_unlock(); +} + +static void ice_eswitch_br_update_work(struct work_struct *work) +{ + struct ice_esw_br_offloads *br_offloads; + + br_offloads = ice_work_to_br_offloads(work); + + ice_eswitch_br_update(br_offloads); + + queue_delayed_work(br_offloads->wq, &br_offloads->update_work, + ICE_ESW_BRIDGE_UPDATE_INTERVAL); +} + +int +ice_eswitch_br_offloads_init(struct ice_pf *pf) +{ + struct ice_esw_br_offloads *br_offloads; + struct device *dev = ice_pf_to_dev(pf); + int err; + + rtnl_lock(); + br_offloads = ice_eswitch_br_offloads_alloc(pf); + rtnl_unlock(); + if (IS_ERR(br_offloads)) { + dev_err(dev, "Failed to init eswitch bridge\n"); + return PTR_ERR(br_offloads); + } + + br_offloads->wq = alloc_ordered_workqueue("ice_bridge_wq", 0); + if (!br_offloads->wq) { + err = -ENOMEM; + dev_err(dev, "Failed to allocate bridge workqueue\n"); + goto err_alloc_wq; + } + + br_offloads->switchdev_nb.notifier_call = + ice_eswitch_br_switchdev_event; + err = register_switchdev_notifier(&br_offloads->switchdev_nb); + if (err) { + dev_err(dev, + "Failed to register switchdev notifier\n"); + goto err_reg_switchdev_nb; + } + + br_offloads->switchdev_blk.notifier_call = + ice_eswitch_br_event_blocking; + err = register_switchdev_blocking_notifier(&br_offloads->switchdev_blk); + if (err) { + dev_err(dev, + "Failed to register bridge blocking switchdev notifier\n"); + goto err_reg_switchdev_blk; + } + + br_offloads->netdev_nb.notifier_call = ice_eswitch_br_port_event; + err = register_netdevice_notifier(&br_offloads->netdev_nb); + if (err) { + dev_err(dev, + "Failed to register bridge port event notifier\n"); + goto err_reg_netdev_nb; + } + + INIT_DELAYED_WORK(&br_offloads->update_work, + ice_eswitch_br_update_work); + queue_delayed_work(br_offloads->wq, &br_offloads->update_work, + ICE_ESW_BRIDGE_UPDATE_INTERVAL); + + return 0; + +err_reg_netdev_nb: + unregister_switchdev_blocking_notifier(&br_offloads->switchdev_blk); +err_reg_switchdev_blk: + unregister_switchdev_notifier(&br_offloads->switchdev_nb); +err_reg_switchdev_nb: + destroy_workqueue(br_offloads->wq); +err_alloc_wq: + rtnl_lock(); + ice_eswitch_br_offloads_dealloc(pf); + rtnl_unlock(); + + return err; +} diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch_br.h b/drivers/net/ethernet/intel/ice/ice_eswitch_br.h new file mode 100644 index 000000000000..85a8fadb2928 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_eswitch_br.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2023, Intel Corporation. */ + +#ifndef _ICE_ESWITCH_BR_H_ +#define _ICE_ESWITCH_BR_H_ + +#include <linux/rhashtable.h> +#include <linux/workqueue.h> + +struct ice_esw_br_fdb_data { + unsigned char addr[ETH_ALEN]; + u16 vid; +}; + +struct ice_esw_br_flow { + struct ice_rule_query_data *fwd_rule; + struct ice_rule_query_data *guard_rule; +}; + +enum { + ICE_ESWITCH_BR_FDB_ADDED_BY_USER = BIT(0), +}; + +struct ice_esw_br_fdb_entry { + struct ice_esw_br_fdb_data data; + struct rhash_head ht_node; + struct list_head list; + + int flags; + + struct net_device *dev; + struct ice_esw_br_port *br_port; + struct ice_esw_br_flow *flow; + + unsigned long last_use; +}; + +enum ice_esw_br_port_type { + ICE_ESWITCH_BR_UPLINK_PORT = 0, + ICE_ESWITCH_BR_VF_REPR_PORT = 1, +}; + +struct ice_esw_br_port { + struct ice_esw_br *bridge; + struct ice_vsi *vsi; + enum ice_esw_br_port_type type; + u16 vsi_idx; + u16 pvid; + struct xarray vlans; +}; + +enum { + ICE_ESWITCH_BR_VLAN_FILTERING = BIT(0), +}; + +struct ice_esw_br { + struct ice_esw_br_offloads *br_offloads; + struct xarray ports; + + struct rhashtable fdb_ht; + struct list_head fdb_list; + + int ifindex; + u32 flags; + unsigned long ageing_time; +}; + +struct ice_esw_br_offloads { + struct ice_pf *pf; + struct ice_esw_br *bridge; + struct notifier_block netdev_nb; + struct notifier_block switchdev_blk; + struct notifier_block switchdev_nb; + + struct workqueue_struct *wq; + struct delayed_work update_work; +}; + +struct ice_esw_br_fdb_work { + struct work_struct work; + struct switchdev_notifier_fdb_info fdb_info; + struct net_device *dev; + unsigned long event; +}; + +struct ice_esw_br_vlan { + u16 vid; + u16 flags; +}; + +#define ice_nb_to_br_offloads(nb, nb_name) \ + container_of(nb, \ + struct ice_esw_br_offloads, \ + nb_name) + +#define ice_work_to_br_offloads(w) \ + container_of(w, \ + struct ice_esw_br_offloads, \ + update_work.work) + +#define ice_work_to_fdb_work(w) \ + container_of(w, \ + struct ice_esw_br_fdb_work, \ + work) + +static inline bool ice_eswitch_br_is_vid_valid(u16 vid) +{ + /* In trunk VLAN mode, for untagged traffic the bridge sends requests + * to offload VLAN 1 with pvid and untagged flags set. Since these + * flags are not supported, add a MAC filter instead. + */ + return vid > 1; +} + +void +ice_eswitch_br_offloads_deinit(struct ice_pf *pf); +int +ice_eswitch_br_offloads_init(struct ice_pf *pf); + +#endif /* _ICE_ESWITCH_BR_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index 5a7753bda324..36b7044717e8 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -4,8 +4,24 @@ /* Link Aggregation code */ #include "ice.h" +#include "ice_lib.h" #include "ice_lag.h" +#define ICE_LAG_RES_SHARED BIT(14) +#define ICE_LAG_RES_VALID BIT(15) + +#define LACP_TRAIN_PKT_LEN 16 +static const u8 lacp_train_pkt[LACP_TRAIN_PKT_LEN] = { 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0x88, 0x09, 0, 0 }; + +#define ICE_RECIPE_LEN 64 +static const u8 ice_dflt_vsi_rcp[ICE_RECIPE_LEN] = { + 0x05, 0, 0, 0, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x85, 0, 0x01, 0, 0, 0, 0xff, 0xff, 0x08, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + /** * ice_lag_set_primary - set PF LAG state as Primary * @lag: LAG info struct @@ -47,16 +63,221 @@ static void ice_lag_set_backup(struct ice_lag *lag) } /** + * netif_is_same_ice - determine if netdev is on the same ice NIC as local PF + * @pf: local PF struct + * @netdev: netdev we are evaluating + */ +static bool netif_is_same_ice(struct ice_pf *pf, struct net_device *netdev) +{ + struct ice_netdev_priv *np; + struct ice_pf *test_pf; + struct ice_vsi *vsi; + + if (!netif_is_ice(netdev)) + return false; + + np = netdev_priv(netdev); + if (!np) + return false; + + vsi = np->vsi; + if (!vsi) + return false; + + test_pf = vsi->back; + if (!test_pf) + return false; + + if (pf->pdev->bus != test_pf->pdev->bus || + pf->pdev->slot != test_pf->pdev->slot) + return false; + + return true; +} + +/** + * ice_netdev_to_lag - return pointer to associated lag struct from netdev + * @netdev: pointer to net_device struct to query + */ +static struct ice_lag *ice_netdev_to_lag(struct net_device *netdev) +{ + struct ice_netdev_priv *np; + struct ice_vsi *vsi; + + if (!netif_is_ice(netdev)) + return NULL; + + np = netdev_priv(netdev); + if (!np) + return NULL; + + vsi = np->vsi; + if (!vsi) + return NULL; + + return vsi->back->lag; +} + +/** + * ice_lag_find_hw_by_lport - return an hw struct from bond members lport + * @lag: lag struct + * @lport: lport value to search for + */ +static struct ice_hw * +ice_lag_find_hw_by_lport(struct ice_lag *lag, u8 lport) +{ + struct ice_lag_netdev_list *entry; + struct net_device *tmp_netdev; + struct ice_netdev_priv *np; + struct list_head *tmp; + struct ice_hw *hw; + + list_for_each(tmp, lag->netdev_head) { + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + tmp_netdev = entry->netdev; + if (!tmp_netdev || !netif_is_ice(tmp_netdev)) + continue; + + np = netdev_priv(tmp_netdev); + if (!np || !np->vsi) + continue; + + hw = &np->vsi->back->hw; + if (hw->port_info->lport == lport) + return hw; + } + + return NULL; +} + +/** + * ice_lag_find_primary - returns pointer to primary interfaces lag struct + * @lag: local interfaces lag struct + */ +static struct ice_lag *ice_lag_find_primary(struct ice_lag *lag) +{ + struct ice_lag *primary_lag = NULL; + struct list_head *tmp; + + list_for_each(tmp, lag->netdev_head) { + struct ice_lag_netdev_list *entry; + struct ice_lag *tmp_lag; + + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + tmp_lag = ice_netdev_to_lag(entry->netdev); + if (tmp_lag && tmp_lag->primary) { + primary_lag = tmp_lag; + break; + } + } + + return primary_lag; +} + +/** + * ice_lag_cfg_dflt_fltr - Add/Remove default VSI rule for LAG + * @lag: lag struct for local interface + * @add: boolean on whether we are adding filters + */ +static int +ice_lag_cfg_dflt_fltr(struct ice_lag *lag, bool add) +{ + struct ice_sw_rule_lkup_rx_tx *s_rule; + u16 s_rule_sz, vsi_num; + struct ice_hw *hw; + u32 act, opc; + u8 *eth_hdr; + int err; + + hw = &lag->pf->hw; + vsi_num = ice_get_hw_vsi_num(hw, 0); + + s_rule_sz = ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s_rule); + s_rule = kzalloc(s_rule_sz, GFP_KERNEL); + if (!s_rule) { + dev_err(ice_pf_to_dev(lag->pf), "error allocating rule for LAG default VSI\n"); + return -ENOMEM; + } + + if (add) { + eth_hdr = s_rule->hdr_data; + ice_fill_eth_hdr(eth_hdr); + + act = (vsi_num << ICE_SINGLE_ACT_VSI_ID_S) & + ICE_SINGLE_ACT_VSI_ID_M; + act |= ICE_SINGLE_ACT_VSI_FORWARDING | + ICE_SINGLE_ACT_VALID_BIT | ICE_SINGLE_ACT_LAN_ENABLE; + + s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_LKUP_RX); + s_rule->recipe_id = cpu_to_le16(lag->pf_recipe); + s_rule->src = cpu_to_le16(hw->port_info->lport); + s_rule->act = cpu_to_le32(act); + s_rule->hdr_len = cpu_to_le16(DUMMY_ETH_HDR_LEN); + opc = ice_aqc_opc_add_sw_rules; + } else { + s_rule->index = cpu_to_le16(lag->pf_rule_id); + opc = ice_aqc_opc_remove_sw_rules; + } + + err = ice_aq_sw_rules(&lag->pf->hw, s_rule, s_rule_sz, 1, opc, NULL); + if (err) + goto dflt_fltr_free; + + if (add) + lag->pf_rule_id = le16_to_cpu(s_rule->index); + else + lag->pf_rule_id = 0; + +dflt_fltr_free: + kfree(s_rule); + return err; +} + +/** + * ice_lag_cfg_pf_fltrs - set filters up for new active port + * @lag: local interfaces lag struct + * @ptr: opaque data containing notifier event + */ +static void +ice_lag_cfg_pf_fltrs(struct ice_lag *lag, void *ptr) +{ + struct netdev_notifier_bonding_info *info; + struct netdev_bonding_info *bonding_info; + struct net_device *event_netdev; + struct device *dev; + + event_netdev = netdev_notifier_info_to_dev(ptr); + /* not for this netdev */ + if (event_netdev != lag->netdev) + return; + + info = (struct netdev_notifier_bonding_info *)ptr; + bonding_info = &info->bonding_info; + dev = ice_pf_to_dev(lag->pf); + + /* interface not active - remove old default VSI rule */ + if (bonding_info->slave.state && lag->pf_rule_id) { + if (ice_lag_cfg_dflt_fltr(lag, false)) + dev_err(dev, "Error removing old default VSI filter\n"); + return; + } + + /* interface becoming active - add new default VSI rule */ + if (!bonding_info->slave.state && !lag->pf_rule_id) + if (ice_lag_cfg_dflt_fltr(lag, true)) + dev_err(dev, "Error adding new default VSI filter\n"); +} + +/** * ice_display_lag_info - print LAG info * @lag: LAG info struct */ static void ice_display_lag_info(struct ice_lag *lag) { - const char *name, *peer, *upper, *role, *bonded, *primary; + const char *name, *upper, *role, *bonded, *primary; struct device *dev = &lag->pf->pdev->dev; name = lag->netdev ? netdev_name(lag->netdev) : "unset"; - peer = lag->peer_netdev ? netdev_name(lag->peer_netdev) : "unset"; upper = lag->upper_netdev ? netdev_name(lag->upper_netdev) : "unset"; primary = lag->primary ? "TRUE" : "FALSE"; bonded = lag->bonded ? "BONDED" : "UNBONDED"; @@ -78,8 +299,410 @@ static void ice_display_lag_info(struct ice_lag *lag) role = "ERROR"; } - dev_dbg(dev, "%s %s, peer:%s, upper:%s, role:%s, primary:%s\n", name, - bonded, peer, upper, role, primary); + dev_dbg(dev, "%s %s, upper:%s, role:%s, primary:%s\n", name, bonded, + upper, role, primary); +} + +/** + * ice_lag_qbuf_recfg - generate a buffer of queues for a reconfigure command + * @hw: HW struct that contains the queue contexts + * @qbuf: pointer to buffer to populate + * @vsi_num: index of the VSI in PF space + * @numq: number of queues to search for + * @tc: traffic class that contains the queues + * + * function returns the number of valid queues in buffer + */ +static u16 +ice_lag_qbuf_recfg(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *qbuf, + u16 vsi_num, u16 numq, u8 tc) +{ + struct ice_q_ctx *q_ctx; + u16 qid, count = 0; + struct ice_pf *pf; + int i; + + pf = hw->back; + for (i = 0; i < numq; i++) { + q_ctx = ice_get_lan_q_ctx(hw, vsi_num, tc, i); + if (!q_ctx) { + dev_dbg(ice_hw_to_dev(hw), "%s queue %d NO Q CONTEXT\n", + __func__, i); + continue; + } + if (q_ctx->q_teid == ICE_INVAL_TEID) { + dev_dbg(ice_hw_to_dev(hw), "%s queue %d INVAL TEID\n", + __func__, i); + continue; + } + if (q_ctx->q_handle == ICE_INVAL_Q_HANDLE) { + dev_dbg(ice_hw_to_dev(hw), "%s queue %d INVAL Q HANDLE\n", + __func__, i); + continue; + } + + qid = pf->vsi[vsi_num]->txq_map[q_ctx->q_handle]; + qbuf->queue_info[count].q_handle = cpu_to_le16(qid); + qbuf->queue_info[count].tc = tc; + qbuf->queue_info[count].q_teid = cpu_to_le32(q_ctx->q_teid); + count++; + } + + return count; +} + +/** + * ice_lag_get_sched_parent - locate or create a sched node parent + * @hw: HW struct for getting parent in + * @tc: traffic class on parent/node + */ +static struct ice_sched_node * +ice_lag_get_sched_parent(struct ice_hw *hw, u8 tc) +{ + struct ice_sched_node *tc_node, *aggnode, *parent = NULL; + u16 num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 }; + struct ice_port_info *pi = hw->port_info; + struct device *dev; + u8 aggl, vsil; + int n; + + dev = ice_hw_to_dev(hw); + + tc_node = ice_sched_get_tc_node(pi, tc); + if (!tc_node) { + dev_warn(dev, "Failure to find TC node for LAG move\n"); + return parent; + } + + aggnode = ice_sched_get_agg_node(pi, tc_node, ICE_DFLT_AGG_ID); + if (!aggnode) { + dev_warn(dev, "Failure to find aggregate node for LAG move\n"); + return parent; + } + + aggl = ice_sched_get_agg_layer(hw); + vsil = ice_sched_get_vsi_layer(hw); + + for (n = aggl + 1; n < vsil; n++) + num_nodes[n] = 1; + + for (n = 0; n < aggnode->num_children; n++) { + parent = ice_sched_get_free_vsi_parent(hw, aggnode->children[n], + num_nodes); + if (parent) + return parent; + } + + /* if free parent not found - add one */ + parent = aggnode; + for (n = aggl + 1; n < vsil; n++) { + u16 num_nodes_added; + u32 first_teid; + int err; + + err = ice_sched_add_nodes_to_layer(pi, tc_node, parent, n, + num_nodes[n], &first_teid, + &num_nodes_added); + if (err || num_nodes[n] != num_nodes_added) + return NULL; + + if (num_nodes_added) + parent = ice_sched_find_node_by_teid(tc_node, + first_teid); + else + parent = parent->children[0]; + if (!parent) { + dev_warn(dev, "Failure to add new parent for LAG move\n"); + return parent; + } + } + + return parent; +} + +/** + * ice_lag_move_vf_node_tc - move scheduling nodes for one VF on one TC + * @lag: lag info struct + * @oldport: lport of previous nodes location + * @newport: lport of destination nodes location + * @vsi_num: array index of VSI in PF space + * @tc: traffic class to move + */ +static void +ice_lag_move_vf_node_tc(struct ice_lag *lag, u8 oldport, u8 newport, + u16 vsi_num, u8 tc) +{ + u16 numq, valq, buf_size, num_moved, qbuf_size; + struct device *dev = ice_pf_to_dev(lag->pf); + struct ice_aqc_cfg_txqs_buf *qbuf; + struct ice_aqc_move_elem *buf; + struct ice_sched_node *n_prt; + struct ice_hw *new_hw = NULL; + __le32 teid, parent_teid; + struct ice_vsi_ctx *ctx; + u32 tmp_teid; + + ctx = ice_get_vsi_ctx(&lag->pf->hw, vsi_num); + if (!ctx) { + dev_warn(dev, "Unable to locate VSI context for LAG failover\n"); + return; + } + + /* check to see if this VF is enabled on this TC */ + if (!ctx->sched.vsi_node[tc]) + return; + + /* locate HW struct for destination port */ + new_hw = ice_lag_find_hw_by_lport(lag, newport); + if (!new_hw) { + dev_warn(dev, "Unable to locate HW struct for LAG node destination\n"); + return; + } + + numq = ctx->num_lan_q_entries[tc]; + teid = ctx->sched.vsi_node[tc]->info.node_teid; + tmp_teid = le32_to_cpu(teid); + parent_teid = ctx->sched.vsi_node[tc]->info.parent_teid; + /* if no teid assigned or numq == 0, then this TC is not active */ + if (!tmp_teid || !numq) + return; + + /* suspend VSI subtree for Traffic Class "tc" on + * this VF's VSI + */ + if (ice_sched_suspend_resume_elems(&lag->pf->hw, 1, &tmp_teid, true)) + dev_dbg(dev, "Problem suspending traffic for LAG node move\n"); + + /* reconfigure all VF's queues on this Traffic Class + * to new port + */ + qbuf_size = struct_size(qbuf, queue_info, numq); + qbuf = kzalloc(qbuf_size, GFP_KERNEL); + if (!qbuf) { + dev_warn(dev, "Failure allocating memory for VF queue recfg buffer\n"); + goto resume_traffic; + } + + /* add the per queue info for the reconfigure command buffer */ + valq = ice_lag_qbuf_recfg(&lag->pf->hw, qbuf, vsi_num, numq, tc); + if (!valq) { + dev_dbg(dev, "No valid queues found for LAG failover\n"); + goto qbuf_none; + } + + if (ice_aq_cfg_lan_txq(&lag->pf->hw, qbuf, qbuf_size, valq, oldport, + newport, NULL)) { + dev_warn(dev, "Failure to configure queues for LAG failover\n"); + goto qbuf_err; + } + +qbuf_none: + kfree(qbuf); + + /* find new parent in destination port's tree for VF VSI node on this + * Traffic Class + */ + n_prt = ice_lag_get_sched_parent(new_hw, tc); + if (!n_prt) + goto resume_traffic; + + /* Move Vf's VSI node for this TC to newport's scheduler tree */ + buf_size = struct_size(buf, teid, 1); + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) { + dev_warn(dev, "Failure to alloc memory for VF node failover\n"); + goto resume_traffic; + } + + buf->hdr.src_parent_teid = parent_teid; + buf->hdr.dest_parent_teid = n_prt->info.node_teid; + buf->hdr.num_elems = cpu_to_le16(1); + buf->hdr.mode = ICE_AQC_MOVE_ELEM_MODE_KEEP_OWN; + buf->teid[0] = teid; + + if (ice_aq_move_sched_elems(&lag->pf->hw, 1, buf, buf_size, &num_moved, + NULL)) + dev_warn(dev, "Failure to move VF nodes for failover\n"); + else + ice_sched_update_parent(n_prt, ctx->sched.vsi_node[tc]); + + kfree(buf); + goto resume_traffic; + +qbuf_err: + kfree(qbuf); + +resume_traffic: + /* restart traffic for VSI node */ + if (ice_sched_suspend_resume_elems(&lag->pf->hw, 1, &tmp_teid, false)) + dev_dbg(dev, "Problem restarting traffic for LAG node move\n"); +} + +/** + * ice_lag_move_single_vf_nodes - Move Tx scheduling nodes for single VF + * @lag: primary interface LAG struct + * @oldport: lport of previous interface + * @newport: lport of destination interface + * @vsi_num: SW index of VF's VSI + */ +static void +ice_lag_move_single_vf_nodes(struct ice_lag *lag, u8 oldport, u8 newport, + u16 vsi_num) +{ + u8 tc; + + ice_for_each_traffic_class(tc) + ice_lag_move_vf_node_tc(lag, oldport, newport, vsi_num, tc); +} + +/** + * ice_lag_move_new_vf_nodes - Move Tx scheduling nodes for a VF if required + * @vf: the VF to move Tx nodes for + * + * Called just after configuring new VF queues. Check whether the VF Tx + * scheduling nodes need to be updated to fail over to the active port. If so, + * move them now. + */ +void ice_lag_move_new_vf_nodes(struct ice_vf *vf) +{ + struct ice_lag_netdev_list ndlist; + struct list_head *tmp, *n; + u8 pri_port, act_port; + struct ice_lag *lag; + struct ice_vsi *vsi; + struct ice_pf *pf; + + vsi = ice_get_vf_vsi(vf); + + if (WARN_ON(!vsi)) + return; + + if (WARN_ON(vsi->type != ICE_VSI_VF)) + return; + + pf = vf->pf; + lag = pf->lag; + + mutex_lock(&pf->lag_mutex); + if (!lag->bonded) + goto new_vf_unlock; + + pri_port = pf->hw.port_info->lport; + act_port = lag->active_port; + + if (lag->upper_netdev) { + struct ice_lag_netdev_list *nl; + struct net_device *tmp_nd; + + INIT_LIST_HEAD(&ndlist.node); + rcu_read_lock(); + for_each_netdev_in_bond_rcu(lag->upper_netdev, tmp_nd) { + nl = kzalloc(sizeof(*nl), GFP_KERNEL); + if (!nl) + break; + + nl->netdev = tmp_nd; + list_add(&nl->node, &ndlist.node); + } + rcu_read_unlock(); + } + + lag->netdev_head = &ndlist.node; + + if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG) && + lag->bonded && lag->primary && pri_port != act_port && + !list_empty(lag->netdev_head)) + ice_lag_move_single_vf_nodes(lag, pri_port, act_port, vsi->idx); + + list_for_each_safe(tmp, n, &ndlist.node) { + struct ice_lag_netdev_list *entry; + + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + list_del(&entry->node); + kfree(entry); + } + lag->netdev_head = NULL; + +new_vf_unlock: + mutex_unlock(&pf->lag_mutex); +} + +/** + * ice_lag_move_vf_nodes - move Tx scheduling nodes for all VFs to new port + * @lag: lag info struct + * @oldport: lport of previous interface + * @newport: lport of destination interface + */ +static void ice_lag_move_vf_nodes(struct ice_lag *lag, u8 oldport, u8 newport) +{ + struct ice_pf *pf; + int i; + + if (!lag->primary) + return; + + pf = lag->pf; + ice_for_each_vsi(pf, i) + if (pf->vsi[i] && (pf->vsi[i]->type == ICE_VSI_VF || + pf->vsi[i]->type == ICE_VSI_SWITCHDEV_CTRL)) + ice_lag_move_single_vf_nodes(lag, oldport, newport, i); +} + +#define ICE_LAG_SRIOV_CP_RECIPE 10 +#define ICE_LAG_SRIOV_TRAIN_PKT_LEN 16 + +/** + * ice_lag_cfg_cp_fltr - configure filter for control packets + * @lag: local interface's lag struct + * @add: add or remove rule + */ +static void +ice_lag_cfg_cp_fltr(struct ice_lag *lag, bool add) +{ + struct ice_sw_rule_lkup_rx_tx *s_rule = NULL; + struct ice_vsi *vsi; + u16 buf_len, opc; + + vsi = lag->pf->vsi[0]; + + buf_len = ICE_SW_RULE_RX_TX_HDR_SIZE(s_rule, + ICE_LAG_SRIOV_TRAIN_PKT_LEN); + s_rule = kzalloc(buf_len, GFP_KERNEL); + if (!s_rule) { + netdev_warn(lag->netdev, "-ENOMEM error configuring CP filter\n"); + return; + } + + if (add) { + s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_LKUP_RX); + s_rule->recipe_id = cpu_to_le16(ICE_LAG_SRIOV_CP_RECIPE); + s_rule->src = cpu_to_le16(vsi->port_info->lport); + s_rule->act = cpu_to_le32(ICE_FWD_TO_VSI | + ICE_SINGLE_ACT_LAN_ENABLE | + ICE_SINGLE_ACT_VALID_BIT | + ((vsi->vsi_num << + ICE_SINGLE_ACT_VSI_ID_S) & + ICE_SINGLE_ACT_VSI_ID_M)); + s_rule->hdr_len = cpu_to_le16(ICE_LAG_SRIOV_TRAIN_PKT_LEN); + memcpy(s_rule->hdr_data, lacp_train_pkt, LACP_TRAIN_PKT_LEN); + opc = ice_aqc_opc_add_sw_rules; + } else { + opc = ice_aqc_opc_remove_sw_rules; + s_rule->index = cpu_to_le16(lag->cp_rule_idx); + } + if (ice_aq_sw_rules(&lag->pf->hw, s_rule, buf_len, 1, opc, NULL)) { + netdev_warn(lag->netdev, "Error %s CP rule for fail-over\n", + add ? "ADDING" : "REMOVING"); + goto cp_free; + } + + if (add) + lag->cp_rule_idx = le16_to_cpu(s_rule->index); + else + lag->cp_rule_idx = 0; + +cp_free: + kfree(s_rule); } /** @@ -124,117 +747,431 @@ lag_out: } /** + * ice_lag_reclaim_vf_tc - move scheduling nodes back to primary interface + * @lag: primary interface lag struct + * @src_hw: HW struct current node location + * @vsi_num: VSI index in PF space + * @tc: traffic class to move + */ +static void +ice_lag_reclaim_vf_tc(struct ice_lag *lag, struct ice_hw *src_hw, u16 vsi_num, + u8 tc) +{ + u16 numq, valq, buf_size, num_moved, qbuf_size; + struct device *dev = ice_pf_to_dev(lag->pf); + struct ice_aqc_cfg_txqs_buf *qbuf; + struct ice_aqc_move_elem *buf; + struct ice_sched_node *n_prt; + __le32 teid, parent_teid; + struct ice_vsi_ctx *ctx; + struct ice_hw *hw; + u32 tmp_teid; + + hw = &lag->pf->hw; + ctx = ice_get_vsi_ctx(hw, vsi_num); + if (!ctx) { + dev_warn(dev, "Unable to locate VSI context for LAG reclaim\n"); + return; + } + + /* check to see if this VF is enabled on this TC */ + if (!ctx->sched.vsi_node[tc]) + return; + + numq = ctx->num_lan_q_entries[tc]; + teid = ctx->sched.vsi_node[tc]->info.node_teid; + tmp_teid = le32_to_cpu(teid); + parent_teid = ctx->sched.vsi_node[tc]->info.parent_teid; + + /* if !teid or !numq, then this TC is not active */ + if (!tmp_teid || !numq) + return; + + /* suspend traffic */ + if (ice_sched_suspend_resume_elems(hw, 1, &tmp_teid, true)) + dev_dbg(dev, "Problem suspending traffic for LAG node move\n"); + + /* reconfig queues for new port */ + qbuf_size = struct_size(qbuf, queue_info, numq); + qbuf = kzalloc(qbuf_size, GFP_KERNEL); + if (!qbuf) { + dev_warn(dev, "Failure allocating memory for VF queue recfg buffer\n"); + goto resume_reclaim; + } + + /* add the per queue info for the reconfigure command buffer */ + valq = ice_lag_qbuf_recfg(hw, qbuf, vsi_num, numq, tc); + if (!valq) { + dev_dbg(dev, "No valid queues found for LAG reclaim\n"); + goto reclaim_none; + } + + if (ice_aq_cfg_lan_txq(hw, qbuf, qbuf_size, numq, + src_hw->port_info->lport, hw->port_info->lport, + NULL)) { + dev_warn(dev, "Failure to configure queues for LAG failover\n"); + goto reclaim_qerr; + } + +reclaim_none: + kfree(qbuf); + + /* find parent in primary tree */ + n_prt = ice_lag_get_sched_parent(hw, tc); + if (!n_prt) + goto resume_reclaim; + + /* Move node to new parent */ + buf_size = struct_size(buf, teid, 1); + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) { + dev_warn(dev, "Failure to alloc memory for VF node failover\n"); + goto resume_reclaim; + } + + buf->hdr.src_parent_teid = parent_teid; + buf->hdr.dest_parent_teid = n_prt->info.node_teid; + buf->hdr.num_elems = cpu_to_le16(1); + buf->hdr.mode = ICE_AQC_MOVE_ELEM_MODE_KEEP_OWN; + buf->teid[0] = teid; + + if (ice_aq_move_sched_elems(&lag->pf->hw, 1, buf, buf_size, &num_moved, + NULL)) + dev_warn(dev, "Failure to move VF nodes for LAG reclaim\n"); + else + ice_sched_update_parent(n_prt, ctx->sched.vsi_node[tc]); + + kfree(buf); + goto resume_reclaim; + +reclaim_qerr: + kfree(qbuf); + +resume_reclaim: + /* restart traffic */ + if (ice_sched_suspend_resume_elems(hw, 1, &tmp_teid, false)) + dev_warn(dev, "Problem restarting traffic for LAG node reclaim\n"); +} + +/** + * ice_lag_reclaim_vf_nodes - When interface leaving bond primary reclaims nodes + * @lag: primary interface lag struct + * @src_hw: HW struct for current node location + */ +static void +ice_lag_reclaim_vf_nodes(struct ice_lag *lag, struct ice_hw *src_hw) +{ + struct ice_pf *pf; + int i, tc; + + if (!lag->primary || !src_hw) + return; + + pf = lag->pf; + ice_for_each_vsi(pf, i) + if (pf->vsi[i] && (pf->vsi[i]->type == ICE_VSI_VF || + pf->vsi[i]->type == ICE_VSI_SWITCHDEV_CTRL)) + ice_for_each_traffic_class(tc) + ice_lag_reclaim_vf_tc(lag, src_hw, i, tc); +} + +/** * ice_lag_link - handle LAG link event * @lag: LAG info struct - * @info: info from the netdev notifier */ -static void -ice_lag_link(struct ice_lag *lag, struct netdev_notifier_changeupper_info *info) +static void ice_lag_link(struct ice_lag *lag) { - struct net_device *netdev_tmp, *upper = info->upper_dev; struct ice_pf *pf = lag->pf; - int peers = 0; if (lag->bonded) dev_warn(ice_pf_to_dev(pf), "%s Already part of a bond\n", netdev_name(lag->netdev)); - rcu_read_lock(); - for_each_netdev_in_bond_rcu(upper, netdev_tmp) - peers++; - rcu_read_unlock(); - - if (lag->upper_netdev != upper) { - dev_hold(upper); - lag->upper_netdev = upper; - } - - ice_clear_rdma_cap(pf); - lag->bonded = true; lag->role = ICE_LAG_UNSET; - - /* if this is the first element in an LAG mark as primary */ - lag->primary = !!(peers == 1); + netdev_info(lag->netdev, "Shared SR-IOV resources in bond are active\n"); } /** * ice_lag_unlink - handle unlink event * @lag: LAG info struct - * @info: info from netdev notification */ -static void -ice_lag_unlink(struct ice_lag *lag, - struct netdev_notifier_changeupper_info *info) +static void ice_lag_unlink(struct ice_lag *lag) { - struct net_device *netdev_tmp, *upper = info->upper_dev; + u8 pri_port, act_port, loc_port; struct ice_pf *pf = lag->pf; - bool found = false; if (!lag->bonded) { netdev_dbg(lag->netdev, "bonding unlink event on non-LAG netdev\n"); return; } - /* determine if we are in the new LAG config or not */ - rcu_read_lock(); - for_each_netdev_in_bond_rcu(upper, netdev_tmp) { - if (netdev_tmp == lag->netdev) { - found = true; - break; + if (lag->primary) { + act_port = lag->active_port; + pri_port = lag->pf->hw.port_info->lport; + if (act_port != pri_port && act_port != ICE_LAG_INVALID_PORT) + ice_lag_move_vf_nodes(lag, act_port, pri_port); + lag->primary = false; + lag->active_port = ICE_LAG_INVALID_PORT; + } else { + struct ice_lag *primary_lag; + + primary_lag = ice_lag_find_primary(lag); + if (primary_lag) { + act_port = primary_lag->active_port; + pri_port = primary_lag->pf->hw.port_info->lport; + loc_port = pf->hw.port_info->lport; + if (act_port == loc_port && + act_port != ICE_LAG_INVALID_PORT) { + ice_lag_reclaim_vf_nodes(primary_lag, + &lag->pf->hw); + primary_lag->active_port = ICE_LAG_INVALID_PORT; + } } } - rcu_read_unlock(); - if (found) + lag->bonded = false; + lag->role = ICE_LAG_NONE; + lag->upper_netdev = NULL; +} + +/** + * ice_lag_link_unlink - helper function to call lag_link/unlink + * @lag: lag info struct + * @ptr: opaque pointer data + */ +static void ice_lag_link_unlink(struct ice_lag *lag, void *ptr) +{ + struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + struct netdev_notifier_changeupper_info *info = ptr; + + if (netdev != lag->netdev) return; - if (lag->upper_netdev) { - dev_put(lag->upper_netdev); - lag->upper_netdev = NULL; + if (info->linking) + ice_lag_link(lag); + else + ice_lag_unlink(lag); +} + +/** + * ice_lag_set_swid - set the SWID on secondary interface + * @primary_swid: primary interface's SWID + * @local_lag: local interfaces LAG struct + * @link: Is this a linking activity + * + * If link is false, then primary_swid should be expected to not be valid + * This function should never be called in interrupt context. + */ +static void +ice_lag_set_swid(u16 primary_swid, struct ice_lag *local_lag, + bool link) +{ + struct ice_aqc_alloc_free_res_elem *buf; + struct ice_aqc_set_port_params *cmd; + struct ice_aq_desc desc; + u16 buf_len, swid; + int status, i; + + buf_len = struct_size(buf, elem, 1); + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) { + dev_err(ice_pf_to_dev(local_lag->pf), "-ENOMEM error setting SWID\n"); + return; } - lag->peer_netdev = NULL; - ice_set_rdma_cap(pf); - lag->bonded = false; - lag->role = ICE_LAG_NONE; + buf->num_elems = cpu_to_le16(1); + buf->res_type = cpu_to_le16(ICE_AQC_RES_TYPE_SWID); + /* if unlinnking need to free the shared resource */ + if (!link && local_lag->bond_swid) { + buf->elem[0].e.sw_resp = cpu_to_le16(local_lag->bond_swid); + status = ice_aq_alloc_free_res(&local_lag->pf->hw, 1, buf, + buf_len, ice_aqc_opc_free_res, + NULL); + if (status) + dev_err(ice_pf_to_dev(local_lag->pf), "Error freeing SWID during LAG unlink\n"); + local_lag->bond_swid = 0; + } + + if (link) { + buf->res_type |= cpu_to_le16(ICE_LAG_RES_SHARED | + ICE_LAG_RES_VALID); + /* store the primary's SWID in case it leaves bond first */ + local_lag->bond_swid = primary_swid; + buf->elem[0].e.sw_resp = cpu_to_le16(local_lag->bond_swid); + } else { + buf->elem[0].e.sw_resp = + cpu_to_le16(local_lag->pf->hw.port_info->sw_id); + } + + status = ice_aq_alloc_free_res(&local_lag->pf->hw, 1, buf, buf_len, + ice_aqc_opc_alloc_res, NULL); + if (status) + dev_err(ice_pf_to_dev(local_lag->pf), "Error subscribing to SWID 0x%04X\n", + local_lag->bond_swid); + + kfree(buf); + + /* Configure port param SWID to correct value */ + if (link) + swid = primary_swid; + else + swid = local_lag->pf->hw.port_info->sw_id; + + cmd = &desc.params.set_port_params; + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_port_params); + + cmd->swid = cpu_to_le16(ICE_AQC_PORT_SWID_VALID | swid); + /* If this is happening in reset context, it is possible that the + * primary interface has not finished setting its SWID to SHARED + * yet. Allow retries to account for this timing issue between + * interfaces. + */ + for (i = 0; i < ICE_LAG_RESET_RETRIES; i++) { + status = ice_aq_send_cmd(&local_lag->pf->hw, &desc, NULL, 0, + NULL); + if (!status) + break; + + usleep_range(1000, 2000); + } + + if (status) + dev_err(ice_pf_to_dev(local_lag->pf), "Error setting SWID in port params %d\n", + status); } /** - * ice_lag_unregister - handle netdev unregister events - * @lag: LAG info struct - * @netdev: netdev reporting the event + * ice_lag_primary_swid - set/clear the SHARED attrib of primary's SWID + * @lag: primary interface's lag struct + * @link: is this a linking activity + * + * Implement setting primary SWID as shared using 0x020B */ -static void ice_lag_unregister(struct ice_lag *lag, struct net_device *netdev) +static void ice_lag_primary_swid(struct ice_lag *lag, bool link) { - struct ice_pf *pf = lag->pf; + struct ice_hw *hw; + u16 swid; - /* check to see if this event is for this netdev - * check that we are in an aggregate - */ - if (netdev != lag->netdev || !lag->bonded) + hw = &lag->pf->hw; + swid = hw->port_info->sw_id; + + if (ice_share_res(hw, ICE_AQC_RES_TYPE_SWID, link, swid)) + dev_warn(ice_pf_to_dev(lag->pf), "Failure to set primary interface shared status\n"); +} + +/** + * ice_lag_add_prune_list - Adds event_pf's VSI to primary's prune list + * @lag: lag info struct + * @event_pf: PF struct for VSI we are adding to primary's prune list + */ +static void ice_lag_add_prune_list(struct ice_lag *lag, struct ice_pf *event_pf) +{ + u16 num_vsi, rule_buf_sz, vsi_list_id, event_vsi_num, prim_vsi_idx; + struct ice_sw_rule_vsi_list *s_rule = NULL; + struct device *dev; + + num_vsi = 1; + + dev = ice_pf_to_dev(lag->pf); + event_vsi_num = event_pf->vsi[0]->vsi_num; + prim_vsi_idx = lag->pf->vsi[0]->idx; + + if (!ice_find_vsi_list_entry(&lag->pf->hw, ICE_SW_LKUP_VLAN, + prim_vsi_idx, &vsi_list_id)) { + dev_warn(dev, "Could not locate prune list when setting up SRIOV LAG\n"); return; + } - if (lag->upper_netdev) { - dev_put(lag->upper_netdev); - lag->upper_netdev = NULL; - ice_set_rdma_cap(pf); + rule_buf_sz = (u16)ICE_SW_RULE_VSI_LIST_SIZE(s_rule, num_vsi); + s_rule = kzalloc(rule_buf_sz, GFP_KERNEL); + if (!s_rule) { + dev_warn(dev, "Error allocating space for prune list when configuring SRIOV LAG\n"); + return; } - /* perform some cleanup in case we come back */ - lag->bonded = false; - lag->role = ICE_LAG_NONE; + + s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_PRUNE_LIST_SET); + s_rule->index = cpu_to_le16(vsi_list_id); + s_rule->number_vsi = cpu_to_le16(num_vsi); + s_rule->vsi[0] = cpu_to_le16(event_vsi_num); + + if (ice_aq_sw_rules(&event_pf->hw, s_rule, rule_buf_sz, 1, + ice_aqc_opc_update_sw_rules, NULL)) + dev_warn(dev, "Error adding VSI prune list\n"); + kfree(s_rule); +} + +/** + * ice_lag_del_prune_list - Remove secondary's vsi from primary's prune list + * @lag: primary interface's ice_lag struct + * @event_pf: PF struct for unlinking interface + */ +static void ice_lag_del_prune_list(struct ice_lag *lag, struct ice_pf *event_pf) +{ + u16 num_vsi, vsi_num, vsi_idx, rule_buf_sz, vsi_list_id; + struct ice_sw_rule_vsi_list *s_rule = NULL; + struct device *dev; + + num_vsi = 1; + + dev = ice_pf_to_dev(lag->pf); + vsi_num = event_pf->vsi[0]->vsi_num; + vsi_idx = lag->pf->vsi[0]->idx; + + if (!ice_find_vsi_list_entry(&lag->pf->hw, ICE_SW_LKUP_VLAN, + vsi_idx, &vsi_list_id)) { + dev_warn(dev, "Could not locate prune list when unwinding SRIOV LAG\n"); + return; + } + + rule_buf_sz = (u16)ICE_SW_RULE_VSI_LIST_SIZE(s_rule, num_vsi); + s_rule = kzalloc(rule_buf_sz, GFP_KERNEL); + if (!s_rule) { + dev_warn(dev, "Error allocating prune list when unwinding SRIOV LAG\n"); + return; + } + + s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_PRUNE_LIST_CLEAR); + s_rule->index = cpu_to_le16(vsi_list_id); + s_rule->number_vsi = cpu_to_le16(num_vsi); + s_rule->vsi[0] = cpu_to_le16(vsi_num); + + if (ice_aq_sw_rules(&event_pf->hw, (struct ice_aqc_sw_rules *)s_rule, + rule_buf_sz, 1, ice_aqc_opc_update_sw_rules, NULL)) + dev_warn(dev, "Error clearing VSI prune list\n"); + + kfree(s_rule); +} + +/** + * ice_lag_init_feature_support_flag - Check for NVM support for LAG + * @pf: PF struct + */ +static void ice_lag_init_feature_support_flag(struct ice_pf *pf) +{ + struct ice_hw_common_caps *caps; + + caps = &pf->hw.dev_caps.common_cap; + if (caps->roce_lag) + ice_set_feature_support(pf, ICE_F_ROCE_LAG); + else + ice_clear_feature_support(pf, ICE_F_ROCE_LAG); + + if (caps->sriov_lag) + ice_set_feature_support(pf, ICE_F_SRIOV_LAG); + else + ice_clear_feature_support(pf, ICE_F_SRIOV_LAG); } /** * ice_lag_changeupper_event - handle LAG changeupper event * @lag: LAG info struct * @ptr: opaque pointer data - * - * ptr is to be cast into netdev_notifier_changeupper_info */ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) { struct netdev_notifier_changeupper_info *info; + struct ice_lag *primary_lag; struct net_device *netdev; info = ptr; @@ -244,44 +1181,444 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) if (netdev != lag->netdev) return; - if (!info->upper_dev) { - netdev_dbg(netdev, "changeupper rcvd, but no upper defined\n"); + primary_lag = ice_lag_find_primary(lag); + if (info->linking) { + lag->upper_netdev = info->upper_dev; + /* If there is not already a primary interface in the LAG, + * then mark this one as primary. + */ + if (!primary_lag) { + lag->primary = true; + /* Configure primary's SWID to be shared */ + ice_lag_primary_swid(lag, true); + primary_lag = lag; + } else { + u16 swid; + + swid = primary_lag->pf->hw.port_info->sw_id; + ice_lag_set_swid(swid, lag, true); + ice_lag_add_prune_list(primary_lag, lag->pf); + } + /* add filter for primary control packets */ + ice_lag_cfg_cp_fltr(lag, true); + } else { + if (!primary_lag && lag->primary) + primary_lag = lag; + + if (!lag->primary) { + ice_lag_set_swid(0, lag, false); + } else { + if (primary_lag && lag->primary) { + ice_lag_primary_swid(lag, false); + ice_lag_del_prune_list(primary_lag, lag->pf); + } + } + /* remove filter for control packets */ + ice_lag_cfg_cp_fltr(lag, false); + } +} + +/** + * ice_lag_monitor_link - monitor interfaces entering/leaving the aggregate + * @lag: lag info struct + * @ptr: opaque data containing notifier event + * + * This function only operates after a primary has been set. + */ +static void ice_lag_monitor_link(struct ice_lag *lag, void *ptr) +{ + struct netdev_notifier_changeupper_info *info; + struct ice_hw *prim_hw, *active_hw; + struct net_device *event_netdev; + struct ice_pf *pf; + u8 prim_port; + + if (!lag->primary) + return; + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (!netif_is_same_ice(lag->pf, event_netdev)) + return; + + pf = lag->pf; + prim_hw = &pf->hw; + prim_port = prim_hw->port_info->lport; + + info = (struct netdev_notifier_changeupper_info *)ptr; + if (info->upper_dev != lag->upper_netdev) return; + + if (!info->linking) { + /* Since there are only two interfaces allowed in SRIOV+LAG, if + * one port is leaving, then nodes need to be on primary + * interface. + */ + if (prim_port != lag->active_port && + lag->active_port != ICE_LAG_INVALID_PORT) { + active_hw = ice_lag_find_hw_by_lport(lag, + lag->active_port); + ice_lag_reclaim_vf_nodes(lag, active_hw); + lag->active_port = ICE_LAG_INVALID_PORT; + } } +} + +/** + * ice_lag_monitor_active - main PF keep track of which port is active + * @lag: lag info struct + * @ptr: opaque data containing notifier event + * + * This function is for the primary PF to monitor changes in which port is + * active and handle changes for SRIOV VF functionality + */ +static void ice_lag_monitor_active(struct ice_lag *lag, void *ptr) +{ + struct net_device *event_netdev, *event_upper; + struct netdev_notifier_bonding_info *info; + struct netdev_bonding_info *bonding_info; + struct ice_netdev_priv *event_np; + struct ice_pf *pf, *event_pf; + u8 prim_port, event_port; + + if (!lag->primary) + return; - netdev_dbg(netdev, "bonding %s\n", info->linking ? "LINK" : "UNLINK"); + pf = lag->pf; + if (!pf) + return; - if (!netif_is_lag_master(info->upper_dev)) { - netdev_dbg(netdev, "changeupper rcvd, but not primary. bail\n"); + event_netdev = netdev_notifier_info_to_dev(ptr); + rcu_read_lock(); + event_upper = netdev_master_upper_dev_get_rcu(event_netdev); + rcu_read_unlock(); + if (!netif_is_ice(event_netdev) || event_upper != lag->upper_netdev) return; + + event_np = netdev_priv(event_netdev); + event_pf = event_np->vsi->back; + event_port = event_pf->hw.port_info->lport; + prim_port = pf->hw.port_info->lport; + + info = (struct netdev_notifier_bonding_info *)ptr; + bonding_info = &info->bonding_info; + + if (!bonding_info->slave.state) { + /* if no port is currently active, then nodes and filters exist + * on primary port, check if we need to move them + */ + if (lag->active_port == ICE_LAG_INVALID_PORT) { + if (event_port != prim_port) + ice_lag_move_vf_nodes(lag, prim_port, + event_port); + lag->active_port = event_port; + return; + } + + /* active port is already set and is current event port */ + if (lag->active_port == event_port) + return; + /* new active port */ + ice_lag_move_vf_nodes(lag, lag->active_port, event_port); + lag->active_port = event_port; + } else { + /* port not set as currently active (e.g. new active port + * has already claimed the nodes and filters + */ + if (lag->active_port != event_port) + return; + /* This is the case when neither port is active (both link down) + * Link down on the bond - set active port to invalid and move + * nodes and filters back to primary if not already there + */ + if (event_port != prim_port) + ice_lag_move_vf_nodes(lag, event_port, prim_port); + lag->active_port = ICE_LAG_INVALID_PORT; } +} - if (info->linking) - ice_lag_link(lag, info); - else - ice_lag_unlink(lag, info); +/** + * ice_lag_chk_comp - evaluate bonded interface for feature support + * @lag: lag info struct + * @ptr: opaque data for netdev event info + */ +static bool +ice_lag_chk_comp(struct ice_lag *lag, void *ptr) +{ + struct net_device *event_netdev, *event_upper; + struct netdev_notifier_bonding_info *info; + struct netdev_bonding_info *bonding_info; + struct list_head *tmp; + struct device *dev; + int count = 0; - ice_display_lag_info(lag); + if (!lag->primary) + return true; + + event_netdev = netdev_notifier_info_to_dev(ptr); + rcu_read_lock(); + event_upper = netdev_master_upper_dev_get_rcu(event_netdev); + rcu_read_unlock(); + if (event_upper != lag->upper_netdev) + return true; + + dev = ice_pf_to_dev(lag->pf); + + /* only supporting switchdev mode for SRIOV VF LAG. + * primary interface has to be in switchdev mode + */ + if (!ice_is_switchdev_running(lag->pf)) { + dev_info(dev, "Primary interface not in switchdev mode - VF LAG disabled\n"); + return false; + } + + info = (struct netdev_notifier_bonding_info *)ptr; + bonding_info = &info->bonding_info; + lag->bond_mode = bonding_info->master.bond_mode; + if (lag->bond_mode != BOND_MODE_ACTIVEBACKUP) { + dev_info(dev, "Bond Mode not ACTIVE-BACKUP - VF LAG disabled\n"); + return false; + } + + list_for_each(tmp, lag->netdev_head) { + struct ice_dcbx_cfg *dcb_cfg, *peer_dcb_cfg; + struct ice_lag_netdev_list *entry; + struct ice_netdev_priv *peer_np; + struct net_device *peer_netdev; + struct ice_vsi *vsi, *peer_vsi; + struct ice_pf *peer_pf; + + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + peer_netdev = entry->netdev; + if (!netif_is_ice(peer_netdev)) { + dev_info(dev, "Found %s non-ice netdev in LAG - VF LAG disabled\n", + netdev_name(peer_netdev)); + return false; + } + + count++; + if (count > 2) { + dev_info(dev, "Found more than two netdevs in LAG - VF LAG disabled\n"); + return false; + } + + peer_np = netdev_priv(peer_netdev); + vsi = ice_get_main_vsi(lag->pf); + peer_vsi = peer_np->vsi; + if (lag->pf->pdev->bus != peer_vsi->back->pdev->bus || + lag->pf->pdev->slot != peer_vsi->back->pdev->slot) { + dev_info(dev, "Found %s on different device in LAG - VF LAG disabled\n", + netdev_name(peer_netdev)); + return false; + } + + dcb_cfg = &vsi->port_info->qos_cfg.local_dcbx_cfg; + peer_dcb_cfg = &peer_vsi->port_info->qos_cfg.local_dcbx_cfg; + if (memcmp(dcb_cfg, peer_dcb_cfg, + sizeof(struct ice_dcbx_cfg))) { + dev_info(dev, "Found %s with different DCB in LAG - VF LAG disabled\n", + netdev_name(peer_netdev)); + return false; + } + + peer_pf = peer_vsi->back; + if (test_bit(ICE_FLAG_FW_LLDP_AGENT, peer_pf->flags)) { + dev_warn(dev, "Found %s with FW LLDP agent active - VF LAG disabled\n", + netdev_name(peer_netdev)); + return false; + } + } + + return true; } /** - * ice_lag_changelower_event - handle LAG changelower event + * ice_lag_unregister - handle netdev unregister events * @lag: LAG info struct - * @ptr: opaque data pointer + * @event_netdev: netdev struct for target of notifier event + */ +static void +ice_lag_unregister(struct ice_lag *lag, struct net_device *event_netdev) +{ + struct ice_netdev_priv *np; + struct ice_pf *event_pf; + struct ice_lag *p_lag; + + p_lag = ice_lag_find_primary(lag); + np = netdev_priv(event_netdev); + event_pf = np->vsi->back; + + if (p_lag) { + if (p_lag->active_port != p_lag->pf->hw.port_info->lport && + p_lag->active_port != ICE_LAG_INVALID_PORT) { + struct ice_hw *active_hw; + + active_hw = ice_lag_find_hw_by_lport(lag, + p_lag->active_port); + if (active_hw) + ice_lag_reclaim_vf_nodes(p_lag, active_hw); + lag->active_port = ICE_LAG_INVALID_PORT; + } + } + + /* primary processing for primary */ + if (lag->primary && lag->netdev == event_netdev) + ice_lag_primary_swid(lag, false); + + /* primary processing for secondary */ + if (lag->primary && lag->netdev != event_netdev) + ice_lag_del_prune_list(lag, event_pf); + + /* secondary processing for secondary */ + if (!lag->primary && lag->netdev == event_netdev) + ice_lag_set_swid(0, lag, false); +} + +/** + * ice_lag_monitor_rdma - set and clear rdma functionality + * @lag: pointer to lag struct + * @ptr: opaque data for netdev event info + */ +static void +ice_lag_monitor_rdma(struct ice_lag *lag, void *ptr) +{ + struct netdev_notifier_changeupper_info *info; + struct net_device *netdev; + + info = ptr; + netdev = netdev_notifier_info_to_dev(ptr); + + if (netdev != lag->netdev) + return; + + if (info->linking) + ice_clear_rdma_cap(lag->pf); + else + ice_set_rdma_cap(lag->pf); +} + +/** + * ice_lag_chk_disabled_bond - monitor interfaces entering/leaving disabled bond + * @lag: lag info struct + * @ptr: opaque data containing event * - * ptr to be cast to netdev_notifier_changelowerstate_info + * as interfaces enter a bond - determine if the bond is currently + * SRIOV LAG compliant and flag if not. As interfaces leave the + * bond, reset their compliant status. */ -static void ice_lag_changelower_event(struct ice_lag *lag, void *ptr) +static void ice_lag_chk_disabled_bond(struct ice_lag *lag, void *ptr) { struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + struct netdev_notifier_changeupper_info *info = ptr; + struct ice_lag *prim_lag; if (netdev != lag->netdev) return; - netdev_dbg(netdev, "bonding info\n"); + if (info->linking) { + prim_lag = ice_lag_find_primary(lag); + if (prim_lag && + !ice_is_feature_supported(prim_lag->pf, ICE_F_SRIOV_LAG)) { + ice_clear_feature_support(lag->pf, ICE_F_SRIOV_LAG); + netdev_info(netdev, "Interface added to non-compliant SRIOV LAG aggregate\n"); + } + } else { + ice_lag_init_feature_support_flag(lag->pf); + } +} + +/** + * ice_lag_disable_sriov_bond - set members of bond as not supporting SRIOV LAG + * @lag: primary interfaces lag struct + */ +static void ice_lag_disable_sriov_bond(struct ice_lag *lag) +{ + struct ice_lag_netdev_list *entry; + struct ice_netdev_priv *np; + struct net_device *netdev; + struct list_head *tmp; + struct ice_pf *pf; + + list_for_each(tmp, lag->netdev_head) { + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + netdev = entry->netdev; + np = netdev_priv(netdev); + pf = np->vsi->back; + + ice_clear_feature_support(pf, ICE_F_SRIOV_LAG); + } +} + +/** + * ice_lag_process_event - process a task assigned to the lag_wq + * @work: pointer to work_struct + */ +static void ice_lag_process_event(struct work_struct *work) +{ + struct netdev_notifier_changeupper_info *info; + struct ice_lag_work *lag_work; + struct net_device *netdev; + struct list_head *tmp, *n; + struct ice_pf *pf; + + lag_work = container_of(work, struct ice_lag_work, lag_task); + pf = lag_work->lag->pf; + + mutex_lock(&pf->lag_mutex); + lag_work->lag->netdev_head = &lag_work->netdev_list.node; + + switch (lag_work->event) { + case NETDEV_CHANGEUPPER: + info = &lag_work->info.changeupper_info; + ice_lag_chk_disabled_bond(lag_work->lag, info); + if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) { + ice_lag_monitor_link(lag_work->lag, info); + ice_lag_changeupper_event(lag_work->lag, info); + ice_lag_link_unlink(lag_work->lag, info); + } + ice_lag_monitor_rdma(lag_work->lag, info); + break; + case NETDEV_BONDING_INFO: + if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) { + if (!ice_lag_chk_comp(lag_work->lag, + &lag_work->info.bonding_info)) { + netdev = lag_work->info.bonding_info.info.dev; + ice_lag_disable_sriov_bond(lag_work->lag); + ice_lag_unregister(lag_work->lag, netdev); + goto lag_cleanup; + } + ice_lag_monitor_active(lag_work->lag, + &lag_work->info.bonding_info); + ice_lag_cfg_pf_fltrs(lag_work->lag, + &lag_work->info.bonding_info); + } + ice_lag_info_event(lag_work->lag, &lag_work->info.bonding_info); + break; + case NETDEV_UNREGISTER: + if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) { + netdev = lag_work->info.bonding_info.info.dev; + if ((netdev == lag_work->lag->netdev || + lag_work->lag->primary) && lag_work->lag->bonded) + ice_lag_unregister(lag_work->lag, netdev); + } + break; + default: + break; + } + +lag_cleanup: + /* cleanup resources allocated for this work item */ + list_for_each_safe(tmp, n, &lag_work->netdev_list.node) { + struct ice_lag_netdev_list *entry; + + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + list_del(&entry->node); + kfree(entry); + } + lag_work->lag->netdev_head = NULL; - if (!netif_is_lag_port(netdev)) - netdev_dbg(netdev, "CHANGELOWER rcvd, but netdev not in LAG. Bail\n"); + mutex_unlock(&pf->lag_mutex); + + kfree(lag_work); } /** @@ -295,34 +1632,79 @@ ice_lag_event_handler(struct notifier_block *notif_blk, unsigned long event, void *ptr) { struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + struct net_device *upper_netdev; + struct ice_lag_work *lag_work; struct ice_lag *lag; - lag = container_of(notif_blk, struct ice_lag, notif_block); + if (!netif_is_ice(netdev)) + return NOTIFY_DONE; + + if (event != NETDEV_CHANGEUPPER && event != NETDEV_BONDING_INFO && + event != NETDEV_UNREGISTER) + return NOTIFY_DONE; + + if (!(netdev->priv_flags & IFF_BONDING)) + return NOTIFY_DONE; + lag = container_of(notif_blk, struct ice_lag, notif_block); if (!lag->netdev) return NOTIFY_DONE; - /* Check that the netdev is in the working namespace */ if (!net_eq(dev_net(netdev), &init_net)) return NOTIFY_DONE; + /* This memory will be freed at the end of ice_lag_process_event */ + lag_work = kzalloc(sizeof(*lag_work), GFP_KERNEL); + if (!lag_work) + return -ENOMEM; + + lag_work->event_netdev = netdev; + lag_work->lag = lag; + lag_work->event = event; + if (event == NETDEV_CHANGEUPPER) { + struct netdev_notifier_changeupper_info *info; + + info = ptr; + upper_netdev = info->upper_dev; + } else { + upper_netdev = netdev_master_upper_dev_get(netdev); + } + + INIT_LIST_HEAD(&lag_work->netdev_list.node); + if (upper_netdev) { + struct ice_lag_netdev_list *nd_list; + struct net_device *tmp_nd; + + rcu_read_lock(); + for_each_netdev_in_bond_rcu(upper_netdev, tmp_nd) { + nd_list = kzalloc(sizeof(*nd_list), GFP_KERNEL); + if (!nd_list) + break; + + nd_list->netdev = tmp_nd; + list_add(&nd_list->node, &lag_work->netdev_list.node); + } + rcu_read_unlock(); + } + switch (event) { case NETDEV_CHANGEUPPER: - ice_lag_changeupper_event(lag, ptr); - break; - case NETDEV_CHANGELOWERSTATE: - ice_lag_changelower_event(lag, ptr); + lag_work->info.changeupper_info = + *((struct netdev_notifier_changeupper_info *)ptr); break; case NETDEV_BONDING_INFO: - ice_lag_info_event(lag, ptr); - break; - case NETDEV_UNREGISTER: - ice_lag_unregister(lag, netdev); + lag_work->info.bonding_info = + *((struct netdev_notifier_bonding_info *)ptr); break; default: + lag_work->info.notifier_info = + *((struct netdev_notifier_info *)ptr); break; } + INIT_WORK(&lag_work->lag_task, ice_lag_process_event); + queue_work(ice_lag_wq, &lag_work->lag_task); + return NOTIFY_DONE; } @@ -366,6 +1748,174 @@ static void ice_unregister_lag_handler(struct ice_lag *lag) } /** + * ice_create_lag_recipe + * @hw: pointer to HW struct + * @rid: pointer to u16 to pass back recipe index + * @base_recipe: recipe to base the new recipe on + * @prio: priority for new recipe + * + * function returns 0 on error + */ +static int ice_create_lag_recipe(struct ice_hw *hw, u16 *rid, + const u8 *base_recipe, u8 prio) +{ + struct ice_aqc_recipe_data_elem *new_rcp; + int err; + + err = ice_alloc_recipe(hw, rid); + if (err) + return err; + + new_rcp = kzalloc(ICE_RECIPE_LEN * ICE_MAX_NUM_RECIPES, GFP_KERNEL); + if (!new_rcp) + return -ENOMEM; + + memcpy(new_rcp, base_recipe, ICE_RECIPE_LEN); + new_rcp->content.act_ctrl_fwd_priority = prio; + new_rcp->content.rid = *rid | ICE_AQ_RECIPE_ID_IS_ROOT; + new_rcp->recipe_indx = *rid; + bitmap_zero((unsigned long *)new_rcp->recipe_bitmap, + ICE_MAX_NUM_RECIPES); + set_bit(*rid, (unsigned long *)new_rcp->recipe_bitmap); + + err = ice_aq_add_recipe(hw, new_rcp, 1, NULL); + if (err) + *rid = 0; + + kfree(new_rcp); + return err; +} + +/** + * ice_lag_move_vf_nodes_tc_sync - move a VF's nodes for a tc during reset + * @lag: primary interfaces lag struct + * @dest_hw: HW struct for destination's interface + * @vsi_num: VSI index in PF space + * @tc: traffic class to move + */ +static void +ice_lag_move_vf_nodes_tc_sync(struct ice_lag *lag, struct ice_hw *dest_hw, + u16 vsi_num, u8 tc) +{ + u16 numq, valq, buf_size, num_moved, qbuf_size; + struct device *dev = ice_pf_to_dev(lag->pf); + struct ice_aqc_cfg_txqs_buf *qbuf; + struct ice_aqc_move_elem *buf; + struct ice_sched_node *n_prt; + __le32 teid, parent_teid; + struct ice_vsi_ctx *ctx; + struct ice_hw *hw; + u32 tmp_teid; + + hw = &lag->pf->hw; + ctx = ice_get_vsi_ctx(hw, vsi_num); + if (!ctx) { + dev_warn(dev, "LAG rebuild failed after reset due to VSI Context failure\n"); + return; + } + + if (!ctx->sched.vsi_node[tc]) + return; + + numq = ctx->num_lan_q_entries[tc]; + teid = ctx->sched.vsi_node[tc]->info.node_teid; + tmp_teid = le32_to_cpu(teid); + parent_teid = ctx->sched.vsi_node[tc]->info.parent_teid; + + if (!tmp_teid || !numq) + return; + + if (ice_sched_suspend_resume_elems(hw, 1, &tmp_teid, true)) + dev_dbg(dev, "Problem suspending traffic during reset rebuild\n"); + + /* reconfig queues for new port */ + qbuf_size = struct_size(qbuf, queue_info, numq); + qbuf = kzalloc(qbuf_size, GFP_KERNEL); + if (!qbuf) { + dev_warn(dev, "Failure allocating VF queue recfg buffer for reset rebuild\n"); + goto resume_sync; + } + + /* add the per queue info for the reconfigure command buffer */ + valq = ice_lag_qbuf_recfg(hw, qbuf, vsi_num, numq, tc); + if (!valq) { + dev_warn(dev, "Failure to reconfig queues for LAG reset rebuild\n"); + goto sync_none; + } + + if (ice_aq_cfg_lan_txq(hw, qbuf, qbuf_size, numq, hw->port_info->lport, + dest_hw->port_info->lport, NULL)) { + dev_warn(dev, "Failure to configure queues for LAG reset rebuild\n"); + goto sync_qerr; + } + +sync_none: + kfree(qbuf); + + /* find parent in destination tree */ + n_prt = ice_lag_get_sched_parent(dest_hw, tc); + if (!n_prt) + goto resume_sync; + + /* Move node to new parent */ + buf_size = struct_size(buf, teid, 1); + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) { + dev_warn(dev, "Failure to alloc for VF node move in reset rebuild\n"); + goto resume_sync; + } + + buf->hdr.src_parent_teid = parent_teid; + buf->hdr.dest_parent_teid = n_prt->info.node_teid; + buf->hdr.num_elems = cpu_to_le16(1); + buf->hdr.mode = ICE_AQC_MOVE_ELEM_MODE_KEEP_OWN; + buf->teid[0] = teid; + + if (ice_aq_move_sched_elems(&lag->pf->hw, 1, buf, buf_size, &num_moved, + NULL)) + dev_warn(dev, "Failure to move VF nodes for LAG reset rebuild\n"); + else + ice_sched_update_parent(n_prt, ctx->sched.vsi_node[tc]); + + kfree(buf); + goto resume_sync; + +sync_qerr: + kfree(qbuf); + +resume_sync: + if (ice_sched_suspend_resume_elems(hw, 1, &tmp_teid, false)) + dev_warn(dev, "Problem restarting traffic for LAG node reset rebuild\n"); +} + +/** + * ice_lag_move_vf_nodes_sync - move vf nodes to active interface + * @lag: primary interfaces lag struct + * @dest_hw: lport value for currently active port + * + * This function is used in a reset context, outside of event handling, + * to move the VF nodes to the secondary interface when that interface + * is the active interface during a reset rebuild + */ +static void +ice_lag_move_vf_nodes_sync(struct ice_lag *lag, struct ice_hw *dest_hw) +{ + struct ice_pf *pf; + int i, tc; + + if (!lag->primary || !dest_hw) + return; + + pf = lag->pf; + ice_for_each_vsi(pf, i) + if (pf->vsi[i] && (pf->vsi[i]->type == ICE_VSI_VF || + pf->vsi[i]->type == ICE_VSI_SWITCHDEV_CTRL)) + ice_for_each_traffic_class(tc) + ice_lag_move_vf_nodes_tc_sync(lag, dest_hw, i, + tc); +} + +/** * ice_init_lag - initialize support for LAG * @pf: PF struct * @@ -377,7 +1927,10 @@ int ice_init_lag(struct ice_pf *pf) struct device *dev = ice_pf_to_dev(pf); struct ice_lag *lag; struct ice_vsi *vsi; - int err; + u64 recipe_bits = 0; + int n, err; + + ice_lag_init_feature_support_flag(pf); pf->lag = kzalloc(sizeof(*lag), GFP_KERNEL); if (!pf->lag) @@ -394,8 +1947,8 @@ int ice_init_lag(struct ice_pf *pf) lag->pf = pf; lag->netdev = vsi->netdev; lag->role = ICE_LAG_NONE; + lag->active_port = ICE_LAG_INVALID_PORT; lag->bonded = false; - lag->peer_netdev = NULL; lag->upper_netdev = NULL; lag->notif_block.notifier_call = NULL; @@ -405,6 +1958,25 @@ int ice_init_lag(struct ice_pf *pf) goto lag_error; } + err = ice_create_lag_recipe(&pf->hw, &lag->pf_recipe, ice_dflt_vsi_rcp, + 1); + if (err) + goto lag_error; + + /* associate recipes to profiles */ + for (n = 0; n < ICE_PROFID_IPV6_GTPU_IPV6_TCP_INNER; n++) { + err = ice_aq_get_recipe_to_profile(&pf->hw, n, + (u8 *)&recipe_bits, NULL); + if (err) + continue; + + if (recipe_bits & BIT(ICE_SW_LKUP_DFLT)) { + recipe_bits |= BIT(lag->pf_recipe); + ice_aq_map_recipe_to_profile(&pf->hw, n, + (u8 *)&recipe_bits, NULL); + } + } + ice_display_lag_info(lag); dev_dbg(dev, "INIT LAG complete\n"); @@ -435,11 +2007,94 @@ void ice_deinit_lag(struct ice_pf *pf) if (lag->pf) ice_unregister_lag_handler(lag); - dev_put(lag->upper_netdev); + flush_workqueue(ice_lag_wq); - dev_put(lag->peer_netdev); + ice_free_hw_res(&pf->hw, ICE_AQC_RES_TYPE_RECIPE, 1, + &pf->lag->pf_recipe); kfree(lag); pf->lag = NULL; } + +/** + * ice_lag_rebuild - rebuild lag resources after reset + * @pf: pointer to local pf struct + * + * PF resets are promoted to CORER resets when interface in an aggregate. This + * means that we need to rebuild the PF resources for the interface. Since + * this will happen outside the normal event processing, need to acquire the lag + * lock. + * + * This function will also evaluate the VF resources if this is the primary + * interface. + */ +void ice_lag_rebuild(struct ice_pf *pf) +{ + struct ice_lag_netdev_list ndlist; + struct ice_lag *lag, *prim_lag; + struct list_head *tmp, *n; + u8 act_port, loc_port; + + if (!pf->lag || !pf->lag->bonded) + return; + + mutex_lock(&pf->lag_mutex); + + lag = pf->lag; + if (lag->primary) { + prim_lag = lag; + } else { + struct ice_lag_netdev_list *nl; + struct net_device *tmp_nd; + + INIT_LIST_HEAD(&ndlist.node); + rcu_read_lock(); + for_each_netdev_in_bond_rcu(lag->upper_netdev, tmp_nd) { + nl = kzalloc(sizeof(*nl), GFP_KERNEL); + if (!nl) + break; + + nl->netdev = tmp_nd; + list_add(&nl->node, &ndlist.node); + } + rcu_read_unlock(); + lag->netdev_head = &ndlist.node; + prim_lag = ice_lag_find_primary(lag); + } + + if (!prim_lag) { + dev_dbg(ice_pf_to_dev(pf), "No primary interface in aggregate, can't rebuild\n"); + goto lag_rebuild_out; + } + + act_port = prim_lag->active_port; + loc_port = lag->pf->hw.port_info->lport; + + /* configure SWID for this port */ + if (lag->primary) { + ice_lag_primary_swid(lag, true); + } else { + ice_lag_set_swid(prim_lag->pf->hw.port_info->sw_id, lag, true); + ice_lag_add_prune_list(prim_lag, pf); + if (act_port == loc_port) + ice_lag_move_vf_nodes_sync(prim_lag, &pf->hw); + } + + ice_lag_cfg_cp_fltr(lag, true); + + if (lag->pf_rule_id) + if (ice_lag_cfg_dflt_fltr(lag, true)) + dev_err(ice_pf_to_dev(pf), "Error adding default VSI rule in rebuild\n"); + + ice_clear_rdma_cap(pf); +lag_rebuild_out: + list_for_each_safe(tmp, n, &ndlist.node) { + struct ice_lag_netdev_list *entry; + + entry = list_entry(tmp, struct ice_lag_netdev_list, node); + list_del(&entry->node); + kfree(entry); + } + mutex_unlock(&pf->lag_mutex); +} diff --git a/drivers/net/ethernet/intel/ice/ice_lag.h b/drivers/net/ethernet/intel/ice/ice_lag.h index 2c373676c42f..18075b82485a 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.h +++ b/drivers/net/ethernet/intel/ice/ice_lag.h @@ -14,20 +14,52 @@ enum ice_lag_role { ICE_LAG_UNSET }; +#define ICE_LAG_INVALID_PORT 0xFF + +#define ICE_LAG_RESET_RETRIES 5 + struct ice_pf; +struct ice_vf; + +struct ice_lag_netdev_list { + struct list_head node; + struct net_device *netdev; +}; /* LAG info struct */ struct ice_lag { struct ice_pf *pf; /* backlink to PF struct */ struct net_device *netdev; /* this PF's netdev */ - struct net_device *peer_netdev; struct net_device *upper_netdev; /* upper bonding netdev */ + struct list_head *netdev_head; struct notifier_block notif_block; + s32 bond_mode; + u16 bond_swid; /* swid for primary interface */ + u8 active_port; /* lport value for the current active port */ u8 bonded:1; /* currently bonded */ u8 primary:1; /* this is primary */ + u16 pf_recipe; + u16 pf_rule_id; + u16 cp_rule_idx; u8 role; }; +/* LAG workqueue struct */ +struct ice_lag_work { + struct work_struct lag_task; + struct ice_lag_netdev_list netdev_list; + struct ice_lag *lag; + unsigned long event; + struct net_device *event_netdev; + union { + struct netdev_notifier_changeupper_info changeupper_info; + struct netdev_notifier_bonding_info bonding_info; + struct netdev_notifier_info notifier_info; + } info; +}; + +void ice_lag_move_new_vf_nodes(struct ice_vf *vf); int ice_init_lag(struct ice_pf *pf); void ice_deinit_lag(struct ice_pf *pf); +void ice_lag_rebuild(struct ice_pf *pf); #endif /* _ICE_LAG_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 0054d7e64ec3..077f2e91ae1a 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -3970,7 +3970,7 @@ bool ice_is_feature_supported(struct ice_pf *pf, enum ice_feature f) * @pf: pointer to the struct ice_pf instance * @f: feature enum to set */ -static void ice_set_feature_support(struct ice_pf *pf, enum ice_feature f) +void ice_set_feature_support(struct ice_pf *pf, enum ice_feature f) { if (f < 0 || f >= ICE_F_MAX) return; @@ -4076,3 +4076,28 @@ void ice_vsi_ctx_clear_allow_override(struct ice_vsi_ctx *ctx) { ctx->info.sec_flags &= ~ICE_AQ_VSI_SEC_FLAG_ALLOW_DEST_OVRD; } + +/** + * ice_vsi_update_local_lb - update sw block in VSI with local loopback bit + * @vsi: pointer to VSI structure + * @set: set or unset the bit + */ +int +ice_vsi_update_local_lb(struct ice_vsi *vsi, bool set) +{ + struct ice_vsi_ctx ctx = { + .info = vsi->info, + }; + + ctx.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); + if (set) + ctx.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_LOCAL_LB; + else + ctx.info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_LOCAL_LB; + + if (ice_update_vsi(&vsi->back->hw, vsi->idx, &ctx, NULL)) + return -ENODEV; + + vsi->info = ctx.info; + return 0; +} diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index e985766e6bb5..dd53fe968ad8 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -157,11 +157,13 @@ void ice_vsi_ctx_clear_antispoof(struct ice_vsi_ctx *ctx); void ice_vsi_ctx_set_allow_override(struct ice_vsi_ctx *ctx); void ice_vsi_ctx_clear_allow_override(struct ice_vsi_ctx *ctx); +int ice_vsi_update_local_lb(struct ice_vsi *vsi, bool set); int ice_vsi_add_vlan_zero(struct ice_vsi *vsi); int ice_vsi_del_vlan_zero(struct ice_vsi *vsi); bool ice_vsi_has_non_zero_vlans(struct ice_vsi *vsi); u16 ice_vsi_num_non_zero_vlans(struct ice_vsi *vsi); bool ice_is_feature_supported(struct ice_pf *pf, enum ice_feature f); +void ice_set_feature_support(struct ice_pf *pf, enum ice_feature f); void ice_clear_feature_support(struct ice_pf *pf, enum ice_feature f); void ice_init_feature_support(struct ice_pf *pf); bool ice_vsi_is_rx_queue_active(struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index cf92c39467c8..2e80d5cd9f56 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -64,6 +64,7 @@ struct device *ice_hw_to_dev(struct ice_hw *hw) } static struct workqueue_struct *ice_wq; +struct workqueue_struct *ice_lag_wq; static const struct net_device_ops ice_netdev_safe_mode_ops; static const struct net_device_ops ice_netdev_ops; @@ -80,7 +81,7 @@ ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, void *data, void (*cleanup)(struct flow_block_cb *block_cb)); -bool netif_is_ice(struct net_device *dev) +bool netif_is_ice(const struct net_device *dev) { return dev && (dev->netdev_ops == &ice_netdev_ops); } @@ -635,6 +636,11 @@ static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); + if (pf->lag && pf->lag->bonded && reset_type == ICE_RESET_PFR) { + dev_dbg(dev, "PFR on a bonded interface, promoting to CORER\n"); + reset_type = ICE_RESET_CORER; + } + ice_prepare_for_reset(pf, reset_type); /* trigger the reset */ @@ -718,8 +724,13 @@ static void ice_reset_subtask(struct ice_pf *pf) } /* No pending resets to finish processing. Check for new resets */ - if (test_bit(ICE_PFR_REQ, pf->state)) + if (test_bit(ICE_PFR_REQ, pf->state)) { reset_type = ICE_RESET_PFR; + if (pf->lag && pf->lag->bonded) { + dev_dbg(ice_pf_to_dev(pf), "PFR on a bonded interface, promoting to CORER\n"); + reset_type = ICE_RESET_CORER; + } + } if (test_bit(ICE_CORER_REQ, pf->state)) reset_type = ICE_RESET_CORER; if (test_bit(ICE_GLOBR_REQ, pf->state)) @@ -3392,6 +3403,7 @@ static void ice_set_ops(struct ice_vsi *vsi) netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_XSK_ZEROCOPY | NETDEV_XDP_ACT_RX_SG; + netdev->xdp_zc_max_segs = ICE_MAX_BUF_TXD; } /** @@ -3794,6 +3806,7 @@ u16 ice_get_avail_rxq_count(struct ice_pf *pf) static void ice_deinit_pf(struct ice_pf *pf) { ice_service_task_stop(pf); + mutex_destroy(&pf->lag_mutex); mutex_destroy(&pf->adev_mutex); mutex_destroy(&pf->sw_mutex); mutex_destroy(&pf->tc_mutex); @@ -3874,6 +3887,7 @@ static int ice_init_pf(struct ice_pf *pf) mutex_init(&pf->sw_mutex); mutex_init(&pf->tc_mutex); mutex_init(&pf->adev_mutex); + mutex_init(&pf->lag_mutex); INIT_HLIST_HEAD(&pf->aq_wait_list); spin_lock_init(&pf->aq_wait_lock); @@ -5570,7 +5584,7 @@ static struct pci_driver ice_driver = { */ static int __init ice_module_init(void) { - int status; + int status = -ENOMEM; pr_info("%s\n", ice_driver_string); pr_info("%s\n", ice_copyright); @@ -5578,15 +5592,27 @@ static int __init ice_module_init(void) ice_wq = alloc_workqueue("%s", 0, 0, KBUILD_MODNAME); if (!ice_wq) { pr_err("Failed to create workqueue\n"); - return -ENOMEM; + return status; + } + + ice_lag_wq = alloc_ordered_workqueue("ice_lag_wq", 0); + if (!ice_lag_wq) { + pr_err("Failed to create LAG workqueue\n"); + goto err_dest_wq; } status = pci_register_driver(&ice_driver); if (status) { pr_err("failed to register PCI driver, err %d\n", status); - destroy_workqueue(ice_wq); + goto err_dest_lag_wq; } + return 0; + +err_dest_lag_wq: + destroy_workqueue(ice_lag_wq); +err_dest_wq: + destroy_workqueue(ice_wq); return status; } module_init(ice_module_init); @@ -5601,6 +5627,7 @@ static void __exit ice_module_exit(void) { pci_unregister_driver(&ice_driver); destroy_workqueue(ice_wq); + destroy_workqueue(ice_lag_wq); pr_info("module unloaded\n"); } module_exit(ice_module_exit); @@ -5703,7 +5730,7 @@ static void ice_set_rx_mode(struct net_device *netdev) struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; - if (!vsi) + if (!vsi || ice_is_switchdev_running(vsi->back)) return; /* Set the flags to synchronize filters @@ -6255,7 +6282,7 @@ static void ice_tx_dim_work(struct work_struct *work) u16 itr; dim = container_of(work, struct dim, work); - rc = (struct ice_ring_container *)dim->priv; + rc = dim->priv; WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile)); @@ -6275,7 +6302,7 @@ static void ice_rx_dim_work(struct work_struct *work) u16 itr; dim = container_of(work, struct dim, work); - rc = (struct ice_ring_container *)dim->priv; + rc = dim->priv; WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile)); @@ -7356,6 +7383,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) clear_bit(ICE_RESET_FAILED, pf->state); ice_plug_aux_dev(pf); + if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) + ice_lag_rebuild(pf); return; err_vsi_rebuild: diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c index e30e12321abd..c686ac0935eb 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.c +++ b/drivers/net/ethernet/intel/ice/ice_repr.c @@ -254,7 +254,7 @@ static const struct net_device_ops ice_repr_netdev_ops = { * ice_is_port_repr_netdev - Check if a given netdevice is a port representor netdev * @netdev: pointer to netdev */ -bool ice_is_port_repr_netdev(struct net_device *netdev) +bool ice_is_port_repr_netdev(const struct net_device *netdev) { return netdev && (netdev->netdev_ops == &ice_repr_netdev_ops); } diff --git a/drivers/net/ethernet/intel/ice/ice_repr.h b/drivers/net/ethernet/intel/ice/ice_repr.h index 9c2a6f496b3b..e1ee2d2c1d2d 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.h +++ b/drivers/net/ethernet/intel/ice/ice_repr.h @@ -12,6 +12,7 @@ struct ice_repr { struct ice_q_vector *q_vector; struct net_device *netdev; struct metadata_dst *dst; + struct ice_esw_br_port *br_port; #ifdef CONFIG_ICE_SWITCHDEV /* info about slow path rule */ struct ice_rule_query_data sp_rule; @@ -27,5 +28,5 @@ void ice_repr_stop_tx_queues(struct ice_repr *repr); void ice_repr_set_traffic_vsi(struct ice_repr *repr, struct ice_vsi *vsi); struct ice_repr *ice_netdev_to_repr(struct net_device *netdev); -bool ice_is_port_repr_netdev(struct net_device *netdev); +bool ice_is_port_repr_netdev(const struct net_device *netdev); #endif diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index b664d60fd037..f4677704b95e 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -447,7 +447,7 @@ ice_aq_cfg_sched_elems(struct ice_hw *hw, u16 elems_req, * * Move scheduling elements (0x0408) */ -static int +int ice_aq_move_sched_elems(struct ice_hw *hw, u16 grps_req, struct ice_aqc_move_elem *buf, u16 buf_size, u16 *grps_movd, struct ice_sq_cd *cd) @@ -526,7 +526,7 @@ ice_aq_query_sched_res(struct ice_hw *hw, u16 buf_size, * * This function suspends or resumes HW nodes */ -static int +int ice_sched_suspend_resume_elems(struct ice_hw *hw, u8 num_nodes, u32 *node_teids, bool suspend) { @@ -569,18 +569,24 @@ ice_alloc_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) { struct ice_vsi_ctx *vsi_ctx; struct ice_q_ctx *q_ctx; + u16 idx; vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); if (!vsi_ctx) return -EINVAL; /* allocate LAN queue contexts */ if (!vsi_ctx->lan_q_ctx[tc]) { - vsi_ctx->lan_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw), - new_numqs, - sizeof(*q_ctx), - GFP_KERNEL); - if (!vsi_ctx->lan_q_ctx[tc]) + q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs, + sizeof(*q_ctx), GFP_KERNEL); + if (!q_ctx) return -ENOMEM; + + for (idx = 0; idx < new_numqs; idx++) { + q_ctx[idx].q_handle = ICE_INVAL_Q_HANDLE; + q_ctx[idx].q_teid = ICE_INVAL_TEID; + } + + vsi_ctx->lan_q_ctx[tc] = q_ctx; vsi_ctx->num_lan_q_entries[tc] = new_numqs; return 0; } @@ -592,9 +598,16 @@ ice_alloc_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) sizeof(*q_ctx), GFP_KERNEL); if (!q_ctx) return -ENOMEM; + memcpy(q_ctx, vsi_ctx->lan_q_ctx[tc], prev_num * sizeof(*q_ctx)); devm_kfree(ice_hw_to_dev(hw), vsi_ctx->lan_q_ctx[tc]); + + for (idx = prev_num; idx < new_numqs; idx++) { + q_ctx[idx].q_handle = ICE_INVAL_Q_HANDLE; + q_ctx[idx].q_teid = ICE_INVAL_TEID; + } + vsi_ctx->lan_q_ctx[tc] = q_ctx; vsi_ctx->num_lan_q_entries[tc] = new_numqs; } @@ -1044,7 +1057,7 @@ ice_sched_add_nodes_to_hw_layer(struct ice_port_info *pi, * * This function add nodes to a given layer. */ -static int +int ice_sched_add_nodes_to_layer(struct ice_port_info *pi, struct ice_sched_node *tc_node, struct ice_sched_node *parent, u8 layer, @@ -1119,7 +1132,7 @@ static u8 ice_sched_get_qgrp_layer(struct ice_hw *hw) * * This function returns the current VSI layer number */ -static u8 ice_sched_get_vsi_layer(struct ice_hw *hw) +u8 ice_sched_get_vsi_layer(struct ice_hw *hw) { /* Num Layers VSI layer * 9 6 @@ -1142,7 +1155,7 @@ static u8 ice_sched_get_vsi_layer(struct ice_hw *hw) * * This function returns the current aggregator layer number */ -static u8 ice_sched_get_agg_layer(struct ice_hw *hw) +u8 ice_sched_get_agg_layer(struct ice_hw *hw) { /* Num Layers aggregator layer * 9 4 @@ -1577,7 +1590,7 @@ ice_sched_get_vsi_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, * This function retrieves an aggregator node for a given aggregator ID from * a given TC branch */ -static struct ice_sched_node * +struct ice_sched_node * ice_sched_get_agg_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, u32 agg_id) { @@ -2139,7 +2152,7 @@ ice_get_agg_info(struct ice_hw *hw, u32 agg_id) * This function walks through the aggregator subtree to find a free parent * node */ -static struct ice_sched_node * +struct ice_sched_node * ice_sched_get_free_vsi_parent(struct ice_hw *hw, struct ice_sched_node *node, u16 *num_nodes) { diff --git a/drivers/net/ethernet/intel/ice/ice_sched.h b/drivers/net/ethernet/intel/ice/ice_sched.h index 9c100747445a..8bd26353d76a 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.h +++ b/drivers/net/ethernet/intel/ice/ice_sched.h @@ -146,8 +146,29 @@ ice_sched_set_node_bw_lmt_per_tc(struct ice_port_info *pi, u32 id, enum ice_agg_type agg_type, u8 tc, enum ice_rl_type rl_type, u32 bw); int ice_cfg_rl_burst_size(struct ice_hw *hw, u32 bytes); +int +ice_sched_suspend_resume_elems(struct ice_hw *hw, u8 num_nodes, u32 *node_teids, + bool suspend); +struct ice_sched_node * +ice_sched_get_agg_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, + u32 agg_id); +u8 ice_sched_get_agg_layer(struct ice_hw *hw); +u8 ice_sched_get_vsi_layer(struct ice_hw *hw); +struct ice_sched_node * +ice_sched_get_free_vsi_parent(struct ice_hw *hw, struct ice_sched_node *node, + u16 *num_nodes); +int +ice_sched_add_nodes_to_layer(struct ice_port_info *pi, + struct ice_sched_node *tc_node, + struct ice_sched_node *parent, u8 layer, + u16 num_nodes, u32 *first_node_teid, + u16 *num_nodes_added); void ice_sched_replay_agg_vsi_preinit(struct ice_hw *hw); void ice_sched_replay_agg(struct ice_hw *hw); +int +ice_aq_move_sched_elems(struct ice_hw *hw, u16 grps_req, + struct ice_aqc_move_elem *buf, u16 buf_size, + u16 *grps_movd, struct ice_sq_cd *cd); int ice_replay_vsi_agg(struct ice_hw *hw, u16 vsi_handle); int ice_sched_replay_q_bw(struct ice_port_info *pi, struct ice_q_ctx *q_ctx); #endif /* _ICE_SCHED_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 6db4ca7978cb..91bc92f5059b 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -20,12 +20,11 @@ * byte 0 = 0x2: to identify it as locally administered DA MAC * byte 6 = 0x2: to identify it as locally administered SA MAC * byte 12 = 0x81 & byte 13 = 0x00: - * In case of VLAN filter first two bytes defines ether type (0x8100) - * and remaining two bytes are placeholder for programming a given VLAN ID - * In case of Ether type filter it is treated as header without VLAN tag - * and byte 12 and 13 is used to program a given Ether type instead + * In case of VLAN filter first two bytes defines ether type (0x8100) + * and remaining two bytes are placeholder for programming a given VLAN ID + * In case of Ether type filter it is treated as header without VLAN tag + * and byte 12 and 13 is used to program a given Ether type instead */ -#define DUMMY_ETH_HDR_LEN 16 static const u8 dummy_eth_header[DUMMY_ETH_HDR_LEN] = { 0x2, 0, 0, 0, 0, 0, 0x2, 0, 0, 0, 0, 0, 0x81, 0, 0, 0}; @@ -1369,14 +1368,6 @@ static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = { ICE_PKT_PROFILE(tcp, 0), }; -#define ICE_SW_RULE_RX_TX_HDR_SIZE(s, l) struct_size((s), hdr_data, (l)) -#define ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s) \ - ICE_SW_RULE_RX_TX_HDR_SIZE((s), DUMMY_ETH_HDR_LEN) -#define ICE_SW_RULE_RX_TX_NO_HDR_SIZE(s) \ - ICE_SW_RULE_RX_TX_HDR_SIZE((s), 0) -#define ICE_SW_RULE_LG_ACT_SIZE(s, n) struct_size((s), act, (n)) -#define ICE_SW_RULE_VSI_LIST_SIZE(s, n) struct_size((s), vsi, (n)) - /* this is a recipe to profile association bitmap */ static DECLARE_BITMAP(recipe_to_profile[ICE_MAX_NUM_RECIPES], ICE_MAX_NUM_PROFILES); @@ -1841,8 +1832,13 @@ ice_aq_alloc_free_vsi_list(struct ice_hw *hw, u16 *vsi_list_id, lkup_type == ICE_SW_LKUP_DFLT) { sw_buf->res_type = cpu_to_le16(ICE_AQC_RES_TYPE_VSI_LIST_REP); } else if (lkup_type == ICE_SW_LKUP_VLAN) { - sw_buf->res_type = - cpu_to_le16(ICE_AQC_RES_TYPE_VSI_LIST_PRUNE); + if (opc == ice_aqc_opc_alloc_res) + sw_buf->res_type = + cpu_to_le16(ICE_AQC_RES_TYPE_VSI_LIST_PRUNE | + ICE_AQC_RES_TYPE_FLAG_SHARED); + else + sw_buf->res_type = + cpu_to_le16(ICE_AQC_RES_TYPE_VSI_LIST_PRUNE); } else { status = -EINVAL; goto ice_aq_alloc_free_vsi_list_exit; @@ -1910,7 +1906,7 @@ ice_aq_sw_rules(struct ice_hw *hw, void *rule_list, u16 rule_list_sz, * * Add(0x0290) */ -static int +int ice_aq_add_recipe(struct ice_hw *hw, struct ice_aqc_recipe_data_elem *s_recipe_list, u16 num_recipes, struct ice_sq_cd *cd) @@ -1947,7 +1943,7 @@ ice_aq_add_recipe(struct ice_hw *hw, * The caller must supply enough space in s_recipe_list to hold all possible * recipes and *num_recipes must equal ICE_MAX_NUM_RECIPES. */ -static int +int ice_aq_get_recipe(struct ice_hw *hw, struct ice_aqc_recipe_data_elem *s_recipe_list, u16 *num_recipes, u16 recipe_root, struct ice_sq_cd *cd) @@ -2040,7 +2036,7 @@ error_out: * @cd: pointer to command details structure or NULL * Recipe to profile association (0x0291) */ -static int +int ice_aq_map_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, struct ice_sq_cd *cd) { @@ -2066,7 +2062,7 @@ ice_aq_map_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, * @cd: pointer to command details structure or NULL * Associate profile ID with given recipe (0x0293) */ -static int +int ice_aq_get_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, struct ice_sq_cd *cd) { @@ -2090,7 +2086,7 @@ ice_aq_get_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, * @hw: pointer to the hardware structure * @rid: recipe ID returned as response to AQ call */ -static int ice_alloc_recipe(struct ice_hw *hw, u16 *rid) +int ice_alloc_recipe(struct ice_hw *hw, u16 *rid) { struct ice_aqc_alloc_free_res_elem *sw_buf; u16 buf_len; @@ -2272,6 +2268,10 @@ ice_get_recp_frm_fw(struct ice_hw *hw, struct ice_sw_recipe *recps, u8 rid, /* Propagate some data to the recipe database */ recps[idx].is_root = !!is_root; recps[idx].priority = root_bufs.content.act_ctrl_fwd_priority; + recps[idx].need_pass_l2 = root_bufs.content.act_ctrl & + ICE_AQ_RECIPE_ACT_NEED_PASS_L2; + recps[idx].allow_pass_l2 = root_bufs.content.act_ctrl & + ICE_AQ_RECIPE_ACT_ALLOW_PASS_L2; bitmap_zero(recps[idx].res_idxs, ICE_MAX_FV_WORDS); if (root_bufs.content.result_indx & ICE_AQ_RECIPE_RESULT_EN) { recps[idx].chain_idx = root_bufs.content.result_indx & @@ -2460,6 +2460,15 @@ static void ice_fill_sw_info(struct ice_hw *hw, struct ice_fltr_info *fi) } /** + * ice_fill_eth_hdr - helper to copy dummy_eth_hdr into supplied buffer + * @eth_hdr: pointer to buffer to populate + */ +void ice_fill_eth_hdr(u8 *eth_hdr) +{ + memcpy(eth_hdr, dummy_eth_header, DUMMY_ETH_HDR_LEN); +} + +/** * ice_fill_sw_rule - Helper function to fill switch rule structure * @hw: pointer to the hardware structure * @f_info: entry containing packet forwarding information @@ -3118,7 +3127,7 @@ ice_find_rule_entry(struct ice_hw *hw, u8 recp_id, struct ice_fltr_info *f_info) * handle element. This can be extended further to search VSI list with more * than 1 vsi_count. Returns pointer to VSI list entry if found. */ -static struct ice_vsi_list_map_info * +struct ice_vsi_list_map_info * ice_find_vsi_list_entry(struct ice_hw *hw, u8 recp_id, u16 vsi_handle, u16 *vsi_list_id) { @@ -3129,7 +3138,7 @@ ice_find_vsi_list_entry(struct ice_hw *hw, u8 recp_id, u16 vsi_handle, list_head = &sw->recp_list[recp_id].filt_rules; list_for_each_entry(list_itr, list_head, list_entry) { - if (list_itr->vsi_count == 1 && list_itr->vsi_list_info) { + if (list_itr->vsi_list_info) { map_info = list_itr->vsi_list_info; if (test_bit(vsi_handle, map_info->vsi_map)) { *vsi_list_id = map_info->vsi_list_id; @@ -4540,6 +4549,45 @@ ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, .offs = {__VA_ARGS__}, \ } +/** + * ice_share_res - set a resource as shared or dedicated + * @hw: hw struct of original owner of resource + * @type: resource type + * @shared: is the resource being set to shared + * @res_id: resource id (descriptor) + */ +int ice_share_res(struct ice_hw *hw, u16 type, u8 shared, u16 res_id) +{ + struct ice_aqc_alloc_free_res_elem *buf; + u16 buf_len; + int status; + + buf_len = struct_size(buf, elem, 1); + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf->num_elems = cpu_to_le16(1); + if (shared) + buf->res_type = cpu_to_le16(((type << ICE_AQC_RES_TYPE_S) & + ICE_AQC_RES_TYPE_M) | + ICE_AQC_RES_TYPE_FLAG_SHARED); + else + buf->res_type = cpu_to_le16(((type << ICE_AQC_RES_TYPE_S) & + ICE_AQC_RES_TYPE_M) & + ~ICE_AQC_RES_TYPE_FLAG_SHARED); + + buf->elem[0].e.sw_resp = cpu_to_le16(res_id); + status = ice_aq_alloc_free_res(hw, 1, buf, buf_len, + ice_aqc_opc_share_res, NULL); + if (status) + ice_debug(hw, ICE_DBG_SW, "Could not set resource type %u id %u to %s\n", + type, res_id, shared ? "SHARED" : "DEDICATED"); + + kfree(buf); + return status; +} + /* This is mapping table entry that maps every word within a given protocol * structure to the real byte offset as per the specification of that * protocol header. @@ -4613,13 +4661,13 @@ static struct ice_protocol_entry ice_prot_id_tbl[ICE_PROTOCOL_LAST] = { * ice_find_recp - find a recipe * @hw: pointer to the hardware structure * @lkup_exts: extension sequence to match - * @tun_type: type of recipe tunnel + * @rinfo: information regarding the rule e.g. priority and action info * * Returns index of matching recipe, or ICE_MAX_NUM_RECIPES if not found. */ static u16 ice_find_recp(struct ice_hw *hw, struct ice_prot_lkup_ext *lkup_exts, - enum ice_sw_tunnel_type tun_type) + const struct ice_adv_rule_info *rinfo) { bool refresh_required = true; struct ice_sw_recipe *recp; @@ -4680,9 +4728,12 @@ ice_find_recp(struct ice_hw *hw, struct ice_prot_lkup_ext *lkup_exts, } /* If for "i"th recipe the found was never set to false * then it means we found our match - * Also tun type of recipe needs to be checked + * Also tun type and *_pass_l2 of recipe needs to be + * checked */ - if (found && recp[i].tun_type == tun_type) + if (found && recp[i].tun_type == rinfo->tun_type && + recp[i].need_pass_l2 == rinfo->need_pass_l2 && + recp[i].allow_pass_l2 == rinfo->allow_pass_l2) return i; /* Return the recipe ID */ } } @@ -4952,6 +5003,7 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, unsigned long *profiles) { DECLARE_BITMAP(result_idx_bm, ICE_MAX_FV_WORDS); + struct ice_aqc_recipe_content *content; struct ice_aqc_recipe_data_elem *tmp; struct ice_aqc_recipe_data_elem *buf; struct ice_recp_grp_entry *entry; @@ -5012,6 +5064,8 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, if (status) goto err_unroll; + content = &buf[recps].content; + /* Clear the result index of the located recipe, as this will be * updated, if needed, later in the recipe creation process. */ @@ -5022,26 +5076,24 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, /* if the recipe is a non-root recipe RID should be programmed * as 0 for the rules to be applied correctly. */ - buf[recps].content.rid = 0; - memset(&buf[recps].content.lkup_indx, 0, - sizeof(buf[recps].content.lkup_indx)); + content->rid = 0; + memset(&content->lkup_indx, 0, + sizeof(content->lkup_indx)); /* All recipes use look-up index 0 to match switch ID. */ - buf[recps].content.lkup_indx[0] = ICE_AQ_SW_ID_LKUP_IDX; - buf[recps].content.mask[0] = - cpu_to_le16(ICE_AQ_SW_ID_LKUP_MASK); + content->lkup_indx[0] = ICE_AQ_SW_ID_LKUP_IDX; + content->mask[0] = cpu_to_le16(ICE_AQ_SW_ID_LKUP_MASK); /* Setup lkup_indx 1..4 to INVALID/ignore and set the mask * to be 0 */ for (i = 1; i <= ICE_NUM_WORDS_RECIPE; i++) { - buf[recps].content.lkup_indx[i] = 0x80; - buf[recps].content.mask[i] = 0; + content->lkup_indx[i] = 0x80; + content->mask[i] = 0; } for (i = 0; i < entry->r_group.n_val_pairs; i++) { - buf[recps].content.lkup_indx[i + 1] = entry->fv_idx[i]; - buf[recps].content.mask[i + 1] = - cpu_to_le16(entry->fv_mask[i]); + content->lkup_indx[i + 1] = entry->fv_idx[i]; + content->mask[i + 1] = cpu_to_le16(entry->fv_mask[i]); } if (rm->n_grp_count > 1) { @@ -5055,7 +5107,7 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, } entry->chain_idx = chain_idx; - buf[recps].content.result_indx = + content->result_indx = ICE_AQ_RECIPE_RESULT_EN | ((chain_idx << ICE_AQ_RECIPE_RESULT_DATA_S) & ICE_AQ_RECIPE_RESULT_DATA_M); @@ -5069,7 +5121,13 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, ICE_MAX_NUM_RECIPES); set_bit(buf[recps].recipe_indx, (unsigned long *)buf[recps].recipe_bitmap); - buf[recps].content.act_ctrl_fwd_priority = rm->priority; + content->act_ctrl_fwd_priority = rm->priority; + + if (rm->need_pass_l2) + content->act_ctrl |= ICE_AQ_RECIPE_ACT_NEED_PASS_L2; + + if (rm->allow_pass_l2) + content->act_ctrl |= ICE_AQ_RECIPE_ACT_ALLOW_PASS_L2; recps++; } @@ -5107,9 +5165,11 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, if (status) goto err_unroll; + content = &buf[recps].content; + buf[recps].recipe_indx = (u8)rid; - buf[recps].content.rid = (u8)rid; - buf[recps].content.rid |= ICE_AQ_RECIPE_ID_IS_ROOT; + content->rid = (u8)rid; + content->rid |= ICE_AQ_RECIPE_ID_IS_ROOT; /* the new entry created should also be part of rg_list to * make sure we have complete recipe */ @@ -5121,16 +5181,13 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, goto err_unroll; } last_chain_entry->rid = rid; - memset(&buf[recps].content.lkup_indx, 0, - sizeof(buf[recps].content.lkup_indx)); + memset(&content->lkup_indx, 0, sizeof(content->lkup_indx)); /* All recipes use look-up index 0 to match switch ID. */ - buf[recps].content.lkup_indx[0] = ICE_AQ_SW_ID_LKUP_IDX; - buf[recps].content.mask[0] = - cpu_to_le16(ICE_AQ_SW_ID_LKUP_MASK); + content->lkup_indx[0] = ICE_AQ_SW_ID_LKUP_IDX; + content->mask[0] = cpu_to_le16(ICE_AQ_SW_ID_LKUP_MASK); for (i = 1; i <= ICE_NUM_WORDS_RECIPE; i++) { - buf[recps].content.lkup_indx[i] = - ICE_AQ_RECIPE_LKUP_IGNORE; - buf[recps].content.mask[i] = 0; + content->lkup_indx[i] = ICE_AQ_RECIPE_LKUP_IGNORE; + content->mask[i] = 0; } i = 1; @@ -5142,8 +5199,8 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, last_chain_entry->chain_idx = ICE_INVAL_CHAIN_IND; list_for_each_entry(entry, &rm->rg_list, l_entry) { last_chain_entry->fv_idx[i] = entry->chain_idx; - buf[recps].content.lkup_indx[i] = entry->chain_idx; - buf[recps].content.mask[i++] = cpu_to_le16(0xFFFF); + content->lkup_indx[i] = entry->chain_idx; + content->mask[i++] = cpu_to_le16(0xFFFF); set_bit(entry->rid, rm->r_bitmap); } list_add(&last_chain_entry->l_entry, &rm->rg_list); @@ -5155,7 +5212,7 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, status = -EINVAL; goto err_unroll; } - buf[recps].content.act_ctrl_fwd_priority = rm->priority; + content->act_ctrl_fwd_priority = rm->priority; recps++; rm->root_rid = (u8)rid; @@ -5220,6 +5277,8 @@ ice_add_sw_recipe(struct ice_hw *hw, struct ice_sw_recipe *rm, recp->priority = buf[buf_idx].content.act_ctrl_fwd_priority; recp->n_grp_count = rm->n_grp_count; recp->tun_type = rm->tun_type; + recp->need_pass_l2 = rm->need_pass_l2; + recp->allow_pass_l2 = rm->allow_pass_l2; recp->recp_created = true; } rm->root_buf = buf; @@ -5388,6 +5447,9 @@ ice_add_adv_recipe(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, /* set the recipe priority if specified */ rm->priority = (u8)rinfo->priority; + rm->need_pass_l2 = rinfo->need_pass_l2; + rm->allow_pass_l2 = rinfo->allow_pass_l2; + /* Find offsets from the field vector. Pick the first one for all the * recipes. */ @@ -5403,7 +5465,7 @@ ice_add_adv_recipe(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, } /* Look for a recipe which matches our requested fv / mask list */ - *rid = ice_find_recp(hw, lkup_exts, rinfo->tun_type); + *rid = ice_find_recp(hw, lkup_exts, rinfo); if (*rid < ICE_MAX_NUM_RECIPES) /* Success if found a recipe that match the existing criteria */ goto err_unroll; @@ -5839,7 +5901,9 @@ static bool ice_rules_equal(const struct ice_adv_rule_info *first, return first->sw_act.flag == second->sw_act.flag && first->tun_type == second->tun_type && first->vlan_type == second->vlan_type && - first->src_vsi == second->src_vsi; + first->src_vsi == second->src_vsi && + first->need_pass_l2 == second->need_pass_l2 && + first->allow_pass_l2 == second->allow_pass_l2; } /** @@ -6078,7 +6142,8 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, if (!(rinfo->sw_act.fltr_act == ICE_FWD_TO_VSI || rinfo->sw_act.fltr_act == ICE_FWD_TO_Q || rinfo->sw_act.fltr_act == ICE_FWD_TO_QGRP || - rinfo->sw_act.fltr_act == ICE_DROP_PACKET)) { + rinfo->sw_act.fltr_act == ICE_DROP_PACKET || + rinfo->sw_act.fltr_act == ICE_NOP)) { status = -EIO; goto free_pkt_profile; } @@ -6089,7 +6154,8 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, goto free_pkt_profile; } - if (rinfo->sw_act.fltr_act == ICE_FWD_TO_VSI) + if (rinfo->sw_act.fltr_act == ICE_FWD_TO_VSI || + rinfo->sw_act.fltr_act == ICE_NOP) rinfo->sw_act.fwd_id.hw_vsi_id = ice_get_hw_vsi_num(hw, vsi_handle); @@ -6159,6 +6225,11 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, act |= ICE_SINGLE_ACT_VSI_FORWARDING | ICE_SINGLE_ACT_DROP | ICE_SINGLE_ACT_VALID_BIT; break; + case ICE_NOP: + act |= FIELD_PREP(ICE_SINGLE_ACT_VSI_ID_M, + rinfo->sw_act.fwd_id.hw_vsi_id); + act &= ~ICE_SINGLE_ACT_VALID_BIT; + break; default: status = -EIO; goto err_ice_add_adv_rule; @@ -6439,7 +6510,7 @@ ice_rem_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, return -EIO; } - rid = ice_find_recp(hw, &lkup_exts, rinfo->tun_type); + rid = ice_find_recp(hw, &lkup_exts, rinfo); /* If did not find a recipe that match the existing criteria */ if (rid == ICE_MAX_NUM_RECIPES) return -EINVAL; @@ -6533,59 +6604,6 @@ ice_rem_adv_rule_by_id(struct ice_hw *hw, } /** - * ice_rem_adv_rule_for_vsi - removes existing advanced switch rules for a - * given VSI handle - * @hw: pointer to the hardware structure - * @vsi_handle: VSI handle for which we are supposed to remove all the rules. - * - * This function is used to remove all the rules for a given VSI and as soon - * as removing a rule fails, it will return immediately with the error code, - * else it will return success. - */ -int ice_rem_adv_rule_for_vsi(struct ice_hw *hw, u16 vsi_handle) -{ - struct ice_adv_fltr_mgmt_list_entry *list_itr, *tmp_entry; - struct ice_vsi_list_map_info *map_info; - struct ice_adv_rule_info rinfo; - struct list_head *list_head; - struct ice_switch_info *sw; - int status; - u8 rid; - - sw = hw->switch_info; - for (rid = 0; rid < ICE_MAX_NUM_RECIPES; rid++) { - if (!sw->recp_list[rid].recp_created) - continue; - if (!sw->recp_list[rid].adv_rule) - continue; - - list_head = &sw->recp_list[rid].filt_rules; - list_for_each_entry_safe(list_itr, tmp_entry, list_head, - list_entry) { - rinfo = list_itr->rule_info; - - if (rinfo.sw_act.fltr_act == ICE_FWD_TO_VSI_LIST) { - map_info = list_itr->vsi_list_info; - if (!map_info) - continue; - - if (!test_bit(vsi_handle, map_info->vsi_map)) - continue; - } else if (rinfo.sw_act.vsi_handle != vsi_handle) { - continue; - } - - rinfo.sw_act.vsi_handle = vsi_handle; - status = ice_rem_adv_rule(hw, list_itr->lkups, - list_itr->lkups_cnt, &rinfo); - if (status) - return status; - } - } - return 0; -} - -/** * ice_replay_vsi_adv_rule - Replay advanced rule for requested VSI * @hw: pointer to the hardware structure * @vsi_handle: driver VSI handle diff --git a/drivers/net/ethernet/intel/ice/ice_switch.h b/drivers/net/ethernet/intel/ice/ice_switch.h index c84b56fe84a5..250823ac173a 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.h +++ b/drivers/net/ethernet/intel/ice/ice_switch.h @@ -22,6 +22,16 @@ #define ICE_PROFID_IPV6_GTPU_TEID 46 #define ICE_PROFID_IPV6_GTPU_IPV6_TCP_INNER 70 +#define ICE_SW_RULE_VSI_LIST_SIZE(s, n) struct_size((s), vsi, (n)) +#define ICE_SW_RULE_RX_TX_HDR_SIZE(s, l) struct_size((s), hdr_data, (l)) +#define ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s) \ + ICE_SW_RULE_RX_TX_HDR_SIZE((s), DUMMY_ETH_HDR_LEN) +#define ICE_SW_RULE_RX_TX_NO_HDR_SIZE(s) \ + ICE_SW_RULE_RX_TX_HDR_SIZE((s), 0) +#define ICE_SW_RULE_LG_ACT_SIZE(s, n) struct_size((s), act, (n)) + +#define DUMMY_ETH_HDR_LEN 16 + /* VSI context structure for add/get/update/free operations */ struct ice_vsi_ctx { u16 vsi_num; @@ -191,6 +201,8 @@ struct ice_adv_rule_info { u16 vlan_type; u16 fltr_rule_id; u32 priority; + u16 need_pass_l2:1; + u16 allow_pass_l2:1; u16 src_vsi; struct ice_sw_act_ctrl sw_act; struct ice_adv_rule_flags_info flags_info; @@ -254,6 +266,9 @@ struct ice_sw_recipe { */ u8 priority; + u8 need_pass_l2:1; + u8 allow_pass_l2:1; + struct list_head rg_list; /* AQ buffer associated with this recipe */ @@ -340,6 +355,7 @@ ice_alloc_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, int ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, u16 counter_id); +int ice_share_res(struct ice_hw *hw, u16 type, u8 shared, u16 res_id); /* Switch/bridge related commands */ void ice_rule_add_tunnel_metadata(struct ice_adv_lkup_elem *lkup); @@ -379,7 +395,6 @@ int ice_set_vlan_vsi_promisc(struct ice_hw *hw, u16 vsi_handle, u8 promisc_mask, bool rm_vlan_promisc); -int ice_rem_adv_rule_for_vsi(struct ice_hw *hw, u16 vsi_handle); int ice_rem_adv_rule_by_id(struct ice_hw *hw, struct ice_rule_query_data *remove_entry); @@ -389,6 +404,7 @@ u16 ice_get_hw_vsi_num(struct ice_hw *hw, u16 vsi_handle); int ice_replay_vsi_all_fltr(struct ice_hw *hw, u16 vsi_handle); void ice_rm_all_sw_replay_rule_info(struct ice_hw *hw); +void ice_fill_eth_hdr(u8 *eth_hdr); int ice_aq_sw_rules(struct ice_hw *hw, void *rule_list, u16 rule_list_sz, @@ -397,4 +413,21 @@ int ice_update_recipe_lkup_idx(struct ice_hw *hw, struct ice_update_recipe_lkup_idx_params *params); void ice_change_proto_id_to_dvm(void); +struct ice_vsi_list_map_info * +ice_find_vsi_list_entry(struct ice_hw *hw, u8 recp_id, u16 vsi_handle, + u16 *vsi_list_id); +int ice_alloc_recipe(struct ice_hw *hw, u16 *rid); +int ice_aq_get_recipe(struct ice_hw *hw, + struct ice_aqc_recipe_data_elem *s_recipe_list, + u16 *num_recipes, u16 recipe_root, struct ice_sq_cd *cd); +int ice_aq_add_recipe(struct ice_hw *hw, + struct ice_aqc_recipe_data_elem *s_recipe_list, + u16 num_recipes, struct ice_sq_cd *cd); +int +ice_aq_get_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, + struct ice_sq_cd *cd); +int +ice_aq_map_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, + struct ice_sq_cd *cd); + #endif /* _ICE_SWITCH_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.c b/drivers/net/ethernet/intel/ice/ice_tc_lib.c index 4a34ef5f58d3..38547db1ec4e 100644 --- a/drivers/net/ethernet/intel/ice/ice_tc_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.c @@ -1343,24 +1343,24 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, dissector = rule->match.dissector; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_CVLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | - BIT(FLOW_DISSECTOR_KEY_IP) | - BIT(FLOW_DISSECTOR_KEY_ENC_IP) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_PPPOE) | - BIT(FLOW_DISSECTOR_KEY_L2TPV3))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PPPOE) | + BIT_ULL(FLOW_DISSECTOR_KEY_L2TPV3))) { NL_SET_ERR_MSG_MOD(fltr->extack, "Unsupported key used"); return -EOPNOTSUPP; } @@ -1382,10 +1382,10 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, */ headers = &fltr->inner_headers; } else if (dissector->used_keys & - (BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))) { + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) { NL_SET_ERR_MSG_MOD(fltr->extack, "Tunnel key used, but device isn't a tunnel"); return -EOPNOTSUPP; } else { diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h index ae98d5a8ff60..b2f5c9fe0149 100644 --- a/drivers/net/ethernet/intel/ice/ice_trace.h +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -21,6 +21,7 @@ #define _ICE_TRACE_H_ #include <linux/tracepoint.h> +#include "ice_eswitch_br.h" /* ice_trace() macro enables shared code to refer to trace points * like: @@ -240,6 +241,95 @@ DEFINE_TX_TSTAMP_OP_EVENT(ice_tx_tstamp_fw_req); DEFINE_TX_TSTAMP_OP_EVENT(ice_tx_tstamp_fw_done); DEFINE_TX_TSTAMP_OP_EVENT(ice_tx_tstamp_complete); +DECLARE_EVENT_CLASS(ice_esw_br_fdb_template, + TP_PROTO(struct ice_esw_br_fdb_entry *fdb), + TP_ARGS(fdb), + TP_STRUCT__entry(__array(char, dev_name, IFNAMSIZ) + __array(unsigned char, addr, ETH_ALEN) + __field(u16, vid) + __field(int, flags)), + TP_fast_assign(strscpy(__entry->dev_name, + netdev_name(fdb->dev), + IFNAMSIZ); + memcpy(__entry->addr, fdb->data.addr, ETH_ALEN); + __entry->vid = fdb->data.vid; + __entry->flags = fdb->flags;), + TP_printk("net_device=%s addr=%pM vid=%u flags=%x", + __entry->dev_name, + __entry->addr, + __entry->vid, + __entry->flags) +); + +DEFINE_EVENT(ice_esw_br_fdb_template, + ice_eswitch_br_fdb_entry_create, + TP_PROTO(struct ice_esw_br_fdb_entry *fdb), + TP_ARGS(fdb) +); + +DEFINE_EVENT(ice_esw_br_fdb_template, + ice_eswitch_br_fdb_entry_find_and_delete, + TP_PROTO(struct ice_esw_br_fdb_entry *fdb), + TP_ARGS(fdb) +); + +DECLARE_EVENT_CLASS(ice_esw_br_vlan_template, + TP_PROTO(struct ice_esw_br_vlan *vlan), + TP_ARGS(vlan), + TP_STRUCT__entry(__field(u16, vid) + __field(u16, flags)), + TP_fast_assign(__entry->vid = vlan->vid; + __entry->flags = vlan->flags;), + TP_printk("vid=%u flags=%x", + __entry->vid, + __entry->flags) +); + +DEFINE_EVENT(ice_esw_br_vlan_template, + ice_eswitch_br_vlan_create, + TP_PROTO(struct ice_esw_br_vlan *vlan), + TP_ARGS(vlan) +); + +DEFINE_EVENT(ice_esw_br_vlan_template, + ice_eswitch_br_vlan_cleanup, + TP_PROTO(struct ice_esw_br_vlan *vlan), + TP_ARGS(vlan) +); + +#define ICE_ESW_BR_PORT_NAME_L 16 + +DECLARE_EVENT_CLASS(ice_esw_br_port_template, + TP_PROTO(struct ice_esw_br_port *port), + TP_ARGS(port), + TP_STRUCT__entry(__field(u16, vport_num) + __array(char, port_type, ICE_ESW_BR_PORT_NAME_L)), + TP_fast_assign(__entry->vport_num = port->vsi_idx; + if (port->type == ICE_ESWITCH_BR_UPLINK_PORT) + strscpy(__entry->port_type, + "Uplink", + ICE_ESW_BR_PORT_NAME_L); + else + strscpy(__entry->port_type, + "VF Representor", + ICE_ESW_BR_PORT_NAME_L);), + TP_printk("vport_num=%u port type=%s", + __entry->vport_num, + __entry->port_type) +); + +DEFINE_EVENT(ice_esw_br_port_template, + ice_eswitch_br_port_link, + TP_PROTO(struct ice_esw_br_port *port), + TP_ARGS(port) +); + +DEFINE_EVENT(ice_esw_br_port_template, + ice_eswitch_br_port_unlink, + TP_PROTO(struct ice_esw_br_port *port), + TP_ARGS(port) +); + /* End tracepoints */ #endif /* _ICE_TRACE_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index a09556e57803..e82f38c2a940 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -277,6 +277,8 @@ struct ice_hw_common_caps { u8 dcb; u8 ieee_1588; u8 rdma; + u8 roce_lag; + u8 sriov_lag; bool nvm_update_pending_nvm; bool nvm_update_pending_orom; @@ -1033,6 +1035,7 @@ enum ice_sw_fwd_act_type { ICE_FWD_TO_Q, ICE_FWD_TO_QGRP, ICE_DROP_PACKET, + ICE_NOP, ICE_INVAL_ACT }; diff --git a/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.c b/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.c index b1ffb81893d4..d7b10dc67f03 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.c @@ -21,6 +21,99 @@ noop_vlan(struct ice_vsi __always_unused *vsi) return 0; } +static void ice_port_vlan_on(struct ice_vsi *vsi) +{ + struct ice_vsi_vlan_ops *vlan_ops; + struct ice_pf *pf = vsi->back; + + if (ice_is_dvm_ena(&pf->hw)) { + vlan_ops = &vsi->outer_vlan_ops; + + /* setup outer VLAN ops */ + vlan_ops->set_port_vlan = ice_vsi_set_outer_port_vlan; + vlan_ops->clear_port_vlan = ice_vsi_clear_outer_port_vlan; + vlan_ops->clear_port_vlan = ice_vsi_clear_outer_port_vlan; + + /* setup inner VLAN ops */ + vlan_ops = &vsi->inner_vlan_ops; + vlan_ops->add_vlan = noop_vlan_arg; + vlan_ops->del_vlan = noop_vlan_arg; + vlan_ops->ena_stripping = ice_vsi_ena_inner_stripping; + vlan_ops->dis_stripping = ice_vsi_dis_inner_stripping; + vlan_ops->ena_insertion = ice_vsi_ena_inner_insertion; + vlan_ops->dis_insertion = ice_vsi_dis_inner_insertion; + } else { + vlan_ops = &vsi->inner_vlan_ops; + + vlan_ops->set_port_vlan = ice_vsi_set_inner_port_vlan; + vlan_ops->clear_port_vlan = ice_vsi_clear_inner_port_vlan; + vlan_ops->clear_port_vlan = ice_vsi_clear_inner_port_vlan; + } + vlan_ops->ena_rx_filtering = ice_vsi_ena_rx_vlan_filtering; +} + +static void ice_port_vlan_off(struct ice_vsi *vsi) +{ + struct ice_vsi_vlan_ops *vlan_ops; + struct ice_pf *pf = vsi->back; + + /* setup inner VLAN ops */ + vlan_ops = &vsi->inner_vlan_ops; + + vlan_ops->ena_stripping = ice_vsi_ena_inner_stripping; + vlan_ops->dis_stripping = ice_vsi_dis_inner_stripping; + vlan_ops->ena_insertion = ice_vsi_ena_inner_insertion; + vlan_ops->dis_insertion = ice_vsi_dis_inner_insertion; + + if (ice_is_dvm_ena(&pf->hw)) { + vlan_ops = &vsi->outer_vlan_ops; + + vlan_ops->del_vlan = ice_vsi_del_vlan; + vlan_ops->ena_stripping = ice_vsi_ena_outer_stripping; + vlan_ops->dis_stripping = ice_vsi_dis_outer_stripping; + vlan_ops->ena_insertion = ice_vsi_ena_outer_insertion; + vlan_ops->dis_insertion = ice_vsi_dis_outer_insertion; + } else { + vlan_ops->del_vlan = ice_vsi_del_vlan; + } + + if (!test_bit(ICE_FLAG_VF_VLAN_PRUNING, pf->flags)) + vlan_ops->ena_rx_filtering = noop_vlan; + else + vlan_ops->ena_rx_filtering = + ice_vsi_ena_rx_vlan_filtering; +} + +/** + * ice_vf_vsi_enable_port_vlan - Set VSI VLAN ops to support port VLAN + * @vsi: VF's VSI being configured + * + * The function won't create port VLAN, it only allows to create port VLAN + * using VLAN ops on the VF VSI. + */ +void ice_vf_vsi_enable_port_vlan(struct ice_vsi *vsi) +{ + if (WARN_ON_ONCE(!vsi->vf)) + return; + + ice_port_vlan_on(vsi); +} + +/** + * ice_vf_vsi_disable_port_vlan - Clear VSI support for creating port VLAN + * @vsi: VF's VSI being configured + * + * The function should be called after removing port VLAN on VSI + * (using VLAN ops) + */ +void ice_vf_vsi_disable_port_vlan(struct ice_vsi *vsi) +{ + if (WARN_ON_ONCE(!vsi->vf)) + return; + + ice_port_vlan_off(vsi); +} + /** * ice_vf_vsi_init_vlan_ops - Initialize default VSI VLAN ops for VF VSI * @vsi: VF's VSI being configured @@ -39,91 +132,18 @@ void ice_vf_vsi_init_vlan_ops(struct ice_vsi *vsi) if (WARN_ON(!vf)) return; - if (ice_is_dvm_ena(&pf->hw)) { - vlan_ops = &vsi->outer_vlan_ops; + if (ice_vf_is_port_vlan_ena(vf)) + ice_port_vlan_on(vsi); + else + ice_port_vlan_off(vsi); - /* outer VLAN ops regardless of port VLAN config */ - vlan_ops->add_vlan = ice_vsi_add_vlan; - vlan_ops->ena_tx_filtering = ice_vsi_ena_tx_vlan_filtering; - vlan_ops->dis_tx_filtering = ice_vsi_dis_tx_vlan_filtering; - - if (ice_vf_is_port_vlan_ena(vf)) { - /* setup outer VLAN ops */ - vlan_ops->set_port_vlan = ice_vsi_set_outer_port_vlan; - /* all Rx traffic should be in the domain of the - * assigned port VLAN, so prevent disabling Rx VLAN - * filtering - */ - vlan_ops->dis_rx_filtering = noop_vlan; - vlan_ops->ena_rx_filtering = - ice_vsi_ena_rx_vlan_filtering; - - /* setup inner VLAN ops */ - vlan_ops = &vsi->inner_vlan_ops; - vlan_ops->add_vlan = noop_vlan_arg; - vlan_ops->del_vlan = noop_vlan_arg; - vlan_ops->ena_stripping = ice_vsi_ena_inner_stripping; - vlan_ops->dis_stripping = ice_vsi_dis_inner_stripping; - vlan_ops->ena_insertion = ice_vsi_ena_inner_insertion; - vlan_ops->dis_insertion = ice_vsi_dis_inner_insertion; - } else { - vlan_ops->dis_rx_filtering = - ice_vsi_dis_rx_vlan_filtering; - - if (!test_bit(ICE_FLAG_VF_VLAN_PRUNING, pf->flags)) - vlan_ops->ena_rx_filtering = noop_vlan; - else - vlan_ops->ena_rx_filtering = - ice_vsi_ena_rx_vlan_filtering; - - vlan_ops->del_vlan = ice_vsi_del_vlan; - vlan_ops->ena_stripping = ice_vsi_ena_outer_stripping; - vlan_ops->dis_stripping = ice_vsi_dis_outer_stripping; - vlan_ops->ena_insertion = ice_vsi_ena_outer_insertion; - vlan_ops->dis_insertion = ice_vsi_dis_outer_insertion; - - /* setup inner VLAN ops */ - vlan_ops = &vsi->inner_vlan_ops; - - vlan_ops->ena_stripping = ice_vsi_ena_inner_stripping; - vlan_ops->dis_stripping = ice_vsi_dis_inner_stripping; - vlan_ops->ena_insertion = ice_vsi_ena_inner_insertion; - vlan_ops->dis_insertion = ice_vsi_dis_inner_insertion; - } - } else { - vlan_ops = &vsi->inner_vlan_ops; + vlan_ops = ice_is_dvm_ena(&pf->hw) ? + &vsi->outer_vlan_ops : &vsi->inner_vlan_ops; - /* inner VLAN ops regardless of port VLAN config */ - vlan_ops->add_vlan = ice_vsi_add_vlan; - vlan_ops->dis_rx_filtering = ice_vsi_dis_rx_vlan_filtering; - vlan_ops->ena_tx_filtering = ice_vsi_ena_tx_vlan_filtering; - vlan_ops->dis_tx_filtering = ice_vsi_dis_tx_vlan_filtering; - - if (ice_vf_is_port_vlan_ena(vf)) { - vlan_ops->set_port_vlan = ice_vsi_set_inner_port_vlan; - vlan_ops->ena_rx_filtering = - ice_vsi_ena_rx_vlan_filtering; - /* all Rx traffic should be in the domain of the - * assigned port VLAN, so prevent disabling Rx VLAN - * filtering - */ - vlan_ops->dis_rx_filtering = noop_vlan; - } else { - vlan_ops->dis_rx_filtering = - ice_vsi_dis_rx_vlan_filtering; - if (!test_bit(ICE_FLAG_VF_VLAN_PRUNING, pf->flags)) - vlan_ops->ena_rx_filtering = noop_vlan; - else - vlan_ops->ena_rx_filtering = - ice_vsi_ena_rx_vlan_filtering; - - vlan_ops->del_vlan = ice_vsi_del_vlan; - vlan_ops->ena_stripping = ice_vsi_ena_inner_stripping; - vlan_ops->dis_stripping = ice_vsi_dis_inner_stripping; - vlan_ops->ena_insertion = ice_vsi_ena_inner_insertion; - vlan_ops->dis_insertion = ice_vsi_dis_inner_insertion; - } - } + vlan_ops->add_vlan = ice_vsi_add_vlan; + vlan_ops->dis_rx_filtering = ice_vsi_dis_rx_vlan_filtering; + vlan_ops->ena_tx_filtering = ice_vsi_ena_tx_vlan_filtering; + vlan_ops->dis_tx_filtering = ice_vsi_dis_tx_vlan_filtering; } /** diff --git a/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.h b/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.h index 875a4e615f39..df8aa09df3e3 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_vsi_vlan_ops.h @@ -13,7 +13,11 @@ void ice_vf_vsi_cfg_svm_legacy_vlan_mode(struct ice_vsi *vsi); #ifdef CONFIG_PCI_IOV void ice_vf_vsi_init_vlan_ops(struct ice_vsi *vsi); +void ice_vf_vsi_enable_port_vlan(struct ice_vsi *vsi); +void ice_vf_vsi_disable_port_vlan(struct ice_vsi *vsi); #else static inline void ice_vf_vsi_init_vlan_ops(struct ice_vsi *vsi) { } +static inline void ice_vf_vsi_enable_port_vlan(struct ice_vsi *vsi) { } +static inline void ice_vf_vsi_disable_port_vlan(struct ice_vsi *vsi) { } #endif /* CONFIG_PCI_IOV */ #endif /* _ICE_PF_VSI_VLAN_OPS_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index efbc2968a7bf..625da88e7965 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -1724,6 +1724,8 @@ error_param: vf->vf_id, i); } + ice_lag_move_new_vf_nodes(vf); + /* send the response to the VF */ return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, VIRTCHNL_STATUS_ERR_PARAM, NULL, 0); diff --git a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.c b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.c index 5b4a0abb4607..76266e709a39 100644 --- a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.c @@ -202,6 +202,24 @@ int ice_vsi_dis_inner_insertion(struct ice_vsi *vsi) return ice_vsi_manage_vlan_insertion(vsi); } +static void +ice_save_vlan_info(struct ice_aqc_vsi_props *info, + struct ice_vsi_vlan_info *vlan) +{ + vlan->sw_flags2 = info->sw_flags2; + vlan->inner_vlan_flags = info->inner_vlan_flags; + vlan->outer_vlan_flags = info->outer_vlan_flags; +} + +static void +ice_restore_vlan_info(struct ice_aqc_vsi_props *info, + struct ice_vsi_vlan_info *vlan) +{ + info->sw_flags2 = vlan->sw_flags2; + info->inner_vlan_flags = vlan->inner_vlan_flags; + info->outer_vlan_flags = vlan->outer_vlan_flags; +} + /** * __ice_vsi_set_inner_port_vlan - set port VLAN VSI context settings to enable a port VLAN * @vsi: the VSI to update @@ -218,6 +236,7 @@ static int __ice_vsi_set_inner_port_vlan(struct ice_vsi *vsi, u16 pvid_info) if (!ctxt) return -ENOMEM; + ice_save_vlan_info(&vsi->info, &vsi->vlan_info); ctxt->info = vsi->info; info = &ctxt->info; info->inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ACCEPTUNTAGGED | @@ -259,6 +278,33 @@ int ice_vsi_set_inner_port_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan) return __ice_vsi_set_inner_port_vlan(vsi, port_vlan_info); } +int ice_vsi_clear_inner_port_vlan(struct ice_vsi *vsi) +{ + struct ice_hw *hw = &vsi->back->hw; + struct ice_aqc_vsi_props *info; + struct ice_vsi_ctx *ctxt; + int ret; + + ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); + if (!ctxt) + return -ENOMEM; + + ice_restore_vlan_info(&vsi->info, &vsi->vlan_info); + vsi->info.port_based_inner_vlan = 0; + ctxt->info = vsi->info; + info = &ctxt->info; + info->valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID | + ICE_AQ_VSI_PROP_SW_VALID); + + ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL); + if (ret) + dev_err(ice_hw_to_dev(hw), "update VSI for port VLAN failed, err %d aq_err %s\n", + ret, ice_aq_str(hw->adminq.sq_last_status)); + + kfree(ctxt); + return ret; +} + /** * ice_cfg_vlan_pruning - enable or disable VLAN pruning on the VSI * @vsi: VSI to enable or disable VLAN pruning on @@ -647,6 +693,7 @@ __ice_vsi_set_outer_port_vlan(struct ice_vsi *vsi, u16 vlan_info, u16 tpid) if (!ctxt) return -ENOMEM; + ice_save_vlan_info(&vsi->info, &vsi->vlan_info); ctxt->info = vsi->info; ctxt->info.sw_flags2 |= ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; @@ -689,9 +736,6 @@ __ice_vsi_set_outer_port_vlan(struct ice_vsi *vsi, u16 vlan_info, u16 tpid) * used if DVM is supported. Also, this function should never be called directly * as it should be part of ice_vsi_vlan_ops if it's needed. * - * This function does not support clearing the port VLAN as there is currently - * no use case for this. - * * Use the ice_vlan structure passed in to set this VSI in a port VLAN. */ int ice_vsi_set_outer_port_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan) @@ -705,3 +749,37 @@ int ice_vsi_set_outer_port_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan) return __ice_vsi_set_outer_port_vlan(vsi, port_vlan_info, vlan->tpid); } + +/** + * ice_vsi_clear_outer_port_vlan - clear outer port vlan + * @vsi: VSI to configure + * + * The function is restoring previously set vlan config (saved in + * vsi->vlan_info). Setting happens in port vlan configuration. + */ +int ice_vsi_clear_outer_port_vlan(struct ice_vsi *vsi) +{ + struct ice_hw *hw = &vsi->back->hw; + struct ice_vsi_ctx *ctxt; + int err; + + ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); + if (!ctxt) + return -ENOMEM; + + ice_restore_vlan_info(&vsi->info, &vsi->vlan_info); + vsi->info.port_based_outer_vlan = 0; + ctxt->info = vsi->info; + + ctxt->info.valid_sections = + cpu_to_le16(ICE_AQ_VSI_PROP_OUTER_TAG_VALID | + ICE_AQ_VSI_PROP_SW_VALID); + + err = ice_update_vsi(hw, vsi->idx, ctxt, NULL); + if (err) + dev_err(ice_pf_to_dev(vsi->back), "update VSI for clearing outer port based VLAN failed, err %d aq_err %s\n", + err, ice_aq_str(hw->adminq.sq_last_status)); + + kfree(ctxt); + return err; +} diff --git a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.h b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.h index f459909490ec..f0d84d11bd5b 100644 --- a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_lib.h @@ -7,6 +7,12 @@ #include <linux/types.h> #include "ice_vlan.h" +struct ice_vsi_vlan_info { + u8 sw_flags2; + u8 inner_vlan_flags; + u8 outer_vlan_flags; +}; + struct ice_vsi; int ice_vsi_add_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan); @@ -17,6 +23,7 @@ int ice_vsi_dis_inner_stripping(struct ice_vsi *vsi); int ice_vsi_ena_inner_insertion(struct ice_vsi *vsi, u16 tpid); int ice_vsi_dis_inner_insertion(struct ice_vsi *vsi); int ice_vsi_set_inner_port_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan); +int ice_vsi_clear_inner_port_vlan(struct ice_vsi *vsi); int ice_vsi_ena_rx_vlan_filtering(struct ice_vsi *vsi); int ice_vsi_dis_rx_vlan_filtering(struct ice_vsi *vsi); @@ -28,5 +35,6 @@ int ice_vsi_dis_outer_stripping(struct ice_vsi *vsi); int ice_vsi_ena_outer_insertion(struct ice_vsi *vsi, u16 tpid); int ice_vsi_dis_outer_insertion(struct ice_vsi *vsi); int ice_vsi_set_outer_port_vlan(struct ice_vsi *vsi, struct ice_vlan *vlan); +int ice_vsi_clear_outer_port_vlan(struct ice_vsi *vsi); #endif /* _ICE_VSI_VLAN_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.h b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.h index 5b47568f6256..b2d2330dedcb 100644 --- a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.h +++ b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.h @@ -21,6 +21,7 @@ struct ice_vsi_vlan_ops { int (*ena_tx_filtering)(struct ice_vsi *vsi); int (*dis_tx_filtering)(struct ice_vsi *vsi); int (*set_port_vlan)(struct ice_vsi *vsi, struct ice_vlan *vlan); + int (*clear_port_vlan)(struct ice_vsi *vsi); }; void ice_vsi_init_vlan_ops(struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index a7fe2b4ce655..2a3f0834e139 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -546,19 +546,6 @@ bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) } /** - * ice_bump_ntc - Bump the next_to_clean counter of an Rx ring - * @rx_ring: Rx ring - */ -static void ice_bump_ntc(struct ice_rx_ring *rx_ring) -{ - int ntc = rx_ring->next_to_clean + 1; - - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - prefetch(ICE_RX_DESC(rx_ring, ntc)); -} - -/** * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer * @rx_ring: Rx ring * @xdp: Pointer to XDP buffer @@ -572,8 +559,14 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) { unsigned int totalsize = xdp->data_end - xdp->data_meta; unsigned int metasize = xdp->data - xdp->data_meta; + struct skb_shared_info *sinfo = NULL; struct sk_buff *skb; + u32 nr_frags = 0; + if (unlikely(xdp_buff_has_frags(xdp))) { + sinfo = xdp_get_shared_info_from_buff(xdp); + nr_frags = sinfo->nr_frags; + } net_prefetch(xdp->data_meta); skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize, @@ -589,6 +582,29 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) __skb_pull(skb, metasize); } + if (likely(!xdp_buff_has_frags(xdp))) + goto out; + + for (int i = 0; i < nr_frags; i++) { + struct skb_shared_info *skinfo = skb_shinfo(skb); + skb_frag_t *frag = &sinfo->frags[i]; + struct page *page; + void *addr; + + page = dev_alloc_page(); + if (!page) { + dev_kfree_skb(skb); + return NULL; + } + addr = page_to_virt(page); + + memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); + + __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++, + addr, 0, skb_frag_size(frag)); + } + +out: xsk_buff_free(xdp); return skb; } @@ -597,7 +613,7 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ * @xdp_ring: XDP Tx ring */ -static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) +static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) { u16 ntc = xdp_ring->next_to_clean; struct ice_tx_desc *tx_desc; @@ -619,7 +635,7 @@ static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) } if (!completed_frames) - return; + return 0; if (likely(!xdp_ring->xdp_tx_active)) { xsk_frames = completed_frames; @@ -649,6 +665,8 @@ skip: xdp_ring->next_to_clean -= cnt; if (xsk_frames) xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); + + return completed_frames; } /** @@ -666,37 +684,72 @@ skip: static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring) { + struct skb_shared_info *sinfo = NULL; u32 size = xdp->data_end - xdp->data; u32 ntu = xdp_ring->next_to_use; struct ice_tx_desc *tx_desc; struct ice_tx_buf *tx_buf; - dma_addr_t dma; - - if (ICE_DESC_UNUSED(xdp_ring) < ICE_RING_QUARTER(xdp_ring)) { - ice_clean_xdp_irq_zc(xdp_ring); - if (!ICE_DESC_UNUSED(xdp_ring)) { - xdp_ring->ring_stats->tx_stats.tx_busy++; - return ICE_XDP_CONSUMED; - } + struct xdp_buff *head; + u32 nr_frags = 0; + u32 free_space; + u32 frag = 0; + + free_space = ICE_DESC_UNUSED(xdp_ring); + if (free_space < ICE_RING_QUARTER(xdp_ring)) + free_space += ice_clean_xdp_irq_zc(xdp_ring); + + if (unlikely(!free_space)) + goto busy; + + if (unlikely(xdp_buff_has_frags(xdp))) { + sinfo = xdp_get_shared_info_from_buff(xdp); + nr_frags = sinfo->nr_frags; + if (free_space < nr_frags + 1) + goto busy; } - dma = xsk_buff_xdp_get_dma(xdp); - xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size); - - tx_buf = &xdp_ring->tx_buf[ntu]; - tx_buf->xdp = xdp; - tx_buf->type = ICE_TX_BUF_XSK_TX; tx_desc = ICE_TX_DESC(xdp_ring, ntu); - tx_desc->buf_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP, - 0, size, 0); - xdp_ring->xdp_tx_active++; + tx_buf = &xdp_ring->tx_buf[ntu]; + head = xdp; + + for (;;) { + dma_addr_t dma; + + dma = xsk_buff_xdp_get_dma(xdp); + xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size); + + tx_buf->xdp = xdp; + tx_buf->type = ICE_TX_BUF_XSK_TX; + tx_desc->buf_addr = cpu_to_le64(dma); + tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0); + /* account for each xdp_buff from xsk_buff_pool */ + xdp_ring->xdp_tx_active++; + + if (++ntu == xdp_ring->count) + ntu = 0; + + if (frag == nr_frags) + break; + + tx_desc = ICE_TX_DESC(xdp_ring, ntu); + tx_buf = &xdp_ring->tx_buf[ntu]; + + xdp = xsk_buff_get_frag(head); + size = skb_frag_size(&sinfo->frags[frag]); + frag++; + } - if (++ntu == xdp_ring->count) - ntu = 0; xdp_ring->next_to_use = ntu; + /* update last descriptor from a frame with EOP */ + tx_desc->cmd_type_offset_bsz |= + cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S); return ICE_XDP_TX; + +busy: + xdp_ring->ring_stats->tx_stats.tx_busy++; + + return ICE_XDP_CONSUMED; } /** @@ -752,6 +805,34 @@ out_failure: return result; } +static int +ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first, + struct xdp_buff *xdp, const unsigned int size) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); + + if (!size) + return 0; + + if (!xdp_buff_has_frags(first)) { + sinfo->nr_frags = 0; + sinfo->xdp_frags_size = 0; + xdp_buff_set_frags_flag(first); + } + + if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { + xsk_buff_free(first); + return -ENOMEM; + } + + __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, + virt_to_page(xdp->data_hard_start), 0, size); + sinfo->xdp_frags_size += size; + xsk_buff_add_frag(xdp); + + return 0; +} + /** * ice_clean_rx_irq_zc - consumes packets from the hardware ring * @rx_ring: AF_XDP Rx ring @@ -762,9 +843,14 @@ out_failure: int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; + struct xsk_buff_pool *xsk_pool = rx_ring->xsk_pool; + u32 ntc = rx_ring->next_to_clean; + u32 ntu = rx_ring->next_to_use; + struct xdp_buff *first = NULL; struct ice_tx_ring *xdp_ring; unsigned int xdp_xmit = 0; struct bpf_prog *xdp_prog; + u32 cnt = rx_ring->count; bool failure = false; int entries_to_alloc; @@ -774,6 +860,9 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) xdp_prog = READ_ONCE(rx_ring->xdp_prog); xdp_ring = rx_ring->xdp_ring; + if (ntc != rx_ring->first_desc) + first = *ice_xdp_buf(rx_ring, rx_ring->first_desc); + while (likely(total_rx_packets < (unsigned int)budget)) { union ice_32b_rx_flex_desc *rx_desc; unsigned int size, xdp_res = 0; @@ -783,7 +872,7 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) u16 vlan_tag = 0; u16 rx_ptype; - rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); + rx_desc = ICE_RX_DESC(rx_ring, ntc); stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits)) @@ -795,51 +884,61 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) */ dma_rmb(); - if (unlikely(rx_ring->next_to_clean == rx_ring->next_to_use)) + if (unlikely(ntc == ntu)) break; - xdp = *ice_xdp_buf(rx_ring, rx_ring->next_to_clean); + xdp = *ice_xdp_buf(rx_ring, ntc); size = le16_to_cpu(rx_desc->wb.pkt_len) & ICE_RX_FLX_DESC_PKT_LEN_M; - if (!size) { - xdp->data = NULL; - xdp->data_end = NULL; - xdp->data_hard_start = NULL; - xdp->data_meta = NULL; - goto construct_skb; - } xsk_buff_set_size(xdp, size); - xsk_buff_dma_sync_for_cpu(xdp, rx_ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); + + if (!first) { + first = xdp; + xdp_buff_clear_frags_flag(first); + } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) { + break; + } + + if (++ntc == cnt) + ntc = 0; - xdp_res = ice_run_xdp_zc(rx_ring, xdp, xdp_prog, xdp_ring); + if (ice_is_non_eop(rx_ring, rx_desc)) + continue; + + xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring); if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) { xdp_xmit |= xdp_res; } else if (xdp_res == ICE_XDP_EXIT) { failure = true; + first = NULL; + rx_ring->first_desc = ntc; break; } else if (xdp_res == ICE_XDP_CONSUMED) { - xsk_buff_free(xdp); + xsk_buff_free(first); } else if (xdp_res == ICE_XDP_PASS) { goto construct_skb; } - total_rx_bytes += size; + total_rx_bytes += xdp_get_buff_len(first); total_rx_packets++; - ice_bump_ntc(rx_ring); + first = NULL; + rx_ring->first_desc = ntc; continue; construct_skb: /* XDP_PASS path */ - skb = ice_construct_skb_zc(rx_ring, xdp); + skb = ice_construct_skb_zc(rx_ring, first); if (!skb) { rx_ring->ring_stats->rx_stats.alloc_buf_failed++; break; } - ice_bump_ntc(rx_ring); + first = NULL; + rx_ring->first_desc = ntc; if (eth_skb_pad(skb)) { skb = NULL; @@ -858,18 +957,22 @@ construct_skb: ice_receive_skb(rx_ring, skb, vlan_tag); } - entries_to_alloc = ICE_DESC_UNUSED(rx_ring); + rx_ring->next_to_clean = ntc; + entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring); if (entries_to_alloc > ICE_RING_QUARTER(rx_ring)) failure |= !ice_alloc_rx_bufs_zc(rx_ring, entries_to_alloc); ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0); ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes); - if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) { - if (failure || rx_ring->next_to_clean == rx_ring->next_to_use) - xsk_set_rx_need_wakeup(rx_ring->xsk_pool); + if (xsk_uses_need_wakeup(xsk_pool)) { + /* ntu could have changed when allocating entries above, so + * use rx_ring value instead of stack based one + */ + if (failure || ntc == rx_ring->next_to_use) + xsk_set_rx_need_wakeup(xsk_pool); else - xsk_clear_rx_need_wakeup(rx_ring->xsk_pool); + xsk_clear_rx_need_wakeup(xsk_pool); return (int)total_rx_packets; } @@ -894,7 +997,7 @@ static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc, tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++); tx_desc->buf_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP, + tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(desc), 0, desc->len, 0); *total_bytes += desc->len; @@ -921,7 +1024,7 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de tx_desc = ICE_TX_DESC(xdp_ring, ntu++); tx_desc->buf_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP, + tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(&descs[i]), 0, descs[i].len, 0); *total_bytes += descs[i].len; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 9a2561409b06..9f63a10c6f80 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -2615,10 +2615,10 @@ static int igb_parse_cls_flower(struct igb_adapter *adapter, struct netlink_ext_ack *extack = f->common.extack; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported"); return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index bdeb36790d77..2a10254edbbd 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -6143,6 +6143,26 @@ static int igc_tsn_clear_schedule(struct igc_adapter *adapter) return 0; } +static void igc_taprio_stats(struct net_device *dev, + struct tc_taprio_qopt_stats *stats) +{ + /* When Strict_End is enabled, the tx_overruns counter + * will always be zero. + */ + stats->tx_overruns = 0; +} + +static void igc_taprio_queue_stats(struct net_device *dev, + struct tc_taprio_qopt_queue_stats *queue_stats) +{ + struct tc_taprio_qopt_stats *stats = &queue_stats->stats; + + /* When Strict_End is enabled, the tx_overruns counter + * will always be zero. + */ + stats->tx_overruns = 0; +} + static int igc_save_qbv_schedule(struct igc_adapter *adapter, struct tc_taprio_qopt_offload *qopt) { @@ -6153,11 +6173,20 @@ static int igc_save_qbv_schedule(struct igc_adapter *adapter, size_t n; int i; - if (qopt->cmd == TAPRIO_CMD_DESTROY) + switch (qopt->cmd) { + case TAPRIO_CMD_REPLACE: + break; + case TAPRIO_CMD_DESTROY: return igc_tsn_clear_schedule(adapter); - - if (qopt->cmd != TAPRIO_CMD_REPLACE) + case TAPRIO_CMD_STATS: + igc_taprio_stats(adapter->netdev, &qopt->stats); + return 0; + case TAPRIO_CMD_QUEUE_STATS: + igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats); + return 0; + default: return -EOPNOTSUPP; + } if (qopt->base_time < 0) return -ERANGE; diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c index 8537578e1cf1..5f6ae11212ae 100644 --- a/drivers/net/ethernet/korina.c +++ b/drivers/net/ethernet/korina.c @@ -43,7 +43,7 @@ #include <linux/ioport.h> #include <linux/iopoll.h> #include <linux/in.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_net.h> #include <linux/slab.h> #include <linux/string.h> diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c index 8662543ca5c8..a1a80f13b1e8 100644 --- a/drivers/net/ethernet/marvell/mvmdio.c +++ b/drivers/net/ethernet/marvell/mvmdio.c @@ -24,8 +24,8 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/phy.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c index 75e83ea2a926..0f9bc4f8ec3b 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c @@ -593,8 +593,6 @@ static int mvpp2_dbgfs_c2_entry_init(struct dentry *parent, sprintf(c2_entry_name, "%03d", id); c2_entry_dir = debugfs_create_dir(c2_entry_name, parent); - if (!c2_entry_dir) - return -ENOMEM; entry = &priv->dbgfs_entries->c2_entries[id]; @@ -626,8 +624,6 @@ static int mvpp2_dbgfs_flow_tbl_entry_init(struct dentry *parent, sprintf(flow_tbl_entry_name, "%03d", id); flow_tbl_entry_dir = debugfs_create_dir(flow_tbl_entry_name, parent); - if (!flow_tbl_entry_dir) - return -ENOMEM; entry = &priv->dbgfs_entries->flt_entries[id]; @@ -646,12 +642,8 @@ static int mvpp2_dbgfs_cls_init(struct dentry *parent, struct mvpp2 *priv) int i, ret; cls_dir = debugfs_create_dir("classifier", parent); - if (!cls_dir) - return -ENOMEM; c2_dir = debugfs_create_dir("c2", cls_dir); - if (!c2_dir) - return -ENOMEM; for (i = 0; i < MVPP22_CLS_C2_N_ENTRIES; i++) { ret = mvpp2_dbgfs_c2_entry_init(c2_dir, priv, i); @@ -660,8 +652,6 @@ static int mvpp2_dbgfs_cls_init(struct dentry *parent, struct mvpp2 *priv) } flow_tbl_dir = debugfs_create_dir("flow_table", cls_dir); - if (!flow_tbl_dir) - return -ENOMEM; for (i = 0; i < MVPP2_CLS_FLOWS_TBL_SIZE; i++) { ret = mvpp2_dbgfs_flow_tbl_entry_init(flow_tbl_dir, priv, i); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 1fec84b4c068..9e1b596c8f08 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -24,7 +24,6 @@ #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/phy.h> #include <linux/phylink.h> #include <linux/phy/phy.h> diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index eba307eee2b2..a8f3c8faf8af 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -235,7 +235,7 @@ M(NPC_GET_KEX_CFG, 0x600c, npc_get_kex_cfg, \ M(NPC_INSTALL_FLOW, 0x600d, npc_install_flow, \ npc_install_flow_req, npc_install_flow_rsp) \ M(NPC_DELETE_FLOW, 0x600e, npc_delete_flow, \ - npc_delete_flow_req, msg_rsp) \ + npc_delete_flow_req, npc_delete_flow_rsp) \ M(NPC_MCAM_READ_ENTRY, 0x600f, npc_mcam_read_entry, \ npc_mcam_read_entry_req, \ npc_mcam_read_entry_rsp) \ @@ -1451,6 +1451,10 @@ struct flow_msg { __be32 ip4dst; __be32 ip6dst[4]; }; + union { + __be32 spi; + }; + u8 tos; u8 ip_ver; u8 ip_proto; @@ -1491,6 +1495,8 @@ struct npc_install_flow_req { u8 vtag0_op; u16 vtag1_def; u8 vtag1_op; + /* old counter value */ + u16 cntr_val; }; struct npc_install_flow_rsp { @@ -1506,6 +1512,11 @@ struct npc_delete_flow_req { u8 all; /* PF + VFs */ }; +struct npc_delete_flow_rsp { + struct mbox_msghdr hdr; + u16 cntr_val; +}; + struct npc_mcam_read_entry_req { struct mbox_msghdr hdr; u16 entry; /* MCAM entry to read */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 9beeead56d7b..534f20c0c5df 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -204,6 +204,7 @@ enum key_fields { NPC_DPORT_UDP, NPC_SPORT_SCTP, NPC_DPORT_SCTP, + NPC_IPSEC_SPI, NPC_HEADER_FIELDS_MAX, NPC_CHAN = NPC_HEADER_FIELDS_MAX, /* Valid when Rx */ NPC_PF_FUNC, /* Valid when Tx */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c index 0ee420a489fc..c55c2c441a1a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c @@ -61,12 +61,12 @@ static const struct pci_device_id ptp_id_table[]; static bool is_ptp_dev_cnf10kb(struct ptp *ptp) { - return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false; + return ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP; } static bool is_ptp_dev_cn10k(struct ptp *ptp) { - return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false; + return ptp->pdev->device == PCI_DEVID_CN10K_PTP; } static bool cn10k_ptp_errata(struct ptp *ptp) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 3b26893efdf8..7cd33edca365 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -2827,6 +2827,10 @@ static void rvu_dbg_npc_mcam_show_flows(struct seq_file *s, seq_printf(s, "%d ", ntohs(rule->packet.dport)); seq_printf(s, "mask 0x%x\n", ntohs(rule->mask.dport)); break; + case NPC_IPSEC_SPI: + seq_printf(s, "0x%x ", ntohl(rule->packet.spi)); + seq_printf(s, "mask 0x%x\n", ntohl(rule->mask.spi)); + break; default: seq_puts(s, "\n"); break; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 952319453701..bb715e616fad 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -41,6 +41,7 @@ static const char * const npc_flow_names[] = { [NPC_SPORT_SCTP] = "sctp source port", [NPC_DPORT_SCTP] = "sctp destination port", [NPC_LXMB] = "Mcast/Bcast header ", + [NPC_IPSEC_SPI] = "SPI ", [NPC_UNKNOWN] = "unknown", }; @@ -513,6 +514,10 @@ do { \ NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); + + NPC_SCAN_HDR(NPC_IPSEC_SPI, NPC_LID_LD, NPC_LT_LD_AH, 4, 4); + NPC_SCAN_HDR(NPC_IPSEC_SPI, NPC_LID_LE, NPC_LT_LE_ESP, 0, 4); + /* SMAC follows the DMAC(which is 6 bytes) */ NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start + 6, 6); /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ @@ -564,6 +569,9 @@ static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) *features &= ~BIT_ULL(NPC_OUTER_VID); + if (*features & (BIT_ULL(NPC_IPPROTO_AH) | BIT_ULL(NPC_IPPROTO_ESP))) + *features |= BIT_ULL(NPC_IPSEC_SPI); + /* for vlan ethertypes corresponding layer type should be in the key */ if (npc_check_field(rvu, blkaddr, NPC_LB, intf)) *features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) | @@ -930,6 +938,9 @@ do { \ NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0, ntohs(mask->dport), 0); + NPC_WRITE_FLOW(NPC_IPSEC_SPI, spi, ntohl(pkt->spi), 0, + ntohl(mask->spi), 0); + NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0, ntohs(mask->vlan_tci), 0); @@ -1192,7 +1203,7 @@ find_rule: write_req.enable_entry = (u8)enable; /* if counter is available then clear and use it */ if (req->set_cntr && rule->has_cntr) { - rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00); + rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), req->cntr_val); write_req.set_cntr = 1; write_req.cntr = rule->cntr; } @@ -1407,12 +1418,13 @@ static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule, int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, struct npc_delete_flow_req *req, - struct msg_rsp *rsp) + struct npc_delete_flow_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_npc_mcam_rule *iter, *tmp; u16 pcifunc = req->hdr.pcifunc; struct list_head del_list; + int blkaddr; INIT_LIST_HEAD(&del_list); @@ -1428,6 +1440,10 @@ int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, list_move_tail(&iter->list, &del_list); /* single rule */ } else if (req->entry == iter->entry) { + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr) + rsp->cntr_val = rvu_read64(rvu, blkaddr, + NPC_AF_MATCH_STATX(iter->cntr)); list_move_tail(&iter->list, &del_list); break; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_switch.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_switch.c index 592b317f4637..854045ed3b06 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_switch.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_switch.c @@ -158,6 +158,7 @@ void rvu_switch_enable(struct rvu *rvu) struct npc_mcam_alloc_entry_req alloc_req = { 0 }; struct npc_mcam_alloc_entry_rsp alloc_rsp = { 0 }; struct npc_delete_flow_req uninstall_req = { 0 }; + struct npc_delete_flow_rsp uninstall_rsp = { 0 }; struct npc_mcam_free_entry_req free_req = { 0 }; struct rvu_switch *rswitch = &rvu->rswitch; struct msg_rsp rsp; @@ -197,7 +198,7 @@ void rvu_switch_enable(struct rvu *rvu) uninstall_rules: uninstall_req.start = rswitch->start_entry; uninstall_req.end = rswitch->start_entry + rswitch->used_entries - 1; - rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &rsp); + rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &uninstall_rsp); kfree(rswitch->entry2pcifunc); free_entries: free_req.all = 1; @@ -209,6 +210,7 @@ exit: void rvu_switch_disable(struct rvu *rvu) { struct npc_delete_flow_req uninstall_req = { 0 }; + struct npc_delete_flow_rsp uninstall_rsp = { 0 }; struct npc_mcam_free_entry_req free_req = { 0 }; struct rvu_switch *rswitch = &rvu->rswitch; struct rvu_hwinfo *hw = rvu->hw; @@ -250,7 +252,7 @@ void rvu_switch_disable(struct rvu *rvu) uninstall_req.start = rswitch->start_entry; uninstall_req.end = rswitch->start_entry + rswitch->used_entries - 1; free_req.all = 1; - rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &rsp); + rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &uninstall_rsp); rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp); rswitch->used_entries = 0; kfree(rswitch->entry2pcifunc); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 77c8f650f7ac..8cdd92dd9762 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -774,6 +774,7 @@ int otx2_txsch_alloc(struct otx2_nic *pfvf) rsp->schq_list[lvl][schq]; pfvf->hw.txschq_link_cfg_lvl = rsp->link_cfg_lvl; + pfvf->hw.txschq_aggr_lvl_rr_prio = rsp->aggr_lvl_rr_prio; return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index ba8091131ec0..25e99fd2e3fd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -224,6 +224,7 @@ struct otx2_hw { /* NIX */ u8 txschq_link_cfg_lvl; + u8 txschq_aggr_lvl_rr_prio; u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; u16 matchall_ipolicer; u32 dwrr_mtu; @@ -360,13 +361,8 @@ struct otx2_flow_config { struct list_head flow_list; u32 dmacflt_max_flows; u16 max_flows; -}; - -struct otx2_tc_info { - /* hash table to store TC offloaded flows */ - struct rhashtable flow_table; - struct rhashtable_params flow_ht_params; - unsigned long *tc_entries_bitmap; + struct list_head flow_list_tc; + bool ntuple; }; struct dev_hw_ops { @@ -491,7 +487,6 @@ struct otx2_nic { /* NPC MCAM */ struct otx2_flow_config *flow_cfg; struct otx2_mac_table *mac_table; - struct otx2_tc_info tc_info; u64 reset_count; struct work_struct reset_task; @@ -1063,7 +1058,6 @@ int otx2_init_tc(struct otx2_nic *nic); void otx2_shutdown_tc(struct otx2_nic *nic); int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, void *type_data); -int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic); /* CGX/RPM DMAC filters support */ int otx2_dmacflt_get_max_cnt(struct otx2_nic *pf); int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u32 bit_pos); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c index 63ef7c41d18d..4e1130496573 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c @@ -41,7 +41,6 @@ static int otx2_dl_mcam_count_set(struct devlink *devlink, u32 id, return 0; otx2_alloc_mcam_entries(pfvf, ctx->val.vu16); - otx2_tc_alloc_ent_bitmap(pfvf); return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index c47d91da32dc..9efcec549834 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -764,6 +764,7 @@ static int otx2_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *nfc) struct otx2_nic *pfvf = netdev_priv(dev); int ret = -EOPNOTSUPP; + pfvf->flow_cfg->ntuple = ntuple; switch (nfc->cmd) { case ETHTOOL_SRXFH: ret = otx2_set_rss_hash_opts(pfvf, nfc); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 2d7713a1a153..4762dbea64a1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -276,6 +276,7 @@ int otx2vf_mcam_flow_init(struct otx2_nic *pfvf) flow_cfg = pfvf->flow_cfg; INIT_LIST_HEAD(&flow_cfg->flow_list); + INIT_LIST_HEAD(&flow_cfg->flow_list_tc); flow_cfg->max_flows = 0; return 0; @@ -298,6 +299,7 @@ int otx2_mcam_flow_init(struct otx2_nic *pf) return -ENOMEM; INIT_LIST_HEAD(&pf->flow_cfg->flow_list); + INIT_LIST_HEAD(&pf->flow_cfg->flow_list_tc); /* Allocate bare minimum number of MCAM entries needed for * unicast and ntuple filters. diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 9551b422622a..61f62a6ec662 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -2027,7 +2027,7 @@ u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb, #endif int txq; - qos_enabled = (netdev->real_num_tx_queues > pf->hw.tx_queues) ? true : false; + qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues; if (unlikely(qos_enabled)) { /* This smp_load_acquire() pairs with smp_store_release() in * otx2_qos_root_add() called from htb offload root creation diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 5e56b6c3e60a..4bb511e3cbe5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -34,9 +34,8 @@ struct otx2_tc_flow_stats { }; struct otx2_tc_flow { - struct rhash_head node; + struct list_head list; unsigned long cookie; - unsigned int bitpos; struct rcu_head rcu; struct otx2_tc_flow_stats stats; spinlock_t lock; /* lock for stats */ @@ -44,31 +43,10 @@ struct otx2_tc_flow { u16 entry; u16 leaf_profile; bool is_act_police; + u32 prio; + struct npc_install_flow_req req; }; -int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) -{ - struct otx2_tc_info *tc = &nic->tc_info; - - if (!nic->flow_cfg->max_flows) - return 0; - - /* Max flows changed, free the existing bitmap */ - kfree(tc->tc_entries_bitmap); - - tc->tc_entries_bitmap = - kcalloc(BITS_TO_LONGS(nic->flow_cfg->max_flows), - sizeof(long), GFP_KERNEL); - if (!tc->tc_entries_bitmap) { - netdev_err(nic->netdev, - "Unable to alloc TC flow entries bitmap\n"); - return -ENOMEM; - } - - return 0; -} -EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap); - static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, u32 *burst_exp, u32 *burst_mantissa) { @@ -476,15 +454,16 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, dissector = rule->match.dissector; if ((dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_IP)))) { - netdev_info(nic->netdev, "unsupported flow used key 0x%x", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_IPSEC) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP)))) { + netdev_info(nic->netdev, "unsupported flow used key 0x%llx", dissector->used_keys); return -EOPNOTSUPP; } @@ -504,6 +483,8 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, match.key->ip_proto != IPPROTO_UDP && match.key->ip_proto != IPPROTO_SCTP && match.key->ip_proto != IPPROTO_ICMP && + match.key->ip_proto != IPPROTO_ESP && + match.key->ip_proto != IPPROTO_AH && match.key->ip_proto != IPPROTO_ICMPV6)) { netdev_info(nic->netdev, "ip_proto=0x%x not supported\n", @@ -523,6 +504,10 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, req->features |= BIT_ULL(NPC_IPPROTO_ICMP); else if (ip_proto == IPPROTO_ICMPV6) req->features |= BIT_ULL(NPC_IPPROTO_ICMP6); + else if (ip_proto == IPPROTO_ESP) + req->features |= BIT_ULL(NPC_IPPROTO_ESP); + else if (ip_proto == IPPROTO_AH) + req->features |= BIT_ULL(NPC_IPPROTO_AH); } if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { @@ -567,6 +552,26 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, } } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPSEC)) { + struct flow_match_ipsec match; + + flow_rule_match_ipsec(rule, &match); + if (!match.mask->spi) { + NL_SET_ERR_MSG_MOD(extack, "spi index not specified"); + return -EOPNOTSUPP; + } + if (ip_proto != IPPROTO_ESP && + ip_proto != IPPROTO_AH) { + NL_SET_ERR_MSG_MOD(extack, + "SPI index is valid only for ESP/AH proto"); + return -EOPNOTSUPP; + } + + flow_spec->spi = match.key->spi; + flow_mask->spi = match.mask->spi; + req->features |= BIT_ULL(NPC_IPSEC_SPI); + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { struct flow_match_ip match; @@ -707,8 +712,117 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, return otx2_tc_parse_actions(nic, &rule->action, req, f, node); } -static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry) +static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf) +{ + struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; + struct otx2_tc_flow *iter, *tmp; + + if (!(pfvf->flags & OTX2_FLAG_MCAM_ENTRIES_ALLOC)) + return; + + list_for_each_entry_safe(iter, tmp, &flow_cfg->flow_list_tc, list) { + list_del(&iter->list); + kfree(iter); + flow_cfg->nr_flows--; + } +} + +static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, + unsigned long cookie) +{ + struct otx2_tc_flow *tmp; + + list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { + if (tmp->cookie == cookie) + return tmp; + } + + return NULL; +} + +static struct otx2_tc_flow *otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, + int index) +{ + struct otx2_tc_flow *tmp; + int i = 0; + + list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { + if (i == index) + return tmp; + i++; + } + + return NULL; +} + +static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct list_head *pos, *n; + struct otx2_tc_flow *tmp; + + list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { + tmp = list_entry(pos, struct otx2_tc_flow, list); + if (node == tmp) { + list_del(&node->list); + return; + } + } +} + +static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct list_head *pos, *n; + struct otx2_tc_flow *tmp; + int index = 0; + + /* If the flow list is empty then add the new node */ + if (list_empty(&flow_cfg->flow_list_tc)) { + list_add(&node->list, &flow_cfg->flow_list_tc); + return index; + } + + list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { + tmp = list_entry(pos, struct otx2_tc_flow, list); + if (node->prio < tmp->prio) + break; + index++; + } + + list_add(&node->list, pos->prev); + return index; +} + +static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flow_req *req) +{ + struct npc_install_flow_req *tmp_req; + int err; + + mutex_lock(&nic->mbox.lock); + tmp_req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); + if (!tmp_req) { + mutex_unlock(&nic->mbox.lock); + return -ENOMEM; + } + + memcpy(tmp_req, req, sizeof(struct npc_install_flow_req)); + /* Send message to AF */ + err = otx2_sync_mbox_msg(&nic->mbox); + if (err) { + netdev_err(nic->netdev, "Failed to install MCAM flow entry %d\n", + req->entry); + mutex_unlock(&nic->mbox.lock); + return -EFAULT; + } + + mutex_unlock(&nic->mbox.lock); + return 0; +} + +static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) { + struct npc_delete_flow_rsp *rsp; struct npc_delete_flow_req *req; int err; @@ -729,22 +843,113 @@ static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry) mutex_unlock(&nic->mbox.lock); return -EFAULT; } + + if (cntr_val) { + rsp = (struct npc_delete_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + netdev_err(nic->netdev, "Failed to get MCAM delete response for entry %d\n", + entry); + mutex_unlock(&nic->mbox.lock); + return -EFAULT; + } + + *cntr_val = rsp->cntr_val; + } + mutex_unlock(&nic->mbox.lock); + return 0; +} + +static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct list_head *pos, *n; + struct otx2_tc_flow *tmp; + int i = 0, index = 0; + u16 cntr_val = 0; + + /* Find and delete the entry from the list and re-install + * all the entries from beginning to the index of the + * deleted entry to higher mcam indexes. + */ + list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { + tmp = list_entry(pos, struct otx2_tc_flow, list); + if (node == tmp) { + list_del(&tmp->list); + break; + } + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + tmp->entry++; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + index++; + } + + list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { + if (i == index) + break; + + tmp = list_entry(pos, struct otx2_tc_flow, list); + otx2_add_mcam_flow_entry(nic, &tmp->req); + i++; + } return 0; } +static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + int mcam_idx = flow_cfg->max_flows - flow_cfg->nr_flows - 1; + struct otx2_tc_flow *tmp; + int list_idx, i; + u16 cntr_val = 0; + + /* Find the index of the entry(list_idx) whose priority + * is greater than the new entry and re-install all + * the entries from beginning to list_idx to higher + * mcam indexes. + */ + list_idx = otx2_tc_add_to_flow_list(flow_cfg, node); + for (i = 0; i < list_idx; i++) { + tmp = otx2_tc_get_entry_by_index(flow_cfg, i); + if (!tmp) + return -ENOMEM; + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + tmp->entry = flow_cfg->flow_ent[mcam_idx]; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + otx2_add_mcam_flow_entry(nic, &tmp->req); + mcam_idx++; + } + + return mcam_idx; +} + +static int otx2_tc_update_mcam_table(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node, + bool add_req) +{ + if (add_req) + return otx2_tc_update_mcam_table_add_req(nic, flow_cfg, node); + + return otx2_tc_update_mcam_table_del_req(nic, flow_cfg, node); +} + static int otx2_tc_del_flow(struct otx2_nic *nic, struct flow_cls_offload *tc_flow_cmd) { struct otx2_flow_config *flow_cfg = nic->flow_cfg; - struct otx2_tc_info *tc_info = &nic->tc_info; struct otx2_tc_flow *flow_node; int err; - flow_node = rhashtable_lookup_fast(&tc_info->flow_table, - &tc_flow_cmd->cookie, - tc_info->flow_ht_params); + flow_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie); if (!flow_node) { netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n", tc_flow_cmd->cookie); @@ -772,16 +977,10 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, mutex_unlock(&nic->mbox.lock); } - otx2_del_mcam_flow_entry(nic, flow_node->entry); - - WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table, - &flow_node->node, - nic->tc_info.flow_ht_params)); + otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL); + otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false); kfree_rcu(flow_node, rcu); - - clear_bit(flow_node->bitpos, tc_info->tc_entries_bitmap); flow_cfg->nr_flows--; - return 0; } @@ -790,15 +989,14 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, { struct netlink_ext_ack *extack = tc_flow_cmd->common.extack; struct otx2_flow_config *flow_cfg = nic->flow_cfg; - struct otx2_tc_info *tc_info = &nic->tc_info; struct otx2_tc_flow *new_node, *old_node; struct npc_install_flow_req *req, dummy; - int rc, err; + int rc, err, mcam_idx; if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) return -ENOMEM; - if (bitmap_full(tc_info->tc_entries_bitmap, flow_cfg->max_flows)) { + if (flow_cfg->nr_flows == flow_cfg->max_flows) { NL_SET_ERR_MSG_MOD(extack, "Free MCAM entry not available to add the flow"); return -ENOMEM; @@ -810,6 +1008,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, return -ENOMEM; spin_lock_init(&new_node->lock); new_node->cookie = tc_flow_cmd->cookie; + new_node->prio = tc_flow_cmd->common.prio; memset(&dummy, 0, sizeof(struct npc_install_flow_req)); @@ -820,12 +1019,11 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, } /* If a flow exists with the same cookie, delete it */ - old_node = rhashtable_lookup_fast(&tc_info->flow_table, - &tc_flow_cmd->cookie, - tc_info->flow_ht_params); + old_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie); if (old_node) otx2_tc_del_flow(nic, tc_flow_cmd); + mcam_idx = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); mutex_lock(&nic->mbox.lock); req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); if (!req) { @@ -836,11 +1034,8 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); - - new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap, - flow_cfg->max_flows); req->channel = nic->hw.rx_chan_base; - req->entry = flow_cfg->flow_ent[flow_cfg->max_flows - new_node->bitpos - 1]; + req->entry = flow_cfg->flow_ent[mcam_idx]; req->intf = NIX_INTF_RX; req->set_cntr = 1; new_node->entry = req->entry; @@ -850,26 +1045,18 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, if (rc) { NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry"); mutex_unlock(&nic->mbox.lock); - kfree_rcu(new_node, rcu); goto free_leaf; } - mutex_unlock(&nic->mbox.lock); - /* add new flow to flow-table */ - rc = rhashtable_insert_fast(&nic->tc_info.flow_table, &new_node->node, - nic->tc_info.flow_ht_params); - if (rc) { - otx2_del_mcam_flow_entry(nic, req->entry); - kfree_rcu(new_node, rcu); - goto free_leaf; - } + mutex_unlock(&nic->mbox.lock); + memcpy(&new_node->req, req, sizeof(struct npc_install_flow_req)); - set_bit(new_node->bitpos, tc_info->tc_entries_bitmap); flow_cfg->nr_flows++; - return 0; free_leaf: + otx2_tc_del_from_flow_list(flow_cfg, new_node); + kfree_rcu(new_node, rcu); if (new_node->is_act_police) { mutex_lock(&nic->mbox.lock); @@ -896,16 +1083,13 @@ free_leaf: static int otx2_tc_get_flow_stats(struct otx2_nic *nic, struct flow_cls_offload *tc_flow_cmd) { - struct otx2_tc_info *tc_info = &nic->tc_info; struct npc_mcam_get_stats_req *req; struct npc_mcam_get_stats_rsp *rsp; struct otx2_tc_flow_stats *stats; struct otx2_tc_flow *flow_node; int err; - flow_node = rhashtable_lookup_fast(&tc_info->flow_table, - &tc_flow_cmd->cookie, - tc_info->flow_ht_params); + flow_node = otx2_tc_get_entry_by_cookie(nic->flow_cfg, tc_flow_cmd->cookie); if (!flow_node) { netdev_info(nic->netdev, "tc flow not found for cookie %lx", tc_flow_cmd->cookie); @@ -1053,12 +1237,20 @@ static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { struct otx2_nic *nic = cb_priv; + bool ntuple; if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) return -EOPNOTSUPP; + ntuple = nic->netdev->features & NETIF_F_NTUPLE; switch (type) { case TC_SETUP_CLSFLOWER: + if (ntuple) { + netdev_warn(nic->netdev, + "Can't install TC flower offload rule when NTUPLE is active"); + return -EOPNOTSUPP; + } + return otx2_setup_tc_cls_flower(nic, type_data); case TC_SETUP_CLSMATCHALL: return otx2_setup_tc_ingress_matchall(nic, type_data); @@ -1143,18 +1335,8 @@ int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, } EXPORT_SYMBOL(otx2_setup_tc); -static const struct rhashtable_params tc_flow_ht_params = { - .head_offset = offsetof(struct otx2_tc_flow, node), - .key_offset = offsetof(struct otx2_tc_flow, cookie), - .key_len = sizeof(((struct otx2_tc_flow *)0)->cookie), - .automatic_shrinking = true, -}; - int otx2_init_tc(struct otx2_nic *nic) { - struct otx2_tc_info *tc = &nic->tc_info; - int err; - /* Exclude receive queue 0 being used for police action */ set_bit(0, &nic->rq_bmap); @@ -1164,25 +1346,12 @@ int otx2_init_tc(struct otx2_nic *nic) return -EINVAL; } - err = otx2_tc_alloc_ent_bitmap(nic); - if (err) - return err; - - tc->flow_ht_params = tc_flow_ht_params; - err = rhashtable_init(&tc->flow_table, &tc->flow_ht_params); - if (err) { - kfree(tc->tc_entries_bitmap); - tc->tc_entries_bitmap = NULL; - } - return err; + return 0; } EXPORT_SYMBOL(otx2_init_tc); void otx2_shutdown_tc(struct otx2_nic *nic) { - struct otx2_tc_info *tc = &nic->tc_info; - - kfree(tc->tc_entries_bitmap); - rhashtable_destroy(&tc->flow_table); + otx2_destroy_tc_flow_list(nic); } EXPORT_SYMBOL(otx2_shutdown_tc); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c index d3a76c5ccda8..1e77bbf5d22a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c @@ -19,6 +19,9 @@ #define OTX2_QOS_CLASS_NONE 0 #define OTX2_QOS_DEFAULT_PRIO 0xF #define OTX2_QOS_INVALID_SQ 0xFFFF +#define OTX2_QOS_INVALID_TXSCHQ_IDX 0xFFFF +#define CN10K_MAX_RR_WEIGHT GENMASK_ULL(13, 0) +#define OTX2_MAX_RR_QUANTUM GENMASK_ULL(23, 0) static void otx2_qos_update_tx_netdev_queues(struct otx2_nic *pfvf) { @@ -65,11 +68,24 @@ static void otx2_qos_get_regaddr(struct otx2_qos_node *node, } } +static int otx2_qos_quantum_to_dwrr_weight(struct otx2_nic *pfvf, u32 quantum) +{ + u32 weight; + + weight = quantum / pfvf->hw.dwrr_mtu; + if (quantum % pfvf->hw.dwrr_mtu) + weight += 1; + + return weight; +} + static void otx2_config_sched_shaping(struct otx2_nic *pfvf, struct otx2_qos_node *node, struct nix_txschq_config *cfg, int *num_regs) { + u32 rr_weight; + u32 quantum; u64 maxrate; otx2_qos_get_regaddr(node, cfg, *num_regs); @@ -86,8 +102,17 @@ static void otx2_config_sched_shaping(struct otx2_nic *pfvf, return; } - /* configure priority */ - cfg->regval[*num_regs] = (node->schq - node->parent->prio_anchor) << 24; + /* configure priority/quantum */ + if (node->is_static) { + cfg->regval[*num_regs] = + (node->schq - node->parent->prio_anchor) << 24; + } else { + quantum = node->quantum ? + node->quantum : pfvf->tx_max_pktlen; + rr_weight = otx2_qos_quantum_to_dwrr_weight(pfvf, quantum); + cfg->regval[*num_regs] = node->parent->child_dwrr_prio << 24 | + rr_weight; + } (*num_regs)++; /* configure PIR */ @@ -195,9 +220,8 @@ static int otx2_qos_txschq_set_parent_topology(struct otx2_nic *pfvf, cfg->reg[0] = NIX_AF_TL1X_TOPOLOGY(parent->schq); cfg->regval[0] = (u64)parent->prio_anchor << 32; - if (parent->level == NIX_TXSCH_LVL_TL1) - cfg->regval[0] |= (u64)TXSCH_TL1_DFLT_RR_PRIO << 1; - + cfg->regval[0] |= ((parent->child_dwrr_prio != OTX2_QOS_DEFAULT_PRIO) ? + parent->child_dwrr_prio : 0) << 1; cfg->num_regs++; rc = otx2_sync_mbox_msg(&pfvf->mbox); @@ -315,9 +339,14 @@ static void otx2_qos_fill_cfg_tl(struct otx2_qos_node *parent, list_for_each_entry(node, &parent->child_list, list) { otx2_qos_fill_cfg_tl(node, cfg); - cfg->schq_contig[node->level]++; otx2_qos_fill_cfg_schq(node, cfg); } + + /* Assign the required number of transmit schedular queues under the + * given class + */ + cfg->schq_contig[parent->level - 1] += parent->child_dwrr_cnt + + parent->max_static_prio + 1; } static void otx2_qos_prepare_txschq_cfg(struct otx2_nic *pfvf, @@ -378,10 +407,12 @@ otx2_qos_alloc_root(struct otx2_nic *pfvf) return ERR_PTR(-ENOMEM); node->parent = NULL; - if (!is_otx2_vf(pfvf->pcifunc)) + if (!is_otx2_vf(pfvf->pcifunc)) { node->level = NIX_TXSCH_LVL_TL1; - else + } else { node->level = NIX_TXSCH_LVL_TL2; + node->child_dwrr_prio = OTX2_QOS_DEFAULT_PRIO; + } WRITE_ONCE(node->qid, OTX2_QOS_QID_INNER); node->classid = OTX2_QOS_ROOT_CLASSID; @@ -401,9 +432,13 @@ static int otx2_qos_add_child_node(struct otx2_qos_node *parent, struct otx2_qos_node *tmp_node; struct list_head *tmp; + if (node->prio > parent->max_static_prio) + parent->max_static_prio = node->prio; + for (tmp = head->next; tmp != head; tmp = tmp->next) { tmp_node = list_entry(tmp, struct otx2_qos_node, list); - if (tmp_node->prio == node->prio) + if (tmp_node->prio == node->prio && + tmp_node->is_static) return -EEXIST; if (tmp_node->prio > node->prio) { list_add_tail(&node->list, tmp); @@ -434,6 +469,10 @@ static int otx2_qos_alloc_txschq_node(struct otx2_nic *pfvf, txschq_node->rate = 0; txschq_node->ceil = 0; txschq_node->prio = 0; + txschq_node->quantum = 0; + txschq_node->is_static = true; + txschq_node->child_dwrr_prio = OTX2_QOS_DEFAULT_PRIO; + txschq_node->txschq_idx = OTX2_QOS_INVALID_TXSCHQ_IDX; mutex_lock(&pfvf->qos.qos_lock); list_add_tail(&txschq_node->list, &node->child_schq_list); @@ -459,7 +498,7 @@ static struct otx2_qos_node * otx2_qos_sw_create_leaf_node(struct otx2_nic *pfvf, struct otx2_qos_node *parent, u16 classid, u32 prio, u64 rate, u64 ceil, - u16 qid) + u32 quantum, u16 qid, bool static_cfg) { struct otx2_qos_node *node; int err; @@ -476,6 +515,10 @@ otx2_qos_sw_create_leaf_node(struct otx2_nic *pfvf, node->rate = otx2_convert_rate(rate); node->ceil = otx2_convert_rate(ceil); node->prio = prio; + node->quantum = quantum; + node->is_static = static_cfg; + node->child_dwrr_prio = OTX2_QOS_DEFAULT_PRIO; + node->txschq_idx = OTX2_QOS_INVALID_TXSCHQ_IDX; __set_bit(qid, pfvf->qos.qos_sq_bmap); @@ -622,12 +665,28 @@ static int otx2_qos_txschq_alloc(struct otx2_nic *pfvf, } pfvf->qos.link_cfg_lvl = rsp->link_cfg_lvl; + pfvf->hw.txschq_aggr_lvl_rr_prio = rsp->aggr_lvl_rr_prio; out: mutex_unlock(&mbox->lock); return rc; } +static void otx2_qos_free_unused_txschq(struct otx2_nic *pfvf, + struct otx2_qos_cfg *cfg) +{ + int lvl, idx, schq; + + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { + for (idx = 0; idx < cfg->schq_contig[lvl]; idx++) { + if (!cfg->schq_index_used[lvl][idx]) { + schq = cfg->schq_contig_list[lvl][idx]; + otx2_txschq_free_one(pfvf, lvl, schq); + } + } + } +} + static void otx2_qos_txschq_fill_cfg_schq(struct otx2_nic *pfvf, struct otx2_qos_node *node, struct otx2_qos_cfg *cfg) @@ -652,9 +711,11 @@ static void otx2_qos_txschq_fill_cfg_tl(struct otx2_nic *pfvf, list_for_each_entry(tmp, &node->child_list, list) { otx2_qos_txschq_fill_cfg_tl(pfvf, tmp, cfg); cnt = cfg->static_node_pos[tmp->level]; - tmp->schq = cfg->schq_contig_list[tmp->level][cnt]; + tmp->schq = cfg->schq_contig_list[tmp->level][tmp->txschq_idx]; + cfg->schq_index_used[tmp->level][tmp->txschq_idx] = true; if (cnt == 0) - node->prio_anchor = tmp->schq; + node->prio_anchor = + cfg->schq_contig_list[tmp->level][0]; cfg->static_node_pos[tmp->level]++; otx2_qos_txschq_fill_cfg_schq(pfvf, tmp, cfg); } @@ -667,9 +728,87 @@ static void otx2_qos_txschq_fill_cfg(struct otx2_nic *pfvf, mutex_lock(&pfvf->qos.qos_lock); otx2_qos_txschq_fill_cfg_tl(pfvf, node, cfg); otx2_qos_txschq_fill_cfg_schq(pfvf, node, cfg); + otx2_qos_free_unused_txschq(pfvf, cfg); mutex_unlock(&pfvf->qos.qos_lock); } +static void __otx2_qos_assign_base_idx_tl(struct otx2_nic *pfvf, + struct otx2_qos_node *tmp, + unsigned long *child_idx_bmap, + int child_cnt) +{ + int idx; + + if (tmp->txschq_idx != OTX2_QOS_INVALID_TXSCHQ_IDX) + return; + + /* assign static nodes 1:1 prio mapping first, then remaining nodes */ + for (idx = 0; idx < child_cnt; idx++) { + if (tmp->is_static && tmp->prio == idx && + !test_bit(idx, child_idx_bmap)) { + tmp->txschq_idx = idx; + set_bit(idx, child_idx_bmap); + return; + } else if (!tmp->is_static && idx >= tmp->prio && + !test_bit(idx, child_idx_bmap)) { + tmp->txschq_idx = idx; + set_bit(idx, child_idx_bmap); + return; + } + } +} + +static int otx2_qos_assign_base_idx_tl(struct otx2_nic *pfvf, + struct otx2_qos_node *node) +{ + unsigned long *child_idx_bmap; + struct otx2_qos_node *tmp; + int child_cnt; + + list_for_each_entry(tmp, &node->child_list, list) + tmp->txschq_idx = OTX2_QOS_INVALID_TXSCHQ_IDX; + + /* allocate child index array */ + child_cnt = node->child_dwrr_cnt + node->max_static_prio + 1; + child_idx_bmap = kcalloc(BITS_TO_LONGS(child_cnt), + sizeof(unsigned long), + GFP_KERNEL); + if (!child_idx_bmap) + return -ENOMEM; + + list_for_each_entry(tmp, &node->child_list, list) + otx2_qos_assign_base_idx_tl(pfvf, tmp); + + /* assign base index of static priority children first */ + list_for_each_entry(tmp, &node->child_list, list) { + if (!tmp->is_static) + continue; + __otx2_qos_assign_base_idx_tl(pfvf, tmp, child_idx_bmap, + child_cnt); + } + + /* assign base index of dwrr priority children */ + list_for_each_entry(tmp, &node->child_list, list) + __otx2_qos_assign_base_idx_tl(pfvf, tmp, child_idx_bmap, + child_cnt); + + kfree(child_idx_bmap); + + return 0; +} + +static int otx2_qos_assign_base_idx(struct otx2_nic *pfvf, + struct otx2_qos_node *node) +{ + int ret = 0; + + mutex_lock(&pfvf->qos.qos_lock); + ret = otx2_qos_assign_base_idx_tl(pfvf, node); + mutex_unlock(&pfvf->qos.qos_lock); + + return ret; +} + static int otx2_qos_txschq_push_cfg_schq(struct otx2_nic *pfvf, struct otx2_qos_node *node, struct otx2_qos_cfg *cfg) @@ -761,8 +900,10 @@ static void otx2_qos_free_cfg(struct otx2_nic *pfvf, struct otx2_qos_cfg *cfg) for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { for (idx = 0; idx < cfg->schq_contig[lvl]; idx++) { - schq = cfg->schq_contig_list[lvl][idx]; - otx2_txschq_free_one(pfvf, lvl, schq); + if (cfg->schq_index_used[lvl][idx]) { + schq = cfg->schq_contig_list[lvl][idx]; + otx2_txschq_free_one(pfvf, lvl, schq); + } } } } @@ -838,6 +979,10 @@ static int otx2_qos_push_txschq_cfg(struct otx2_nic *pfvf, if (ret) return -ENOSPC; + ret = otx2_qos_assign_base_idx(pfvf, node); + if (ret) + return -ENOMEM; + if (!(pfvf->netdev->flags & IFF_UP)) { otx2_qos_txschq_fill_cfg(pfvf, node, cfg); return 0; @@ -894,6 +1039,13 @@ static int otx2_qos_root_add(struct otx2_nic *pfvf, u16 htb_maj_id, u16 htb_defc goto free_root_node; } + /* Update TL1 RR PRIO */ + if (root->level == NIX_TXSCH_LVL_TL1) { + root->child_dwrr_prio = pfvf->hw.txschq_aggr_lvl_rr_prio; + netdev_dbg(pfvf->netdev, + "TL1 DWRR Priority %d\n", root->child_dwrr_prio); + } + if (!(pfvf->netdev->flags & IFF_UP) || root->level == NIX_TXSCH_LVL_TL1) { root->schq = new_cfg->schq_list[root->level][0]; @@ -940,37 +1092,126 @@ static int otx2_qos_root_destroy(struct otx2_nic *pfvf) return 0; } +static int otx2_qos_validate_quantum(struct otx2_nic *pfvf, u32 quantum) +{ + u32 rr_weight = otx2_qos_quantum_to_dwrr_weight(pfvf, quantum); + int err = 0; + + /* Max Round robin weight supported by octeontx2 and CN10K + * is different. Validate accordingly + */ + if (is_dev_otx2(pfvf->pdev)) + err = (rr_weight > OTX2_MAX_RR_QUANTUM) ? -EINVAL : 0; + else if (rr_weight > CN10K_MAX_RR_WEIGHT) + err = -EINVAL; + + return err; +} + +static int otx2_qos_validate_dwrr_cfg(struct otx2_qos_node *parent, + struct netlink_ext_ack *extack, + struct otx2_nic *pfvf, + u64 prio, u64 quantum) +{ + int err; + + err = otx2_qos_validate_quantum(pfvf, quantum); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Unsupported quantum value"); + return err; + } + + if (parent->child_dwrr_prio == OTX2_QOS_DEFAULT_PRIO) { + parent->child_dwrr_prio = prio; + } else if (prio != parent->child_dwrr_prio) { + NL_SET_ERR_MSG_MOD(extack, "Only one DWRR group is allowed"); + return -EOPNOTSUPP; + } + + return 0; +} + static int otx2_qos_validate_configuration(struct otx2_qos_node *parent, struct netlink_ext_ack *extack, struct otx2_nic *pfvf, - u64 prio) + u64 prio, bool static_cfg) { - if (test_bit(prio, parent->prio_bmap)) { - NL_SET_ERR_MSG_MOD(extack, - "Static priority child with same priority exists"); + if (prio == parent->child_dwrr_prio && static_cfg) { + NL_SET_ERR_MSG_MOD(extack, "DWRR child group with same priority exists"); return -EEXIST; } - if (prio == TXSCH_TL1_DFLT_RR_PRIO) { + if (static_cfg && test_bit(prio, parent->prio_bmap)) { NL_SET_ERR_MSG_MOD(extack, - "Priority is reserved for Round Robin"); - return -EINVAL; + "Static priority child with same priority exists"); + return -EEXIST; } return 0; } +static void otx2_reset_dwrr_prio(struct otx2_qos_node *parent, u64 prio) +{ + /* For PF, root node dwrr priority is static */ + if (parent->level == NIX_TXSCH_LVL_TL1) + return; + + if (parent->child_dwrr_prio != OTX2_QOS_DEFAULT_PRIO) { + parent->child_dwrr_prio = OTX2_QOS_DEFAULT_PRIO; + clear_bit(prio, parent->prio_bmap); + } +} + +static bool is_qos_node_dwrr(struct otx2_qos_node *parent, + struct otx2_nic *pfvf, + u64 prio) +{ + struct otx2_qos_node *node; + bool ret = false; + + if (parent->child_dwrr_prio == prio) + return true; + + mutex_lock(&pfvf->qos.qos_lock); + list_for_each_entry(node, &parent->child_list, list) { + if (prio == node->prio) { + if (parent->child_dwrr_prio != OTX2_QOS_DEFAULT_PRIO && + parent->child_dwrr_prio != prio) + continue; + + if (otx2_qos_validate_quantum(pfvf, node->quantum)) { + netdev_err(pfvf->netdev, + "Unsupported quantum value for existing classid=0x%x quantum=%d prio=%d", + node->classid, node->quantum, + node->prio); + break; + } + /* mark old node as dwrr */ + node->is_static = false; + parent->child_dwrr_cnt++; + parent->child_static_cnt--; + ret = true; + break; + } + } + mutex_unlock(&pfvf->qos.qos_lock); + + return ret; +} + static int otx2_qos_leaf_alloc_queue(struct otx2_nic *pfvf, u16 classid, u32 parent_classid, u64 rate, u64 ceil, - u64 prio, struct netlink_ext_ack *extack) + u64 prio, u32 quantum, + struct netlink_ext_ack *extack) { struct otx2_qos_cfg *old_cfg, *new_cfg; struct otx2_qos_node *node, *parent; int qid, ret, err; + bool static_cfg; netdev_dbg(pfvf->netdev, - "TC_HTB_LEAF_ALLOC_QUEUE: classid=0x%x parent_classid=0x%x rate=%lld ceil=%lld prio=%lld\n", - classid, parent_classid, rate, ceil, prio); + "TC_HTB_LEAF_ALLOC_QUEUE: classid=0x%x parent_classid=0x%x rate=%lld ceil=%lld prio=%lld quantum=%d\n", + classid, parent_classid, rate, ceil, prio, quantum); if (prio > OTX2_QOS_MAX_PRIO) { NL_SET_ERR_MSG_MOD(extack, "Valid priority range 0 to 7"); @@ -978,6 +1219,12 @@ static int otx2_qos_leaf_alloc_queue(struct otx2_nic *pfvf, u16 classid, goto out; } + if (!quantum || quantum > INT_MAX) { + NL_SET_ERR_MSG_MOD(extack, "Invalid quantum, range 1 - 2147483647 bytes"); + ret = -EOPNOTSUPP; + goto out; + } + /* get parent node */ parent = otx2_sw_node_find(pfvf, parent_classid); if (!parent) { @@ -991,10 +1238,24 @@ static int otx2_qos_leaf_alloc_queue(struct otx2_nic *pfvf, u16 classid, goto out; } - ret = otx2_qos_validate_configuration(parent, extack, pfvf, prio); + static_cfg = !is_qos_node_dwrr(parent, pfvf, prio); + ret = otx2_qos_validate_configuration(parent, extack, pfvf, prio, + static_cfg); if (ret) goto out; + if (!static_cfg) { + ret = otx2_qos_validate_dwrr_cfg(parent, extack, pfvf, prio, + quantum); + if (ret) + goto out; + } + + if (static_cfg) + parent->child_static_cnt++; + else + parent->child_dwrr_cnt++; + set_bit(prio, parent->prio_bmap); /* read current txschq configuration */ @@ -1019,7 +1280,7 @@ static int otx2_qos_leaf_alloc_queue(struct otx2_nic *pfvf, u16 classid, /* allocate and initialize a new child node */ node = otx2_qos_sw_create_leaf_node(pfvf, parent, classid, prio, rate, - ceil, qid); + ceil, quantum, qid, static_cfg); if (IS_ERR(node)) { NL_SET_ERR_MSG_MOD(extack, "Unable to allocate leaf node"); ret = PTR_ERR(node); @@ -1067,6 +1328,11 @@ free_node: free_old_cfg: kfree(old_cfg); reset_prio: + if (static_cfg) + parent->child_static_cnt--; + else + parent->child_dwrr_cnt--; + clear_bit(prio, parent->prio_bmap); out: return ret; @@ -1074,10 +1340,11 @@ out: static int otx2_qos_leaf_to_inner(struct otx2_nic *pfvf, u16 classid, u16 child_classid, u64 rate, u64 ceil, u64 prio, - struct netlink_ext_ack *extack) + u32 quantum, struct netlink_ext_ack *extack) { struct otx2_qos_cfg *old_cfg, *new_cfg; struct otx2_qos_node *node, *child; + bool static_cfg; int ret, err; u16 qid; @@ -1091,6 +1358,12 @@ static int otx2_qos_leaf_to_inner(struct otx2_nic *pfvf, u16 classid, goto out; } + if (!quantum || quantum > INT_MAX) { + NL_SET_ERR_MSG_MOD(extack, "Invalid quantum, range 1 - 2147483647 bytes"); + ret = -EOPNOTSUPP; + goto out; + } + /* find node related to classid */ node = otx2_sw_node_find(pfvf, classid); if (!node) { @@ -1105,6 +1378,19 @@ static int otx2_qos_leaf_to_inner(struct otx2_nic *pfvf, u16 classid, goto out; } + static_cfg = !is_qos_node_dwrr(node, pfvf, prio); + if (!static_cfg) { + ret = otx2_qos_validate_dwrr_cfg(node, extack, pfvf, prio, + quantum); + if (ret) + goto out; + } + + if (static_cfg) + node->child_static_cnt++; + else + node->child_dwrr_cnt++; + set_bit(prio, node->prio_bmap); /* store the qid to assign to leaf node */ @@ -1127,7 +1413,8 @@ static int otx2_qos_leaf_to_inner(struct otx2_nic *pfvf, u16 classid, /* allocate and initialize a new child node */ child = otx2_qos_sw_create_leaf_node(pfvf, node, child_classid, - prio, rate, ceil, qid); + prio, rate, ceil, quantum, + qid, static_cfg); if (IS_ERR(child)) { NL_SET_ERR_MSG_MOD(extack, "Unable to allocate leaf node"); ret = PTR_ERR(child); @@ -1178,6 +1465,10 @@ free_node: free_old_cfg: kfree(old_cfg); reset_prio: + if (static_cfg) + node->child_static_cnt--; + else + node->child_dwrr_cnt--; clear_bit(prio, node->prio_bmap); out: return ret; @@ -1187,6 +1478,7 @@ static int otx2_qos_leaf_del(struct otx2_nic *pfvf, u16 *classid, struct netlink_ext_ack *extack) { struct otx2_qos_node *node, *parent; + int dwrr_del_node = false; u64 prio; u16 qid; @@ -1202,12 +1494,27 @@ static int otx2_qos_leaf_del(struct otx2_nic *pfvf, u16 *classid, prio = node->prio; qid = node->qid; + if (!node->is_static) + dwrr_del_node = true; + otx2_qos_disable_sq(pfvf, node->qid); otx2_qos_destroy_node(pfvf, node); pfvf->qos.qid_to_sqmap[qid] = OTX2_QOS_INVALID_SQ; - clear_bit(prio, parent->prio_bmap); + if (dwrr_del_node) { + parent->child_dwrr_cnt--; + } else { + parent->child_static_cnt--; + clear_bit(prio, parent->prio_bmap); + } + + /* Reset DWRR priority if all dwrr nodes are deleted */ + if (!parent->child_dwrr_cnt) + otx2_reset_dwrr_prio(parent, prio); + + if (!parent->child_static_cnt) + parent->max_static_prio = 0; return 0; } @@ -1217,6 +1524,7 @@ static int otx2_qos_leaf_del_last(struct otx2_nic *pfvf, u16 classid, bool force { struct otx2_qos_node *node, *parent; struct otx2_qos_cfg *new_cfg; + int dwrr_del_node = false; u64 prio; int err; u16 qid; @@ -1241,11 +1549,26 @@ static int otx2_qos_leaf_del_last(struct otx2_nic *pfvf, u16 classid, bool force return -ENOENT; } + if (!node->is_static) + dwrr_del_node = true; + /* destroy the leaf node */ otx2_qos_destroy_node(pfvf, node); pfvf->qos.qid_to_sqmap[qid] = OTX2_QOS_INVALID_SQ; - clear_bit(prio, parent->prio_bmap); + if (dwrr_del_node) { + parent->child_dwrr_cnt--; + } else { + parent->child_static_cnt--; + clear_bit(prio, parent->prio_bmap); + } + + /* Reset DWRR priority if all dwrr nodes are deleted */ + if (!parent->child_dwrr_cnt) + otx2_reset_dwrr_prio(parent, prio); + + if (!parent->child_static_cnt) + parent->max_static_prio = 0; /* create downstream txschq entries to parent */ err = otx2_qos_alloc_txschq_node(pfvf, parent); @@ -1298,10 +1621,12 @@ void otx2_qos_config_txschq(struct otx2_nic *pfvf) if (!root) return; - err = otx2_qos_txschq_config(pfvf, root); - if (err) { - netdev_err(pfvf->netdev, "Error update txschq configuration\n"); - goto root_destroy; + if (root->level != NIX_TXSCH_LVL_TL1) { + err = otx2_qos_txschq_config(pfvf, root); + if (err) { + netdev_err(pfvf->netdev, "Error update txschq configuration\n"); + goto root_destroy; + } } err = otx2_qos_txschq_push_cfg_tl(pfvf, root, NULL); @@ -1334,7 +1659,8 @@ int otx2_setup_tc_htb(struct net_device *ndev, struct tc_htb_qopt_offload *htb) res = otx2_qos_leaf_alloc_queue(pfvf, htb->classid, htb->parent_classid, htb->rate, htb->ceil, - htb->prio, htb->extack); + htb->prio, htb->quantum, + htb->extack); if (res < 0) return res; htb->qid = res; @@ -1343,7 +1669,7 @@ int otx2_setup_tc_htb(struct net_device *ndev, struct tc_htb_qopt_offload *htb) return otx2_qos_leaf_to_inner(pfvf, htb->parent_classid, htb->classid, htb->rate, htb->ceil, htb->prio, - htb->extack); + htb->quantum, htb->extack); case TC_HTB_LEAF_DEL: return otx2_qos_leaf_del(pfvf, &htb->classid, htb->extack); case TC_HTB_LEAF_DEL_LAST: diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.h b/drivers/net/ethernet/marvell/octeontx2/nic/qos.h index 19773284be27..221bd0438f60 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.h @@ -35,6 +35,7 @@ struct otx2_qos_cfg { int dwrr_node_pos[NIX_TXSCH_LVL_CNT]; u16 schq_contig_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; u16 schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + bool schq_index_used[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; }; struct otx2_qos { @@ -59,10 +60,18 @@ struct otx2_qos_node { u64 ceil; u32 classid; u32 prio; - u16 schq; /* hw txschq */ + u32 quantum; + /* hw txschq */ + u16 schq; u16 qid; u16 prio_anchor; + u16 max_static_prio; + u16 child_dwrr_cnt; + u16 child_static_cnt; + u16 child_dwrr_prio; + u16 txschq_idx; /* txschq allocation index */ u8 level; + bool is_static; }; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index 3e20e71b0f81..8b9455d8a4f7 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -202,16 +202,16 @@ static int prestera_flower_parse(struct prestera_flow_block *block, int err; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_META) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ICMP) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_PORTS_RANGE) | - BIT(FLOW_DISSECTOR_KEY_VLAN))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_META) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ICMP) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS_RANGE) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { NL_SET_ERR_MSG_MOD(f->common.extack, "Unsupported key"); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index 9277a8fd1339..cc2a9ae794be 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -5,9 +5,6 @@ #include <linux/dmapool.h> #include <linux/etherdevice.h> #include <linux/if_vlan.h> -#include <linux/of_address.h> -#include <linux/of_device.h> -#include <linux/of.h> #include <linux/platform_device.h> #include "prestera_dsa.h" diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 7c487f9b36ec..c4cca27fb0d5 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -32,7 +32,6 @@ #include <linux/prefetch.h> #include <linux/debugfs.h> #include <linux/mii.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/dmi.h> diff --git a/drivers/net/ethernet/mediatek/mtk_eth_path.c b/drivers/net/ethernet/mediatek/mtk_eth_path.c index 317e447f4991..7c27a19c4d8f 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_path.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_path.c @@ -15,10 +15,10 @@ struct mtk_eth_muxc { const char *name; int cap_bit; - int (*set_path)(struct mtk_eth *eth, int path); + int (*set_path)(struct mtk_eth *eth, u64 path); }; -static const char *mtk_eth_path_name(int path) +static const char *mtk_eth_path_name(u64 path) { switch (path) { case MTK_ETH_PATH_GMAC1_RGMII: @@ -40,10 +40,10 @@ static const char *mtk_eth_path_name(int path) } } -static int set_mux_gdm1_to_gmac1_esw(struct mtk_eth *eth, int path) +static int set_mux_gdm1_to_gmac1_esw(struct mtk_eth *eth, u64 path) { bool updated = true; - u32 val, mask, set; + u32 mask, set, reg; switch (path) { case MTK_ETH_PATH_GMAC1_SGMII: @@ -59,11 +59,13 @@ static int set_mux_gdm1_to_gmac1_esw(struct mtk_eth *eth, int path) break; } - if (updated) { - val = mtk_r32(eth, MTK_MAC_MISC); - val = (val & mask) | set; - mtk_w32(eth, val, MTK_MAC_MISC); - } + if (mtk_is_netsys_v3_or_greater(eth)) + reg = MTK_MAC_MISC_V3; + else + reg = MTK_MAC_MISC; + + if (updated) + mtk_m32(eth, mask, set, reg); dev_dbg(eth->dev, "path %s in %s updated = %d\n", mtk_eth_path_name(path), __func__, updated); @@ -71,7 +73,7 @@ static int set_mux_gdm1_to_gmac1_esw(struct mtk_eth *eth, int path) return 0; } -static int set_mux_gmac2_gmac0_to_gephy(struct mtk_eth *eth, int path) +static int set_mux_gmac2_gmac0_to_gephy(struct mtk_eth *eth, u64 path) { unsigned int val = 0; bool updated = true; @@ -94,7 +96,7 @@ static int set_mux_gmac2_gmac0_to_gephy(struct mtk_eth *eth, int path) return 0; } -static int set_mux_u3_gmac2_to_qphy(struct mtk_eth *eth, int path) +static int set_mux_u3_gmac2_to_qphy(struct mtk_eth *eth, u64 path) { unsigned int val = 0, mask = 0, reg = 0; bool updated = true; @@ -125,7 +127,7 @@ static int set_mux_u3_gmac2_to_qphy(struct mtk_eth *eth, int path) return 0; } -static int set_mux_gmac1_gmac2_to_sgmii_rgmii(struct mtk_eth *eth, int path) +static int set_mux_gmac1_gmac2_to_sgmii_rgmii(struct mtk_eth *eth, u64 path) { unsigned int val = 0; bool updated = true; @@ -163,7 +165,7 @@ static int set_mux_gmac1_gmac2_to_sgmii_rgmii(struct mtk_eth *eth, int path) return 0; } -static int set_mux_gmac12_to_gephy_sgmii(struct mtk_eth *eth, int path) +static int set_mux_gmac12_to_gephy_sgmii(struct mtk_eth *eth, u64 path) { unsigned int val = 0; bool updated = true; @@ -218,7 +220,7 @@ static const struct mtk_eth_muxc mtk_eth_muxc[] = { }, }; -static int mtk_eth_mux_setup(struct mtk_eth *eth, int path) +static int mtk_eth_mux_setup(struct mtk_eth *eth, u64 path) { int i, err = 0; @@ -249,7 +251,7 @@ out: int mtk_gmac_sgmii_path_setup(struct mtk_eth *eth, int mac_id) { - int path; + u64 path; path = (mac_id == 0) ? MTK_ETH_PATH_GMAC1_SGMII : MTK_ETH_PATH_GMAC2_SGMII; @@ -260,7 +262,7 @@ int mtk_gmac_sgmii_path_setup(struct mtk_eth *eth, int mac_id) int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id) { - int path = 0; + u64 path = 0; if (mac_id == 1) path = MTK_ETH_PATH_GMAC2_GEPHY; @@ -274,7 +276,7 @@ int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id) int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id) { - int path; + u64 path; path = (mac_id == 0) ? MTK_ETH_PATH_GMAC1_RGMII : MTK_ETH_PATH_GMAC2_RGMII; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 2d15342c260a..1b89f800f6df 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -6,11 +6,12 @@ * Copyright (C) 2013-2016 Michael Lee <igvtee@gmail.com> */ -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/of_address.h> #include <linux/mfd/syscon.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/clk.h> #include <linux/pm_runtime.h> @@ -152,6 +153,54 @@ static const struct mtk_reg_map mt7986_reg_map = { .pse_oq_sta = 0x01a0, }; +static const struct mtk_reg_map mt7988_reg_map = { + .tx_irq_mask = 0x461c, + .tx_irq_status = 0x4618, + .pdma = { + .rx_ptr = 0x6900, + .rx_cnt_cfg = 0x6904, + .pcrx_ptr = 0x6908, + .glo_cfg = 0x6a04, + .rst_idx = 0x6a08, + .delay_irq = 0x6a0c, + .irq_status = 0x6a20, + .irq_mask = 0x6a28, + .adma_rx_dbg0 = 0x6a38, + .int_grp = 0x6a50, + }, + .qdma = { + .qtx_cfg = 0x4400, + .qtx_sch = 0x4404, + .rx_ptr = 0x4500, + .rx_cnt_cfg = 0x4504, + .qcrx_ptr = 0x4508, + .glo_cfg = 0x4604, + .rst_idx = 0x4608, + .delay_irq = 0x460c, + .fc_th = 0x4610, + .int_grp = 0x4620, + .hred = 0x4644, + .ctx_ptr = 0x4700, + .dtx_ptr = 0x4704, + .crx_ptr = 0x4710, + .drx_ptr = 0x4714, + .fq_head = 0x4720, + .fq_tail = 0x4724, + .fq_count = 0x4728, + .fq_blen = 0x472c, + .tx_sch_rate = 0x4798, + }, + .gdm1_cnt = 0x1c00, + .gdma_to_ppe = 0x3333, + .ppe_base = 0x2000, + .wdma_base = { + [0] = 0x4800, + [1] = 0x4c00, + }, + .pse_iq_sta = 0x0180, + .pse_oq_sta = 0x01a0, +}; + /* strings used by ethtool */ static const struct mtk_ethtool_stats { char str[ETH_GSTRING_LEN]; @@ -179,10 +228,54 @@ static const struct mtk_ethtool_stats { }; static const char * const mtk_clks_source_name[] = { - "ethif", "sgmiitop", "esw", "gp0", "gp1", "gp2", "fe", "trgpll", - "sgmii_tx250m", "sgmii_rx250m", "sgmii_cdr_ref", "sgmii_cdr_fb", - "sgmii2_tx250m", "sgmii2_rx250m", "sgmii2_cdr_ref", "sgmii2_cdr_fb", - "sgmii_ck", "eth2pll", "wocpu0", "wocpu1", "netsys0", "netsys1" + "ethif", + "sgmiitop", + "esw", + "gp0", + "gp1", + "gp2", + "gp3", + "xgp1", + "xgp2", + "xgp3", + "crypto", + "fe", + "trgpll", + "sgmii_tx250m", + "sgmii_rx250m", + "sgmii_cdr_ref", + "sgmii_cdr_fb", + "sgmii2_tx250m", + "sgmii2_rx250m", + "sgmii2_cdr_ref", + "sgmii2_cdr_fb", + "sgmii_ck", + "eth2pll", + "wocpu0", + "wocpu1", + "netsys0", + "netsys1", + "ethwarp_wocpu2", + "ethwarp_wocpu1", + "ethwarp_wocpu0", + "top_usxgmii0_sel", + "top_usxgmii1_sel", + "top_sgm0_sel", + "top_sgm1_sel", + "top_xfi_phy0_xtal_sel", + "top_xfi_phy1_xtal_sel", + "top_eth_gmii_sel", + "top_eth_refck_50m_sel", + "top_eth_sys_200m_sel", + "top_eth_sys_sel", + "top_eth_xgmii_sel", + "top_eth_mii_sel", + "top_netsys_sel", + "top_netsys_500m_sel", + "top_netsys_pao_2x_sel", + "top_netsys_sync_250m_sel", + "top_netsys_ppefb_250m_sel", + "top_netsys_warp_sel", }; void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg) @@ -195,7 +288,7 @@ u32 mtk_r32(struct mtk_eth *eth, unsigned reg) return __raw_readl(eth->base + reg); } -static u32 mtk_m32(struct mtk_eth *eth, u32 mask, u32 set, unsigned reg) +u32 mtk_m32(struct mtk_eth *eth, u32 mask, u32 set, unsigned int reg) { u32 val; @@ -385,10 +478,8 @@ static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth, } static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth, - phy_interface_t interface, int speed) + phy_interface_t interface) { - unsigned long rate; - u32 tck, rck, intf; int ret; if (interface == PHY_INTERFACE_MODE_TRGMII) { @@ -399,30 +490,20 @@ static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth, return; } - if (speed == SPEED_1000) { - intf = INTF_MODE_RGMII_1000; - rate = 250000000; - rck = RCK_CTRL_RGMII_1000; - tck = TCK_CTRL_RGMII_1000; - } else { - intf = INTF_MODE_RGMII_10_100; - rate = 500000000; - rck = RCK_CTRL_RGMII_10_100; - tck = TCK_CTRL_RGMII_10_100; - } - - mtk_w32(eth, intf, INTF_MODE); - - regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0, - ETHSYS_TRGMII_CLK_SEL362_5, - ETHSYS_TRGMII_CLK_SEL362_5); + dev_err(eth->dev, "Missing PLL configuration, ethernet may not work\n"); +} - ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], rate); - if (ret) - dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret); +static void mtk_setup_bridge_switch(struct mtk_eth *eth) +{ + /* Force Port1 XGMAC Link Up */ + mtk_m32(eth, 0, MTK_XGMAC_FORCE_LINK(MTK_GMAC1_ID), + MTK_XGMAC_STS(MTK_GMAC1_ID)); - mtk_w32(eth, rck, TRGMII_RCK_CTRL); - mtk_w32(eth, tck, TRGMII_TCK_CTRL); + /* Adjust GSW bridge IPG to 11 */ + mtk_m32(eth, GSWTX_IPG_MASK | GSWRX_IPG_MASK, + (GSW_IPG_11 << GSWTX_IPG_SHIFT) | + (GSW_IPG_11 << GSWRX_IPG_SHIFT), + MTK_GSW_CFG); } static struct phylink_pcs *mtk_mac_select_pcs(struct phylink_config *config, @@ -484,6 +565,8 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode, goto init_err; } break; + case PHY_INTERFACE_MODE_INTERNAL: + break; default: goto err_phy; } @@ -498,17 +581,8 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode, state->interface)) goto err_phy; } else { - /* FIXME: this is incorrect. Not only does it - * use state->speed (which is not guaranteed - * to be correct) but it also makes use of it - * in a code path that will only be reachable - * when the PHY interface mode changes, not - * when the speed changes. Consequently, RGMII - * is probably broken. - */ mtk_gmac0_rgmii_adjust(mac->hw, - state->interface, - state->speed); + state->interface); /* mt7623_pad_clk_setup */ for (i = 0 ; i < NUM_TRGMII_CTRL; i++) @@ -562,6 +636,15 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode, return; } + /* Setup gmac */ + if (mtk_is_netsys_v3_or_greater(eth) && + mac->interface == PHY_INTERFACE_MODE_INTERNAL) { + mtk_w32(mac->hw, MTK_GDMA_XGDM_SEL, MTK_GDMA_EG_CTRL(mac->id)); + mtk_w32(mac->hw, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(mac->id)); + + mtk_setup_bridge_switch(eth); + } + return; err_phy: @@ -602,38 +685,6 @@ static int mtk_mac_finish(struct phylink_config *config, unsigned int mode, return 0; } -static void mtk_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - struct mtk_mac *mac = container_of(config, struct mtk_mac, - phylink_config); - u32 pmsr = mtk_r32(mac->hw, MTK_MAC_MSR(mac->id)); - - state->link = (pmsr & MAC_MSR_LINK); - state->duplex = (pmsr & MAC_MSR_DPX) >> 1; - - switch (pmsr & (MAC_MSR_SPEED_1000 | MAC_MSR_SPEED_100)) { - case 0: - state->speed = SPEED_10; - break; - case MAC_MSR_SPEED_100: - state->speed = SPEED_100; - break; - case MAC_MSR_SPEED_1000: - state->speed = SPEED_1000; - break; - default: - state->speed = SPEED_UNKNOWN; - break; - } - - state->pause &= (MLO_PAUSE_RX | MLO_PAUSE_TX); - if (pmsr & MAC_MSR_RX_FC) - state->pause |= MLO_PAUSE_RX; - if (pmsr & MAC_MSR_TX_FC) - state->pause |= MLO_PAUSE_TX; -} - static void mtk_mac_link_down(struct phylink_config *config, unsigned int mode, phy_interface_t interface) { @@ -659,7 +710,7 @@ static void mtk_set_queue_speed(struct mtk_eth *eth, unsigned int idx, FIELD_PREP(MTK_QTX_SCH_MIN_RATE_MAN, 1) | FIELD_PREP(MTK_QTX_SCH_MIN_RATE_EXP, 4) | MTK_QTX_SCH_LEAKY_BUCKET_SIZE; - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v1(eth)) val |= MTK_QTX_SCH_LEAKY_BUCKET_EN; if (IS_ENABLED(CONFIG_SOC_MT7621)) { @@ -756,7 +807,6 @@ static void mtk_mac_link_up(struct phylink_config *config, static const struct phylink_mac_ops mtk_phylink_ops = { .mac_select_pcs = mtk_mac_select_pcs, - .mac_pcs_get_state = mtk_mac_pcs_get_state, .mac_config = mtk_mac_config, .mac_finish = mtk_mac_finish, .mac_link_down = mtk_mac_link_down, @@ -807,11 +857,15 @@ static int mtk_mdio_init(struct mtk_eth *eth) } divider = min_t(unsigned int, DIV_ROUND_UP(MDC_MAX_FREQ, max_clk), 63); + /* Configure MDC Turbo Mode */ + if (mtk_is_netsys_v3_or_greater(eth)) + mtk_m32(eth, 0, MISC_MDC_TURBO, MTK_MAC_MISC_V3); + /* Configure MDC Divider */ - val = mtk_r32(eth, MTK_PPSC); - val &= ~PPSC_MDC_CFG; - val |= FIELD_PREP(PPSC_MDC_CFG, divider) | PPSC_MDC_TURBO; - mtk_w32(eth, val, MTK_PPSC); + val = FIELD_PREP(PPSC_MDC_CFG, divider); + if (!mtk_is_netsys_v3_or_greater(eth)) + val |= PPSC_MDC_TURBO; + mtk_m32(eth, PPSC_MDC_CFG, val, MTK_PPSC); dev_dbg(eth->dev, "MDC is running on %d Hz\n", MDC_MAX_FREQ / divider); @@ -943,17 +997,32 @@ void mtk_stats_update_mac(struct mtk_mac *mac) mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x20 + offs); hw_stats->rx_flow_control_packets += mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x24 + offs); - hw_stats->tx_skip += - mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x28 + offs); - hw_stats->tx_collisions += - mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x2c + offs); - hw_stats->tx_bytes += - mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x30 + offs); - stats = mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x34 + offs); - if (stats) - hw_stats->tx_bytes += (stats << 32); - hw_stats->tx_packets += - mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x38 + offs); + + if (mtk_is_netsys_v3_or_greater(eth)) { + hw_stats->tx_skip += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x50 + offs); + hw_stats->tx_collisions += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x54 + offs); + hw_stats->tx_bytes += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x40 + offs); + stats = mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x44 + offs); + if (stats) + hw_stats->tx_bytes += (stats << 32); + hw_stats->tx_packets += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x48 + offs); + } else { + hw_stats->tx_skip += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x28 + offs); + hw_stats->tx_collisions += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x2c + offs); + hw_stats->tx_bytes += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x30 + offs); + stats = mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x34 + offs); + if (stats) + hw_stats->tx_bytes += (stats << 32); + hw_stats->tx_packets += + mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x38 + offs); + } } u64_stats_update_end(&hw_stats->syncp); @@ -963,7 +1032,7 @@ static void mtk_stats_update(struct mtk_eth *eth) { int i; - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->mac[i] || !eth->mac[i]->hw_stats) continue; if (spin_trylock(ð->mac[i]->hw_stats->stats_lock)) { @@ -1037,7 +1106,7 @@ static bool mtk_rx_get_desc(struct mtk_eth *eth, struct mtk_rx_dma_v2 *rxd, rxd->rxd1 = READ_ONCE(dma_rxd->rxd1); rxd->rxd3 = READ_ONCE(dma_rxd->rxd3); rxd->rxd4 = READ_ONCE(dma_rxd->rxd4); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { rxd->rxd5 = READ_ONCE(dma_rxd->rxd5); rxd->rxd6 = READ_ONCE(dma_rxd->rxd6); } @@ -1095,7 +1164,7 @@ static int mtk_init_fq_dma(struct mtk_eth *eth) txd->txd3 = TX_DMA_PLEN0(MTK_QDMA_PAGE_SIZE); txd->txd4 = 0; - if (MTK_HAS_CAPS(soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { txd->txd5 = 0; txd->txd6 = 0; txd->txd7 = 0; @@ -1257,7 +1326,19 @@ static void mtk_tx_set_dma_desc_v2(struct net_device *dev, void *txd, data |= TX_DMA_LS0; WRITE_ONCE(desc->txd3, data); - data = (mac->id + 1) << TX_DMA_FPORT_SHIFT_V2; /* forward port */ + /* set forward port */ + switch (mac->id) { + case MTK_GMAC1_ID: + data = PSE_GDM1_PORT << TX_DMA_FPORT_SHIFT_V2; + break; + case MTK_GMAC2_ID: + data = PSE_GDM2_PORT << TX_DMA_FPORT_SHIFT_V2; + break; + case MTK_GMAC3_ID: + data = PSE_GDM3_PORT << TX_DMA_FPORT_SHIFT_V2; + break; + } + data |= TX_DMA_SWC_V2 | QID_BITS_V2(info->qid); WRITE_ONCE(desc->txd4, data); @@ -1268,6 +1349,8 @@ static void mtk_tx_set_dma_desc_v2(struct net_device *dev, void *txd, /* tx checksum offload */ if (info->csum) data |= TX_DMA_CHKSUM_V2; + if (mtk_is_netsys_v3_or_greater(eth) && netdev_uses_dsa(dev)) + data |= TX_DMA_SPTAG_V3; } WRITE_ONCE(desc->txd5, data); @@ -1286,7 +1369,7 @@ static void mtk_tx_set_dma_desc(struct net_device *dev, void *txd, struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) mtk_tx_set_dma_desc_v2(dev, txd, info); else mtk_tx_set_dma_desc_v1(dev, txd, info); @@ -1333,8 +1416,7 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, mtk_tx_set_dma_desc(dev, itxd, &txd_info); itx_buf->flags |= MTK_TX_FLAGS_SINGLE0; - itx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 : - MTK_TX_FLAGS_FPORT1; + itx_buf->mac_id = mac->id; setup_tx_buf(eth, itx_buf, itxd_pdma, txd_info.addr, txd_info.size, k++); @@ -1382,8 +1464,7 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, memset(tx_buf, 0, sizeof(*tx_buf)); tx_buf->data = (void *)MTK_DMA_DUMMY_DESC; tx_buf->flags |= MTK_TX_FLAGS_PAGE0; - tx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 : - MTK_TX_FLAGS_FPORT1; + tx_buf->mac_id = mac->id; setup_tx_buf(eth, tx_buf, txd_pdma, txd_info.addr, txd_info.size, k++); @@ -1468,7 +1549,7 @@ static int mtk_queue_stopped(struct mtk_eth *eth) { int i; - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i]) continue; if (netif_queue_stopped(eth->netdev[i])) @@ -1482,7 +1563,7 @@ static void mtk_wake_queue(struct mtk_eth *eth) { int i; - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i]) continue; netif_tx_wake_all_queues(eth->netdev[i]); @@ -1593,7 +1674,7 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth) static bool mtk_page_pool_enabled(struct mtk_eth *eth) { - return MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2); + return mtk_is_netsys_v2_or_greater(eth); } static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth, @@ -1685,7 +1766,7 @@ static int mtk_xdp_frame_map(struct mtk_eth *eth, struct net_device *dev, } mtk_tx_set_dma_desc(dev, txd, txd_info); - tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1; + tx_buf->mac_id = mac->id; tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX; tx_buf->data = (void *)MTK_DMA_DUMMY_DESC; @@ -1935,13 +2016,26 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, break; /* find out which mac the packet come from. values start at 1 */ - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) - mac = RX_DMA_GET_SPORT_V2(trxd.rxd5) - 1; - else if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) && - !(trxd.rxd4 & RX_DMA_SPECIAL_TAG)) + if (mtk_is_netsys_v2_or_greater(eth)) { + u32 val = RX_DMA_GET_SPORT_V2(trxd.rxd5); + + switch (val) { + case PSE_GDM1_PORT: + case PSE_GDM2_PORT: + mac = val - 1; + break; + case PSE_GDM3_PORT: + mac = MTK_GMAC3_ID; + break; + default: + break; + } + } else if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) && + !(trxd.rxd4 & RX_DMA_SPECIAL_TAG)) { mac = RX_DMA_GET_SPORT(trxd.rxd4) - 1; + } - if (unlikely(mac < 0 || mac >= MTK_MAC_COUNT || + if (unlikely(mac < 0 || mac >= MTK_MAX_DEVS || !eth->netdev[mac])) goto release_desc; @@ -2031,7 +2125,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skb->dev = netdev; bytes += skb->len; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { reason = FIELD_GET(MTK_RXD5_PPE_CPU_REASON, trxd.rxd5); hash = trxd.rxd5 & MTK_RXD5_FOE_ENTRY; if (hash != MTK_RXD5_FOE_ENTRY) @@ -2056,8 +2150,8 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, /* When using VLAN untagging in combination with DSA, the * hardware treats the MTK special tag as a VLAN and untags it. */ - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) && - (trxd.rxd2 & RX_DMA_VTAG) && netdev_uses_dsa(netdev)) { + if (mtk_is_netsys_v1(eth) && (trxd.rxd2 & RX_DMA_VTAG) && + netdev_uses_dsa(netdev)) { unsigned int port = RX_DMA_VPID(trxd.rxd3) & GENMASK(2, 0); if (port < ARRAY_SIZE(eth->dsa_meta) && @@ -2161,7 +2255,6 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, while ((cpu != dma) && budget) { u32 next_cpu = desc->txd2; - int mac = 0; desc = mtk_qdma_phys_to_virt(ring, desc->txd2); if ((desc->txd3 & TX_DMA_OWNER_CPU) == 0) @@ -2169,15 +2262,13 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, tx_buf = mtk_desc_to_tx_buf(ring, desc, eth->soc->txrx.txd_size); - if (tx_buf->flags & MTK_TX_FLAGS_FPORT1) - mac = 1; - if (!tx_buf->data) break; if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { if (tx_buf->type == MTK_TYPE_SKB) - mtk_poll_tx_done(eth, state, mac, tx_buf->data); + mtk_poll_tx_done(eth, state, tx_buf->mac_id, + tx_buf->data); budget--; } @@ -2367,7 +2458,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth) txd->txd2 = next_ptr; txd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU; txd->txd4 = 0; - if (MTK_HAS_CAPS(soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { txd->txd5 = 0; txd->txd6 = 0; txd->txd7 = 0; @@ -2420,14 +2511,14 @@ static int mtk_tx_alloc(struct mtk_eth *eth) FIELD_PREP(MTK_QTX_SCH_MIN_RATE_MAN, 1) | FIELD_PREP(MTK_QTX_SCH_MIN_RATE_EXP, 4) | MTK_QTX_SCH_LEAKY_BUCKET_SIZE; - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v1(eth)) val |= MTK_QTX_SCH_LEAKY_BUCKET_EN; mtk_w32(eth, val, soc->reg_map->qdma.qtx_sch + ofs); ofs += MTK_QTX_OFFSET; } val = MTK_QDMA_TX_SCH_MAX_WFQ | (MTK_QDMA_TX_SCH_MAX_WFQ << 16); mtk_w32(eth, val, soc->reg_map->qdma.tx_sch_rate); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) mtk_w32(eth, val, soc->reg_map->qdma.tx_sch_rate + 4); } else { mtk_w32(eth, ring->phys_pdma, MT7628_TX_BASE_PTR0); @@ -2556,7 +2647,7 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag) rxd->rxd3 = 0; rxd->rxd4 = 0; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { rxd->rxd5 = 0; rxd->rxd6 = 0; rxd->rxd7 = 0; @@ -2978,7 +3069,7 @@ static void mtk_dma_free(struct mtk_eth *eth) const struct mtk_soc_data *soc = eth->soc; int i; - for (i = 0; i < MTK_MAC_COUNT; i++) + for (i = 0; i < MTK_MAX_DEVS; i++) if (eth->netdev[i]) netdev_reset_queue(eth->netdev[i]); if (eth->scratch_ring) { @@ -3104,7 +3195,7 @@ static int mtk_start_dma(struct mtk_eth *eth) MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO | MTK_RX_2B_OFFSET | MTK_TX_WB_DDONE; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) val |= MTK_MUTLI_CNT | MTK_RESV_BUF | MTK_WCOMP_EN | MTK_DMAD_WR_WDONE | MTK_CHK_DDONE_EN | MTK_LEAKY_BUCKET_EN; @@ -3132,8 +3223,13 @@ static void mtk_gdm_config(struct mtk_eth *eth, u32 config) if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) return; - for (i = 0; i < MTK_MAC_COUNT; i++) { - u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i)); + for (i = 0; i < MTK_MAX_DEVS; i++) { + u32 val; + + if (!eth->netdev[i]) + continue; + + val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i)); /* default setup the forward port to send frame to PDMA */ val &= ~0xffff; @@ -3143,7 +3239,7 @@ static void mtk_gdm_config(struct mtk_eth *eth, u32 config) val |= config; - if (eth->netdev[i] && netdev_uses_dsa(eth->netdev[i])) + if (netdev_uses_dsa(eth->netdev[i])) val |= MTK_GDMA_SPECIAL_TAG; mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i)); @@ -3250,7 +3346,7 @@ static int mtk_open(struct net_device *dev) phylink_start(mac->phylink); netif_tx_start_all_queues(dev); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return 0; if (mtk_uses_dsa(dev) && !eth->prog) { @@ -3516,7 +3612,7 @@ static void mtk_hw_reset(struct mtk_eth *eth) { u32 val; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { regmap_write(eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN, 0); val = RSTCTRL_PPE0_V2; } else { @@ -3528,7 +3624,7 @@ static void mtk_hw_reset(struct mtk_eth *eth) ethsys_reset(eth, RSTCTRL_ETH | RSTCTRL_FE | val); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) regmap_write(eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN, 0x3ffffff); } @@ -3554,7 +3650,7 @@ static void mtk_hw_warm_reset(struct mtk_eth *eth) return; } - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) rst_mask = RSTCTRL_ETH | RSTCTRL_PPE0_V2; else rst_mask = RSTCTRL_ETH | RSTCTRL_PPE0; @@ -3724,7 +3820,7 @@ static int mtk_hw_init(struct mtk_eth *eth, bool reset) else mtk_hw_reset(eth); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { /* Set FE to PDMAv2 if necessary */ val = mtk_r32(eth, MTK_FE_GLO_MISC); mtk_w32(eth, val | BIT(4), MTK_FE_GLO_MISC); @@ -3745,15 +3841,15 @@ static int mtk_hw_init(struct mtk_eth *eth, bool reset) * up with the more appropriate value when mtk_mac_config call is being * invoked. */ - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { struct net_device *dev = eth->netdev[i]; - mtk_w32(eth, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(i)); - if (dev) { - struct mtk_mac *mac = netdev_priv(dev); + if (!dev) + continue; - mtk_set_mcr_max_rx(mac, dev->mtu + MTK_RX_ETH_HLEN); - } + mtk_w32(eth, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(i)); + mtk_set_mcr_max_rx(netdev_priv(dev), + dev->mtu + MTK_RX_ETH_HLEN); } /* Indicates CDM to parse the MTK special tag from CPU @@ -3761,7 +3857,7 @@ static int mtk_hw_init(struct mtk_eth *eth, bool reset) */ val = mtk_r32(eth, MTK_CDMQ_IG_CTRL); mtk_w32(eth, val | MTK_CDMQ_STAG_EN, MTK_CDMQ_IG_CTRL); - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v1(eth)) { val = mtk_r32(eth, MTK_CDMP_IG_CTRL); mtk_w32(eth, val | MTK_CDMP_STAG_EN, MTK_CDMP_IG_CTRL); @@ -3783,7 +3879,24 @@ static int mtk_hw_init(struct mtk_eth *eth, bool reset) mtk_w32(eth, eth->soc->txrx.rx_irq_done_mask, reg_map->qdma.int_grp + 4); mtk_w32(eth, 0x21021000, MTK_FE_INT_GRP); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v3_or_greater(eth)) { + /* PSE should not drop port1, port8 and port9 packets */ + mtk_w32(eth, 0x00000302, PSE_DROP_CFG); + + /* GDM and CDM Threshold */ + mtk_w32(eth, 0x00000707, MTK_CDMW0_THRES); + mtk_w32(eth, 0x00000077, MTK_CDMW1_THRES); + + /* Disable GDM1 RX CRC stripping */ + mtk_m32(eth, MTK_GDMA_STRP_CRC, 0, MTK_GDMA_FWD_CFG(0)); + + /* PSE GDM3 MIB counter has incorrect hw default values, + * so the driver ought to read clear the values beforehand + * in case ethtool retrieve wrong mib values. + */ + for (i = 0; i < 0x80; i += 0x4) + mtk_r32(eth, reg_map->gdm1_cnt + 0x100 + i); + } else if (!mtk_is_netsys_v1(eth)) { /* PSE should not drop port8 and port9 packets from WDMA Tx */ mtk_w32(eth, 0x00000300, PSE_DROP_CFG); @@ -3933,7 +4046,7 @@ static void mtk_pending_work(struct work_struct *work) mtk_prepare_for_reset(eth); /* stop all devices to make sure that dma is properly shut down */ - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i] || !netif_running(eth->netdev[i])) continue; @@ -3949,8 +4062,8 @@ static void mtk_pending_work(struct work_struct *work) mtk_hw_init(eth, true); /* restart DMA and enable IRQs */ - for (i = 0; i < MTK_MAC_COUNT; i++) { - if (!test_bit(i, &restart)) + for (i = 0; i < MTK_MAX_DEVS; i++) { + if (!eth->netdev[i] || !test_bit(i, &restart)) continue; if (mtk_open(eth->netdev[i])) { @@ -3977,7 +4090,7 @@ static int mtk_free_dev(struct mtk_eth *eth) { int i; - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i]) continue; free_netdev(eth->netdev[i]); @@ -3996,7 +4109,7 @@ static int mtk_unreg_dev(struct mtk_eth *eth) { int i; - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { struct mtk_mac *mac; if (!eth->netdev[i]) continue; @@ -4298,7 +4411,7 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) } id = be32_to_cpup(_id); - if (id >= MTK_MAC_COUNT) { + if (id >= MTK_MAX_DEVS) { dev_err(eth->dev, "%d is not a valid mac id\n", id); return -EINVAL; } @@ -4346,7 +4459,11 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) } spin_lock_init(&mac->hw_stats->stats_lock); u64_stats_init(&mac->hw_stats->syncp); - mac->hw_stats->reg_offset = id * MTK_STAT_OFFSET; + + if (mtk_is_netsys_v3_or_greater(eth)) + mac->hw_stats->reg_offset = id * 0x80; + else + mac->hw_stats->reg_offset = id * 0x40; /* phylink create */ err = of_get_phy_mode(np, &phy_mode); @@ -4361,18 +4478,22 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.dev = ð->netdev[id]->dev; mac->phylink_config.type = PHYLINK_NETDEV; - /* This driver makes use of state->speed in mac_config */ - mac->phylink_config.legacy_pre_march2020 = true; mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD; - __set_bit(PHY_INTERFACE_MODE_MII, - mac->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_GMII, - mac->phylink_config.supported_interfaces); + /* MT7623 gmac0 is now missing its speed-specific PLL configuration + * in its .mac_config method (since state->speed is not valid there. + * Disable support for MII, GMII and RGMII. + */ + if (!mac->hw->soc->disable_pll_modes || mac->id != 0) { + __set_bit(PHY_INTERFACE_MODE_MII, + mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + mac->phylink_config.supported_interfaces); - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII)) - phy_interface_set_rgmii(mac->phylink_config.supported_interfaces); + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII)) + phy_interface_set_rgmii(mac->phylink_config.supported_interfaces); + } if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) && !mac->id) __set_bit(PHY_INTERFACE_MODE_TRGMII, @@ -4396,6 +4517,17 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.supported_interfaces); } + if (mtk_is_netsys_v3_or_greater(mac->hw) && + MTK_HAS_CAPS(mac->hw->soc->caps, MTK_ESW_BIT) && + id == MTK_GMAC1_ID) { + mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_SYM_PAUSE | + MAC_10000FD; + phy_interface_zero(mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + mac->phylink_config.supported_interfaces); + } + phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), phy_mode, &mtk_phylink_ops); @@ -4454,7 +4586,7 @@ void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev) rtnl_lock(); - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { dev = eth->netdev[i]; if (!dev || !(dev->flags & IFF_UP)) @@ -4584,7 +4716,7 @@ static int mtk_probe(struct platform_device *pdev) } } - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { err = -EINVAL; @@ -4692,9 +4824,8 @@ static int mtk_probe(struct platform_device *pdev) } if (eth->soc->offload_version) { - u32 num_ppe; + u32 num_ppe = mtk_is_netsys_v2_or_greater(eth) ? 2 : 1; - num_ppe = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1; num_ppe = min_t(u32, ARRAY_SIZE(eth->ppe), num_ppe); for (i = 0; i < num_ppe; i++) { u32 ppe_addr = eth->soc->reg_map->ppe_base + i * 0x400; @@ -4761,7 +4892,7 @@ static int mtk_remove(struct platform_device *pdev) int i; /* stop all devices to make sure that dma is properly shut down */ - for (i = 0; i < MTK_MAC_COUNT; i++) { + for (i = 0; i < MTK_MAX_DEVS; i++) { if (!eth->netdev[i]) continue; mtk_stop(eth->netdev[i]); @@ -4786,6 +4917,7 @@ static const struct mtk_soc_data mt2701_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7623_CLKS_BITMAP, .required_pctl = true, + .version = 1, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4802,9 +4934,10 @@ static const struct mtk_soc_data mt7621_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7621_CLKS_BITMAP, .required_pctl = false, + .version = 1, .offload_version = 1, .hash_offset = 2, - .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, + .foe_entry_size = MTK_FOE_ENTRY_V1_SIZE, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4822,10 +4955,11 @@ static const struct mtk_soc_data mt7622_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7622_CLKS_BITMAP, .required_pctl = false, + .version = 1, .offload_version = 2, .hash_offset = 2, .has_accounting = true, - .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, + .foe_entry_size = MTK_FOE_ENTRY_V1_SIZE, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4842,9 +4976,11 @@ static const struct mtk_soc_data mt7623_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7623_CLKS_BITMAP, .required_pctl = true, + .version = 1, .offload_version = 1, .hash_offset = 2, - .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, + .foe_entry_size = MTK_FOE_ENTRY_V1_SIZE, + .disable_pll_modes = true, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4863,6 +4999,7 @@ static const struct mtk_soc_data mt7629_data = { .required_clks = MT7629_CLKS_BITMAP, .required_pctl = false, .has_accounting = true, + .version = 1, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4880,10 +5017,11 @@ static const struct mtk_soc_data mt7981_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7981_CLKS_BITMAP, .required_pctl = false, + .version = 2, .offload_version = 2, .hash_offset = 4, - .foe_entry_size = sizeof(struct mtk_foe_entry), .has_accounting = true, + .foe_entry_size = MTK_FOE_ENTRY_V2_SIZE, .txrx = { .txd_size = sizeof(struct mtk_tx_dma_v2), .rxd_size = sizeof(struct mtk_rx_dma_v2), @@ -4901,10 +5039,33 @@ static const struct mtk_soc_data mt7986_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7986_CLKS_BITMAP, .required_pctl = false, + .version = 2, + .offload_version = 2, + .hash_offset = 4, + .has_accounting = true, + .foe_entry_size = MTK_FOE_ENTRY_V2_SIZE, + .txrx = { + .txd_size = sizeof(struct mtk_tx_dma_v2), + .rxd_size = sizeof(struct mtk_rx_dma_v2), + .rx_irq_done_mask = MTK_RX_DONE_INT_V2, + .rx_dma_l4_valid = RX_DMA_L4_VALID_V2, + .dma_max_len = MTK_TX_DMA_BUF_LEN_V2, + .dma_len_offset = 8, + }, +}; + +static const struct mtk_soc_data mt7988_data = { + .reg_map = &mt7988_reg_map, + .ana_rgc3 = 0x128, + .caps = MT7988_CAPS, + .hw_features = MTK_HW_FEATURES, + .required_clks = MT7988_CLKS_BITMAP, + .required_pctl = false, + .version = 3, .offload_version = 2, .hash_offset = 4, - .foe_entry_size = sizeof(struct mtk_foe_entry), .has_accounting = true, + .foe_entry_size = MTK_FOE_ENTRY_V3_SIZE, .txrx = { .txd_size = sizeof(struct mtk_tx_dma_v2), .rxd_size = sizeof(struct mtk_rx_dma_v2), @@ -4921,6 +5082,7 @@ static const struct mtk_soc_data rt5350_data = { .hw_features = MTK_HW_FEATURES_MT7628, .required_clks = MT7628_CLKS_BITMAP, .required_pctl = false, + .version = 1, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4932,14 +5094,15 @@ static const struct mtk_soc_data rt5350_data = { }; const struct of_device_id of_mtk_match[] = { - { .compatible = "mediatek,mt2701-eth", .data = &mt2701_data}, - { .compatible = "mediatek,mt7621-eth", .data = &mt7621_data}, - { .compatible = "mediatek,mt7622-eth", .data = &mt7622_data}, - { .compatible = "mediatek,mt7623-eth", .data = &mt7623_data}, - { .compatible = "mediatek,mt7629-eth", .data = &mt7629_data}, - { .compatible = "mediatek,mt7981-eth", .data = &mt7981_data}, - { .compatible = "mediatek,mt7986-eth", .data = &mt7986_data}, - { .compatible = "ralink,rt5350-eth", .data = &rt5350_data}, + { .compatible = "mediatek,mt2701-eth", .data = &mt2701_data }, + { .compatible = "mediatek,mt7621-eth", .data = &mt7621_data }, + { .compatible = "mediatek,mt7622-eth", .data = &mt7622_data }, + { .compatible = "mediatek,mt7623-eth", .data = &mt7623_data }, + { .compatible = "mediatek,mt7629-eth", .data = &mt7629_data }, + { .compatible = "mediatek,mt7981-eth", .data = &mt7981_data }, + { .compatible = "mediatek,mt7986-eth", .data = &mt7986_data }, + { .compatible = "mediatek,mt7988-eth", .data = &mt7988_data }, + { .compatible = "ralink,rt5350-eth", .data = &rt5350_data }, {}, }; MODULE_DEVICE_TABLE(of, of_mtk_match); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 707445f6bcb1..80d17729e557 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -33,7 +33,6 @@ #define MTK_TX_DMA_BUF_LEN_V2 0xffff #define MTK_QDMA_RING_SIZE 2048 #define MTK_DMA_SIZE 512 -#define MTK_MAC_COUNT 2 #define MTK_RX_ETH_HLEN (ETH_HLEN + ETH_FCS_LEN) #define MTK_RX_HLEN (NET_SKB_PAD + MTK_RX_ETH_HLEN + NET_IP_ALIGN) #define MTK_DMA_DUMMY_DESC 0xffffffff @@ -118,14 +117,21 @@ #define MTK_CDMP_EG_CTRL 0x404 /* GDM Exgress Control Register */ -#define MTK_GDMA_FWD_CFG(x) (0x500 + (x * 0x1000)) +#define MTK_GDMA_FWD_CFG(x) ({ typeof(x) _x = (x); (_x == MTK_GMAC3_ID) ? \ + 0x540 : 0x500 + (_x * 0x1000); }) #define MTK_GDMA_SPECIAL_TAG BIT(24) #define MTK_GDMA_ICS_EN BIT(22) #define MTK_GDMA_TCS_EN BIT(21) #define MTK_GDMA_UCS_EN BIT(20) +#define MTK_GDMA_STRP_CRC BIT(16) #define MTK_GDMA_TO_PDMA 0x0 #define MTK_GDMA_DROP_ALL 0x7777 +/* GDM Egress Control Register */ +#define MTK_GDMA_EG_CTRL(x) ({ typeof(x) _x = (x); (_x == MTK_GMAC3_ID) ? \ + 0x544 : 0x504 + (_x * 0x1000); }) +#define MTK_GDMA_XGDM_SEL BIT(31) + /* Unicast Filter MAC Address Register - Low */ #define MTK_GDMA_MAC_ADRL(x) (0x508 + (x * 0x1000)) @@ -288,8 +294,6 @@ /* QDMA Interrupt grouping registers */ #define MTK_RLS_DONE_INT BIT(0) -#define MTK_STAT_OFFSET 0x40 - /* QDMA TX NUM */ #define QID_BITS_V2(x) (((x) & 0x3f) << 16) #define MTK_QDMA_GMAC2_QID 8 @@ -302,6 +306,8 @@ #define TX_DMA_CHKSUM_V2 (0x7 << 28) #define TX_DMA_TSO_V2 BIT(31) +#define TX_DMA_SPTAG_V3 BIT(27) + /* QDMA V2 descriptor txd4 */ #define TX_DMA_FPORT_SHIFT_V2 8 #define TX_DMA_FPORT_MASK_V2 0xf @@ -389,7 +395,26 @@ #define PHY_IAC_TIMEOUT HZ #define MTK_MAC_MISC 0x1000c +#define MTK_MAC_MISC_V3 0x10010 #define MTK_MUX_TO_ESW BIT(0) +#define MISC_MDC_TURBO BIT(4) + +/* XMAC status registers */ +#define MTK_XGMAC_STS(x) (((x) == MTK_GMAC3_ID) ? 0x1001C : 0x1000C) +#define MTK_XGMAC_FORCE_LINK(x) (((x) == MTK_GMAC2_ID) ? BIT(31) : BIT(15)) +#define MTK_USXGMII_PCS_LINK BIT(8) +#define MTK_XGMAC_RX_FC BIT(5) +#define MTK_XGMAC_TX_FC BIT(4) +#define MTK_USXGMII_PCS_MODE GENMASK(3, 1) +#define MTK_XGMAC_LINK_STS BIT(0) + +/* GSW bridge registers */ +#define MTK_GSW_CFG (0x10080) +#define GSWTX_IPG_MASK GENMASK(19, 16) +#define GSWTX_IPG_SHIFT 16 +#define GSWRX_IPG_MASK GENMASK(3, 0) +#define GSWRX_IPG_SHIFT 0 +#define GSW_IPG_11 11 /* Mac control registers */ #define MTK_MAC_MCR(x) (0x10100 + (x * 0x100)) @@ -635,12 +660,6 @@ enum mtk_tx_flags { */ MTK_TX_FLAGS_SINGLE0 = 0x01, MTK_TX_FLAGS_PAGE0 = 0x02, - - /* MTK_TX_FLAGS_FPORTx allows tracking which port the transmitted - * SKB out instead of looking up through hardware TX descriptor. - */ - MTK_TX_FLAGS_FPORT0 = 0x04, - MTK_TX_FLAGS_FPORT1 = 0x08, }; /* This enum allows us to identify how the clock is defined on the array of the @@ -653,6 +672,11 @@ enum mtk_clks_map { MTK_CLK_GP0, MTK_CLK_GP1, MTK_CLK_GP2, + MTK_CLK_GP3, + MTK_CLK_XGP1, + MTK_CLK_XGP2, + MTK_CLK_XGP3, + MTK_CLK_CRYPTO, MTK_CLK_FE, MTK_CLK_TRGPLL, MTK_CLK_SGMII_TX_250M, @@ -669,63 +693,145 @@ enum mtk_clks_map { MTK_CLK_WOCPU1, MTK_CLK_NETSYS0, MTK_CLK_NETSYS1, + MTK_CLK_ETHWARP_WOCPU2, + MTK_CLK_ETHWARP_WOCPU1, + MTK_CLK_ETHWARP_WOCPU0, + MTK_CLK_TOP_USXGMII_SBUS_0_SEL, + MTK_CLK_TOP_USXGMII_SBUS_1_SEL, + MTK_CLK_TOP_SGM_0_SEL, + MTK_CLK_TOP_SGM_1_SEL, + MTK_CLK_TOP_XFI_PHY_0_XTAL_SEL, + MTK_CLK_TOP_XFI_PHY_1_XTAL_SEL, + MTK_CLK_TOP_ETH_GMII_SEL, + MTK_CLK_TOP_ETH_REFCK_50M_SEL, + MTK_CLK_TOP_ETH_SYS_200M_SEL, + MTK_CLK_TOP_ETH_SYS_SEL, + MTK_CLK_TOP_ETH_XGMII_SEL, + MTK_CLK_TOP_ETH_MII_SEL, + MTK_CLK_TOP_NETSYS_SEL, + MTK_CLK_TOP_NETSYS_500M_SEL, + MTK_CLK_TOP_NETSYS_PAO_2X_SEL, + MTK_CLK_TOP_NETSYS_SYNC_250M_SEL, + MTK_CLK_TOP_NETSYS_PPEFB_250M_SEL, + MTK_CLK_TOP_NETSYS_WARP_SEL, MTK_CLK_MAX }; -#define MT7623_CLKS_BITMAP (BIT(MTK_CLK_ETHIF) | BIT(MTK_CLK_ESW) | \ - BIT(MTK_CLK_GP1) | BIT(MTK_CLK_GP2) | \ - BIT(MTK_CLK_TRGPLL)) -#define MT7622_CLKS_BITMAP (BIT(MTK_CLK_ETHIF) | BIT(MTK_CLK_ESW) | \ - BIT(MTK_CLK_GP0) | BIT(MTK_CLK_GP1) | \ - BIT(MTK_CLK_GP2) | \ - BIT(MTK_CLK_SGMII_TX_250M) | \ - BIT(MTK_CLK_SGMII_RX_250M) | \ - BIT(MTK_CLK_SGMII_CDR_REF) | \ - BIT(MTK_CLK_SGMII_CDR_FB) | \ - BIT(MTK_CLK_SGMII_CK) | \ - BIT(MTK_CLK_ETH2PLL)) +#define MT7623_CLKS_BITMAP (BIT_ULL(MTK_CLK_ETHIF) | BIT_ULL(MTK_CLK_ESW) | \ + BIT_ULL(MTK_CLK_GP1) | BIT_ULL(MTK_CLK_GP2) | \ + BIT_ULL(MTK_CLK_TRGPLL)) +#define MT7622_CLKS_BITMAP (BIT_ULL(MTK_CLK_ETHIF) | BIT_ULL(MTK_CLK_ESW) | \ + BIT_ULL(MTK_CLK_GP0) | BIT_ULL(MTK_CLK_GP1) | \ + BIT_ULL(MTK_CLK_GP2) | \ + BIT_ULL(MTK_CLK_SGMII_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII_CK) | \ + BIT_ULL(MTK_CLK_ETH2PLL)) #define MT7621_CLKS_BITMAP (0) #define MT7628_CLKS_BITMAP (0) -#define MT7629_CLKS_BITMAP (BIT(MTK_CLK_ETHIF) | BIT(MTK_CLK_ESW) | \ - BIT(MTK_CLK_GP0) | BIT(MTK_CLK_GP1) | \ - BIT(MTK_CLK_GP2) | BIT(MTK_CLK_FE) | \ - BIT(MTK_CLK_SGMII_TX_250M) | \ - BIT(MTK_CLK_SGMII_RX_250M) | \ - BIT(MTK_CLK_SGMII_CDR_REF) | \ - BIT(MTK_CLK_SGMII_CDR_FB) | \ - BIT(MTK_CLK_SGMII2_TX_250M) | \ - BIT(MTK_CLK_SGMII2_RX_250M) | \ - BIT(MTK_CLK_SGMII2_CDR_REF) | \ - BIT(MTK_CLK_SGMII2_CDR_FB) | \ - BIT(MTK_CLK_SGMII_CK) | \ - BIT(MTK_CLK_ETH2PLL) | BIT(MTK_CLK_SGMIITOP)) -#define MT7981_CLKS_BITMAP (BIT(MTK_CLK_FE) | BIT(MTK_CLK_GP2) | BIT(MTK_CLK_GP1) | \ - BIT(MTK_CLK_WOCPU0) | \ - BIT(MTK_CLK_SGMII_TX_250M) | \ - BIT(MTK_CLK_SGMII_RX_250M) | \ - BIT(MTK_CLK_SGMII_CDR_REF) | \ - BIT(MTK_CLK_SGMII_CDR_FB) | \ - BIT(MTK_CLK_SGMII2_TX_250M) | \ - BIT(MTK_CLK_SGMII2_RX_250M) | \ - BIT(MTK_CLK_SGMII2_CDR_REF) | \ - BIT(MTK_CLK_SGMII2_CDR_FB) | \ - BIT(MTK_CLK_SGMII_CK)) -#define MT7986_CLKS_BITMAP (BIT(MTK_CLK_FE) | BIT(MTK_CLK_GP2) | BIT(MTK_CLK_GP1) | \ - BIT(MTK_CLK_WOCPU1) | BIT(MTK_CLK_WOCPU0) | \ - BIT(MTK_CLK_SGMII_TX_250M) | \ - BIT(MTK_CLK_SGMII_RX_250M) | \ - BIT(MTK_CLK_SGMII_CDR_REF) | \ - BIT(MTK_CLK_SGMII_CDR_FB) | \ - BIT(MTK_CLK_SGMII2_TX_250M) | \ - BIT(MTK_CLK_SGMII2_RX_250M) | \ - BIT(MTK_CLK_SGMII2_CDR_REF) | \ - BIT(MTK_CLK_SGMII2_CDR_FB)) +#define MT7629_CLKS_BITMAP (BIT_ULL(MTK_CLK_ETHIF) | BIT_ULL(MTK_CLK_ESW) | \ + BIT_ULL(MTK_CLK_GP0) | BIT_ULL(MTK_CLK_GP1) | \ + BIT_ULL(MTK_CLK_GP2) | BIT_ULL(MTK_CLK_FE) | \ + BIT_ULL(MTK_CLK_SGMII_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII2_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII_CK) | \ + BIT_ULL(MTK_CLK_ETH2PLL) | BIT_ULL(MTK_CLK_SGMIITOP)) +#define MT7981_CLKS_BITMAP (BIT_ULL(MTK_CLK_FE) | BIT_ULL(MTK_CLK_GP2) | \ + BIT_ULL(MTK_CLK_GP1) | \ + BIT_ULL(MTK_CLK_WOCPU0) | \ + BIT_ULL(MTK_CLK_SGMII_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII2_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII_CK)) +#define MT7986_CLKS_BITMAP (BIT_ULL(MTK_CLK_FE) | BIT_ULL(MTK_CLK_GP2) | \ + BIT_ULL(MTK_CLK_GP1) | \ + BIT_ULL(MTK_CLK_WOCPU1) | BIT_ULL(MTK_CLK_WOCPU0) | \ + BIT_ULL(MTK_CLK_SGMII_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII_CDR_FB) | \ + BIT_ULL(MTK_CLK_SGMII2_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_REF) | \ + BIT_ULL(MTK_CLK_SGMII2_CDR_FB)) +#define MT7988_CLKS_BITMAP (BIT_ULL(MTK_CLK_FE) | BIT_ULL(MTK_CLK_ESW) | \ + BIT_ULL(MTK_CLK_GP1) | BIT_ULL(MTK_CLK_GP2) | \ + BIT_ULL(MTK_CLK_GP3) | BIT_ULL(MTK_CLK_XGP1) | \ + BIT_ULL(MTK_CLK_XGP2) | BIT_ULL(MTK_CLK_XGP3) | \ + BIT_ULL(MTK_CLK_CRYPTO) | \ + BIT_ULL(MTK_CLK_SGMII_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII_RX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_TX_250M) | \ + BIT_ULL(MTK_CLK_SGMII2_RX_250M) | \ + BIT_ULL(MTK_CLK_ETHWARP_WOCPU2) | \ + BIT_ULL(MTK_CLK_ETHWARP_WOCPU1) | \ + BIT_ULL(MTK_CLK_ETHWARP_WOCPU0) | \ + BIT_ULL(MTK_CLK_TOP_USXGMII_SBUS_0_SEL) | \ + BIT_ULL(MTK_CLK_TOP_USXGMII_SBUS_1_SEL) | \ + BIT_ULL(MTK_CLK_TOP_SGM_0_SEL) | \ + BIT_ULL(MTK_CLK_TOP_SGM_1_SEL) | \ + BIT_ULL(MTK_CLK_TOP_XFI_PHY_0_XTAL_SEL) | \ + BIT_ULL(MTK_CLK_TOP_XFI_PHY_1_XTAL_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_GMII_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_REFCK_50M_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_SYS_200M_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_SYS_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_XGMII_SEL) | \ + BIT_ULL(MTK_CLK_TOP_ETH_MII_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_500M_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_PAO_2X_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_SYNC_250M_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_PPEFB_250M_SEL) | \ + BIT_ULL(MTK_CLK_TOP_NETSYS_WARP_SEL)) enum mtk_dev_state { MTK_HW_INIT, MTK_RESETTING }; +/* PSE Port Definition */ +enum mtk_pse_port { + PSE_ADMA_PORT = 0, + PSE_GDM1_PORT, + PSE_GDM2_PORT, + PSE_PPE0_PORT, + PSE_PPE1_PORT, + PSE_QDMA_TX_PORT, + PSE_QDMA_RX_PORT, + PSE_DROP_PORT, + PSE_WDMA0_PORT, + PSE_WDMA1_PORT, + PSE_TDMA_PORT, + PSE_NONE_PORT, + PSE_PPE2_PORT, + PSE_WDMA2_PORT, + PSE_EIP197_PORT, + PSE_GDM3_PORT, + PSE_PORT_MAX +}; + +/* GMAC Identifier */ +enum mtk_gmac_id { + MTK_GMAC1_ID = 0, + MTK_GMAC2_ID, + MTK_GMAC3_ID, + MTK_GMAC_ID_MAX +}; + enum mtk_tx_buf_type { MTK_TYPE_SKB, MTK_TYPE_XDP_TX, @@ -744,7 +850,8 @@ struct mtk_tx_buf { enum mtk_tx_buf_type type; void *data; - u32 flags; + u16 mac_id; + u16 flags; DEFINE_DMA_UNMAP_ADDR(dma_addr0); DEFINE_DMA_UNMAP_LEN(dma_len0); DEFINE_DMA_UNMAP_ADDR(dma_addr1); @@ -820,7 +927,6 @@ enum mkt_eth_capabilities { MTK_SHARED_INT_BIT, MTK_TRGMII_MT7621_CLK_BIT, MTK_QDMA_BIT, - MTK_NETSYS_V2_BIT, MTK_SOC_MT7628_BIT, MTK_RSTCTRL_PPE1_BIT, MTK_U3_COPHY_V2_BIT, @@ -843,42 +949,41 @@ enum mkt_eth_capabilities { }; /* Supported hardware group on SoCs */ -#define MTK_RGMII BIT(MTK_RGMII_BIT) -#define MTK_TRGMII BIT(MTK_TRGMII_BIT) -#define MTK_SGMII BIT(MTK_SGMII_BIT) -#define MTK_ESW BIT(MTK_ESW_BIT) -#define MTK_GEPHY BIT(MTK_GEPHY_BIT) -#define MTK_MUX BIT(MTK_MUX_BIT) -#define MTK_INFRA BIT(MTK_INFRA_BIT) -#define MTK_SHARED_SGMII BIT(MTK_SHARED_SGMII_BIT) -#define MTK_HWLRO BIT(MTK_HWLRO_BIT) -#define MTK_SHARED_INT BIT(MTK_SHARED_INT_BIT) -#define MTK_TRGMII_MT7621_CLK BIT(MTK_TRGMII_MT7621_CLK_BIT) -#define MTK_QDMA BIT(MTK_QDMA_BIT) -#define MTK_NETSYS_V2 BIT(MTK_NETSYS_V2_BIT) -#define MTK_SOC_MT7628 BIT(MTK_SOC_MT7628_BIT) -#define MTK_RSTCTRL_PPE1 BIT(MTK_RSTCTRL_PPE1_BIT) -#define MTK_U3_COPHY_V2 BIT(MTK_U3_COPHY_V2_BIT) +#define MTK_RGMII BIT_ULL(MTK_RGMII_BIT) +#define MTK_TRGMII BIT_ULL(MTK_TRGMII_BIT) +#define MTK_SGMII BIT_ULL(MTK_SGMII_BIT) +#define MTK_ESW BIT_ULL(MTK_ESW_BIT) +#define MTK_GEPHY BIT_ULL(MTK_GEPHY_BIT) +#define MTK_MUX BIT_ULL(MTK_MUX_BIT) +#define MTK_INFRA BIT_ULL(MTK_INFRA_BIT) +#define MTK_SHARED_SGMII BIT_ULL(MTK_SHARED_SGMII_BIT) +#define MTK_HWLRO BIT_ULL(MTK_HWLRO_BIT) +#define MTK_SHARED_INT BIT_ULL(MTK_SHARED_INT_BIT) +#define MTK_TRGMII_MT7621_CLK BIT_ULL(MTK_TRGMII_MT7621_CLK_BIT) +#define MTK_QDMA BIT_ULL(MTK_QDMA_BIT) +#define MTK_SOC_MT7628 BIT_ULL(MTK_SOC_MT7628_BIT) +#define MTK_RSTCTRL_PPE1 BIT_ULL(MTK_RSTCTRL_PPE1_BIT) +#define MTK_U3_COPHY_V2 BIT_ULL(MTK_U3_COPHY_V2_BIT) #define MTK_ETH_MUX_GDM1_TO_GMAC1_ESW \ - BIT(MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT) + BIT_ULL(MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT) #define MTK_ETH_MUX_GMAC2_GMAC0_TO_GEPHY \ - BIT(MTK_ETH_MUX_GMAC2_GMAC0_TO_GEPHY_BIT) + BIT_ULL(MTK_ETH_MUX_GMAC2_GMAC0_TO_GEPHY_BIT) #define MTK_ETH_MUX_U3_GMAC2_TO_QPHY \ - BIT(MTK_ETH_MUX_U3_GMAC2_TO_QPHY_BIT) + BIT_ULL(MTK_ETH_MUX_U3_GMAC2_TO_QPHY_BIT) #define MTK_ETH_MUX_GMAC1_GMAC2_TO_SGMII_RGMII \ - BIT(MTK_ETH_MUX_GMAC1_GMAC2_TO_SGMII_RGMII_BIT) + BIT_ULL(MTK_ETH_MUX_GMAC1_GMAC2_TO_SGMII_RGMII_BIT) #define MTK_ETH_MUX_GMAC12_TO_GEPHY_SGMII \ - BIT(MTK_ETH_MUX_GMAC12_TO_GEPHY_SGMII_BIT) + BIT_ULL(MTK_ETH_MUX_GMAC12_TO_GEPHY_SGMII_BIT) /* Supported path present on SoCs */ -#define MTK_ETH_PATH_GMAC1_RGMII BIT(MTK_ETH_PATH_GMAC1_RGMII_BIT) -#define MTK_ETH_PATH_GMAC1_TRGMII BIT(MTK_ETH_PATH_GMAC1_TRGMII_BIT) -#define MTK_ETH_PATH_GMAC1_SGMII BIT(MTK_ETH_PATH_GMAC1_SGMII_BIT) -#define MTK_ETH_PATH_GMAC2_RGMII BIT(MTK_ETH_PATH_GMAC2_RGMII_BIT) -#define MTK_ETH_PATH_GMAC2_SGMII BIT(MTK_ETH_PATH_GMAC2_SGMII_BIT) -#define MTK_ETH_PATH_GMAC2_GEPHY BIT(MTK_ETH_PATH_GMAC2_GEPHY_BIT) -#define MTK_ETH_PATH_GDM1_ESW BIT(MTK_ETH_PATH_GDM1_ESW_BIT) +#define MTK_ETH_PATH_GMAC1_RGMII BIT_ULL(MTK_ETH_PATH_GMAC1_RGMII_BIT) +#define MTK_ETH_PATH_GMAC1_TRGMII BIT_ULL(MTK_ETH_PATH_GMAC1_TRGMII_BIT) +#define MTK_ETH_PATH_GMAC1_SGMII BIT_ULL(MTK_ETH_PATH_GMAC1_SGMII_BIT) +#define MTK_ETH_PATH_GMAC2_RGMII BIT_ULL(MTK_ETH_PATH_GMAC2_RGMII_BIT) +#define MTK_ETH_PATH_GMAC2_SGMII BIT_ULL(MTK_ETH_PATH_GMAC2_SGMII_BIT) +#define MTK_ETH_PATH_GMAC2_GEPHY BIT_ULL(MTK_ETH_PATH_GMAC2_GEPHY_BIT) +#define MTK_ETH_PATH_GDM1_ESW BIT_ULL(MTK_ETH_PATH_GDM1_ESW_BIT) #define MTK_GMAC1_RGMII (MTK_ETH_PATH_GMAC1_RGMII | MTK_RGMII) #define MTK_GMAC1_TRGMII (MTK_ETH_PATH_GMAC1_TRGMII | MTK_TRGMII) @@ -934,11 +1039,13 @@ enum mkt_eth_capabilities { #define MT7981_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | MTK_GMAC2_GEPHY | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ MTK_MUX_U3_GMAC2_TO_QPHY | MTK_U3_COPHY_V2 | \ - MTK_NETSYS_V2 | MTK_RSTCTRL_PPE1) + MTK_RSTCTRL_PPE1) #define MT7986_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ - MTK_NETSYS_V2 | MTK_RSTCTRL_PPE1) + MTK_RSTCTRL_PPE1) + +#define MT7988_CAPS (MTK_GDM1_ESW | MTK_QDMA | MTK_RSTCTRL_PPE1) struct mtk_tx_dma_desc_info { dma_addr_t addr; @@ -1009,6 +1116,7 @@ struct mtk_reg_map { * @required_pctl A bool value to show whether the SoC requires * the extra setup for those pins used by GMAC. * @hash_offset Flow table hash offset. + * @version SoC version. * @foe_entry_size Foe table entry size. * @has_accounting Bool indicating support for accounting of * offloaded flows. @@ -1022,14 +1130,16 @@ struct mtk_reg_map { struct mtk_soc_data { const struct mtk_reg_map *reg_map; u32 ana_rgc3; - u32 caps; - u32 required_clks; + u64 caps; + u64 required_clks; bool required_pctl; u8 offload_version; u8 hash_offset; + u8 version; u16 foe_entry_size; netdev_features_t hw_features; bool has_accounting; + bool disable_pll_modes; struct { u32 txd_size; u32 rxd_size; @@ -1042,8 +1152,8 @@ struct mtk_soc_data { #define MTK_DMA_MONITOR_TIMEOUT msecs_to_jiffies(1000) -/* currently no SoC has more than 2 macs */ -#define MTK_MAX_DEVS 2 +/* currently no SoC has more than 3 macs */ +#define MTK_MAX_DEVS 3 /* struct mtk_eth - This is the main datasructure for holding the state * of the driver @@ -1182,6 +1292,21 @@ struct mtk_mac { /* the struct describing the SoC. these are declared in the soc_xyz.c files */ extern const struct of_device_id of_mtk_match[]; +static inline bool mtk_is_netsys_v1(struct mtk_eth *eth) +{ + return eth->soc->version == 1; +} + +static inline bool mtk_is_netsys_v2_or_greater(struct mtk_eth *eth) +{ + return eth->soc->version > 1; +} + +static inline bool mtk_is_netsys_v3_or_greater(struct mtk_eth *eth) +{ + return eth->soc->version > 2; +} + static inline struct mtk_foe_entry * mtk_foe_get_entry(struct mtk_ppe *ppe, u16 hash) { @@ -1192,7 +1317,7 @@ mtk_foe_get_entry(struct mtk_ppe *ppe, u16 hash) static inline u32 mtk_get_ib1_ts_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB1_BIND_TIMESTAMP_V2; return MTK_FOE_IB1_BIND_TIMESTAMP; @@ -1200,7 +1325,7 @@ static inline u32 mtk_get_ib1_ts_mask(struct mtk_eth *eth) static inline u32 mtk_get_ib1_ppoe_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB1_BIND_PPPOE_V2; return MTK_FOE_IB1_BIND_PPPOE; @@ -1208,7 +1333,7 @@ static inline u32 mtk_get_ib1_ppoe_mask(struct mtk_eth *eth) static inline u32 mtk_get_ib1_vlan_tag_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB1_BIND_VLAN_TAG_V2; return MTK_FOE_IB1_BIND_VLAN_TAG; @@ -1216,7 +1341,7 @@ static inline u32 mtk_get_ib1_vlan_tag_mask(struct mtk_eth *eth) static inline u32 mtk_get_ib1_vlan_layer_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB1_BIND_VLAN_LAYER_V2; return MTK_FOE_IB1_BIND_VLAN_LAYER; @@ -1224,7 +1349,7 @@ static inline u32 mtk_get_ib1_vlan_layer_mask(struct mtk_eth *eth) static inline u32 mtk_prep_ib1_vlan_layer(struct mtk_eth *eth, u32 val) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val); return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, val); @@ -1232,7 +1357,7 @@ static inline u32 mtk_prep_ib1_vlan_layer(struct mtk_eth *eth, u32 val) static inline u32 mtk_get_ib1_vlan_layer(struct mtk_eth *eth, u32 val) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val); return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, val); @@ -1240,7 +1365,7 @@ static inline u32 mtk_get_ib1_vlan_layer(struct mtk_eth *eth, u32 val) static inline u32 mtk_get_ib1_pkt_type_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB1_PACKET_TYPE_V2; return MTK_FOE_IB1_PACKET_TYPE; @@ -1248,7 +1373,7 @@ static inline u32 mtk_get_ib1_pkt_type_mask(struct mtk_eth *eth) static inline u32 mtk_get_ib1_pkt_type(struct mtk_eth *eth, u32 val) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE_V2, val); return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, val); @@ -1256,7 +1381,7 @@ static inline u32 mtk_get_ib1_pkt_type(struct mtk_eth *eth, u32 val) static inline u32 mtk_get_ib2_multicast_mask(struct mtk_eth *eth) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) return MTK_FOE_IB2_MULTICAST_V2; return MTK_FOE_IB2_MULTICAST; @@ -1267,6 +1392,7 @@ void mtk_stats_update_mac(struct mtk_mac *mac); void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg); u32 mtk_r32(struct mtk_eth *eth, unsigned reg); +u32 mtk_m32(struct mtk_eth *eth, u32 mask, u32 set, unsigned int reg); int mtk_gmac_sgmii_path_setup(struct mtk_eth *eth, int mac_id); int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id); diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 9129821f3ab8..86f32f486043 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -92,7 +92,6 @@ static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe) static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets) { - u32 byte_cnt_low, byte_cnt_high, pkt_cnt_low, pkt_cnt_high; u32 val, cnt_r0, cnt_r1, cnt_r2; int ret; @@ -107,12 +106,20 @@ static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *p cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1); cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2); - byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0); - byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1); - pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1); - pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2); - *bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low; - *packets = (pkt_cnt_high << 16) | pkt_cnt_low; + if (mtk_is_netsys_v3_or_greater(ppe->eth)) { + /* 64 bit for each counter */ + u32 cnt_r3 = readl(ppe->base + MTK_PPE_MIB_SER_R3); + *bytes = ((u64)cnt_r1 << 32) | cnt_r0; + *packets = ((u64)cnt_r3 << 32) | cnt_r2; + } else { + /* 48 bit byte counter, 40 bit packet counter */ + u32 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0); + u32 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1); + u32 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1); + u32 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2); + *bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low; + *packets = ((u64)pkt_cnt_high << 16) | pkt_cnt_low; + } return 0; } @@ -208,7 +215,7 @@ int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, memset(entry, 0, sizeof(*entry)); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | @@ -272,7 +279,7 @@ int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, u32 *ib2 = mtk_foe_entry_ib2(eth, entry); u32 val = *ib2; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { val &= ~MTK_FOE_IB2_DEST_PORT_V2; val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); } else { @@ -423,13 +430,22 @@ int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); u32 *ib2 = mtk_foe_entry_ib2(eth, entry); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + switch (eth->soc->version) { + case 3: + *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; + *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | + MTK_FOE_IB2_WDMA_WINFO_V2; + l2->w3info = FIELD_PREP(MTK_FOE_WINFO_WCID_V3, wcid) | + FIELD_PREP(MTK_FOE_WINFO_BSS_V3, bss); + break; + case 2: *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | MTK_FOE_IB2_WDMA_WINFO_V2; l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | FIELD_PREP(MTK_FOE_WINFO_BSS, bss); - } else { + break; + default: *ib2 &= ~MTK_FOE_IB2_PORT_MG; *ib2 |= MTK_FOE_IB2_WDMA_WINFO; if (wdma_idx) @@ -437,6 +453,7 @@ int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); + break; } return 0; @@ -447,7 +464,7 @@ int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry, { u32 *ib2 = mtk_foe_entry_ib2(eth, entry); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { *ib2 &= ~MTK_FOE_IB2_QID_V2; *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue); *ib2 |= MTK_FOE_IB2_PSE_QOS_V2; @@ -603,7 +620,7 @@ __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, struct mtk_foe_entry *hwe; u32 val; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, timestamp); @@ -619,7 +636,7 @@ __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, hwe->ib1 = entry->ib1; if (ppe->accounting) { - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) val = MTK_FOE_IB2_MIB_CNT_V2; else val = MTK_FOE_IB2_MIB_CNT; @@ -964,8 +981,7 @@ void mtk_ppe_start(struct mtk_ppe *ppe) mtk_ppe_init_foe_table(ppe); ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); - val = MTK_PPE_TB_CFG_ENTRY_80B | - MTK_PPE_TB_CFG_AGE_NON_L4 | + val = MTK_PPE_TB_CFG_AGE_NON_L4 | MTK_PPE_TB_CFG_AGE_UNBIND | MTK_PPE_TB_CFG_AGE_TCP | MTK_PPE_TB_CFG_AGE_UDP | @@ -979,8 +995,10 @@ void mtk_ppe_start(struct mtk_ppe *ppe) MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, MTK_PPE_ENTRIES_SHIFT); - if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(ppe->eth)) val |= MTK_PPE_TB_CFG_INFO_SEL; + if (!mtk_is_netsys_v3_or_greater(ppe->eth)) + val |= MTK_PPE_TB_CFG_ENTRY_80B; ppe_w32(ppe, MTK_PPE_TB_CFG, val); ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, @@ -995,7 +1013,7 @@ void mtk_ppe_start(struct mtk_ppe *ppe) MTK_PPE_FLOW_CFG_IP4_NAPT | MTK_PPE_FLOW_CFG_IP4_DSLITE | MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; - if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(ppe->eth)) val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | MTK_PPE_MD_TOAP_BYP_CRSN1 | MTK_PPE_MD_TOAP_BYP_CRSN2 | @@ -1037,7 +1055,7 @@ void mtk_ppe_start(struct mtk_ppe *ppe) ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); - if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(ppe->eth)) { ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index e51de31a52ec..e3d0ec72bc69 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -85,6 +85,17 @@ enum { #define MTK_FOE_WINFO_BSS GENMASK(5, 0) #define MTK_FOE_WINFO_WCID GENMASK(15, 6) +#define MTK_FOE_WINFO_BSS_V3 GENMASK(23, 16) +#define MTK_FOE_WINFO_WCID_V3 GENMASK(15, 0) + +#define MTK_FOE_WINFO_PAO_USR_INFO GENMASK(15, 0) +#define MTK_FOE_WINFO_PAO_TID GENMASK(19, 16) +#define MTK_FOE_WINFO_PAO_IS_FIXEDRATE BIT(20) +#define MTK_FOE_WINFO_PAO_IS_PRIOR BIT(21) +#define MTK_FOE_WINFO_PAO_IS_SP BIT(22) +#define MTK_FOE_WINFO_PAO_HF BIT(23) +#define MTK_FOE_WINFO_PAO_AMSDU_EN BIT(24) + enum { MTK_FOE_STATE_INVALID, MTK_FOE_STATE_UNBIND, @@ -106,8 +117,13 @@ struct mtk_foe_mac_info { u16 pppoe_id; u16 src_mac_lo; + /* netsys_v2 */ u16 minfo; u16 winfo; + + /* netsys_v3 */ + u32 w3info; + u32 wpao; }; /* software-only entry type */ @@ -216,6 +232,10 @@ struct mtk_foe_ipv6_6rd { struct mtk_foe_mac_info l2; }; +#define MTK_FOE_ENTRY_V1_SIZE 80 +#define MTK_FOE_ENTRY_V2_SIZE 96 +#define MTK_FOE_ENTRY_V3_SIZE 128 + struct mtk_foe_entry { u32 ib1; @@ -225,7 +245,7 @@ struct mtk_foe_entry { struct mtk_foe_ipv4_dslite dslite; struct mtk_foe_ipv6 ipv6; struct mtk_foe_ipv6_6rd ipv6_6rd; - u32 data[23]; + u32 data[31]; }; }; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index 02eebff02d45..a70a5417c173 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -193,7 +193,7 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, if (mtk_flow_get_wdma_info(dev, dest_mac, &info) == 0) { mtk_foe_entry_set_wdma(eth, foe, info.wdma_idx, info.queue, info.bss, info.wcid); - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + if (mtk_is_netsys_v2_or_greater(eth)) { switch (info.wdma_idx) { case 0: pse_port = 8; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h index a2e61b3eb006..3ce088eef0ef 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h @@ -163,6 +163,8 @@ enum { #define MTK_PPE_MIB_SER_R2 0x348 #define MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH GENMASK(23, 0) +#define MTK_PPE_MIB_SER_R3 0x34c + #define MTK_PPE_MIB_CACHE_CTL 0x350 #define MTK_PPE_MIB_CACHE_CTL_EN BIT(0) #define MTK_PPE_MIB_CACHE_CTL_FLUSH BIT(2) diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c index 02c03325911f..31aebeb2e285 100644 --- a/drivers/net/ethernet/mediatek/mtk_star_emac.c +++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c @@ -17,7 +17,6 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index 985cff910f30..00aeee0d5e45 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -2,6 +2,7 @@ /* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */ #include <linux/kernel.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/bitfield.h> @@ -1091,7 +1092,7 @@ mtk_wed_rx_reset(struct mtk_wed_device *dev) } else { struct mtk_eth *eth = dev->hw->eth; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (mtk_is_netsys_v2_or_greater(eth)) wed_set(dev, MTK_WED_RESET_IDX, MTK_WED_RESET_IDX_RX_V2); else @@ -1907,7 +1908,7 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, hw->wdma = wdma; hw->index = index; hw->irq = irq; - hw->version = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1; + hw->version = mtk_is_netsys_v1(eth) ? 1 : 2; if (hw->version == 1) { hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, diff --git a/drivers/net/ethernet/mediatek/mtk_wed_wo.c b/drivers/net/ethernet/mediatek/mtk_wed_wo.c index 69fba29055e9..3bd51a3d6650 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_wo.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_wo.c @@ -7,10 +7,9 @@ #include <linux/kernel.h> #include <linux/dma-mapping.h> -#include <linux/of_platform.h> #include <linux/interrupt.h> -#include <linux/of_address.h> #include <linux/mfd/syscon.h> +#include <linux/of.h> #include <linux/of_irq.h> #include <linux/bitfield.h> diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 7d45f1d55f79..164a13272faa 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -1467,8 +1467,8 @@ static int add_ip_rule(struct mlx4_en_priv *priv, struct list_head *list_h) { int err; - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; + struct mlx4_spec_list *spec_l2; + struct mlx4_spec_list *spec_l3; struct ethtool_usrip4_spec *l3_mask = &cmd->fs.m_u.usr_ip4_spec; spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); @@ -1505,9 +1505,9 @@ static int add_tcp_udp_rule(struct mlx4_en_priv *priv, struct list_head *list_h, int proto) { int err; - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; - struct mlx4_spec_list *spec_l4 = NULL; + struct mlx4_spec_list *spec_l2; + struct mlx4_spec_list *spec_l3; + struct mlx4_spec_list *spec_l4; struct ethtool_tcpip4_spec *l4_mask = &cmd->fs.m_u.tcp_ip4_spec; spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index e11bc0ac880e..403604ceebc8 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -291,7 +291,7 @@ mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, __be32 dst_ip, u8 ip_proto, __be16 src_port, __be16 dst_port, u32 flow_id) { - struct mlx4_en_filter *filter = NULL; + struct mlx4_en_filter *filter; filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); if (!filter) @@ -2935,7 +2935,7 @@ static void mlx4_en_bond_work(struct work_struct *work) static int mlx4_en_queue_bond_work(struct mlx4_en_priv *priv, int is_bonded, u8 v2p_p1, u8 v2p_p2) { - struct mlx4_en_bond *bond = NULL; + struct mlx4_en_bond *bond; bond = kzalloc(sizeof(*bond), GFP_ATOMIC); if (!bond) diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 61286b0d9b0c..8a5409b00530 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -864,7 +864,7 @@ static void mlx4_slave_destroy_special_qp_cap(struct mlx4_dev *dev) static int mlx4_slave_special_qp_cap(struct mlx4_dev *dev) { - struct mlx4_func_cap *func_cap = NULL; + struct mlx4_func_cap *func_cap; struct mlx4_caps *caps = &dev->caps; int i, err = 0; @@ -908,9 +908,9 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) { int err; u32 page_size; - struct mlx4_dev_cap *dev_cap = NULL; - struct mlx4_func_cap *func_cap = NULL; - struct mlx4_init_hca_param *hca_param = NULL; + struct mlx4_dev_cap *dev_cap; + struct mlx4_func_cap *func_cap; + struct mlx4_init_hca_param *hca_param; hca_param = kzalloc(sizeof(*hca_param), GFP_KERNEL); func_cap = kzalloc(sizeof(*func_cap), GFP_KERNEL); diff --git a/drivers/net/ethernet/mellanox/mlx4/mcg.c b/drivers/net/ethernet/mellanox/mlx4/mcg.c index f1716a83a4d3..24d0c7c46878 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mcg.c +++ b/drivers/net/ethernet/mellanox/mlx4/mcg.c @@ -294,7 +294,7 @@ static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port, struct mlx4_promisc_qp *dqp, *tmp_dqp; if (port < 1 || port > dev->caps.num_ports) - return NULL; + return false; s_steer = &mlx4_priv(dev)->steer[port - 1]; @@ -375,7 +375,7 @@ static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port, bool ret = false; if (port < 1 || port > dev->caps.num_ports) - return NULL; + return false; s_steer = &mlx4_priv(dev)->steer[port - 1]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 35f00700a4d6..63a2f2bb80a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -75,6 +75,10 @@ mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \ esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o +ifneq ($(CONFIG_MLX5_EN_IPSEC),) + mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/ipsec_fs.o +endif + mlx5_core-$(CONFIG_MLX5_BRIDGE) += esw/bridge.o esw/bridge_mcast.o esw/bridge_debugfs.o \ en/rep/bridge.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index d532883b42d7..afb348579577 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -162,18 +162,18 @@ static int cmd_alloc_index(struct mlx5_cmd *cmd) int ret; spin_lock_irqsave(&cmd->alloc_lock, flags); - ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds); - if (ret < cmd->max_reg_cmds) - clear_bit(ret, &cmd->bitmask); + ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds); + if (ret < cmd->vars.max_reg_cmds) + clear_bit(ret, &cmd->vars.bitmask); spin_unlock_irqrestore(&cmd->alloc_lock, flags); - return ret < cmd->max_reg_cmds ? ret : -ENOMEM; + return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM; } static void cmd_free_index(struct mlx5_cmd *cmd, int idx) { lockdep_assert_held(&cmd->alloc_lock); - set_bit(idx, &cmd->bitmask); + set_bit(idx, &cmd->vars.bitmask); } static void cmd_ent_get(struct mlx5_cmd_work_ent *ent) @@ -192,7 +192,7 @@ static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) if (ent->idx >= 0) { cmd_free_index(cmd, ent->idx); - up(ent->page_queue ? &cmd->pages_sem : &cmd->sem); + up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem); } cmd_free_ent(ent); @@ -202,7 +202,7 @@ out: static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) { - return cmd->cmd_buf + (idx << cmd->log_stride); + return cmd->cmd_buf + (idx << cmd->vars.log_stride); } static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg) @@ -974,7 +974,7 @@ static void cmd_work_handler(struct work_struct *work) cb_timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); complete(&ent->handling); - sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; + sem = ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem; down(sem); if (!ent->page_queue) { alloc_ret = cmd_alloc_index(cmd); @@ -994,9 +994,9 @@ static void cmd_work_handler(struct work_struct *work) } ent->idx = alloc_ret; } else { - ent->idx = cmd->max_reg_cmds; + ent->idx = cmd->vars.max_reg_cmds; spin_lock_irqsave(&cmd->alloc_lock, flags); - clear_bit(ent->idx, &cmd->bitmask); + clear_bit(ent->idx, &cmd->vars.bitmask); spin_unlock_irqrestore(&cmd->alloc_lock, flags); } @@ -1225,8 +1225,8 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, goto out_free; ds = ent->ts2 - ent->ts1; - if (ent->op < MLX5_CMD_OP_MAX) { - stats = &cmd->stats[ent->op]; + stats = xa_load(&cmd->stats, ent->op); + if (stats) { spin_lock_irq(&stats->lock); stats->sum += ds; ++stats->n; @@ -1548,7 +1548,6 @@ static void clean_debug_files(struct mlx5_core_dev *dev) if (!mlx5_debugfs_root) return; - mlx5_cmdif_debugfs_cleanup(dev); debugfs_remove_recursive(dbg->dbg_root); } @@ -1563,8 +1562,6 @@ static void create_debugfs_files(struct mlx5_core_dev *dev) debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops); debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops); - - mlx5_cmdif_debugfs_init(dev); } void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) @@ -1572,15 +1569,15 @@ void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) struct mlx5_cmd *cmd = &dev->cmd; int i; - for (i = 0; i < cmd->max_reg_cmds; i++) - down(&cmd->sem); - down(&cmd->pages_sem); + for (i = 0; i < cmd->vars.max_reg_cmds; i++) + down(&cmd->vars.sem); + down(&cmd->vars.pages_sem); cmd->allowed_opcode = opcode; - up(&cmd->pages_sem); - for (i = 0; i < cmd->max_reg_cmds; i++) - up(&cmd->sem); + up(&cmd->vars.pages_sem); + for (i = 0; i < cmd->vars.max_reg_cmds; i++) + up(&cmd->vars.sem); } static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) @@ -1588,15 +1585,15 @@ static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) struct mlx5_cmd *cmd = &dev->cmd; int i; - for (i = 0; i < cmd->max_reg_cmds; i++) - down(&cmd->sem); - down(&cmd->pages_sem); + for (i = 0; i < cmd->vars.max_reg_cmds; i++) + down(&cmd->vars.sem); + down(&cmd->vars.pages_sem); cmd->mode = mode; - up(&cmd->pages_sem); - for (i = 0; i < cmd->max_reg_cmds; i++) - up(&cmd->sem); + up(&cmd->vars.pages_sem); + for (i = 0; i < cmd->vars.max_reg_cmds; i++) + up(&cmd->vars.sem); } static int cmd_comp_notifier(struct notifier_block *nb, @@ -1655,7 +1652,7 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force /* there can be at most 32 command queues */ vector = vec & 0xffffffff; - for (i = 0; i < (1 << cmd->log_sz); i++) { + for (i = 0; i < (1 << cmd->vars.log_sz); i++) { if (test_bit(i, &vector)) { ent = cmd->ent_arr[i]; @@ -1698,8 +1695,8 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force if (ent->callback) { ds = ent->ts2 - ent->ts1; - if (ent->op < MLX5_CMD_OP_MAX) { - stats = &cmd->stats[ent->op]; + stats = xa_load(&cmd->stats, ent->op); + if (stats) { spin_lock_irqsave(&stats->lock, flags); stats->sum += ds; ++stats->n; @@ -1744,7 +1741,7 @@ static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) /* wait for pending handlers to complete */ mlx5_eq_synchronize_cmd_irq(dev); spin_lock_irqsave(&dev->cmd.alloc_lock, flags); - vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); + vector = ~dev->cmd.vars.bitmask & ((1ul << (1 << dev->cmd.vars.log_sz)) - 1); if (!vector) goto no_trig; @@ -1753,14 +1750,14 @@ static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) * to guarantee pending commands will not get freed in the meanwhile. * For that reason, it also has to be done inside the alloc_lock. */ - for_each_set_bit(i, &bitmask, (1 << cmd->log_sz)) + for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) cmd_ent_get(cmd->ent_arr[i]); vector |= MLX5_TRIGGERED_CMD_COMP; spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); mlx5_core_dbg(dev, "vector 0x%llx\n", vector); mlx5_cmd_comp_handler(dev, vector, true); - for_each_set_bit(i, &bitmask, (1 << cmd->log_sz)) + for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) cmd_ent_put(cmd->ent_arr[i]); return; @@ -1773,22 +1770,22 @@ void mlx5_cmd_flush(struct mlx5_core_dev *dev) struct mlx5_cmd *cmd = &dev->cmd; int i; - for (i = 0; i < cmd->max_reg_cmds; i++) { - while (down_trylock(&cmd->sem)) { + for (i = 0; i < cmd->vars.max_reg_cmds; i++) { + while (down_trylock(&cmd->vars.sem)) { mlx5_cmd_trigger_completions(dev); cond_resched(); } } - while (down_trylock(&cmd->pages_sem)) { + while (down_trylock(&cmd->vars.pages_sem)) { mlx5_cmd_trigger_completions(dev); cond_resched(); } /* Unlock cmdif */ - up(&cmd->pages_sem); - for (i = 0; i < cmd->max_reg_cmds; i++) - up(&cmd->sem); + up(&cmd->vars.pages_sem); + for (i = 0; i < cmd->vars.max_reg_cmds; i++) + up(&cmd->vars.sem); } static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, @@ -1858,7 +1855,7 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, /* atomic context may not sleep */ if (callback) return -EINVAL; - down(&dev->cmd.throttle_sem); + down(&dev->cmd.vars.throttle_sem); } pages_queue = is_manage_pages(in); @@ -1903,7 +1900,7 @@ out_in: free_msg(dev, inb); out_up: if (throttle_op) - up(&dev->cmd.throttle_sem); + up(&dev->cmd.vars.throttle_sem); return err; } @@ -1926,7 +1923,9 @@ static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status, if (!err || !(strcmp(namep, "unknown command opcode"))) return; - stats = &dev->cmd.stats[opcode]; + stats = xa_load(&dev->cmd.stats, opcode); + if (!stats) + return; spin_lock_irq(&stats->lock); stats->failed++; if (err < 0) @@ -2190,19 +2189,8 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) int size = sizeof(struct mlx5_cmd_prot_block); int align = roundup_pow_of_two(size); struct mlx5_cmd *cmd = &dev->cmd; - u32 cmd_h, cmd_l; - u16 cmd_if_rev; + u32 cmd_l; int err; - int i; - - memset(cmd, 0, sizeof(*cmd)); - cmd_if_rev = cmdif_rev(dev); - if (cmd_if_rev != CMD_IF_REV) { - mlx5_core_err(dev, - "Driver cmdif rev(%d) differs from firmware's(%d)\n", - CMD_IF_REV, cmd_if_rev); - return -EINVAL; - } cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); if (!cmd->pool) @@ -2212,62 +2200,16 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) if (err) goto err_free_pool; - cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; - cmd->log_sz = cmd_l >> 4 & 0xf; - cmd->log_stride = cmd_l & 0xf; - if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) { - mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", - 1 << cmd->log_sz); - err = -EINVAL; - goto err_free_page; - } - - if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) { - mlx5_core_err(dev, "command queue size overflow\n"); - err = -EINVAL; - goto err_free_page; - } - - cmd->state = MLX5_CMDIF_STATE_DOWN; - cmd->checksum_disabled = 1; - cmd->max_reg_cmds = (1 << cmd->log_sz) - 1; - cmd->bitmask = (1UL << cmd->max_reg_cmds) - 1; - - cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; - if (cmd->cmdif_rev > CMD_IF_REV) { - mlx5_core_err(dev, "driver does not support command interface version. driver %d, firmware %d\n", - CMD_IF_REV, cmd->cmdif_rev); - err = -EOPNOTSUPP; - goto err_free_page; - } - - spin_lock_init(&cmd->alloc_lock); - spin_lock_init(&cmd->token_lock); - for (i = 0; i < MLX5_CMD_OP_MAX; i++) - spin_lock_init(&cmd->stats[i].lock); - - sema_init(&cmd->sem, cmd->max_reg_cmds); - sema_init(&cmd->pages_sem, 1); - sema_init(&cmd->throttle_sem, DIV_ROUND_UP(cmd->max_reg_cmds, 2)); - - cmd_h = (u32)((u64)(cmd->dma) >> 32); cmd_l = (u32)(cmd->dma); if (cmd_l & 0xfff) { mlx5_core_err(dev, "invalid command queue address\n"); err = -ENOMEM; - goto err_free_page; + goto err_cmd_page; } + cmd->checksum_disabled = 1; - iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); - iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); - - /* Make sure firmware sees the complete address before we proceed */ - wmb(); - - mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); - - cmd->mode = CMD_MODE_POLLING; - cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; + spin_lock_init(&cmd->alloc_lock); + spin_lock_init(&cmd->token_lock); create_msg_cache(dev); @@ -2279,16 +2221,14 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) goto err_cache; } - create_debugfs_files(dev); + mlx5_cmdif_debugfs_init(dev); return 0; err_cache: destroy_msg_cache(dev); - -err_free_page: +err_cmd_page: free_cmd_page(dev, cmd); - err_free_pool: dma_pool_destroy(cmd->pool); return err; @@ -2298,13 +2238,78 @@ void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) { struct mlx5_cmd *cmd = &dev->cmd; - clean_debug_files(dev); + mlx5_cmdif_debugfs_cleanup(dev); destroy_workqueue(cmd->wq); destroy_msg_cache(dev); free_cmd_page(dev, cmd); dma_pool_destroy(cmd->pool); } +int mlx5_cmd_enable(struct mlx5_core_dev *dev) +{ + struct mlx5_cmd *cmd = &dev->cmd; + u32 cmd_h, cmd_l; + + memset(&cmd->vars, 0, sizeof(cmd->vars)); + cmd->vars.cmdif_rev = cmdif_rev(dev); + if (cmd->vars.cmdif_rev != CMD_IF_REV) { + mlx5_core_err(dev, + "Driver cmdif rev(%d) differs from firmware's(%d)\n", + CMD_IF_REV, cmd->vars.cmdif_rev); + return -EINVAL; + } + + cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; + cmd->vars.log_sz = cmd_l >> 4 & 0xf; + cmd->vars.log_stride = cmd_l & 0xf; + if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) { + mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", + 1 << cmd->vars.log_sz); + return -EINVAL; + } + + if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) { + mlx5_core_err(dev, "command queue size overflow\n"); + return -EINVAL; + } + + cmd->state = MLX5_CMDIF_STATE_DOWN; + cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1; + cmd->vars.bitmask = (1UL << cmd->vars.max_reg_cmds) - 1; + + sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds); + sema_init(&cmd->vars.pages_sem, 1); + sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2)); + + cmd_h = (u32)((u64)(cmd->dma) >> 32); + cmd_l = (u32)(cmd->dma); + if (WARN_ON(cmd_l & 0xfff)) + return -EINVAL; + + iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); + iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); + + /* Make sure firmware sees the complete address before we proceed */ + wmb(); + + mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); + + cmd->mode = CMD_MODE_POLLING; + cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; + + create_debugfs_files(dev); + + return 0; +} + +void mlx5_cmd_disable(struct mlx5_core_dev *dev) +{ + struct mlx5_cmd *cmd = &dev->cmd; + + clean_debug_files(dev); + flush_workqueue(cmd->wq); +} + void mlx5_cmd_set_state(struct mlx5_core_dev *dev, enum mlx5_cmdif_state cmdif_state) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index 2138f28a2931..09652dc89115 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -176,8 +176,8 @@ static ssize_t slots_read(struct file *filp, char __user *buf, size_t count, int ret; cmd = filp->private_data; - weight = bitmap_weight(&cmd->bitmask, cmd->max_reg_cmds); - field = cmd->max_reg_cmds - weight; + weight = bitmap_weight(&cmd->vars.bitmask, cmd->vars.max_reg_cmds); + field = cmd->vars.max_reg_cmds - weight; ret = snprintf(tbuf, sizeof(tbuf), "%d\n", field); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } @@ -188,6 +188,24 @@ static const struct file_operations slots_fops = { .read = slots_read, }; +static struct mlx5_cmd_stats * +mlx5_cmdif_alloc_stats(struct xarray *stats_xa, int opcode) +{ + struct mlx5_cmd_stats *stats = kzalloc(sizeof(*stats), GFP_KERNEL); + int err; + + if (!stats) + return NULL; + + err = xa_insert(stats_xa, opcode, stats, GFP_KERNEL); + if (err) { + kfree(stats); + return NULL; + } + spin_lock_init(&stats->lock); + return stats; +} + void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) { struct mlx5_cmd_stats *stats; @@ -200,10 +218,14 @@ void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) debugfs_create_file("slots_inuse", 0400, *cmd, &dev->cmd, &slots_fops); + xa_init(&dev->cmd.stats); + for (i = 0; i < MLX5_CMD_OP_MAX; i++) { - stats = &dev->cmd.stats[i]; namep = mlx5_command_str(i); if (strcmp(namep, "unknown command opcode")) { + stats = mlx5_cmdif_alloc_stats(&dev->cmd.stats, i); + if (!stats) + continue; stats->root = debugfs_create_dir(namep, *cmd); debugfs_create_file("average", 0400, stats->root, stats, @@ -224,7 +246,13 @@ void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) void mlx5_cmdif_debugfs_cleanup(struct mlx5_core_dev *dev) { + struct mlx5_cmd_stats *stats; + unsigned long i; + debugfs_remove_recursive(dev->priv.dbg.cmdif_debugfs); + xa_for_each(&dev->cmd.stats, i, stats) + kfree(stats); + xa_destroy(&dev->cmd.stats); } void mlx5_cq_debugfs_init(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index edb06fb9bbc5..7909f378dc93 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -36,6 +36,7 @@ #include <linux/mlx5/vport.h> #include "mlx5_core.h" #include "devlink.h" +#include "lag/lag.h" /* intf dev list mutex */ static DEFINE_MUTEX(mlx5_intf_mutex); @@ -587,10 +588,7 @@ static int next_phys_dev_lag(struct device *dev, const void *data) if (!mdev) return 0; - if (!MLX5_CAP_GEN(mdev, vport_group_manager) || - !MLX5_CAP_GEN(mdev, lag_master) || - (MLX5_CAP_GEN(mdev, num_lag_ports) > MLX5_MAX_PORTS || - MLX5_CAP_GEN(mdev, num_lag_ports) <= 1)) + if (!mlx5_lag_is_supported(mdev)) return 0; return _next_phys_dev(mdev, data); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b1807bfb815f..0f8f70b91485 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1167,9 +1167,6 @@ int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv, int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc); int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc); -int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, - u32 *rule_locs); -int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd); u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv); u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv); int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 1874c2f0587f..244bc15a42ab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -379,9 +379,9 @@ int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_ if (!htb && htb_qopt->command != TC_HTB_CREATE) return -EINVAL; - if (htb_qopt->prio) { + if (htb_qopt->prio || htb_qopt->quantum) { NL_SET_ERR_MSG_MOD(htb_qopt->extack, - "prio parameter is not supported by device with HTB offload enabled."); + "prio and quantum parameters are not supported by device with HTB offload enabled."); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index b5c773ffc763..b12fe3c5a258 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -715,9 +715,20 @@ void mlx5e_rep_tc_receive(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq, uplink_priv = &uplink_rpriv->uplink_priv; ct_priv = uplink_priv->ct_priv; - if (!mlx5_ipsec_is_rx_flow(cqe) && - !mlx5e_tc_update_skb(cqe, skb, mapping_ctx, reg_c0, ct_priv, zone_restore_id, tunnel_id, - &tc_priv)) +#ifdef CONFIG_MLX5_EN_IPSEC + if (!(tunnel_id >> ESW_TUN_OPTS_BITS)) { + u32 mapped_id; + u32 metadata; + + mapped_id = tunnel_id & ESW_IPSEC_RX_MAPPED_ID_MASK; + if (mapped_id && + !mlx5_esw_ipsec_rx_make_metadata(priv, mapped_id, &metadata)) + mlx5e_ipsec_offload_handle_rx_skb(priv->netdev, skb, metadata); + } +#endif + + if (!mlx5e_tc_update_skb(cqe, skb, mapping_ctx, reg_c0, ct_priv, + zone_restore_id, tunnel_id, &tc_priv)) goto free_skb; forward: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index e1095bc36543..56e6b8c7501f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -218,17 +218,32 @@ int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, res->rss_nch); } -u8 mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) +int mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, + enum mlx5_traffic_types tt) { - struct mlx5e_rss *rss = res->rss[0]; + struct mlx5e_rss *rss; + + if (rss_idx >= MLX5E_MAX_NUM_RSS) + return -EINVAL; + + rss = res->rss[rss_idx]; + if (!rss) + return -ENOENT; return mlx5e_rss_get_hash_fields(rss, tt); } -int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt, - u8 rx_hash_fields) +int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, + enum mlx5_traffic_types tt, u8 rx_hash_fields) { - struct mlx5e_rss *rss = res->rss[0]; + struct mlx5e_rss *rss; + + if (rss_idx >= MLX5E_MAX_NUM_RSS) + return -EINVAL; + + rss = res->rss[rss_idx]; + if (!rss) + return -ENOENT; return mlx5e_rss_set_hash_fields(rss, tt, rx_hash_fields); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 5d5f64fab60f..580fe8bc3cd2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -48,9 +48,10 @@ int mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, const u32 *indir, const u8 *key, const u8 *hfunc); -u8 mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt); -int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt, - u8 rx_hash_fields); +int mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, + enum mlx5_traffic_types tt); +int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, + enum mlx5_traffic_types tt, u8 rx_hash_fields); int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res, struct mlx5e_packet_merge_param *pkt_merge_param); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c index 2b80fe73549d..8c531f4ec912 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c @@ -221,16 +221,21 @@ mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs *fs) } static inline bool -mlx5_tc_ct_valid_used_dissector_keys(const u32 used_keys) +mlx5_tc_ct_valid_used_dissector_keys(const u64 used_keys) { -#define DISS_BIT(name) BIT(FLOW_DISSECTOR_KEY_ ## name) - const u32 basic_keys = DISS_BIT(BASIC) | DISS_BIT(CONTROL) | DISS_BIT(META); - const u32 ipv4_tcp = basic_keys | DISS_BIT(IPV4_ADDRS) | DISS_BIT(PORTS) | DISS_BIT(TCP); - const u32 ipv6_tcp = basic_keys | DISS_BIT(IPV6_ADDRS) | DISS_BIT(PORTS) | DISS_BIT(TCP); - const u32 ipv4_udp = basic_keys | DISS_BIT(IPV4_ADDRS) | DISS_BIT(PORTS); - const u32 ipv6_udp = basic_keys | DISS_BIT(IPV6_ADDRS) | DISS_BIT(PORTS); - const u32 ipv4_gre = basic_keys | DISS_BIT(IPV4_ADDRS); - const u32 ipv6_gre = basic_keys | DISS_BIT(IPV6_ADDRS); +#define DISS_BIT(name) BIT_ULL(FLOW_DISSECTOR_KEY_ ## name) + const u64 basic_keys = DISS_BIT(BASIC) | DISS_BIT(CONTROL) | + DISS_BIT(META); + const u64 ipv4_tcp = basic_keys | DISS_BIT(IPV4_ADDRS) | + DISS_BIT(PORTS) | DISS_BIT(TCP); + const u64 ipv6_tcp = basic_keys | DISS_BIT(IPV6_ADDRS) | + DISS_BIT(PORTS) | DISS_BIT(TCP); + const u64 ipv4_udp = basic_keys | DISS_BIT(IPV4_ADDRS) | + DISS_BIT(PORTS); + const u64 ipv6_udp = basic_keys | DISS_BIT(IPV6_ADDRS) | + DISS_BIT(PORTS); + const u64 ipv4_gre = basic_keys | DISS_BIT(IPV4_ADDRS); + const u64 ipv6_gre = basic_keys | DISS_BIT(IPV6_ADDRS); return (used_keys == ipv4_tcp || used_keys == ipv4_udp || used_keys == ipv6_tcp || used_keys == ipv6_udp || used_keys == ipv4_gre || used_keys == ipv6_gre); @@ -247,7 +252,7 @@ mlx5_ct_fs_smfs_ct_validate_flow_rule(struct mlx5_ct_fs *fs, struct flow_rule *f struct flow_match_tcp tcp; if (!mlx5_tc_ct_valid_used_dissector_keys(flow_rule->match.dissector->used_keys)) { - ct_dbg("rule uses unexpected dissectors (0x%08x)", + ct_dbg("rule uses unexpected dissectors (0x%016llx)", flow_rule->match.dissector->used_keys); return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 891d39b4bfd4..a577f0edabe8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -40,6 +40,7 @@ #include "en.h" #include "ipsec.h" #include "ipsec_rxtx.h" +#include "en_rep.h" #define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000) #define MLX5E_IPSEC_TUNNEL_SA XA_MARK_1 @@ -354,6 +355,12 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, mlx5e_ipsec_init_limits(sa_entry, attrs); mlx5e_ipsec_init_macs(sa_entry, attrs); + + if (x->encap) { + attrs->encap = true; + attrs->sport = x->encap->encap_sport; + attrs->dport = x->encap->encap_dport; + } } static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, @@ -387,8 +394,25 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, return -EINVAL; } if (x->encap) { - NL_SET_ERR_MSG_MOD(extack, "Encapsulated xfrm state may not be offloaded"); - return -EINVAL; + if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) { + NL_SET_ERR_MSG_MOD(extack, "Encapsulation is not supported"); + return -EINVAL; + } + + if (x->encap->encap_type != UDP_ENCAP_ESPINUDP) { + NL_SET_ERR_MSG_MOD(extack, "Encapsulation other than UDP is not supported"); + return -EINVAL; + } + + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) { + NL_SET_ERR_MSG_MOD(extack, "Encapsulation is supported in packet offload mode only"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT) { + NL_SET_ERR_MSG_MOD(extack, "Encapsulation is supported in transport mode only"); + return -EINVAL; + } } if (!x->aead) { NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without aead"); @@ -835,6 +859,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) goto clear_aso; } + ipsec->is_uplink_rep = mlx5e_is_uplink_rep(priv); ret = mlx5e_accel_ipsec_fs_init(ipsec); if (ret) goto err_fs_init; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 4e9887171508..9e7c42c2f77b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -94,13 +94,20 @@ struct mlx5_accel_esp_xfrm_attrs { u8 dir : 2; u8 type : 2; u8 drop : 1; + u8 encap : 1; u8 family; struct mlx5_replay_esn replay_esn; u32 authsize; u32 reqid; struct mlx5_ipsec_lft lft; - u8 smac[ETH_ALEN]; - u8 dmac[ETH_ALEN]; + union { + u8 smac[ETH_ALEN]; + __be16 sport; + }; + union { + u8 dmac[ETH_ALEN]; + __be16 dport; + }; }; enum mlx5_ipsec_cap { @@ -110,6 +117,7 @@ enum mlx5_ipsec_cap { MLX5_IPSEC_CAP_ROCE = 1 << 3, MLX5_IPSEC_CAP_PRIO = 1 << 4, MLX5_IPSEC_CAP_TUNNEL = 1 << 5, + MLX5_IPSEC_CAP_ESPINUDP = 1 << 6, }; struct mlx5e_priv; @@ -135,7 +143,7 @@ struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_tx_drop_trailer; }; -struct mlx5e_ipsec_rx; +struct mlx5e_ipsec_fc; struct mlx5e_ipsec_tx; struct mlx5e_ipsec_work { @@ -161,6 +169,58 @@ struct mlx5e_ipsec_aso { spinlock_t lock; }; +struct mlx5e_ipsec_rx_create_attr { + struct mlx5_flow_namespace *ns; + struct mlx5_ttc_table *ttc; + u32 family; + int prio; + int pol_level; + int sa_level; + int status_level; + enum mlx5_flow_namespace_type chains_ns; +}; + +struct mlx5e_ipsec_ft { + struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *pol; + struct mlx5_flow_table *sa; + struct mlx5_flow_table *status; + u32 refcnt; +}; + +struct mlx5e_ipsec_rule { + struct mlx5_flow_handle *rule; + struct mlx5_modify_hdr *modify_hdr; + struct mlx5_pkt_reformat *pkt_reformat; + struct mlx5_fc *fc; +}; + +struct mlx5e_ipsec_miss { + struct mlx5_flow_group *group; + struct mlx5_flow_handle *rule; +}; + +struct mlx5e_ipsec_rx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; + struct mlx5e_ipsec_miss sa; + struct mlx5e_ipsec_rule status; + struct mlx5e_ipsec_miss status_drop; + struct mlx5_fc *status_drop_cnt; + struct mlx5e_ipsec_fc *fc; + struct mlx5_fs_chains *chains; + u8 allow_tunnel_mode : 1; + struct xarray ipsec_obj_id_map; +}; + +struct mlx5e_ipsec_tx_create_attr { + int prio; + int pol_level; + int sa_level; + int cnt_level; + enum mlx5_flow_namespace_type chains_ns; +}; + struct mlx5e_ipsec { struct mlx5_core_dev *mdev; struct xarray sadb; @@ -170,11 +230,14 @@ struct mlx5e_ipsec { struct mlx5e_flow_steering *fs; struct mlx5e_ipsec_rx *rx_ipv4; struct mlx5e_ipsec_rx *rx_ipv6; + struct mlx5e_ipsec_rx *rx_esw; struct mlx5e_ipsec_tx *tx; + struct mlx5e_ipsec_tx *tx_esw; struct mlx5e_ipsec_aso *aso; struct notifier_block nb; struct notifier_block netevent_nb; struct mlx5_ipsec_fs *roce; + u8 is_uplink_rep: 1; }; struct mlx5e_ipsec_esn_state { @@ -183,13 +246,6 @@ struct mlx5e_ipsec_esn_state { u8 overlap: 1; }; -struct mlx5e_ipsec_rule { - struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *modify_hdr; - struct mlx5_pkt_reformat *pkt_reformat; - struct mlx5_fc *fc; -}; - struct mlx5e_ipsec_limits { u64 round; u8 soft_limit_hit : 1; @@ -209,6 +265,7 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_work *work; struct mlx5e_ipsec_dwork *dwork; struct mlx5e_ipsec_limits limits; + u32 rx_mapped_id; }; struct mlx5_accel_pol_xfrm_attrs { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 832d36be4a17..3781c72d97f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,6 +9,8 @@ #include "fs_core.h" #include "lib/ipsec_fs_roce.h" #include "lib/fs_chains.h" +#include "esw/ipsec_fs.h" +#include "en_rep.h" #define NUM_IPSEC_FTE BIT(15) #define MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE 16 @@ -19,32 +21,10 @@ struct mlx5e_ipsec_fc { struct mlx5_fc *drop; }; -struct mlx5e_ipsec_ft { - struct mutex mutex; /* Protect changes to this struct */ - struct mlx5_flow_table *pol; - struct mlx5_flow_table *sa; - struct mlx5_flow_table *status; - u32 refcnt; -}; - -struct mlx5e_ipsec_miss { - struct mlx5_flow_group *group; - struct mlx5_flow_handle *rule; -}; - -struct mlx5e_ipsec_rx { - struct mlx5e_ipsec_ft ft; - struct mlx5e_ipsec_miss pol; - struct mlx5e_ipsec_miss sa; - struct mlx5e_ipsec_rule status; - struct mlx5e_ipsec_fc *fc; - struct mlx5_fs_chains *chains; - u8 allow_tunnel_mode : 1; -}; - struct mlx5e_ipsec_tx { struct mlx5e_ipsec_ft ft; struct mlx5e_ipsec_miss pol; + struct mlx5e_ipsec_miss sa; struct mlx5e_ipsec_rule status; struct mlx5_flow_namespace *ns; struct mlx5e_ipsec_fc *fc; @@ -60,14 +40,25 @@ static enum mlx5_traffic_types family2tt(u32 family) return MLX5_TT_IPV6_IPSEC_ESP; } -static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family) +static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family, int type) { + if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) + return ipsec->rx_esw; + if (family == AF_INET) return ipsec->rx_ipv4; return ipsec->rx_ipv6; } +static struct mlx5e_ipsec_tx *ipsec_tx(struct mlx5e_ipsec *ipsec, int type) +{ + if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) + return ipsec->tx_esw; + + return ipsec->tx; +} + static struct mlx5_fs_chains * ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, enum mlx5_flow_namespace_type ns, int base_prio, @@ -238,13 +229,19 @@ out: return err; } -static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, - struct mlx5e_ipsec_rx *rx, u32 family) +static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); - /* disconnect */ mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); +} + +static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) +{ + /* disconnect */ + if (rx != ipsec->rx_esw) + ipsec_rx_ft_disconnect(ipsec, family); if (rx->chains) { ipsec_chains_destroy(rx->chains); @@ -257,63 +254,112 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, mlx5_del_flow_rules(rx->sa.rule); mlx5_destroy_flow_group(rx->sa.group); mlx5_destroy_flow_table(rx->ft.sa); - if (rx->allow_tunnel_mode) - mlx5_eswitch_unblock_encap(mdev); - mlx5_del_flow_rules(rx->status.rule); - mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + if (rx == ipsec->rx_esw) { + mlx5_esw_ipsec_rx_status_destroy(ipsec, rx); + } else { + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + } mlx5_destroy_flow_table(rx->ft.status); mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family); } +static void ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + u32 family, + struct mlx5e_ipsec_rx_create_attr *attr) +{ + if (rx == ipsec->rx_esw) { + /* For packet offload in switchdev mode, RX & TX use FDB namespace */ + attr->ns = ipsec->tx_esw->ns; + mlx5_esw_ipsec_rx_create_attr_set(ipsec, attr); + return; + } + + attr->ns = mlx5e_fs_get_ns(ipsec->fs, false); + attr->ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + attr->family = family; + attr->prio = MLX5E_NIC_PRIO; + attr->pol_level = MLX5E_ACCEL_FS_POL_FT_LEVEL; + attr->sa_level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; + attr->status_level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; + attr->chains_ns = MLX5_FLOW_NAMESPACE_KERNEL; +} + +static int ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5e_ipsec_rx_create_attr *attr, + struct mlx5_flow_destination *dest) +{ + struct mlx5_flow_table *ft; + int err; + + if (rx == ipsec->rx_esw) + return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest); + + *dest = mlx5_ttc_get_default_dest(attr->ttc, family2tt(attr->family)); + err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest, + attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, + attr->prio); + if (err) + return err; + + ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, attr->family); + if (ft) { + dest->type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest->ft = ft; + } + + return 0; +} + +static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5e_ipsec_rx_create_attr *attr) +{ + struct mlx5_flow_destination dest = {}; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = rx->ft.pol; + mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest); +} + static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, u32 family) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); - struct mlx5_flow_destination default_dest; + struct mlx5e_ipsec_rx_create_attr attr; struct mlx5_flow_destination dest[2]; struct mlx5_flow_table *ft; u32 flags = 0; int err; - default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); - err = mlx5_ipsec_fs_roce_rx_create(mdev, ipsec->roce, ns, &default_dest, - family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, - MLX5E_NIC_PRIO); + ipsec_rx_create_attr_set(ipsec, rx, family, &attr); + + err = ipsec_rx_status_pass_dest_get(ipsec, rx, &attr, &dest[0]); if (err) return err; - ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, - MLX5E_NIC_PRIO, 1, 0); + ft = ipsec_ft_create(attr.ns, attr.status_level, attr.prio, 1, 0); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft_status; } - rx->ft.status = ft; - ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); - if (ft) { - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest[0].ft = ft; - } else { - dest[0] = default_dest; - } - dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); - err = ipsec_status_rule(mdev, rx, dest); + if (rx == ipsec->rx_esw) + err = mlx5_esw_ipsec_rx_status_create(ipsec, rx, dest); + else + err = ipsec_status_rule(mdev, rx, dest); if (err) goto err_add; /* Create FT */ - if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) - rx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); if (rx->allow_tunnel_mode) flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; - ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, 2, - flags); + ft = ipsec_ft_create(attr.ns, attr.sa_level, attr.prio, 2, flags); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; @@ -326,9 +372,9 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { rx->chains = ipsec_chains_create(mdev, rx->ft.sa, - MLX5_FLOW_NAMESPACE_KERNEL, - MLX5E_NIC_PRIO, - MLX5E_ACCEL_FS_POL_FT_LEVEL, + attr.chains_ns, + attr.prio, + attr.pol_level, &rx->ft.pol); if (IS_ERR(rx->chains)) { err = PTR_ERR(rx->chains); @@ -338,8 +384,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, goto connect; } - ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, - 2, 0); + ft = ipsec_ft_create(attr.ns, attr.pol_level, attr.prio, 2, 0); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_pol_ft; @@ -354,10 +399,8 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, connect: /* connect */ - memset(dest, 0x00, sizeof(*dest)); - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest[0].ft = rx->ft.pol; - mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest[0]); + if (rx != ipsec->rx_esw) + ipsec_rx_ft_connect(ipsec, rx, &attr); return 0; err_pol_miss: @@ -368,8 +411,6 @@ err_pol_ft: err_fs: mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: - if (rx->allow_tunnel_mode) - mlx5_eswitch_unblock_encap(mdev); mlx5_del_flow_rules(rx->status.rule); mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); err_add: @@ -387,13 +428,26 @@ static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, if (rx->ft.refcnt) goto skip; + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) + rx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); + + err = mlx5_eswitch_block_mode_trylock(mdev); + if (err) + goto err_out; + err = rx_create(mdev, ipsec, rx, family); + mlx5_eswitch_block_mode_unlock(mdev, err); if (err) - return err; + goto err_out; skip: rx->ft.refcnt++; return 0; + +err_out: + if (rx->allow_tunnel_mode) + mlx5_eswitch_unblock_encap(mdev); + return err; } static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, @@ -402,13 +456,19 @@ static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, if (--rx->ft.refcnt) return; + mlx5_eswitch_unblock_mode_lock(ipsec->mdev); rx_destroy(ipsec->mdev, ipsec, rx, family); + mlx5_eswitch_unblock_mode_unlock(ipsec->mdev); + + if (rx->allow_tunnel_mode) + mlx5_eswitch_unblock_encap(ipsec->mdev); } static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, - struct mlx5e_ipsec *ipsec, u32 family) + struct mlx5e_ipsec *ipsec, u32 family, + int type) { - struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); + struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); int err; mutex_lock(&rx->ft.mutex); @@ -422,9 +482,9 @@ static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, - u32 family, u32 prio) + u32 family, u32 prio, int type) { - struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); + struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); struct mlx5_flow_table *ft; int err; @@ -449,18 +509,18 @@ err_get: return ERR_PTR(err); } -static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family) +static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family, int type) { - struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); + struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); mutex_lock(&rx->ft.mutex); rx_put(ipsec, rx, family); mutex_unlock(&rx->ft.mutex); } -static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio) +static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio, int type) { - struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); + struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); mutex_lock(&rx->ft.mutex); if (rx->chains) @@ -504,7 +564,7 @@ err_rule: } /* IPsec TX flow steering */ -static void tx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, +static void tx_destroy(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, struct mlx5_ipsec_fs *roce) { mlx5_ipsec_fs_roce_tx_destroy(roce); @@ -516,22 +576,43 @@ static void tx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, mlx5_destroy_flow_table(tx->ft.pol); } + if (tx == ipsec->tx_esw) { + mlx5_del_flow_rules(tx->sa.rule); + mlx5_destroy_flow_group(tx->sa.group); + } mlx5_destroy_flow_table(tx->ft.sa); - if (tx->allow_tunnel_mode) - mlx5_eswitch_unblock_encap(mdev); mlx5_del_flow_rules(tx->status.rule); mlx5_destroy_flow_table(tx->ft.status); } -static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, +static void ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_tx *tx, + struct mlx5e_ipsec_tx_create_attr *attr) +{ + if (tx == ipsec->tx_esw) { + mlx5_esw_ipsec_tx_create_attr_set(ipsec, attr); + return; + } + + attr->prio = 0; + attr->pol_level = 0; + attr->sa_level = 1; + attr->cnt_level = 2; + attr->chains_ns = MLX5_FLOW_NAMESPACE_EGRESS_IPSEC; +} + +static int tx_create(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, struct mlx5_ipsec_fs *roce) { + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx_create_attr attr; struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft; u32 flags = 0; int err; - ft = ipsec_ft_create(tx->ns, 2, 0, 1, 0); + ipsec_tx_create_attr_set(ipsec, tx, &attr); + ft = ipsec_ft_create(tx->ns, attr.cnt_level, attr.prio, 1, 0); if (IS_ERR(ft)) return PTR_ERR(ft); tx->ft.status = ft; @@ -540,20 +621,27 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, if (err) goto err_status_rule; - if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) - tx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); if (tx->allow_tunnel_mode) flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; - ft = ipsec_ft_create(tx->ns, 1, 0, 4, flags); + ft = ipsec_ft_create(tx->ns, attr.sa_level, attr.prio, 4, flags); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_sa_ft; } tx->ft.sa = ft; + if (tx == ipsec->tx_esw) { + dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; + dest.vport.num = MLX5_VPORT_UPLINK; + err = ipsec_miss_create(mdev, tx->ft.sa, &tx->sa, &dest); + if (err) + goto err_sa_miss; + memset(&dest, 0, sizeof(dest)); + } + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { tx->chains = ipsec_chains_create( - mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 0, + mdev, tx->ft.sa, attr.chains_ns, attr.prio, attr.pol_level, &tx->ft.pol); if (IS_ERR(tx->chains)) { err = PTR_ERR(tx->chains); @@ -563,7 +651,7 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, goto connect_roce; } - ft = ipsec_ft_create(tx->ns, 0, 0, 2, 0); + ft = ipsec_ft_create(tx->ns, attr.pol_level, attr.prio, 2, 0); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_pol_ft; @@ -592,16 +680,38 @@ err_roce: mlx5_destroy_flow_table(tx->ft.pol); } err_pol_ft: + if (tx == ipsec->tx_esw) { + mlx5_del_flow_rules(tx->sa.rule); + mlx5_destroy_flow_group(tx->sa.group); + } +err_sa_miss: mlx5_destroy_flow_table(tx->ft.sa); err_sa_ft: - if (tx->allow_tunnel_mode) - mlx5_eswitch_unblock_encap(mdev); mlx5_del_flow_rules(tx->status.rule); err_status_rule: mlx5_destroy_flow_table(tx->ft.status); return err; } +static void ipsec_esw_tx_ft_policy_set(struct mlx5_core_dev *mdev, + struct mlx5_flow_table *ft) +{ +#ifdef CONFIG_MLX5_ESWITCH + struct mlx5_eswitch *esw = mdev->priv.eswitch; + struct mlx5e_rep_priv *uplink_rpriv; + struct mlx5e_priv *priv; + + esw->offloads.ft_ipsec_tx_pol = ft; + uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); + priv = netdev_priv(uplink_rpriv->netdev); + if (!priv->channels.num) + return; + + mlx5e_rep_deactivate_channels(priv); + mlx5e_rep_activate_channels(priv); +#endif +} + static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) { @@ -610,13 +720,32 @@ static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, if (tx->ft.refcnt) goto skip; - err = tx_create(mdev, tx, ipsec->roce); + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) + tx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); + + err = mlx5_eswitch_block_mode_trylock(mdev); if (err) - return err; + goto err_out; + + err = tx_create(ipsec, tx, ipsec->roce); + if (err) { + mlx5_eswitch_block_mode_unlock(mdev, err); + goto err_out; + } + + if (tx == ipsec->tx_esw) + ipsec_esw_tx_ft_policy_set(mdev, tx->ft.pol); + + mlx5_eswitch_block_mode_unlock(mdev, err); skip: tx->ft.refcnt++; return 0; + +err_out: + if (tx->allow_tunnel_mode) + mlx5_eswitch_unblock_encap(mdev); + return err; } static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) @@ -624,14 +753,26 @@ static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) if (--tx->ft.refcnt) return; - tx_destroy(ipsec->mdev, tx, ipsec->roce); + mlx5_eswitch_unblock_mode_lock(ipsec->mdev); + + if (tx == ipsec->tx_esw) { + mlx5_esw_ipsec_restore_dest_uplink(ipsec->mdev); + ipsec_esw_tx_ft_policy_set(ipsec->mdev, NULL); + } + + tx_destroy(ipsec, tx, ipsec->roce); + + mlx5_eswitch_unblock_mode_unlock(ipsec->mdev); + + if (tx->allow_tunnel_mode) + mlx5_eswitch_unblock_encap(ipsec->mdev); } static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, - u32 prio) + u32 prio, int type) { - struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); struct mlx5_flow_table *ft; int err; @@ -657,9 +798,9 @@ err_get: } static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, - struct mlx5e_ipsec *ipsec) + struct mlx5e_ipsec *ipsec, int type) { - struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); int err; mutex_lock(&tx->ft.mutex); @@ -671,18 +812,18 @@ static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, return tx; } -static void tx_ft_put(struct mlx5e_ipsec *ipsec) +static void tx_ft_put(struct mlx5e_ipsec *ipsec, int type) { - struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); mutex_lock(&tx->ft.mutex); tx_put(ipsec, tx); mutex_unlock(&tx->ft.mutex); } -static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio) +static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio, int type) { - struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); mutex_lock(&tx->ft.mutex); if (tx->chains) @@ -782,15 +923,15 @@ static void setup_fte_reg_a(struct mlx5_flow_spec *spec) misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); } -static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) +static void setup_fte_reg_c4(struct mlx5_flow_spec *spec, u32 reqid) { /* Pass policy check before choosing this SA */ spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; - MLX5_SET(fte_match_param, spec->match_criteria, - misc_parameters_2.metadata_reg_c_0, reqid); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_4); MLX5_SET(fte_match_param, spec->match_value, - misc_parameters_2.metadata_reg_c_0, reqid); + misc_parameters_2.metadata_reg_c_4, reqid); } static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) @@ -814,11 +955,24 @@ static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upsp } } -static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, +static enum mlx5_flow_namespace_type ipsec_fs_get_ns(struct mlx5e_ipsec *ipsec, + int type, u8 dir) +{ + if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) + return MLX5_FLOW_NAMESPACE_FDB; + + if (dir == XFRM_DEV_OFFLOAD_IN) + return MLX5_FLOW_NAMESPACE_KERNEL; + + return MLX5_FLOW_NAMESPACE_EGRESS; +} + +static int setup_modify_header(struct mlx5e_ipsec *ipsec, int type, u32 val, u8 dir, struct mlx5_flow_act *flow_act) { + enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, type, dir); u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - enum mlx5_flow_namespace_type ns_type; + struct mlx5_core_dev *mdev = ipsec->mdev; struct mlx5_modify_hdr *modify_hdr; MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); @@ -826,12 +980,10 @@ static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, case XFRM_DEV_OFFLOAD_IN: MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - ns_type = MLX5_FLOW_NAMESPACE_KERNEL; break; case XFRM_DEV_OFFLOAD_OUT: MLX5_SET(set_action_in, action, field, - MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); - ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + MLX5_ACTION_IN_FIELD_METADATA_REG_C_4); break; default: return -EINVAL; @@ -951,37 +1103,70 @@ free_reformatbf: return -EINVAL; } +static int get_reformat_type(struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + switch (attrs->dir) { + case XFRM_DEV_OFFLOAD_IN: + if (attrs->encap) + return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP; + return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; + case XFRM_DEV_OFFLOAD_OUT: + if (attrs->family == AF_INET) { + if (attrs->encap) + return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4; + return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; + } + + if (attrs->encap) + return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6; + return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; + default: + WARN_ON(true); + } + + return -EINVAL; +} + static int setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, struct mlx5_pkt_reformat_params *reformat_params) { - u8 *reformatbf; + struct udphdr *udphdr; + char *reformatbf; + size_t bfflen; __be32 spi; + void *hdr; + + reformat_params->type = get_reformat_type(attrs); + if (reformat_params->type < 0) + return reformat_params->type; switch (attrs->dir) { case XFRM_DEV_OFFLOAD_IN: - reformat_params->type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; break; case XFRM_DEV_OFFLOAD_OUT: - if (attrs->family == AF_INET) - reformat_params->type = - MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; - else - reformat_params->type = - MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; - - reformatbf = kzalloc(MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE, - GFP_KERNEL); + bfflen = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE; + if (attrs->encap) + bfflen += sizeof(*udphdr); + + reformatbf = kzalloc(bfflen, GFP_KERNEL); if (!reformatbf) return -ENOMEM; + hdr = reformatbf; + if (attrs->encap) { + udphdr = (struct udphdr *)reformatbf; + udphdr->source = attrs->sport; + udphdr->dest = attrs->dport; + hdr += sizeof(*udphdr); + } + /* convert to network format */ spi = htonl(attrs->spi); - memcpy(reformatbf, &spi, sizeof(spi)); + memcpy(hdr, &spi, sizeof(spi)); reformat_params->param_0 = attrs->authsize; - reformat_params->size = - MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE; + reformat_params->size = bfflen; reformat_params->data = reformatbf; break; default: @@ -991,26 +1176,17 @@ setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, return 0; } -static int setup_pkt_reformat(struct mlx5_core_dev *mdev, +static int setup_pkt_reformat(struct mlx5e_ipsec *ipsec, struct mlx5_accel_esp_xfrm_attrs *attrs, struct mlx5_flow_act *flow_act) { + enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, attrs->type, + attrs->dir); struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_core_dev *mdev = ipsec->mdev; struct mlx5_pkt_reformat *pkt_reformat; - enum mlx5_flow_namespace_type ns_type; int ret; - switch (attrs->dir) { - case XFRM_DEV_OFFLOAD_IN: - ns_type = MLX5_FLOW_NAMESPACE_KERNEL; - break; - case XFRM_DEV_OFFLOAD_OUT: - ns_type = MLX5_FLOW_NAMESPACE_EGRESS; - break; - default: - return -EINVAL; - } - switch (attrs->mode) { case XFRM_MODE_TRANSPORT: ret = setup_pkt_transport_reformat(attrs, &reformat_params); @@ -1047,9 +1223,9 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_flow_spec *spec; struct mlx5e_ipsec_rx *rx; struct mlx5_fc *counter; - int err; + int err = 0; - rx = rx_ft_get(mdev, ipsec, attrs->family); + rx = rx_ft_get(mdev, ipsec, attrs->family, attrs->type); if (IS_ERR(rx)) return PTR_ERR(rx); @@ -1068,14 +1244,19 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) setup_fte_esp(spec); setup_fte_no_frags(spec); - err = setup_modify_header(mdev, sa_entry->ipsec_obj_id | BIT(31), - XFRM_DEV_OFFLOAD_IN, &flow_act); + if (rx != ipsec->rx_esw) + err = setup_modify_header(ipsec, attrs->type, + sa_entry->ipsec_obj_id | BIT(31), + XFRM_DEV_OFFLOAD_IN, &flow_act); + else + err = mlx5_esw_ipsec_rx_setup_modify_header(sa_entry, &flow_act); + if (err) goto err_mod_header; switch (attrs->type) { case XFRM_DEV_OFFLOAD_PACKET: - err = setup_pkt_reformat(mdev, attrs, &flow_act); + err = setup_pkt_reformat(ipsec, attrs, &flow_act); if (err) goto err_pkt_reformat; break; @@ -1125,7 +1306,7 @@ err_pkt_reformat: err_mod_header: kvfree(spec); err_alloc: - rx_ft_put(ipsec, attrs->family); + rx_ft_put(ipsec, attrs->family, attrs->type); return err; } @@ -1142,7 +1323,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_fc *counter; int err; - tx = tx_ft_get(mdev, ipsec); + tx = tx_ft_get(mdev, ipsec, attrs->type); if (IS_ERR(tx)) return PTR_ERR(tx); @@ -1168,8 +1349,8 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) break; case XFRM_DEV_OFFLOAD_PACKET: if (attrs->reqid) - setup_fte_reg_c0(spec, attrs->reqid); - err = setup_pkt_reformat(mdev, attrs, &flow_act); + setup_fte_reg_c4(spec, attrs->reqid); + err = setup_pkt_reformat(ipsec, attrs, &flow_act); if (err) goto err_pkt_reformat; break; @@ -1218,7 +1399,7 @@ err_add_cnt: err_pkt_reformat: kvfree(spec); err_alloc: - tx_ft_put(ipsec); + tx_ft_put(ipsec, attrs->type); return err; } @@ -1226,15 +1407,16 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx; + struct mlx5e_ipsec *ipsec = pol_entry->ipsec; struct mlx5_flow_destination dest[2] = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; struct mlx5_flow_table *ft; + struct mlx5e_ipsec_tx *tx; int err, dstn = 0; - ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio); + ft = tx_ft_get_policy(mdev, ipsec, attrs->prio, attrs->type); if (IS_ERR(ft)) return PTR_ERR(ft); @@ -1244,6 +1426,7 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) goto err_alloc; } + tx = ipsec_tx(ipsec, attrs->type); if (attrs->family == AF_INET) setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); else @@ -1258,7 +1441,7 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) if (!attrs->reqid) break; - err = setup_modify_header(mdev, attrs->reqid, + err = setup_modify_header(ipsec, attrs->type, attrs->reqid, XFRM_DEV_OFFLOAD_OUT, &flow_act); if (err) goto err_mod_header; @@ -1277,6 +1460,8 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } flow_act.flags |= FLOW_ACT_NO_APPEND; + if (tx == ipsec->tx_esw && tx->chains) + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; dest[dstn].ft = tx->ft.sa; dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dstn++; @@ -1298,7 +1483,7 @@ err_action: err_mod_header: kvfree(spec); err_alloc: - tx_ft_put_policy(pol_entry->ipsec, attrs->prio); + tx_ft_put_policy(ipsec, attrs->prio, attrs->type); return err; } @@ -1306,6 +1491,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5e_ipsec *ipsec = pol_entry->ipsec; struct mlx5_flow_destination dest[2]; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; @@ -1314,11 +1500,12 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) struct mlx5e_ipsec_rx *rx; int err, dstn = 0; - ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio); + ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio, + attrs->type); if (IS_ERR(ft)) return PTR_ERR(ft); - rx = ipsec_rx(pol_entry->ipsec, attrs->family); + rx = ipsec_rx(pol_entry->ipsec, attrs->family, attrs->type); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { @@ -1350,6 +1537,8 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } flow_act.flags |= FLOW_ACT_NO_APPEND; + if (rx == ipsec->rx_esw && rx->chains) + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest[dstn].ft = rx->ft.sa; dstn++; @@ -1367,88 +1556,110 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) err_action: kvfree(spec); err_alloc: - rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio); + rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio, attrs->type); return err; } +static void ipsec_fs_destroy_single_counter(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec_fc *fc) +{ + mlx5_fc_destroy(mdev, fc->drop); + mlx5_fc_destroy(mdev, fc->cnt); + kfree(fc); +} + static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; struct mlx5_core_dev *mdev = ipsec->mdev; - struct mlx5e_ipsec_tx *tx = ipsec->tx; - mlx5_fc_destroy(mdev, tx->fc->drop); - mlx5_fc_destroy(mdev, tx->fc->cnt); - kfree(tx->fc); - mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); - mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); - kfree(rx_ipv4->fc); + ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); + ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); + if (ipsec->is_uplink_rep) { + ipsec_fs_destroy_single_counter(mdev, ipsec->tx_esw->fc); + ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); + } } -static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) +static struct mlx5e_ipsec_fc *ipsec_fs_init_single_counter(struct mlx5_core_dev *mdev) { - struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; - struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; - struct mlx5_core_dev *mdev = ipsec->mdev; - struct mlx5e_ipsec_tx *tx = ipsec->tx; struct mlx5e_ipsec_fc *fc; struct mlx5_fc *counter; int err; - fc = kzalloc(sizeof(*rx_ipv4->fc), GFP_KERNEL); + fc = kzalloc(sizeof(*fc), GFP_KERNEL); if (!fc) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - /* Both IPv4 and IPv6 point to same flow counters struct. */ - rx_ipv4->fc = fc; - rx_ipv6->fc = fc; counter = mlx5_fc_create(mdev, false); if (IS_ERR(counter)) { err = PTR_ERR(counter); - goto err_rx_cnt; + goto err_cnt; } - fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); if (IS_ERR(counter)) { err = PTR_ERR(counter); - goto err_rx_drop; + goto err_drop; } - fc->drop = counter; - fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); - if (!fc) { - err = -ENOMEM; - goto err_tx_fc; + + return fc; + +err_drop: + mlx5_fc_destroy(mdev, fc->cnt); +err_cnt: + kfree(fc); + return ERR_PTR(err); +} + +static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_fc *fc; + int err; + + fc = ipsec_fs_init_single_counter(mdev); + if (IS_ERR(fc)) { + err = PTR_ERR(fc); + goto err_rx_cnt; } + ipsec->rx_ipv4->fc = fc; - tx->fc = fc; - counter = mlx5_fc_create(mdev, false); - if (IS_ERR(counter)) { - err = PTR_ERR(counter); + fc = ipsec_fs_init_single_counter(mdev); + if (IS_ERR(fc)) { + err = PTR_ERR(fc); goto err_tx_cnt; } + ipsec->tx->fc = fc; - fc->cnt = counter; - counter = mlx5_fc_create(mdev, false); - if (IS_ERR(counter)) { - err = PTR_ERR(counter); - goto err_tx_drop; + if (ipsec->is_uplink_rep) { + fc = ipsec_fs_init_single_counter(mdev); + if (IS_ERR(fc)) { + err = PTR_ERR(fc); + goto err_rx_esw_cnt; + } + ipsec->rx_esw->fc = fc; + + fc = ipsec_fs_init_single_counter(mdev); + if (IS_ERR(fc)) { + err = PTR_ERR(fc); + goto err_tx_esw_cnt; + } + ipsec->tx_esw->fc = fc; } - fc->drop = counter; + /* Both IPv4 and IPv6 point to same flow counters struct. */ + ipsec->rx_ipv6->fc = ipsec->rx_ipv4->fc; return 0; -err_tx_drop: - mlx5_fc_destroy(mdev, tx->fc->cnt); +err_tx_esw_cnt: + ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); +err_rx_esw_cnt: + ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); err_tx_cnt: - kfree(tx->fc); -err_tx_fc: - mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); -err_rx_drop: - mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); + ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); err_rx_cnt: - kfree(rx_ipv4->fc); return err; } @@ -1458,6 +1669,7 @@ void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) struct mlx5e_ipsec *ipsec = priv->ipsec; struct mlx5e_ipsec_hw_stats *stats; struct mlx5e_ipsec_fc *fc; + u64 packets, bytes; stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; @@ -1479,14 +1691,94 @@ void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, &stats->ipsec_tx_drop_bytes); + + if (ipsec->is_uplink_rep) { + fc = ipsec->rx_esw->fc; + if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { + stats->ipsec_rx_pkts += packets; + stats->ipsec_rx_bytes += bytes; + } + + if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { + stats->ipsec_rx_drop_pkts += packets; + stats->ipsec_rx_drop_bytes += bytes; + } + + fc = ipsec->tx_esw->fc; + if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { + stats->ipsec_tx_pkts += packets; + stats->ipsec_tx_bytes += bytes; + } + + if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { + stats->ipsec_tx_drop_pkts += packets; + stats->ipsec_tx_drop_bytes += bytes; + } + } +} + +#ifdef CONFIG_MLX5_ESWITCH +static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) +{ + struct mlx5_eswitch *esw = mdev->priv.eswitch; + int err = 0; + + if (esw) + down_write(&esw->mode_lock); + + if (mdev->num_block_ipsec) { + err = -EBUSY; + goto unlock; + } + + mdev->num_block_tc++; + +unlock: + if (esw) + up_write(&esw->mode_lock); + + return err; +} +#else +static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) +{ + if (mdev->num_block_ipsec) + return -EBUSY; + + mdev->num_block_tc++; + return 0; +} +#endif + +static void mlx5e_ipsec_unblock_tc_offload(struct mlx5_core_dev *mdev) +{ + mdev->num_block_tc++; } int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { + int err; + + if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) { + err = mlx5e_ipsec_block_tc_offload(sa_entry->ipsec->mdev); + if (err) + return err; + } + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) - return tx_add_rule(sa_entry); + err = tx_add_rule(sa_entry); + else + err = rx_add_rule(sa_entry); + + if (err) + goto err_out; + + return 0; - return rx_add_rule(sa_entry); +err_out: + if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) + mlx5e_ipsec_unblock_tc_offload(sa_entry->ipsec->mdev); + return err; } void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) @@ -1499,21 +1791,40 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) if (ipsec_rule->pkt_reformat) mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); + if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) + mlx5e_ipsec_unblock_tc_offload(mdev); + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { - tx_ft_put(sa_entry->ipsec); + tx_ft_put(sa_entry->ipsec, sa_entry->attrs.type); return; } mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); - rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family); + mlx5_esw_ipsec_rx_id_mapping_remove(sa_entry); + rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family, sa_entry->attrs.type); } int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) { + int err; + + err = mlx5e_ipsec_block_tc_offload(pol_entry->ipsec->mdev); + if (err) + return err; + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) - return tx_add_policy(pol_entry); + err = tx_add_policy(pol_entry); + else + err = rx_add_policy(pol_entry); + + if (err) + goto err_out; + + return 0; - return rx_add_policy(pol_entry); +err_out: + mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); + return err; } void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) @@ -1523,16 +1834,18 @@ void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) mlx5_del_flow_rules(ipsec_rule->rule); + mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { rx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.family, - pol_entry->attrs.prio); + pol_entry->attrs.prio, pol_entry->attrs.type); return; } if (ipsec_rule->modify_hdr) mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); - tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio); + tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio, pol_entry->attrs.type); } void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) @@ -1540,7 +1853,7 @@ void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) if (!ipsec->tx) return; - if (mlx5_ipsec_device_caps(ipsec->mdev) & MLX5_IPSEC_CAP_ROCE) + if (ipsec->roce) mlx5_ipsec_fs_roce_cleanup(ipsec->roce); ipsec_fs_destroy_counters(ipsec); @@ -1555,12 +1868,24 @@ void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) mutex_destroy(&ipsec->rx_ipv6->ft.mutex); WARN_ON(ipsec->rx_ipv6->ft.refcnt); kfree(ipsec->rx_ipv6); + + if (ipsec->is_uplink_rep) { + xa_destroy(&ipsec->rx_esw->ipsec_obj_id_map); + + mutex_destroy(&ipsec->tx_esw->ft.mutex); + WARN_ON(ipsec->tx_esw->ft.refcnt); + kfree(ipsec->tx_esw); + + mutex_destroy(&ipsec->rx_esw->ft.mutex); + WARN_ON(ipsec->rx_esw->ft.refcnt); + kfree(ipsec->rx_esw); + } } int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) { struct mlx5_core_dev *mdev = ipsec->mdev; - struct mlx5_flow_namespace *ns; + struct mlx5_flow_namespace *ns, *ns_esw; int err = -ENOMEM; ns = mlx5_get_flow_namespace(ipsec->mdev, @@ -1568,9 +1893,23 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) if (!ns) return -EOPNOTSUPP; + if (ipsec->is_uplink_rep) { + ns_esw = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_FDB); + if (!ns_esw) + return -EOPNOTSUPP; + + ipsec->tx_esw = kzalloc(sizeof(*ipsec->tx_esw), GFP_KERNEL); + if (!ipsec->tx_esw) + return -ENOMEM; + + ipsec->rx_esw = kzalloc(sizeof(*ipsec->rx_esw), GFP_KERNEL); + if (!ipsec->rx_esw) + goto err_rx_esw; + } + ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); if (!ipsec->tx) - return -ENOMEM; + goto err_tx; ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); if (!ipsec->rx_ipv4) @@ -1589,8 +1928,14 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) mutex_init(&ipsec->rx_ipv6->ft.mutex); ipsec->tx->ns = ns; - if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) + if (ipsec->is_uplink_rep) { + mutex_init(&ipsec->tx_esw->ft.mutex); + mutex_init(&ipsec->rx_esw->ft.mutex); + ipsec->tx_esw->ns = ns_esw; + xa_init_flags(&ipsec->rx_esw->ipsec_obj_id_map, XA_FLAGS_ALLOC1); + } else if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) { ipsec->roce = mlx5_ipsec_fs_roce_init(mdev); + } return 0; @@ -1600,6 +1945,10 @@ err_rx_ipv6: kfree(ipsec->rx_ipv4); err_rx_ipv4: kfree(ipsec->tx); +err_tx: + kfree(ipsec->rx_esw); +err_rx_esw: + kfree(ipsec->tx_esw); return err; } @@ -1621,10 +1970,12 @@ void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) bool mlx5e_ipsec_fs_tunnel_enabled(struct mlx5e_ipsec_sa_entry *sa_entry) { - struct mlx5e_ipsec_rx *rx = - ipsec_rx(sa_entry->ipsec, sa_entry->attrs.family); - struct mlx5e_ipsec_tx *tx = sa_entry->ipsec->tx; + struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; + struct mlx5e_ipsec_rx *rx; + struct mlx5e_ipsec_tx *tx; + rx = ipsec_rx(sa_entry->ipsec, attrs->family, attrs->type); + tx = ipsec_tx(sa_entry->ipsec, attrs->type); if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) return tx->allow_tunnel_mode; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index a3554bde3e07..3245d1c9d539 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -45,8 +45,9 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap)) caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD; - if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) && - MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level)) + if ((MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level)) || + MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, ignore_flow_level)) caps |= MLX5_IPSEC_CAP_PRIO; if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, @@ -54,6 +55,12 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_l3_esp_tunnel_to_l2)) caps |= MLX5_IPSEC_CAP_TUNNEL; + + if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, + reformat_add_esp_transport_over_udp) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, + reformat_del_esp_transport_over_udp)) + caps |= MLX5_IPSEC_CAP_ESPINUDP; } if (mlx5_get_roce_state(mdev) && diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 8d995e304869..51a144246ea6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -37,6 +37,7 @@ #include "ipsec.h" #include "ipsec_rxtx.h" #include "en.h" +#include "esw/ipsec_fs.h" enum { MLX5E_IPSEC_TX_SYNDROME_OFFLOAD = 0x8, @@ -311,9 +312,8 @@ enum { void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - struct mlx5_cqe64 *cqe) + u32 ipsec_meta_data) { - u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_ipsec *ipsec = priv->ipsec; struct mlx5e_ipsec_sa_entry *sa_entry; @@ -358,3 +358,24 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_syndrome); } } + +int mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv *priv, u32 id, u32 *metadata) +{ + struct mlx5e_ipsec *ipsec = priv->ipsec; + u32 ipsec_obj_id; + int err; + + if (!ipsec || !ipsec->is_uplink_rep) + return -EINVAL; + + err = mlx5_esw_ipsec_rx_ipsec_obj_id_search(priv, id, &ipsec_obj_id); + if (err) { + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); + return err; + } + + *metadata = MLX5_IPSEC_METADATA_CREATE(ipsec_obj_id, + MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED); + + return 0; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index 1878a70b9031..9ee014a8ad24 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -43,6 +43,7 @@ #define MLX5_IPSEC_METADATA_MARKER(metadata) (((metadata) >> 31) & 0x1) #define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(5, 0)) #define MLX5_IPSEC_METADATA_HANDLE(metadata) ((metadata) & GENMASK(23, 0)) +#define MLX5_IPSEC_METADATA_CREATE(id, syndrome) ((id) | ((syndrome) << 24)) struct mlx5e_accel_tx_ipsec_state { struct xfrm_offload *xo; @@ -66,7 +67,8 @@ void mlx5e_ipsec_handle_tx_wqe(struct mlx5e_tx_wqe *wqe, struct mlx5_wqe_inline_seg *inlseg); void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - struct mlx5_cqe64 *cqe); + u32 ipsec_meta_data); +int mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv *priv, u32 id, u32 *metadata); static inline unsigned int mlx5e_ipsec_tx_ids_len(struct mlx5e_accel_tx_ipsec_state *ipsec_st) { return ipsec_st->tailen; @@ -145,7 +147,7 @@ mlx5e_ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, static inline void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - struct mlx5_cqe64 *cqe) + u32 ipsec_meta_data) {} static inline bool mlx5e_ipsec_eseg_meta(struct mlx5_wqe_eth_seg *eseg) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 27861b68ced5..04195a673a6b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -2163,8 +2163,8 @@ static u32 mlx5e_get_priv_flags(struct net_device *netdev) return priv->channels.params.pflags; } -int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, - u32 *rule_locs) +static int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, + u32 *rule_locs) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -2181,7 +2181,7 @@ int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, return mlx5e_ethtool_get_rxnfc(priv, info, rule_locs); } -int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +static int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) { struct mlx5e_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c index aac32e505c14..3eccdadc0357 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c @@ -96,10 +96,6 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv, case UDP_V4_FLOW: case TCP_V6_FLOW: case UDP_V6_FLOW: - max_tuples = ETHTOOL_NUM_L3_L4_FTS; - prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples); - eth_ft = ðtool->l3_l4_ft[prio]; - break; case IP_USER_FLOW: case IPV6_USER_FLOW: max_tuples = ETHTOOL_NUM_L3_L4_FTS; @@ -900,10 +896,16 @@ static int mlx5e_set_rss_hash_opt(struct mlx5e_priv *priv, struct ethtool_rxnfc *nfc) { u8 rx_hash_field = 0; + u32 flow_type = 0; + u32 rss_idx = 0; int err; int tt; - tt = flow_type_to_traffic_type(nfc->flow_type); + if (nfc->flow_type & FLOW_RSS) + rss_idx = nfc->rss_context; + + flow_type = flow_type_mask(nfc->flow_type); + tt = flow_type_to_traffic_type(flow_type); if (tt < 0) return tt; @@ -911,10 +913,10 @@ static int mlx5e_set_rss_hash_opt(struct mlx5e_priv *priv, * on src IP, dest IP, TCP/UDP src port and TCP/UDP dest * port. */ - if (nfc->flow_type != TCP_V4_FLOW && - nfc->flow_type != TCP_V6_FLOW && - nfc->flow_type != UDP_V4_FLOW && - nfc->flow_type != UDP_V6_FLOW) + if (flow_type != TCP_V4_FLOW && + flow_type != TCP_V6_FLOW && + flow_type != UDP_V4_FLOW && + flow_type != UDP_V6_FLOW) return -EOPNOTSUPP; if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST | @@ -931,7 +933,7 @@ static int mlx5e_set_rss_hash_opt(struct mlx5e_priv *priv, rx_hash_field |= MLX5_HASH_FIELD_SEL_L4_DPORT; mutex_lock(&priv->state_lock); - err = mlx5e_rx_res_rss_set_hash_fields(priv->rx_res, tt, rx_hash_field); + err = mlx5e_rx_res_rss_set_hash_fields(priv->rx_res, rss_idx, tt, rx_hash_field); mutex_unlock(&priv->state_lock); return err; @@ -940,14 +942,23 @@ static int mlx5e_set_rss_hash_opt(struct mlx5e_priv *priv, static int mlx5e_get_rss_hash_opt(struct mlx5e_priv *priv, struct ethtool_rxnfc *nfc) { - u32 hash_field = 0; + int hash_field = 0; + u32 flow_type = 0; + u32 rss_idx = 0; int tt; - tt = flow_type_to_traffic_type(nfc->flow_type); + if (nfc->flow_type & FLOW_RSS) + rss_idx = nfc->rss_context; + + flow_type = flow_type_mask(nfc->flow_type); + tt = flow_type_to_traffic_type(flow_type); if (tt < 0) return tt; - hash_field = mlx5e_rx_res_rss_get_hash_fields(priv->rx_res, tt); + hash_field = mlx5e_rx_res_rss_get_hash_fields(priv->rx_res, rss_idx, tt); + if (hash_field < 0) + return hash_field; + nfc->data = 0; if (hash_field & MLX5_HASH_FIELD_SEL_SRC_IP) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 99b3843396f3..2fdb8895aecd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -399,15 +399,13 @@ static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw, } static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, - struct mlx5_devcom *devcom, struct mlx5e_rep_sq *rep_sq, int i) { - struct mlx5_eswitch *peer_esw = NULL; struct mlx5_flow_handle *flow_rule; - int tmp; + struct mlx5_devcom_comp_dev *tmp; + struct mlx5_eswitch *peer_esw; - mlx5_devcom_for_each_peer_entry(devcom, MLX5_DEVCOM_ESW_OFFLOADS, - peer_esw, tmp) { + mlx5_devcom_for_each_peer_entry(esw->devcom, peer_esw, tmp) { u16 peer_rule_idx = MLX5_CAP_GEN(peer_esw->dev, vhca_id); struct mlx5e_rep_sq_peer *sq_peer; int err; @@ -443,7 +441,6 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, struct mlx5_flow_handle *flow_rule; struct mlx5e_rep_priv *rpriv; struct mlx5e_rep_sq *rep_sq; - struct mlx5_devcom *devcom; bool devcom_locked = false; int err; int i; @@ -451,10 +448,10 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, if (esw->mode != MLX5_ESWITCH_OFFLOADS) return 0; - devcom = esw->dev->priv.devcom; rpriv = mlx5e_rep_to_rep_priv(rep); - if (mlx5_devcom_comp_is_ready(devcom, MLX5_DEVCOM_ESW_OFFLOADS) && - mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) + + if (mlx5_devcom_comp_is_ready(esw->devcom) && + mlx5_devcom_for_each_peer_begin(esw->devcom)) devcom_locked = true; for (i = 0; i < sqns_num; i++) { @@ -477,7 +474,7 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, xa_init(&rep_sq->sq_peer); if (devcom_locked) { - err = mlx5e_sqs2vport_add_peers_rules(esw, rep, devcom, rep_sq, i); + err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i); if (err) { mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); xa_destroy(&rep_sq->sq_peer); @@ -490,7 +487,7 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, } if (devcom_locked) - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_for_each_peer_end(esw->devcom); return 0; @@ -498,7 +495,7 @@ out_err: mlx5e_sqs2vport_stop(esw, rep); if (devcom_locked) - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_for_each_peer_end(esw->devcom); return err; } @@ -1339,6 +1336,7 @@ static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = { &MLX5E_STATS_GRP(channels), &MLX5E_STATS_GRP(per_port_buff_congest), #ifdef CONFIG_MLX5_EN_IPSEC + &MLX5E_STATS_GRP(ipsec_hw), &MLX5E_STATS_GRP(ipsec_sw), #endif &MLX5E_STATS_GRP(ptp), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 41d37159e027..f7bb5f4aaaca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1543,7 +1543,8 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, mlx5e_ktls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt); if (unlikely(mlx5_ipsec_is_rx_flow(cqe))) - mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe); + mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, + be32_to_cpu(cqe->ft_metadata)); if (unlikely(mlx5e_macsec_is_rx_flow(cqe))) mlx5e_macsec_offload_handle_rx_skb(netdev, skb, cqe); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 92377632f9e0..a2f2f344f206 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1668,11 +1668,10 @@ int mlx5e_tc_query_route_vport(struct net_device *out_dev, struct net_device *ro { struct mlx5e_priv *out_priv, *route_priv; struct mlx5_core_dev *route_mdev; - struct mlx5_devcom *devcom; + struct mlx5_devcom_comp_dev *pos; struct mlx5_eswitch *esw; u16 vhca_id; int err; - int i; out_priv = netdev_priv(out_dev); esw = out_priv->mdev->priv.eswitch; @@ -1688,10 +1687,8 @@ int mlx5e_tc_query_route_vport(struct net_device *out_dev, struct net_device *ro return err; rcu_read_lock(); - devcom = out_priv->mdev->priv.devcom; err = -ENODEV; - mlx5_devcom_for_each_peer_entry_rcu(devcom, MLX5_DEVCOM_ESW_OFFLOADS, - esw, i) { + mlx5_devcom_for_each_peer_entry_rcu(esw->devcom, esw, pos) { err = mlx5_eswitch_vhca_id_to_vport(esw, vhca_id, vport); if (!err) break; @@ -2046,15 +2043,15 @@ static void mlx5e_tc_del_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { if (mlx5e_is_eswitch_flow(flow)) { - struct mlx5_devcom *devcom = flow->priv->mdev->priv.devcom; + struct mlx5_devcom_comp_dev *devcom = flow->priv->mdev->priv.eswitch->devcom; - if (!mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) { + if (!mlx5_devcom_for_each_peer_begin(devcom)) { mlx5e_tc_del_fdb_flow(priv, flow); return; } mlx5e_tc_del_fdb_peers_flow(flow); - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_for_each_peer_end(devcom); mlx5e_tc_del_fdb_flow(priv, flow); } else { mlx5e_tc_del_nic_flow(priv, flow); @@ -2608,29 +2605,29 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, match_level = outer_match_level; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_META) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_CVLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_TCP) | - BIT(FLOW_DISSECTOR_KEY_IP) | - BIT(FLOW_DISSECTOR_KEY_CT) | - BIT(FLOW_DISSECTOR_KEY_ENC_IP) | - BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | - BIT(FLOW_DISSECTOR_KEY_ICMP) | - BIT(FLOW_DISSECTOR_KEY_MPLS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_META) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_CT) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ICMP) | + BIT_ULL(FLOW_DISSECTOR_KEY_MPLS))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported key"); - netdev_dbg(priv->netdev, "Unsupported key used: 0x%x\n", + netdev_dbg(priv->netdev, "Unsupported key used: 0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } @@ -4231,8 +4228,7 @@ static bool is_peer_flow_needed(struct mlx5e_tc_flow *flow) flow_flag_test(flow, INGRESS); bool act_is_encap = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT); - bool esw_paired = mlx5_devcom_comp_is_ready(esw_attr->in_mdev->priv.devcom, - MLX5_DEVCOM_ESW_OFFLOADS); + bool esw_paired = mlx5_devcom_comp_is_ready(esw_attr->in_mdev->priv.eswitch->devcom); if (!esw_paired) return false; @@ -4486,14 +4482,13 @@ mlx5e_add_fdb_flow(struct mlx5e_priv *priv, struct net_device *filter_dev, struct mlx5e_tc_flow **__flow) { - struct mlx5_devcom *devcom = priv->mdev->priv.devcom; + struct mlx5_devcom_comp_dev *devcom = priv->mdev->priv.eswitch->devcom, *pos; struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *in_rep = rpriv->rep; struct mlx5_core_dev *in_mdev = priv->mdev; struct mlx5_eswitch *peer_esw; struct mlx5e_tc_flow *flow; int err; - int i; flow = __mlx5e_add_fdb_flow(priv, f, flow_flags, filter_dev, in_rep, in_mdev); @@ -4505,27 +4500,25 @@ mlx5e_add_fdb_flow(struct mlx5e_priv *priv, return 0; } - if (!mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) { + if (!mlx5_devcom_for_each_peer_begin(devcom)) { err = -ENODEV; goto clean_flow; } - mlx5_devcom_for_each_peer_entry(devcom, - MLX5_DEVCOM_ESW_OFFLOADS, - peer_esw, i) { + mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { err = mlx5e_tc_add_fdb_peer_flow(f, flow, flow_flags, peer_esw); if (err) goto peer_clean; } - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_for_each_peer_end(devcom); *__flow = flow; return 0; peer_clean: mlx5e_tc_del_fdb_peers_flow(flow); - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_for_each_peer_end(devcom); clean_flow: mlx5e_tc_del_fdb_flow(priv, flow); return err; @@ -4628,6 +4621,46 @@ static bool is_flow_rule_duplicate_allowed(struct net_device *dev, return netif_is_lag_port(dev) && rpriv && rpriv->rep->vport != MLX5_VPORT_UPLINK; } +/* As IPsec and TC order is not aligned between software and hardware-offload, + * either IPsec offload or TC offload, not both, is allowed for a specific interface. + */ +static bool is_tc_ipsec_order_check_needed(struct net_device *filter, struct mlx5e_priv *priv) +{ + if (!IS_ENABLED(CONFIG_MLX5_EN_IPSEC)) + return false; + + if (filter != priv->netdev) + return false; + + if (mlx5e_eswitch_vf_rep(priv->netdev)) + return false; + + return true; +} + +static int mlx5e_tc_block_ipsec_offload(struct net_device *filter, struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + + if (!is_tc_ipsec_order_check_needed(filter, priv)) + return 0; + + if (mdev->num_block_tc) + return -EBUSY; + + mdev->num_block_ipsec++; + + return 0; +} + +static void mlx5e_tc_unblock_ipsec_offload(struct net_device *filter, struct mlx5e_priv *priv) +{ + if (!is_tc_ipsec_order_check_needed(filter, priv)) + return; + + priv->mdev->num_block_ipsec--; +} + int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags) { @@ -4640,6 +4673,10 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, if (!mlx5_esw_hold(priv->mdev)) return -EBUSY; + err = mlx5e_tc_block_ipsec_offload(dev, priv); + if (err) + goto esw_release; + mlx5_esw_get(priv->mdev); rcu_read_lock(); @@ -4685,7 +4722,9 @@ rcu_unlock: err_free: mlx5e_flow_put(priv, flow); out: + mlx5e_tc_unblock_ipsec_offload(dev, priv); mlx5_esw_put(priv->mdev); +esw_release: mlx5_esw_release(priv->mdev); return err; } @@ -4726,6 +4765,7 @@ int mlx5e_delete_flower(struct net_device *dev, struct mlx5e_priv *priv, trace_mlx5e_delete_flower(f); mlx5e_flow_put(priv, flow); + mlx5e_tc_unblock_ipsec_offload(dev, priv); mlx5_esw_put(priv->mdev); return 0; @@ -4743,7 +4783,7 @@ int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv, int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags) { - struct mlx5_devcom *devcom = priv->mdev->priv.devcom; + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct rhashtable *tc_ht = get_tc_ht(priv, flags); struct mlx5e_tc_flow *flow; struct mlx5_fc *counter; @@ -4779,7 +4819,7 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, /* Under multipath it's possible for one rule to be currently * un-offloaded while the other rule is offloaded. */ - if (!mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) + if (esw && !mlx5_devcom_for_each_peer_begin(esw->devcom)) goto out; if (flow_flag_test(flow, DUP)) { @@ -4810,7 +4850,8 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, } no_peer_counter: - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + if (esw) + mlx5_devcom_for_each_peer_end(esw->devcom); out: flow_stats_update(&f->stats, bytes, packets, 0, lastuse, FLOW_ACTION_HW_STATS_DELAYED); @@ -5215,11 +5256,12 @@ void mlx5e_tc_ht_cleanup(struct rhashtable *tc_ht) int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) { const size_t sz_enc_opts = sizeof(struct tunnel_match_enc_opts); + struct netdev_phys_item_id ppid; struct mlx5e_rep_priv *rpriv; struct mapping_ctx *mapping; struct mlx5_eswitch *esw; struct mlx5e_priv *priv; - u64 mapping_id; + u64 mapping_id, key; int err = 0; rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv); @@ -5273,7 +5315,11 @@ int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) goto err_action_counter; } - mlx5_esw_offloads_devcom_init(esw); + err = dev_get_port_parent_id(priv->netdev, &ppid, false); + if (!err) { + memcpy(&key, &ppid.id, sizeof(key)); + mlx5_esw_offloads_devcom_init(esw, key); + } return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index f4fe1daa4afd..e36294b7ade2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -652,30 +652,30 @@ mlx5_esw_bridge_ingress_flow_peer_create(u16 vport_num, u16 esw_owner_vhca_id, struct mlx5_esw_bridge_vlan *vlan, u32 counter_id, struct mlx5_esw_bridge *bridge) { - struct mlx5_devcom *devcom = bridge->br_offloads->esw->dev->priv.devcom; + struct mlx5_devcom_comp_dev *devcom = bridge->br_offloads->esw->devcom, *pos; struct mlx5_eswitch *tmp, *peer_esw = NULL; static struct mlx5_flow_handle *handle; - int i; - if (!mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) + if (!mlx5_devcom_for_each_peer_begin(devcom)) return ERR_PTR(-ENODEV); - mlx5_devcom_for_each_peer_entry(devcom, - MLX5_DEVCOM_ESW_OFFLOADS, - tmp, i) { + mlx5_devcom_for_each_peer_entry(devcom, tmp, pos) { if (mlx5_esw_is_owner(tmp, vport_num, esw_owner_vhca_id)) { peer_esw = tmp; break; } } + if (!peer_esw) { - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); - return ERR_PTR(-ENODEV); + handle = ERR_PTR(-ENODEV); + goto out; } handle = mlx5_esw_bridge_ingress_flow_with_esw_create(vport_num, addr, vlan, counter_id, bridge, peer_esw); - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + +out: + mlx5_devcom_for_each_peer_end(devcom); return handle; } @@ -1391,8 +1391,8 @@ mlx5_esw_bridge_fdb_entry_init(struct net_device *dev, u16 vport_num, u16 esw_ow mlx5_fc_id(counter), bridge); if (IS_ERR(handle)) { err = PTR_ERR(handle); - esw_warn(esw->dev, "Failed to create ingress flow(vport=%u,err=%d)\n", - vport_num, err); + esw_warn(esw->dev, "Failed to create ingress flow(vport=%u,err=%d,peer=%d)\n", + vport_num, err, peer); goto err_ingress_flow_create; } entry->ingress_handle = handle; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c index 2455f8b93c1e..7a01714b3780 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c @@ -539,30 +539,29 @@ mlx5_esw_bridge_mcast_filter_flow_create(struct mlx5_esw_bridge_port *port) static struct mlx5_flow_handle * mlx5_esw_bridge_mcast_filter_flow_peer_create(struct mlx5_esw_bridge_port *port) { - struct mlx5_devcom *devcom = port->bridge->br_offloads->esw->dev->priv.devcom; + struct mlx5_devcom_comp_dev *devcom = port->bridge->br_offloads->esw->devcom, *pos; struct mlx5_eswitch *tmp, *peer_esw = NULL; static struct mlx5_flow_handle *handle; - int i; - if (!mlx5_devcom_for_each_peer_begin(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) + if (!mlx5_devcom_for_each_peer_begin(devcom)) return ERR_PTR(-ENODEV); - mlx5_devcom_for_each_peer_entry(devcom, - MLX5_DEVCOM_ESW_OFFLOADS, - tmp, i) { + mlx5_devcom_for_each_peer_entry(devcom, tmp, pos) { if (mlx5_esw_is_owner(tmp, port->vport_num, port->esw_owner_vhca_id)) { peer_esw = tmp; break; } } + if (!peer_esw) { - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); - return ERR_PTR(-ENODEV); + handle = ERR_PTR(-ENODEV); + goto out; } handle = mlx5_esw_bridge_mcast_flow_with_esw_create(port, peer_esw); - mlx5_devcom_for_each_peer_end(devcom, MLX5_DEVCOM_ESW_OFFLOADS); +out: + mlx5_devcom_for_each_peer_end(devcom); return handle; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index af779c700278..433541ac36a7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -132,10 +132,8 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vpo if (IS_ERR(vport)) return; - if (vport->dl_port->devlink_rate) { - mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devl_rate_leaf_destroy(vport->dl_port); - } + mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); + devl_rate_leaf_destroy(vport->dl_port); devl_port_unregister(vport->dl_port); mlx5_esw_dl_port_free(vport->dl_port); @@ -211,10 +209,8 @@ void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num if (IS_ERR(vport)) return; - if (vport->dl_port->devlink_rate) { - mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devl_rate_leaf_destroy(vport->dl_port); - } + mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); + devl_rate_leaf_destroy(vport->dl_port); devl_port_unregister(vport->dl_port); vport->dl_port = NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c new file mode 100644 index 000000000000..455746952260 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "fs_core.h" +#include "eswitch.h" +#include "en_accel/ipsec.h" +#include "esw/ipsec_fs.h" +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) +#include "en/tc_priv.h" +#endif + +enum { + MLX5_ESW_IPSEC_RX_POL_FT_LEVEL, + MLX5_ESW_IPSEC_RX_ESP_FT_LEVEL, + MLX5_ESW_IPSEC_RX_ESP_FT_CHK_LEVEL, +}; + +enum { + MLX5_ESW_IPSEC_TX_POL_FT_LEVEL, + MLX5_ESW_IPSEC_TX_ESP_FT_LEVEL, + MLX5_ESW_IPSEC_TX_ESP_FT_CNT_LEVEL, +}; + +static void esw_ipsec_rx_status_drop_destroy(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx) +{ + mlx5_del_flow_rules(rx->status_drop.rule); + mlx5_destroy_flow_group(rx->status_drop.group); + mlx5_fc_destroy(ipsec->mdev, rx->status_drop_cnt); +} + +static void esw_ipsec_rx_status_pass_destroy(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx) +{ + mlx5_del_flow_rules(rx->status.rule); + mlx5_chains_put_table(esw_chains(ipsec->mdev->priv.eswitch), 0, 1, 0); +} + +static int esw_ipsec_rx_status_drop_create(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table *ft = rx->ft.status; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_fc *flow_counter; + struct mlx5_flow_spec *spec; + struct mlx5_flow_group *g; + u32 *flow_group_in; + int err = 0; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!flow_group_in || !spec) { + err = -ENOMEM; + goto err_out; + } + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); + g = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + mlx5_core_err(mdev, + "Failed to add ipsec rx status drop flow group, err=%d\n", err); + goto err_out; + } + + flow_counter = mlx5_fc_create(mdev, false); + if (IS_ERR(flow_counter)) { + err = PTR_ERR(flow_counter); + mlx5_core_err(mdev, + "Failed to add ipsec rx status drop rule counter, err=%d\n", err); + goto err_cnt; + } + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(flow_counter); + spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; + rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, + "Failed to add ipsec rx status drop rule, err=%d\n", err); + goto err_rule; + } + + rx->status_drop.group = g; + rx->status_drop.rule = rule; + rx->status_drop_cnt = flow_counter; + + kvfree(flow_group_in); + kvfree(spec); + return 0; + +err_rule: + mlx5_fc_destroy(mdev, flow_counter); +err_cnt: + mlx5_destroy_flow_group(g); +err_out: + kvfree(flow_group_in); + kvfree(spec); + return err; +} + +static int esw_ipsec_rx_status_pass_create(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) +{ + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int err; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_2.ipsec_syndrome); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.ipsec_syndrome, 0); + spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; + flow_act.flags = FLOW_ACT_NO_APPEND; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_warn(ipsec->mdev, + "Failed to add ipsec rx status pass rule, err=%d\n", err); + goto err_rule; + } + + rx->status.rule = rule; + kvfree(spec); + return 0; + +err_rule: + kvfree(spec); + return err; +} + +void mlx5_esw_ipsec_rx_status_destroy(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx) +{ + esw_ipsec_rx_status_pass_destroy(ipsec, rx); + esw_ipsec_rx_status_drop_destroy(ipsec, rx); +} + +int mlx5_esw_ipsec_rx_status_create(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) +{ + int err; + + err = esw_ipsec_rx_status_drop_create(ipsec, rx); + if (err) + return err; + + err = esw_ipsec_rx_status_pass_create(ipsec, rx, dest); + if (err) + goto err_pass_create; + + return 0; + +err_pass_create: + esw_ipsec_rx_status_drop_destroy(ipsec, rx); + return err; +} + +void mlx5_esw_ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx_create_attr *attr) +{ + attr->prio = FDB_CRYPTO_INGRESS; + attr->pol_level = MLX5_ESW_IPSEC_RX_POL_FT_LEVEL; + attr->sa_level = MLX5_ESW_IPSEC_RX_ESP_FT_LEVEL; + attr->status_level = MLX5_ESW_IPSEC_RX_ESP_FT_CHK_LEVEL; + attr->chains_ns = MLX5_FLOW_NAMESPACE_FDB; +} + +int mlx5_esw_ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, + struct mlx5_flow_destination *dest) +{ + dest->type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest->ft = mlx5_chains_get_table(esw_chains(ipsec->mdev->priv.eswitch), 0, 1, 0); + + return 0; +} + +int mlx5_esw_ipsec_rx_setup_modify_header(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_flow_act *flow_act) +{ + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5_modify_hdr *modify_hdr; + u32 mapped_id; + int err; + + err = xa_alloc_bh(&ipsec->rx_esw->ipsec_obj_id_map, &mapped_id, + xa_mk_value(sa_entry->ipsec_obj_id), + XA_LIMIT(1, ESW_IPSEC_RX_MAPPED_ID_MASK), 0); + if (err) + return err; + + /* reuse tunnel bits for ipsec, + * tun_id is always 0 and tun_opts is mapped to ipsec_obj_id. + */ + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); + MLX5_SET(set_action_in, action, offset, ESW_ZONE_ID_BITS); + MLX5_SET(set_action_in, action, length, + ESW_TUN_ID_BITS + ESW_TUN_OPTS_BITS); + MLX5_SET(set_action_in, action, data, mapped_id); + + modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_FDB, + 1, action); + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + goto err_header_alloc; + } + + sa_entry->rx_mapped_id = mapped_id; + flow_act->modify_hdr = modify_hdr; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + + return 0; + +err_header_alloc: + xa_erase_bh(&ipsec->rx_esw->ipsec_obj_id_map, mapped_id); + return err; +} + +void mlx5_esw_ipsec_rx_id_mapping_remove(struct mlx5e_ipsec_sa_entry *sa_entry) +{ + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + + if (sa_entry->rx_mapped_id) + xa_erase_bh(&ipsec->rx_esw->ipsec_obj_id_map, + sa_entry->rx_mapped_id); +} + +int mlx5_esw_ipsec_rx_ipsec_obj_id_search(struct mlx5e_priv *priv, u32 id, + u32 *ipsec_obj_id) +{ + struct mlx5e_ipsec *ipsec = priv->ipsec; + void *val; + + val = xa_load(&ipsec->rx_esw->ipsec_obj_id_map, id); + if (!val) + return -ENOENT; + + *ipsec_obj_id = xa_to_value(val); + + return 0; +} + +void mlx5_esw_ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_tx_create_attr *attr) +{ + attr->prio = FDB_CRYPTO_EGRESS; + attr->pol_level = MLX5_ESW_IPSEC_TX_POL_FT_LEVEL; + attr->sa_level = MLX5_ESW_IPSEC_TX_ESP_FT_LEVEL; + attr->cnt_level = MLX5_ESW_IPSEC_TX_ESP_FT_CNT_LEVEL; + attr->chains_ns = MLX5_FLOW_NAMESPACE_FDB; +} + +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) +static int mlx5_esw_ipsec_modify_flow_dests(struct mlx5_eswitch *esw, + struct mlx5e_tc_flow *flow) +{ + struct mlx5_esw_flow_attr *esw_attr; + struct mlx5_flow_attr *attr; + int err; + + attr = flow->attr; + esw_attr = attr->esw_attr; + if (esw_attr->out_count - esw_attr->split_count > 1) + return 0; + + err = mlx5_eswitch_restore_ipsec_rule(esw, flow->rule[0], esw_attr, + esw_attr->out_count - 1); + + return err; +} +#endif + +void mlx5_esw_ipsec_restore_dest_uplink(struct mlx5_core_dev *mdev) +{ +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) + struct mlx5_eswitch *esw = mdev->priv.eswitch; + struct mlx5_eswitch_rep *rep; + struct mlx5e_rep_priv *rpriv; + struct rhashtable_iter iter; + struct mlx5e_tc_flow *flow; + unsigned long i; + int err; + + xa_for_each(&esw->offloads.vport_reps, i, rep) { + rpriv = rep->rep_data[REP_ETH].priv; + if (!rpriv || !rpriv->netdev) + continue; + + rhashtable_walk_enter(&rpriv->tc_ht, &iter); + rhashtable_walk_start(&iter); + while ((flow = rhashtable_walk_next(&iter)) != NULL) { + if (IS_ERR(flow)) + continue; + + err = mlx5_esw_ipsec_modify_flow_dests(esw, flow); + if (err) + mlx5_core_warn_once(mdev, + "Faided to modify flow dests for IPsec"); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + } +#endif +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.h new file mode 100644 index 000000000000..0c90f7a8b0d3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_ESW_IPSEC_FS_H__ +#define __MLX5_ESW_IPSEC_FS_H__ + +struct mlx5e_ipsec; +struct mlx5e_ipsec_sa_entry; + +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_esw_ipsec_rx_status_destroy(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx); +int mlx5_esw_ipsec_rx_status_create(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest); +void mlx5_esw_ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx_create_attr *attr); +int mlx5_esw_ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, + struct mlx5_flow_destination *dest); +int mlx5_esw_ipsec_rx_setup_modify_header(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_flow_act *flow_act); +void mlx5_esw_ipsec_rx_id_mapping_remove(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5_esw_ipsec_rx_ipsec_obj_id_search(struct mlx5e_priv *priv, u32 id, + u32 *ipsec_obj_id); +void mlx5_esw_ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_tx_create_attr *attr); +void mlx5_esw_ipsec_restore_dest_uplink(struct mlx5_core_dev *mdev); +#else +static inline void mlx5_esw_ipsec_rx_status_destroy(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx) {} + +static inline int mlx5_esw_ipsec_rx_status_create(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) +{ + return -EINVAL; +} + +static inline void mlx5_esw_ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx_create_attr *attr) {} + +static inline int mlx5_esw_ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, + struct mlx5_flow_destination *dest) +{ + return -EINVAL; +} + +static inline int mlx5_esw_ipsec_rx_setup_modify_header(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_flow_act *flow_act) +{ + return -EINVAL; +} + +static inline void mlx5_esw_ipsec_rx_id_mapping_remove(struct mlx5e_ipsec_sa_entry *sa_entry) {} + +static inline int mlx5_esw_ipsec_rx_ipsec_obj_id_search(struct mlx5e_priv *priv, u32 id, + u32 *ipsec_obj_id) +{ + return -EINVAL; +} + +static inline void mlx5_esw_ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_tx_create_attr *attr) {} + +static inline void mlx5_esw_ipsec_restore_dest_uplink(struct mlx5_core_dev *mdev) {} +#endif /* CONFIG_MLX5_ESWITCH */ +#endif /* __MLX5_ESW_IPSEC_FS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 243c455f1029..4a7a13169a90 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1068,9 +1068,8 @@ static void mlx5_eswitch_clear_ec_vf_vports_info(struct mlx5_eswitch *esw) } } -/* Public E-Switch API */ -int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, - enum mlx5_eswitch_vport_event enabled_events) +static int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, + enum mlx5_eswitch_vport_event enabled_events) { int err; @@ -1078,7 +1077,7 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, if (err) return err; - err = esw_offloads_load_rep(esw, vport_num); + err = mlx5_esw_offloads_load_rep(esw, vport_num); if (err) goto err_rep; @@ -1089,9 +1088,9 @@ err_rep: return err; } -void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num) +static void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num) { - esw_offloads_unload_rep(esw, vport_num); + mlx5_esw_offloads_unload_rep(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index ae0dc8a3060d..f3a6a1826e00 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -254,6 +254,7 @@ struct mlx5_esw_offload { struct mlx5_flow_group *vport_rx_group; struct mlx5_flow_group *vport_rx_drop_group; struct mlx5_flow_handle *vport_rx_drop_rule; + struct mlx5_flow_table *ft_ipsec_tx_pol; struct xarray vport_reps; struct list_head peer_flows[MLX5_MAX_PORTS]; struct mutex peer_mutex; @@ -269,6 +270,7 @@ struct mlx5_esw_offload { u8 inline_mode; atomic64_t num_flows; u64 num_block_encap; + u64 num_block_mode; enum devlink_eswitch_encap_mode encap; struct ida vport_metadata_ida; unsigned int host_number; /* ECPF supports one external host */ @@ -354,6 +356,7 @@ struct mlx5_eswitch { } params; struct blocking_notifier_head n_head; struct xarray paired; + struct mlx5_devcom_comp_dev *devcom; }; void esw_offloads_disable(struct mlx5_eswitch *esw); @@ -381,8 +384,9 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf); void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw); void mlx5_eswitch_disable(struct mlx5_eswitch *esw); -void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw); +void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key); void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw); +bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, u16 vport, const u8 *mac); int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, @@ -725,15 +729,8 @@ void mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, u16 vport, struct mlx5_flow_spec *spec); -int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num); -void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num); - -int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num); -void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num); - -int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, - enum mlx5_eswitch_vport_event enabled_events); -void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num); +int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num); +void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num); int mlx5_eswitch_load_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs, enum mlx5_eswitch_vport_event enabled_events); @@ -788,6 +785,11 @@ int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw); bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev); void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev); +int mlx5_eswitch_block_mode_trylock(struct mlx5_core_dev *dev); +void mlx5_eswitch_block_mode_unlock(struct mlx5_core_dev *dev, int err); +void mlx5_eswitch_unblock_mode_lock(struct mlx5_core_dev *dev); +void mlx5_eswitch_unblock_mode_unlock(struct mlx5_core_dev *dev); + static inline int mlx5_eswitch_num_vfs(struct mlx5_eswitch *esw) { if (mlx5_esw_allowed(esw)) @@ -809,6 +811,8 @@ mlx5_eswitch_get_slow_fdb(struct mlx5_eswitch *esw) return esw->fdb_table.offloads.slow_fdb; } +int mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, + struct mlx5_esw_flow_attr *esw_attr, int attr_idx); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } @@ -816,8 +820,9 @@ static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {} static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; } static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {} static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {} -static inline void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw) {} +static inline void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) {} static inline void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) {} +static inline bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) { return false; } static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } static inline int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; } @@ -866,6 +871,14 @@ static inline bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev) static inline void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) { } + +static inline int mlx5_eswitch_block_mode_trylock(struct mlx5_core_dev *dev) { return 0; } + +static inline void mlx5_eswitch_block_mode_unlock(struct mlx5_core_dev *dev, int err) {} + +static inline void mlx5_eswitch_unblock_mode_lock(struct mlx5_core_dev *dev) {} + +static inline void mlx5_eswitch_unblock_mode_unlock(struct mlx5_core_dev *dev) {} #endif /* CONFIG_MLX5_ESWITCH */ #endif /* __MLX5_ESWITCH_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index e59380ee1ead..723dff87e6d5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -424,10 +424,51 @@ esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 l mlx5_chains_put_table(chains, chain, prio, level); } +static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) +{ + return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); +} + +static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, + struct mlx5_esw_flow_attr *esw_attr, + int attr_idx) +{ + if (esw->offloads.ft_ipsec_tx_pol && + esw_attr->dests[attr_idx].rep && + esw_attr->dests[attr_idx].rep->vport == MLX5_VPORT_UPLINK && + /* To be aligned with software, encryption is needed only for tunnel device */ + (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && + esw_attr->dests[attr_idx].rep != esw_attr->in_rep && + esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) + return true; + + return false; +} + +static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, + struct mlx5_esw_flow_attr *esw_attr) +{ + int i; + + if (!esw->offloads.ft_ipsec_tx_pol) + return true; + + for (i = 0; i < esw_attr->split_count; i++) + if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) + return false; + + for (i = esw_attr->split_count; i < esw_attr->out_count; i++) + if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && + (esw_attr->out_count - esw_attr->split_count > 1)) + return false; + + return true; +} + static void -esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, - struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, - int attr_idx, int dest_idx, bool pkt_reformat) +esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, + struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, + int attr_idx, int dest_idx, bool pkt_reformat) { dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport; @@ -449,6 +490,33 @@ esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *f } } +static void +esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, + struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, + int attr_idx, int dest_idx, bool pkt_reformat) +{ + dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; + dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + if (pkt_reformat && + esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; + } +} + +static void +esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, + struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, + int attr_idx, int dest_idx, bool pkt_reformat) +{ + if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) + esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, + attr_idx, dest_idx, pkt_reformat); + else + esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, + attr_idx, dest_idx, pkt_reformat); +} + static int esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, @@ -575,6 +643,9 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) return ERR_PTR(-EOPNOTSUPP); + if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) + return ERR_PTR(-EOPNOTSUPP); + dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); if (!dest) return ERR_PTR(-ENOMEM); @@ -884,6 +955,17 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + if (rep->vport == MLX5_VPORT_UPLINK && on_esw->offloads.ft_ipsec_tx_pol) { + dest.ft = on_esw->offloads.ft_ipsec_tx_pol; + flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + } else { + dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; + dest.vport.num = rep->vport; + dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); + dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; + } + if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && rep->vport == MLX5_VPORT_UPLINK) spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; @@ -2390,7 +2472,7 @@ static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) __esw_offloads_unload_rep(esw, rep, rep_type); } -int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) +static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_eswitch_rep *rep; int rep_type; @@ -2414,7 +2496,7 @@ err_reps: return err; } -void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) +static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_eswitch_rep *rep; int rep_type; @@ -2424,7 +2506,7 @@ void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) __esw_offloads_unload_rep(esw, rep, rep_type); } -int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) +int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) { int err; @@ -2448,7 +2530,7 @@ load_err: return err; } -void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) +void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) { if (esw->mode != MLX5_ESWITCH_OFFLOADS) return; @@ -2810,7 +2892,6 @@ static int mlx5_esw_offloads_devcom_event(int event, void *event_data) { struct mlx5_eswitch *esw = my_data; - struct mlx5_devcom *devcom = esw->dev->priv.devcom; struct mlx5_eswitch *peer_esw = event_data; u16 esw_i, peer_esw_i; bool esw_paired; @@ -2832,6 +2913,7 @@ static int mlx5_esw_offloads_devcom_event(int event, err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); if (err) goto err_out; + err = mlx5_esw_offloads_pair(esw, peer_esw); if (err) goto err_peer; @@ -2850,7 +2932,7 @@ static int mlx5_esw_offloads_devcom_event(int event, esw->num_peers++; peer_esw->num_peers++; - mlx5_devcom_comp_set_ready(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); + mlx5_devcom_comp_set_ready(esw->devcom, true); break; case ESW_OFFLOADS_DEVCOM_UNPAIR: @@ -2860,7 +2942,7 @@ static int mlx5_esw_offloads_devcom_event(int event, peer_esw->num_peers--; esw->num_peers--; if (!esw->num_peers && !peer_esw->num_peers) - mlx5_devcom_comp_set_ready(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); + mlx5_devcom_comp_set_ready(esw->devcom, false); xa_erase(&peer_esw->paired, esw_i); xa_erase(&esw->paired, peer_esw_i); mlx5_esw_offloads_unpair(peer_esw, esw); @@ -2885,9 +2967,8 @@ err_out: return err; } -void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw) +void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) { - struct mlx5_devcom *devcom = esw->dev->priv.devcom; int i; for (i = 0; i < MLX5_MAX_PORTS; i++) @@ -2897,38 +2978,44 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw) if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) return; - if (!mlx5_lag_is_supported(esw->dev)) + if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && + !mlx5_lag_is_supported(esw->dev)) return; xa_init(&esw->paired); - mlx5_devcom_register_component(devcom, - MLX5_DEVCOM_ESW_OFFLOADS, - mlx5_esw_offloads_devcom_event, - esw); - esw->num_peers = 0; - mlx5_devcom_send_event(devcom, - MLX5_DEVCOM_ESW_OFFLOADS, + esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, + MLX5_DEVCOM_ESW_OFFLOADS, + key, + mlx5_esw_offloads_devcom_event, + esw); + if (IS_ERR_OR_NULL(esw->devcom)) + return; + + mlx5_devcom_send_event(esw->devcom, ESW_OFFLOADS_DEVCOM_PAIR, - ESW_OFFLOADS_DEVCOM_UNPAIR, esw); + ESW_OFFLOADS_DEVCOM_UNPAIR, + esw); } void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) { - struct mlx5_devcom *devcom = esw->dev->priv.devcom; - - if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) + if (IS_ERR_OR_NULL(esw->devcom)) return; - if (!mlx5_lag_is_supported(esw->dev)) - return; - - mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, + mlx5_devcom_send_event(esw->devcom, + ESW_OFFLOADS_DEVCOM_UNPAIR, ESW_OFFLOADS_DEVCOM_UNPAIR, - ESW_OFFLOADS_DEVCOM_UNPAIR, esw); + esw); - mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); + mlx5_devcom_unregister_component(esw->devcom); xa_destroy(&esw->paired); + esw->devcom = NULL; +} + +bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) +{ + return mlx5_devcom_comp_is_ready(esw->devcom); } bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) @@ -3355,7 +3442,7 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; /* Uplink vport rep must load first. */ - err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); + err = mlx5_esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); if (err) goto err_uplink; @@ -3366,7 +3453,7 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) return 0; err_vports: - esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); + mlx5_esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); err_uplink: esw_offloads_steering_cleanup(esw); err_steering_init: @@ -3404,7 +3491,7 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw, void esw_offloads_disable(struct mlx5_eswitch *esw) { mlx5_eswitch_disable_pf_vf_vports(esw); - esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); + mlx5_esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); esw_set_passing_vport_metadata(esw, false); esw_offloads_steering_cleanup(esw); mapping_destroy(esw->offloads.reg_c0_obj_pool); @@ -3501,6 +3588,69 @@ static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink *devlink) return net_eq(devl_net, netdev_net); } +int mlx5_eswitch_block_mode_trylock(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + struct mlx5_eswitch *esw; + int err; + + devl_lock(devlink); + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) { + /* Failure means no eswitch => not possible to change eswitch mode */ + devl_unlock(devlink); + return 0; + } + + err = mlx5_esw_try_lock(esw); + if (err < 0) { + devl_unlock(devlink); + return err; + } + + return 0; +} + +void mlx5_eswitch_block_mode_unlock(struct mlx5_core_dev *dev, int err) +{ + struct devlink *devlink = priv_to_devlink(dev); + struct mlx5_eswitch *esw; + + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return; + + if (!err) + esw->offloads.num_block_mode++; + mlx5_esw_unlock(esw); + devl_unlock(devlink); +} + +void mlx5_eswitch_unblock_mode_lock(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + struct mlx5_eswitch *esw; + + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return; + + down_write(&esw->mode_lock); +} + +void mlx5_eswitch_unblock_mode_unlock(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + struct mlx5_eswitch *esw; + + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return; + + esw->offloads.num_block_mode--; + up_write(&esw->mode_lock); +} + int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3534,6 +3684,13 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (cur_mlx5_mode == mlx5_mode) goto unlock; + if (esw->offloads.num_block_mode) { + NL_SET_ERR_MSG_MOD(extack, + "Can't change eswitch mode when IPsec SA and/or policies are configured"); + err = -EOPNOTSUPP; + goto unlock; + } + mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { @@ -4119,7 +4276,6 @@ int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enab { struct mlx5_eswitch *esw; struct mlx5_vport *vport; - int err = -EOPNOTSUPP; esw = mlx5_devlink_eswitch_get(port->devlink); if (IS_ERR(esw)) @@ -4127,7 +4283,7 @@ int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enab if (!MLX5_CAP_GEN(esw->dev, migration)) { NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); - return err; + return -EOPNOTSUPP; } vport = mlx5_devlink_port_fn_get_vport(port, esw); @@ -4137,12 +4293,9 @@ int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enab } mutex_lock(&esw->state_lock); - if (vport->enabled) { - *is_enabled = vport->info.mig_enabled; - err = 0; - } + *is_enabled = vport->info.mig_enabled; mutex_unlock(&esw->state_lock); - return err; + return 0; } int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, @@ -4171,10 +4324,6 @@ int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, } mutex_lock(&esw->state_lock); - if (!vport->enabled) { - NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); - goto out; - } if (vport->info.mig_enabled == enable) { err = 0; @@ -4218,7 +4367,6 @@ int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, { struct mlx5_eswitch *esw; struct mlx5_vport *vport; - int err = -EOPNOTSUPP; esw = mlx5_devlink_eswitch_get(port->devlink); if (IS_ERR(esw)) @@ -4231,12 +4379,9 @@ int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, } mutex_lock(&esw->state_lock); - if (vport->enabled) { - *is_enabled = vport->info.roce_enabled; - err = 0; - } + *is_enabled = vport->info.roce_enabled; mutex_unlock(&esw->state_lock); - return err; + return 0; } int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, @@ -4245,10 +4390,10 @@ int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); struct mlx5_eswitch *esw; struct mlx5_vport *vport; - int err = -EOPNOTSUPP; void *query_ctx; void *hca_caps; u16 vport_num; + int err; esw = mlx5_devlink_eswitch_get(port->devlink); if (IS_ERR(esw)) @@ -4262,10 +4407,6 @@ int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, vport_num = vport->vport; mutex_lock(&esw->state_lock); - if (!vport->enabled) { - NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); - goto out; - } if (vport->info.roce_enabled == enable) { err = 0; @@ -4303,3 +4444,19 @@ out: mutex_unlock(&esw->state_lock); return err; } + +int +mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, + struct mlx5_esw_flow_attr *esw_attr, int attr_idx) +{ + struct mlx5_flow_destination new_dest = {}; + struct mlx5_flow_destination old_dest = {}; + + if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) + return 0; + + esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); + esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); + + return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 6b069fa411c5..a3228502f866 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -1122,7 +1122,7 @@ int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle, } for (i = 0; i < handle->num_rules; i++) { - if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr)) + if (mlx5_flow_dests_cmp(old_dest, &handle->rule[i]->dest_attr)) return _mlx5_modify_rule_destination(handle->rule[i], new_dest); } @@ -3050,6 +3050,12 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) if (err) goto out_err; + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_CRYPTO_INGRESS, 3); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + err = create_fdb_fast_path(steering); if (err) goto out_err; @@ -3072,6 +3078,12 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) goto out_err; } + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_CRYPTO_EGRESS, 3); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + /* We put this priority last, knowing that nothing will get here * unless explicitly forwarded to. This is possible because the * slow path tables have catch all rules and nothing gets passed diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index f0a074b2fcdf..af3fac090b82 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -835,7 +835,7 @@ static bool mlx5_shared_fdb_supported(struct mlx5_lag *ldev) dev = ldev->pf[MLX5_LAG_P1].dev; if (is_mdev_switchdev_mode(dev) && mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && - mlx5_devcom_comp_is_ready(dev->priv.devcom, MLX5_DEVCOM_ESW_OFFLOADS) && + mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) && MLX5_CAP_ESW(dev, esw_shared_ingress_acl) && mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1) return true; @@ -1268,16 +1268,6 @@ recheck: mlx5_ldev_put(ldev); } -bool mlx5_lag_is_supported(struct mlx5_core_dev *dev) -{ - if (!MLX5_CAP_GEN(dev, vport_group_manager) || - !MLX5_CAP_GEN(dev, lag_master) || - MLX5_CAP_GEN(dev, num_lag_ports) < 2 || - MLX5_CAP_GEN(dev, num_lag_ports) > MLX5_MAX_PORTS) - return false; - return true; -} - void mlx5_lag_add_mdev(struct mlx5_core_dev *dev) { int err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index a061b1873e27..481e92f39fe6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -74,8 +74,6 @@ struct mlx5_lag { struct lag_mpesw lag_mpesw; }; -bool mlx5_lag_is_supported(struct mlx5_core_dev *dev); - static inline struct mlx5_lag * mlx5_lag_dev(struct mlx5_core_dev *dev) { @@ -115,4 +113,14 @@ void mlx5_lag_remove_devices(struct mlx5_lag *ldev); int mlx5_deactivate_lag(struct mlx5_lag *ldev); void mlx5_lag_add_devices(struct mlx5_lag *ldev); +static inline bool mlx5_lag_is_supported(struct mlx5_core_dev *dev) +{ + if (!MLX5_CAP_GEN(dev, vport_group_manager) || + !MLX5_CAP_GEN(dev, lag_master) || + MLX5_CAP_GEN(dev, num_lag_ports) < 2 || + MLX5_CAP_GEN(dev, num_lag_ports) > MLX5_MAX_PORTS) + return false; + return true; +} + #endif /* __MLX5_LAG_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 78c94b22bdc0..feb62d952643 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -2,214 +2,273 @@ /* Copyright (c) 2018 Mellanox Technologies */ #include <linux/mlx5/vport.h> +#include <linux/list.h> #include "lib/devcom.h" #include "mlx5_core.h" -static LIST_HEAD(devcom_list); +static LIST_HEAD(devcom_dev_list); +static LIST_HEAD(devcom_comp_list); +/* protect device list */ +static DEFINE_MUTEX(dev_list_lock); +/* protect component list */ +static DEFINE_MUTEX(comp_list_lock); -#define devcom_for_each_component(priv, comp, iter) \ - for (iter = 0; \ - comp = &(priv)->components[iter], iter < MLX5_DEVCOM_NUM_COMPONENTS; \ - iter++) +#define devcom_for_each_component(iter) \ + list_for_each_entry(iter, &devcom_comp_list, comp_list) -struct mlx5_devcom_component { - struct { - void __rcu *data; - } device[MLX5_DEVCOM_PORTS_SUPPORTED]; +struct mlx5_devcom_dev { + struct list_head list; + struct mlx5_core_dev *dev; + struct kref ref; +}; +struct mlx5_devcom_comp { + struct list_head comp_list; + enum mlx5_devcom_component id; + u64 key; + struct list_head comp_dev_list_head; mlx5_devcom_event_handler_t handler; - struct rw_semaphore sem; + struct kref ref; bool ready; + struct rw_semaphore sem; }; -struct mlx5_devcom_list { +struct mlx5_devcom_comp_dev { struct list_head list; - - struct mlx5_devcom_component components[MLX5_DEVCOM_NUM_COMPONENTS]; - struct mlx5_core_dev *devs[MLX5_DEVCOM_PORTS_SUPPORTED]; + struct mlx5_devcom_comp *comp; + struct mlx5_devcom_dev *devc; + void __rcu *data; }; -struct mlx5_devcom { - struct mlx5_devcom_list *priv; - int idx; -}; - -static struct mlx5_devcom_list *mlx5_devcom_list_alloc(void) +static bool devcom_dev_exists(struct mlx5_core_dev *dev) { - struct mlx5_devcom_component *comp; - struct mlx5_devcom_list *priv; - int i; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return NULL; + struct mlx5_devcom_dev *iter; - devcom_for_each_component(priv, comp, i) - init_rwsem(&comp->sem); + list_for_each_entry(iter, &devcom_dev_list, list) + if (iter->dev == dev) + return true; - return priv; + return false; } -static struct mlx5_devcom *mlx5_devcom_alloc(struct mlx5_devcom_list *priv, - u8 idx) +static struct mlx5_devcom_dev * +mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev) { - struct mlx5_devcom *devcom; + struct mlx5_devcom_dev *devc; - devcom = kzalloc(sizeof(*devcom), GFP_KERNEL); - if (!devcom) + devc = kzalloc(sizeof(*devc), GFP_KERNEL); + if (!devc) return NULL; - devcom->priv = priv; - devcom->idx = idx; - return devcom; + devc->dev = dev; + kref_init(&devc->ref); + return devc; } -/* Must be called with intf_mutex held */ -struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev) +struct mlx5_devcom_dev * +mlx5_devcom_register_device(struct mlx5_core_dev *dev) { - struct mlx5_devcom_list *priv = NULL, *iter; - struct mlx5_devcom *devcom = NULL; - bool new_priv = false; - u64 sguid0, sguid1; - int idx, i; - - if (!mlx5_core_is_pf(dev)) - return NULL; - if (MLX5_CAP_GEN(dev, num_lag_ports) > MLX5_DEVCOM_PORTS_SUPPORTED) - return NULL; - - mlx5_dev_list_lock(); - sguid0 = mlx5_query_nic_system_image_guid(dev); - list_for_each_entry(iter, &devcom_list, list) { - /* There is at least one device in iter */ - struct mlx5_core_dev *tmp_dev; - - idx = -1; - for (i = 0; i < MLX5_DEVCOM_PORTS_SUPPORTED; i++) { - if (iter->devs[i]) - tmp_dev = iter->devs[i]; - else - idx = i; - } - - if (idx == -1) - continue; - - sguid1 = mlx5_query_nic_system_image_guid(tmp_dev); - if (sguid0 != sguid1) - continue; - - priv = iter; - break; - } + struct mlx5_devcom_dev *devc; - if (!priv) { - priv = mlx5_devcom_list_alloc(); - if (!priv) { - devcom = ERR_PTR(-ENOMEM); - goto out; - } + mutex_lock(&dev_list_lock); - idx = 0; - new_priv = true; + if (devcom_dev_exists(dev)) { + devc = ERR_PTR(-EEXIST); + goto out; } - priv->devs[idx] = dev; - devcom = mlx5_devcom_alloc(priv, idx); - if (!devcom) { - if (new_priv) - kfree(priv); - devcom = ERR_PTR(-ENOMEM); + devc = mlx5_devcom_dev_alloc(dev); + if (!devc) { + devc = ERR_PTR(-ENOMEM); goto out; } - if (new_priv) - list_add(&priv->list, &devcom_list); + list_add_tail(&devc->list, &devcom_dev_list); out: - mlx5_dev_list_unlock(); - return devcom; + mutex_unlock(&dev_list_lock); + return devc; } -/* Must be called with intf_mutex held */ -void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom) +static void +mlx5_devcom_dev_release(struct kref *ref) { - struct mlx5_devcom_list *priv; - int i; + struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref); - if (IS_ERR_OR_NULL(devcom)) - return; + mutex_lock(&dev_list_lock); + list_del(&devc->list); + mutex_unlock(&dev_list_lock); + kfree(devc); +} - mlx5_dev_list_lock(); - priv = devcom->priv; - priv->devs[devcom->idx] = NULL; +void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc) +{ + if (!IS_ERR_OR_NULL(devc)) + kref_put(&devc->ref, mlx5_devcom_dev_release); +} - kfree(devcom); +static struct mlx5_devcom_comp * +mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler) +{ + struct mlx5_devcom_comp *comp; - for (i = 0; i < MLX5_DEVCOM_PORTS_SUPPORTED; i++) - if (priv->devs[i]) - break; + comp = kzalloc(sizeof(*comp), GFP_KERNEL); + if (!comp) + return ERR_PTR(-ENOMEM); - if (i != MLX5_DEVCOM_PORTS_SUPPORTED) - goto out; + comp->id = id; + comp->key = key; + comp->handler = handler; + init_rwsem(&comp->sem); + kref_init(&comp->ref); + INIT_LIST_HEAD(&comp->comp_dev_list_head); - list_del(&priv->list); - kfree(priv); -out: - mlx5_dev_list_unlock(); + return comp; } -void mlx5_devcom_register_component(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - mlx5_devcom_event_handler_t handler, - void *data) +static void +mlx5_devcom_comp_release(struct kref *ref) { - struct mlx5_devcom_component *comp; + struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref); - if (IS_ERR_OR_NULL(devcom)) - return; + mutex_lock(&comp_list_lock); + list_del(&comp->comp_list); + mutex_unlock(&comp_list_lock); + kfree(comp); +} + +static struct mlx5_devcom_comp_dev * +devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, + struct mlx5_devcom_comp *comp, + void *data) +{ + struct mlx5_devcom_comp_dev *devcom; - WARN_ON(!data); + devcom = kzalloc(sizeof(*devcom), GFP_KERNEL); + if (!devcom) + return ERR_PTR(-ENOMEM); + + kref_get(&devc->ref); + devcom->devc = devc; + devcom->comp = comp; + rcu_assign_pointer(devcom->data, data); - comp = &devcom->priv->components[id]; down_write(&comp->sem); - comp->handler = handler; - rcu_assign_pointer(comp->device[devcom->idx].data, data); + list_add_tail(&devcom->list, &comp->comp_dev_list_head); up_write(&comp->sem); + + return devcom; } -void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id) +static void +devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) { - struct mlx5_devcom_component *comp; - - if (IS_ERR_OR_NULL(devcom)) - return; + struct mlx5_devcom_comp *comp = devcom->comp; - comp = &devcom->priv->components[id]; down_write(&comp->sem); - RCU_INIT_POINTER(comp->device[devcom->idx].data, NULL); + list_del(&devcom->list); up_write(&comp->sem); - synchronize_rcu(); + + kref_put(&devcom->devc->ref, mlx5_devcom_dev_release); + kfree(devcom); + kref_put(&comp->ref, mlx5_devcom_comp_release); } -int mlx5_devcom_send_event(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, +static bool +devcom_component_equal(struct mlx5_devcom_comp *devcom, + enum mlx5_devcom_component id, + u64 key) +{ + return devcom->id == id && devcom->key == key; +} + +static struct mlx5_devcom_comp * +devcom_component_get(struct mlx5_devcom_dev *devc, + enum mlx5_devcom_component id, + u64 key, + mlx5_devcom_event_handler_t handler) +{ + struct mlx5_devcom_comp *comp; + + devcom_for_each_component(comp) { + if (devcom_component_equal(comp, id, key)) { + if (handler == comp->handler) { + kref_get(&comp->ref); + return comp; + } + + mlx5_core_err(devc->dev, + "Cannot register existing devcom component with different handler\n"); + return ERR_PTR(-EINVAL); + } + } + + return NULL; +} + +struct mlx5_devcom_comp_dev * +mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, + enum mlx5_devcom_component id, + u64 key, + mlx5_devcom_event_handler_t handler, + void *data) +{ + struct mlx5_devcom_comp_dev *devcom; + struct mlx5_devcom_comp *comp; + + if (IS_ERR_OR_NULL(devc)) + return NULL; + + mutex_lock(&comp_list_lock); + comp = devcom_component_get(devc, id, key, handler); + if (IS_ERR(comp)) { + devcom = ERR_PTR(-EINVAL); + goto out_unlock; + } + + if (!comp) { + comp = mlx5_devcom_comp_alloc(id, key, handler); + if (IS_ERR(comp)) { + devcom = ERR_CAST(comp); + goto out_unlock; + } + list_add_tail(&comp->comp_list, &devcom_comp_list); + } + mutex_unlock(&comp_list_lock); + + devcom = devcom_alloc_comp_dev(devc, comp, data); + if (IS_ERR(devcom)) + kref_put(&comp->ref, mlx5_devcom_comp_release); + + return devcom; + +out_unlock: + mutex_unlock(&comp_list_lock); + return devcom; +} + +void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom) +{ + if (!IS_ERR_OR_NULL(devcom)) + devcom_free_comp_dev(devcom); +} + +int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, int event, int rollback_event, void *event_data) { - struct mlx5_devcom_component *comp; - int err = -ENODEV, i; + struct mlx5_devcom_comp *comp = devcom->comp; + struct mlx5_devcom_comp_dev *pos; + int err = 0; + void *data; if (IS_ERR_OR_NULL(devcom)) - return err; + return -ENODEV; - comp = &devcom->priv->components[id]; down_write(&comp->sem); - for (i = 0; i < MLX5_DEVCOM_PORTS_SUPPORTED; i++) { - void *data = rcu_dereference_protected(comp->device[i].data, - lockdep_is_held(&comp->sem)); + list_for_each_entry(pos, &comp->comp_dev_list_head, list) { + data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); - if (i != devcom->idx && data) { + if (pos != devcom && data) { err = comp->handler(event, data, event_data); if (err) goto rollback; @@ -220,48 +279,43 @@ int mlx5_devcom_send_event(struct mlx5_devcom *devcom, return 0; rollback: - while (i--) { - void *data = rcu_dereference_protected(comp->device[i].data, - lockdep_is_held(&comp->sem)); + if (list_entry_is_head(pos, &comp->comp_dev_list_head, list)) + goto out; + pos = list_prev_entry(pos, list); + list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) { + data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); - if (i != devcom->idx && data) + if (pos != devcom && data) comp->handler(rollback_event, data, event_data); } - +out: up_write(&comp->sem); return err; } -void mlx5_devcom_comp_set_ready(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - bool ready) +void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready) { - struct mlx5_devcom_component *comp; - - comp = &devcom->priv->components[id]; - WARN_ON(!rwsem_is_locked(&comp->sem)); + WARN_ON(!rwsem_is_locked(&devcom->comp->sem)); - WRITE_ONCE(comp->ready, ready); + WRITE_ONCE(devcom->comp->ready, ready); } -bool mlx5_devcom_comp_is_ready(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id) +bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom) { if (IS_ERR_OR_NULL(devcom)) return false; - return READ_ONCE(devcom->priv->components[id].ready); + return READ_ONCE(devcom->comp->ready); } -bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id) +bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom) { - struct mlx5_devcom_component *comp; + struct mlx5_devcom_comp *comp; if (IS_ERR_OR_NULL(devcom)) return false; - comp = &devcom->priv->components[id]; + comp = devcom->comp; down_read(&comp->sem); if (!READ_ONCE(comp->ready)) { up_read(&comp->sem); @@ -271,74 +325,60 @@ bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom *devcom, return true; } -void mlx5_devcom_for_each_peer_end(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id) +void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom) { - struct mlx5_devcom_component *comp = &devcom->priv->components[id]; - - up_read(&comp->sem); + up_read(&devcom->comp->sem); } -void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - int *i) +void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom, + struct mlx5_devcom_comp_dev **pos) { - struct mlx5_devcom_component *comp; - void *ret; - int idx; + struct mlx5_devcom_comp *comp = devcom->comp; + struct mlx5_devcom_comp_dev *tmp; + void *data; - comp = &devcom->priv->components[id]; + tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); - if (*i == MLX5_DEVCOM_PORTS_SUPPORTED) - return NULL; - for (idx = *i; idx < MLX5_DEVCOM_PORTS_SUPPORTED; idx++) { - if (idx != devcom->idx) { - ret = rcu_dereference_protected(comp->device[idx].data, - lockdep_is_held(&comp->sem)); - if (ret) + list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { + if (tmp != devcom) { + data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem)); + if (data) break; } } - if (idx == MLX5_DEVCOM_PORTS_SUPPORTED) { - *i = idx; + if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) return NULL; - } - *i = idx + 1; - return ret; + *pos = tmp; + return data; } -void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - int *i) +void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, + struct mlx5_devcom_comp_dev **pos) { - struct mlx5_devcom_component *comp; - void *ret; - int idx; + struct mlx5_devcom_comp *comp = devcom->comp; + struct mlx5_devcom_comp_dev *tmp; + void *data; - comp = &devcom->priv->components[id]; + tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); - if (*i == MLX5_DEVCOM_PORTS_SUPPORTED) - return NULL; - for (idx = *i; idx < MLX5_DEVCOM_PORTS_SUPPORTED; idx++) { - if (idx != devcom->idx) { + list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { + if (tmp != devcom) { /* This can change concurrently, however 'data' pointer will remain * valid for the duration of RCU read section. */ if (!READ_ONCE(comp->ready)) return NULL; - ret = rcu_dereference(comp->device[idx].data); - if (ret) + data = rcu_dereference(tmp->data); + if (data) break; } } - if (idx == MLX5_DEVCOM_PORTS_SUPPORTED) { - *i = idx; + if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) return NULL; - } - *i = idx + 1; - return ret; + *pos = tmp; + return data; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index d953a01b8eaa..8389ac0af708 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -6,11 +6,8 @@ #include <linux/mlx5/driver.h> -#define MLX5_DEVCOM_PORTS_SUPPORTED 4 - -enum mlx5_devcom_components { +enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, - MLX5_DEVCOM_NUM_COMPONENTS, }; @@ -18,45 +15,40 @@ typedef int (*mlx5_devcom_event_handler_t)(int event, void *my_data, void *event_data); -struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev); -void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom); +struct mlx5_devcom_dev *mlx5_devcom_register_device(struct mlx5_core_dev *dev); +void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc); -void mlx5_devcom_register_component(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - mlx5_devcom_event_handler_t handler, - void *data); -void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id); +struct mlx5_devcom_comp_dev * +mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, + enum mlx5_devcom_component id, + u64 key, + mlx5_devcom_event_handler_t handler, + void *data); +void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom); -int mlx5_devcom_send_event(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, +int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, int event, int rollback_event, void *event_data); -void mlx5_devcom_comp_set_ready(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, - bool ready); -bool mlx5_devcom_comp_is_ready(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id); - -bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id); -void mlx5_devcom_for_each_peer_end(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id); -void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, int *i); - -#define mlx5_devcom_for_each_peer_entry(devcom, id, data, i) \ - for (i = 0, data = mlx5_devcom_get_next_peer_data(devcom, id, &i); \ - data; \ - data = mlx5_devcom_get_next_peer_data(devcom, id, &i)) - -void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom *devcom, - enum mlx5_devcom_components id, int *i); - -#define mlx5_devcom_for_each_peer_entry_rcu(devcom, id, data, i) \ - for (i = 0, data = mlx5_devcom_get_next_peer_data_rcu(devcom, id, &i); \ - data; \ - data = mlx5_devcom_get_next_peer_data_rcu(devcom, id, &i)) - -#endif +void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready); +bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom); + +bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom); +void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom); +void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom, + struct mlx5_devcom_comp_dev **pos); + +#define mlx5_devcom_for_each_peer_entry(devcom, data, pos) \ + for (pos = NULL, data = mlx5_devcom_get_next_peer_data(devcom, &pos); \ + data; \ + data = mlx5_devcom_get_next_peer_data(devcom, &pos)) + +void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, + struct mlx5_devcom_comp_dev **pos); + +#define mlx5_devcom_for_each_peer_entry_rcu(devcom, data, pos) \ + for (pos = NULL, data = mlx5_devcom_get_next_peer_data_rcu(devcom, &pos); \ + data; \ + data = mlx5_devcom_get_next_peer_data_rcu(devcom, &pos)) + +#endif /* __LIB_MLX5_DEVCOM_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index f42abc2ea73c..db2e8e4f848d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -951,10 +951,10 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) { int err; - dev->priv.devcom = mlx5_devcom_register_device(dev); - if (IS_ERR(dev->priv.devcom)) - mlx5_core_err(dev, "failed to register with devcom (0x%p)\n", - dev->priv.devcom); + dev->priv.devc = mlx5_devcom_register_device(dev); + if (IS_ERR(dev->priv.devc)) + mlx5_core_warn(dev, "failed to register devcom device %ld\n", + PTR_ERR(dev->priv.devc)); err = mlx5_query_board_id(dev); if (err) { @@ -1089,7 +1089,7 @@ err_eq_cleanup: err_irq_cleanup: mlx5_irq_table_cleanup(dev); err_devcom: - mlx5_devcom_unregister_device(dev->priv.devcom); + mlx5_devcom_unregister_device(dev->priv.devc); return err; } @@ -1118,7 +1118,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_events_cleanup(dev); mlx5_eq_table_cleanup(dev); mlx5_irq_table_cleanup(dev); - mlx5_devcom_unregister_device(dev->priv.devcom); + mlx5_devcom_unregister_device(dev->priv.devc); } static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout) @@ -1142,7 +1142,7 @@ static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeou return err; } - err = mlx5_cmd_init(dev); + err = mlx5_cmd_enable(dev); if (err) { mlx5_core_err(dev, "Failed initializing command interface, aborting\n"); return err; @@ -1196,7 +1196,7 @@ stop_health_poll: mlx5_stop_health_poll(dev, boot); err_cmd_cleanup: mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); - mlx5_cmd_cleanup(dev); + mlx5_cmd_disable(dev); return err; } @@ -1207,7 +1207,7 @@ static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot) mlx5_core_disable_hca(dev, 0); mlx5_stop_health_poll(dev, boot); mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); - mlx5_cmd_cleanup(dev); + mlx5_cmd_disable(dev); } static int mlx5_function_open(struct mlx5_core_dev *dev) @@ -1797,6 +1797,12 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops); INIT_LIST_HEAD(&priv->traps); + err = mlx5_cmd_init(dev); + if (err) { + mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n"); + goto err_cmd_init; + } + err = mlx5_tout_init(dev); if (err) { mlx5_core_err(dev, "Failed initializing timeouts, aborting\n"); @@ -1842,6 +1848,8 @@ err_pagealloc_init: err_health_init: mlx5_tout_cleanup(dev); err_timeout_init: + mlx5_cmd_cleanup(dev); +err_cmd_init: debugfs_remove(dev->priv.dbg.dbg_root); mutex_destroy(&priv->pgdir_mutex); mutex_destroy(&priv->alloc_mutex); @@ -1864,6 +1872,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) mlx5_pagealloc_cleanup(dev); mlx5_health_cleanup(dev); mlx5_tout_cleanup(dev); + mlx5_cmd_cleanup(dev); debugfs_remove_recursive(dev->priv.dbg.dbg_root); mutex_destroy(&priv->pgdir_mutex); mutex_destroy(&priv->alloc_mutex); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index c4be257c043d..43b0144121ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -178,6 +178,8 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_cmd_init(struct mlx5_core_dev *dev); void mlx5_cmd_cleanup(struct mlx5_core_dev *dev); +int mlx5_cmd_enable(struct mlx5_core_dev *dev); +void mlx5_cmd_disable(struct mlx5_core_dev *dev); void mlx5_cmd_set_state(struct mlx5_core_dev *dev, enum mlx5_cmdif_state cmdif_state); int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id); diff --git a/drivers/net/ethernet/mellanox/mlxsw/Makefile b/drivers/net/ethernet/mellanox/mlxsw/Makefile index 3ca9fce759ea..71cad6bb6e62 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Makefile +++ b/drivers/net/ethernet/mellanox/mlxsw/Makefile @@ -29,7 +29,7 @@ mlxsw_spectrum-objs := spectrum.o spectrum_buffers.o \ spectrum_nve.o spectrum_nve_vxlan.o \ spectrum_dpipe.o spectrum_trap.o \ spectrum_ethtool.o spectrum_policer.o \ - spectrum_pgt.o + spectrum_pgt.o spectrum_port_range.o mlxsw_spectrum-$(CONFIG_MLXSW_SPECTRUM_DCB) += spectrum_dcb.o mlxsw_spectrum-$(CONFIG_PTP_1588_CLOCK) += spectrum_ptp.o obj-$(CONFIG_MLXSW_MINIMAL) += mlxsw_minimal.o diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c index f0b2963ebac3..7870327d921b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c @@ -43,6 +43,7 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = { MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4), MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4), MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1), + MLXSW_AFK_ELEMENT_INFO_U32(L4_PORT_RANGE, 0x40, 1, 16), }; struct mlxsw_afk { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h index 65a4abadc7db..2eac7582c31a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h @@ -36,6 +36,7 @@ enum mlxsw_afk_element { MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB, MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB, MLXSW_AFK_ELEMENT_FDB_MISS, + MLXSW_AFK_ELEMENT_L4_PORT_RANGE, MLXSW_AFK_ELEMENT_MAX, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index 0107cbc32fc7..d637c0348fa1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -32,6 +32,7 @@ struct mlxsw_env { const struct mlxsw_bus_info *bus_info; u8 max_module_count; /* Maximum number of modules per-slot. */ u8 num_of_slots; /* Including the main board. */ + u8 max_eeprom_len; /* Maximum module EEPROM transaction length. */ struct mutex line_cards_lock; /* Protects line cards. */ struct mlxsw_env_line_card *line_cards[]; }; @@ -111,7 +112,7 @@ mlxsw_env_validate_cable_ident(struct mlxsw_core *core, u8 slot_index, int id, if (err) return err; - mlxsw_reg_mcia_pack(mcia_pl, slot_index, id, 0, + mlxsw_reg_mcia_pack(mcia_pl, slot_index, id, MLXSW_REG_MCIA_PAGE0_LO_OFF, 0, 1, MLXSW_REG_MCIA_I2C_ADDR_LOW); err = mlxsw_reg_query(core, MLXSW_REG(mcia), mcia_pl); @@ -146,6 +147,7 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, u8 slot_index, int module, u16 offset, u16 size, void *data, bool qsfp, unsigned int *p_read_size) { + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); char mcia_pl[MLXSW_REG_MCIA_LEN]; char *eeprom_tmp; u16 i2c_addr; @@ -153,11 +155,7 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, u8 slot_index, int status; int err; - /* MCIA register accepts buffer size <= 48. Page of size 128 should be - * read by chunks of size 48, 48, 32. Align the size of the last chunk - * to avoid reading after the end of the page. - */ - size = min_t(u16, size, MLXSW_REG_MCIA_EEPROM_SIZE); + size = min_t(u16, size, mlxsw_env->max_eeprom_len); if (offset < MLXSW_REG_MCIA_EEPROM_PAGE_LENGTH && offset + size > MLXSW_REG_MCIA_EEPROM_PAGE_LENGTH) @@ -188,7 +186,7 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, u8 slot_index, } } - mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, 0, page, offset, size, + mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, page, offset, size, i2c_addr); err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcia), mcia_pl); @@ -266,12 +264,12 @@ mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, u8 slot_index, page = MLXSW_REG_MCIA_TH_PAGE_CMIS_NUM; else page = MLXSW_REG_MCIA_TH_PAGE_NUM; - mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, 0, page, + mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, page, MLXSW_REG_MCIA_TH_PAGE_OFF + off, MLXSW_REG_MCIA_TH_ITEM_SIZE, MLXSW_REG_MCIA_I2C_ADDR_LOW); } else { - mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, 0, + mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, MLXSW_REG_MCIA_PAGE0_LO, off, MLXSW_REG_MCIA_TH_ITEM_SIZE, MLXSW_REG_MCIA_I2C_ADDR_HIGH); @@ -489,9 +487,9 @@ mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 size; size = min_t(u8, page->length - bytes_read, - MLXSW_REG_MCIA_EEPROM_SIZE); + mlxsw_env->max_eeprom_len); - mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, 0, page->page, + mlxsw_reg_mcia_pack(mcia_pl, slot_index, module, page->page, device_addr + bytes_read, size, page->i2c_address); mlxsw_reg_mcia_bank_number_set(mcia_pl, page->bank); @@ -1359,6 +1357,26 @@ static struct mlxsw_linecards_event_ops mlxsw_env_event_ops = { .got_inactive = mlxsw_env_got_inactive, }; +static int mlxsw_env_max_module_eeprom_len_query(struct mlxsw_env *mlxsw_env) +{ + char mcam_pl[MLXSW_REG_MCAM_LEN]; + bool mcia_128b_supported; + int err; + + mlxsw_reg_mcam_pack(mcam_pl, + MLXSW_REG_MCAM_FEATURE_GROUP_ENHANCED_FEATURES); + err = mlxsw_reg_query(mlxsw_env->core, MLXSW_REG(mcam), mcam_pl); + if (err) + return err; + + mlxsw_reg_mcam_unpack(mcam_pl, MLXSW_REG_MCAM_MCIA_128B, + &mcia_128b_supported); + + mlxsw_env->max_eeprom_len = mcia_128b_supported ? 128 : 48; + + return 0; +} + int mlxsw_env_init(struct mlxsw_core *mlxsw_core, const struct mlxsw_bus_info *bus_info, struct mlxsw_env **p_env) @@ -1427,10 +1445,15 @@ int mlxsw_env_init(struct mlxsw_core *mlxsw_core, if (err) goto err_type_set; + err = mlxsw_env_max_module_eeprom_len_query(env); + if (err) + goto err_eeprom_len_query; + env->line_cards[0]->active = true; return 0; +err_eeprom_len_query: err_type_set: mlxsw_env_module_event_disable(env, 0); err_mlxsw_env_module_event_enable: diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 8165bf31a99a..4b90ae44b476 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -2799,6 +2799,78 @@ static inline void mlxsw_reg_ptar_unpack(char *payload, char *tcam_region_info) mlxsw_reg_ptar_tcam_region_info_memcpy_from(payload, tcam_region_info); } +/* PPRR - Policy-Engine Port Range Register + * ---------------------------------------- + * This register is used for configuring port range identification. + */ +#define MLXSW_REG_PPRR_ID 0x3008 +#define MLXSW_REG_PPRR_LEN 0x14 + +MLXSW_REG_DEFINE(pprr, MLXSW_REG_PPRR_ID, MLXSW_REG_PPRR_LEN); + +/* reg_pprr_ipv4 + * Apply port range register to IPv4 packets. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, ipv4, 0x00, 31, 1); + +/* reg_pprr_ipv6 + * Apply port range register to IPv6 packets. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, ipv6, 0x00, 30, 1); + +/* reg_pprr_src + * Apply port range register to source L4 ports. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, src, 0x00, 29, 1); + +/* reg_pprr_dst + * Apply port range register to destination L4 ports. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, dst, 0x00, 28, 1); + +/* reg_pprr_tcp + * Apply port range register to TCP packets. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, tcp, 0x00, 27, 1); + +/* reg_pprr_udp + * Apply port range register to UDP packets. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, udp, 0x00, 26, 1); + +/* reg_pprr_register_index + * Index of Port Range Register being accessed. + * Range is 0..cap_max_acl_l4_port_range-1. + * Access: Index + */ +MLXSW_ITEM32(reg, pprr, register_index, 0x00, 0, 8); + +/* reg_prrr_port_range_min + * Minimum port range for comparison. + * Match is defined as: + * port_range_min <= packet_port <= port_range_max. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, port_range_min, 0x04, 16, 16); + +/* reg_prrr_port_range_max + * Maximum port range for comparison. + * Access: RW + */ +MLXSW_ITEM32(reg, pprr, port_range_max, 0x04, 0, 16); + +static inline void mlxsw_reg_pprr_pack(char *payload, u8 register_index) +{ + MLXSW_REG_ZERO(pprr, payload); + mlxsw_reg_pprr_register_index_set(payload, register_index); +} + /* PPBS - Policy-Engine Policy Based Switching Register * ---------------------------------------------------- * This register retrieves and sets Policy Based Switching Table entries. @@ -9568,18 +9640,10 @@ static inline void mlxsw_reg_mtbr_temp_unpack(char *payload, int rec_ind, */ #define MLXSW_REG_MCIA_ID 0x9014 -#define MLXSW_REG_MCIA_LEN 0x40 +#define MLXSW_REG_MCIA_LEN 0x94 MLXSW_REG_DEFINE(mcia, MLXSW_REG_MCIA_ID, MLXSW_REG_MCIA_LEN); -/* reg_mcia_l - * Lock bit. Setting this bit will lock the access to the specific - * cable. Used for updating a full page in a cable EPROM. Any access - * other then subsequence writes will fail while the port is locked. - * Access: RW - */ -MLXSW_ITEM32(reg, mcia, l, 0x00, 31, 1); - /* reg_mcia_module * Module number. * Access: Index @@ -9644,7 +9708,6 @@ MLXSW_ITEM32(reg, mcia, size, 0x08, 0, 16); #define MLXSW_REG_MCIA_EEPROM_PAGE_LENGTH 256 #define MLXSW_REG_MCIA_EEPROM_UP_PAGE_LENGTH 128 -#define MLXSW_REG_MCIA_EEPROM_SIZE 48 #define MLXSW_REG_MCIA_I2C_ADDR_LOW 0x50 #define MLXSW_REG_MCIA_I2C_ADDR_HIGH 0x51 #define MLXSW_REG_MCIA_PAGE0_LO_OFF 0xa0 @@ -9681,7 +9744,7 @@ enum mlxsw_reg_mcia_eeprom_module_info { * Bytes to read/write. * Access: RW */ -MLXSW_ITEM_BUF(reg, mcia, eeprom, 0x10, MLXSW_REG_MCIA_EEPROM_SIZE); +MLXSW_ITEM_BUF(reg, mcia, eeprom, 0x10, 128); /* This is used to access the optional upper pages (1-3) in the QSFP+ * memory map. Page 1 is available on offset 256 through 383, page 2 - @@ -9692,14 +9755,12 @@ MLXSW_ITEM_BUF(reg, mcia, eeprom, 0x10, MLXSW_REG_MCIA_EEPROM_SIZE); MLXSW_REG_MCIA_EEPROM_UP_PAGE_LENGTH + 1) static inline void mlxsw_reg_mcia_pack(char *payload, u8 slot_index, u8 module, - u8 lock, u8 page_number, - u16 device_addr, u8 size, + u8 page_number, u16 device_addr, u8 size, u8 i2c_device_addr) { MLXSW_REG_ZERO(mcia, payload); mlxsw_reg_mcia_slot_set(payload, slot_index); mlxsw_reg_mcia_module_set(payload, module); - mlxsw_reg_mcia_l_set(payload, lock); mlxsw_reg_mcia_page_number_set(payload, page_number); mlxsw_reg_mcia_device_address_set(payload, device_addr); mlxsw_reg_mcia_size_set(payload, size); @@ -10509,6 +10570,79 @@ static inline void mlxsw_reg_mcda_pack(char *payload, u32 update_handle, mlxsw_reg_mcda_data_set(payload, i, *(u32 *) &data[i * 4]); } +/* MCAM - Management Capabilities Mask Register + * -------------------------------------------- + * Reports the device supported management features. + */ +#define MLXSW_REG_MCAM_ID 0x907F +#define MLXSW_REG_MCAM_LEN 0x48 + +MLXSW_REG_DEFINE(mcam, MLXSW_REG_MCAM_ID, MLXSW_REG_MCAM_LEN); + +enum mlxsw_reg_mcam_feature_group { + /* Enhanced features. */ + MLXSW_REG_MCAM_FEATURE_GROUP_ENHANCED_FEATURES, +}; + +/* reg_mcam_feature_group + * Feature list mask index. + * Access: Index + */ +MLXSW_ITEM32(reg, mcam, feature_group, 0x00, 16, 8); + +enum mlxsw_reg_mcam_mng_feature_cap_mask_bits { + /* If set, MCIA supports 128 bytes payloads. Otherwise, 48 bytes. */ + MLXSW_REG_MCAM_MCIA_128B = 34, +}; + +#define MLXSW_REG_BYTES_PER_DWORD 0x4 + +/* reg_mcam_mng_feature_cap_mask + * Supported port's enhanced features. + * Based on feature_group index. + * When bit is set, the feature is supported in the device. + * Access: RO + */ +#define MLXSW_REG_MCAM_MNG_FEATURE_CAP_MASK_DWORD(_dw_num, _offset) \ + MLXSW_ITEM_BIT_ARRAY(reg, mcam, mng_feature_cap_mask_dw##_dw_num, \ + _offset, MLXSW_REG_BYTES_PER_DWORD, 1) + +/* The access to the bits in the field 'mng_feature_cap_mask' is not same to + * other mask fields in other registers. In most of the cases bit #0 is the + * first one in the last dword. In MCAM register, the first dword contains bits + * #0-#31 and so on, so the access to the bits is simpler using bit array per + * dword. Declare each dword of 'mng_feature_cap_mask' field separately. + */ +MLXSW_REG_MCAM_MNG_FEATURE_CAP_MASK_DWORD(0, 0x28); +MLXSW_REG_MCAM_MNG_FEATURE_CAP_MASK_DWORD(1, 0x2C); +MLXSW_REG_MCAM_MNG_FEATURE_CAP_MASK_DWORD(2, 0x30); +MLXSW_REG_MCAM_MNG_FEATURE_CAP_MASK_DWORD(3, 0x34); + +static inline void +mlxsw_reg_mcam_pack(char *payload, enum mlxsw_reg_mcam_feature_group feat_group) +{ + MLXSW_REG_ZERO(mcam, payload); + mlxsw_reg_mcam_feature_group_set(payload, feat_group); +} + +static inline void +mlxsw_reg_mcam_unpack(char *payload, + enum mlxsw_reg_mcam_mng_feature_cap_mask_bits bit, + bool *p_mng_feature_cap_val) +{ + int offset = bit % (MLXSW_REG_BYTES_PER_DWORD * BITS_PER_BYTE); + int dword = bit / (MLXSW_REG_BYTES_PER_DWORD * BITS_PER_BYTE); + u8 (*getters[])(const char *, u16) = { + mlxsw_reg_mcam_mng_feature_cap_mask_dw0_get, + mlxsw_reg_mcam_mng_feature_cap_mask_dw1_get, + mlxsw_reg_mcam_mng_feature_cap_mask_dw2_get, + mlxsw_reg_mcam_mng_feature_cap_mask_dw3_get, + }; + + if (!WARN_ON_ONCE(dword >= ARRAY_SIZE(getters))) + *p_mng_feature_cap_val = getters[dword](payload, offset); +} + /* MPSC - Monitoring Packet Sampling Configuration Register * -------------------------------------------------------- * MPSC Register is used to configure the Packet Sampling mechanism. @@ -12819,6 +12953,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(pacl), MLXSW_REG(pagt), MLXSW_REG(ptar), + MLXSW_REG(pprr), MLXSW_REG(ppbs), MLXSW_REG(prcr), MLXSW_REG(pefa), @@ -12901,10 +13036,11 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(mcion), MLXSW_REG(mtpps), MLXSW_REG(mtutc), - MLXSW_REG(mpsc), MLXSW_REG(mcqi), MLXSW_REG(mcc), MLXSW_REG(mcda), + MLXSW_REG(mcam), + MLXSW_REG(mpsc), MLXSW_REG(mgpc), MLXSW_REG(mprs), MLXSW_REG(mogcr), diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h index 19ae0d1c74a8..89dd2777ec4d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/resources.h +++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h @@ -39,6 +39,7 @@ enum mlxsw_res_id { MLXSW_RES_ID_ACL_FLEX_KEYS, MLXSW_RES_ID_ACL_MAX_ACTION_PER_RULE, MLXSW_RES_ID_ACL_ACTIONS_PER_SET, + MLXSW_RES_ID_ACL_MAX_L4_PORT_RANGE, MLXSW_RES_ID_ACL_MAX_ERPT_BANKS, MLXSW_RES_ID_ACL_MAX_ERPT_BANK_SIZE, MLXSW_RES_ID_ACL_MAX_LARGE_KEY_ID, @@ -99,6 +100,7 @@ static u16 mlxsw_res_ids[] = { [MLXSW_RES_ID_ACL_FLEX_KEYS] = 0x2910, [MLXSW_RES_ID_ACL_MAX_ACTION_PER_RULE] = 0x2911, [MLXSW_RES_ID_ACL_ACTIONS_PER_SET] = 0x2912, + [MLXSW_RES_ID_ACL_MAX_L4_PORT_RANGE] = 0x2920, [MLXSW_RES_ID_ACL_MAX_ERPT_BANKS] = 0x2940, [MLXSW_RES_ID_ACL_MAX_ERPT_BANK_SIZE] = 0x2941, [MLXSW_RES_ID_ACL_MAX_LARGE_KEY_ID] = 0x2942, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 25a01dafde1b..9dbd5edff0b0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -1132,8 +1132,8 @@ static int mlxsw_sp_port_add_vid(struct net_device *dev, return PTR_ERR_OR_ZERO(mlxsw_sp_port_vlan_create(mlxsw_sp_port, vid)); } -static int mlxsw_sp_port_kill_vid(struct net_device *dev, - __be16 __always_unused proto, u16 vid) +int mlxsw_sp_port_kill_vid(struct net_device *dev, + __be16 __always_unused proto, u16 vid) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; @@ -3188,6 +3188,12 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core, goto err_nve_init; } + err = mlxsw_sp_port_range_init(mlxsw_sp); + if (err) { + dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize port ranges\n"); + goto err_port_range_init; + } + err = mlxsw_sp_acl_init(mlxsw_sp); if (err) { dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize ACL\n"); @@ -3280,6 +3286,8 @@ err_ptp_clock_init: err_router_init: mlxsw_sp_acl_fini(mlxsw_sp); err_acl_init: + mlxsw_sp_port_range_fini(mlxsw_sp); +err_port_range_init: mlxsw_sp_nve_fini(mlxsw_sp); err_nve_init: mlxsw_sp_ipv6_addr_ht_fini(mlxsw_sp); @@ -3462,6 +3470,7 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core) } mlxsw_sp_router_fini(mlxsw_sp); mlxsw_sp_acl_fini(mlxsw_sp); + mlxsw_sp_port_range_fini(mlxsw_sp); mlxsw_sp_nve_fini(mlxsw_sp); mlxsw_sp_ipv6_addr_ht_fini(mlxsw_sp); mlxsw_sp_afa_fini(mlxsw_sp); @@ -3730,6 +3739,26 @@ static int mlxsw_sp_resources_rifs_register(struct mlxsw_core *mlxsw_core) &size_params); } +static int +mlxsw_sp_resources_port_range_register(struct mlxsw_core *mlxsw_core) +{ + struct devlink *devlink = priv_to_devlink(mlxsw_core); + struct devlink_resource_size_params size_params; + u64 max; + + if (!MLXSW_CORE_RES_VALID(mlxsw_core, ACL_MAX_L4_PORT_RANGE)) + return -EIO; + + max = MLXSW_CORE_RES_GET(mlxsw_core, ACL_MAX_L4_PORT_RANGE); + devlink_resource_size_params_init(&size_params, max, max, 1, + DEVLINK_RESOURCE_UNIT_ENTRY); + + return devl_resource_register(devlink, "port_range_registers", max, + MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); +} + static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core) { int err; @@ -3758,8 +3787,13 @@ static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core) if (err) goto err_resources_rifs_register; + err = mlxsw_sp_resources_port_range_register(mlxsw_core); + if (err) + goto err_resources_port_range_register; + return 0; +err_resources_port_range_register: err_resources_rifs_register: err_resources_rif_mac_profile_register: err_policer_resources_register: @@ -3797,8 +3831,13 @@ static int mlxsw_sp2_resources_register(struct mlxsw_core *mlxsw_core) if (err) goto err_resources_rifs_register; + err = mlxsw_sp_resources_port_range_register(mlxsw_core); + if (err) + goto err_resources_port_range_register; + return 0; +err_resources_port_range_register: err_resources_rifs_register: err_resources_rif_mac_profile_register: err_policer_resources_register: @@ -4073,23 +4112,6 @@ struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find_rcu(struct net_device *dev) return (struct mlxsw_sp_port *)priv.data; } -struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev) -{ - struct mlxsw_sp_port *mlxsw_sp_port; - - rcu_read_lock(); - mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev); - if (mlxsw_sp_port) - dev_hold(mlxsw_sp_port->dev); - rcu_read_unlock(); - return mlxsw_sp_port; -} - -void mlxsw_sp_port_dev_put(struct mlxsw_sp_port *mlxsw_sp_port) -{ - dev_put(mlxsw_sp_port->dev); -} - int mlxsw_sp_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp) { char mprs_pl[MLXSW_REG_MPRS_LEN]; @@ -4298,6 +4320,88 @@ static int mlxsw_sp_port_lag_index_get(struct mlxsw_sp *mlxsw_sp, return -EBUSY; } +static int mlxsw_sp_lag_uppers_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port, + struct net_device *lag_dev, + struct netlink_ext_ack *extack) +{ + struct net_device *upper_dev; + struct net_device *master; + struct list_head *iter; + int done = 0; + int err; + + master = netdev_master_upper_dev_get(lag_dev); + if (master && netif_is_bridge_master(master)) { + err = mlxsw_sp_port_bridge_join(mlxsw_sp_port, lag_dev, master, + extack); + if (err) + return err; + } + + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + + master = netdev_master_upper_dev_get(upper_dev); + if (master && netif_is_bridge_master(master)) { + err = mlxsw_sp_port_bridge_join(mlxsw_sp_port, + upper_dev, master, + extack); + if (err) + goto err_port_bridge_join; + } + + ++done; + } + + return 0; + +err_port_bridge_join: + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + + master = netdev_master_upper_dev_get(upper_dev); + if (!master || !netif_is_bridge_master(master)) + continue; + + if (!done--) + break; + + mlxsw_sp_port_bridge_leave(mlxsw_sp_port, upper_dev, master); + } + + master = netdev_master_upper_dev_get(lag_dev); + if (master && netif_is_bridge_master(master)) + mlxsw_sp_port_bridge_leave(mlxsw_sp_port, lag_dev, master); + + return err; +} + +static void +mlxsw_sp_lag_uppers_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_port, + struct net_device *lag_dev) +{ + struct net_device *upper_dev; + struct net_device *master; + struct list_head *iter; + + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + + master = netdev_master_upper_dev_get(upper_dev); + if (!master) + continue; + + mlxsw_sp_port_bridge_leave(mlxsw_sp_port, upper_dev, master); + } + + master = netdev_master_upper_dev_get(lag_dev); + if (master) + mlxsw_sp_port_bridge_leave(mlxsw_sp_port, lag_dev, master); +} + static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, struct net_device *lag_dev, struct netlink_ext_ack *extack) @@ -4322,6 +4426,12 @@ static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, err = mlxsw_sp_port_lag_index_get(mlxsw_sp, lag_id, &port_index); if (err) return err; + + err = mlxsw_sp_lag_uppers_bridge_join(mlxsw_sp_port, lag_dev, + extack); + if (err) + goto err_lag_uppers_bridge_join; + err = mlxsw_sp_lag_col_port_add(mlxsw_sp_port, lag_id, port_index); if (err) goto err_col_port_add; @@ -4342,8 +4452,14 @@ static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, if (err) goto err_router_join; + err = mlxsw_sp_netdevice_enslavement_replay(mlxsw_sp, lag_dev, extack); + if (err) + goto err_replay; + return 0; +err_replay: + mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev); err_router_join: lag->ref_count--; mlxsw_sp_port->lagged = 0; @@ -4351,6 +4467,8 @@ err_router_join: mlxsw_sp_port->local_port); mlxsw_sp_lag_col_port_remove(mlxsw_sp_port, lag_id); err_col_port_add: + mlxsw_sp_lag_uppers_bridge_leave(mlxsw_sp_port, lag_dev); +err_lag_uppers_bridge_join: if (!lag->ref_count) mlxsw_sp_lag_destroy(mlxsw_sp, lag_id); return err; @@ -4600,9 +4718,62 @@ static bool mlxsw_sp_bridge_vxlan_is_valid(struct net_device *br_dev, return true; } +static bool mlxsw_sp_netdev_is_master(struct net_device *upper_dev, + struct net_device *dev) +{ + return upper_dev == netdev_master_upper_dev_get(dev); +} + +static int __mlxsw_sp_netdevice_event(struct mlxsw_sp *mlxsw_sp, + unsigned long event, void *ptr, + bool process_foreign); + +static int mlxsw_sp_netdevice_validate_uppers(struct mlxsw_sp *mlxsw_sp, + struct net_device *dev, + struct netlink_ext_ack *extack) +{ + struct net_device *upper_dev; + struct list_head *iter; + int err; + + netdev_for_each_upper_dev_rcu(dev, upper_dev, iter) { + struct netdev_notifier_changeupper_info info = { + .info = { + .dev = dev, + .extack = extack, + }, + .master = mlxsw_sp_netdev_is_master(upper_dev, dev), + .upper_dev = upper_dev, + .linking = true, + + /* upper_info is relevant for LAG devices. But we would + * only need this if LAG were a valid upper above + * another upper (e.g. a bridge that is a member of a + * LAG), and that is never a valid configuration. So we + * can keep this as NULL. + */ + .upper_info = NULL, + }; + + err = __mlxsw_sp_netdevice_event(mlxsw_sp, + NETDEV_PRECHANGEUPPER, + &info, true); + if (err) + return err; + + err = mlxsw_sp_netdevice_validate_uppers(mlxsw_sp, upper_dev, + extack); + if (err) + return err; + } + + return 0; +} + static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev, struct net_device *dev, - unsigned long event, void *ptr) + unsigned long event, void *ptr, + bool replay_deslavement) { struct netdev_notifier_changeupper_info *info; struct mlxsw_sp_port *mlxsw_sp_port; @@ -4640,8 +4811,11 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev, (!netif_is_bridge_master(upper_dev) || !mlxsw_sp_bridge_device_is_offloaded(mlxsw_sp, upper_dev))) { - NL_SET_ERR_MSG_MOD(extack, "Enslaving a port to a device that already has an upper device is not supported"); - return -EINVAL; + err = mlxsw_sp_netdevice_validate_uppers(mlxsw_sp, + upper_dev, + extack); + if (err) + return err; } if (netif_is_lag_master(upper_dev) && !mlxsw_sp_master_lag_check(mlxsw_sp, upper_dev, @@ -4656,11 +4830,6 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev, NL_SET_ERR_MSG_MOD(extack, "Can not put a VLAN on a LAG port"); return -EINVAL; } - if (netif_is_macvlan(upper_dev) && - !mlxsw_sp_rif_exists(mlxsw_sp, lower_dev)) { - NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); - return -EOPNOTSUPP; - } if (netif_is_ovs_master(upper_dev) && vlan_uses_dev(dev)) { NL_SET_ERR_MSG_MOD(extack, "Master device is an OVS master and this device has a VLAN"); return -EINVAL; @@ -4707,15 +4876,20 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev, case NETDEV_CHANGEUPPER: upper_dev = info->upper_dev; if (netif_is_bridge_master(upper_dev)) { - if (info->linking) + if (info->linking) { err = mlxsw_sp_port_bridge_join(mlxsw_sp_port, lower_dev, upper_dev, extack); - else + } else { mlxsw_sp_port_bridge_leave(mlxsw_sp_port, lower_dev, upper_dev); + if (!replay_deslavement) + break; + mlxsw_sp_netdevice_deslavement_replay(mlxsw_sp, + lower_dev); + } } else if (netif_is_lag_master(upper_dev)) { if (info->linking) { err = mlxsw_sp_port_lag_join(mlxsw_sp_port, @@ -4724,6 +4898,8 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev, mlxsw_sp_port_lag_col_dist_disable(mlxsw_sp_port); mlxsw_sp_port_lag_leave(mlxsw_sp_port, upper_dev); + mlxsw_sp_netdevice_deslavement_replay(mlxsw_sp, + dev); } } else if (netif_is_ovs_master(upper_dev)) { if (info->linking) @@ -4776,13 +4952,15 @@ static int mlxsw_sp_netdevice_port_lower_event(struct net_device *dev, static int mlxsw_sp_netdevice_port_event(struct net_device *lower_dev, struct net_device *port_dev, - unsigned long event, void *ptr) + unsigned long event, void *ptr, + bool replay_deslavement) { switch (event) { case NETDEV_PRECHANGEUPPER: case NETDEV_CHANGEUPPER: return mlxsw_sp_netdevice_port_upper_event(lower_dev, port_dev, - event, ptr); + event, ptr, + replay_deslavement); case NETDEV_CHANGELOWERSTATE: return mlxsw_sp_netdevice_port_lower_event(port_dev, event, ptr); @@ -4791,6 +4969,30 @@ static int mlxsw_sp_netdevice_port_event(struct net_device *lower_dev, return 0; } +/* Called for LAG or its upper VLAN after the per-LAG-lower processing was done, + * to do any per-LAG / per-LAG-upper processing. + */ +static int mlxsw_sp_netdevice_post_lag_event(struct net_device *dev, + unsigned long event, + void *ptr) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(dev); + struct netdev_notifier_changeupper_info *info = ptr; + + if (!mlxsw_sp) + return 0; + + switch (event) { + case NETDEV_CHANGEUPPER: + if (info->linking) + break; + if (netif_is_bridge_master(info->upper_dev)) + mlxsw_sp_netdevice_deslavement_replay(mlxsw_sp, dev); + break; + } + return 0; +} + static int mlxsw_sp_netdevice_lag_event(struct net_device *lag_dev, unsigned long event, void *ptr) { @@ -4801,19 +5003,19 @@ static int mlxsw_sp_netdevice_lag_event(struct net_device *lag_dev, netdev_for_each_lower_dev(lag_dev, dev, iter) { if (mlxsw_sp_port_dev_check(dev)) { ret = mlxsw_sp_netdevice_port_event(lag_dev, dev, event, - ptr); + ptr, false); if (ret) return ret; } } - return 0; + return mlxsw_sp_netdevice_post_lag_event(lag_dev, event, ptr); } static int mlxsw_sp_netdevice_port_vlan_event(struct net_device *vlan_dev, struct net_device *dev, unsigned long event, void *ptr, - u16 vid) + u16 vid, bool replay_deslavement) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; @@ -4844,27 +5046,30 @@ static int mlxsw_sp_netdevice_port_vlan_event(struct net_device *vlan_dev, (!netif_is_bridge_master(upper_dev) || !mlxsw_sp_bridge_device_is_offloaded(mlxsw_sp, upper_dev))) { - NL_SET_ERR_MSG_MOD(extack, "Enslaving a port to a device that already has an upper device is not supported"); - return -EINVAL; - } - if (netif_is_macvlan(upper_dev) && - !mlxsw_sp_rif_exists(mlxsw_sp, vlan_dev)) { - NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); - return -EOPNOTSUPP; + err = mlxsw_sp_netdevice_validate_uppers(mlxsw_sp, + upper_dev, + extack); + if (err) + return err; } break; case NETDEV_CHANGEUPPER: upper_dev = info->upper_dev; if (netif_is_bridge_master(upper_dev)) { - if (info->linking) + if (info->linking) { err = mlxsw_sp_port_bridge_join(mlxsw_sp_port, vlan_dev, upper_dev, extack); - else + } else { mlxsw_sp_port_bridge_leave(mlxsw_sp_port, vlan_dev, upper_dev); + if (!replay_deslavement) + break; + mlxsw_sp_netdevice_deslavement_replay(mlxsw_sp, + vlan_dev); + } } else if (netif_is_macvlan(upper_dev)) { if (!info->linking) mlxsw_sp_rif_macvlan_del(mlxsw_sp, upper_dev); @@ -4888,26 +5093,26 @@ static int mlxsw_sp_netdevice_lag_port_vlan_event(struct net_device *vlan_dev, if (mlxsw_sp_port_dev_check(dev)) { ret = mlxsw_sp_netdevice_port_vlan_event(vlan_dev, dev, event, ptr, - vid); + vid, false); if (ret) return ret; } } - return 0; + return mlxsw_sp_netdevice_post_lag_event(vlan_dev, event, ptr); } -static int mlxsw_sp_netdevice_bridge_vlan_event(struct net_device *vlan_dev, +static int mlxsw_sp_netdevice_bridge_vlan_event(struct mlxsw_sp *mlxsw_sp, + struct net_device *vlan_dev, struct net_device *br_dev, unsigned long event, void *ptr, - u16 vid) + u16 vid, bool process_foreign) { - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev); struct netdev_notifier_changeupper_info *info = ptr; struct netlink_ext_ack *extack; struct net_device *upper_dev; - if (!mlxsw_sp) + if (!process_foreign && !mlxsw_sp_lower_get(vlan_dev)) return 0; extack = netdev_notifier_info_to_extack(&info->info); @@ -4920,13 +5125,6 @@ static int mlxsw_sp_netdevice_bridge_vlan_event(struct net_device *vlan_dev, NL_SET_ERR_MSG_MOD(extack, "Unknown upper device type"); return -EOPNOTSUPP; } - if (!info->linking) - break; - if (netif_is_macvlan(upper_dev) && - !mlxsw_sp_rif_exists(mlxsw_sp, vlan_dev)) { - NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); - return -EOPNOTSUPP; - } break; case NETDEV_CHANGEUPPER: upper_dev = info->upper_dev; @@ -4940,36 +5138,42 @@ static int mlxsw_sp_netdevice_bridge_vlan_event(struct net_device *vlan_dev, return 0; } -static int mlxsw_sp_netdevice_vlan_event(struct net_device *vlan_dev, - unsigned long event, void *ptr) +static int mlxsw_sp_netdevice_vlan_event(struct mlxsw_sp *mlxsw_sp, + struct net_device *vlan_dev, + unsigned long event, void *ptr, + bool process_foreign) { struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); u16 vid = vlan_dev_vlan_id(vlan_dev); if (mlxsw_sp_port_dev_check(real_dev)) return mlxsw_sp_netdevice_port_vlan_event(vlan_dev, real_dev, - event, ptr, vid); + event, ptr, vid, + true); else if (netif_is_lag_master(real_dev)) return mlxsw_sp_netdevice_lag_port_vlan_event(vlan_dev, real_dev, event, ptr, vid); else if (netif_is_bridge_master(real_dev)) - return mlxsw_sp_netdevice_bridge_vlan_event(vlan_dev, real_dev, - event, ptr, vid); + return mlxsw_sp_netdevice_bridge_vlan_event(mlxsw_sp, vlan_dev, + real_dev, event, + ptr, vid, + process_foreign); return 0; } -static int mlxsw_sp_netdevice_bridge_event(struct net_device *br_dev, - unsigned long event, void *ptr) +static int mlxsw_sp_netdevice_bridge_event(struct mlxsw_sp *mlxsw_sp, + struct net_device *br_dev, + unsigned long event, void *ptr, + bool process_foreign) { - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(br_dev); struct netdev_notifier_changeupper_info *info = ptr; struct netlink_ext_ack *extack; struct net_device *upper_dev; u16 proto; - if (!mlxsw_sp) + if (!process_foreign && !mlxsw_sp_lower_get(br_dev)) return 0; extack = netdev_notifier_info_to_extack(&info->info); @@ -4997,11 +5201,6 @@ static int mlxsw_sp_netdevice_bridge_event(struct net_device *br_dev, NL_SET_ERR_MSG_MOD(extack, "VLAN uppers are only supported with 802.1q VLAN protocol"); return -EOPNOTSUPP; } - if (netif_is_macvlan(upper_dev) && - !mlxsw_sp_rif_exists(mlxsw_sp, br_dev)) { - NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); - return -EOPNOTSUPP; - } break; case NETDEV_CHANGEUPPER: upper_dev = info->upper_dev; @@ -5107,35 +5306,48 @@ static int mlxsw_sp_netdevice_vxlan_event(struct mlxsw_sp *mlxsw_sp, return 0; } -static int mlxsw_sp_netdevice_event(struct notifier_block *nb, - unsigned long event, void *ptr) +static int __mlxsw_sp_netdevice_event(struct mlxsw_sp *mlxsw_sp, + unsigned long event, void *ptr, + bool process_foreign) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct mlxsw_sp_span_entry *span_entry; - struct mlxsw_sp *mlxsw_sp; int err = 0; - mlxsw_sp = container_of(nb, struct mlxsw_sp, netdevice_nb); if (event == NETDEV_UNREGISTER) { span_entry = mlxsw_sp_span_entry_find_by_port(mlxsw_sp, dev); if (span_entry) mlxsw_sp_span_entry_invalidate(mlxsw_sp, span_entry); } - mlxsw_sp_span_respin(mlxsw_sp); if (netif_is_vxlan(dev)) err = mlxsw_sp_netdevice_vxlan_event(mlxsw_sp, dev, event, ptr); else if (mlxsw_sp_port_dev_check(dev)) - err = mlxsw_sp_netdevice_port_event(dev, dev, event, ptr); + err = mlxsw_sp_netdevice_port_event(dev, dev, event, ptr, true); else if (netif_is_lag_master(dev)) err = mlxsw_sp_netdevice_lag_event(dev, event, ptr); else if (is_vlan_dev(dev)) - err = mlxsw_sp_netdevice_vlan_event(dev, event, ptr); + err = mlxsw_sp_netdevice_vlan_event(mlxsw_sp, dev, event, ptr, + process_foreign); else if (netif_is_bridge_master(dev)) - err = mlxsw_sp_netdevice_bridge_event(dev, event, ptr); + err = mlxsw_sp_netdevice_bridge_event(mlxsw_sp, dev, event, ptr, + process_foreign); else if (netif_is_macvlan(dev)) err = mlxsw_sp_netdevice_macvlan_event(dev, event, ptr); + return err; +} + +static int mlxsw_sp_netdevice_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct mlxsw_sp *mlxsw_sp; + int err; + + mlxsw_sp = container_of(nb, struct mlxsw_sp, netdevice_nb); + mlxsw_sp_span_respin(mlxsw_sp); + err = __mlxsw_sp_netdevice_event(mlxsw_sp, event, ptr, false); + return notifier_from_errno(err); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index 231e364cbb7c..62151f0531ae 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -69,6 +69,7 @@ enum mlxsw_sp_resource_id { MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, MLXSW_SP_RESOURCE_RIFS, + MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS, }; struct mlxsw_sp_port; @@ -175,6 +176,7 @@ struct mlxsw_sp { struct mlxsw_sp_acl *acl; struct mlxsw_sp_fid_core *fid_core; struct mlxsw_sp_policer_core *policer_core; + struct mlxsw_sp_port_range_core *pr_core; struct mlxsw_sp_kvdl *kvdl; struct mlxsw_sp_nve *nve; struct notifier_block netdevice_nb; @@ -698,6 +700,8 @@ int mlxsw_sp_port_pvid_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, struct mlxsw_sp_port_vlan * mlxsw_sp_port_vlan_create(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid); void mlxsw_sp_port_vlan_destroy(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan); +int mlxsw_sp_port_kill_vid(struct net_device *dev, + __be16 __always_unused proto, u16 vid); int mlxsw_sp_port_vlan_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid_begin, u16 vid_end, bool is_member, bool untagged); int mlxsw_sp_flow_counter_get(struct mlxsw_sp *mlxsw_sp, @@ -716,8 +720,6 @@ int mlxsw_sp_txhdr_ptp_data_construct(struct mlxsw_core *mlxsw_core, bool mlxsw_sp_port_dev_check(const struct net_device *dev); struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev); struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev); -struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev); -void mlxsw_sp_port_dev_put(struct mlxsw_sp_port *mlxsw_sp_port); struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find_rcu(struct net_device *dev); int mlxsw_sp_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp); @@ -865,9 +867,13 @@ struct mlxsw_sp_acl_rule_info { egress_bind_blocker:1, counter_valid:1, policer_index_valid:1, - ipv6_valid:1; + ipv6_valid:1, + src_port_range_reg_valid:1, + dst_port_range_reg_valid:1; unsigned int counter_index; u16 policer_index; + u8 src_port_range_reg_index; + u8 dst_port_range_reg_index; struct { u32 prev_val; enum mlxsw_sp_acl_mangle_field prev_field; @@ -992,7 +998,8 @@ void mlxsw_sp_acl_ruleset_prio_get(struct mlxsw_sp_acl_ruleset *ruleset, struct mlxsw_sp_acl_rule_info * mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl, struct mlxsw_afa_block *afa_block); -void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp_acl_rule_info *rulei); +void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei); int mlxsw_sp_acl_rulei_commit(struct mlxsw_sp_acl_rule_info *rulei); void mlxsw_sp_acl_rulei_priority(struct mlxsw_sp_acl_rule_info *rulei, unsigned int priority); @@ -1484,4 +1491,18 @@ int mlxsw_sp_pgt_entry_port_set(struct mlxsw_sp *mlxsw_sp, u16 mid, int mlxsw_sp_pgt_init(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp_pgt_fini(struct mlxsw_sp *mlxsw_sp); +/* spectrum_port_range.c */ +struct mlxsw_sp_port_range { + u16 min; + u16 max; + u8 source:1; /* Source or destination */ +}; + +int mlxsw_sp_port_range_reg_get(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_port_range *range, + struct netlink_ext_ack *extack, + u8 *p_prr_index); +void mlxsw_sp_port_range_reg_put(struct mlxsw_sp *mlxsw_sp, u8 prr_index); +int mlxsw_sp_port_range_init(struct mlxsw_sp *mlxsw_sp); +void mlxsw_sp_port_range_fini(struct mlxsw_sp *mlxsw_sp); #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum1_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum1_acl_tcam.c index 3a636f753607..dfcdd37e797b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum1_acl_tcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum1_acl_tcam.c @@ -90,7 +90,7 @@ mlxsw_sp1_acl_ctcam_region_catchall_add(struct mlxsw_sp *mlxsw_sp, err_entry_add: err_rulei_commit: err_rulei_act_continue: - mlxsw_sp_acl_rulei_destroy(rulei); + mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rulei); err_rulei_create: mlxsw_sp_acl_ctcam_chunk_fini(®ion->catchall.cchunk); return err; @@ -105,7 +105,7 @@ mlxsw_sp1_acl_ctcam_region_catchall_del(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, ®ion->cregion, ®ion->catchall.cchunk, ®ion->catchall.centry); - mlxsw_sp_acl_rulei_destroy(rulei); + mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rulei); mlxsw_sp_acl_ctcam_chunk_fini(®ion->catchall.cchunk); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index 0423ac262d89..186161a3459d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -339,10 +339,17 @@ err_afa_block_create: return ERR_PTR(err); } -void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp_acl_rule_info *rulei) +void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei) { if (rulei->action_created) mlxsw_afa_block_destroy(rulei->act_block); + if (rulei->src_port_range_reg_valid) + mlxsw_sp_port_range_reg_put(mlxsw_sp, + rulei->src_port_range_reg_index); + if (rulei->dst_port_range_reg_valid) + mlxsw_sp_port_range_reg_put(mlxsw_sp, + rulei->dst_port_range_reg_index); kfree(rulei); } @@ -834,7 +841,7 @@ void mlxsw_sp_acl_rule_destroy(struct mlxsw_sp *mlxsw_sp, { struct mlxsw_sp_acl_ruleset *ruleset = rule->ruleset; - mlxsw_sp_acl_rulei_destroy(rule->rulei); + mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rule->rulei); kfree(rule); mlxsw_sp_acl_ruleset_ref_dec(mlxsw_sp, ruleset); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c index 4dea39f2b304..b7f58605b6c7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c @@ -31,12 +31,14 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l2_smac_ex[] = { static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_sip[] = { MLXSW_AFK_ELEMENT_INST_BUF(SRC_IP_0_31, 0x00, 4), + MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 16, 16), MLXSW_AFK_ELEMENT_INST_U32(IP_PROTO, 0x08, 0, 8), MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16), }; static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_dip[] = { MLXSW_AFK_ELEMENT_INST_BUF(DST_IP_0_31, 0x00, 4), + MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 16, 16), MLXSW_AFK_ELEMENT_INST_U32(IP_PROTO, 0x08, 0, 8), MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16), }; @@ -205,6 +207,7 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l4_0[] = { static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l4_2[] = { MLXSW_AFK_ELEMENT_INST_U32(TCP_FLAGS, 0x04, 16, 9), /* TCP_CONTROL + TCP_ECN */ + MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 0, 16), }; static const struct mlxsw_afk_block mlxsw_sp2_afk_blocks[] = { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c index 72917f09e806..af3f57d017ec 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c @@ -418,6 +418,68 @@ static int mlxsw_sp_flower_parse_ports(struct mlxsw_sp *mlxsw_sp, return 0; } +static int +mlxsw_sp_flower_parse_ports_range(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei, + struct flow_cls_offload *f, u8 ip_proto) +{ + const struct flow_rule *rule = flow_cls_offload_flow_rule(f); + struct flow_match_ports_range match; + u32 key_mask_value = 0; + + if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS_RANGE)) + return 0; + + if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) { + NL_SET_ERR_MSG_MOD(f->common.extack, "Only UDP and TCP keys are supported"); + return -EINVAL; + } + + flow_rule_match_ports_range(rule, &match); + + if (match.mask->tp_min.src) { + struct mlxsw_sp_port_range range = { + .min = ntohs(match.key->tp_min.src), + .max = ntohs(match.key->tp_max.src), + .source = true, + }; + u8 prr_index; + int err; + + err = mlxsw_sp_port_range_reg_get(mlxsw_sp, &range, + f->common.extack, &prr_index); + if (err) + return err; + + rulei->src_port_range_reg_index = prr_index; + rulei->src_port_range_reg_valid = true; + key_mask_value |= BIT(prr_index); + } + + if (match.mask->tp_min.dst) { + struct mlxsw_sp_port_range range = { + .min = ntohs(match.key->tp_min.dst), + .max = ntohs(match.key->tp_max.dst), + }; + u8 prr_index; + int err; + + err = mlxsw_sp_port_range_reg_get(mlxsw_sp, &range, + f->common.extack, &prr_index); + if (err) + return err; + + rulei->dst_port_range_reg_index = prr_index; + rulei->dst_port_range_reg_valid = true; + key_mask_value |= BIT(prr_index); + } + + mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_L4_PORT_RANGE, + key_mask_value, key_mask_value); + + return 0; +} + static int mlxsw_sp_flower_parse_tcp(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, struct flow_cls_offload *f, @@ -496,16 +558,17 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp, int err; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_META) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_TCP) | - BIT(FLOW_DISSECTOR_KEY_IP) | - BIT(FLOW_DISSECTOR_KEY_VLAN))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_META) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS_RANGE) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { dev_err(mlxsw_sp->bus_info->dev, "Unsupported key\n"); NL_SET_ERR_MSG_MOD(f->common.extack, "Unsupported key"); return -EOPNOTSUPP; @@ -604,6 +667,11 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_flower_parse_ports(mlxsw_sp, rulei, f, ip_proto); if (err) return err; + + err = mlxsw_sp_flower_parse_ports_range(mlxsw_sp, rulei, f, ip_proto); + if (err) + return err; + err = mlxsw_sp_flower_parse_tcp(mlxsw_sp, rulei, f, ip_proto); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c index d2b57a045aa4..5479a1c19d2e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c @@ -989,6 +989,9 @@ void mlxsw_sp_nve_fid_disable(struct mlxsw_sp *mlxsw_sp, int nve_ifindex; __be32 vni; + /* Necessary for __dev_get_by_index() below. */ + ASSERT_RTNL(); + mlxsw_sp_nve_flood_ip_flush(mlxsw_sp, fid); mlxsw_sp_nve_fdb_flush_by_fid(mlxsw_sp, fid_index); mlxsw_sp_nve_ipv6_addr_flush_by_fid(mlxsw_sp, fid_index); @@ -997,15 +1000,13 @@ void mlxsw_sp_nve_fid_disable(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_fid_vni(fid, &vni))) goto out; - nve_dev = dev_get_by_index(mlxsw_sp_net(mlxsw_sp), nve_ifindex); + nve_dev = __dev_get_by_index(mlxsw_sp_net(mlxsw_sp), nve_ifindex); if (!nve_dev) goto out; mlxsw_sp_nve_fdb_clear_offload(mlxsw_sp, fid, nve_dev, vni); mlxsw_sp_fid_fdb_clear_offload(fid, nve_dev); - dev_put(nve_dev); - out: mlxsw_sp_fid_vni_clear(fid); mlxsw_sp_nve_tunnel_fini(mlxsw_sp); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c new file mode 100644 index 000000000000..2d193de12be6 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <linux/bits.h> +#include <linux/netlink.h> +#include <linux/refcount.h> +#include <linux/xarray.h> +#include <net/devlink.h> + +#include "spectrum.h" + +struct mlxsw_sp_port_range_reg { + struct mlxsw_sp_port_range range; + refcount_t refcount; + u32 index; +}; + +struct mlxsw_sp_port_range_core { + struct xarray prr_xa; + struct xa_limit prr_ids; + atomic_t prr_count; +}; + +static int +mlxsw_sp_port_range_reg_configure(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_port_range_reg *prr) +{ + char pprr_pl[MLXSW_REG_PPRR_LEN]; + + /* We do not care if packet is IPv4/IPv6 and TCP/UDP, so set all four + * fields. + */ + mlxsw_reg_pprr_pack(pprr_pl, prr->index); + mlxsw_reg_pprr_ipv4_set(pprr_pl, true); + mlxsw_reg_pprr_ipv6_set(pprr_pl, true); + mlxsw_reg_pprr_src_set(pprr_pl, prr->range.source); + mlxsw_reg_pprr_dst_set(pprr_pl, !prr->range.source); + mlxsw_reg_pprr_tcp_set(pprr_pl, true); + mlxsw_reg_pprr_udp_set(pprr_pl, true); + mlxsw_reg_pprr_port_range_min_set(pprr_pl, prr->range.min); + mlxsw_reg_pprr_port_range_max_set(pprr_pl, prr->range.max); + + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pprr), pprr_pl); +} + +static struct mlxsw_sp_port_range_reg * +mlxsw_sp_port_range_reg_create(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_port_range *range, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core; + struct mlxsw_sp_port_range_reg *prr; + int err; + + prr = kzalloc(sizeof(*prr), GFP_KERNEL); + if (!prr) + return ERR_PTR(-ENOMEM); + + prr->range = *range; + refcount_set(&prr->refcount, 1); + + err = xa_alloc(&pr_core->prr_xa, &prr->index, prr, pr_core->prr_ids, + GFP_KERNEL); + if (err) { + if (err == -EBUSY) + NL_SET_ERR_MSG_MOD(extack, "Exceeded number of port range registers"); + goto err_xa_alloc; + } + + err = mlxsw_sp_port_range_reg_configure(mlxsw_sp, prr); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to configure port range register"); + goto err_reg_configure; + } + + atomic_inc(&pr_core->prr_count); + + return prr; + +err_reg_configure: + xa_erase(&pr_core->prr_xa, prr->index); +err_xa_alloc: + kfree(prr); + return ERR_PTR(err); +} + +static void mlxsw_sp_port_range_reg_destroy(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_port_range_reg *prr) +{ + struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core; + + atomic_dec(&pr_core->prr_count); + xa_erase(&pr_core->prr_xa, prr->index); + kfree(prr); +} + +static struct mlxsw_sp_port_range_reg * +mlxsw_sp_port_range_reg_find(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_port_range *range) +{ + struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core; + struct mlxsw_sp_port_range_reg *prr; + unsigned long index; + + xa_for_each(&pr_core->prr_xa, index, prr) { + if (prr->range.min == range->min && + prr->range.max == range->max && + prr->range.source == range->source) + return prr; + } + + return NULL; +} + +int mlxsw_sp_port_range_reg_get(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_port_range *range, + struct netlink_ext_ack *extack, + u8 *p_prr_index) +{ + struct mlxsw_sp_port_range_reg *prr; + + prr = mlxsw_sp_port_range_reg_find(mlxsw_sp, range); + if (prr) { + refcount_inc(&prr->refcount); + *p_prr_index = prr->index; + return 0; + } + + prr = mlxsw_sp_port_range_reg_create(mlxsw_sp, range, extack); + if (IS_ERR(prr)) + return PTR_ERR(prr); + + *p_prr_index = prr->index; + + return 0; +} + +void mlxsw_sp_port_range_reg_put(struct mlxsw_sp *mlxsw_sp, u8 prr_index) +{ + struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core; + struct mlxsw_sp_port_range_reg *prr; + + prr = xa_load(&pr_core->prr_xa, prr_index); + if (WARN_ON(!prr)) + return; + + if (!refcount_dec_and_test(&prr->refcount)) + return; + + mlxsw_sp_port_range_reg_destroy(mlxsw_sp, prr); +} + +static u64 mlxsw_sp_port_range_reg_occ_get(void *priv) +{ + struct mlxsw_sp_port_range_core *pr_core = priv; + + return atomic_read(&pr_core->prr_count); +} + +int mlxsw_sp_port_range_init(struct mlxsw_sp *mlxsw_sp) +{ + struct mlxsw_sp_port_range_core *pr_core; + struct mlxsw_core *core = mlxsw_sp->core; + u64 max; + + if (!MLXSW_CORE_RES_VALID(core, ACL_MAX_L4_PORT_RANGE)) + return -EIO; + max = MLXSW_CORE_RES_GET(core, ACL_MAX_L4_PORT_RANGE); + + /* Each port range register is represented using a single bit in the + * two bytes "l4_port_range" ACL key element. + */ + WARN_ON(max > BITS_PER_BYTE * sizeof(u16)); + + pr_core = kzalloc(sizeof(*mlxsw_sp->pr_core), GFP_KERNEL); + if (!pr_core) + return -ENOMEM; + mlxsw_sp->pr_core = pr_core; + + pr_core->prr_ids.max = max - 1; + xa_init_flags(&pr_core->prr_xa, XA_FLAGS_ALLOC); + + devl_resource_occ_get_register(priv_to_devlink(core), + MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS, + mlxsw_sp_port_range_reg_occ_get, + pr_core); + + return 0; +} + +void mlxsw_sp_port_range_fini(struct mlxsw_sp *mlxsw_sp) +{ + struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core; + + devl_resource_occ_get_unregister(priv_to_devlink(mlxsw_sp->core), + MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS); + WARN_ON(!xa_empty(&pr_core->prr_xa)); + xa_destroy(&pr_core->prr_xa); + kfree(pr_core); +} diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index b32adf277a22..debd2c466f11 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -71,6 +71,7 @@ static const struct rhashtable_params mlxsw_sp_crif_ht_params = { struct mlxsw_sp_rif { struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */ + netdevice_tracker dev_tracker; struct list_head neigh_list; struct mlxsw_sp_fid *fid; unsigned char addr[ETH_ALEN]; @@ -139,6 +140,7 @@ struct mlxsw_sp_rif_ops { struct netlink_ext_ack *extack); void (*deconfigure)(struct mlxsw_sp_rif *rif); struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, + const struct mlxsw_sp_rif_params *params, struct netlink_ext_ack *extack); void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); }; @@ -2871,6 +2873,21 @@ static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev) return !!mlxsw_sp_port; } +static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router, + struct neighbour *n) +{ + struct net *net; + + net = neigh_parms_net(n->parms); + + /* Take a reference to ensure the neighbour won't be destructed until we + * drop the reference in delayed work. + */ + neigh_clone(n); + return mlxsw_sp_router_schedule_work(net, router, n, + mlxsw_sp_router_neigh_event_work); +} + static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -2878,7 +2895,6 @@ static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, unsigned long interval; struct neigh_parms *p; struct neighbour *n; - struct net *net; router = container_of(nb, struct mlxsw_sp_router, netevent_nb); @@ -2902,7 +2918,6 @@ static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, break; case NETEVENT_NEIGH_UPDATE: n = ptr; - net = neigh_parms_net(n->parms); if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) return NOTIFY_DONE; @@ -2910,13 +2925,7 @@ static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, if (!mlxsw_sp_dev_lower_is_port(n->dev)) return NOTIFY_DONE; - /* Take a reference to ensure the neighbour won't be - * destructed until we drop the reference in delayed - * work. - */ - neigh_clone(n); - return mlxsw_sp_router_schedule_work(net, router, n, - mlxsw_sp_router_neigh_event_work); + return mlxsw_sp_router_schedule_neigh_work(router, n); case NETEVENT_IPV4_MPATH_HASH_UPDATE: case NETEVENT_IPV6_MPATH_HASH_UPDATE: @@ -2975,6 +2984,52 @@ static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, } } +struct mlxsw_sp_neigh_rif_made_sync { + struct mlxsw_sp *mlxsw_sp; + struct mlxsw_sp_rif *rif; + int err; +}; + +static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data) +{ + struct mlxsw_sp_neigh_rif_made_sync *rms = data; + int rc; + + if (rms->err) + return; + if (n->dev != mlxsw_sp_rif_dev(rms->rif)) + return; + rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n); + if (rc != NOTIFY_DONE) + rms->err = -ENOMEM; +} + +static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_rif *rif) +{ + struct mlxsw_sp_neigh_rif_made_sync rms = { + .mlxsw_sp = mlxsw_sp, + .rif = rif, + }; + + neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); + if (rms.err) + goto err_arp; + +#if IS_ENABLED(CONFIG_IPV6) + neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); +#endif + if (rms.err) + goto err_nd; + + return 0; + +err_nd: +err_arp: + mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); + return rms.err; +} + enum mlxsw_sp_nexthop_type { MLXSW_SP_NEXTHOP_TYPE_ETH, MLXSW_SP_NEXTHOP_TYPE_IPIP, @@ -4396,6 +4451,19 @@ err_neigh_init: return err; } +static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh) +{ + switch (nh->type) { + case MLXSW_SP_NEXTHOP_TYPE_ETH: + return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); + case MLXSW_SP_NEXTHOP_TYPE_IPIP: + break; + } + + return 0; +} + static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh) { @@ -4524,6 +4592,35 @@ static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, } } +static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_rif *rif) +{ + struct mlxsw_sp_nexthop *nh, *tmp; + unsigned int n = 0; + int err; + + list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, + crif_list_node) { + err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh); + if (err) + goto err_nexthop_type_rif; + mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); + n++; + } + + return 0; + +err_nexthop_type_rif: + list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, + crif_list_node) { + if (!n--) + break; + mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); + mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); + } + return err; +} + static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_rif *rif) { @@ -7451,6 +7548,7 @@ struct mlxsw_sp_fib6_event_work { struct mlxsw_sp_fib_event_work { struct work_struct work; + netdevice_tracker dev_tracker; union { struct mlxsw_sp_fib6_event_work fib6_work; struct fib_entry_notifier_info fen_info; @@ -7624,12 +7722,12 @@ static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) &fib_work->ven_info); if (err) dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); - dev_put(fib_work->ven_info.dev); + netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); break; case FIB_EVENT_VIF_DEL: mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_work->ven_info); - dev_put(fib_work->ven_info.dev); + netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); break; } mutex_unlock(&mlxsw_sp->router->lock); @@ -7700,7 +7798,8 @@ mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, case FIB_EVENT_VIF_ADD: case FIB_EVENT_VIF_DEL: memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); - dev_hold(fib_work->ven_info.dev); + netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker, + GFP_ATOMIC); break; } } @@ -7884,6 +7983,26 @@ static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); } +static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_rif *rif) +{ + int err; + + err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif); + if (err) + return err; + + err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif); + if (err) + goto err_nexthop; + + return 0; + +err_nexthop: + mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); + return err; +} + static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_rif *rif) { @@ -8190,6 +8309,7 @@ mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, struct mlxsw_sp_router_hwstats_notify_work { struct work_struct work; struct net_device *dev; + netdevice_tracker dev_tracker; }; static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) @@ -8201,7 +8321,7 @@ static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) rtnl_lock(); rtnl_offload_xstats_notify(hws_work->dev); rtnl_unlock(); - dev_put(hws_work->dev); + netdev_put(hws_work->dev, &hws_work->dev_tracker); kfree(hws_work); } @@ -8221,7 +8341,7 @@ mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) return; INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); - dev_hold(dev); + netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL); hws_work->dev = dev; mlxsw_core_schedule_work(&hws_work->work); } @@ -8293,14 +8413,14 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, err = -ENOMEM; goto err_rif_alloc; } - dev_hold(params->dev); + netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL); mlxsw_sp->router->rifs[rif_index] = rif; rif->mlxsw_sp = mlxsw_sp; rif->ops = ops; rif->rif_entries = rif_entries; if (ops->fid_get) { - fid = ops->fid_get(rif, extack); + fid = ops->fid_get(rif, params, extack); if (IS_ERR(fid)) { err = PTR_ERR(fid); goto err_fid_get; @@ -8321,6 +8441,10 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, goto err_mr_rif_add; } + err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif); + if (err) + goto err_rif_made_sync; + if (netdev_offload_xstats_enabled(params->dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { err = mlxsw_sp_router_port_l3_stats_enable(rif); @@ -8335,6 +8459,8 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, return rif; err_stats_enable: + mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); +err_rif_made_sync: err_mr_rif_add: for (i--; i >= 0; i--) mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); @@ -8344,7 +8470,7 @@ err_configure: mlxsw_sp_fid_put(fid); err_fid_get: mlxsw_sp->router->rifs[rif_index] = NULL; - dev_put(params->dev); + netdev_put(params->dev, &rif->dev_tracker); mlxsw_sp_rif_free(rif); err_rif_alloc: err_crif_lookup: @@ -8386,7 +8512,7 @@ static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) /* Loopback RIFs are not associated with a FID. */ mlxsw_sp_fid_put(fid); mlxsw_sp->router->rifs[rif->rif_index] = NULL; - dev_put(dev); + netdev_put(dev, &rif->dev_tracker); mlxsw_sp_rif_free(rif); mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); vr->rif_count--; @@ -8410,6 +8536,110 @@ out: mutex_unlock(&mlxsw_sp->router->lock); } +static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp, + struct net_device *br_dev, + u16 vid) +{ + struct net_device *upper_dev; + struct mlxsw_sp_crif *crif; + + rcu_read_lock(); + upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid); + rcu_read_unlock(); + + if (!upper_dev) + return; + + crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev); + if (!crif || !crif->rif) + return; + + mlxsw_sp_rif_destroy(crif->rif); +} + +static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, + struct net_device *l3_dev, + int lower_pvid, + unsigned long event, + struct netlink_ext_ack *extack); + +int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp, + struct net_device *br_dev, + u16 new_vid, bool is_pvid, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_rif *old_rif; + struct mlxsw_sp_rif *new_rif; + struct net_device *upper_dev; + u16 old_pvid = 0; + u16 new_pvid; + int err = 0; + + mutex_lock(&mlxsw_sp->router->lock); + old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev); + if (old_rif) { + /* If the RIF on the bridge is not a VLAN RIF, we shouldn't have + * gotten a PVID notification. + */ + if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)) + old_rif = NULL; + else + old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid); + } + + if (is_pvid) + new_pvid = new_vid; + else if (old_pvid == new_vid) + new_pvid = 0; + else + goto out; + + if (old_pvid == new_pvid) + goto out; + + if (new_pvid) { + struct mlxsw_sp_rif_params params = { + .dev = br_dev, + .vid = new_pvid, + }; + + /* If there is a VLAN upper with the same VID as the new PVID, + * kill its RIF, if there is one. + */ + mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid); + + if (mlxsw_sp_dev_addr_list_empty(br_dev)) + goto out; + new_rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); + if (IS_ERR(new_rif)) { + err = PTR_ERR(new_rif); + goto out; + } + + if (old_pvid) + mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif, + true); + } else { + mlxsw_sp_rif_destroy(old_rif); + } + + if (old_pvid) { + rcu_read_lock(); + upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), + old_pvid); + rcu_read_unlock(); + if (upper_dev) + err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, + upper_dev, + new_pvid, + NETDEV_UP, extack); + } + +out: + mutex_unlock(&mlxsw_sp->router->lock); + return err; +} + static void mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) @@ -8664,21 +8894,24 @@ __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - struct mlxsw_sp_rif_params params = { - .dev = l3_dev, - }; + struct mlxsw_sp_rif_params params; u16 vid = mlxsw_sp_port_vlan->vid; struct mlxsw_sp_rif *rif; struct mlxsw_sp_fid *fid; int err; + params = (struct mlxsw_sp_rif_params) { + .dev = l3_dev, + .vid = vid, + }; + mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); if (IS_ERR(rif)) return PTR_ERR(rif); /* FID was already created, just take a reference */ - fid = rif->ops->fid_get(rif, extack); + fid = rif->ops->fid_get(rif, ¶ms, extack); err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); if (err) goto err_fid_port_vid_map; @@ -8776,10 +9009,11 @@ static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, } static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, - unsigned long event, + unsigned long event, bool nomaster, struct netlink_ext_ack *extack) { - if (netif_is_any_bridge_port(port_dev) || netif_is_lag_port(port_dev)) + if (!nomaster && (netif_is_any_bridge_port(port_dev) || + netif_is_lag_port(port_dev))) return 0; return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, @@ -8810,10 +9044,10 @@ static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, } static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, - unsigned long event, + unsigned long event, bool nomaster, struct netlink_ext_ack *extack) { - if (netif_is_bridge_port(lag_dev)) + if (!nomaster && netif_is_bridge_port(lag_dev)) return 0; return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, @@ -8822,6 +9056,7 @@ static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, struct net_device *l3_dev, + int lower_pvid, unsigned long event, struct netlink_ext_ack *extack) { @@ -8829,6 +9064,7 @@ static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, .dev = l3_dev, }; struct mlxsw_sp_rif *rif; + int err; switch (event) { case NETDEV_UP: @@ -8840,7 +9076,21 @@ static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); return -EOPNOTSUPP; } + err = br_vlan_get_pvid(l3_dev, ¶ms.vid); + if (err) + return err; + if (!params.vid) + return 0; + } else if (is_vlan_dev(l3_dev)) { + params.vid = vlan_dev_vlan_id(l3_dev); + + /* If the VID matches PVID of the bridge below, the + * bridge owns the RIF for this VLAN. Don't do anything. + */ + if ((int)params.vid == lower_pvid) + return 0; } + rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); if (IS_ERR(rif)) return PTR_ERR(rif); @@ -8856,24 +9106,32 @@ static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, struct net_device *vlan_dev, - unsigned long event, + unsigned long event, bool nomaster, struct netlink_ext_ack *extack) { struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); u16 vid = vlan_dev_vlan_id(vlan_dev); + u16 lower_pvid; + int err; - if (netif_is_bridge_port(vlan_dev)) + if (!nomaster && netif_is_bridge_port(vlan_dev)) return 0; - if (mlxsw_sp_port_dev_check(real_dev)) + if (mlxsw_sp_port_dev_check(real_dev)) { return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, event, vid, extack); - else if (netif_is_lag_master(real_dev)) + } else if (netif_is_lag_master(real_dev)) { return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, vid, extack); - else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) - return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, + } else if (netif_is_bridge_master(real_dev) && + br_vlan_enabled(real_dev)) { + err = br_vlan_get_pvid(real_dev, &lower_pvid); + if (err) + return err; + return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, + lower_pvid, event, extack); + } return 0; } @@ -8927,10 +9185,8 @@ static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, int err; rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); - if (!rif) { - NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); - return -EOPNOTSUPP; - } + if (!rif) + return 0; err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, mlxsw_sp_fid_index(rif->fid), true); @@ -9000,19 +9256,21 @@ static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, struct net_device *dev, - unsigned long event, + unsigned long event, bool nomaster, struct netlink_ext_ack *extack) { if (mlxsw_sp_port_dev_check(dev)) - return mlxsw_sp_inetaddr_port_event(dev, event, extack); + return mlxsw_sp_inetaddr_port_event(dev, event, nomaster, + extack); else if (netif_is_lag_master(dev)) - return mlxsw_sp_inetaddr_lag_event(dev, event, extack); + return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster, + extack); else if (netif_is_bridge_master(dev)) - return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, + return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event, extack); else if (is_vlan_dev(dev)) return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, - extack); + nomaster, extack); else if (netif_is_macvlan(dev)) return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, extack); @@ -9039,7 +9297,8 @@ static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, if (!mlxsw_sp_rif_should_config(rif, dev, event)) goto out; - err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); + err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false, + NULL); out: mutex_unlock(&router->lock); return notifier_from_errno(err); @@ -9063,7 +9322,8 @@ static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, if (!mlxsw_sp_rif_should_config(rif, dev, event)) goto out; - err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); + err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, + ivi->extack); out: mutex_unlock(&mlxsw_sp->router->lock); return notifier_from_errno(err); @@ -9073,6 +9333,7 @@ struct mlxsw_sp_inet6addr_event_work { struct work_struct work; struct mlxsw_sp *mlxsw_sp; struct net_device *dev; + netdevice_tracker dev_tracker; unsigned long event; }; @@ -9092,11 +9353,11 @@ static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) if (!mlxsw_sp_rif_should_config(rif, dev, event)) goto out; - __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); + __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL); out: mutex_unlock(&mlxsw_sp->router->lock); rtnl_unlock(); - dev_put(dev); + netdev_put(dev, &inet6addr_work->dev_tracker); kfree(inet6addr_work); } @@ -9122,7 +9383,7 @@ static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, inet6addr_work->mlxsw_sp = router->mlxsw_sp; inet6addr_work->dev = dev; inet6addr_work->event = event; - dev_hold(dev); + netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC); mlxsw_core_schedule_work(&inet6addr_work->work); return NOTIFY_DONE; @@ -9146,7 +9407,8 @@ static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, if (!mlxsw_sp_rif_should_config(rif, dev, event)) goto out; - err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); + err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, + i6vi->extack); out: mutex_unlock(&mlxsw_sp->router->lock); return notifier_from_errno(err); @@ -9466,10 +9728,11 @@ static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, */ rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); if (rif) - __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, + __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, extack); - return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); + return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false, + extack); } static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, @@ -9480,7 +9743,7 @@ static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); if (!rif) return; - __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); + __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL); } static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) @@ -9523,6 +9786,116 @@ mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, return err; } +struct mlxsw_sp_router_replay_inetaddr_up { + struct mlxsw_sp *mlxsw_sp; + struct netlink_ext_ack *extack; + unsigned int done; + bool deslavement; +}; + +static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev, + struct netdev_nested_priv *priv) +{ + struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; + bool nomaster = ctx->deslavement; + struct mlxsw_sp_crif *crif; + int err; + + if (mlxsw_sp_dev_addr_list_empty(dev)) + return 0; + + crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); + if (!crif || crif->rif) + return 0; + + if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) + return 0; + + err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP, + nomaster, ctx->extack); + if (err) + return err; + + ctx->done++; + return 0; +} + +static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev, + struct netdev_nested_priv *priv) +{ + struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; + bool nomaster = ctx->deslavement; + struct mlxsw_sp_crif *crif; + + if (!ctx->done) + return 0; + + if (mlxsw_sp_dev_addr_list_empty(dev)) + return 0; + + crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); + if (!crif || !crif->rif) + return 0; + + /* We are rolling back NETDEV_UP, so ask for that. */ + if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) + return 0; + + __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster, + NULL); + + ctx->done--; + return 0; +} + +int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp, + struct net_device *upper_dev, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_router_replay_inetaddr_up ctx = { + .mlxsw_sp = mlxsw_sp, + .extack = extack, + .deslavement = false, + }; + struct netdev_nested_priv priv = { + .data = &ctx, + }; + int err; + + err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv); + if (err) + return err; + + err = netdev_walk_all_upper_dev_rcu(upper_dev, + mlxsw_sp_router_replay_inetaddr_up, + &priv); + if (err) + goto err_replay_up; + + return 0; + +err_replay_up: + netdev_walk_all_upper_dev_rcu(upper_dev, + mlxsw_sp_router_unreplay_inetaddr_up, + &priv); + mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv); + return err; +} + +void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp, + struct net_device *dev) +{ + struct mlxsw_sp_router_replay_inetaddr_up ctx = { + .mlxsw_sp = mlxsw_sp, + .deslavement = true, + }; + struct netdev_nested_priv priv = { + .data = &ctx, + }; + + mlxsw_sp_router_replay_inetaddr_up(dev, &priv); +} + static int mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, struct net_device *dev, @@ -9539,15 +9912,84 @@ mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, dev, extack); } +static void +mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, + struct net_device *dev) +{ + struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; + + mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, + vid); + if (WARN_ON(!mlxsw_sp_port_vlan)) + return; + + __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); +} + static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, struct net_device *lag_dev, struct netlink_ext_ack *extack) { u16 default_vid = MLXSW_SP_DEFAULT_VID; + struct net_device *upper_dev; + struct list_head *iter; + int done = 0; + u16 vid; + int err; - return mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, - default_vid, lag_dev, - extack); + err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid, + lag_dev, extack); + if (err) + return err; + + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + + vid = vlan_dev_vlan_id(upper_dev); + err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid, + upper_dev, extack); + if (err) + goto err_router_join_dev; + + ++done; + } + + return 0; + +err_router_join_dev: + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + if (!done--) + break; + + vid = vlan_dev_vlan_id(upper_dev); + mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); + } + + mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); + return err; +} + +static void +__mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, + struct net_device *lag_dev) +{ + u16 default_vid = MLXSW_SP_DEFAULT_VID; + struct net_device *upper_dev; + struct list_head *iter; + u16 vid; + + netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { + if (!is_vlan_dev(upper_dev)) + continue; + + vid = vlan_dev_vlan_id(upper_dev); + mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); + } + + mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); } int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, @@ -9563,6 +10005,14 @@ int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, return err; } +void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, + struct net_device *lag_dev) +{ + mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); + __mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev); + mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); +} + static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -9608,6 +10058,40 @@ out: return notifier_from_errno(err); } +struct mlxsw_sp_macvlan_replay { + struct mlxsw_sp *mlxsw_sp; + struct netlink_ext_ack *extack; +}; + +static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev, + struct netdev_nested_priv *priv) +{ + const struct mlxsw_sp_macvlan_replay *rms = priv->data; + struct netlink_ext_ack *extack = rms->extack; + struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp; + + if (!netif_is_macvlan(dev)) + return 0; + + return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack); +} + +static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_macvlan_replay rms = { + .mlxsw_sp = rif->mlxsw_sp, + .extack = extack, + }; + struct netdev_nested_priv priv = { + .data = &rms, + }; + + return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif), + mlxsw_sp_macvlan_replay_upper, + &priv); +} + static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, struct netdev_nested_priv *priv) { @@ -9630,7 +10114,6 @@ static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) if (!netif_is_macvlan_port(dev)) return 0; - netdev_warn(dev, "Router interface is deleted. Upper macvlans will not work\n"); return netdev_walk_all_upper_dev_rcu(dev, __mlxsw_sp_rif_macvlan_flush, &priv); } @@ -9688,6 +10171,10 @@ static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, if (err) goto err_rif_subport_op; + err = mlxsw_sp_macvlan_replay(rif, extack); + if (err) + goto err_macvlan_replay; + err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), true); if (err) @@ -9703,6 +10190,8 @@ err_fid_rif_set: mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), false); err_rif_fdb_op: + mlxsw_sp_rif_macvlan_flush(rif); +err_macvlan_replay: mlxsw_sp_rif_subport_op(rif, false); err_rif_subport_op: mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); @@ -9724,6 +10213,7 @@ static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) static struct mlxsw_sp_fid * mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, + const struct mlxsw_sp_rif_params *params, struct netlink_ext_ack *extack) { return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); @@ -9788,6 +10278,10 @@ static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, if (err) goto err_fid_bc_flood_set; + err = mlxsw_sp_macvlan_replay(rif, extack); + if (err) + goto err_macvlan_replay; + err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), true); if (err) @@ -9803,6 +10297,8 @@ err_fid_rif_set: mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), false); err_rif_fdb_op: + mlxsw_sp_rif_macvlan_flush(rif); +err_macvlan_replay: mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, mlxsw_sp_router_port(mlxsw_sp), false); err_fid_bc_flood_set: @@ -9836,6 +10332,7 @@ static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) static struct mlxsw_sp_fid * mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, + const struct mlxsw_sp_rif_params *params, struct netlink_ext_ack *extack) { int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif); @@ -9869,27 +10366,22 @@ static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { static struct mlxsw_sp_fid * mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, + const struct mlxsw_sp_rif_params *params, struct netlink_ext_ack *extack) { struct net_device *dev = mlxsw_sp_rif_dev(rif); struct net_device *br_dev; - u16 vid; - int err; + + if (WARN_ON(!params->vid)) + return ERR_PTR(-EINVAL); if (is_vlan_dev(dev)) { - vid = vlan_dev_vlan_id(dev); br_dev = vlan_dev_real_dev(dev); if (WARN_ON(!netif_is_bridge_master(br_dev))) return ERR_PTR(-EINVAL); - } else { - err = br_vlan_get_pvid(dev, &vid); - if (err < 0 || !vid) { - NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); - return ERR_PTR(-EINVAL); - } } - return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); + return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid); } static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) @@ -9954,6 +10446,10 @@ static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid, if (err) goto err_fid_bc_flood_set; + err = mlxsw_sp_macvlan_replay(rif, extack); + if (err) + goto err_macvlan_replay; + err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), true); if (err) @@ -9969,6 +10465,8 @@ err_fid_rif_set: mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, mlxsw_sp_fid_index(rif->fid), false); err_rif_fdb_op: + mlxsw_sp_rif_macvlan_flush(rif); +err_macvlan_replay: mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, mlxsw_sp_router_port(mlxsw_sp), false); err_fid_bc_flood_set: diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h index 9a2669a08480..ed3b628caafe 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h @@ -171,8 +171,19 @@ int mlxsw_sp_ipip_ecn_encap_init(struct mlxsw_sp *mlxsw_sp); int mlxsw_sp_ipip_ecn_decap_init(struct mlxsw_sp *mlxsw_sp); struct net_device * mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev); +int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp, + struct net_device *dev, + u16 new_vid, bool is_pvid, + struct netlink_ext_ack *extack); int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, struct net_device *lag_dev, struct netlink_ext_ack *extack); +void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, + struct net_device *lag_dev); +int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp, + struct net_device *upper_dev, + struct netlink_ext_ack *extack); +void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp, + struct net_device *dev); #endif /* _MLXSW_ROUTER_H_*/ diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c index d88e62bc759f..5376d4af5f91 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c @@ -384,6 +384,91 @@ mlxsw_sp_bridge_port_find(struct mlxsw_sp_bridge *bridge, return __mlxsw_sp_bridge_port_find(bridge_device, brport_dev); } +static int mlxsw_sp_port_obj_add(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack); +static int mlxsw_sp_port_obj_del(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj); + +struct mlxsw_sp_bridge_port_replay_switchdev_objs { + struct net_device *brport_dev; + struct mlxsw_sp_port *mlxsw_sp_port; + int done; +}; + +static int +mlxsw_sp_bridge_port_replay_switchdev_objs(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + struct switchdev_notifier_port_obj_info *port_obj_info = ptr; + struct netlink_ext_ack *extack = port_obj_info->info.extack; + struct mlxsw_sp_bridge_port_replay_switchdev_objs *rso; + int err = 0; + + rso = (void *)port_obj_info->info.ctx; + + if (event != SWITCHDEV_PORT_OBJ_ADD || + dev != rso->brport_dev) + goto out; + + /* When a port is joining the bridge through a LAG, there likely are + * VLANs configured on that LAG already. The replay will thus attempt to + * have the given port-vlans join the corresponding FIDs. But the LAG + * netdevice has already called the ndo_vlan_rx_add_vid NDO for its VLAN + * memberships, back before CHANGEUPPER was distributed and netdevice + * master set. So now before propagating the VLAN events further, we + * first need to kill the corresponding VID at the mlxsw_sp_port. + * + * Note that this doesn't need to be rolled back on failure -- if the + * replay fails, the enslavement is off, and the VIDs would be killed by + * LAG anyway as part of its rollback. + */ + if (port_obj_info->obj->id == SWITCHDEV_OBJ_ID_PORT_VLAN) { + u16 vid = SWITCHDEV_OBJ_PORT_VLAN(port_obj_info->obj)->vid; + + err = mlxsw_sp_port_kill_vid(rso->mlxsw_sp_port->dev, 0, vid); + if (err) + goto out; + } + + ++rso->done; + err = mlxsw_sp_port_obj_add(rso->mlxsw_sp_port->dev, NULL, + port_obj_info->obj, extack); + +out: + return notifier_from_errno(err); +} + +static struct notifier_block mlxsw_sp_bridge_port_replay_switchdev_objs_nb = { + .notifier_call = mlxsw_sp_bridge_port_replay_switchdev_objs, +}; + +static int +mlxsw_sp_bridge_port_unreplay_switchdev_objs(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + struct switchdev_notifier_port_obj_info *port_obj_info = ptr; + struct mlxsw_sp_bridge_port_replay_switchdev_objs *rso; + + rso = (void *)port_obj_info->info.ctx; + + if (event != SWITCHDEV_PORT_OBJ_ADD || + dev != rso->brport_dev) + return NOTIFY_DONE; + if (!rso->done--) + return NOTIFY_STOP; + + mlxsw_sp_port_obj_del(rso->mlxsw_sp_port->dev, NULL, + port_obj_info->obj); + return NOTIFY_DONE; +} + +static struct notifier_block mlxsw_sp_bridge_port_unreplay_switchdev_objs_nb = { + .notifier_call = mlxsw_sp_bridge_port_unreplay_switchdev_objs, +}; + static struct mlxsw_sp_bridge_port * mlxsw_sp_bridge_port_create(struct mlxsw_sp_bridge_device *bridge_device, struct net_device *brport_dev, @@ -1479,29 +1564,15 @@ err_port_vlan_set: } static int -mlxsw_sp_br_ban_rif_pvid_change(struct mlxsw_sp *mlxsw_sp, - const struct net_device *br_dev, - const struct switchdev_obj_port_vlan *vlan) +mlxsw_sp_br_rif_pvid_change(struct mlxsw_sp *mlxsw_sp, + struct net_device *br_dev, + const struct switchdev_obj_port_vlan *vlan, + struct netlink_ext_ack *extack) { - u16 pvid; - - pvid = mlxsw_sp_rif_vid(mlxsw_sp, br_dev); - if (!pvid) - return 0; - - if (vlan->flags & BRIDGE_VLAN_INFO_PVID) { - if (vlan->vid != pvid) { - netdev_err(br_dev, "Can't change PVID, it's used by router interface\n"); - return -EBUSY; - } - } else { - if (vlan->vid == pvid) { - netdev_err(br_dev, "Can't remove PVID, it's used by router interface\n"); - return -EBUSY; - } - } + bool flag_pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID; - return 0; + return mlxsw_sp_router_bridge_vlan_add(mlxsw_sp, br_dev, vlan->vid, + flag_pvid, extack); } static int mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port, @@ -1518,8 +1589,8 @@ static int mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port, int err = 0; if (br_vlan_enabled(orig_dev)) - err = mlxsw_sp_br_ban_rif_pvid_change(mlxsw_sp, - orig_dev, vlan); + err = mlxsw_sp_br_rif_pvid_change(mlxsw_sp, orig_dev, + vlan, extack); if (!err) err = -EOPNOTSUPP; return err; @@ -2365,6 +2436,33 @@ static struct mlxsw_sp_port *mlxsw_sp_lag_rep_port(struct mlxsw_sp *mlxsw_sp, } static int +mlxsw_sp_bridge_port_replay(struct mlxsw_sp_bridge_port *bridge_port, + struct mlxsw_sp_port *mlxsw_sp_port, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_bridge_port_replay_switchdev_objs rso = { + .brport_dev = bridge_port->dev, + .mlxsw_sp_port = mlxsw_sp_port, + }; + struct notifier_block *nb; + int err; + + nb = &mlxsw_sp_bridge_port_replay_switchdev_objs_nb; + err = switchdev_bridge_port_replay(bridge_port->dev, mlxsw_sp_port->dev, + &rso, NULL, nb, extack); + if (err) + goto err_replay; + + return 0; + +err_replay: + nb = &mlxsw_sp_bridge_port_unreplay_switchdev_objs_nb; + switchdev_bridge_port_replay(bridge_port->dev, mlxsw_sp_port->dev, + &rso, NULL, nb, extack); + return err; +} + +static int mlxsw_sp_bridge_vlan_aware_port_join(struct mlxsw_sp_bridge_port *bridge_port, struct mlxsw_sp_port *mlxsw_sp_port, struct netlink_ext_ack *extack) @@ -2378,7 +2476,7 @@ mlxsw_sp_bridge_vlan_aware_port_join(struct mlxsw_sp_bridge_port *bridge_port, if (mlxsw_sp_port->default_vlan->fid) mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port->default_vlan); - return 0; + return mlxsw_sp_bridge_port_replay(bridge_port, mlxsw_sp_port, extack); } static int @@ -2550,6 +2648,7 @@ mlxsw_sp_bridge_8021d_port_join(struct mlxsw_sp_bridge_device *bridge_device, struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; struct net_device *dev = bridge_port->dev; u16 vid; + int err; vid = is_vlan_dev(dev) ? vlan_dev_vlan_id(dev) : MLXSW_SP_DEFAULT_VID; mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); @@ -2565,8 +2664,20 @@ mlxsw_sp_bridge_8021d_port_join(struct mlxsw_sp_bridge_device *bridge_device, if (mlxsw_sp_port_vlan->fid) mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); - return mlxsw_sp_port_vlan_bridge_join(mlxsw_sp_port_vlan, bridge_port, - extack); + err = mlxsw_sp_port_vlan_bridge_join(mlxsw_sp_port_vlan, bridge_port, + extack); + if (err) + return err; + + err = mlxsw_sp_bridge_port_replay(bridge_port, mlxsw_sp_port, extack); + if (err) + goto err_replay; + + return 0; + +err_replay: + mlxsw_sp_port_vlan_bridge_leave(mlxsw_sp_port_vlan); + return err; } static void @@ -2783,8 +2894,15 @@ int mlxsw_sp_port_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port, if (err) goto err_port_join; + err = mlxsw_sp_netdevice_enslavement_replay(mlxsw_sp, br_dev, extack); + if (err) + goto err_replay; + return 0; +err_replay: + bridge_device->ops->port_leave(bridge_device, bridge_port, + mlxsw_sp_port); err_port_join: mlxsw_sp_bridge_port_put(mlxsw_sp->bridge, bridge_port); return err; @@ -3262,6 +3380,7 @@ out: struct mlxsw_sp_switchdev_event_work { struct work_struct work; + netdevice_tracker dev_tracker; union { struct switchdev_notifier_fdb_info fdb_info; struct switchdev_notifier_vxlan_fdb_info vxlan_fdb_info; @@ -3418,8 +3537,8 @@ static void mlxsw_sp_switchdev_bridge_fdb_event_work(struct work_struct *work) out: rtnl_unlock(); kfree(switchdev_work->fdb_info.addr); + netdev_put(dev, &switchdev_work->dev_tracker); kfree(switchdev_work); - dev_put(dev); } static void @@ -3574,8 +3693,8 @@ static void mlxsw_sp_switchdev_vxlan_fdb_event_work(struct work_struct *work) out: rtnl_unlock(); + netdev_put(dev, &switchdev_work->dev_tracker); kfree(switchdev_work); - dev_put(dev); } static int @@ -3675,7 +3794,7 @@ static int mlxsw_sp_switchdev_event(struct notifier_block *unused, * upper device containig mlxsw_sp_port or just a * mlxsw_sp_port */ - dev_hold(dev); + netdev_hold(dev, &switchdev_work->dev_tracker, GFP_ATOMIC); break; case SWITCHDEV_VXLAN_FDB_ADD_TO_DEVICE: case SWITCHDEV_VXLAN_FDB_DEL_TO_DEVICE: @@ -3685,7 +3804,7 @@ static int mlxsw_sp_switchdev_event(struct notifier_block *unused, info); if (err) goto err_vxlan_work_prepare; - dev_hold(dev); + netdev_hold(dev, &switchdev_work->dev_tracker, GFP_ATOMIC); break; default: kfree(switchdev_work); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index fbb0bb4594cd..4a1acc7234f6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -5,9 +5,10 @@ #include <linux/if_vlan.h> #include <linux/iopoll.h> #include <linux/ip.h> -#include <linux/of_platform.h> +#include <linux/of.h> #include <linux/of_net.h> #include <linux/phy/phy.h> +#include <linux/platform_device.h> #include <linux/reset.h> #include <net/addrconf.h> @@ -449,39 +450,46 @@ static int lan966x_port_get_parent_id(struct net_device *dev, return 0; } -static int lan966x_port_ioctl(struct net_device *dev, struct ifreq *ifr, - int cmd) +static int lan966x_port_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) +{ + struct lan966x_port *port = netdev_priv(dev); + + if (!port->lan966x->ptp) + return -EOPNOTSUPP; + + lan966x_ptp_hwtstamp_get(port, cfg); + + return 0; +} + +static int lan966x_port_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct lan966x_port *port = netdev_priv(dev); int err; - if (cmd == SIOCSHWTSTAMP) { - err = lan966x_ptp_setup_traps(port, ifr); - if (err) - return err; - } + if (cfg->source != HWTSTAMP_SOURCE_NETDEV && + cfg->source != HWTSTAMP_SOURCE_PHYLIB) + return -EOPNOTSUPP; - if (!phy_has_hwtstamp(dev->phydev) && port->lan966x->ptp) { - switch (cmd) { - case SIOCSHWTSTAMP: - err = lan966x_ptp_hwtstamp_set(port, ifr); - if (err) - lan966x_ptp_del_traps(port); + err = lan966x_ptp_setup_traps(port, cfg); + if (err) + return err; + + if (cfg->source == HWTSTAMP_SOURCE_NETDEV) { + if (!port->lan966x->ptp) + return -EOPNOTSUPP; + err = lan966x_ptp_hwtstamp_set(port, cfg, extack); + if (err) { + lan966x_ptp_del_traps(port); return err; - case SIOCGHWTSTAMP: - return lan966x_ptp_hwtstamp_get(port, ifr); } } - if (!dev->phydev) - return -ENODEV; - - err = phy_mii_ioctl(dev->phydev, ifr, cmd); - if (err && cmd == SIOCSHWTSTAMP) - lan966x_ptp_del_traps(port); - - return err; + return 0; } static const struct net_device_ops lan966x_port_netdev_ops = { @@ -494,10 +502,12 @@ static const struct net_device_ops lan966x_port_netdev_ops = { .ndo_get_stats64 = lan966x_stats_get, .ndo_set_mac_address = lan966x_port_set_mac_address, .ndo_get_port_parent_id = lan966x_port_get_parent_id, - .ndo_eth_ioctl = lan966x_port_ioctl, + .ndo_eth_ioctl = phy_do_ioctl, .ndo_setup_tc = lan966x_tc_setup, .ndo_bpf = lan966x_xdp, .ndo_xdp_xmit = lan966x_xdp_xmit, + .ndo_hwtstamp_get = lan966x_port_hwtstamp_get, + .ndo_hwtstamp_set = lan966x_port_hwtstamp_set, }; bool lan966x_netdevice_check(const struct net_device *dev) @@ -807,6 +817,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_TC; dev->hw_features |= NETIF_F_HW_TC; + dev->priv_flags |= IFF_SEE_ALL_HWTSTAMP_REQUESTS; dev->needed_headroom = IFH_LEN_BYTES; eth_hw_addr_gen(dev, lan966x->base_mac, p + 1); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index 27f272831ea5..b538d496e8d7 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -298,7 +298,7 @@ struct lan966x_phc { struct ptp_clock *clock; struct ptp_clock_info info; struct ptp_pin_desc pins[LAN966X_PHC_PINS_NUM]; - struct hwtstamp_config hwtstamp_config; + struct kernel_hwtstamp_config hwtstamp_config; struct lan966x *lan966x; u8 index; }; @@ -578,8 +578,11 @@ void lan966x_mdb_restore_entries(struct lan966x *lan966x); int lan966x_ptp_init(struct lan966x *lan966x); void lan966x_ptp_deinit(struct lan966x *lan966x); -int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr); -int lan966x_ptp_hwtstamp_get(struct lan966x_port *port, struct ifreq *ifr); +int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack); +void lan966x_ptp_hwtstamp_get(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg); void lan966x_ptp_rxtstamp(struct lan966x *lan966x, struct sk_buff *skb, u64 src_port, u64 timestamp); int lan966x_ptp_txtstamp_request(struct lan966x_port *port, @@ -590,7 +593,8 @@ irqreturn_t lan966x_ptp_irq_handler(int irq, void *args); irqreturn_t lan966x_ptp_ext_irq_handler(int irq, void *args); u32 lan966x_ptp_get_period_ps(void); int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts); -int lan966x_ptp_setup_traps(struct lan966x_port *port, struct ifreq *ifr); +int lan966x_ptp_setup_traps(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg); int lan966x_ptp_del_traps(struct lan966x_port *port); int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c index 266a21a2d124..60bd0cff6677 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c @@ -248,29 +248,23 @@ int lan966x_ptp_del_traps(struct lan966x_port *port) return err; } -int lan966x_ptp_setup_traps(struct lan966x_port *port, struct ifreq *ifr) +int lan966x_ptp_setup_traps(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg) { - struct hwtstamp_config cfg; - - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - if (cfg.rx_filter == HWTSTAMP_FILTER_NONE) + if (cfg->rx_filter == HWTSTAMP_FILTER_NONE) return lan966x_ptp_del_traps(port); else return lan966x_ptp_add_traps(port); } -int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr) +int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct lan966x *lan966x = port->lan966x; - struct hwtstamp_config cfg; struct lan966x_phc *phc; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - switch (cfg.tx_type) { + switch (cfg->tx_type) { case HWTSTAMP_TX_ON: port->ptp_tx_cmd = IFH_REW_OP_TWO_STEP_PTP; break; @@ -284,7 +278,7 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr) return -ERANGE; } - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: port->ptp_rx_cmd = false; break; @@ -303,7 +297,7 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr) case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: case HWTSTAMP_FILTER_NTP_ALL: port->ptp_rx_cmd = true; - cfg.rx_filter = HWTSTAMP_FILTER_ALL; + cfg->rx_filter = HWTSTAMP_FILTER_ALL; break; default: return -ERANGE; @@ -312,20 +306,20 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr) /* Commit back the result & save it */ mutex_lock(&lan966x->ptp_lock); phc = &lan966x->phc[LAN966X_PHC_PORT]; - memcpy(&phc->hwtstamp_config, &cfg, sizeof(cfg)); + phc->hwtstamp_config = *cfg; mutex_unlock(&lan966x->ptp_lock); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } -int lan966x_ptp_hwtstamp_get(struct lan966x_port *port, struct ifreq *ifr) +void lan966x_ptp_hwtstamp_get(struct lan966x_port *port, + struct kernel_hwtstamp_config *cfg) { struct lan966x *lan966x = port->lan966x; struct lan966x_phc *phc; phc = &lan966x->phc[LAN966X_PHC_PORT]; - return copy_to_user(ifr->ifr_data, &phc->hwtstamp_config, - sizeof(phc->hwtstamp_config)) ? -EFAULT : 0; + *cfg = phc->hwtstamp_config; } static int lan966x_ptp_classify(struct lan966x_port *port, struct sk_buff *skb) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c index 96b3def6c474..d696cf9dbd19 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c @@ -75,7 +75,7 @@ lan966x_tc_flower_handler_control_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL); return err; @@ -172,7 +172,7 @@ lan966x_tc_flower_handler_basic_usage(struct vcap_tc_flower_parse_usage *st) } } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_BASIC); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); return err; out: NL_SET_ERR_MSG_MOD(st->fco->common.extack, "ip_proto parse error"); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h index 62c85463b634..89a9a7afa32c 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h @@ -205,7 +205,7 @@ enum sparx5_core_clockfreq { struct sparx5_phc { struct ptp_clock *clock; struct ptp_clock_info info; - struct hwtstamp_config hwtstamp_config; + struct kernel_hwtstamp_config hwtstamp_config; struct sparx5 *sparx5; u8 index; }; @@ -388,8 +388,11 @@ void sparx5_unregister_netdevs(struct sparx5 *sparx5); /* sparx5_ptp.c */ int sparx5_ptp_init(struct sparx5 *sparx5); void sparx5_ptp_deinit(struct sparx5 *sparx5); -int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr); -int sparx5_ptp_hwtstamp_get(struct sparx5_port *port, struct ifreq *ifr); +int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack); +void sparx5_ptp_hwtstamp_get(struct sparx5_port *port, + struct kernel_hwtstamp_config *cfg); void sparx5_ptp_rxtstamp(struct sparx5 *sparx5, struct sk_buff *skb, u64 timestamp); int sparx5_ptp_txtstamp_request(struct sparx5_port *port, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c index d078156581d5..705a004b324f 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -210,22 +210,31 @@ static int sparx5_get_port_parent_id(struct net_device *dev, return 0; } -static int sparx5_port_ioctl(struct net_device *dev, struct ifreq *ifr, - int cmd) +static int sparx5_port_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) { struct sparx5_port *sparx5_port = netdev_priv(dev); struct sparx5 *sparx5 = sparx5_port->sparx5; - if (!phy_has_hwtstamp(dev->phydev) && sparx5->ptp) { - switch (cmd) { - case SIOCSHWTSTAMP: - return sparx5_ptp_hwtstamp_set(sparx5_port, ifr); - case SIOCGHWTSTAMP: - return sparx5_ptp_hwtstamp_get(sparx5_port, ifr); - } - } + if (!sparx5->ptp) + return -EOPNOTSUPP; + + sparx5_ptp_hwtstamp_get(sparx5_port, cfg); + + return 0; +} + +static int sparx5_port_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + struct sparx5_port *sparx5_port = netdev_priv(dev); + struct sparx5 *sparx5 = sparx5_port->sparx5; + + if (!sparx5->ptp) + return -EOPNOTSUPP; - return phy_mii_ioctl(dev->phydev, ifr, cmd); + return sparx5_ptp_hwtstamp_set(sparx5_port, cfg, extack); } static const struct net_device_ops sparx5_port_netdev_ops = { @@ -238,8 +247,10 @@ static const struct net_device_ops sparx5_port_netdev_ops = { .ndo_validate_addr = eth_validate_addr, .ndo_get_stats64 = sparx5_get_stats64, .ndo_get_port_parent_id = sparx5_get_port_parent_id, - .ndo_eth_ioctl = sparx5_port_ioctl, + .ndo_eth_ioctl = phy_do_ioctl, .ndo_setup_tc = sparx5_port_setup_tc, + .ndo_hwtstamp_get = sparx5_port_hwtstamp_get, + .ndo_hwtstamp_set = sparx5_port_hwtstamp_set, }; bool sparx5_netdevice_check(const struct net_device *dev) diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 0edb98cef7e4..5a932460db58 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -74,10 +74,11 @@ static u64 sparx5_ptp_get_nominal_value(struct sparx5 *sparx5) return res; } -int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr) +int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct sparx5 *sparx5 = port->sparx5; - struct hwtstamp_config cfg; struct sparx5_phc *phc; /* For now don't allow to run ptp on ports that are part of a bridge, @@ -88,10 +89,7 @@ int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr) if (test_bit(port->portno, sparx5->bridge_mask)) return -EINVAL; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - switch (cfg.tx_type) { + switch (cfg->tx_type) { case HWTSTAMP_TX_ON: port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; break; @@ -105,7 +103,7 @@ int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr) return -ERANGE; } - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: break; case HWTSTAMP_FILTER_ALL: @@ -122,7 +120,7 @@ int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr) case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: case HWTSTAMP_FILTER_NTP_ALL: - cfg.rx_filter = HWTSTAMP_FILTER_ALL; + cfg->rx_filter = HWTSTAMP_FILTER_ALL; break; default: return -ERANGE; @@ -131,20 +129,20 @@ int sparx5_ptp_hwtstamp_set(struct sparx5_port *port, struct ifreq *ifr) /* Commit back the result & save it */ mutex_lock(&sparx5->ptp_lock); phc = &sparx5->phc[SPARX5_PHC_PORT]; - memcpy(&phc->hwtstamp_config, &cfg, sizeof(cfg)); + phc->hwtstamp_config = *cfg; mutex_unlock(&sparx5->ptp_lock); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } -int sparx5_ptp_hwtstamp_get(struct sparx5_port *port, struct ifreq *ifr) +void sparx5_ptp_hwtstamp_get(struct sparx5_port *port, + struct kernel_hwtstamp_config *cfg) { struct sparx5 *sparx5 = port->sparx5; struct sparx5_phc *phc; phc = &sparx5->phc[SPARX5_PHC_PORT]; - return copy_to_user(ifr->ifr_data, &phc->hwtstamp_config, - sizeof(phc->hwtstamp_config)) ? -EFAULT : 0; + *cfg = phc->hwtstamp_config; } static void sparx5_ptp_classify(struct sparx5_port *port, struct sk_buff *skb, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c index 3f87a5285a6d..906299ad8425 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c @@ -126,7 +126,7 @@ sparx5_tc_flower_handler_basic_usage(struct vcap_tc_flower_parse_usage *st) } } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_BASIC); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); return err; @@ -175,7 +175,7 @@ sparx5_tc_flower_handler_control_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL); return err; diff --git a/drivers/net/ethernet/microchip/vcap/vcap_tc.c b/drivers/net/ethernet/microchip/vcap/vcap_tc.c index 09abe7944af6..27e2dffb65e6 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_tc.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_tc.c @@ -50,7 +50,7 @@ int vcap_tc_flower_handler_ethaddr_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS); return err; @@ -86,7 +86,7 @@ int vcap_tc_flower_handler_ipv4_usage(struct vcap_tc_flower_parse_usage *st) } } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS); return err; @@ -124,7 +124,7 @@ int vcap_tc_flower_handler_ipv6_usage(struct vcap_tc_flower_parse_usage *st) goto out; } } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS); return err; out: NL_SET_ERR_MSG_MOD(st->fco->common.extack, "ipv6_addr parse error"); @@ -158,7 +158,7 @@ int vcap_tc_flower_handler_portnum_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_PORTS); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); return err; @@ -201,7 +201,7 @@ int vcap_tc_flower_handler_cvlan_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN); return 0; out: @@ -238,7 +238,7 @@ int vcap_tc_flower_handler_vlan_usage(struct vcap_tc_flower_parse_usage *st, if (mt.mask->vlan_tpid) st->tpid = be16_to_cpu(mt.key->vlan_tpid); - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_VLAN); return 0; out: @@ -313,7 +313,7 @@ int vcap_tc_flower_handler_tcp_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP); return err; @@ -376,7 +376,7 @@ int vcap_tc_flower_handler_arp_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_ARP); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ARP); return 0; @@ -401,7 +401,7 @@ int vcap_tc_flower_handler_ip_usage(struct vcap_tc_flower_parse_usage *st) goto out; } - st->used_keys |= BIT(FLOW_DISSECTOR_KEY_IP); + st->used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_IP); return err; diff --git a/drivers/net/ethernet/microchip/vcap/vcap_tc.h b/drivers/net/ethernet/microchip/vcap/vcap_tc.h index 071f892f9aa4..49b02d032906 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_tc.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_tc.h @@ -14,7 +14,7 @@ struct vcap_tc_flower_parse_usage { u16 l3_proto; u8 l4_proto; u16 tpid; - unsigned int used_keys; + unsigned long long used_keys; }; int vcap_tc_flower_handler_ethaddr_usage(struct vcap_tc_flower_parse_usage *st); diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 8f3f78b68592..3765d3389a9a 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -300,8 +300,11 @@ static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) { + /* Hardware Spec specifies that software client should set 0 for + * wqe_cnt for Receive Queues. This value is not used in Send Queues. + */ mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, - queue->id, queue->head * GDMA_WQE_BU_SIZE, 1); + queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); } void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index a499e460594b..ac2acc9aca9d 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1386,8 +1386,8 @@ static void mana_post_pkt_rxq(struct mana_rxq *rxq) recv_buf_oob = &rxq->rx_oobs[curr_index]; - err = mana_gd_post_and_ring(rxq->gdma_rq, &recv_buf_oob->wqe_req, - &recv_buf_oob->wqe_inf); + err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, + &recv_buf_oob->wqe_inf); if (WARN_ON_ONCE(err)) return; @@ -1657,6 +1657,12 @@ static void mana_poll_rx_cq(struct mana_cq *cq) mana_process_rx_cqe(rxq, cq, &comp[i]); } + if (comp_read > 0) { + struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; + + mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); + } + if (rxq->xdp_flush) xdp_do_flush(); } diff --git a/drivers/net/ethernet/mscc/ocelot_fdma.c b/drivers/net/ethernet/mscc/ocelot_fdma.c index 83a3ce0c568e..312a46832154 100644 --- a/drivers/net/ethernet/mscc/ocelot_fdma.c +++ b/drivers/net/ethernet/mscc/ocelot_fdma.c @@ -12,7 +12,6 @@ #include <linux/dmapool.h> #include <linux/dsa/ocelot.h> #include <linux/netdevice.h> -#include <linux/of_platform.h> #include <linux/skbuff.h> #include "ocelot_fdma.h" diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c index e0916afcddfb..33b438c6aec5 100644 --- a/drivers/net/ethernet/mscc/ocelot_flower.c +++ b/drivers/net/ethernet/mscc/ocelot_flower.c @@ -581,14 +581,14 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress, int ret; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_META) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_META) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { return -EOPNOTSUPP; } @@ -641,12 +641,12 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress, * then just bail out */ if ((dissector->used_keys & - (BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL))) != - (BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL))) + (BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL))) != + (BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL))) return -EOPNOTSUPP; flow_rule_match_eth_addrs(rule, &match); diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index 97e90e2869d4..151b42465348 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -10,8 +10,9 @@ #include <linux/of_net.h> #include <linux/netdevice.h> #include <linux/phylink.h> +#include <linux/of.h> #include <linux/of_mdio.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/mfd/syscon.h> #include <linux/skbuff.h> #include <net/switchdev.h> diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c index 73032173ac4e..2643c4b3ff1f 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c +++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c @@ -61,7 +61,7 @@ bool is_pre_ct_flow(struct flow_cls_offload *flow) struct flow_match_ct ct; int i; - if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { + if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { flow_rule_match_ct(rule, &ct); if (ct.key->ct_state) return false; @@ -94,7 +94,7 @@ bool is_post_ct_flow(struct flow_cls_offload *flow) struct flow_match_ct ct; int i; - if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { + if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { flow_rule_match_ct(rule, &ct); if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) return true; @@ -236,10 +236,11 @@ static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1, static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, struct nfp_fl_ct_flow_entry *entry2) { - unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys & - entry2->rule->match.dissector->used_keys; + unsigned long long ovlp_keys; bool out, is_v6 = false; u8 ip_proto = 0; + ovlp_keys = entry1->rule->match.dissector->used_keys & + entry2->rule->match.dissector->used_keys; /* Temporary buffer for mangling keys, 64 is enough to cover max * struct size of key in various fields that may be mangled. * Supported fields to mangle: @@ -257,7 +258,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, /* Check the overlapped fields one by one, the unmasked part * should not conflict with each other. */ - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) { struct flow_match_control match1, match2; flow_rule_match_control(entry1->rule, &match1); @@ -267,7 +268,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) { struct flow_match_basic match1, match2; flow_rule_match_basic(entry1->rule, &match1); @@ -289,7 +290,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, * will be do merge check when do nft and post ct merge, * so skip this ip merge check here. */ - if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) && + if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) && nfp_ct_merge_check_cannot_skip(entry1, entry2)) { struct flow_match_ipv4_addrs match1, match2; @@ -311,7 +312,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, * will be do merge check when do nft and post ct merge, * so skip this ip merge check here. */ - if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) && + if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) && nfp_ct_merge_check_cannot_skip(entry1, entry2)) { struct flow_match_ipv6_addrs match1, match2; @@ -333,7 +334,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, * will be do merge check when do nft and post ct merge, * so skip this tport merge check here. */ - if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) && + if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_PORTS)) && nfp_ct_merge_check_cannot_skip(entry1, entry2)) { enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC; struct flow_match_ports match1, match2; @@ -355,7 +356,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS)) { struct flow_match_eth_addrs match1, match2; flow_rule_match_eth_addrs(entry1->rule, &match1); @@ -371,7 +372,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN)) { struct flow_match_vlan match1, match2; flow_rule_match_vlan(entry1->rule, &match1); @@ -381,7 +382,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_MPLS)) { struct flow_match_mpls match1, match2; flow_rule_match_mpls(entry1->rule, &match1); @@ -391,7 +392,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_TCP)) { struct flow_match_tcp match1, match2; flow_rule_match_tcp(entry1->rule, &match1); @@ -401,7 +402,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IP)) { struct flow_match_ip match1, match2; flow_rule_match_ip(entry1->rule, &match1); @@ -413,7 +414,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID)) { struct flow_match_enc_keyid match1, match2; flow_rule_match_enc_keyid(entry1->rule, &match1); @@ -423,7 +424,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { struct flow_match_ipv4_addrs match1, match2; flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1); @@ -433,7 +434,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { struct flow_match_ipv6_addrs match1, match2; flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1); @@ -443,7 +444,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL)) { struct flow_match_control match1, match2; flow_rule_match_enc_control(entry1->rule, &match1); @@ -453,7 +454,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) { struct flow_match_ip match1, match2; flow_rule_match_enc_ip(entry1->rule, &match1); @@ -463,7 +464,7 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) { + if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS)) { struct flow_match_enc_opts match1, match2; flow_rule_match_enc_opts(entry1->rule, &match1); @@ -589,7 +590,7 @@ static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry, int i; ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA); - if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) { + if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) { u32 *act_lbl; act_lbl = ct_met->ct_metadata.labels; diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index 18328eb7f5c3..c153f0575b92 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -24,43 +24,43 @@ FLOW_DIS_FIRST_FRAG) #define NFP_FLOWER_WHITELIST_DISSECTOR \ - (BIT(FLOW_DISSECTOR_KEY_CONTROL) | \ - BIT(FLOW_DISSECTOR_KEY_BASIC) | \ - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_TCP) | \ - BIT(FLOW_DISSECTOR_KEY_PORTS) | \ - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_VLAN) | \ - BIT(FLOW_DISSECTOR_KEY_CVLAN) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IP) | \ - BIT(FLOW_DISSECTOR_KEY_MPLS) | \ - BIT(FLOW_DISSECTOR_KEY_CT) | \ - BIT(FLOW_DISSECTOR_KEY_META) | \ - BIT(FLOW_DISSECTOR_KEY_IP)) + (BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_MPLS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_CT) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_META) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_IP)) #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR \ - (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IP)) + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R \ - (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_V6_R \ - (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \ + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) #define NFP_FLOWER_MERGE_FIELDS \ (NFP_FLOWER_LAYER_PORT | \ @@ -1303,7 +1303,7 @@ static bool offload_pre_check(struct flow_cls_offload *flow) struct flow_dissector *dissector = rule->match.dissector; struct flow_match_ct ct; - if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { + if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { flow_rule_match_ct(rule, &ct); /* Allow special case where CT match is all 0 */ if (memchr_inv(ct.key, 0, sizeof(*ct.key))) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 6b1fb5708434..f18c791cf698 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -924,7 +924,7 @@ static void nfp_net_write_mac_addr(struct nfp_net *nn, const u8 *addr) */ static void nfp_net_clear_config_and_disable(struct nfp_net *nn) { - u32 new_ctrl, update; + u32 new_ctrl, new_ctrl_w1, update; unsigned int r; int err; @@ -937,14 +937,29 @@ static void nfp_net_clear_config_and_disable(struct nfp_net *nn) if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG) new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG; - nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0); - nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0); + if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN)) { + nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0); + nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0); + } nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); err = nfp_net_reconfig(nn, update); if (err) nn_err(nn, "Could not disable device: %d\n", err); + if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) { + new_ctrl_w1 = nn->dp.ctrl_w1; + new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_FREELIST_EN; + nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0); + nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0); + + nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1); + err = nfp_net_reconfig(nn, update); + if (err) + nn_err(nn, "Could not disable FREELIST_EN: %d\n", err); + nn->dp.ctrl_w1 = new_ctrl_w1; + } + for (r = 0; r < nn->dp.num_rx_rings; r++) { nfp_net_rx_ring_reset(&nn->dp.rx_rings[r]); if (nfp_net_has_xsk_pool_slow(&nn->dp, nn->dp.rx_rings[r].idx)) @@ -964,11 +979,12 @@ static void nfp_net_clear_config_and_disable(struct nfp_net *nn) */ static int nfp_net_set_config_and_enable(struct nfp_net *nn) { - u32 bufsz, new_ctrl, update = 0; + u32 bufsz, new_ctrl, new_ctrl_w1, update = 0; unsigned int r; int err; new_ctrl = nn->dp.ctrl; + new_ctrl_w1 = nn->dp.ctrl_w1; if (nn->dp.ctrl & NFP_NET_CFG_CTRL_RSS_ANY) { nfp_net_rss_write_key(nn); @@ -1001,16 +1017,25 @@ static int nfp_net_set_config_and_enable(struct nfp_net *nn) bufsz = nn->dp.fl_bufsz - nn->dp.rx_dma_off - NFP_NET_RX_BUF_NON_DATA; nn_writel(nn, NFP_NET_CFG_FLBUFSZ, bufsz); - /* Enable device */ - new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; + /* Enable device + * Step 1: Replace the CTRL_ENABLE by NFP_NET_CFG_CTRL_FREELIST_EN if + * FREELIST_EN exits. + */ + if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) + new_ctrl_w1 |= NFP_NET_CFG_CTRL_FREELIST_EN; + else + new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; update |= NFP_NET_CFG_UPDATE_GEN; update |= NFP_NET_CFG_UPDATE_MSIX; update |= NFP_NET_CFG_UPDATE_RING; if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG) new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; + /* Step 2: Send the configuration and write the freelist. + * - The freelist only need to be written once. + */ nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); - nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, nn->dp.ctrl_w1); + nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1); err = nfp_net_reconfig(nn, update); if (err) { nfp_net_clear_config_and_disable(nn); @@ -1018,10 +1043,25 @@ static int nfp_net_set_config_and_enable(struct nfp_net *nn) } nn->dp.ctrl = new_ctrl; + nn->dp.ctrl_w1 = new_ctrl_w1; for (r = 0; r < nn->dp.num_rx_rings; r++) nfp_net_rx_ring_fill_freelist(&nn->dp, &nn->dp.rx_rings[r]); + /* Step 3: Do the NFP_NET_CFG_CTRL_ENABLE. Send the configuration. + */ + if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) { + new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; + nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); + + err = nfp_net_reconfig(nn, update); + if (err) { + nfp_net_clear_config_and_disable(nn); + return err; + } + nn->dp.ctrl = new_ctrl; + } + return 0; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h index 669b9dccb6a9..3e63f6d6a563 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h @@ -268,6 +268,7 @@ #define NFP_NET_CFG_CTRL_PKT_TYPE (0x1 << 0) /* Pkttype offload */ #define NFP_NET_CFG_CTRL_IPSEC (0x1 << 1) /* IPsec offload */ #define NFP_NET_CFG_CTRL_MCAST_FILTER (0x1 << 2) /* Multicast Filter */ +#define NFP_NET_CFG_CTRL_FREELIST_EN (0x1 << 6) /* Freelist enable flag bit */ #define NFP_NET_CFG_CAP_WORD1 0x00a4 diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c index 0fd156286d4d..ba27bbc68f85 100644 --- a/drivers/net/ethernet/ni/nixge.c +++ b/drivers/net/ethernet/ni/nixge.c @@ -7,11 +7,10 @@ #include <linux/etherdevice.h> #include <linux/module.h> #include <linux/netdevice.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> -#include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/skbuff.h> #include <linux/phy.h> #include <linux/mii.h> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c index ab7d217b98b3..d6ce113a4210 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c @@ -213,29 +213,18 @@ out: return ret; } -static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +static void ionic_clear_pci(struct ionic *ionic) { - struct device *dev = &pdev->dev; - struct ionic *ionic; - int num_vfs; - int err; - - ionic = ionic_devlink_alloc(dev); - if (!ionic) - return -ENOMEM; - - ionic->pdev = pdev; - ionic->dev = dev; - pci_set_drvdata(pdev, ionic); - mutex_init(&ionic->dev_cmd_lock); + ionic_unmap_bars(ionic); + pci_release_regions(ionic->pdev); + pci_disable_device(ionic->pdev); +} - /* Query system for DMA addressing limitation for the device. */ - err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN)); - if (err) { - dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n", - err); - goto err_out_clear_drvdata; - } +static int ionic_setup_one(struct ionic *ionic) +{ + struct pci_dev *pdev = ionic->pdev; + struct device *dev = ionic->dev; + int err; ionic_debugfs_add_dev(ionic); @@ -249,20 +238,19 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = pci_request_regions(pdev, IONIC_DRV_NAME); if (err) { dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err); - goto err_out_pci_disable_device; + goto err_out_clear_pci; } - pcie_print_link_status(pdev); err = ionic_map_bars(ionic); if (err) - goto err_out_pci_release_regions; + goto err_out_clear_pci; /* Configure the device */ err = ionic_setup(ionic); if (err) { dev_err(dev, "Cannot setup device: %d, aborting\n", err); - goto err_out_unmap_bars; + goto err_out_clear_pci; } pci_set_master(pdev); @@ -279,24 +267,64 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_teardown; } - /* Configure the ports */ + /* Configure the port */ err = ionic_port_identify(ionic); if (err) { dev_err(dev, "Cannot identify port: %d, aborting\n", err); - goto err_out_reset; + goto err_out_teardown; } err = ionic_port_init(ionic); if (err) { dev_err(dev, "Cannot init port: %d, aborting\n", err); - goto err_out_reset; + goto err_out_teardown; + } + + return 0; + +err_out_teardown: + ionic_dev_teardown(ionic); +err_out_clear_pci: + ionic_clear_pci(ionic); +err_out_debugfs_del_dev: + ionic_debugfs_del_dev(ionic); + + return err; +} + +static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct ionic *ionic; + int num_vfs; + int err; + + ionic = ionic_devlink_alloc(dev); + if (!ionic) + return -ENOMEM; + + ionic->pdev = pdev; + ionic->dev = dev; + pci_set_drvdata(pdev, ionic); + mutex_init(&ionic->dev_cmd_lock); + + /* Query system for DMA addressing limitation for the device. */ + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN)); + if (err) { + dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n", + err); + goto err_out; } + err = ionic_setup_one(ionic); + if (err) + goto err_out; + /* Allocate and init the LIF */ err = ionic_lif_size(ionic); if (err) { dev_err(dev, "Cannot size LIF: %d, aborting\n", err); - goto err_out_port_reset; + goto err_out_pci; } err = ionic_lif_alloc(ionic); @@ -347,21 +375,10 @@ err_out_free_lifs: ionic->lif = NULL; err_out_free_irqs: ionic_bus_free_irq_vectors(ionic); -err_out_port_reset: - ionic_port_reset(ionic); -err_out_reset: - ionic_reset(ionic); -err_out_teardown: +err_out_pci: ionic_dev_teardown(ionic); -err_out_unmap_bars: - ionic_unmap_bars(ionic); -err_out_pci_release_regions: - pci_release_regions(pdev); -err_out_pci_disable_device: - pci_disable_device(pdev); -err_out_debugfs_del_dev: - ionic_debugfs_del_dev(ionic); -err_out_clear_drvdata: + ionic_clear_pci(ionic); +err_out: mutex_destroy(&ionic->dev_cmd_lock); ionic_devlink_free(ionic); @@ -386,20 +403,71 @@ static void ionic_remove(struct pci_dev *pdev) ionic_port_reset(ionic); ionic_reset(ionic); ionic_dev_teardown(ionic); - ionic_unmap_bars(ionic); - pci_release_regions(pdev); - pci_disable_device(pdev); + ionic_clear_pci(ionic); ionic_debugfs_del_dev(ionic); mutex_destroy(&ionic->dev_cmd_lock); ionic_devlink_free(ionic); } +static void ionic_reset_prepare(struct pci_dev *pdev) +{ + struct ionic *ionic = pci_get_drvdata(pdev); + struct ionic_lif *lif = ionic->lif; + + dev_dbg(ionic->dev, "%s: device stopping\n", __func__); + + del_timer_sync(&ionic->watchdog_timer); + cancel_work_sync(&lif->deferred.work); + + mutex_lock(&lif->queue_lock); + ionic_stop_queues_reconfig(lif); + ionic_txrx_free(lif); + ionic_lif_deinit(lif); + ionic_qcqs_free(lif); + mutex_unlock(&lif->queue_lock); + + ionic_dev_teardown(ionic); + ionic_clear_pci(ionic); + ionic_debugfs_del_dev(ionic); +} + +static void ionic_reset_done(struct pci_dev *pdev) +{ + struct ionic *ionic = pci_get_drvdata(pdev); + struct ionic_lif *lif = ionic->lif; + int err; + + err = ionic_setup_one(ionic); + if (err) + goto err_out; + + ionic_debugfs_add_sizes(ionic); + ionic_debugfs_add_lif(ionic->lif); + + err = ionic_restart_lif(lif); + if (err) + goto err_out; + + mod_timer(&ionic->watchdog_timer, jiffies + 1); + +err_out: + dev_dbg(ionic->dev, "%s: device recovery %s\n", + __func__, err ? "failed" : "done"); +} + +static const struct pci_error_handlers ionic_err_handler = { + /* FLR handling */ + .reset_prepare = ionic_reset_prepare, + .reset_done = ionic_reset_done, +}; + static struct pci_driver ionic_driver = { .name = IONIC_DRV_NAME, .id_table = ionic_id_table, .probe = ionic_probe, .remove = ionic_remove, .sriov_configure = ionic_sriov_configure, + .err_handler = &ionic_err_handler }; int ionic_bus_register_driver(void) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index 612b0015dc43..adc05f944c14 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -434,7 +434,7 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq) } } -static void ionic_qcqs_free(struct ionic_lif *lif) +void ionic_qcqs_free(struct ionic_lif *lif) { struct device *dev = lif->ionic->dev; struct ionic_qcq *adminqcq; @@ -1754,7 +1754,7 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa) return ionic_lif_addr_add(netdev_priv(netdev), mac); } -static void ionic_stop_queues_reconfig(struct ionic_lif *lif) +void ionic_stop_queues_reconfig(struct ionic_lif *lif) { /* Stop and clean the queues before reconfiguration */ netif_device_detach(lif->netdev); @@ -2009,7 +2009,7 @@ static void ionic_txrx_deinit(struct ionic_lif *lif) } } -static void ionic_txrx_free(struct ionic_lif *lif) +void ionic_txrx_free(struct ionic_lif *lif) { unsigned int i; @@ -3266,27 +3266,11 @@ static void ionic_lif_handle_fw_down(struct ionic_lif *lif) dev_info(ionic->dev, "FW Down: LIFs stopped\n"); } -static void ionic_lif_handle_fw_up(struct ionic_lif *lif) +int ionic_restart_lif(struct ionic_lif *lif) { struct ionic *ionic = lif->ionic; int err; - if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) - return; - - dev_info(ionic->dev, "FW Up: restarting LIFs\n"); - - ionic_init_devinfo(ionic); - err = ionic_identify(ionic); - if (err) - goto err_out; - err = ionic_port_identify(ionic); - if (err) - goto err_out; - err = ionic_port_init(ionic); - if (err) - goto err_out; - mutex_lock(&lif->queue_lock); if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state)) @@ -3322,12 +3306,8 @@ static void ionic_lif_handle_fw_up(struct ionic_lif *lif) clear_bit(IONIC_LIF_F_FW_RESET, lif->state); ionic_link_status_check_request(lif, CAN_SLEEP); netif_device_attach(lif->netdev); - dev_info(ionic->dev, "FW Up: LIFs restarted\n"); - - /* restore the hardware timestamping queues */ - ionic_lif_hwstamp_replay(lif); - return; + return 0; err_txrx_free: ionic_txrx_free(lif); @@ -3337,6 +3317,46 @@ err_qcqs_free: ionic_qcqs_free(lif); err_unlock: mutex_unlock(&lif->queue_lock); + + return err; +} + +static void ionic_lif_handle_fw_up(struct ionic_lif *lif) +{ + struct ionic *ionic = lif->ionic; + int err; + + if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) + return; + + dev_info(ionic->dev, "FW Up: restarting LIFs\n"); + + /* This is a little different from what happens at + * probe time because the LIF already exists so we + * just need to reanimate it. + */ + ionic_init_devinfo(ionic); + err = ionic_identify(ionic); + if (err) + goto err_out; + err = ionic_port_identify(ionic); + if (err) + goto err_out; + err = ionic_port_init(ionic); + if (err) + goto err_out; + + err = ionic_restart_lif(lif); + if (err) + goto err_out; + + dev_info(ionic->dev, "FW Up: LIFs restarted\n"); + + /* restore the hardware timestamping queues */ + ionic_lif_hwstamp_replay(lif); + + return; + err_out: dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err); } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h index fd2ea670e7d8..457c24195ca6 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h @@ -325,6 +325,11 @@ void ionic_lif_deinit(struct ionic_lif *lif); int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr); int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr); +void ionic_stop_queues_reconfig(struct ionic_lif *lif); +void ionic_txrx_free(struct ionic_lif *lif); +void ionic_qcqs_free(struct ionic_lif *lif); +int ionic_restart_lif(struct ionic_lif *lif); + int ionic_lif_register(struct ionic_lif *lif); void ionic_lif_unregister(struct ionic_lif *lif); int ionic_lif_identify(struct ionic *ionic, u8 lif_type, diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c index 3010833ddde3..a5ac21a0ee33 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_filter.c +++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c @@ -1827,12 +1827,12 @@ qede_parse_flow_attr(struct qede_dev *edev, __be16 proto, memset(tuple, 0, sizeof(*tuple)); if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS))) { - DP_NOTICE(edev, "Unsupported key set:0x%x\n", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS))) { + DP_NOTICE(edev, "Unsupported key set:0x%llx\n", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c index 802ef81493e0..e4bc18009d08 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c @@ -8,7 +8,9 @@ #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/acpi.h> +#include <linux/of.h> #include <linux/of_device.h> +#include <linux/of_platform.h> #include "emac.h" #include "emac-mac.h" #include "emac-sgmii.h" diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c index eaa50050aa0b..19bb16daf4e7 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac.c @@ -11,7 +11,6 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_net.h> -#include <linux/of_device.h> #include <linux/phy.h> #include <linux/platform_device.h> #include <linux/acpi.h> diff --git a/drivers/net/ethernet/qualcomm/qca_spi.c b/drivers/net/ethernet/qualcomm/qca_spi.c index 4a1b94e5a8ea..bec723028e96 100644 --- a/drivers/net/ethernet/qualcomm/qca_spi.c +++ b/drivers/net/ethernet/qualcomm/qca_spi.c @@ -35,7 +35,6 @@ #include <linux/moduleparam.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/sched.h> #include <linux/skbuff.h> diff --git a/drivers/net/ethernet/qualcomm/qca_uart.c b/drivers/net/ethernet/qualcomm/qca_uart.c index 26646cb6a20a..ace99c62d03a 100644 --- a/drivers/net/ethernet/qualcomm/qca_uart.c +++ b/drivers/net/ethernet/qualcomm/qca_uart.c @@ -32,7 +32,6 @@ #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/sched.h> #include <linux/serdev.h> diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 4d6b3b7d6abb..7df9f9f8e134 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -21,10 +21,9 @@ #include <linux/module.h> #include <linux/net_tstamp.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> #include <linux/spinlock.h> diff --git a/drivers/net/ethernet/renesas/rswitch.c b/drivers/net/ethernet/renesas/rswitch.c index 4e412ac0965a..0ba7fb75d589 100644 --- a/drivers/net/ethernet/renesas/rswitch.c +++ b/drivers/net/ethernet/renesas/rswitch.c @@ -12,11 +12,10 @@ #include <linux/module.h> #include <linux/net_tstamp.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/phy/phy.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/rtnetlink.h> #include <linux/slab.h> diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index d8ec729825be..274ea16c0a1f 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -19,8 +19,6 @@ #include <linux/mdio-bitbang.h> #include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_irq.h> #include <linux/of_net.h> #include <linux/phy.h> #include <linux/cache.h> diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index 8c019f382a7f..6dfa062feebc 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -2209,7 +2209,7 @@ static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue) /* low two bits of label are what we want for type */ BUILD_BUG_ON((EFX_TXQ_TYPE_OUTER_CSUM | EFX_TXQ_TYPE_INNER_CSUM) != 3); tx_queue->type = tx_queue->label & 3; - return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf, + return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd, (tx_queue->ptr_mask + 1) * sizeof(efx_qword_t), GFP_KERNEL); @@ -4267,8 +4267,6 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { .sriov_init = efx_ef10_sriov_init, .sriov_fini = efx_ef10_sriov_fini, .sriov_wanted = efx_ef10_sriov_wanted, - .sriov_reset = efx_ef10_sriov_reset, - .sriov_flr = efx_ef10_sriov_flr, .sriov_set_vf_mac = efx_ef10_sriov_set_vf_mac, .sriov_set_vf_vlan = efx_ef10_sriov_set_vf_vlan, .sriov_set_vf_spoofchk = efx_ef10_sriov_set_vf_spoofchk, diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index 7adde9639c8a..f3e8ed578c09 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -224,7 +224,7 @@ int efx_ef100_init_datapath_caps(struct efx_nic *efx) static int ef100_ev_probe(struct efx_channel *channel) { /* Allocate an extra descriptor for the QMDA status completion entry */ - return efx_nic_alloc_buffer(channel->efx, &channel->eventq.buf, + return efx_nic_alloc_buffer(channel->efx, &channel->eventq, (channel->eventq_mask + 2) * sizeof(efx_qword_t), GFP_KERNEL); diff --git a/drivers/net/ethernet/sfc/ef100_tx.c b/drivers/net/ethernet/sfc/ef100_tx.c index 849e5555bd12..e6b6be549581 100644 --- a/drivers/net/ethernet/sfc/ef100_tx.c +++ b/drivers/net/ethernet/sfc/ef100_tx.c @@ -23,7 +23,7 @@ int ef100_tx_probe(struct efx_tx_queue *tx_queue) { /* Allocate an extra descriptor for the QMDA status completion entry */ - return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf, + return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd, (tx_queue->ptr_mask + 2) * sizeof(efx_oword_t), GFP_KERNEL); @@ -101,8 +101,8 @@ static bool ef100_tx_can_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb) static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) { - if (likely(tx_queue->txd.buf.addr)) - return ((efx_oword_t *)tx_queue->txd.buf.addr) + index; + if (likely(tx_queue->txd.addr)) + return ((efx_oword_t *)tx_queue->txd.addr) + index; else return NULL; } diff --git a/drivers/net/ethernet/sfc/ef10_sriov.h b/drivers/net/ethernet/sfc/ef10_sriov.h index 3c703ca878b0..be419c9c5dec 100644 --- a/drivers/net/ethernet/sfc/ef10_sriov.h +++ b/drivers/net/ethernet/sfc/ef10_sriov.h @@ -35,9 +35,7 @@ static inline bool efx_ef10_sriov_wanted(struct efx_nic *efx) int efx_ef10_sriov_configure(struct efx_nic *efx, int num_vfs); int efx_ef10_sriov_init(struct efx_nic *efx); -static inline void efx_ef10_sriov_reset(struct efx_nic *efx) {} void efx_ef10_sriov_fini(struct efx_nic *efx); -static inline void efx_ef10_sriov_flr(struct efx_nic *efx, unsigned vf_i) {} int efx_ef10_sriov_set_vf_mac(struct efx_nic *efx, int vf, const u8 *mac); diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index d670a319b379..19f4b4d0b851 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -605,7 +605,6 @@ static const struct net_device_ops efx_netdev_ops = { #endif .ndo_get_phys_port_id = efx_get_phys_port_id, .ndo_get_phys_port_name = efx_get_phys_port_name, - .ndo_setup_tc = efx_setup_tc, #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = efx_filter_rfs, #endif diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h index 4239c7ece123..48d3623735ba 100644 --- a/drivers/net/ethernet/sfc/efx.h +++ b/drivers/net/ethernet/sfc/efx.h @@ -30,8 +30,6 @@ static inline netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct tx_queue, skb); } void efx_xmit_done_single(struct efx_tx_queue *tx_queue); -int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, - void *type_data); extern unsigned int efx_piobuf_size; /* RX */ diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 41b33a75333c..8d2d7ea2ebef 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -713,9 +713,6 @@ int efx_probe_channels(struct efx_nic *efx) struct efx_channel *channel; int rc; - /* Restart special buffer allocation */ - efx->next_buffer_table = 0; - /* Probe channels in reverse, so that any 'extra' channels * use the start of the buffer table. This allows the traffic * channels to be resized without moving them or wasting the @@ -849,36 +846,14 @@ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel, *ptp_channel = efx_ptp_channel(efx); struct efx_ptp_data *ptp_data = efx->ptp_data; - unsigned int i, next_buffer_table = 0; u32 old_rxq_entries, old_txq_entries; + unsigned int i; int rc, rc2; rc = efx_check_disabled(efx); if (rc) return rc; - /* Not all channels should be reallocated. We must avoid - * reallocating their buffer table entries. - */ - efx_for_each_channel(channel, efx) { - struct efx_rx_queue *rx_queue; - struct efx_tx_queue *tx_queue; - - if (channel->type->copy) - continue; - next_buffer_table = max(next_buffer_table, - channel->eventq.index + - channel->eventq.entries); - efx_for_each_channel_rx_queue(rx_queue, channel) - next_buffer_table = max(next_buffer_table, - rx_queue->rxd.index + - rx_queue->rxd.entries); - efx_for_each_channel_tx_queue(tx_queue, channel) - next_buffer_table = max(next_buffer_table, - tx_queue->txd.index + - tx_queue->txd.entries); - } - efx_device_detach_sync(efx); efx_stop_all(efx); efx_soft_disable_interrupts(efx); @@ -904,9 +879,6 @@ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) for (i = 0; i < efx->n_channels; i++) swap(efx->channel[i], other_channel[i]); - /* Restart buffer table allocation */ - efx->next_buffer_table = next_buffer_table; - for (i = 0; i < efx->n_channels; i++) { channel = efx->channel[i]; if (!channel->type->copy) diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index 361687de308d..175bd9cdfdac 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -35,11 +35,6 @@ MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); /* This is the time (in jiffies) between invocations of the hardware * monitor. - * On Falcon-based NICs, this will: - * - Check the on-board hardware monitor; - * - Poll the link state and reconfigure the hardware as necessary. - * On Siena-based NICs for power systems with EEH support, this will give EEH a - * chance to start. */ static unsigned int efx_monitor_interval = 1 * HZ; @@ -785,8 +780,6 @@ int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) mutex_unlock(&efx->rss_lock); efx->type->filter_table_restore(efx); up_write(&efx->filter_sem); - if (efx->type->sriov_reset) - efx->type->sriov_reset(efx); mutex_unlock(&efx->mac_lock); diff --git a/drivers/net/ethernet/sfc/farch_regs.h b/drivers/net/ethernet/sfc/farch_regs.h deleted file mode 100644 index d138be423e63..000000000000 --- a/drivers/net/ethernet/sfc/farch_regs.h +++ /dev/null @@ -1,2929 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/**************************************************************************** - * Driver for Solarflare network controllers and boards - * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2012 Solarflare Communications Inc. - */ - -#ifndef EFX_FARCH_REGS_H -#define EFX_FARCH_REGS_H - -/* - * Falcon hardware architecture definitions have a name prefix following - * the format: - * - * F<type>_<min-rev><max-rev>_ - * - * The following <type> strings are used: - * - * MMIO register MC register Host memory structure - * ------------------------------------------------------------- - * Address R MCR - * Bitfield RF MCRF SF - * Enumerator FE MCFE SE - * - * <min-rev> is the first revision to which the definition applies: - * - * A: Falcon A1 (SFC4000AB) - * B: Falcon B0 (SFC4000BA) - * C: Siena A0 (SFL9021AA) - * - * If the definition has been changed or removed in later revisions - * then <max-rev> is the last revision to which the definition applies; - * otherwise it is "Z". - */ - -/************************************************************************** - * - * Falcon/Siena registers and descriptors - * - ************************************************************************** - */ - -/* ADR_REGION_REG: Address region register */ -#define FR_AZ_ADR_REGION 0x00000000 -#define FRF_AZ_ADR_REGION3_LBN 96 -#define FRF_AZ_ADR_REGION3_WIDTH 18 -#define FRF_AZ_ADR_REGION2_LBN 64 -#define FRF_AZ_ADR_REGION2_WIDTH 18 -#define FRF_AZ_ADR_REGION1_LBN 32 -#define FRF_AZ_ADR_REGION1_WIDTH 18 -#define FRF_AZ_ADR_REGION0_LBN 0 -#define FRF_AZ_ADR_REGION0_WIDTH 18 - -/* INT_EN_REG_KER: Kernel driver Interrupt enable register */ -#define FR_AZ_INT_EN_KER 0x00000010 -#define FRF_AZ_KER_INT_LEVE_SEL_LBN 8 -#define FRF_AZ_KER_INT_LEVE_SEL_WIDTH 6 -#define FRF_AZ_KER_INT_CHAR_LBN 4 -#define FRF_AZ_KER_INT_CHAR_WIDTH 1 -#define FRF_AZ_KER_INT_KER_LBN 3 -#define FRF_AZ_KER_INT_KER_WIDTH 1 -#define FRF_AZ_DRV_INT_EN_KER_LBN 0 -#define FRF_AZ_DRV_INT_EN_KER_WIDTH 1 - -/* INT_EN_REG_CHAR: Char Driver interrupt enable register */ -#define FR_BZ_INT_EN_CHAR 0x00000020 -#define FRF_BZ_CHAR_INT_LEVE_SEL_LBN 8 -#define FRF_BZ_CHAR_INT_LEVE_SEL_WIDTH 6 -#define FRF_BZ_CHAR_INT_CHAR_LBN 4 -#define FRF_BZ_CHAR_INT_CHAR_WIDTH 1 -#define FRF_BZ_CHAR_INT_KER_LBN 3 -#define FRF_BZ_CHAR_INT_KER_WIDTH 1 -#define FRF_BZ_DRV_INT_EN_CHAR_LBN 0 -#define FRF_BZ_DRV_INT_EN_CHAR_WIDTH 1 - -/* INT_ADR_REG_KER: Interrupt host address for Kernel driver */ -#define FR_AZ_INT_ADR_KER 0x00000030 -#define FRF_AZ_NORM_INT_VEC_DIS_KER_LBN 64 -#define FRF_AZ_NORM_INT_VEC_DIS_KER_WIDTH 1 -#define FRF_AZ_INT_ADR_KER_LBN 0 -#define FRF_AZ_INT_ADR_KER_WIDTH 64 - -/* INT_ADR_REG_CHAR: Interrupt host address for Char driver */ -#define FR_BZ_INT_ADR_CHAR 0x00000040 -#define FRF_BZ_NORM_INT_VEC_DIS_CHAR_LBN 64 -#define FRF_BZ_NORM_INT_VEC_DIS_CHAR_WIDTH 1 -#define FRF_BZ_INT_ADR_CHAR_LBN 0 -#define FRF_BZ_INT_ADR_CHAR_WIDTH 64 - -/* INT_ACK_KER: Kernel interrupt acknowledge register */ -#define FR_AA_INT_ACK_KER 0x00000050 -#define FRF_AA_INT_ACK_KER_FIELD_LBN 0 -#define FRF_AA_INT_ACK_KER_FIELD_WIDTH 32 - -/* INT_ISR0_REG: Function 0 Interrupt Acknowledge Status register */ -#define FR_BZ_INT_ISR0 0x00000090 -#define FRF_BZ_INT_ISR_REG_LBN 0 -#define FRF_BZ_INT_ISR_REG_WIDTH 64 - -/* HW_INIT_REG: Hardware initialization register */ -#define FR_AZ_HW_INIT 0x000000c0 -#define FRF_BB_BDMRD_CPLF_FULL_LBN 124 -#define FRF_BB_BDMRD_CPLF_FULL_WIDTH 1 -#define FRF_BB_PCIE_CPL_TIMEOUT_CTRL_LBN 121 -#define FRF_BB_PCIE_CPL_TIMEOUT_CTRL_WIDTH 3 -#define FRF_CZ_TX_MRG_TAGS_LBN 120 -#define FRF_CZ_TX_MRG_TAGS_WIDTH 1 -#define FRF_AB_TRGT_MASK_ALL_LBN 100 -#define FRF_AB_TRGT_MASK_ALL_WIDTH 1 -#define FRF_AZ_DOORBELL_DROP_LBN 92 -#define FRF_AZ_DOORBELL_DROP_WIDTH 8 -#define FRF_AB_TX_RREQ_MASK_EN_LBN 76 -#define FRF_AB_TX_RREQ_MASK_EN_WIDTH 1 -#define FRF_AB_PE_EIDLE_DIS_LBN 75 -#define FRF_AB_PE_EIDLE_DIS_WIDTH 1 -#define FRF_AA_FC_BLOCKING_EN_LBN 45 -#define FRF_AA_FC_BLOCKING_EN_WIDTH 1 -#define FRF_BZ_B2B_REQ_EN_LBN 45 -#define FRF_BZ_B2B_REQ_EN_WIDTH 1 -#define FRF_AA_B2B_REQ_EN_LBN 44 -#define FRF_AA_B2B_REQ_EN_WIDTH 1 -#define FRF_BB_FC_BLOCKING_EN_LBN 44 -#define FRF_BB_FC_BLOCKING_EN_WIDTH 1 -#define FRF_AZ_POST_WR_MASK_LBN 40 -#define FRF_AZ_POST_WR_MASK_WIDTH 4 -#define FRF_AZ_TLP_TC_LBN 34 -#define FRF_AZ_TLP_TC_WIDTH 3 -#define FRF_AZ_TLP_ATTR_LBN 32 -#define FRF_AZ_TLP_ATTR_WIDTH 2 -#define FRF_AB_INTB_VEC_LBN 24 -#define FRF_AB_INTB_VEC_WIDTH 5 -#define FRF_AB_INTA_VEC_LBN 16 -#define FRF_AB_INTA_VEC_WIDTH 5 -#define FRF_AZ_WD_TIMER_LBN 8 -#define FRF_AZ_WD_TIMER_WIDTH 8 -#define FRF_AZ_US_DISABLE_LBN 5 -#define FRF_AZ_US_DISABLE_WIDTH 1 -#define FRF_AZ_TLP_EP_LBN 4 -#define FRF_AZ_TLP_EP_WIDTH 1 -#define FRF_AZ_ATTR_SEL_LBN 3 -#define FRF_AZ_ATTR_SEL_WIDTH 1 -#define FRF_AZ_TD_SEL_LBN 1 -#define FRF_AZ_TD_SEL_WIDTH 1 -#define FRF_AZ_TLP_TD_LBN 0 -#define FRF_AZ_TLP_TD_WIDTH 1 - -/* EE_SPI_HCMD_REG: SPI host command register */ -#define FR_AB_EE_SPI_HCMD 0x00000100 -#define FRF_AB_EE_SPI_HCMD_CMD_EN_LBN 31 -#define FRF_AB_EE_SPI_HCMD_CMD_EN_WIDTH 1 -#define FRF_AB_EE_WR_TIMER_ACTIVE_LBN 28 -#define FRF_AB_EE_WR_TIMER_ACTIVE_WIDTH 1 -#define FRF_AB_EE_SPI_HCMD_SF_SEL_LBN 24 -#define FRF_AB_EE_SPI_HCMD_SF_SEL_WIDTH 1 -#define FRF_AB_EE_SPI_HCMD_DABCNT_LBN 16 -#define FRF_AB_EE_SPI_HCMD_DABCNT_WIDTH 5 -#define FRF_AB_EE_SPI_HCMD_READ_LBN 15 -#define FRF_AB_EE_SPI_HCMD_READ_WIDTH 1 -#define FRF_AB_EE_SPI_HCMD_DUBCNT_LBN 12 -#define FRF_AB_EE_SPI_HCMD_DUBCNT_WIDTH 2 -#define FRF_AB_EE_SPI_HCMD_ADBCNT_LBN 8 -#define FRF_AB_EE_SPI_HCMD_ADBCNT_WIDTH 2 -#define FRF_AB_EE_SPI_HCMD_ENC_LBN 0 -#define FRF_AB_EE_SPI_HCMD_ENC_WIDTH 8 - -/* USR_EV_CFG: User Level Event Configuration register */ -#define FR_CZ_USR_EV_CFG 0x00000100 -#define FRF_CZ_USREV_DIS_LBN 16 -#define FRF_CZ_USREV_DIS_WIDTH 1 -#define FRF_CZ_DFLT_EVQ_LBN 0 -#define FRF_CZ_DFLT_EVQ_WIDTH 10 - -/* EE_SPI_HADR_REG: SPI host address register */ -#define FR_AB_EE_SPI_HADR 0x00000110 -#define FRF_AB_EE_SPI_HADR_DUBYTE_LBN 24 -#define FRF_AB_EE_SPI_HADR_DUBYTE_WIDTH 8 -#define FRF_AB_EE_SPI_HADR_ADR_LBN 0 -#define FRF_AB_EE_SPI_HADR_ADR_WIDTH 24 - -/* EE_SPI_HDATA_REG: SPI host data register */ -#define FR_AB_EE_SPI_HDATA 0x00000120 -#define FRF_AB_EE_SPI_HDATA3_LBN 96 -#define FRF_AB_EE_SPI_HDATA3_WIDTH 32 -#define FRF_AB_EE_SPI_HDATA2_LBN 64 -#define FRF_AB_EE_SPI_HDATA2_WIDTH 32 -#define FRF_AB_EE_SPI_HDATA1_LBN 32 -#define FRF_AB_EE_SPI_HDATA1_WIDTH 32 -#define FRF_AB_EE_SPI_HDATA0_LBN 0 -#define FRF_AB_EE_SPI_HDATA0_WIDTH 32 - -/* EE_BASE_PAGE_REG: Expansion ROM base mirror register */ -#define FR_AB_EE_BASE_PAGE 0x00000130 -#define FRF_AB_EE_EXPROM_MASK_LBN 16 -#define FRF_AB_EE_EXPROM_MASK_WIDTH 13 -#define FRF_AB_EE_EXP_ROM_WINDOW_BASE_LBN 0 -#define FRF_AB_EE_EXP_ROM_WINDOW_BASE_WIDTH 13 - -/* EE_VPD_CFG0_REG: SPI/VPD configuration register 0 */ -#define FR_AB_EE_VPD_CFG0 0x00000140 -#define FRF_AB_EE_SF_FASTRD_EN_LBN 127 -#define FRF_AB_EE_SF_FASTRD_EN_WIDTH 1 -#define FRF_AB_EE_SF_CLOCK_DIV_LBN 120 -#define FRF_AB_EE_SF_CLOCK_DIV_WIDTH 7 -#define FRF_AB_EE_VPD_WIP_POLL_LBN 119 -#define FRF_AB_EE_VPD_WIP_POLL_WIDTH 1 -#define FRF_AB_EE_EE_CLOCK_DIV_LBN 112 -#define FRF_AB_EE_EE_CLOCK_DIV_WIDTH 7 -#define FRF_AB_EE_EE_WR_TMR_VALUE_LBN 96 -#define FRF_AB_EE_EE_WR_TMR_VALUE_WIDTH 16 -#define FRF_AB_EE_VPDW_LENGTH_LBN 80 -#define FRF_AB_EE_VPDW_LENGTH_WIDTH 15 -#define FRF_AB_EE_VPDW_BASE_LBN 64 -#define FRF_AB_EE_VPDW_BASE_WIDTH 15 -#define FRF_AB_EE_VPD_WR_CMD_EN_LBN 56 -#define FRF_AB_EE_VPD_WR_CMD_EN_WIDTH 8 -#define FRF_AB_EE_VPD_BASE_LBN 32 -#define FRF_AB_EE_VPD_BASE_WIDTH 24 -#define FRF_AB_EE_VPD_LENGTH_LBN 16 -#define FRF_AB_EE_VPD_LENGTH_WIDTH 15 -#define FRF_AB_EE_VPD_AD_SIZE_LBN 8 -#define FRF_AB_EE_VPD_AD_SIZE_WIDTH 5 -#define FRF_AB_EE_VPD_ACCESS_ON_LBN 5 -#define FRF_AB_EE_VPD_ACCESS_ON_WIDTH 1 -#define FRF_AB_EE_VPD_ACCESS_BLOCK_LBN 4 -#define FRF_AB_EE_VPD_ACCESS_BLOCK_WIDTH 1 -#define FRF_AB_EE_VPD_DEV_SF_SEL_LBN 2 -#define FRF_AB_EE_VPD_DEV_SF_SEL_WIDTH 1 -#define FRF_AB_EE_VPD_EN_AD9_MODE_LBN 1 -#define FRF_AB_EE_VPD_EN_AD9_MODE_WIDTH 1 -#define FRF_AB_EE_VPD_EN_LBN 0 -#define FRF_AB_EE_VPD_EN_WIDTH 1 - -/* EE_VPD_SW_CNTL_REG: VPD access SW control register */ -#define FR_AB_EE_VPD_SW_CNTL 0x00000150 -#define FRF_AB_EE_VPD_CYCLE_PENDING_LBN 31 -#define FRF_AB_EE_VPD_CYCLE_PENDING_WIDTH 1 -#define FRF_AB_EE_VPD_CYC_WRITE_LBN 28 -#define FRF_AB_EE_VPD_CYC_WRITE_WIDTH 1 -#define FRF_AB_EE_VPD_CYC_ADR_LBN 0 -#define FRF_AB_EE_VPD_CYC_ADR_WIDTH 15 - -/* EE_VPD_SW_DATA_REG: VPD access SW data register */ -#define FR_AB_EE_VPD_SW_DATA 0x00000160 -#define FRF_AB_EE_VPD_CYC_DAT_LBN 0 -#define FRF_AB_EE_VPD_CYC_DAT_WIDTH 32 - -/* PBMX_DBG_IADDR_REG: Capture Module address register */ -#define FR_CZ_PBMX_DBG_IADDR 0x000001f0 -#define FRF_CZ_PBMX_DBG_IADDR_LBN 0 -#define FRF_CZ_PBMX_DBG_IADDR_WIDTH 32 - -/* PCIE_CORE_INDIRECT_REG: Indirect Access to PCIE Core registers */ -#define FR_BB_PCIE_CORE_INDIRECT 0x000001f0 -#define FRF_BB_PCIE_CORE_TARGET_DATA_LBN 32 -#define FRF_BB_PCIE_CORE_TARGET_DATA_WIDTH 32 -#define FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_LBN 15 -#define FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_WIDTH 1 -#define FRF_BB_PCIE_CORE_TARGET_REG_ADRS_LBN 0 -#define FRF_BB_PCIE_CORE_TARGET_REG_ADRS_WIDTH 12 - -/* PBMX_DBG_IDATA_REG: Capture Module data register */ -#define FR_CZ_PBMX_DBG_IDATA 0x000001f8 -#define FRF_CZ_PBMX_DBG_IDATA_LBN 0 -#define FRF_CZ_PBMX_DBG_IDATA_WIDTH 64 - -/* NIC_STAT_REG: NIC status register */ -#define FR_AB_NIC_STAT 0x00000200 -#define FRF_BB_AER_DIS_LBN 34 -#define FRF_BB_AER_DIS_WIDTH 1 -#define FRF_BB_EE_STRAP_EN_LBN 31 -#define FRF_BB_EE_STRAP_EN_WIDTH 1 -#define FRF_BB_EE_STRAP_LBN 24 -#define FRF_BB_EE_STRAP_WIDTH 4 -#define FRF_BB_REVISION_ID_LBN 17 -#define FRF_BB_REVISION_ID_WIDTH 7 -#define FRF_AB_ONCHIP_SRAM_LBN 16 -#define FRF_AB_ONCHIP_SRAM_WIDTH 1 -#define FRF_AB_SF_PRST_LBN 9 -#define FRF_AB_SF_PRST_WIDTH 1 -#define FRF_AB_EE_PRST_LBN 8 -#define FRF_AB_EE_PRST_WIDTH 1 -#define FRF_AB_ATE_MODE_LBN 3 -#define FRF_AB_ATE_MODE_WIDTH 1 -#define FRF_AB_STRAP_PINS_LBN 0 -#define FRF_AB_STRAP_PINS_WIDTH 3 - -/* GPIO_CTL_REG: GPIO control register */ -#define FR_AB_GPIO_CTL 0x00000210 -#define FRF_AB_GPIO_OUT3_LBN 112 -#define FRF_AB_GPIO_OUT3_WIDTH 16 -#define FRF_AB_GPIO_IN3_LBN 104 -#define FRF_AB_GPIO_IN3_WIDTH 8 -#define FRF_AB_GPIO_PWRUP_VALUE3_LBN 96 -#define FRF_AB_GPIO_PWRUP_VALUE3_WIDTH 8 -#define FRF_AB_GPIO_OUT2_LBN 80 -#define FRF_AB_GPIO_OUT2_WIDTH 16 -#define FRF_AB_GPIO_IN2_LBN 72 -#define FRF_AB_GPIO_IN2_WIDTH 8 -#define FRF_AB_GPIO_PWRUP_VALUE2_LBN 64 -#define FRF_AB_GPIO_PWRUP_VALUE2_WIDTH 8 -#define FRF_AB_GPIO15_OEN_LBN 63 -#define FRF_AB_GPIO15_OEN_WIDTH 1 -#define FRF_AB_GPIO14_OEN_LBN 62 -#define FRF_AB_GPIO14_OEN_WIDTH 1 -#define FRF_AB_GPIO13_OEN_LBN 61 -#define FRF_AB_GPIO13_OEN_WIDTH 1 -#define FRF_AB_GPIO12_OEN_LBN 60 -#define FRF_AB_GPIO12_OEN_WIDTH 1 -#define FRF_AB_GPIO11_OEN_LBN 59 -#define FRF_AB_GPIO11_OEN_WIDTH 1 -#define FRF_AB_GPIO10_OEN_LBN 58 -#define FRF_AB_GPIO10_OEN_WIDTH 1 -#define FRF_AB_GPIO9_OEN_LBN 57 -#define FRF_AB_GPIO9_OEN_WIDTH 1 -#define FRF_AB_GPIO8_OEN_LBN 56 -#define FRF_AB_GPIO8_OEN_WIDTH 1 -#define FRF_AB_GPIO15_OUT_LBN 55 -#define FRF_AB_GPIO15_OUT_WIDTH 1 -#define FRF_AB_GPIO14_OUT_LBN 54 -#define FRF_AB_GPIO14_OUT_WIDTH 1 -#define FRF_AB_GPIO13_OUT_LBN 53 -#define FRF_AB_GPIO13_OUT_WIDTH 1 -#define FRF_AB_GPIO12_OUT_LBN 52 -#define FRF_AB_GPIO12_OUT_WIDTH 1 -#define FRF_AB_GPIO11_OUT_LBN 51 -#define FRF_AB_GPIO11_OUT_WIDTH 1 -#define FRF_AB_GPIO10_OUT_LBN 50 -#define FRF_AB_GPIO10_OUT_WIDTH 1 -#define FRF_AB_GPIO9_OUT_LBN 49 -#define FRF_AB_GPIO9_OUT_WIDTH 1 -#define FRF_AB_GPIO8_OUT_LBN 48 -#define FRF_AB_GPIO8_OUT_WIDTH 1 -#define FRF_AB_GPIO15_IN_LBN 47 -#define FRF_AB_GPIO15_IN_WIDTH 1 -#define FRF_AB_GPIO14_IN_LBN 46 -#define FRF_AB_GPIO14_IN_WIDTH 1 -#define FRF_AB_GPIO13_IN_LBN 45 -#define FRF_AB_GPIO13_IN_WIDTH 1 -#define FRF_AB_GPIO12_IN_LBN 44 -#define FRF_AB_GPIO12_IN_WIDTH 1 -#define FRF_AB_GPIO11_IN_LBN 43 -#define FRF_AB_GPIO11_IN_WIDTH 1 -#define FRF_AB_GPIO10_IN_LBN 42 -#define FRF_AB_GPIO10_IN_WIDTH 1 -#define FRF_AB_GPIO9_IN_LBN 41 -#define FRF_AB_GPIO9_IN_WIDTH 1 -#define FRF_AB_GPIO8_IN_LBN 40 -#define FRF_AB_GPIO8_IN_WIDTH 1 -#define FRF_AB_GPIO15_PWRUP_VALUE_LBN 39 -#define FRF_AB_GPIO15_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO14_PWRUP_VALUE_LBN 38 -#define FRF_AB_GPIO14_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO13_PWRUP_VALUE_LBN 37 -#define FRF_AB_GPIO13_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO12_PWRUP_VALUE_LBN 36 -#define FRF_AB_GPIO12_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO11_PWRUP_VALUE_LBN 35 -#define FRF_AB_GPIO11_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO10_PWRUP_VALUE_LBN 34 -#define FRF_AB_GPIO10_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO9_PWRUP_VALUE_LBN 33 -#define FRF_AB_GPIO9_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO8_PWRUP_VALUE_LBN 32 -#define FRF_AB_GPIO8_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_CLK156_OUT_EN_LBN 31 -#define FRF_AB_CLK156_OUT_EN_WIDTH 1 -#define FRF_AB_USE_NIC_CLK_LBN 30 -#define FRF_AB_USE_NIC_CLK_WIDTH 1 -#define FRF_AB_GPIO5_OEN_LBN 29 -#define FRF_AB_GPIO5_OEN_WIDTH 1 -#define FRF_AB_GPIO4_OEN_LBN 28 -#define FRF_AB_GPIO4_OEN_WIDTH 1 -#define FRF_AB_GPIO3_OEN_LBN 27 -#define FRF_AB_GPIO3_OEN_WIDTH 1 -#define FRF_AB_GPIO2_OEN_LBN 26 -#define FRF_AB_GPIO2_OEN_WIDTH 1 -#define FRF_AB_GPIO1_OEN_LBN 25 -#define FRF_AB_GPIO1_OEN_WIDTH 1 -#define FRF_AB_GPIO0_OEN_LBN 24 -#define FRF_AB_GPIO0_OEN_WIDTH 1 -#define FRF_AB_GPIO7_OUT_LBN 23 -#define FRF_AB_GPIO7_OUT_WIDTH 1 -#define FRF_AB_GPIO6_OUT_LBN 22 -#define FRF_AB_GPIO6_OUT_WIDTH 1 -#define FRF_AB_GPIO5_OUT_LBN 21 -#define FRF_AB_GPIO5_OUT_WIDTH 1 -#define FRF_AB_GPIO4_OUT_LBN 20 -#define FRF_AB_GPIO4_OUT_WIDTH 1 -#define FRF_AB_GPIO3_OUT_LBN 19 -#define FRF_AB_GPIO3_OUT_WIDTH 1 -#define FRF_AB_GPIO2_OUT_LBN 18 -#define FRF_AB_GPIO2_OUT_WIDTH 1 -#define FRF_AB_GPIO1_OUT_LBN 17 -#define FRF_AB_GPIO1_OUT_WIDTH 1 -#define FRF_AB_GPIO0_OUT_LBN 16 -#define FRF_AB_GPIO0_OUT_WIDTH 1 -#define FRF_AB_GPIO7_IN_LBN 15 -#define FRF_AB_GPIO7_IN_WIDTH 1 -#define FRF_AB_GPIO6_IN_LBN 14 -#define FRF_AB_GPIO6_IN_WIDTH 1 -#define FRF_AB_GPIO5_IN_LBN 13 -#define FRF_AB_GPIO5_IN_WIDTH 1 -#define FRF_AB_GPIO4_IN_LBN 12 -#define FRF_AB_GPIO4_IN_WIDTH 1 -#define FRF_AB_GPIO3_IN_LBN 11 -#define FRF_AB_GPIO3_IN_WIDTH 1 -#define FRF_AB_GPIO2_IN_LBN 10 -#define FRF_AB_GPIO2_IN_WIDTH 1 -#define FRF_AB_GPIO1_IN_LBN 9 -#define FRF_AB_GPIO1_IN_WIDTH 1 -#define FRF_AB_GPIO0_IN_LBN 8 -#define FRF_AB_GPIO0_IN_WIDTH 1 -#define FRF_AB_GPIO7_PWRUP_VALUE_LBN 7 -#define FRF_AB_GPIO7_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO6_PWRUP_VALUE_LBN 6 -#define FRF_AB_GPIO6_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO5_PWRUP_VALUE_LBN 5 -#define FRF_AB_GPIO5_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO4_PWRUP_VALUE_LBN 4 -#define FRF_AB_GPIO4_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO3_PWRUP_VALUE_LBN 3 -#define FRF_AB_GPIO3_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO2_PWRUP_VALUE_LBN 2 -#define FRF_AB_GPIO2_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO1_PWRUP_VALUE_LBN 1 -#define FRF_AB_GPIO1_PWRUP_VALUE_WIDTH 1 -#define FRF_AB_GPIO0_PWRUP_VALUE_LBN 0 -#define FRF_AB_GPIO0_PWRUP_VALUE_WIDTH 1 - -/* GLB_CTL_REG: Global control register */ -#define FR_AB_GLB_CTL 0x00000220 -#define FRF_AB_EXT_PHY_RST_CTL_LBN 63 -#define FRF_AB_EXT_PHY_RST_CTL_WIDTH 1 -#define FRF_AB_XAUI_SD_RST_CTL_LBN 62 -#define FRF_AB_XAUI_SD_RST_CTL_WIDTH 1 -#define FRF_AB_PCIE_SD_RST_CTL_LBN 61 -#define FRF_AB_PCIE_SD_RST_CTL_WIDTH 1 -#define FRF_AA_PCIX_RST_CTL_LBN 60 -#define FRF_AA_PCIX_RST_CTL_WIDTH 1 -#define FRF_BB_BIU_RST_CTL_LBN 60 -#define FRF_BB_BIU_RST_CTL_WIDTH 1 -#define FRF_AB_PCIE_STKY_RST_CTL_LBN 59 -#define FRF_AB_PCIE_STKY_RST_CTL_WIDTH 1 -#define FRF_AB_PCIE_NSTKY_RST_CTL_LBN 58 -#define FRF_AB_PCIE_NSTKY_RST_CTL_WIDTH 1 -#define FRF_AB_PCIE_CORE_RST_CTL_LBN 57 -#define FRF_AB_PCIE_CORE_RST_CTL_WIDTH 1 -#define FRF_AB_XGRX_RST_CTL_LBN 56 -#define FRF_AB_XGRX_RST_CTL_WIDTH 1 -#define FRF_AB_XGTX_RST_CTL_LBN 55 -#define FRF_AB_XGTX_RST_CTL_WIDTH 1 -#define FRF_AB_EM_RST_CTL_LBN 54 -#define FRF_AB_EM_RST_CTL_WIDTH 1 -#define FRF_AB_EV_RST_CTL_LBN 53 -#define FRF_AB_EV_RST_CTL_WIDTH 1 -#define FRF_AB_SR_RST_CTL_LBN 52 -#define FRF_AB_SR_RST_CTL_WIDTH 1 -#define FRF_AB_RX_RST_CTL_LBN 51 -#define FRF_AB_RX_RST_CTL_WIDTH 1 -#define FRF_AB_TX_RST_CTL_LBN 50 -#define FRF_AB_TX_RST_CTL_WIDTH 1 -#define FRF_AB_EE_RST_CTL_LBN 49 -#define FRF_AB_EE_RST_CTL_WIDTH 1 -#define FRF_AB_CS_RST_CTL_LBN 48 -#define FRF_AB_CS_RST_CTL_WIDTH 1 -#define FRF_AB_HOT_RST_CTL_LBN 40 -#define FRF_AB_HOT_RST_CTL_WIDTH 2 -#define FRF_AB_RST_EXT_PHY_LBN 31 -#define FRF_AB_RST_EXT_PHY_WIDTH 1 -#define FRF_AB_RST_XAUI_SD_LBN 30 -#define FRF_AB_RST_XAUI_SD_WIDTH 1 -#define FRF_AB_RST_PCIE_SD_LBN 29 -#define FRF_AB_RST_PCIE_SD_WIDTH 1 -#define FRF_AA_RST_PCIX_LBN 28 -#define FRF_AA_RST_PCIX_WIDTH 1 -#define FRF_BB_RST_BIU_LBN 28 -#define FRF_BB_RST_BIU_WIDTH 1 -#define FRF_AB_RST_PCIE_STKY_LBN 27 -#define FRF_AB_RST_PCIE_STKY_WIDTH 1 -#define FRF_AB_RST_PCIE_NSTKY_LBN 26 -#define FRF_AB_RST_PCIE_NSTKY_WIDTH 1 -#define FRF_AB_RST_PCIE_CORE_LBN 25 -#define FRF_AB_RST_PCIE_CORE_WIDTH 1 -#define FRF_AB_RST_XGRX_LBN 24 -#define FRF_AB_RST_XGRX_WIDTH 1 -#define FRF_AB_RST_XGTX_LBN 23 -#define FRF_AB_RST_XGTX_WIDTH 1 -#define FRF_AB_RST_EM_LBN 22 -#define FRF_AB_RST_EM_WIDTH 1 -#define FRF_AB_RST_EV_LBN 21 -#define FRF_AB_RST_EV_WIDTH 1 -#define FRF_AB_RST_SR_LBN 20 -#define FRF_AB_RST_SR_WIDTH 1 -#define FRF_AB_RST_RX_LBN 19 -#define FRF_AB_RST_RX_WIDTH 1 -#define FRF_AB_RST_TX_LBN 18 -#define FRF_AB_RST_TX_WIDTH 1 -#define FRF_AB_RST_SF_LBN 17 -#define FRF_AB_RST_SF_WIDTH 1 -#define FRF_AB_RST_CS_LBN 16 -#define FRF_AB_RST_CS_WIDTH 1 -#define FRF_AB_INT_RST_DUR_LBN 4 -#define FRF_AB_INT_RST_DUR_WIDTH 3 -#define FRF_AB_EXT_PHY_RST_DUR_LBN 1 -#define FRF_AB_EXT_PHY_RST_DUR_WIDTH 3 -#define FFE_AB_EXT_PHY_RST_DUR_10240US 7 -#define FFE_AB_EXT_PHY_RST_DUR_5120US 6 -#define FFE_AB_EXT_PHY_RST_DUR_2560US 5 -#define FFE_AB_EXT_PHY_RST_DUR_1280US 4 -#define FFE_AB_EXT_PHY_RST_DUR_640US 3 -#define FFE_AB_EXT_PHY_RST_DUR_320US 2 -#define FFE_AB_EXT_PHY_RST_DUR_160US 1 -#define FFE_AB_EXT_PHY_RST_DUR_80US 0 -#define FRF_AB_SWRST_LBN 0 -#define FRF_AB_SWRST_WIDTH 1 - -/* FATAL_INTR_REG_KER: Fatal interrupt register for Kernel */ -#define FR_AZ_FATAL_INTR_KER 0x00000230 -#define FRF_CZ_SRAM_PERR_INT_P_KER_EN_LBN 44 -#define FRF_CZ_SRAM_PERR_INT_P_KER_EN_WIDTH 1 -#define FRF_AB_PCI_BUSERR_INT_KER_EN_LBN 43 -#define FRF_AB_PCI_BUSERR_INT_KER_EN_WIDTH 1 -#define FRF_CZ_MBU_PERR_INT_KER_EN_LBN 43 -#define FRF_CZ_MBU_PERR_INT_KER_EN_WIDTH 1 -#define FRF_AZ_SRAM_OOB_INT_KER_EN_LBN 42 -#define FRF_AZ_SRAM_OOB_INT_KER_EN_WIDTH 1 -#define FRF_AZ_BUFID_OOB_INT_KER_EN_LBN 41 -#define FRF_AZ_BUFID_OOB_INT_KER_EN_WIDTH 1 -#define FRF_AZ_MEM_PERR_INT_KER_EN_LBN 40 -#define FRF_AZ_MEM_PERR_INT_KER_EN_WIDTH 1 -#define FRF_AZ_RBUF_OWN_INT_KER_EN_LBN 39 -#define FRF_AZ_RBUF_OWN_INT_KER_EN_WIDTH 1 -#define FRF_AZ_TBUF_OWN_INT_KER_EN_LBN 38 -#define FRF_AZ_TBUF_OWN_INT_KER_EN_WIDTH 1 -#define FRF_AZ_RDESCQ_OWN_INT_KER_EN_LBN 37 -#define FRF_AZ_RDESCQ_OWN_INT_KER_EN_WIDTH 1 -#define FRF_AZ_TDESCQ_OWN_INT_KER_EN_LBN 36 -#define FRF_AZ_TDESCQ_OWN_INT_KER_EN_WIDTH 1 -#define FRF_AZ_EVQ_OWN_INT_KER_EN_LBN 35 -#define FRF_AZ_EVQ_OWN_INT_KER_EN_WIDTH 1 -#define FRF_AZ_EVF_OFLO_INT_KER_EN_LBN 34 -#define FRF_AZ_EVF_OFLO_INT_KER_EN_WIDTH 1 -#define FRF_AZ_ILL_ADR_INT_KER_EN_LBN 33 -#define FRF_AZ_ILL_ADR_INT_KER_EN_WIDTH 1 -#define FRF_AZ_SRM_PERR_INT_KER_EN_LBN 32 -#define FRF_AZ_SRM_PERR_INT_KER_EN_WIDTH 1 -#define FRF_CZ_SRAM_PERR_INT_P_KER_LBN 12 -#define FRF_CZ_SRAM_PERR_INT_P_KER_WIDTH 1 -#define FRF_AB_PCI_BUSERR_INT_KER_LBN 11 -#define FRF_AB_PCI_BUSERR_INT_KER_WIDTH 1 -#define FRF_CZ_MBU_PERR_INT_KER_LBN 11 -#define FRF_CZ_MBU_PERR_INT_KER_WIDTH 1 -#define FRF_AZ_SRAM_OOB_INT_KER_LBN 10 -#define FRF_AZ_SRAM_OOB_INT_KER_WIDTH 1 -#define FRF_AZ_BUFID_DC_OOB_INT_KER_LBN 9 -#define FRF_AZ_BUFID_DC_OOB_INT_KER_WIDTH 1 -#define FRF_AZ_MEM_PERR_INT_KER_LBN 8 -#define FRF_AZ_MEM_PERR_INT_KER_WIDTH 1 -#define FRF_AZ_RBUF_OWN_INT_KER_LBN 7 -#define FRF_AZ_RBUF_OWN_INT_KER_WIDTH 1 -#define FRF_AZ_TBUF_OWN_INT_KER_LBN 6 -#define FRF_AZ_TBUF_OWN_INT_KER_WIDTH 1 -#define FRF_AZ_RDESCQ_OWN_INT_KER_LBN 5 -#define FRF_AZ_RDESCQ_OWN_INT_KER_WIDTH 1 -#define FRF_AZ_TDESCQ_OWN_INT_KER_LBN 4 -#define FRF_AZ_TDESCQ_OWN_INT_KER_WIDTH 1 -#define FRF_AZ_EVQ_OWN_INT_KER_LBN 3 -#define FRF_AZ_EVQ_OWN_INT_KER_WIDTH 1 -#define FRF_AZ_EVF_OFLO_INT_KER_LBN 2 -#define FRF_AZ_EVF_OFLO_INT_KER_WIDTH 1 -#define FRF_AZ_ILL_ADR_INT_KER_LBN 1 -#define FRF_AZ_ILL_ADR_INT_KER_WIDTH 1 -#define FRF_AZ_SRM_PERR_INT_KER_LBN 0 -#define FRF_AZ_SRM_PERR_INT_KER_WIDTH 1 - -/* FATAL_INTR_REG_CHAR: Fatal interrupt register for Char */ -#define FR_BZ_FATAL_INTR_CHAR 0x00000240 -#define FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_LBN 44 -#define FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_WIDTH 1 -#define FRF_BB_PCI_BUSERR_INT_CHAR_EN_LBN 43 -#define FRF_BB_PCI_BUSERR_INT_CHAR_EN_WIDTH 1 -#define FRF_CZ_MBU_PERR_INT_CHAR_EN_LBN 43 -#define FRF_CZ_MBU_PERR_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_SRAM_OOB_INT_CHAR_EN_LBN 42 -#define FRF_BZ_SRAM_OOB_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_BUFID_OOB_INT_CHAR_EN_LBN 41 -#define FRF_BZ_BUFID_OOB_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_MEM_PERR_INT_CHAR_EN_LBN 40 -#define FRF_BZ_MEM_PERR_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_RBUF_OWN_INT_CHAR_EN_LBN 39 -#define FRF_BZ_RBUF_OWN_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_TBUF_OWN_INT_CHAR_EN_LBN 38 -#define FRF_BZ_TBUF_OWN_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_RDESCQ_OWN_INT_CHAR_EN_LBN 37 -#define FRF_BZ_RDESCQ_OWN_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_TDESCQ_OWN_INT_CHAR_EN_LBN 36 -#define FRF_BZ_TDESCQ_OWN_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_EVQ_OWN_INT_CHAR_EN_LBN 35 -#define FRF_BZ_EVQ_OWN_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_EVF_OFLO_INT_CHAR_EN_LBN 34 -#define FRF_BZ_EVF_OFLO_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_ILL_ADR_INT_CHAR_EN_LBN 33 -#define FRF_BZ_ILL_ADR_INT_CHAR_EN_WIDTH 1 -#define FRF_BZ_SRM_PERR_INT_CHAR_EN_LBN 32 -#define FRF_BZ_SRM_PERR_INT_CHAR_EN_WIDTH 1 -#define FRF_CZ_SRAM_PERR_INT_P_CHAR_LBN 12 -#define FRF_CZ_SRAM_PERR_INT_P_CHAR_WIDTH 1 -#define FRF_BB_PCI_BUSERR_INT_CHAR_LBN 11 -#define FRF_BB_PCI_BUSERR_INT_CHAR_WIDTH 1 -#define FRF_CZ_MBU_PERR_INT_CHAR_LBN 11 -#define FRF_CZ_MBU_PERR_INT_CHAR_WIDTH 1 -#define FRF_BZ_SRAM_OOB_INT_CHAR_LBN 10 -#define FRF_BZ_SRAM_OOB_INT_CHAR_WIDTH 1 -#define FRF_BZ_BUFID_DC_OOB_INT_CHAR_LBN 9 -#define FRF_BZ_BUFID_DC_OOB_INT_CHAR_WIDTH 1 -#define FRF_BZ_MEM_PERR_INT_CHAR_LBN 8 -#define FRF_BZ_MEM_PERR_INT_CHAR_WIDTH 1 -#define FRF_BZ_RBUF_OWN_INT_CHAR_LBN 7 -#define FRF_BZ_RBUF_OWN_INT_CHAR_WIDTH 1 -#define FRF_BZ_TBUF_OWN_INT_CHAR_LBN 6 -#define FRF_BZ_TBUF_OWN_INT_CHAR_WIDTH 1 -#define FRF_BZ_RDESCQ_OWN_INT_CHAR_LBN 5 -#define FRF_BZ_RDESCQ_OWN_INT_CHAR_WIDTH 1 -#define FRF_BZ_TDESCQ_OWN_INT_CHAR_LBN 4 -#define FRF_BZ_TDESCQ_OWN_INT_CHAR_WIDTH 1 -#define FRF_BZ_EVQ_OWN_INT_CHAR_LBN 3 -#define FRF_BZ_EVQ_OWN_INT_CHAR_WIDTH 1 -#define FRF_BZ_EVF_OFLO_INT_CHAR_LBN 2 -#define FRF_BZ_EVF_OFLO_INT_CHAR_WIDTH 1 -#define FRF_BZ_ILL_ADR_INT_CHAR_LBN 1 -#define FRF_BZ_ILL_ADR_INT_CHAR_WIDTH 1 -#define FRF_BZ_SRM_PERR_INT_CHAR_LBN 0 -#define FRF_BZ_SRM_PERR_INT_CHAR_WIDTH 1 - -/* DP_CTRL_REG: Datapath control register */ -#define FR_BZ_DP_CTRL 0x00000250 -#define FRF_BZ_FLS_EVQ_ID_LBN 0 -#define FRF_BZ_FLS_EVQ_ID_WIDTH 12 - -/* MEM_STAT_REG: Memory status register */ -#define FR_AZ_MEM_STAT 0x00000260 -#define FRF_AB_MEM_PERR_VEC_LBN 53 -#define FRF_AB_MEM_PERR_VEC_WIDTH 38 -#define FRF_AB_MBIST_CORR_LBN 38 -#define FRF_AB_MBIST_CORR_WIDTH 15 -#define FRF_AB_MBIST_ERR_LBN 0 -#define FRF_AB_MBIST_ERR_WIDTH 40 -#define FRF_CZ_MEM_PERR_VEC_LBN 0 -#define FRF_CZ_MEM_PERR_VEC_WIDTH 35 - -/* CS_DEBUG_REG: Debug register */ -#define FR_AZ_CS_DEBUG 0x00000270 -#define FRF_AB_GLB_DEBUG2_SEL_LBN 50 -#define FRF_AB_GLB_DEBUG2_SEL_WIDTH 3 -#define FRF_AB_DEBUG_BLK_SEL2_LBN 47 -#define FRF_AB_DEBUG_BLK_SEL2_WIDTH 3 -#define FRF_AB_DEBUG_BLK_SEL1_LBN 44 -#define FRF_AB_DEBUG_BLK_SEL1_WIDTH 3 -#define FRF_AB_DEBUG_BLK_SEL0_LBN 41 -#define FRF_AB_DEBUG_BLK_SEL0_WIDTH 3 -#define FRF_CZ_CS_PORT_NUM_LBN 40 -#define FRF_CZ_CS_PORT_NUM_WIDTH 2 -#define FRF_AB_MISC_DEBUG_ADDR_LBN 36 -#define FRF_AB_MISC_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_SERDES_DEBUG_ADDR_LBN 31 -#define FRF_AB_SERDES_DEBUG_ADDR_WIDTH 5 -#define FRF_CZ_CS_PORT_FPE_LBN 1 -#define FRF_CZ_CS_PORT_FPE_WIDTH 35 -#define FRF_AB_EM_DEBUG_ADDR_LBN 26 -#define FRF_AB_EM_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_SR_DEBUG_ADDR_LBN 21 -#define FRF_AB_SR_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_EV_DEBUG_ADDR_LBN 16 -#define FRF_AB_EV_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_RX_DEBUG_ADDR_LBN 11 -#define FRF_AB_RX_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_TX_DEBUG_ADDR_LBN 6 -#define FRF_AB_TX_DEBUG_ADDR_WIDTH 5 -#define FRF_AB_CS_BIU_DEBUG_ADDR_LBN 1 -#define FRF_AB_CS_BIU_DEBUG_ADDR_WIDTH 5 -#define FRF_AZ_CS_DEBUG_EN_LBN 0 -#define FRF_AZ_CS_DEBUG_EN_WIDTH 1 - -/* DRIVER_REG: Driver scratch register [0-7] */ -#define FR_AZ_DRIVER 0x00000280 -#define FR_AZ_DRIVER_STEP 16 -#define FR_AZ_DRIVER_ROWS 8 -#define FRF_AZ_DRIVER_DW0_LBN 0 -#define FRF_AZ_DRIVER_DW0_WIDTH 32 - -/* ALTERA_BUILD_REG: Altera build register */ -#define FR_AZ_ALTERA_BUILD 0x00000300 -#define FRF_AZ_ALTERA_BUILD_VER_LBN 0 -#define FRF_AZ_ALTERA_BUILD_VER_WIDTH 32 - -/* CSR_SPARE_REG: Spare register */ -#define FR_AZ_CSR_SPARE 0x00000310 -#define FRF_AB_MEM_PERR_EN_LBN 64 -#define FRF_AB_MEM_PERR_EN_WIDTH 38 -#define FRF_CZ_MEM_PERR_EN_LBN 64 -#define FRF_CZ_MEM_PERR_EN_WIDTH 35 -#define FRF_AB_MEM_PERR_EN_TX_DATA_LBN 72 -#define FRF_AB_MEM_PERR_EN_TX_DATA_WIDTH 2 -#define FRF_AZ_CSR_SPARE_BITS_LBN 0 -#define FRF_AZ_CSR_SPARE_BITS_WIDTH 32 - -/* PCIE_SD_CTL0123_REG: PCIE SerDes control register 0 to 3 */ -#define FR_AB_PCIE_SD_CTL0123 0x00000320 -#define FRF_AB_PCIE_TESTSIG_H_LBN 96 -#define FRF_AB_PCIE_TESTSIG_H_WIDTH 19 -#define FRF_AB_PCIE_TESTSIG_L_LBN 64 -#define FRF_AB_PCIE_TESTSIG_L_WIDTH 19 -#define FRF_AB_PCIE_OFFSET_LBN 56 -#define FRF_AB_PCIE_OFFSET_WIDTH 8 -#define FRF_AB_PCIE_OFFSETEN_H_LBN 55 -#define FRF_AB_PCIE_OFFSETEN_H_WIDTH 1 -#define FRF_AB_PCIE_OFFSETEN_L_LBN 54 -#define FRF_AB_PCIE_OFFSETEN_L_WIDTH 1 -#define FRF_AB_PCIE_HIVMODE_H_LBN 53 -#define FRF_AB_PCIE_HIVMODE_H_WIDTH 1 -#define FRF_AB_PCIE_HIVMODE_L_LBN 52 -#define FRF_AB_PCIE_HIVMODE_L_WIDTH 1 -#define FRF_AB_PCIE_PARRESET_H_LBN 51 -#define FRF_AB_PCIE_PARRESET_H_WIDTH 1 -#define FRF_AB_PCIE_PARRESET_L_LBN 50 -#define FRF_AB_PCIE_PARRESET_L_WIDTH 1 -#define FRF_AB_PCIE_LPBKWDRV_H_LBN 49 -#define FRF_AB_PCIE_LPBKWDRV_H_WIDTH 1 -#define FRF_AB_PCIE_LPBKWDRV_L_LBN 48 -#define FRF_AB_PCIE_LPBKWDRV_L_WIDTH 1 -#define FRF_AB_PCIE_LPBK_LBN 40 -#define FRF_AB_PCIE_LPBK_WIDTH 8 -#define FRF_AB_PCIE_PARLPBK_LBN 32 -#define FRF_AB_PCIE_PARLPBK_WIDTH 8 -#define FRF_AB_PCIE_RXTERMADJ_H_LBN 30 -#define FRF_AB_PCIE_RXTERMADJ_H_WIDTH 2 -#define FRF_AB_PCIE_RXTERMADJ_L_LBN 28 -#define FRF_AB_PCIE_RXTERMADJ_L_WIDTH 2 -#define FFE_AB_PCIE_RXTERMADJ_MIN15PCNT 3 -#define FFE_AB_PCIE_RXTERMADJ_PL10PCNT 2 -#define FFE_AB_PCIE_RXTERMADJ_MIN17PCNT 1 -#define FFE_AB_PCIE_RXTERMADJ_NOMNL 0 -#define FRF_AB_PCIE_TXTERMADJ_H_LBN 26 -#define FRF_AB_PCIE_TXTERMADJ_H_WIDTH 2 -#define FRF_AB_PCIE_TXTERMADJ_L_LBN 24 -#define FRF_AB_PCIE_TXTERMADJ_L_WIDTH 2 -#define FFE_AB_PCIE_TXTERMADJ_MIN15PCNT 3 -#define FFE_AB_PCIE_TXTERMADJ_PL10PCNT 2 -#define FFE_AB_PCIE_TXTERMADJ_MIN17PCNT 1 -#define FFE_AB_PCIE_TXTERMADJ_NOMNL 0 -#define FRF_AB_PCIE_RXEQCTL_H_LBN 18 -#define FRF_AB_PCIE_RXEQCTL_H_WIDTH 2 -#define FRF_AB_PCIE_RXEQCTL_L_LBN 16 -#define FRF_AB_PCIE_RXEQCTL_L_WIDTH 2 -#define FFE_AB_PCIE_RXEQCTL_OFF_ALT 3 -#define FFE_AB_PCIE_RXEQCTL_OFF 2 -#define FFE_AB_PCIE_RXEQCTL_MIN 1 -#define FFE_AB_PCIE_RXEQCTL_MAX 0 -#define FRF_AB_PCIE_HIDRV_LBN 8 -#define FRF_AB_PCIE_HIDRV_WIDTH 8 -#define FRF_AB_PCIE_LODRV_LBN 0 -#define FRF_AB_PCIE_LODRV_WIDTH 8 - -/* PCIE_SD_CTL45_REG: PCIE SerDes control register 4 and 5 */ -#define FR_AB_PCIE_SD_CTL45 0x00000330 -#define FRF_AB_PCIE_DTX7_LBN 60 -#define FRF_AB_PCIE_DTX7_WIDTH 4 -#define FRF_AB_PCIE_DTX6_LBN 56 -#define FRF_AB_PCIE_DTX6_WIDTH 4 -#define FRF_AB_PCIE_DTX5_LBN 52 -#define FRF_AB_PCIE_DTX5_WIDTH 4 -#define FRF_AB_PCIE_DTX4_LBN 48 -#define FRF_AB_PCIE_DTX4_WIDTH 4 -#define FRF_AB_PCIE_DTX3_LBN 44 -#define FRF_AB_PCIE_DTX3_WIDTH 4 -#define FRF_AB_PCIE_DTX2_LBN 40 -#define FRF_AB_PCIE_DTX2_WIDTH 4 -#define FRF_AB_PCIE_DTX1_LBN 36 -#define FRF_AB_PCIE_DTX1_WIDTH 4 -#define FRF_AB_PCIE_DTX0_LBN 32 -#define FRF_AB_PCIE_DTX0_WIDTH 4 -#define FRF_AB_PCIE_DEQ7_LBN 28 -#define FRF_AB_PCIE_DEQ7_WIDTH 4 -#define FRF_AB_PCIE_DEQ6_LBN 24 -#define FRF_AB_PCIE_DEQ6_WIDTH 4 -#define FRF_AB_PCIE_DEQ5_LBN 20 -#define FRF_AB_PCIE_DEQ5_WIDTH 4 -#define FRF_AB_PCIE_DEQ4_LBN 16 -#define FRF_AB_PCIE_DEQ4_WIDTH 4 -#define FRF_AB_PCIE_DEQ3_LBN 12 -#define FRF_AB_PCIE_DEQ3_WIDTH 4 -#define FRF_AB_PCIE_DEQ2_LBN 8 -#define FRF_AB_PCIE_DEQ2_WIDTH 4 -#define FRF_AB_PCIE_DEQ1_LBN 4 -#define FRF_AB_PCIE_DEQ1_WIDTH 4 -#define FRF_AB_PCIE_DEQ0_LBN 0 -#define FRF_AB_PCIE_DEQ0_WIDTH 4 - -/* PCIE_PCS_CTL_STAT_REG: PCIE PCS control and status register */ -#define FR_AB_PCIE_PCS_CTL_STAT 0x00000340 -#define FRF_AB_PCIE_PRBSERRCOUNT0_H_LBN 52 -#define FRF_AB_PCIE_PRBSERRCOUNT0_H_WIDTH 4 -#define FRF_AB_PCIE_PRBSERRCOUNT0_L_LBN 48 -#define FRF_AB_PCIE_PRBSERRCOUNT0_L_WIDTH 4 -#define FRF_AB_PCIE_PRBSERR_LBN 40 -#define FRF_AB_PCIE_PRBSERR_WIDTH 8 -#define FRF_AB_PCIE_PRBSERRH0_LBN 32 -#define FRF_AB_PCIE_PRBSERRH0_WIDTH 8 -#define FRF_AB_PCIE_FASTINIT_H_LBN 15 -#define FRF_AB_PCIE_FASTINIT_H_WIDTH 1 -#define FRF_AB_PCIE_FASTINIT_L_LBN 14 -#define FRF_AB_PCIE_FASTINIT_L_WIDTH 1 -#define FRF_AB_PCIE_CTCDISABLE_H_LBN 13 -#define FRF_AB_PCIE_CTCDISABLE_H_WIDTH 1 -#define FRF_AB_PCIE_CTCDISABLE_L_LBN 12 -#define FRF_AB_PCIE_CTCDISABLE_L_WIDTH 1 -#define FRF_AB_PCIE_PRBSSYNC_H_LBN 11 -#define FRF_AB_PCIE_PRBSSYNC_H_WIDTH 1 -#define FRF_AB_PCIE_PRBSSYNC_L_LBN 10 -#define FRF_AB_PCIE_PRBSSYNC_L_WIDTH 1 -#define FRF_AB_PCIE_PRBSERRACK_H_LBN 9 -#define FRF_AB_PCIE_PRBSERRACK_H_WIDTH 1 -#define FRF_AB_PCIE_PRBSERRACK_L_LBN 8 -#define FRF_AB_PCIE_PRBSERRACK_L_WIDTH 1 -#define FRF_AB_PCIE_PRBSSEL_LBN 0 -#define FRF_AB_PCIE_PRBSSEL_WIDTH 8 - -/* DEBUG_DATA_OUT_REG: Live Debug and Debug 2 out ports */ -#define FR_BB_DEBUG_DATA_OUT 0x00000350 -#define FRF_BB_DEBUG2_PORT_LBN 25 -#define FRF_BB_DEBUG2_PORT_WIDTH 15 -#define FRF_BB_DEBUG1_PORT_LBN 0 -#define FRF_BB_DEBUG1_PORT_WIDTH 25 - -/* EVQ_RPTR_REGP0: Event queue read pointer register */ -#define FR_BZ_EVQ_RPTR_P0 0x00000400 -#define FR_BZ_EVQ_RPTR_P0_STEP 8192 -#define FR_BZ_EVQ_RPTR_P0_ROWS 1024 -/* EVQ_RPTR_REG_KER: Event queue read pointer register */ -#define FR_AA_EVQ_RPTR_KER 0x00011b00 -#define FR_AA_EVQ_RPTR_KER_STEP 4 -#define FR_AA_EVQ_RPTR_KER_ROWS 4 -/* EVQ_RPTR_REG: Event queue read pointer register */ -#define FR_BZ_EVQ_RPTR 0x00fa0000 -#define FR_BZ_EVQ_RPTR_STEP 16 -#define FR_BB_EVQ_RPTR_ROWS 4096 -#define FR_CZ_EVQ_RPTR_ROWS 1024 -/* EVQ_RPTR_REGP123: Event queue read pointer register */ -#define FR_BB_EVQ_RPTR_P123 0x01000400 -#define FR_BB_EVQ_RPTR_P123_STEP 8192 -#define FR_BB_EVQ_RPTR_P123_ROWS 3072 -#define FRF_AZ_EVQ_RPTR_VLD_LBN 15 -#define FRF_AZ_EVQ_RPTR_VLD_WIDTH 1 -#define FRF_AZ_EVQ_RPTR_LBN 0 -#define FRF_AZ_EVQ_RPTR_WIDTH 15 - -/* TIMER_COMMAND_REGP0: Timer Command Registers */ -#define FR_BZ_TIMER_COMMAND_P0 0x00000420 -#define FR_BZ_TIMER_COMMAND_P0_STEP 8192 -#define FR_BZ_TIMER_COMMAND_P0_ROWS 1024 -/* TIMER_COMMAND_REG_KER: Timer Command Registers */ -#define FR_AA_TIMER_COMMAND_KER 0x00000420 -#define FR_AA_TIMER_COMMAND_KER_STEP 8192 -#define FR_AA_TIMER_COMMAND_KER_ROWS 4 -/* TIMER_COMMAND_REGP123: Timer Command Registers */ -#define FR_BB_TIMER_COMMAND_P123 0x01000420 -#define FR_BB_TIMER_COMMAND_P123_STEP 8192 -#define FR_BB_TIMER_COMMAND_P123_ROWS 3072 -#define FRF_CZ_TC_TIMER_MODE_LBN 14 -#define FRF_CZ_TC_TIMER_MODE_WIDTH 2 -#define FRF_AB_TC_TIMER_MODE_LBN 12 -#define FRF_AB_TC_TIMER_MODE_WIDTH 2 -#define FRF_CZ_TC_TIMER_VAL_LBN 0 -#define FRF_CZ_TC_TIMER_VAL_WIDTH 14 -#define FRF_AB_TC_TIMER_VAL_LBN 0 -#define FRF_AB_TC_TIMER_VAL_WIDTH 12 - -/* DRV_EV_REG: Driver generated event register */ -#define FR_AZ_DRV_EV 0x00000440 -#define FRF_AZ_DRV_EV_QID_LBN 64 -#define FRF_AZ_DRV_EV_QID_WIDTH 12 -#define FRF_AZ_DRV_EV_DATA_LBN 0 -#define FRF_AZ_DRV_EV_DATA_WIDTH 64 - -/* EVQ_CTL_REG: Event queue control register */ -#define FR_AZ_EVQ_CTL 0x00000450 -#define FRF_CZ_RX_EVQ_WAKEUP_MASK_LBN 15 -#define FRF_CZ_RX_EVQ_WAKEUP_MASK_WIDTH 10 -#define FRF_BB_RX_EVQ_WAKEUP_MASK_LBN 15 -#define FRF_BB_RX_EVQ_WAKEUP_MASK_WIDTH 6 -#define FRF_AZ_EVQ_OWNERR_CTL_LBN 14 -#define FRF_AZ_EVQ_OWNERR_CTL_WIDTH 1 -#define FRF_AZ_EVQ_FIFO_AF_TH_LBN 7 -#define FRF_AZ_EVQ_FIFO_AF_TH_WIDTH 7 -#define FRF_AZ_EVQ_FIFO_NOTAF_TH_LBN 0 -#define FRF_AZ_EVQ_FIFO_NOTAF_TH_WIDTH 7 - -/* EVQ_CNT1_REG: Event counter 1 register */ -#define FR_AZ_EVQ_CNT1 0x00000460 -#define FRF_AZ_EVQ_CNT_PRE_FIFO_LBN 120 -#define FRF_AZ_EVQ_CNT_PRE_FIFO_WIDTH 7 -#define FRF_AZ_EVQ_CNT_TOBIU_LBN 100 -#define FRF_AZ_EVQ_CNT_TOBIU_WIDTH 20 -#define FRF_AZ_EVQ_TX_REQ_CNT_LBN 80 -#define FRF_AZ_EVQ_TX_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_RX_REQ_CNT_LBN 60 -#define FRF_AZ_EVQ_RX_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_EM_REQ_CNT_LBN 40 -#define FRF_AZ_EVQ_EM_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_CSR_REQ_CNT_LBN 20 -#define FRF_AZ_EVQ_CSR_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_ERR_REQ_CNT_LBN 0 -#define FRF_AZ_EVQ_ERR_REQ_CNT_WIDTH 20 - -/* EVQ_CNT2_REG: Event counter 2 register */ -#define FR_AZ_EVQ_CNT2 0x00000470 -#define FRF_AZ_EVQ_UPD_REQ_CNT_LBN 104 -#define FRF_AZ_EVQ_UPD_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_CLR_REQ_CNT_LBN 84 -#define FRF_AZ_EVQ_CLR_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_RDY_CNT_LBN 80 -#define FRF_AZ_EVQ_RDY_CNT_WIDTH 4 -#define FRF_AZ_EVQ_WU_REQ_CNT_LBN 60 -#define FRF_AZ_EVQ_WU_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_WET_REQ_CNT_LBN 40 -#define FRF_AZ_EVQ_WET_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_INIT_REQ_CNT_LBN 20 -#define FRF_AZ_EVQ_INIT_REQ_CNT_WIDTH 20 -#define FRF_AZ_EVQ_TM_REQ_CNT_LBN 0 -#define FRF_AZ_EVQ_TM_REQ_CNT_WIDTH 20 - -/* USR_EV_REG: Event mailbox register */ -#define FR_CZ_USR_EV 0x00000540 -#define FR_CZ_USR_EV_STEP 8192 -#define FR_CZ_USR_EV_ROWS 1024 -#define FRF_CZ_USR_EV_DATA_LBN 0 -#define FRF_CZ_USR_EV_DATA_WIDTH 32 - -/* BUF_TBL_CFG_REG: Buffer table configuration register */ -#define FR_AZ_BUF_TBL_CFG 0x00000600 -#define FRF_AZ_BUF_TBL_MODE_LBN 3 -#define FRF_AZ_BUF_TBL_MODE_WIDTH 1 - -/* SRM_RX_DC_CFG_REG: SRAM receive descriptor cache configuration register */ -#define FR_AZ_SRM_RX_DC_CFG 0x00000610 -#define FRF_AZ_SRM_CLK_TMP_EN_LBN 21 -#define FRF_AZ_SRM_CLK_TMP_EN_WIDTH 1 -#define FRF_AZ_SRM_RX_DC_BASE_ADR_LBN 0 -#define FRF_AZ_SRM_RX_DC_BASE_ADR_WIDTH 21 - -/* SRM_TX_DC_CFG_REG: SRAM transmit descriptor cache configuration register */ -#define FR_AZ_SRM_TX_DC_CFG 0x00000620 -#define FRF_AZ_SRM_TX_DC_BASE_ADR_LBN 0 -#define FRF_AZ_SRM_TX_DC_BASE_ADR_WIDTH 21 - -/* SRM_CFG_REG: SRAM configuration register */ -#define FR_AZ_SRM_CFG 0x00000630 -#define FRF_AZ_SRM_OOB_ADR_INTEN_LBN 5 -#define FRF_AZ_SRM_OOB_ADR_INTEN_WIDTH 1 -#define FRF_AZ_SRM_OOB_BUF_INTEN_LBN 4 -#define FRF_AZ_SRM_OOB_BUF_INTEN_WIDTH 1 -#define FRF_AZ_SRM_INIT_EN_LBN 3 -#define FRF_AZ_SRM_INIT_EN_WIDTH 1 -#define FRF_AZ_SRM_NUM_BANK_LBN 2 -#define FRF_AZ_SRM_NUM_BANK_WIDTH 1 -#define FRF_AZ_SRM_BANK_SIZE_LBN 0 -#define FRF_AZ_SRM_BANK_SIZE_WIDTH 2 - -/* BUF_TBL_UPD_REG: Buffer table update register */ -#define FR_AZ_BUF_TBL_UPD 0x00000650 -#define FRF_AZ_BUF_UPD_CMD_LBN 63 -#define FRF_AZ_BUF_UPD_CMD_WIDTH 1 -#define FRF_AZ_BUF_CLR_CMD_LBN 62 -#define FRF_AZ_BUF_CLR_CMD_WIDTH 1 -#define FRF_AZ_BUF_CLR_END_ID_LBN 32 -#define FRF_AZ_BUF_CLR_END_ID_WIDTH 20 -#define FRF_AZ_BUF_CLR_START_ID_LBN 0 -#define FRF_AZ_BUF_CLR_START_ID_WIDTH 20 - -/* SRM_UPD_EVQ_REG: Buffer table update register */ -#define FR_AZ_SRM_UPD_EVQ 0x00000660 -#define FRF_AZ_SRM_UPD_EVQ_ID_LBN 0 -#define FRF_AZ_SRM_UPD_EVQ_ID_WIDTH 12 - -/* SRAM_PARITY_REG: SRAM parity register. */ -#define FR_AZ_SRAM_PARITY 0x00000670 -#define FRF_CZ_BYPASS_ECC_LBN 3 -#define FRF_CZ_BYPASS_ECC_WIDTH 1 -#define FRF_CZ_SEC_INT_LBN 2 -#define FRF_CZ_SEC_INT_WIDTH 1 -#define FRF_CZ_FORCE_SRAM_DOUBLE_ERR_LBN 1 -#define FRF_CZ_FORCE_SRAM_DOUBLE_ERR_WIDTH 1 -#define FRF_AB_FORCE_SRAM_PERR_LBN 0 -#define FRF_AB_FORCE_SRAM_PERR_WIDTH 1 -#define FRF_CZ_FORCE_SRAM_SINGLE_ERR_LBN 0 -#define FRF_CZ_FORCE_SRAM_SINGLE_ERR_WIDTH 1 - -/* RX_CFG_REG: Receive configuration register */ -#define FR_AZ_RX_CFG 0x00000800 -#define FRF_CZ_RX_MIN_KBUF_SIZE_LBN 72 -#define FRF_CZ_RX_MIN_KBUF_SIZE_WIDTH 14 -#define FRF_CZ_RX_HDR_SPLIT_EN_LBN 71 -#define FRF_CZ_RX_HDR_SPLIT_EN_WIDTH 1 -#define FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_LBN 62 -#define FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH 9 -#define FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_LBN 53 -#define FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH 9 -#define FRF_CZ_RX_PRE_RFF_IPG_LBN 49 -#define FRF_CZ_RX_PRE_RFF_IPG_WIDTH 4 -#define FRF_BZ_RX_TCP_SUP_LBN 48 -#define FRF_BZ_RX_TCP_SUP_WIDTH 1 -#define FRF_BZ_RX_INGR_EN_LBN 47 -#define FRF_BZ_RX_INGR_EN_WIDTH 1 -#define FRF_BZ_RX_IP_HASH_LBN 46 -#define FRF_BZ_RX_IP_HASH_WIDTH 1 -#define FRF_BZ_RX_HASH_ALG_LBN 45 -#define FRF_BZ_RX_HASH_ALG_WIDTH 1 -#define FRF_BZ_RX_HASH_INSRT_HDR_LBN 44 -#define FRF_BZ_RX_HASH_INSRT_HDR_WIDTH 1 -#define FRF_BZ_RX_DESC_PUSH_EN_LBN 43 -#define FRF_BZ_RX_DESC_PUSH_EN_WIDTH 1 -#define FRF_BZ_RX_RDW_PATCH_EN_LBN 42 -#define FRF_BZ_RX_RDW_PATCH_EN_WIDTH 1 -#define FRF_BB_RX_PCI_BURST_SIZE_LBN 39 -#define FRF_BB_RX_PCI_BURST_SIZE_WIDTH 3 -#define FRF_BZ_RX_OWNERR_CTL_LBN 38 -#define FRF_BZ_RX_OWNERR_CTL_WIDTH 1 -#define FRF_BZ_RX_XON_TX_TH_LBN 33 -#define FRF_BZ_RX_XON_TX_TH_WIDTH 5 -#define FRF_AA_RX_DESC_PUSH_EN_LBN 35 -#define FRF_AA_RX_DESC_PUSH_EN_WIDTH 1 -#define FRF_AA_RX_RDW_PATCH_EN_LBN 34 -#define FRF_AA_RX_RDW_PATCH_EN_WIDTH 1 -#define FRF_AA_RX_PCI_BURST_SIZE_LBN 31 -#define FRF_AA_RX_PCI_BURST_SIZE_WIDTH 3 -#define FRF_BZ_RX_XOFF_TX_TH_LBN 28 -#define FRF_BZ_RX_XOFF_TX_TH_WIDTH 5 -#define FRF_AA_RX_OWNERR_CTL_LBN 30 -#define FRF_AA_RX_OWNERR_CTL_WIDTH 1 -#define FRF_AA_RX_XON_TX_TH_LBN 25 -#define FRF_AA_RX_XON_TX_TH_WIDTH 5 -#define FRF_BZ_RX_USR_BUF_SIZE_LBN 19 -#define FRF_BZ_RX_USR_BUF_SIZE_WIDTH 9 -#define FRF_AA_RX_XOFF_TX_TH_LBN 20 -#define FRF_AA_RX_XOFF_TX_TH_WIDTH 5 -#define FRF_AA_RX_USR_BUF_SIZE_LBN 11 -#define FRF_AA_RX_USR_BUF_SIZE_WIDTH 9 -#define FRF_BZ_RX_XON_MAC_TH_LBN 10 -#define FRF_BZ_RX_XON_MAC_TH_WIDTH 9 -#define FRF_AA_RX_XON_MAC_TH_LBN 6 -#define FRF_AA_RX_XON_MAC_TH_WIDTH 5 -#define FRF_BZ_RX_XOFF_MAC_TH_LBN 1 -#define FRF_BZ_RX_XOFF_MAC_TH_WIDTH 9 -#define FRF_AA_RX_XOFF_MAC_TH_LBN 1 -#define FRF_AA_RX_XOFF_MAC_TH_WIDTH 5 -#define FRF_AZ_RX_XOFF_MAC_EN_LBN 0 -#define FRF_AZ_RX_XOFF_MAC_EN_WIDTH 1 - -/* RX_FILTER_CTL_REG: Receive filter control registers */ -#define FR_BZ_RX_FILTER_CTL 0x00000810 -#define FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_LBN 94 -#define FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_WIDTH 8 -#define FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_LBN 86 -#define FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_WIDTH 8 -#define FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_LBN 85 -#define FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_WIDTH 1 -#define FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_LBN 69 -#define FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_WIDTH 16 -#define FRF_CZ_MULTICAST_NOMATCH_Q_ID_LBN 57 -#define FRF_CZ_MULTICAST_NOMATCH_Q_ID_WIDTH 12 -#define FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_LBN 56 -#define FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_WIDTH 1 -#define FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_LBN 55 -#define FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_WIDTH 1 -#define FRF_CZ_UNICAST_NOMATCH_Q_ID_LBN 43 -#define FRF_CZ_UNICAST_NOMATCH_Q_ID_WIDTH 12 -#define FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_LBN 42 -#define FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_WIDTH 1 -#define FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_LBN 41 -#define FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_WIDTH 1 -#define FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_LBN 40 -#define FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_WIDTH 1 -#define FRF_BZ_UDP_FULL_SRCH_LIMIT_LBN 32 -#define FRF_BZ_UDP_FULL_SRCH_LIMIT_WIDTH 8 -#define FRF_BZ_NUM_KER_LBN 24 -#define FRF_BZ_NUM_KER_WIDTH 2 -#define FRF_BZ_UDP_WILD_SRCH_LIMIT_LBN 16 -#define FRF_BZ_UDP_WILD_SRCH_LIMIT_WIDTH 8 -#define FRF_BZ_TCP_WILD_SRCH_LIMIT_LBN 8 -#define FRF_BZ_TCP_WILD_SRCH_LIMIT_WIDTH 8 -#define FRF_BZ_TCP_FULL_SRCH_LIMIT_LBN 0 -#define FRF_BZ_TCP_FULL_SRCH_LIMIT_WIDTH 8 - -/* RX_FLUSH_DESCQ_REG: Receive flush descriptor queue register */ -#define FR_AZ_RX_FLUSH_DESCQ 0x00000820 -#define FRF_AZ_RX_FLUSH_DESCQ_CMD_LBN 24 -#define FRF_AZ_RX_FLUSH_DESCQ_CMD_WIDTH 1 -#define FRF_AZ_RX_FLUSH_DESCQ_LBN 0 -#define FRF_AZ_RX_FLUSH_DESCQ_WIDTH 12 - -/* RX_DESC_UPD_REGP0: Receive descriptor update register. */ -#define FR_BZ_RX_DESC_UPD_P0 0x00000830 -#define FR_BZ_RX_DESC_UPD_P0_STEP 8192 -#define FR_BZ_RX_DESC_UPD_P0_ROWS 1024 -/* RX_DESC_UPD_REG_KER: Receive descriptor update register. */ -#define FR_AA_RX_DESC_UPD_KER 0x00000830 -#define FR_AA_RX_DESC_UPD_KER_STEP 8192 -#define FR_AA_RX_DESC_UPD_KER_ROWS 4 -/* RX_DESC_UPD_REGP123: Receive descriptor update register. */ -#define FR_BB_RX_DESC_UPD_P123 0x01000830 -#define FR_BB_RX_DESC_UPD_P123_STEP 8192 -#define FR_BB_RX_DESC_UPD_P123_ROWS 3072 -#define FRF_AZ_RX_DESC_WPTR_LBN 96 -#define FRF_AZ_RX_DESC_WPTR_WIDTH 12 -#define FRF_AZ_RX_DESC_PUSH_CMD_LBN 95 -#define FRF_AZ_RX_DESC_PUSH_CMD_WIDTH 1 -#define FRF_AZ_RX_DESC_LBN 0 -#define FRF_AZ_RX_DESC_WIDTH 64 - -/* RX_DC_CFG_REG: Receive descriptor cache configuration register */ -#define FR_AZ_RX_DC_CFG 0x00000840 -#define FRF_AB_RX_MAX_PF_LBN 2 -#define FRF_AB_RX_MAX_PF_WIDTH 2 -#define FRF_AZ_RX_DC_SIZE_LBN 0 -#define FRF_AZ_RX_DC_SIZE_WIDTH 2 -#define FFE_AZ_RX_DC_SIZE_64 3 -#define FFE_AZ_RX_DC_SIZE_32 2 -#define FFE_AZ_RX_DC_SIZE_16 1 -#define FFE_AZ_RX_DC_SIZE_8 0 - -/* RX_DC_PF_WM_REG: Receive descriptor cache pre-fetch watermark register */ -#define FR_AZ_RX_DC_PF_WM 0x00000850 -#define FRF_AZ_RX_DC_PF_HWM_LBN 6 -#define FRF_AZ_RX_DC_PF_HWM_WIDTH 6 -#define FRF_AZ_RX_DC_PF_LWM_LBN 0 -#define FRF_AZ_RX_DC_PF_LWM_WIDTH 6 - -/* RX_RSS_TKEY_REG: RSS Toeplitz hash key */ -#define FR_BZ_RX_RSS_TKEY 0x00000860 -#define FRF_BZ_RX_RSS_TKEY_HI_LBN 64 -#define FRF_BZ_RX_RSS_TKEY_HI_WIDTH 64 -#define FRF_BZ_RX_RSS_TKEY_LO_LBN 0 -#define FRF_BZ_RX_RSS_TKEY_LO_WIDTH 64 - -/* RX_NODESC_DROP_REG: Receive dropped packet counter register */ -#define FR_AZ_RX_NODESC_DROP 0x00000880 -#define FRF_CZ_RX_NODESC_DROP_CNT_LBN 0 -#define FRF_CZ_RX_NODESC_DROP_CNT_WIDTH 32 -#define FRF_AB_RX_NODESC_DROP_CNT_LBN 0 -#define FRF_AB_RX_NODESC_DROP_CNT_WIDTH 16 - -/* RX_SELF_RST_REG: Receive self reset register */ -#define FR_AA_RX_SELF_RST 0x00000890 -#define FRF_AA_RX_ISCSI_DIS_LBN 17 -#define FRF_AA_RX_ISCSI_DIS_WIDTH 1 -#define FRF_AA_RX_SW_RST_REG_LBN 16 -#define FRF_AA_RX_SW_RST_REG_WIDTH 1 -#define FRF_AA_RX_NODESC_WAIT_DIS_LBN 9 -#define FRF_AA_RX_NODESC_WAIT_DIS_WIDTH 1 -#define FRF_AA_RX_SELF_RST_EN_LBN 8 -#define FRF_AA_RX_SELF_RST_EN_WIDTH 1 -#define FRF_AA_RX_MAX_PF_LAT_LBN 4 -#define FRF_AA_RX_MAX_PF_LAT_WIDTH 4 -#define FRF_AA_RX_MAX_LU_LAT_LBN 0 -#define FRF_AA_RX_MAX_LU_LAT_WIDTH 4 - -/* RX_DEBUG_REG: undocumented register */ -#define FR_AZ_RX_DEBUG 0x000008a0 -#define FRF_AZ_RX_DEBUG_LBN 0 -#define FRF_AZ_RX_DEBUG_WIDTH 64 - -/* RX_PUSH_DROP_REG: Receive descriptor push dropped counter register */ -#define FR_AZ_RX_PUSH_DROP 0x000008b0 -#define FRF_AZ_RX_PUSH_DROP_CNT_LBN 0 -#define FRF_AZ_RX_PUSH_DROP_CNT_WIDTH 32 - -/* RX_RSS_IPV6_REG1: IPv6 RSS Toeplitz hash key low bytes */ -#define FR_CZ_RX_RSS_IPV6_REG1 0x000008d0 -#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN 0 -#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH 128 - -/* RX_RSS_IPV6_REG2: IPv6 RSS Toeplitz hash key middle bytes */ -#define FR_CZ_RX_RSS_IPV6_REG2 0x000008e0 -#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN 0 -#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH 128 - -/* RX_RSS_IPV6_REG3: IPv6 RSS Toeplitz hash key upper bytes and IPv6 RSS settings */ -#define FR_CZ_RX_RSS_IPV6_REG3 0x000008f0 -#define FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_LBN 66 -#define FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_WIDTH 1 -#define FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_LBN 65 -#define FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_WIDTH 1 -#define FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_LBN 64 -#define FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_WIDTH 1 -#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN 0 -#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH 64 - -/* TX_FLUSH_DESCQ_REG: Transmit flush descriptor queue register */ -#define FR_AZ_TX_FLUSH_DESCQ 0x00000a00 -#define FRF_AZ_TX_FLUSH_DESCQ_CMD_LBN 12 -#define FRF_AZ_TX_FLUSH_DESCQ_CMD_WIDTH 1 -#define FRF_AZ_TX_FLUSH_DESCQ_LBN 0 -#define FRF_AZ_TX_FLUSH_DESCQ_WIDTH 12 - -/* TX_DESC_UPD_REGP0: Transmit descriptor update register. */ -#define FR_BZ_TX_DESC_UPD_P0 0x00000a10 -#define FR_BZ_TX_DESC_UPD_P0_STEP 8192 -#define FR_BZ_TX_DESC_UPD_P0_ROWS 1024 -/* TX_DESC_UPD_REG_KER: Transmit descriptor update register. */ -#define FR_AA_TX_DESC_UPD_KER 0x00000a10 -#define FR_AA_TX_DESC_UPD_KER_STEP 8192 -#define FR_AA_TX_DESC_UPD_KER_ROWS 8 -/* TX_DESC_UPD_REGP123: Transmit descriptor update register. */ -#define FR_BB_TX_DESC_UPD_P123 0x01000a10 -#define FR_BB_TX_DESC_UPD_P123_STEP 8192 -#define FR_BB_TX_DESC_UPD_P123_ROWS 3072 -#define FRF_AZ_TX_DESC_WPTR_LBN 96 -#define FRF_AZ_TX_DESC_WPTR_WIDTH 12 -#define FRF_AZ_TX_DESC_PUSH_CMD_LBN 95 -#define FRF_AZ_TX_DESC_PUSH_CMD_WIDTH 1 -#define FRF_AZ_TX_DESC_LBN 0 -#define FRF_AZ_TX_DESC_WIDTH 95 - -/* TX_DC_CFG_REG: Transmit descriptor cache configuration register */ -#define FR_AZ_TX_DC_CFG 0x00000a20 -#define FRF_AZ_TX_DC_SIZE_LBN 0 -#define FRF_AZ_TX_DC_SIZE_WIDTH 2 -#define FFE_AZ_TX_DC_SIZE_32 2 -#define FFE_AZ_TX_DC_SIZE_16 1 -#define FFE_AZ_TX_DC_SIZE_8 0 - -/* TX_CHKSM_CFG_REG: Transmit checksum configuration register */ -#define FR_AA_TX_CHKSM_CFG 0x00000a30 -#define FRF_AA_TX_Q_CHKSM_DIS_96_127_LBN 96 -#define FRF_AA_TX_Q_CHKSM_DIS_96_127_WIDTH 32 -#define FRF_AA_TX_Q_CHKSM_DIS_64_95_LBN 64 -#define FRF_AA_TX_Q_CHKSM_DIS_64_95_WIDTH 32 -#define FRF_AA_TX_Q_CHKSM_DIS_32_63_LBN 32 -#define FRF_AA_TX_Q_CHKSM_DIS_32_63_WIDTH 32 -#define FRF_AA_TX_Q_CHKSM_DIS_0_31_LBN 0 -#define FRF_AA_TX_Q_CHKSM_DIS_0_31_WIDTH 32 - -/* TX_CFG_REG: Transmit configuration register */ -#define FR_AZ_TX_CFG 0x00000a50 -#define FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_LBN 114 -#define FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_WIDTH 8 -#define FRF_CZ_TX_FILTER_TEST_MODE_BIT_LBN 113 -#define FRF_CZ_TX_FILTER_TEST_MODE_BIT_WIDTH 1 -#define FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_LBN 105 -#define FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_LBN 97 -#define FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_LBN 89 -#define FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_LBN 81 -#define FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_LBN 73 -#define FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_LBN 65 -#define FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8 -#define FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_LBN 64 -#define FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_WIDTH 1 -#define FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_LBN 48 -#define FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_WIDTH 16 -#define FRF_CZ_TX_FILTER_EN_BIT_LBN 47 -#define FRF_CZ_TX_FILTER_EN_BIT_WIDTH 1 -#define FRF_AZ_TX_IP_ID_P0_OFS_LBN 16 -#define FRF_AZ_TX_IP_ID_P0_OFS_WIDTH 15 -#define FRF_AZ_TX_NO_EOP_DISC_EN_LBN 5 -#define FRF_AZ_TX_NO_EOP_DISC_EN_WIDTH 1 -#define FRF_AZ_TX_P1_PRI_EN_LBN 4 -#define FRF_AZ_TX_P1_PRI_EN_WIDTH 1 -#define FRF_AZ_TX_OWNERR_CTL_LBN 2 -#define FRF_AZ_TX_OWNERR_CTL_WIDTH 1 -#define FRF_AA_TX_NON_IP_DROP_DIS_LBN 1 -#define FRF_AA_TX_NON_IP_DROP_DIS_WIDTH 1 -#define FRF_AZ_TX_IP_ID_REP_EN_LBN 0 -#define FRF_AZ_TX_IP_ID_REP_EN_WIDTH 1 - -/* TX_PUSH_DROP_REG: Transmit push dropped register */ -#define FR_AZ_TX_PUSH_DROP 0x00000a60 -#define FRF_AZ_TX_PUSH_DROP_CNT_LBN 0 -#define FRF_AZ_TX_PUSH_DROP_CNT_WIDTH 32 - -/* TX_RESERVED_REG: Transmit configuration register */ -#define FR_AZ_TX_RESERVED 0x00000a80 -#define FRF_AZ_TX_EVT_CNT_LBN 121 -#define FRF_AZ_TX_EVT_CNT_WIDTH 7 -#define FRF_AZ_TX_PREF_AGE_CNT_LBN 119 -#define FRF_AZ_TX_PREF_AGE_CNT_WIDTH 2 -#define FRF_AZ_TX_RD_COMP_TMR_LBN 96 -#define FRF_AZ_TX_RD_COMP_TMR_WIDTH 23 -#define FRF_AZ_TX_PUSH_EN_LBN 89 -#define FRF_AZ_TX_PUSH_EN_WIDTH 1 -#define FRF_AZ_TX_PUSH_CHK_DIS_LBN 88 -#define FRF_AZ_TX_PUSH_CHK_DIS_WIDTH 1 -#define FRF_AZ_TX_D_FF_FULL_P0_LBN 85 -#define FRF_AZ_TX_D_FF_FULL_P0_WIDTH 1 -#define FRF_AZ_TX_DMAR_ST_P0_LBN 81 -#define FRF_AZ_TX_DMAR_ST_P0_WIDTH 1 -#define FRF_AZ_TX_DMAQ_ST_LBN 78 -#define FRF_AZ_TX_DMAQ_ST_WIDTH 1 -#define FRF_AZ_TX_RX_SPACER_LBN 64 -#define FRF_AZ_TX_RX_SPACER_WIDTH 8 -#define FRF_AZ_TX_DROP_ABORT_EN_LBN 60 -#define FRF_AZ_TX_DROP_ABORT_EN_WIDTH 1 -#define FRF_AZ_TX_SOFT_EVT_EN_LBN 59 -#define FRF_AZ_TX_SOFT_EVT_EN_WIDTH 1 -#define FRF_AZ_TX_PS_EVT_DIS_LBN 58 -#define FRF_AZ_TX_PS_EVT_DIS_WIDTH 1 -#define FRF_AZ_TX_RX_SPACER_EN_LBN 57 -#define FRF_AZ_TX_RX_SPACER_EN_WIDTH 1 -#define FRF_AZ_TX_XP_TIMER_LBN 52 -#define FRF_AZ_TX_XP_TIMER_WIDTH 5 -#define FRF_AZ_TX_PREF_SPACER_LBN 44 -#define FRF_AZ_TX_PREF_SPACER_WIDTH 8 -#define FRF_AZ_TX_PREF_WD_TMR_LBN 22 -#define FRF_AZ_TX_PREF_WD_TMR_WIDTH 22 -#define FRF_AZ_TX_ONLY1TAG_LBN 21 -#define FRF_AZ_TX_ONLY1TAG_WIDTH 1 -#define FRF_AZ_TX_PREF_THRESHOLD_LBN 19 -#define FRF_AZ_TX_PREF_THRESHOLD_WIDTH 2 -#define FRF_AZ_TX_ONE_PKT_PER_Q_LBN 18 -#define FRF_AZ_TX_ONE_PKT_PER_Q_WIDTH 1 -#define FRF_AZ_TX_DIS_NON_IP_EV_LBN 17 -#define FRF_AZ_TX_DIS_NON_IP_EV_WIDTH 1 -#define FRF_AA_TX_DMA_FF_THR_LBN 16 -#define FRF_AA_TX_DMA_FF_THR_WIDTH 1 -#define FRF_AZ_TX_DMA_SPACER_LBN 8 -#define FRF_AZ_TX_DMA_SPACER_WIDTH 8 -#define FRF_AA_TX_TCP_DIS_LBN 7 -#define FRF_AA_TX_TCP_DIS_WIDTH 1 -#define FRF_BZ_TX_FLUSH_MIN_LEN_EN_LBN 7 -#define FRF_BZ_TX_FLUSH_MIN_LEN_EN_WIDTH 1 -#define FRF_AA_TX_IP_DIS_LBN 6 -#define FRF_AA_TX_IP_DIS_WIDTH 1 -#define FRF_AZ_TX_MAX_CPL_LBN 2 -#define FRF_AZ_TX_MAX_CPL_WIDTH 2 -#define FFE_AZ_TX_MAX_CPL_16 3 -#define FFE_AZ_TX_MAX_CPL_8 2 -#define FFE_AZ_TX_MAX_CPL_4 1 -#define FFE_AZ_TX_MAX_CPL_NOLIMIT 0 -#define FRF_AZ_TX_MAX_PREF_LBN 0 -#define FRF_AZ_TX_MAX_PREF_WIDTH 2 -#define FFE_AZ_TX_MAX_PREF_32 3 -#define FFE_AZ_TX_MAX_PREF_16 2 -#define FFE_AZ_TX_MAX_PREF_8 1 -#define FFE_AZ_TX_MAX_PREF_OFF 0 - -/* TX_PACE_REG: Transmit pace control register */ -#define FR_BZ_TX_PACE 0x00000a90 -#define FRF_BZ_TX_PACE_SB_NOT_AF_LBN 19 -#define FRF_BZ_TX_PACE_SB_NOT_AF_WIDTH 10 -#define FRF_BZ_TX_PACE_SB_AF_LBN 9 -#define FRF_BZ_TX_PACE_SB_AF_WIDTH 10 -#define FRF_BZ_TX_PACE_FB_BASE_LBN 5 -#define FRF_BZ_TX_PACE_FB_BASE_WIDTH 4 -#define FRF_BZ_TX_PACE_BIN_TH_LBN 0 -#define FRF_BZ_TX_PACE_BIN_TH_WIDTH 5 - -/* TX_PACE_DROP_QID_REG: PACE Drop QID Counter */ -#define FR_BZ_TX_PACE_DROP_QID 0x00000aa0 -#define FRF_BZ_TX_PACE_QID_DRP_CNT_LBN 0 -#define FRF_BZ_TX_PACE_QID_DRP_CNT_WIDTH 16 - -/* TX_VLAN_REG: Transmit VLAN tag register */ -#define FR_BB_TX_VLAN 0x00000ae0 -#define FRF_BB_TX_VLAN_EN_LBN 127 -#define FRF_BB_TX_VLAN_EN_WIDTH 1 -#define FRF_BB_TX_VLAN7_PORT1_EN_LBN 125 -#define FRF_BB_TX_VLAN7_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN7_PORT0_EN_LBN 124 -#define FRF_BB_TX_VLAN7_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN7_LBN 112 -#define FRF_BB_TX_VLAN7_WIDTH 12 -#define FRF_BB_TX_VLAN6_PORT1_EN_LBN 109 -#define FRF_BB_TX_VLAN6_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN6_PORT0_EN_LBN 108 -#define FRF_BB_TX_VLAN6_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN6_LBN 96 -#define FRF_BB_TX_VLAN6_WIDTH 12 -#define FRF_BB_TX_VLAN5_PORT1_EN_LBN 93 -#define FRF_BB_TX_VLAN5_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN5_PORT0_EN_LBN 92 -#define FRF_BB_TX_VLAN5_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN5_LBN 80 -#define FRF_BB_TX_VLAN5_WIDTH 12 -#define FRF_BB_TX_VLAN4_PORT1_EN_LBN 77 -#define FRF_BB_TX_VLAN4_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN4_PORT0_EN_LBN 76 -#define FRF_BB_TX_VLAN4_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN4_LBN 64 -#define FRF_BB_TX_VLAN4_WIDTH 12 -#define FRF_BB_TX_VLAN3_PORT1_EN_LBN 61 -#define FRF_BB_TX_VLAN3_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN3_PORT0_EN_LBN 60 -#define FRF_BB_TX_VLAN3_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN3_LBN 48 -#define FRF_BB_TX_VLAN3_WIDTH 12 -#define FRF_BB_TX_VLAN2_PORT1_EN_LBN 45 -#define FRF_BB_TX_VLAN2_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN2_PORT0_EN_LBN 44 -#define FRF_BB_TX_VLAN2_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN2_LBN 32 -#define FRF_BB_TX_VLAN2_WIDTH 12 -#define FRF_BB_TX_VLAN1_PORT1_EN_LBN 29 -#define FRF_BB_TX_VLAN1_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN1_PORT0_EN_LBN 28 -#define FRF_BB_TX_VLAN1_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN1_LBN 16 -#define FRF_BB_TX_VLAN1_WIDTH 12 -#define FRF_BB_TX_VLAN0_PORT1_EN_LBN 13 -#define FRF_BB_TX_VLAN0_PORT1_EN_WIDTH 1 -#define FRF_BB_TX_VLAN0_PORT0_EN_LBN 12 -#define FRF_BB_TX_VLAN0_PORT0_EN_WIDTH 1 -#define FRF_BB_TX_VLAN0_LBN 0 -#define FRF_BB_TX_VLAN0_WIDTH 12 - -/* TX_IPFIL_PORTEN_REG: Transmit filter control register */ -#define FR_BZ_TX_IPFIL_PORTEN 0x00000af0 -#define FRF_BZ_TX_MADR0_FIL_EN_LBN 64 -#define FRF_BZ_TX_MADR0_FIL_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL31_PORT_EN_LBN 62 -#define FRF_BB_TX_IPFIL31_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL30_PORT_EN_LBN 60 -#define FRF_BB_TX_IPFIL30_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL29_PORT_EN_LBN 58 -#define FRF_BB_TX_IPFIL29_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL28_PORT_EN_LBN 56 -#define FRF_BB_TX_IPFIL28_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL27_PORT_EN_LBN 54 -#define FRF_BB_TX_IPFIL27_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL26_PORT_EN_LBN 52 -#define FRF_BB_TX_IPFIL26_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL25_PORT_EN_LBN 50 -#define FRF_BB_TX_IPFIL25_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL24_PORT_EN_LBN 48 -#define FRF_BB_TX_IPFIL24_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL23_PORT_EN_LBN 46 -#define FRF_BB_TX_IPFIL23_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL22_PORT_EN_LBN 44 -#define FRF_BB_TX_IPFIL22_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL21_PORT_EN_LBN 42 -#define FRF_BB_TX_IPFIL21_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL20_PORT_EN_LBN 40 -#define FRF_BB_TX_IPFIL20_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL19_PORT_EN_LBN 38 -#define FRF_BB_TX_IPFIL19_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL18_PORT_EN_LBN 36 -#define FRF_BB_TX_IPFIL18_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL17_PORT_EN_LBN 34 -#define FRF_BB_TX_IPFIL17_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL16_PORT_EN_LBN 32 -#define FRF_BB_TX_IPFIL16_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL15_PORT_EN_LBN 30 -#define FRF_BB_TX_IPFIL15_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL14_PORT_EN_LBN 28 -#define FRF_BB_TX_IPFIL14_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL13_PORT_EN_LBN 26 -#define FRF_BB_TX_IPFIL13_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL12_PORT_EN_LBN 24 -#define FRF_BB_TX_IPFIL12_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL11_PORT_EN_LBN 22 -#define FRF_BB_TX_IPFIL11_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL10_PORT_EN_LBN 20 -#define FRF_BB_TX_IPFIL10_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL9_PORT_EN_LBN 18 -#define FRF_BB_TX_IPFIL9_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL8_PORT_EN_LBN 16 -#define FRF_BB_TX_IPFIL8_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL7_PORT_EN_LBN 14 -#define FRF_BB_TX_IPFIL7_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL6_PORT_EN_LBN 12 -#define FRF_BB_TX_IPFIL6_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL5_PORT_EN_LBN 10 -#define FRF_BB_TX_IPFIL5_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL4_PORT_EN_LBN 8 -#define FRF_BB_TX_IPFIL4_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL3_PORT_EN_LBN 6 -#define FRF_BB_TX_IPFIL3_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL2_PORT_EN_LBN 4 -#define FRF_BB_TX_IPFIL2_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL1_PORT_EN_LBN 2 -#define FRF_BB_TX_IPFIL1_PORT_EN_WIDTH 1 -#define FRF_BB_TX_IPFIL0_PORT_EN_LBN 0 -#define FRF_BB_TX_IPFIL0_PORT_EN_WIDTH 1 - -/* TX_IPFIL_TBL: Transmit IP source address filter table */ -#define FR_BB_TX_IPFIL_TBL 0x00000b00 -#define FR_BB_TX_IPFIL_TBL_STEP 16 -#define FR_BB_TX_IPFIL_TBL_ROWS 16 -#define FRF_BB_TX_IPFIL_MASK_1_LBN 96 -#define FRF_BB_TX_IPFIL_MASK_1_WIDTH 32 -#define FRF_BB_TX_IP_SRC_ADR_1_LBN 64 -#define FRF_BB_TX_IP_SRC_ADR_1_WIDTH 32 -#define FRF_BB_TX_IPFIL_MASK_0_LBN 32 -#define FRF_BB_TX_IPFIL_MASK_0_WIDTH 32 -#define FRF_BB_TX_IP_SRC_ADR_0_LBN 0 -#define FRF_BB_TX_IP_SRC_ADR_0_WIDTH 32 - -/* MD_TXD_REG: PHY management transmit data register */ -#define FR_AB_MD_TXD 0x00000c00 -#define FRF_AB_MD_TXD_LBN 0 -#define FRF_AB_MD_TXD_WIDTH 16 - -/* MD_RXD_REG: PHY management receive data register */ -#define FR_AB_MD_RXD 0x00000c10 -#define FRF_AB_MD_RXD_LBN 0 -#define FRF_AB_MD_RXD_WIDTH 16 - -/* MD_CS_REG: PHY management configuration & status register */ -#define FR_AB_MD_CS 0x00000c20 -#define FRF_AB_MD_RD_EN_CMD_LBN 15 -#define FRF_AB_MD_RD_EN_CMD_WIDTH 1 -#define FRF_AB_MD_WR_EN_CMD_LBN 14 -#define FRF_AB_MD_WR_EN_CMD_WIDTH 1 -#define FRF_AB_MD_ADDR_CMD_LBN 13 -#define FRF_AB_MD_ADDR_CMD_WIDTH 1 -#define FRF_AB_MD_PT_LBN 7 -#define FRF_AB_MD_PT_WIDTH 3 -#define FRF_AB_MD_PL_LBN 6 -#define FRF_AB_MD_PL_WIDTH 1 -#define FRF_AB_MD_INT_CLR_LBN 5 -#define FRF_AB_MD_INT_CLR_WIDTH 1 -#define FRF_AB_MD_GC_LBN 4 -#define FRF_AB_MD_GC_WIDTH 1 -#define FRF_AB_MD_PRSP_LBN 3 -#define FRF_AB_MD_PRSP_WIDTH 1 -#define FRF_AB_MD_RIC_LBN 2 -#define FRF_AB_MD_RIC_WIDTH 1 -#define FRF_AB_MD_RDC_LBN 1 -#define FRF_AB_MD_RDC_WIDTH 1 -#define FRF_AB_MD_WRC_LBN 0 -#define FRF_AB_MD_WRC_WIDTH 1 - -/* MD_PHY_ADR_REG: PHY management PHY address register */ -#define FR_AB_MD_PHY_ADR 0x00000c30 -#define FRF_AB_MD_PHY_ADR_LBN 0 -#define FRF_AB_MD_PHY_ADR_WIDTH 16 - -/* MD_ID_REG: PHY management ID register */ -#define FR_AB_MD_ID 0x00000c40 -#define FRF_AB_MD_PRT_ADR_LBN 11 -#define FRF_AB_MD_PRT_ADR_WIDTH 5 -#define FRF_AB_MD_DEV_ADR_LBN 6 -#define FRF_AB_MD_DEV_ADR_WIDTH 5 - -/* MD_STAT_REG: PHY management status & mask register */ -#define FR_AB_MD_STAT 0x00000c50 -#define FRF_AB_MD_PINT_LBN 4 -#define FRF_AB_MD_PINT_WIDTH 1 -#define FRF_AB_MD_DONE_LBN 3 -#define FRF_AB_MD_DONE_WIDTH 1 -#define FRF_AB_MD_BSERR_LBN 2 -#define FRF_AB_MD_BSERR_WIDTH 1 -#define FRF_AB_MD_LNFL_LBN 1 -#define FRF_AB_MD_LNFL_WIDTH 1 -#define FRF_AB_MD_BSY_LBN 0 -#define FRF_AB_MD_BSY_WIDTH 1 - -/* MAC_STAT_DMA_REG: Port MAC statistical counter DMA register */ -#define FR_AB_MAC_STAT_DMA 0x00000c60 -#define FRF_AB_MAC_STAT_DMA_CMD_LBN 48 -#define FRF_AB_MAC_STAT_DMA_CMD_WIDTH 1 -#define FRF_AB_MAC_STAT_DMA_ADR_LBN 0 -#define FRF_AB_MAC_STAT_DMA_ADR_WIDTH 48 - -/* MAC_CTRL_REG: Port MAC control register */ -#define FR_AB_MAC_CTRL 0x00000c80 -#define FRF_AB_MAC_XOFF_VAL_LBN 16 -#define FRF_AB_MAC_XOFF_VAL_WIDTH 16 -#define FRF_BB_TXFIFO_DRAIN_EN_LBN 7 -#define FRF_BB_TXFIFO_DRAIN_EN_WIDTH 1 -#define FRF_AB_MAC_XG_DISTXCRC_LBN 5 -#define FRF_AB_MAC_XG_DISTXCRC_WIDTH 1 -#define FRF_AB_MAC_BCAD_ACPT_LBN 4 -#define FRF_AB_MAC_BCAD_ACPT_WIDTH 1 -#define FRF_AB_MAC_UC_PROM_LBN 3 -#define FRF_AB_MAC_UC_PROM_WIDTH 1 -#define FRF_AB_MAC_LINK_STATUS_LBN 2 -#define FRF_AB_MAC_LINK_STATUS_WIDTH 1 -#define FRF_AB_MAC_SPEED_LBN 0 -#define FRF_AB_MAC_SPEED_WIDTH 2 -#define FFE_AB_MAC_SPEED_10G 3 -#define FFE_AB_MAC_SPEED_1G 2 -#define FFE_AB_MAC_SPEED_100M 1 -#define FFE_AB_MAC_SPEED_10M 0 - -/* GEN_MODE_REG: General Purpose mode register (external interrupt mask) */ -#define FR_BB_GEN_MODE 0x00000c90 -#define FRF_BB_XFP_PHY_INT_POL_SEL_LBN 3 -#define FRF_BB_XFP_PHY_INT_POL_SEL_WIDTH 1 -#define FRF_BB_XG_PHY_INT_POL_SEL_LBN 2 -#define FRF_BB_XG_PHY_INT_POL_SEL_WIDTH 1 -#define FRF_BB_XFP_PHY_INT_MASK_LBN 1 -#define FRF_BB_XFP_PHY_INT_MASK_WIDTH 1 -#define FRF_BB_XG_PHY_INT_MASK_LBN 0 -#define FRF_BB_XG_PHY_INT_MASK_WIDTH 1 - -/* MAC_MC_HASH_REG0: Multicast address hash table */ -#define FR_AB_MAC_MC_HASH_REG0 0x00000ca0 -#define FRF_AB_MAC_MCAST_HASH0_LBN 0 -#define FRF_AB_MAC_MCAST_HASH0_WIDTH 128 - -/* MAC_MC_HASH_REG1: Multicast address hash table */ -#define FR_AB_MAC_MC_HASH_REG1 0x00000cb0 -#define FRF_AB_MAC_MCAST_HASH1_LBN 0 -#define FRF_AB_MAC_MCAST_HASH1_WIDTH 128 - -/* GM_CFG1_REG: GMAC configuration register 1 */ -#define FR_AB_GM_CFG1 0x00000e00 -#define FRF_AB_GM_SW_RST_LBN 31 -#define FRF_AB_GM_SW_RST_WIDTH 1 -#define FRF_AB_GM_SIM_RST_LBN 30 -#define FRF_AB_GM_SIM_RST_WIDTH 1 -#define FRF_AB_GM_RST_RX_MAC_CTL_LBN 19 -#define FRF_AB_GM_RST_RX_MAC_CTL_WIDTH 1 -#define FRF_AB_GM_RST_TX_MAC_CTL_LBN 18 -#define FRF_AB_GM_RST_TX_MAC_CTL_WIDTH 1 -#define FRF_AB_GM_RST_RX_FUNC_LBN 17 -#define FRF_AB_GM_RST_RX_FUNC_WIDTH 1 -#define FRF_AB_GM_RST_TX_FUNC_LBN 16 -#define FRF_AB_GM_RST_TX_FUNC_WIDTH 1 -#define FRF_AB_GM_LOOP_LBN 8 -#define FRF_AB_GM_LOOP_WIDTH 1 -#define FRF_AB_GM_RX_FC_EN_LBN 5 -#define FRF_AB_GM_RX_FC_EN_WIDTH 1 -#define FRF_AB_GM_TX_FC_EN_LBN 4 -#define FRF_AB_GM_TX_FC_EN_WIDTH 1 -#define FRF_AB_GM_SYNC_RXEN_LBN 3 -#define FRF_AB_GM_SYNC_RXEN_WIDTH 1 -#define FRF_AB_GM_RX_EN_LBN 2 -#define FRF_AB_GM_RX_EN_WIDTH 1 -#define FRF_AB_GM_SYNC_TXEN_LBN 1 -#define FRF_AB_GM_SYNC_TXEN_WIDTH 1 -#define FRF_AB_GM_TX_EN_LBN 0 -#define FRF_AB_GM_TX_EN_WIDTH 1 - -/* GM_CFG2_REG: GMAC configuration register 2 */ -#define FR_AB_GM_CFG2 0x00000e10 -#define FRF_AB_GM_PAMBL_LEN_LBN 12 -#define FRF_AB_GM_PAMBL_LEN_WIDTH 4 -#define FRF_AB_GM_IF_MODE_LBN 8 -#define FRF_AB_GM_IF_MODE_WIDTH 2 -#define FFE_AB_IF_MODE_BYTE_MODE 2 -#define FFE_AB_IF_MODE_NIBBLE_MODE 1 -#define FRF_AB_GM_HUGE_FRM_EN_LBN 5 -#define FRF_AB_GM_HUGE_FRM_EN_WIDTH 1 -#define FRF_AB_GM_LEN_CHK_LBN 4 -#define FRF_AB_GM_LEN_CHK_WIDTH 1 -#define FRF_AB_GM_PAD_CRC_EN_LBN 2 -#define FRF_AB_GM_PAD_CRC_EN_WIDTH 1 -#define FRF_AB_GM_CRC_EN_LBN 1 -#define FRF_AB_GM_CRC_EN_WIDTH 1 -#define FRF_AB_GM_FD_LBN 0 -#define FRF_AB_GM_FD_WIDTH 1 - -/* GM_IPG_REG: GMAC IPG register */ -#define FR_AB_GM_IPG 0x00000e20 -#define FRF_AB_GM_NONB2B_IPG1_LBN 24 -#define FRF_AB_GM_NONB2B_IPG1_WIDTH 7 -#define FRF_AB_GM_NONB2B_IPG2_LBN 16 -#define FRF_AB_GM_NONB2B_IPG2_WIDTH 7 -#define FRF_AB_GM_MIN_IPG_ENF_LBN 8 -#define FRF_AB_GM_MIN_IPG_ENF_WIDTH 8 -#define FRF_AB_GM_B2B_IPG_LBN 0 -#define FRF_AB_GM_B2B_IPG_WIDTH 7 - -/* GM_HD_REG: GMAC half duplex register */ -#define FR_AB_GM_HD 0x00000e30 -#define FRF_AB_GM_ALT_BOFF_VAL_LBN 20 -#define FRF_AB_GM_ALT_BOFF_VAL_WIDTH 4 -#define FRF_AB_GM_ALT_BOFF_EN_LBN 19 -#define FRF_AB_GM_ALT_BOFF_EN_WIDTH 1 -#define FRF_AB_GM_BP_NO_BOFF_LBN 18 -#define FRF_AB_GM_BP_NO_BOFF_WIDTH 1 -#define FRF_AB_GM_DIS_BOFF_LBN 17 -#define FRF_AB_GM_DIS_BOFF_WIDTH 1 -#define FRF_AB_GM_EXDEF_TX_EN_LBN 16 -#define FRF_AB_GM_EXDEF_TX_EN_WIDTH 1 -#define FRF_AB_GM_RTRY_LIMIT_LBN 12 -#define FRF_AB_GM_RTRY_LIMIT_WIDTH 4 -#define FRF_AB_GM_COL_WIN_LBN 0 -#define FRF_AB_GM_COL_WIN_WIDTH 10 - -/* GM_MAX_FLEN_REG: GMAC maximum frame length register */ -#define FR_AB_GM_MAX_FLEN 0x00000e40 -#define FRF_AB_GM_MAX_FLEN_LBN 0 -#define FRF_AB_GM_MAX_FLEN_WIDTH 16 - -/* GM_TEST_REG: GMAC test register */ -#define FR_AB_GM_TEST 0x00000e70 -#define FRF_AB_GM_MAX_BOFF_LBN 3 -#define FRF_AB_GM_MAX_BOFF_WIDTH 1 -#define FRF_AB_GM_REG_TX_FLOW_EN_LBN 2 -#define FRF_AB_GM_REG_TX_FLOW_EN_WIDTH 1 -#define FRF_AB_GM_TEST_PAUSE_LBN 1 -#define FRF_AB_GM_TEST_PAUSE_WIDTH 1 -#define FRF_AB_GM_SHORT_SLOT_LBN 0 -#define FRF_AB_GM_SHORT_SLOT_WIDTH 1 - -/* GM_ADR1_REG: GMAC station address register 1 */ -#define FR_AB_GM_ADR1 0x00000f00 -#define FRF_AB_GM_ADR_B0_LBN 24 -#define FRF_AB_GM_ADR_B0_WIDTH 8 -#define FRF_AB_GM_ADR_B1_LBN 16 -#define FRF_AB_GM_ADR_B1_WIDTH 8 -#define FRF_AB_GM_ADR_B2_LBN 8 -#define FRF_AB_GM_ADR_B2_WIDTH 8 -#define FRF_AB_GM_ADR_B3_LBN 0 -#define FRF_AB_GM_ADR_B3_WIDTH 8 - -/* GM_ADR2_REG: GMAC station address register 2 */ -#define FR_AB_GM_ADR2 0x00000f10 -#define FRF_AB_GM_ADR_B4_LBN 24 -#define FRF_AB_GM_ADR_B4_WIDTH 8 -#define FRF_AB_GM_ADR_B5_LBN 16 -#define FRF_AB_GM_ADR_B5_WIDTH 8 - -/* GMF_CFG0_REG: GMAC FIFO configuration register 0 */ -#define FR_AB_GMF_CFG0 0x00000f20 -#define FRF_AB_GMF_FTFENRPLY_LBN 20 -#define FRF_AB_GMF_FTFENRPLY_WIDTH 1 -#define FRF_AB_GMF_STFENRPLY_LBN 19 -#define FRF_AB_GMF_STFENRPLY_WIDTH 1 -#define FRF_AB_GMF_FRFENRPLY_LBN 18 -#define FRF_AB_GMF_FRFENRPLY_WIDTH 1 -#define FRF_AB_GMF_SRFENRPLY_LBN 17 -#define FRF_AB_GMF_SRFENRPLY_WIDTH 1 -#define FRF_AB_GMF_WTMENRPLY_LBN 16 -#define FRF_AB_GMF_WTMENRPLY_WIDTH 1 -#define FRF_AB_GMF_FTFENREQ_LBN 12 -#define FRF_AB_GMF_FTFENREQ_WIDTH 1 -#define FRF_AB_GMF_STFENREQ_LBN 11 -#define FRF_AB_GMF_STFENREQ_WIDTH 1 -#define FRF_AB_GMF_FRFENREQ_LBN 10 -#define FRF_AB_GMF_FRFENREQ_WIDTH 1 -#define FRF_AB_GMF_SRFENREQ_LBN 9 -#define FRF_AB_GMF_SRFENREQ_WIDTH 1 -#define FRF_AB_GMF_WTMENREQ_LBN 8 -#define FRF_AB_GMF_WTMENREQ_WIDTH 1 -#define FRF_AB_GMF_HSTRSTFT_LBN 4 -#define FRF_AB_GMF_HSTRSTFT_WIDTH 1 -#define FRF_AB_GMF_HSTRSTST_LBN 3 -#define FRF_AB_GMF_HSTRSTST_WIDTH 1 -#define FRF_AB_GMF_HSTRSTFR_LBN 2 -#define FRF_AB_GMF_HSTRSTFR_WIDTH 1 -#define FRF_AB_GMF_HSTRSTSR_LBN 1 -#define FRF_AB_GMF_HSTRSTSR_WIDTH 1 -#define FRF_AB_GMF_HSTRSTWT_LBN 0 -#define FRF_AB_GMF_HSTRSTWT_WIDTH 1 - -/* GMF_CFG1_REG: GMAC FIFO configuration register 1 */ -#define FR_AB_GMF_CFG1 0x00000f30 -#define FRF_AB_GMF_CFGFRTH_LBN 16 -#define FRF_AB_GMF_CFGFRTH_WIDTH 5 -#define FRF_AB_GMF_CFGXOFFRTX_LBN 0 -#define FRF_AB_GMF_CFGXOFFRTX_WIDTH 16 - -/* GMF_CFG2_REG: GMAC FIFO configuration register 2 */ -#define FR_AB_GMF_CFG2 0x00000f40 -#define FRF_AB_GMF_CFGHWM_LBN 16 -#define FRF_AB_GMF_CFGHWM_WIDTH 6 -#define FRF_AB_GMF_CFGLWM_LBN 0 -#define FRF_AB_GMF_CFGLWM_WIDTH 6 - -/* GMF_CFG3_REG: GMAC FIFO configuration register 3 */ -#define FR_AB_GMF_CFG3 0x00000f50 -#define FRF_AB_GMF_CFGHWMFT_LBN 16 -#define FRF_AB_GMF_CFGHWMFT_WIDTH 6 -#define FRF_AB_GMF_CFGFTTH_LBN 0 -#define FRF_AB_GMF_CFGFTTH_WIDTH 6 - -/* GMF_CFG4_REG: GMAC FIFO configuration register 4 */ -#define FR_AB_GMF_CFG4 0x00000f60 -#define FRF_AB_GMF_HSTFLTRFRM_LBN 0 -#define FRF_AB_GMF_HSTFLTRFRM_WIDTH 18 - -/* GMF_CFG5_REG: GMAC FIFO configuration register 5 */ -#define FR_AB_GMF_CFG5 0x00000f70 -#define FRF_AB_GMF_CFGHDPLX_LBN 22 -#define FRF_AB_GMF_CFGHDPLX_WIDTH 1 -#define FRF_AB_GMF_SRFULL_LBN 21 -#define FRF_AB_GMF_SRFULL_WIDTH 1 -#define FRF_AB_GMF_HSTSRFULLCLR_LBN 20 -#define FRF_AB_GMF_HSTSRFULLCLR_WIDTH 1 -#define FRF_AB_GMF_CFGBYTMODE_LBN 19 -#define FRF_AB_GMF_CFGBYTMODE_WIDTH 1 -#define FRF_AB_GMF_HSTDRPLT64_LBN 18 -#define FRF_AB_GMF_HSTDRPLT64_WIDTH 1 -#define FRF_AB_GMF_HSTFLTRFRMDC_LBN 0 -#define FRF_AB_GMF_HSTFLTRFRMDC_WIDTH 18 - -/* TX_SRC_MAC_TBL: Transmit IP source address filter table */ -#define FR_BB_TX_SRC_MAC_TBL 0x00001000 -#define FR_BB_TX_SRC_MAC_TBL_STEP 16 -#define FR_BB_TX_SRC_MAC_TBL_ROWS 16 -#define FRF_BB_TX_SRC_MAC_ADR_1_LBN 64 -#define FRF_BB_TX_SRC_MAC_ADR_1_WIDTH 48 -#define FRF_BB_TX_SRC_MAC_ADR_0_LBN 0 -#define FRF_BB_TX_SRC_MAC_ADR_0_WIDTH 48 - -/* TX_SRC_MAC_CTL_REG: Transmit MAC source address filter control */ -#define FR_BB_TX_SRC_MAC_CTL 0x00001100 -#define FRF_BB_TX_SRC_DROP_CTR_LBN 16 -#define FRF_BB_TX_SRC_DROP_CTR_WIDTH 16 -#define FRF_BB_TX_SRC_FLTR_EN_LBN 15 -#define FRF_BB_TX_SRC_FLTR_EN_WIDTH 1 -#define FRF_BB_TX_DROP_CTR_CLR_LBN 12 -#define FRF_BB_TX_DROP_CTR_CLR_WIDTH 1 -#define FRF_BB_TX_MAC_QID_SEL_LBN 0 -#define FRF_BB_TX_MAC_QID_SEL_WIDTH 3 - -/* XM_ADR_LO_REG: XGMAC address register low */ -#define FR_AB_XM_ADR_LO 0x00001200 -#define FRF_AB_XM_ADR_LO_LBN 0 -#define FRF_AB_XM_ADR_LO_WIDTH 32 - -/* XM_ADR_HI_REG: XGMAC address register high */ -#define FR_AB_XM_ADR_HI 0x00001210 -#define FRF_AB_XM_ADR_HI_LBN 0 -#define FRF_AB_XM_ADR_HI_WIDTH 16 - -/* XM_GLB_CFG_REG: XGMAC global configuration */ -#define FR_AB_XM_GLB_CFG 0x00001220 -#define FRF_AB_XM_RMTFLT_GEN_LBN 17 -#define FRF_AB_XM_RMTFLT_GEN_WIDTH 1 -#define FRF_AB_XM_DEBUG_MODE_LBN 16 -#define FRF_AB_XM_DEBUG_MODE_WIDTH 1 -#define FRF_AB_XM_RX_STAT_EN_LBN 11 -#define FRF_AB_XM_RX_STAT_EN_WIDTH 1 -#define FRF_AB_XM_TX_STAT_EN_LBN 10 -#define FRF_AB_XM_TX_STAT_EN_WIDTH 1 -#define FRF_AB_XM_RX_JUMBO_MODE_LBN 6 -#define FRF_AB_XM_RX_JUMBO_MODE_WIDTH 1 -#define FRF_AB_XM_WAN_MODE_LBN 5 -#define FRF_AB_XM_WAN_MODE_WIDTH 1 -#define FRF_AB_XM_INTCLR_MODE_LBN 3 -#define FRF_AB_XM_INTCLR_MODE_WIDTH 1 -#define FRF_AB_XM_CORE_RST_LBN 0 -#define FRF_AB_XM_CORE_RST_WIDTH 1 - -/* XM_TX_CFG_REG: XGMAC transmit configuration */ -#define FR_AB_XM_TX_CFG 0x00001230 -#define FRF_AB_XM_TX_PROG_LBN 24 -#define FRF_AB_XM_TX_PROG_WIDTH 1 -#define FRF_AB_XM_IPG_LBN 16 -#define FRF_AB_XM_IPG_WIDTH 4 -#define FRF_AB_XM_FCNTL_LBN 10 -#define FRF_AB_XM_FCNTL_WIDTH 1 -#define FRF_AB_XM_TXCRC_LBN 8 -#define FRF_AB_XM_TXCRC_WIDTH 1 -#define FRF_AB_XM_EDRC_LBN 6 -#define FRF_AB_XM_EDRC_WIDTH 1 -#define FRF_AB_XM_AUTO_PAD_LBN 5 -#define FRF_AB_XM_AUTO_PAD_WIDTH 1 -#define FRF_AB_XM_TX_PRMBL_LBN 2 -#define FRF_AB_XM_TX_PRMBL_WIDTH 1 -#define FRF_AB_XM_TXEN_LBN 1 -#define FRF_AB_XM_TXEN_WIDTH 1 -#define FRF_AB_XM_TX_RST_LBN 0 -#define FRF_AB_XM_TX_RST_WIDTH 1 - -/* XM_RX_CFG_REG: XGMAC receive configuration */ -#define FR_AB_XM_RX_CFG 0x00001240 -#define FRF_AB_XM_PASS_LENERR_LBN 26 -#define FRF_AB_XM_PASS_LENERR_WIDTH 1 -#define FRF_AB_XM_PASS_CRC_ERR_LBN 25 -#define FRF_AB_XM_PASS_CRC_ERR_WIDTH 1 -#define FRF_AB_XM_PASS_PRMBLE_ERR_LBN 24 -#define FRF_AB_XM_PASS_PRMBLE_ERR_WIDTH 1 -#define FRF_AB_XM_REJ_BCAST_LBN 20 -#define FRF_AB_XM_REJ_BCAST_WIDTH 1 -#define FRF_AB_XM_ACPT_ALL_MCAST_LBN 11 -#define FRF_AB_XM_ACPT_ALL_MCAST_WIDTH 1 -#define FRF_AB_XM_ACPT_ALL_UCAST_LBN 9 -#define FRF_AB_XM_ACPT_ALL_UCAST_WIDTH 1 -#define FRF_AB_XM_AUTO_DEPAD_LBN 8 -#define FRF_AB_XM_AUTO_DEPAD_WIDTH 1 -#define FRF_AB_XM_RXCRC_LBN 3 -#define FRF_AB_XM_RXCRC_WIDTH 1 -#define FRF_AB_XM_RX_PRMBL_LBN 2 -#define FRF_AB_XM_RX_PRMBL_WIDTH 1 -#define FRF_AB_XM_RXEN_LBN 1 -#define FRF_AB_XM_RXEN_WIDTH 1 -#define FRF_AB_XM_RX_RST_LBN 0 -#define FRF_AB_XM_RX_RST_WIDTH 1 - -/* XM_MGT_INT_MASK: documentation to be written for sum_XM_MGT_INT_MASK */ -#define FR_AB_XM_MGT_INT_MASK 0x00001250 -#define FRF_AB_XM_MSK_STA_INTR_LBN 16 -#define FRF_AB_XM_MSK_STA_INTR_WIDTH 1 -#define FRF_AB_XM_MSK_STAT_CNTR_HF_LBN 9 -#define FRF_AB_XM_MSK_STAT_CNTR_HF_WIDTH 1 -#define FRF_AB_XM_MSK_STAT_CNTR_OF_LBN 8 -#define FRF_AB_XM_MSK_STAT_CNTR_OF_WIDTH 1 -#define FRF_AB_XM_MSK_PRMBLE_ERR_LBN 2 -#define FRF_AB_XM_MSK_PRMBLE_ERR_WIDTH 1 -#define FRF_AB_XM_MSK_RMTFLT_LBN 1 -#define FRF_AB_XM_MSK_RMTFLT_WIDTH 1 -#define FRF_AB_XM_MSK_LCLFLT_LBN 0 -#define FRF_AB_XM_MSK_LCLFLT_WIDTH 1 - -/* XM_FC_REG: XGMAC flow control register */ -#define FR_AB_XM_FC 0x00001270 -#define FRF_AB_XM_PAUSE_TIME_LBN 16 -#define FRF_AB_XM_PAUSE_TIME_WIDTH 16 -#define FRF_AB_XM_RX_MAC_STAT_LBN 11 -#define FRF_AB_XM_RX_MAC_STAT_WIDTH 1 -#define FRF_AB_XM_TX_MAC_STAT_LBN 10 -#define FRF_AB_XM_TX_MAC_STAT_WIDTH 1 -#define FRF_AB_XM_MCNTL_PASS_LBN 8 -#define FRF_AB_XM_MCNTL_PASS_WIDTH 2 -#define FRF_AB_XM_REJ_CNTL_UCAST_LBN 6 -#define FRF_AB_XM_REJ_CNTL_UCAST_WIDTH 1 -#define FRF_AB_XM_REJ_CNTL_MCAST_LBN 5 -#define FRF_AB_XM_REJ_CNTL_MCAST_WIDTH 1 -#define FRF_AB_XM_ZPAUSE_LBN 2 -#define FRF_AB_XM_ZPAUSE_WIDTH 1 -#define FRF_AB_XM_XMIT_PAUSE_LBN 1 -#define FRF_AB_XM_XMIT_PAUSE_WIDTH 1 -#define FRF_AB_XM_DIS_FCNTL_LBN 0 -#define FRF_AB_XM_DIS_FCNTL_WIDTH 1 - -/* XM_PAUSE_TIME_REG: XGMAC pause time register */ -#define FR_AB_XM_PAUSE_TIME 0x00001290 -#define FRF_AB_XM_TX_PAUSE_CNT_LBN 16 -#define FRF_AB_XM_TX_PAUSE_CNT_WIDTH 16 -#define FRF_AB_XM_RX_PAUSE_CNT_LBN 0 -#define FRF_AB_XM_RX_PAUSE_CNT_WIDTH 16 - -/* XM_TX_PARAM_REG: XGMAC transmit parameter register */ -#define FR_AB_XM_TX_PARAM 0x000012d0 -#define FRF_AB_XM_TX_JUMBO_MODE_LBN 31 -#define FRF_AB_XM_TX_JUMBO_MODE_WIDTH 1 -#define FRF_AB_XM_MAX_TX_FRM_SIZE_HI_LBN 19 -#define FRF_AB_XM_MAX_TX_FRM_SIZE_HI_WIDTH 11 -#define FRF_AB_XM_MAX_TX_FRM_SIZE_LO_LBN 16 -#define FRF_AB_XM_MAX_TX_FRM_SIZE_LO_WIDTH 3 -#define FRF_AB_XM_PAD_CHAR_LBN 0 -#define FRF_AB_XM_PAD_CHAR_WIDTH 8 - -/* XM_RX_PARAM_REG: XGMAC receive parameter register */ -#define FR_AB_XM_RX_PARAM 0x000012e0 -#define FRF_AB_XM_MAX_RX_FRM_SIZE_HI_LBN 3 -#define FRF_AB_XM_MAX_RX_FRM_SIZE_HI_WIDTH 11 -#define FRF_AB_XM_MAX_RX_FRM_SIZE_LO_LBN 0 -#define FRF_AB_XM_MAX_RX_FRM_SIZE_LO_WIDTH 3 - -/* XM_MGT_INT_MSK_REG: XGMAC management interrupt mask register */ -#define FR_AB_XM_MGT_INT_MSK 0x000012f0 -#define FRF_AB_XM_STAT_CNTR_OF_LBN 9 -#define FRF_AB_XM_STAT_CNTR_OF_WIDTH 1 -#define FRF_AB_XM_STAT_CNTR_HF_LBN 8 -#define FRF_AB_XM_STAT_CNTR_HF_WIDTH 1 -#define FRF_AB_XM_PRMBLE_ERR_LBN 2 -#define FRF_AB_XM_PRMBLE_ERR_WIDTH 1 -#define FRF_AB_XM_RMTFLT_LBN 1 -#define FRF_AB_XM_RMTFLT_WIDTH 1 -#define FRF_AB_XM_LCLFLT_LBN 0 -#define FRF_AB_XM_LCLFLT_WIDTH 1 - -/* XX_PWR_RST_REG: XGXS/XAUI powerdown/reset register */ -#define FR_AB_XX_PWR_RST 0x00001300 -#define FRF_AB_XX_PWRDND_SIG_LBN 31 -#define FRF_AB_XX_PWRDND_SIG_WIDTH 1 -#define FRF_AB_XX_PWRDNC_SIG_LBN 30 -#define FRF_AB_XX_PWRDNC_SIG_WIDTH 1 -#define FRF_AB_XX_PWRDNB_SIG_LBN 29 -#define FRF_AB_XX_PWRDNB_SIG_WIDTH 1 -#define FRF_AB_XX_PWRDNA_SIG_LBN 28 -#define FRF_AB_XX_PWRDNA_SIG_WIDTH 1 -#define FRF_AB_XX_SIM_MODE_LBN 27 -#define FRF_AB_XX_SIM_MODE_WIDTH 1 -#define FRF_AB_XX_RSTPLLCD_SIG_LBN 25 -#define FRF_AB_XX_RSTPLLCD_SIG_WIDTH 1 -#define FRF_AB_XX_RSTPLLAB_SIG_LBN 24 -#define FRF_AB_XX_RSTPLLAB_SIG_WIDTH 1 -#define FRF_AB_XX_RESETD_SIG_LBN 23 -#define FRF_AB_XX_RESETD_SIG_WIDTH 1 -#define FRF_AB_XX_RESETC_SIG_LBN 22 -#define FRF_AB_XX_RESETC_SIG_WIDTH 1 -#define FRF_AB_XX_RESETB_SIG_LBN 21 -#define FRF_AB_XX_RESETB_SIG_WIDTH 1 -#define FRF_AB_XX_RESETA_SIG_LBN 20 -#define FRF_AB_XX_RESETA_SIG_WIDTH 1 -#define FRF_AB_XX_RSTXGXSRX_SIG_LBN 18 -#define FRF_AB_XX_RSTXGXSRX_SIG_WIDTH 1 -#define FRF_AB_XX_RSTXGXSTX_SIG_LBN 17 -#define FRF_AB_XX_RSTXGXSTX_SIG_WIDTH 1 -#define FRF_AB_XX_SD_RST_ACT_LBN 16 -#define FRF_AB_XX_SD_RST_ACT_WIDTH 1 -#define FRF_AB_XX_PWRDND_EN_LBN 15 -#define FRF_AB_XX_PWRDND_EN_WIDTH 1 -#define FRF_AB_XX_PWRDNC_EN_LBN 14 -#define FRF_AB_XX_PWRDNC_EN_WIDTH 1 -#define FRF_AB_XX_PWRDNB_EN_LBN 13 -#define FRF_AB_XX_PWRDNB_EN_WIDTH 1 -#define FRF_AB_XX_PWRDNA_EN_LBN 12 -#define FRF_AB_XX_PWRDNA_EN_WIDTH 1 -#define FRF_AB_XX_RSTPLLCD_EN_LBN 9 -#define FRF_AB_XX_RSTPLLCD_EN_WIDTH 1 -#define FRF_AB_XX_RSTPLLAB_EN_LBN 8 -#define FRF_AB_XX_RSTPLLAB_EN_WIDTH 1 -#define FRF_AB_XX_RESETD_EN_LBN 7 -#define FRF_AB_XX_RESETD_EN_WIDTH 1 -#define FRF_AB_XX_RESETC_EN_LBN 6 -#define FRF_AB_XX_RESETC_EN_WIDTH 1 -#define FRF_AB_XX_RESETB_EN_LBN 5 -#define FRF_AB_XX_RESETB_EN_WIDTH 1 -#define FRF_AB_XX_RESETA_EN_LBN 4 -#define FRF_AB_XX_RESETA_EN_WIDTH 1 -#define FRF_AB_XX_RSTXGXSRX_EN_LBN 2 -#define FRF_AB_XX_RSTXGXSRX_EN_WIDTH 1 -#define FRF_AB_XX_RSTXGXSTX_EN_LBN 1 -#define FRF_AB_XX_RSTXGXSTX_EN_WIDTH 1 -#define FRF_AB_XX_RST_XX_EN_LBN 0 -#define FRF_AB_XX_RST_XX_EN_WIDTH 1 - -/* XX_SD_CTL_REG: XGXS/XAUI powerdown/reset control register */ -#define FR_AB_XX_SD_CTL 0x00001310 -#define FRF_AB_XX_TERMADJ1_LBN 17 -#define FRF_AB_XX_TERMADJ1_WIDTH 1 -#define FRF_AB_XX_TERMADJ0_LBN 16 -#define FRF_AB_XX_TERMADJ0_WIDTH 1 -#define FRF_AB_XX_HIDRVD_LBN 15 -#define FRF_AB_XX_HIDRVD_WIDTH 1 -#define FRF_AB_XX_LODRVD_LBN 14 -#define FRF_AB_XX_LODRVD_WIDTH 1 -#define FRF_AB_XX_HIDRVC_LBN 13 -#define FRF_AB_XX_HIDRVC_WIDTH 1 -#define FRF_AB_XX_LODRVC_LBN 12 -#define FRF_AB_XX_LODRVC_WIDTH 1 -#define FRF_AB_XX_HIDRVB_LBN 11 -#define FRF_AB_XX_HIDRVB_WIDTH 1 -#define FRF_AB_XX_LODRVB_LBN 10 -#define FRF_AB_XX_LODRVB_WIDTH 1 -#define FRF_AB_XX_HIDRVA_LBN 9 -#define FRF_AB_XX_HIDRVA_WIDTH 1 -#define FRF_AB_XX_LODRVA_LBN 8 -#define FRF_AB_XX_LODRVA_WIDTH 1 -#define FRF_AB_XX_LPBKD_LBN 3 -#define FRF_AB_XX_LPBKD_WIDTH 1 -#define FRF_AB_XX_LPBKC_LBN 2 -#define FRF_AB_XX_LPBKC_WIDTH 1 -#define FRF_AB_XX_LPBKB_LBN 1 -#define FRF_AB_XX_LPBKB_WIDTH 1 -#define FRF_AB_XX_LPBKA_LBN 0 -#define FRF_AB_XX_LPBKA_WIDTH 1 - -/* XX_TXDRV_CTL_REG: XAUI SerDes transmit drive control register */ -#define FR_AB_XX_TXDRV_CTL 0x00001320 -#define FRF_AB_XX_DEQD_LBN 28 -#define FRF_AB_XX_DEQD_WIDTH 4 -#define FRF_AB_XX_DEQC_LBN 24 -#define FRF_AB_XX_DEQC_WIDTH 4 -#define FRF_AB_XX_DEQB_LBN 20 -#define FRF_AB_XX_DEQB_WIDTH 4 -#define FRF_AB_XX_DEQA_LBN 16 -#define FRF_AB_XX_DEQA_WIDTH 4 -#define FRF_AB_XX_DTXD_LBN 12 -#define FRF_AB_XX_DTXD_WIDTH 4 -#define FRF_AB_XX_DTXC_LBN 8 -#define FRF_AB_XX_DTXC_WIDTH 4 -#define FRF_AB_XX_DTXB_LBN 4 -#define FRF_AB_XX_DTXB_WIDTH 4 -#define FRF_AB_XX_DTXA_LBN 0 -#define FRF_AB_XX_DTXA_WIDTH 4 - -/* XX_PRBS_CTL_REG: documentation to be written for sum_XX_PRBS_CTL_REG */ -#define FR_AB_XX_PRBS_CTL 0x00001330 -#define FRF_AB_XX_CH3_RX_PRBS_SEL_LBN 30 -#define FRF_AB_XX_CH3_RX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH3_RX_PRBS_INV_LBN 29 -#define FRF_AB_XX_CH3_RX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH3_RX_PRBS_CHKEN_LBN 28 -#define FRF_AB_XX_CH3_RX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH2_RX_PRBS_SEL_LBN 26 -#define FRF_AB_XX_CH2_RX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH2_RX_PRBS_INV_LBN 25 -#define FRF_AB_XX_CH2_RX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH2_RX_PRBS_CHKEN_LBN 24 -#define FRF_AB_XX_CH2_RX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH1_RX_PRBS_SEL_LBN 22 -#define FRF_AB_XX_CH1_RX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH1_RX_PRBS_INV_LBN 21 -#define FRF_AB_XX_CH1_RX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH1_RX_PRBS_CHKEN_LBN 20 -#define FRF_AB_XX_CH1_RX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH0_RX_PRBS_SEL_LBN 18 -#define FRF_AB_XX_CH0_RX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH0_RX_PRBS_INV_LBN 17 -#define FRF_AB_XX_CH0_RX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH0_RX_PRBS_CHKEN_LBN 16 -#define FRF_AB_XX_CH0_RX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH3_TX_PRBS_SEL_LBN 14 -#define FRF_AB_XX_CH3_TX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH3_TX_PRBS_INV_LBN 13 -#define FRF_AB_XX_CH3_TX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH3_TX_PRBS_CHKEN_LBN 12 -#define FRF_AB_XX_CH3_TX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH2_TX_PRBS_SEL_LBN 10 -#define FRF_AB_XX_CH2_TX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH2_TX_PRBS_INV_LBN 9 -#define FRF_AB_XX_CH2_TX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH2_TX_PRBS_CHKEN_LBN 8 -#define FRF_AB_XX_CH2_TX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH1_TX_PRBS_SEL_LBN 6 -#define FRF_AB_XX_CH1_TX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH1_TX_PRBS_INV_LBN 5 -#define FRF_AB_XX_CH1_TX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH1_TX_PRBS_CHKEN_LBN 4 -#define FRF_AB_XX_CH1_TX_PRBS_CHKEN_WIDTH 1 -#define FRF_AB_XX_CH0_TX_PRBS_SEL_LBN 2 -#define FRF_AB_XX_CH0_TX_PRBS_SEL_WIDTH 2 -#define FRF_AB_XX_CH0_TX_PRBS_INV_LBN 1 -#define FRF_AB_XX_CH0_TX_PRBS_INV_WIDTH 1 -#define FRF_AB_XX_CH0_TX_PRBS_CHKEN_LBN 0 -#define FRF_AB_XX_CH0_TX_PRBS_CHKEN_WIDTH 1 - -/* XX_PRBS_CHK_REG: documentation to be written for sum_XX_PRBS_CHK_REG */ -#define FR_AB_XX_PRBS_CHK 0x00001340 -#define FRF_AB_XX_REV_LB_EN_LBN 16 -#define FRF_AB_XX_REV_LB_EN_WIDTH 1 -#define FRF_AB_XX_CH3_DEG_DET_LBN 15 -#define FRF_AB_XX_CH3_DEG_DET_WIDTH 1 -#define FRF_AB_XX_CH3_LFSR_LOCK_IND_LBN 14 -#define FRF_AB_XX_CH3_LFSR_LOCK_IND_WIDTH 1 -#define FRF_AB_XX_CH3_PRBS_FRUN_LBN 13 -#define FRF_AB_XX_CH3_PRBS_FRUN_WIDTH 1 -#define FRF_AB_XX_CH3_ERR_CHK_LBN 12 -#define FRF_AB_XX_CH3_ERR_CHK_WIDTH 1 -#define FRF_AB_XX_CH2_DEG_DET_LBN 11 -#define FRF_AB_XX_CH2_DEG_DET_WIDTH 1 -#define FRF_AB_XX_CH2_LFSR_LOCK_IND_LBN 10 -#define FRF_AB_XX_CH2_LFSR_LOCK_IND_WIDTH 1 -#define FRF_AB_XX_CH2_PRBS_FRUN_LBN 9 -#define FRF_AB_XX_CH2_PRBS_FRUN_WIDTH 1 -#define FRF_AB_XX_CH2_ERR_CHK_LBN 8 -#define FRF_AB_XX_CH2_ERR_CHK_WIDTH 1 -#define FRF_AB_XX_CH1_DEG_DET_LBN 7 -#define FRF_AB_XX_CH1_DEG_DET_WIDTH 1 -#define FRF_AB_XX_CH1_LFSR_LOCK_IND_LBN 6 -#define FRF_AB_XX_CH1_LFSR_LOCK_IND_WIDTH 1 -#define FRF_AB_XX_CH1_PRBS_FRUN_LBN 5 -#define FRF_AB_XX_CH1_PRBS_FRUN_WIDTH 1 -#define FRF_AB_XX_CH1_ERR_CHK_LBN 4 -#define FRF_AB_XX_CH1_ERR_CHK_WIDTH 1 -#define FRF_AB_XX_CH0_DEG_DET_LBN 3 -#define FRF_AB_XX_CH0_DEG_DET_WIDTH 1 -#define FRF_AB_XX_CH0_LFSR_LOCK_IND_LBN 2 -#define FRF_AB_XX_CH0_LFSR_LOCK_IND_WIDTH 1 -#define FRF_AB_XX_CH0_PRBS_FRUN_LBN 1 -#define FRF_AB_XX_CH0_PRBS_FRUN_WIDTH 1 -#define FRF_AB_XX_CH0_ERR_CHK_LBN 0 -#define FRF_AB_XX_CH0_ERR_CHK_WIDTH 1 - -/* XX_PRBS_ERR_REG: documentation to be written for sum_XX_PRBS_ERR_REG */ -#define FR_AB_XX_PRBS_ERR 0x00001350 -#define FRF_AB_XX_CH3_PRBS_ERR_CNT_LBN 24 -#define FRF_AB_XX_CH3_PRBS_ERR_CNT_WIDTH 8 -#define FRF_AB_XX_CH2_PRBS_ERR_CNT_LBN 16 -#define FRF_AB_XX_CH2_PRBS_ERR_CNT_WIDTH 8 -#define FRF_AB_XX_CH1_PRBS_ERR_CNT_LBN 8 -#define FRF_AB_XX_CH1_PRBS_ERR_CNT_WIDTH 8 -#define FRF_AB_XX_CH0_PRBS_ERR_CNT_LBN 0 -#define FRF_AB_XX_CH0_PRBS_ERR_CNT_WIDTH 8 - -/* XX_CORE_STAT_REG: XAUI XGXS core status register */ -#define FR_AB_XX_CORE_STAT 0x00001360 -#define FRF_AB_XX_FORCE_SIG3_LBN 31 -#define FRF_AB_XX_FORCE_SIG3_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG3_VAL_LBN 30 -#define FRF_AB_XX_FORCE_SIG3_VAL_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG2_LBN 29 -#define FRF_AB_XX_FORCE_SIG2_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG2_VAL_LBN 28 -#define FRF_AB_XX_FORCE_SIG2_VAL_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG1_LBN 27 -#define FRF_AB_XX_FORCE_SIG1_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG1_VAL_LBN 26 -#define FRF_AB_XX_FORCE_SIG1_VAL_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG0_LBN 25 -#define FRF_AB_XX_FORCE_SIG0_WIDTH 1 -#define FRF_AB_XX_FORCE_SIG0_VAL_LBN 24 -#define FRF_AB_XX_FORCE_SIG0_VAL_WIDTH 1 -#define FRF_AB_XX_XGXS_LB_EN_LBN 23 -#define FRF_AB_XX_XGXS_LB_EN_WIDTH 1 -#define FRF_AB_XX_XGMII_LB_EN_LBN 22 -#define FRF_AB_XX_XGMII_LB_EN_WIDTH 1 -#define FRF_AB_XX_MATCH_FAULT_LBN 21 -#define FRF_AB_XX_MATCH_FAULT_WIDTH 1 -#define FRF_AB_XX_ALIGN_DONE_LBN 20 -#define FRF_AB_XX_ALIGN_DONE_WIDTH 1 -#define FRF_AB_XX_SYNC_STAT3_LBN 19 -#define FRF_AB_XX_SYNC_STAT3_WIDTH 1 -#define FRF_AB_XX_SYNC_STAT2_LBN 18 -#define FRF_AB_XX_SYNC_STAT2_WIDTH 1 -#define FRF_AB_XX_SYNC_STAT1_LBN 17 -#define FRF_AB_XX_SYNC_STAT1_WIDTH 1 -#define FRF_AB_XX_SYNC_STAT0_LBN 16 -#define FRF_AB_XX_SYNC_STAT0_WIDTH 1 -#define FRF_AB_XX_COMMA_DET_CH3_LBN 15 -#define FRF_AB_XX_COMMA_DET_CH3_WIDTH 1 -#define FRF_AB_XX_COMMA_DET_CH2_LBN 14 -#define FRF_AB_XX_COMMA_DET_CH2_WIDTH 1 -#define FRF_AB_XX_COMMA_DET_CH1_LBN 13 -#define FRF_AB_XX_COMMA_DET_CH1_WIDTH 1 -#define FRF_AB_XX_COMMA_DET_CH0_LBN 12 -#define FRF_AB_XX_COMMA_DET_CH0_WIDTH 1 -#define FRF_AB_XX_CGRP_ALIGN_CH3_LBN 11 -#define FRF_AB_XX_CGRP_ALIGN_CH3_WIDTH 1 -#define FRF_AB_XX_CGRP_ALIGN_CH2_LBN 10 -#define FRF_AB_XX_CGRP_ALIGN_CH2_WIDTH 1 -#define FRF_AB_XX_CGRP_ALIGN_CH1_LBN 9 -#define FRF_AB_XX_CGRP_ALIGN_CH1_WIDTH 1 -#define FRF_AB_XX_CGRP_ALIGN_CH0_LBN 8 -#define FRF_AB_XX_CGRP_ALIGN_CH0_WIDTH 1 -#define FRF_AB_XX_CHAR_ERR_CH3_LBN 7 -#define FRF_AB_XX_CHAR_ERR_CH3_WIDTH 1 -#define FRF_AB_XX_CHAR_ERR_CH2_LBN 6 -#define FRF_AB_XX_CHAR_ERR_CH2_WIDTH 1 -#define FRF_AB_XX_CHAR_ERR_CH1_LBN 5 -#define FRF_AB_XX_CHAR_ERR_CH1_WIDTH 1 -#define FRF_AB_XX_CHAR_ERR_CH0_LBN 4 -#define FRF_AB_XX_CHAR_ERR_CH0_WIDTH 1 -#define FRF_AB_XX_DISPERR_CH3_LBN 3 -#define FRF_AB_XX_DISPERR_CH3_WIDTH 1 -#define FRF_AB_XX_DISPERR_CH2_LBN 2 -#define FRF_AB_XX_DISPERR_CH2_WIDTH 1 -#define FRF_AB_XX_DISPERR_CH1_LBN 1 -#define FRF_AB_XX_DISPERR_CH1_WIDTH 1 -#define FRF_AB_XX_DISPERR_CH0_LBN 0 -#define FRF_AB_XX_DISPERR_CH0_WIDTH 1 - -/* RX_DESC_PTR_TBL_KER: Receive descriptor pointer table */ -#define FR_AA_RX_DESC_PTR_TBL_KER 0x00011800 -#define FR_AA_RX_DESC_PTR_TBL_KER_STEP 16 -#define FR_AA_RX_DESC_PTR_TBL_KER_ROWS 4 -/* RX_DESC_PTR_TBL: Receive descriptor pointer table */ -#define FR_BZ_RX_DESC_PTR_TBL 0x00f40000 -#define FR_BZ_RX_DESC_PTR_TBL_STEP 16 -#define FR_BB_RX_DESC_PTR_TBL_ROWS 4096 -#define FR_CZ_RX_DESC_PTR_TBL_ROWS 1024 -#define FRF_CZ_RX_HDR_SPLIT_LBN 90 -#define FRF_CZ_RX_HDR_SPLIT_WIDTH 1 -#define FRF_AA_RX_RESET_LBN 89 -#define FRF_AA_RX_RESET_WIDTH 1 -#define FRF_AZ_RX_ISCSI_DDIG_EN_LBN 88 -#define FRF_AZ_RX_ISCSI_DDIG_EN_WIDTH 1 -#define FRF_AZ_RX_ISCSI_HDIG_EN_LBN 87 -#define FRF_AZ_RX_ISCSI_HDIG_EN_WIDTH 1 -#define FRF_AZ_RX_DESC_PREF_ACT_LBN 86 -#define FRF_AZ_RX_DESC_PREF_ACT_WIDTH 1 -#define FRF_AZ_RX_DC_HW_RPTR_LBN 80 -#define FRF_AZ_RX_DC_HW_RPTR_WIDTH 6 -#define FRF_AZ_RX_DESCQ_HW_RPTR_LBN 68 -#define FRF_AZ_RX_DESCQ_HW_RPTR_WIDTH 12 -#define FRF_AZ_RX_DESCQ_SW_WPTR_LBN 56 -#define FRF_AZ_RX_DESCQ_SW_WPTR_WIDTH 12 -#define FRF_AZ_RX_DESCQ_BUF_BASE_ID_LBN 36 -#define FRF_AZ_RX_DESCQ_BUF_BASE_ID_WIDTH 20 -#define FRF_AZ_RX_DESCQ_EVQ_ID_LBN 24 -#define FRF_AZ_RX_DESCQ_EVQ_ID_WIDTH 12 -#define FRF_AZ_RX_DESCQ_OWNER_ID_LBN 10 -#define FRF_AZ_RX_DESCQ_OWNER_ID_WIDTH 14 -#define FRF_AZ_RX_DESCQ_LABEL_LBN 5 -#define FRF_AZ_RX_DESCQ_LABEL_WIDTH 5 -#define FRF_AZ_RX_DESCQ_SIZE_LBN 3 -#define FRF_AZ_RX_DESCQ_SIZE_WIDTH 2 -#define FFE_AZ_RX_DESCQ_SIZE_4K 3 -#define FFE_AZ_RX_DESCQ_SIZE_2K 2 -#define FFE_AZ_RX_DESCQ_SIZE_1K 1 -#define FFE_AZ_RX_DESCQ_SIZE_512 0 -#define FRF_AZ_RX_DESCQ_TYPE_LBN 2 -#define FRF_AZ_RX_DESCQ_TYPE_WIDTH 1 -#define FRF_AZ_RX_DESCQ_JUMBO_LBN 1 -#define FRF_AZ_RX_DESCQ_JUMBO_WIDTH 1 -#define FRF_AZ_RX_DESCQ_EN_LBN 0 -#define FRF_AZ_RX_DESCQ_EN_WIDTH 1 - -/* TX_DESC_PTR_TBL_KER: Transmit descriptor pointer */ -#define FR_AA_TX_DESC_PTR_TBL_KER 0x00011900 -#define FR_AA_TX_DESC_PTR_TBL_KER_STEP 16 -#define FR_AA_TX_DESC_PTR_TBL_KER_ROWS 8 -/* TX_DESC_PTR_TBL: Transmit descriptor pointer */ -#define FR_BZ_TX_DESC_PTR_TBL 0x00f50000 -#define FR_BZ_TX_DESC_PTR_TBL_STEP 16 -#define FR_BB_TX_DESC_PTR_TBL_ROWS 4096 -#define FR_CZ_TX_DESC_PTR_TBL_ROWS 1024 -#define FRF_CZ_TX_DPT_Q_MASK_WIDTH_LBN 94 -#define FRF_CZ_TX_DPT_Q_MASK_WIDTH_WIDTH 2 -#define FRF_CZ_TX_DPT_ETH_FILT_EN_LBN 93 -#define FRF_CZ_TX_DPT_ETH_FILT_EN_WIDTH 1 -#define FRF_CZ_TX_DPT_IP_FILT_EN_LBN 92 -#define FRF_CZ_TX_DPT_IP_FILT_EN_WIDTH 1 -#define FRF_BZ_TX_NON_IP_DROP_DIS_LBN 91 -#define FRF_BZ_TX_NON_IP_DROP_DIS_WIDTH 1 -#define FRF_BZ_TX_IP_CHKSM_DIS_LBN 90 -#define FRF_BZ_TX_IP_CHKSM_DIS_WIDTH 1 -#define FRF_BZ_TX_TCP_CHKSM_DIS_LBN 89 -#define FRF_BZ_TX_TCP_CHKSM_DIS_WIDTH 1 -#define FRF_AZ_TX_DESCQ_EN_LBN 88 -#define FRF_AZ_TX_DESCQ_EN_WIDTH 1 -#define FRF_AZ_TX_ISCSI_DDIG_EN_LBN 87 -#define FRF_AZ_TX_ISCSI_DDIG_EN_WIDTH 1 -#define FRF_AZ_TX_ISCSI_HDIG_EN_LBN 86 -#define FRF_AZ_TX_ISCSI_HDIG_EN_WIDTH 1 -#define FRF_AZ_TX_DC_HW_RPTR_LBN 80 -#define FRF_AZ_TX_DC_HW_RPTR_WIDTH 6 -#define FRF_AZ_TX_DESCQ_HW_RPTR_LBN 68 -#define FRF_AZ_TX_DESCQ_HW_RPTR_WIDTH 12 -#define FRF_AZ_TX_DESCQ_SW_WPTR_LBN 56 -#define FRF_AZ_TX_DESCQ_SW_WPTR_WIDTH 12 -#define FRF_AZ_TX_DESCQ_BUF_BASE_ID_LBN 36 -#define FRF_AZ_TX_DESCQ_BUF_BASE_ID_WIDTH 20 -#define FRF_AZ_TX_DESCQ_EVQ_ID_LBN 24 -#define FRF_AZ_TX_DESCQ_EVQ_ID_WIDTH 12 -#define FRF_AZ_TX_DESCQ_OWNER_ID_LBN 10 -#define FRF_AZ_TX_DESCQ_OWNER_ID_WIDTH 14 -#define FRF_AZ_TX_DESCQ_LABEL_LBN 5 -#define FRF_AZ_TX_DESCQ_LABEL_WIDTH 5 -#define FRF_AZ_TX_DESCQ_SIZE_LBN 3 -#define FRF_AZ_TX_DESCQ_SIZE_WIDTH 2 -#define FFE_AZ_TX_DESCQ_SIZE_4K 3 -#define FFE_AZ_TX_DESCQ_SIZE_2K 2 -#define FFE_AZ_TX_DESCQ_SIZE_1K 1 -#define FFE_AZ_TX_DESCQ_SIZE_512 0 -#define FRF_AZ_TX_DESCQ_TYPE_LBN 1 -#define FRF_AZ_TX_DESCQ_TYPE_WIDTH 2 -#define FRF_AZ_TX_DESCQ_FLUSH_LBN 0 -#define FRF_AZ_TX_DESCQ_FLUSH_WIDTH 1 - -/* EVQ_PTR_TBL_KER: Event queue pointer table */ -#define FR_AA_EVQ_PTR_TBL_KER 0x00011a00 -#define FR_AA_EVQ_PTR_TBL_KER_STEP 16 -#define FR_AA_EVQ_PTR_TBL_KER_ROWS 4 -/* EVQ_PTR_TBL: Event queue pointer table */ -#define FR_BZ_EVQ_PTR_TBL 0x00f60000 -#define FR_BZ_EVQ_PTR_TBL_STEP 16 -#define FR_CZ_EVQ_PTR_TBL_ROWS 1024 -#define FR_BB_EVQ_PTR_TBL_ROWS 4096 -#define FRF_BZ_EVQ_RPTR_IGN_LBN 40 -#define FRF_BZ_EVQ_RPTR_IGN_WIDTH 1 -#define FRF_AB_EVQ_WKUP_OR_INT_EN_LBN 39 -#define FRF_AB_EVQ_WKUP_OR_INT_EN_WIDTH 1 -#define FRF_CZ_EVQ_DOS_PROTECT_EN_LBN 39 -#define FRF_CZ_EVQ_DOS_PROTECT_EN_WIDTH 1 -#define FRF_AZ_EVQ_NXT_WPTR_LBN 24 -#define FRF_AZ_EVQ_NXT_WPTR_WIDTH 15 -#define FRF_AZ_EVQ_EN_LBN 23 -#define FRF_AZ_EVQ_EN_WIDTH 1 -#define FRF_AZ_EVQ_SIZE_LBN 20 -#define FRF_AZ_EVQ_SIZE_WIDTH 3 -#define FFE_AZ_EVQ_SIZE_32K 6 -#define FFE_AZ_EVQ_SIZE_16K 5 -#define FFE_AZ_EVQ_SIZE_8K 4 -#define FFE_AZ_EVQ_SIZE_4K 3 -#define FFE_AZ_EVQ_SIZE_2K 2 -#define FFE_AZ_EVQ_SIZE_1K 1 -#define FFE_AZ_EVQ_SIZE_512 0 -#define FRF_AZ_EVQ_BUF_BASE_ID_LBN 0 -#define FRF_AZ_EVQ_BUF_BASE_ID_WIDTH 20 - -/* BUF_HALF_TBL_KER: Buffer table in half buffer table mode direct access by driver */ -#define FR_AA_BUF_HALF_TBL_KER 0x00018000 -#define FR_AA_BUF_HALF_TBL_KER_STEP 8 -#define FR_AA_BUF_HALF_TBL_KER_ROWS 4096 -/* BUF_HALF_TBL: Buffer table in half buffer table mode direct access by driver */ -#define FR_BZ_BUF_HALF_TBL 0x00800000 -#define FR_BZ_BUF_HALF_TBL_STEP 8 -#define FR_CZ_BUF_HALF_TBL_ROWS 147456 -#define FR_BB_BUF_HALF_TBL_ROWS 524288 -#define FRF_AZ_BUF_ADR_HBUF_ODD_LBN 44 -#define FRF_AZ_BUF_ADR_HBUF_ODD_WIDTH 20 -#define FRF_AZ_BUF_OWNER_ID_HBUF_ODD_LBN 32 -#define FRF_AZ_BUF_OWNER_ID_HBUF_ODD_WIDTH 12 -#define FRF_AZ_BUF_ADR_HBUF_EVEN_LBN 12 -#define FRF_AZ_BUF_ADR_HBUF_EVEN_WIDTH 20 -#define FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_LBN 0 -#define FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_WIDTH 12 - -/* BUF_FULL_TBL_KER: Buffer table in full buffer table mode direct access by driver */ -#define FR_AA_BUF_FULL_TBL_KER 0x00018000 -#define FR_AA_BUF_FULL_TBL_KER_STEP 8 -#define FR_AA_BUF_FULL_TBL_KER_ROWS 4096 -/* BUF_FULL_TBL: Buffer table in full buffer table mode direct access by driver */ -#define FR_BZ_BUF_FULL_TBL 0x00800000 -#define FR_BZ_BUF_FULL_TBL_STEP 8 -#define FR_CZ_BUF_FULL_TBL_ROWS 147456 -#define FR_BB_BUF_FULL_TBL_ROWS 917504 -#define FRF_AZ_BUF_FULL_UNUSED_LBN 51 -#define FRF_AZ_BUF_FULL_UNUSED_WIDTH 13 -#define FRF_AZ_IP_DAT_BUF_SIZE_LBN 50 -#define FRF_AZ_IP_DAT_BUF_SIZE_WIDTH 1 -#define FRF_AZ_BUF_ADR_REGION_LBN 48 -#define FRF_AZ_BUF_ADR_REGION_WIDTH 2 -#define FFE_AZ_BUF_ADR_REGN3 3 -#define FFE_AZ_BUF_ADR_REGN2 2 -#define FFE_AZ_BUF_ADR_REGN1 1 -#define FFE_AZ_BUF_ADR_REGN0 0 -#define FRF_AZ_BUF_ADR_FBUF_LBN 14 -#define FRF_AZ_BUF_ADR_FBUF_WIDTH 34 -#define FRF_AZ_BUF_OWNER_ID_FBUF_LBN 0 -#define FRF_AZ_BUF_OWNER_ID_FBUF_WIDTH 14 - -/* RX_FILTER_TBL0: TCP/IPv4 Receive filter table */ -#define FR_BZ_RX_FILTER_TBL0 0x00f00000 -#define FR_BZ_RX_FILTER_TBL0_STEP 32 -#define FR_BZ_RX_FILTER_TBL0_ROWS 8192 -/* RX_FILTER_TBL1: TCP/IPv4 Receive filter table */ -#define FR_BB_RX_FILTER_TBL1 0x00f00010 -#define FR_BB_RX_FILTER_TBL1_STEP 32 -#define FR_BB_RX_FILTER_TBL1_ROWS 8192 -#define FRF_BZ_RSS_EN_LBN 110 -#define FRF_BZ_RSS_EN_WIDTH 1 -#define FRF_BZ_SCATTER_EN_LBN 109 -#define FRF_BZ_SCATTER_EN_WIDTH 1 -#define FRF_BZ_TCP_UDP_LBN 108 -#define FRF_BZ_TCP_UDP_WIDTH 1 -#define FRF_BZ_RXQ_ID_LBN 96 -#define FRF_BZ_RXQ_ID_WIDTH 12 -#define FRF_BZ_DEST_IP_LBN 64 -#define FRF_BZ_DEST_IP_WIDTH 32 -#define FRF_BZ_DEST_PORT_TCP_LBN 48 -#define FRF_BZ_DEST_PORT_TCP_WIDTH 16 -#define FRF_BZ_SRC_IP_LBN 16 -#define FRF_BZ_SRC_IP_WIDTH 32 -#define FRF_BZ_SRC_TCP_DEST_UDP_LBN 0 -#define FRF_BZ_SRC_TCP_DEST_UDP_WIDTH 16 - -/* RX_MAC_FILTER_TBL0: Receive Ethernet filter table */ -#define FR_CZ_RX_MAC_FILTER_TBL0 0x00f00010 -#define FR_CZ_RX_MAC_FILTER_TBL0_STEP 32 -#define FR_CZ_RX_MAC_FILTER_TBL0_ROWS 512 -#define FRF_CZ_RMFT_RSS_EN_LBN 75 -#define FRF_CZ_RMFT_RSS_EN_WIDTH 1 -#define FRF_CZ_RMFT_SCATTER_EN_LBN 74 -#define FRF_CZ_RMFT_SCATTER_EN_WIDTH 1 -#define FRF_CZ_RMFT_IP_OVERRIDE_LBN 73 -#define FRF_CZ_RMFT_IP_OVERRIDE_WIDTH 1 -#define FRF_CZ_RMFT_RXQ_ID_LBN 61 -#define FRF_CZ_RMFT_RXQ_ID_WIDTH 12 -#define FRF_CZ_RMFT_WILDCARD_MATCH_LBN 60 -#define FRF_CZ_RMFT_WILDCARD_MATCH_WIDTH 1 -#define FRF_CZ_RMFT_DEST_MAC_LBN 12 -#define FRF_CZ_RMFT_DEST_MAC_WIDTH 48 -#define FRF_CZ_RMFT_VLAN_ID_LBN 0 -#define FRF_CZ_RMFT_VLAN_ID_WIDTH 12 - -/* TIMER_TBL: Timer table */ -#define FR_BZ_TIMER_TBL 0x00f70000 -#define FR_BZ_TIMER_TBL_STEP 16 -#define FR_CZ_TIMER_TBL_ROWS 1024 -#define FR_BB_TIMER_TBL_ROWS 4096 -#define FRF_CZ_TIMER_Q_EN_LBN 33 -#define FRF_CZ_TIMER_Q_EN_WIDTH 1 -#define FRF_CZ_INT_ARMD_LBN 32 -#define FRF_CZ_INT_ARMD_WIDTH 1 -#define FRF_CZ_INT_PEND_LBN 31 -#define FRF_CZ_INT_PEND_WIDTH 1 -#define FRF_CZ_HOST_NOTIFY_MODE_LBN 30 -#define FRF_CZ_HOST_NOTIFY_MODE_WIDTH 1 -#define FRF_CZ_RELOAD_TIMER_VAL_LBN 16 -#define FRF_CZ_RELOAD_TIMER_VAL_WIDTH 14 -#define FRF_CZ_TIMER_MODE_LBN 14 -#define FRF_CZ_TIMER_MODE_WIDTH 2 -#define FFE_CZ_TIMER_MODE_INT_HLDOFF 3 -#define FFE_CZ_TIMER_MODE_TRIG_START 2 -#define FFE_CZ_TIMER_MODE_IMMED_START 1 -#define FFE_CZ_TIMER_MODE_DIS 0 -#define FRF_BB_TIMER_MODE_LBN 12 -#define FRF_BB_TIMER_MODE_WIDTH 2 -#define FFE_BB_TIMER_MODE_INT_HLDOFF 2 -#define FFE_BB_TIMER_MODE_TRIG_START 2 -#define FFE_BB_TIMER_MODE_IMMED_START 1 -#define FFE_BB_TIMER_MODE_DIS 0 -#define FRF_CZ_TIMER_VAL_LBN 0 -#define FRF_CZ_TIMER_VAL_WIDTH 14 -#define FRF_BB_TIMER_VAL_LBN 0 -#define FRF_BB_TIMER_VAL_WIDTH 12 - -/* TX_PACE_TBL: Transmit pacing table */ -#define FR_BZ_TX_PACE_TBL 0x00f80000 -#define FR_BZ_TX_PACE_TBL_STEP 16 -#define FR_CZ_TX_PACE_TBL_ROWS 1024 -#define FR_BB_TX_PACE_TBL_ROWS 4096 -#define FRF_BZ_TX_PACE_LBN 0 -#define FRF_BZ_TX_PACE_WIDTH 5 - -/* RX_INDIRECTION_TBL: RX Indirection Table */ -#define FR_BZ_RX_INDIRECTION_TBL 0x00fb0000 -#define FR_BZ_RX_INDIRECTION_TBL_STEP 16 -#define FR_BZ_RX_INDIRECTION_TBL_ROWS 128 -#define FRF_BZ_IT_QUEUE_LBN 0 -#define FRF_BZ_IT_QUEUE_WIDTH 6 - -/* TX_FILTER_TBL0: TCP/IPv4 Transmit filter table */ -#define FR_CZ_TX_FILTER_TBL0 0x00fc0000 -#define FR_CZ_TX_FILTER_TBL0_STEP 16 -#define FR_CZ_TX_FILTER_TBL0_ROWS 8192 -#define FRF_CZ_TIFT_TCP_UDP_LBN 108 -#define FRF_CZ_TIFT_TCP_UDP_WIDTH 1 -#define FRF_CZ_TIFT_TXQ_ID_LBN 96 -#define FRF_CZ_TIFT_TXQ_ID_WIDTH 12 -#define FRF_CZ_TIFT_DEST_IP_LBN 64 -#define FRF_CZ_TIFT_DEST_IP_WIDTH 32 -#define FRF_CZ_TIFT_DEST_PORT_TCP_LBN 48 -#define FRF_CZ_TIFT_DEST_PORT_TCP_WIDTH 16 -#define FRF_CZ_TIFT_SRC_IP_LBN 16 -#define FRF_CZ_TIFT_SRC_IP_WIDTH 32 -#define FRF_CZ_TIFT_SRC_TCP_DEST_UDP_LBN 0 -#define FRF_CZ_TIFT_SRC_TCP_DEST_UDP_WIDTH 16 - -/* TX_MAC_FILTER_TBL0: Transmit Ethernet filter table */ -#define FR_CZ_TX_MAC_FILTER_TBL0 0x00fe0000 -#define FR_CZ_TX_MAC_FILTER_TBL0_STEP 16 -#define FR_CZ_TX_MAC_FILTER_TBL0_ROWS 512 -#define FRF_CZ_TMFT_TXQ_ID_LBN 61 -#define FRF_CZ_TMFT_TXQ_ID_WIDTH 12 -#define FRF_CZ_TMFT_WILDCARD_MATCH_LBN 60 -#define FRF_CZ_TMFT_WILDCARD_MATCH_WIDTH 1 -#define FRF_CZ_TMFT_SRC_MAC_LBN 12 -#define FRF_CZ_TMFT_SRC_MAC_WIDTH 48 -#define FRF_CZ_TMFT_VLAN_ID_LBN 0 -#define FRF_CZ_TMFT_VLAN_ID_WIDTH 12 - -/* MC_TREG_SMEM: MC Shared Memory */ -#define FR_CZ_MC_TREG_SMEM 0x00ff0000 -#define FR_CZ_MC_TREG_SMEM_STEP 4 -#define FR_CZ_MC_TREG_SMEM_ROWS 512 -#define FRF_CZ_MC_TREG_SMEM_ROW_LBN 0 -#define FRF_CZ_MC_TREG_SMEM_ROW_WIDTH 32 - -/* MSIX_VECTOR_TABLE: MSIX Vector Table */ -#define FR_BB_MSIX_VECTOR_TABLE 0x00ff0000 -#define FR_BZ_MSIX_VECTOR_TABLE_STEP 16 -#define FR_BB_MSIX_VECTOR_TABLE_ROWS 64 -/* MSIX_VECTOR_TABLE: MSIX Vector Table */ -#define FR_CZ_MSIX_VECTOR_TABLE 0x00000000 -/* FR_BZ_MSIX_VECTOR_TABLE_STEP 16 */ -#define FR_CZ_MSIX_VECTOR_TABLE_ROWS 1024 -#define FRF_BZ_MSIX_VECTOR_RESERVED_LBN 97 -#define FRF_BZ_MSIX_VECTOR_RESERVED_WIDTH 31 -#define FRF_BZ_MSIX_VECTOR_MASK_LBN 96 -#define FRF_BZ_MSIX_VECTOR_MASK_WIDTH 1 -#define FRF_BZ_MSIX_MESSAGE_DATA_LBN 64 -#define FRF_BZ_MSIX_MESSAGE_DATA_WIDTH 32 -#define FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_LBN 32 -#define FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_WIDTH 32 -#define FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_LBN 0 -#define FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_WIDTH 32 - -/* MSIX_PBA_TABLE: MSIX Pending Bit Array */ -#define FR_BB_MSIX_PBA_TABLE 0x00ff2000 -#define FR_BZ_MSIX_PBA_TABLE_STEP 4 -#define FR_BB_MSIX_PBA_TABLE_ROWS 2 -/* MSIX_PBA_TABLE: MSIX Pending Bit Array */ -#define FR_CZ_MSIX_PBA_TABLE 0x00008000 -/* FR_BZ_MSIX_PBA_TABLE_STEP 4 */ -#define FR_CZ_MSIX_PBA_TABLE_ROWS 32 -#define FRF_BZ_MSIX_PBA_PEND_DWORD_LBN 0 -#define FRF_BZ_MSIX_PBA_PEND_DWORD_WIDTH 32 - -/* SRM_DBG_REG: SRAM debug access */ -#define FR_BZ_SRM_DBG 0x03000000 -#define FR_BZ_SRM_DBG_STEP 8 -#define FR_CZ_SRM_DBG_ROWS 262144 -#define FR_BB_SRM_DBG_ROWS 2097152 -#define FRF_BZ_SRM_DBG_LBN 0 -#define FRF_BZ_SRM_DBG_WIDTH 64 - -/* TB_MSIX_PBA_TABLE: MSIX Pending Bit Array */ -#define FR_CZ_TB_MSIX_PBA_TABLE 0x00008000 -#define FR_CZ_TB_MSIX_PBA_TABLE_STEP 4 -#define FR_CZ_TB_MSIX_PBA_TABLE_ROWS 1024 -#define FRF_CZ_TB_MSIX_PBA_PEND_DWORD_LBN 0 -#define FRF_CZ_TB_MSIX_PBA_PEND_DWORD_WIDTH 32 - -/* DRIVER_EV */ -#define FSF_AZ_DRIVER_EV_SUBCODE_LBN 56 -#define FSF_AZ_DRIVER_EV_SUBCODE_WIDTH 4 -#define FSE_BZ_TX_DSC_ERROR_EV 15 -#define FSE_BZ_RX_DSC_ERROR_EV 14 -#define FSE_AA_RX_RECOVER_EV 11 -#define FSE_AZ_TIMER_EV 10 -#define FSE_AZ_TX_PKT_NON_TCP_UDP 9 -#define FSE_AZ_WAKE_UP_EV 6 -#define FSE_AZ_SRM_UPD_DONE_EV 5 -#define FSE_AB_EVQ_NOT_EN_EV 3 -#define FSE_AZ_EVQ_INIT_DONE_EV 2 -#define FSE_AZ_RX_DESCQ_FLS_DONE_EV 1 -#define FSE_AZ_TX_DESCQ_FLS_DONE_EV 0 -#define FSF_AZ_DRIVER_EV_SUBDATA_LBN 0 -#define FSF_AZ_DRIVER_EV_SUBDATA_WIDTH 14 - -/* EVENT_ENTRY */ -#define FSF_AZ_EV_CODE_LBN 60 -#define FSF_AZ_EV_CODE_WIDTH 4 -#define FSE_CZ_EV_CODE_MCDI_EV 12 -#define FSE_CZ_EV_CODE_USER_EV 8 -#define FSE_AZ_EV_CODE_DRV_GEN_EV 7 -#define FSE_AZ_EV_CODE_GLOBAL_EV 6 -#define FSE_AZ_EV_CODE_DRIVER_EV 5 -#define FSE_AZ_EV_CODE_TX_EV 2 -#define FSE_AZ_EV_CODE_RX_EV 0 -#define FSF_AZ_EV_DATA_LBN 0 -#define FSF_AZ_EV_DATA_WIDTH 60 - -/* GLOBAL_EV */ -#define FSF_BB_GLB_EV_RX_RECOVERY_LBN 12 -#define FSF_BB_GLB_EV_RX_RECOVERY_WIDTH 1 -#define FSF_AA_GLB_EV_RX_RECOVERY_LBN 11 -#define FSF_AA_GLB_EV_RX_RECOVERY_WIDTH 1 -#define FSF_BB_GLB_EV_XG_MGT_INTR_LBN 11 -#define FSF_BB_GLB_EV_XG_MGT_INTR_WIDTH 1 -#define FSF_AB_GLB_EV_XFP_PHY0_INTR_LBN 10 -#define FSF_AB_GLB_EV_XFP_PHY0_INTR_WIDTH 1 -#define FSF_AB_GLB_EV_XG_PHY0_INTR_LBN 9 -#define FSF_AB_GLB_EV_XG_PHY0_INTR_WIDTH 1 -#define FSF_AB_GLB_EV_G_PHY0_INTR_LBN 7 -#define FSF_AB_GLB_EV_G_PHY0_INTR_WIDTH 1 - -/* LEGACY_INT_VEC */ -#define FSF_AZ_NET_IVEC_FATAL_INT_LBN 64 -#define FSF_AZ_NET_IVEC_FATAL_INT_WIDTH 1 -#define FSF_AZ_NET_IVEC_INT_Q_LBN 40 -#define FSF_AZ_NET_IVEC_INT_Q_WIDTH 4 -#define FSF_AZ_NET_IVEC_INT_FLAG_LBN 32 -#define FSF_AZ_NET_IVEC_INT_FLAG_WIDTH 1 -#define FSF_AZ_NET_IVEC_EVQ_FIFO_HF_LBN 1 -#define FSF_AZ_NET_IVEC_EVQ_FIFO_HF_WIDTH 1 -#define FSF_AZ_NET_IVEC_EVQ_FIFO_AF_LBN 0 -#define FSF_AZ_NET_IVEC_EVQ_FIFO_AF_WIDTH 1 - -/* MC_XGMAC_FLTR_RULE_DEF */ -#define FSF_CZ_MC_XFRC_MODE_LBN 416 -#define FSF_CZ_MC_XFRC_MODE_WIDTH 1 -#define FSE_CZ_MC_XFRC_MODE_LAYERED 1 -#define FSE_CZ_MC_XFRC_MODE_SIMPLE 0 -#define FSF_CZ_MC_XFRC_HASH_LBN 384 -#define FSF_CZ_MC_XFRC_HASH_WIDTH 32 -#define FSF_CZ_MC_XFRC_LAYER4_BYTE_MASK_LBN 256 -#define FSF_CZ_MC_XFRC_LAYER4_BYTE_MASK_WIDTH 128 -#define FSF_CZ_MC_XFRC_LAYER3_BYTE_MASK_LBN 128 -#define FSF_CZ_MC_XFRC_LAYER3_BYTE_MASK_WIDTH 128 -#define FSF_CZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_LBN 0 -#define FSF_CZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_WIDTH 128 - -/* RX_EV */ -#define FSF_CZ_RX_EV_PKT_NOT_PARSED_LBN 58 -#define FSF_CZ_RX_EV_PKT_NOT_PARSED_WIDTH 1 -#define FSF_CZ_RX_EV_IPV6_PKT_LBN 57 -#define FSF_CZ_RX_EV_IPV6_PKT_WIDTH 1 -#define FSF_AZ_RX_EV_PKT_OK_LBN 56 -#define FSF_AZ_RX_EV_PKT_OK_WIDTH 1 -#define FSF_AZ_RX_EV_PAUSE_FRM_ERR_LBN 55 -#define FSF_AZ_RX_EV_PAUSE_FRM_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_LBN 54 -#define FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_IP_FRAG_ERR_LBN 53 -#define FSF_AZ_RX_EV_IP_FRAG_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_LBN 52 -#define FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_LBN 51 -#define FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_ETH_CRC_ERR_LBN 50 -#define FSF_AZ_RX_EV_ETH_CRC_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_FRM_TRUNC_LBN 49 -#define FSF_AZ_RX_EV_FRM_TRUNC_WIDTH 1 -#define FSF_AA_RX_EV_DRIB_NIB_LBN 49 -#define FSF_AA_RX_EV_DRIB_NIB_WIDTH 1 -#define FSF_AZ_RX_EV_TOBE_DISC_LBN 47 -#define FSF_AZ_RX_EV_TOBE_DISC_WIDTH 1 -#define FSF_AZ_RX_EV_PKT_TYPE_LBN 44 -#define FSF_AZ_RX_EV_PKT_TYPE_WIDTH 3 -#define FSE_AZ_RX_EV_PKT_TYPE_VLAN_JUMBO 5 -#define FSE_AZ_RX_EV_PKT_TYPE_VLAN_LLC 4 -#define FSE_AZ_RX_EV_PKT_TYPE_VLAN 3 -#define FSE_AZ_RX_EV_PKT_TYPE_JUMBO 2 -#define FSE_AZ_RX_EV_PKT_TYPE_LLC 1 -#define FSE_AZ_RX_EV_PKT_TYPE_ETH 0 -#define FSF_AZ_RX_EV_HDR_TYPE_LBN 42 -#define FSF_AZ_RX_EV_HDR_TYPE_WIDTH 2 -#define FSE_AZ_RX_EV_HDR_TYPE_OTHER 3 -#define FSE_AB_RX_EV_HDR_TYPE_IPV4_OTHER 2 -#define FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_OTHER 2 -#define FSE_AB_RX_EV_HDR_TYPE_IPV4_UDP 1 -#define FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP 1 -#define FSE_AB_RX_EV_HDR_TYPE_IPV4_TCP 0 -#define FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP 0 -#define FSF_AZ_RX_EV_DESC_Q_EMPTY_LBN 41 -#define FSF_AZ_RX_EV_DESC_Q_EMPTY_WIDTH 1 -#define FSF_AZ_RX_EV_MCAST_HASH_MATCH_LBN 40 -#define FSF_AZ_RX_EV_MCAST_HASH_MATCH_WIDTH 1 -#define FSF_AZ_RX_EV_MCAST_PKT_LBN 39 -#define FSF_AZ_RX_EV_MCAST_PKT_WIDTH 1 -#define FSF_AA_RX_EV_RECOVERY_FLAG_LBN 37 -#define FSF_AA_RX_EV_RECOVERY_FLAG_WIDTH 1 -#define FSF_AZ_RX_EV_Q_LABEL_LBN 32 -#define FSF_AZ_RX_EV_Q_LABEL_WIDTH 5 -#define FSF_AZ_RX_EV_JUMBO_CONT_LBN 31 -#define FSF_AZ_RX_EV_JUMBO_CONT_WIDTH 1 -#define FSF_AZ_RX_EV_PORT_LBN 30 -#define FSF_AZ_RX_EV_PORT_WIDTH 1 -#define FSF_AZ_RX_EV_BYTE_CNT_LBN 16 -#define FSF_AZ_RX_EV_BYTE_CNT_WIDTH 14 -#define FSF_AZ_RX_EV_SOP_LBN 15 -#define FSF_AZ_RX_EV_SOP_WIDTH 1 -#define FSF_AZ_RX_EV_ISCSI_PKT_OK_LBN 14 -#define FSF_AZ_RX_EV_ISCSI_PKT_OK_WIDTH 1 -#define FSF_AZ_RX_EV_ISCSI_DDIG_ERR_LBN 13 -#define FSF_AZ_RX_EV_ISCSI_DDIG_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_ISCSI_HDIG_ERR_LBN 12 -#define FSF_AZ_RX_EV_ISCSI_HDIG_ERR_WIDTH 1 -#define FSF_AZ_RX_EV_DESC_PTR_LBN 0 -#define FSF_AZ_RX_EV_DESC_PTR_WIDTH 12 - -/* RX_KER_DESC */ -#define FSF_AZ_RX_KER_BUF_SIZE_LBN 48 -#define FSF_AZ_RX_KER_BUF_SIZE_WIDTH 14 -#define FSF_AZ_RX_KER_BUF_REGION_LBN 46 -#define FSF_AZ_RX_KER_BUF_REGION_WIDTH 2 -#define FSF_AZ_RX_KER_BUF_ADDR_LBN 0 -#define FSF_AZ_RX_KER_BUF_ADDR_WIDTH 46 - -/* RX_USER_DESC */ -#define FSF_AZ_RX_USER_2BYTE_OFFSET_LBN 20 -#define FSF_AZ_RX_USER_2BYTE_OFFSET_WIDTH 12 -#define FSF_AZ_RX_USER_BUF_ID_LBN 0 -#define FSF_AZ_RX_USER_BUF_ID_WIDTH 20 - -/* TX_EV */ -#define FSF_AZ_TX_EV_PKT_ERR_LBN 38 -#define FSF_AZ_TX_EV_PKT_ERR_WIDTH 1 -#define FSF_AZ_TX_EV_PKT_TOO_BIG_LBN 37 -#define FSF_AZ_TX_EV_PKT_TOO_BIG_WIDTH 1 -#define FSF_AZ_TX_EV_Q_LABEL_LBN 32 -#define FSF_AZ_TX_EV_Q_LABEL_WIDTH 5 -#define FSF_AZ_TX_EV_PORT_LBN 16 -#define FSF_AZ_TX_EV_PORT_WIDTH 1 -#define FSF_AZ_TX_EV_WQ_FF_FULL_LBN 15 -#define FSF_AZ_TX_EV_WQ_FF_FULL_WIDTH 1 -#define FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_LBN 14 -#define FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_WIDTH 1 -#define FSF_AZ_TX_EV_COMP_LBN 12 -#define FSF_AZ_TX_EV_COMP_WIDTH 1 -#define FSF_AZ_TX_EV_DESC_PTR_LBN 0 -#define FSF_AZ_TX_EV_DESC_PTR_WIDTH 12 - -/* TX_KER_DESC */ -#define FSF_AZ_TX_KER_CONT_LBN 62 -#define FSF_AZ_TX_KER_CONT_WIDTH 1 -#define FSF_AZ_TX_KER_BYTE_COUNT_LBN 48 -#define FSF_AZ_TX_KER_BYTE_COUNT_WIDTH 14 -#define FSF_AZ_TX_KER_BUF_REGION_LBN 46 -#define FSF_AZ_TX_KER_BUF_REGION_WIDTH 2 -#define FSF_AZ_TX_KER_BUF_ADDR_LBN 0 -#define FSF_AZ_TX_KER_BUF_ADDR_WIDTH 46 - -/* TX_USER_DESC */ -#define FSF_AZ_TX_USER_SW_EV_EN_LBN 48 -#define FSF_AZ_TX_USER_SW_EV_EN_WIDTH 1 -#define FSF_AZ_TX_USER_CONT_LBN 46 -#define FSF_AZ_TX_USER_CONT_WIDTH 1 -#define FSF_AZ_TX_USER_BYTE_CNT_LBN 33 -#define FSF_AZ_TX_USER_BYTE_CNT_WIDTH 13 -#define FSF_AZ_TX_USER_BUF_ID_LBN 13 -#define FSF_AZ_TX_USER_BUF_ID_WIDTH 20 -#define FSF_AZ_TX_USER_BYTE_OFS_LBN 0 -#define FSF_AZ_TX_USER_BYTE_OFS_WIDTH 13 - -/* USER_EV */ -#define FSF_CZ_USER_QID_LBN 32 -#define FSF_CZ_USER_QID_WIDTH 10 -#define FSF_CZ_USER_EV_REG_VALUE_LBN 0 -#define FSF_CZ_USER_EV_REG_VALUE_WIDTH 32 - -/************************************************************************** - * - * Falcon B0 PCIe core indirect registers - * - ************************************************************************** - */ - -#define FPCR_BB_PCIE_DEVICE_CTRL_STAT 0x68 - -#define FPCR_BB_PCIE_LINK_CTRL_STAT 0x70 - -#define FPCR_BB_ACK_RPL_TIMER 0x700 -#define FPCRF_BB_ACK_TL_LBN 0 -#define FPCRF_BB_ACK_TL_WIDTH 16 -#define FPCRF_BB_RPL_TL_LBN 16 -#define FPCRF_BB_RPL_TL_WIDTH 16 - -#define FPCR_BB_ACK_FREQ 0x70C -#define FPCRF_BB_ACK_FREQ_LBN 0 -#define FPCRF_BB_ACK_FREQ_WIDTH 7 - -/************************************************************************** - * - * Pseudo-registers and fields - * - ************************************************************************** - */ - -/* Interrupt acknowledge work-around register (A0/A1 only) */ -#define FR_AA_WORK_AROUND_BROKEN_PCI_READS 0x0070 - -/* EE_SPI_HCMD_REG: SPI host command register */ -/* Values for the EE_SPI_HCMD_SF_SEL register field */ -#define FFE_AB_SPI_DEVICE_EEPROM 0 -#define FFE_AB_SPI_DEVICE_FLASH 1 - -/* NIC_STAT_REG: NIC status register */ -#define FRF_AB_STRAP_10G_LBN 2 -#define FRF_AB_STRAP_10G_WIDTH 1 -#define FRF_AA_STRAP_PCIE_LBN 0 -#define FRF_AA_STRAP_PCIE_WIDTH 1 - -/* FATAL_INTR_REG_KER: Fatal interrupt register for Kernel */ -#define FRF_AZ_FATAL_INTR_LBN 0 -#define FRF_AZ_FATAL_INTR_WIDTH 12 - -/* SRM_CFG_REG: SRAM configuration register */ -/* We treat the number of SRAM banks and bank size as a single field */ -#define FRF_AZ_SRM_NB_SZ_LBN FRF_AZ_SRM_BANK_SIZE_LBN -#define FRF_AZ_SRM_NB_SZ_WIDTH \ - (FRF_AZ_SRM_BANK_SIZE_WIDTH + FRF_AZ_SRM_NUM_BANK_WIDTH) -#define FFE_AB_SRM_NB1_SZ2M 0 -#define FFE_AB_SRM_NB1_SZ4M 1 -#define FFE_AB_SRM_NB1_SZ8M 2 -#define FFE_AB_SRM_NB_SZ_DEF 3 -#define FFE_AB_SRM_NB2_SZ4M 4 -#define FFE_AB_SRM_NB2_SZ8M 5 -#define FFE_AB_SRM_NB2_SZ16M 6 -#define FFE_AB_SRM_NB_SZ_RES 7 - -/* RX_DESC_UPD_REGP0: Receive descriptor update register. */ -/* We write just the last dword of these registers */ -#define FR_AZ_RX_DESC_UPD_DWORD_P0 \ - (BUILD_BUG_ON_ZERO(FR_AA_RX_DESC_UPD_KER != FR_BZ_RX_DESC_UPD_P0) + \ - FR_BZ_RX_DESC_UPD_P0 + 3 * 4) -#define FRF_AZ_RX_DESC_WPTR_DWORD_LBN (FRF_AZ_RX_DESC_WPTR_LBN - 3 * 32) -#define FRF_AZ_RX_DESC_WPTR_DWORD_WIDTH FRF_AZ_RX_DESC_WPTR_WIDTH - -/* TX_DESC_UPD_REGP0: Transmit descriptor update register. */ -#define FR_AZ_TX_DESC_UPD_DWORD_P0 \ - (BUILD_BUG_ON_ZERO(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0) + \ - FR_BZ_TX_DESC_UPD_P0 + 3 * 4) -#define FRF_AZ_TX_DESC_WPTR_DWORD_LBN (FRF_AZ_TX_DESC_WPTR_LBN - 3 * 32) -#define FRF_AZ_TX_DESC_WPTR_DWORD_WIDTH FRF_AZ_TX_DESC_WPTR_WIDTH - -/* GMF_CFG4_REG: GMAC FIFO configuration register 4 */ -#define FRF_AB_GMF_HSTFLTRFRM_PAUSE_LBN 12 -#define FRF_AB_GMF_HSTFLTRFRM_PAUSE_WIDTH 1 - -/* GMF_CFG5_REG: GMAC FIFO configuration register 5 */ -#define FRF_AB_GMF_HSTFLTRFRMDC_PAUSE_LBN 12 -#define FRF_AB_GMF_HSTFLTRFRMDC_PAUSE_WIDTH 1 - -/* XM_TX_PARAM_REG: XGMAC transmit parameter register */ -#define FRF_AB_XM_MAX_TX_FRM_SIZE_LBN FRF_AB_XM_MAX_TX_FRM_SIZE_LO_LBN -#define FRF_AB_XM_MAX_TX_FRM_SIZE_WIDTH (FRF_AB_XM_MAX_TX_FRM_SIZE_HI_WIDTH + \ - FRF_AB_XM_MAX_TX_FRM_SIZE_LO_WIDTH) - -/* XM_RX_PARAM_REG: XGMAC receive parameter register */ -#define FRF_AB_XM_MAX_RX_FRM_SIZE_LBN FRF_AB_XM_MAX_RX_FRM_SIZE_LO_LBN -#define FRF_AB_XM_MAX_RX_FRM_SIZE_WIDTH (FRF_AB_XM_MAX_RX_FRM_SIZE_HI_WIDTH + \ - FRF_AB_XM_MAX_RX_FRM_SIZE_LO_WIDTH) - -/* XX_TXDRV_CTL_REG: XAUI SerDes transmit drive control register */ -/* Default values */ -#define FFE_AB_XX_TXDRV_DEQ_DEF 0xe /* deq=.6 */ -#define FFE_AB_XX_TXDRV_DTX_DEF 0x5 /* 1.25 */ -#define FFE_AB_XX_SD_CTL_DRV_DEF 0 /* 20mA */ - -/* XX_CORE_STAT_REG: XAUI XGXS core status register */ -/* XGXS all-lanes status fields */ -#define FRF_AB_XX_SYNC_STAT_LBN FRF_AB_XX_SYNC_STAT0_LBN -#define FRF_AB_XX_SYNC_STAT_WIDTH 4 -#define FRF_AB_XX_COMMA_DET_LBN FRF_AB_XX_COMMA_DET_CH0_LBN -#define FRF_AB_XX_COMMA_DET_WIDTH 4 -#define FRF_AB_XX_CHAR_ERR_LBN FRF_AB_XX_CHAR_ERR_CH0_LBN -#define FRF_AB_XX_CHAR_ERR_WIDTH 4 -#define FRF_AB_XX_DISPERR_LBN FRF_AB_XX_DISPERR_CH0_LBN -#define FRF_AB_XX_DISPERR_WIDTH 4 -#define FFE_AB_XX_STAT_ALL_LANES 0xf -#define FRF_AB_XX_FORCE_SIG_LBN FRF_AB_XX_FORCE_SIG0_VAL_LBN -#define FRF_AB_XX_FORCE_SIG_WIDTH 8 -#define FFE_AB_XX_FORCE_SIG_ALL_LANES 0xff - -/* RX_MAC_FILTER_TBL0 */ -/* RMFT_DEST_MAC is wider than 32 bits */ -#define FRF_CZ_RMFT_DEST_MAC_LO_LBN FRF_CZ_RMFT_DEST_MAC_LBN -#define FRF_CZ_RMFT_DEST_MAC_LO_WIDTH 32 -#define FRF_CZ_RMFT_DEST_MAC_HI_LBN (FRF_CZ_RMFT_DEST_MAC_LBN + 32) -#define FRF_CZ_RMFT_DEST_MAC_HI_WIDTH (FRF_CZ_RMFT_DEST_MAC_WIDTH - 32) - -/* TX_MAC_FILTER_TBL0 */ -/* TMFT_SRC_MAC is wider than 32 bits */ -#define FRF_CZ_TMFT_SRC_MAC_LO_LBN FRF_CZ_TMFT_SRC_MAC_LBN -#define FRF_CZ_TMFT_SRC_MAC_LO_WIDTH 32 -#define FRF_CZ_TMFT_SRC_MAC_HI_LBN (FRF_CZ_TMFT_SRC_MAC_LBN + 32) -#define FRF_CZ_TMFT_SRC_MAC_HI_WIDTH (FRF_CZ_TMFT_SRC_MAC_WIDTH - 32) - -/* TX_PACE_TBL */ -/* Values >20 are documented as reserved, but will result in a queue going - * into the fast bin with a pace value of zero. */ -#define FFE_BZ_TX_PACE_OFF 0 -#define FFE_BZ_TX_PACE_RESERVED 21 - -/* DRIVER_EV */ -/* Sub-fields of an RX flush completion event */ -#define FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_LBN 12 -#define FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_WIDTH 1 -#define FSF_AZ_DRIVER_EV_RX_DESCQ_ID_LBN 0 -#define FSF_AZ_DRIVER_EV_RX_DESCQ_ID_WIDTH 12 - -/* EVENT_ENTRY */ -/* Magic number field for event test */ -#define FSF_AZ_DRV_GEN_EV_MAGIC_LBN 0 -#define FSF_AZ_DRV_GEN_EV_MAGIC_WIDTH 32 - -/* RX packet prefix */ -#define FS_BZ_RX_PREFIX_HASH_OFST 12 -#define FS_BZ_RX_PREFIX_SIZE 16 - -#endif /* EFX_FARCH_REGS_H */ diff --git a/drivers/net/ethernet/sfc/filter.h b/drivers/net/ethernet/sfc/filter.h index 5f201a547e5b..0d45900afa76 100644 --- a/drivers/net/ethernet/sfc/filter.h +++ b/drivers/net/ethernet/sfc/filter.h @@ -30,13 +30,6 @@ * * Only some combinations are supported, depending on NIC type: * - * - Falcon supports RX filters matching by {TCP,UDP}/IPv4 4-tuple or - * local 2-tuple (only implemented for Falcon B0) - * - * - Siena supports RX and TX filters matching by {TCP,UDP}/IPv4 4-tuple - * or local 2-tuple, or local MAC with or without outer VID, and RX - * default filters - * * - Huntington supports filter matching controlled by firmware, potentially * using {TCP,UDP}/IPv{4,6} 4-tuple or local 2-tuple, local MAC or I/G bit, * with or without outer and inner VID diff --git a/drivers/net/ethernet/sfc/io.h b/drivers/net/ethernet/sfc/io.h index 30439cc83a89..7432c09010d6 100644 --- a/drivers/net/ethernet/sfc/io.h +++ b/drivers/net/ethernet/sfc/io.h @@ -17,46 +17,22 @@ * ************************************************************************** * - * Notes on locking strategy for the Falcon architecture: - * - * Many CSRs are very wide and cannot be read or written atomically. - * Writes from the host are buffered by the Bus Interface Unit (BIU) - * up to 128 bits. Whenever the host writes part of such a register, - * the BIU collects the written value and does not write to the - * underlying register until all 4 dwords have been written. A - * similar buffering scheme applies to host access to the NIC's 64-bit - * SRAM. - * - * Writes to different CSRs and 64-bit SRAM words must be serialised, - * since interleaved access can result in lost writes. We use - * efx_nic::biu_lock for this. - * - * We also serialise reads from 128-bit CSRs and SRAM with the same - * spinlock. This may not be necessary, but it doesn't really matter - * as there are no such reads on the fast path. + * The EF10 architecture exposes very few registers to the host and + * most of them are only 32 bits wide. The only exceptions are the MC + * doorbell register pair, which has its own latching, and + * TX_DESC_UPD. * - * The DMA descriptor pointers (RX_DESC_UPD and TX_DESC_UPD) are - * 128-bit but are special-cased in the BIU to avoid the need for - * locking in the host: + * The TX_DESC_UPD DMA descriptor pointer is 128-bits but is a special + * case in the BIU to avoid the need for locking in the host: * - * - They are write-only. - * - The semantics of writing to these registers are such that + * - It is write-only. + * - The semantics of writing to this register is such that * replacing the low 96 bits with zero does not affect functionality. - * - If the host writes to the last dword address of such a register + * - If the host writes to the last dword address of the register * (i.e. the high 32 bits) the underlying register will always be * written. If the collector and the current write together do not * provide values for all 128 bits of the register, the low 96 bits * will be written as zero. - * - If the host writes to the address of any other part of such a - * register while the collector already holds values for some other - * register, the write is discarded and the collector maintains its - * current state. - * - * The EF10 architecture exposes very few registers to the host and - * most of them are only 32 bits wide. The only exceptions are the MC - * doorbell register pair, which has its own latching, and - * TX_DESC_UPD, which works in a similar way to the Falcon - * architecture. */ #if BITS_PER_LONG == 64 @@ -125,27 +101,6 @@ static inline void efx_writeo(struct efx_nic *efx, const efx_oword_t *value, spin_unlock_irqrestore(&efx->biu_lock, flags); } -/* Write 64-bit SRAM through the supplied mapping, locking as appropriate. */ -static inline void efx_sram_writeq(struct efx_nic *efx, void __iomem *membase, - const efx_qword_t *value, unsigned int index) -{ - unsigned int addr = index * sizeof(*value); - unsigned long flags __attribute__ ((unused)); - - netif_vdbg(efx, hw, efx->net_dev, - "writing SRAM address %x with " EFX_QWORD_FMT "\n", - addr, EFX_QWORD_VAL(*value)); - - spin_lock_irqsave(&efx->biu_lock, flags); -#ifdef EFX_USE_QWORD_IO - __raw_writeq((__force u64)value->u64[0], membase + addr); -#else - __raw_writel((__force u32)value->u32[0], membase + addr); - __raw_writel((__force u32)value->u32[1], membase + addr + 4); -#endif - spin_unlock_irqrestore(&efx->biu_lock, flags); -} - /* Write a 32-bit CSR or the last dword of a special 128-bit CSR */ static inline void efx_writed(struct efx_nic *efx, const efx_dword_t *value, unsigned int reg) @@ -176,27 +131,6 @@ static inline void efx_reado(struct efx_nic *efx, efx_oword_t *value, EFX_OWORD_VAL(*value)); } -/* Read 64-bit SRAM through the supplied mapping, locking as appropriate. */ -static inline void efx_sram_readq(struct efx_nic *efx, void __iomem *membase, - efx_qword_t *value, unsigned int index) -{ - unsigned int addr = index * sizeof(*value); - unsigned long flags __attribute__ ((unused)); - - spin_lock_irqsave(&efx->biu_lock, flags); -#ifdef EFX_USE_QWORD_IO - value->u64[0] = (__force __le64)__raw_readq(membase + addr); -#else - value->u32[0] = (__force __le32)__raw_readl(membase + addr); - value->u32[1] = (__force __le32)__raw_readl(membase + addr + 4); -#endif - spin_unlock_irqrestore(&efx->biu_lock, flags); - - netif_vdbg(efx, hw, efx->net_dev, - "read from SRAM address %x, got "EFX_QWORD_FMT"\n", - addr, EFX_QWORD_VAL(*value)); -} - /* Read a 32-bit CSR or SRAM */ static inline void efx_readd(struct efx_nic *efx, efx_dword_t *value, unsigned int reg) diff --git a/drivers/net/ethernet/sfc/mcdi.c b/drivers/net/ethernet/sfc/mcdi.c index a7f2c31071e8..d23da9627338 100644 --- a/drivers/net/ethernet/sfc/mcdi.c +++ b/drivers/net/ethernet/sfc/mcdi.c @@ -10,7 +10,6 @@ #include "net_driver.h" #include "nic.h" #include "io.h" -#include "farch_regs.h" #include "mcdi_pcol.h" /************************************************************************** @@ -1353,12 +1352,6 @@ void efx_mcdi_process_event(struct efx_channel *channel, case MCDI_EVENT_CODE_MAC_STATS_DMA: /* MAC stats are gather lazily. We can ignore this. */ break; - case MCDI_EVENT_CODE_FLR: - if (efx->type->sriov_flr) - efx->type->sriov_flr(efx, - MCDI_EVENT_FIELD(*event, FLR_VF)); - break; - case MCDI_EVENT_CODE_PTP_RX: case MCDI_EVENT_CODE_PTP_FAULT: case MCDI_EVENT_CODE_PTP_PPS: efx_ptp_event(efx, event); diff --git a/drivers/net/ethernet/sfc/mcdi_functions.c b/drivers/net/ethernet/sfc/mcdi_functions.c index d3e6d8239f5c..ff8424167384 100644 --- a/drivers/net/ethernet/sfc/mcdi_functions.c +++ b/drivers/net/ethernet/sfc/mcdi_functions.c @@ -62,7 +62,7 @@ int efx_mcdi_alloc_vis(struct efx_nic *efx, unsigned int min_vis, int efx_mcdi_ev_probe(struct efx_channel *channel) { - return efx_nic_alloc_buffer(channel->efx, &channel->eventq.buf, + return efx_nic_alloc_buffer(channel->efx, &channel->eventq, (channel->eventq_mask + 1) * sizeof(efx_qword_t), GFP_KERNEL); @@ -74,14 +74,14 @@ int efx_mcdi_ev_init(struct efx_channel *channel, bool v1_cut_thru, bool v2) MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_MAX_EVQ_SIZE * 8 / EFX_BUF_SIZE)); MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_EVQ_V2_OUT_LEN); - size_t entries = channel->eventq.buf.len / EFX_BUF_SIZE; + size_t entries = channel->eventq.len / EFX_BUF_SIZE; struct efx_nic *efx = channel->efx; size_t inlen, outlen; dma_addr_t dma_addr; int rc, i; /* Fill event queue with all ones (i.e. empty events) */ - memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len); + memset(channel->eventq.addr, 0xff, channel->eventq.len); MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_SIZE, channel->eventq_mask + 1); MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_INSTANCE, channel->channel); @@ -112,7 +112,7 @@ int efx_mcdi_ev_init(struct efx_channel *channel, bool v1_cut_thru, bool v2) INIT_EVQ_IN_FLAG_CUT_THRU, v1_cut_thru); } - dma_addr = channel->eventq.buf.dma_addr; + dma_addr = channel->eventq.dma_addr; for (i = 0; i < entries; ++i) { MCDI_SET_ARRAY_QWORD(inbuf, INIT_EVQ_IN_DMA_ADDR, i, dma_addr); dma_addr += EFX_BUF_SIZE; @@ -134,7 +134,7 @@ int efx_mcdi_ev_init(struct efx_channel *channel, bool v1_cut_thru, bool v2) void efx_mcdi_ev_remove(struct efx_channel *channel) { - efx_nic_free_buffer(channel->efx, &channel->eventq.buf); + efx_nic_free_buffer(channel->efx, &channel->eventq); } void efx_mcdi_ev_fini(struct efx_channel *channel) @@ -166,7 +166,7 @@ int efx_mcdi_tx_init(struct efx_tx_queue *tx_queue) EFX_BUF_SIZE)); bool csum_offload = tx_queue->type & EFX_TXQ_TYPE_OUTER_CSUM; bool inner_csum = tx_queue->type & EFX_TXQ_TYPE_INNER_CSUM; - size_t entries = tx_queue->txd.buf.len / EFX_BUF_SIZE; + size_t entries = tx_queue->txd.len / EFX_BUF_SIZE; struct efx_channel *channel = tx_queue->channel; struct efx_nic *efx = tx_queue->efx; dma_addr_t dma_addr; @@ -182,7 +182,7 @@ int efx_mcdi_tx_init(struct efx_tx_queue *tx_queue) MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_OWNER_ID, 0); MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_PORT_ID, efx->vport_id); - dma_addr = tx_queue->txd.buf.dma_addr; + dma_addr = tx_queue->txd.dma_addr; netif_dbg(efx, hw, efx->net_dev, "pushing TXQ %d. %zu entries (%llx)\n", tx_queue->queue, entries, (u64)dma_addr); @@ -240,7 +240,7 @@ fail: void efx_mcdi_tx_remove(struct efx_tx_queue *tx_queue) { - efx_nic_free_buffer(tx_queue->efx, &tx_queue->txd.buf); + efx_nic_free_buffer(tx_queue->efx, &tx_queue->txd); } void efx_mcdi_tx_fini(struct efx_tx_queue *tx_queue) @@ -269,7 +269,7 @@ fail: int efx_mcdi_rx_probe(struct efx_rx_queue *rx_queue) { - return efx_nic_alloc_buffer(rx_queue->efx, &rx_queue->rxd.buf, + return efx_nic_alloc_buffer(rx_queue->efx, &rx_queue->rxd, (rx_queue->ptr_mask + 1) * sizeof(efx_qword_t), GFP_KERNEL); @@ -278,7 +278,7 @@ int efx_mcdi_rx_probe(struct efx_rx_queue *rx_queue) void efx_mcdi_rx_init(struct efx_rx_queue *rx_queue) { struct efx_channel *channel = efx_rx_queue_channel(rx_queue); - size_t entries = rx_queue->rxd.buf.len / EFX_BUF_SIZE; + size_t entries = rx_queue->rxd.len / EFX_BUF_SIZE; MCDI_DECLARE_BUF(inbuf, MC_CMD_INIT_RXQ_V4_IN_LEN); struct efx_nic *efx = rx_queue->efx; unsigned int buffer_size; @@ -306,7 +306,7 @@ void efx_mcdi_rx_init(struct efx_rx_queue *rx_queue) MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_PORT_ID, efx->vport_id); MCDI_SET_DWORD(inbuf, INIT_RXQ_V4_IN_BUFFER_SIZE_BYTES, buffer_size); - dma_addr = rx_queue->rxd.buf.dma_addr; + dma_addr = rx_queue->rxd.dma_addr; netif_dbg(efx, hw, efx->net_dev, "pushing RXQ %d. %zu entries (%llx)\n", efx_rx_queue_index(rx_queue), entries, (u64)dma_addr); @@ -325,7 +325,7 @@ void efx_mcdi_rx_init(struct efx_rx_queue *rx_queue) void efx_mcdi_rx_remove(struct efx_rx_queue *rx_queue) { - efx_nic_free_buffer(rx_queue->efx, &rx_queue->rxd.buf); + efx_nic_free_buffer(rx_queue->efx, &rx_queue->rxd); } void efx_mcdi_rx_fini(struct efx_rx_queue *rx_queue) diff --git a/drivers/net/ethernet/sfc/mcdi_port_common.c b/drivers/net/ethernet/sfc/mcdi_port_common.c index 0ab14f3d01d4..76ea26722ca4 100644 --- a/drivers/net/ethernet/sfc/mcdi_port_common.c +++ b/drivers/net/ethernet/sfc/mcdi_port_common.c @@ -1106,11 +1106,6 @@ int efx_mcdi_set_mac(struct efx_nic *efx) MCDI_SET_DWORD(cmdbytes, SET_MAC_IN_MTU, efx_calc_mac_mtu(efx)); MCDI_SET_DWORD(cmdbytes, SET_MAC_IN_DRAIN, 0); - - /* Set simple MAC filter for Siena */ - MCDI_POPULATE_DWORD_1(cmdbytes, SET_MAC_IN_REJECT, - SET_MAC_IN_REJECT_UNCST, efx->unicast_filter); - MCDI_POPULATE_DWORD_1(cmdbytes, SET_MAC_IN_FLAGS, SET_MAC_IN_FLAG_INCLUDE_FCS, !!(efx->net_dev->features & NETIF_F_RXFCS)); diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index a7a22b019794..27d86e90a3bb 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -67,9 +67,7 @@ #define EFX_MAX_CORE_TX_QUEUES (EFX_MAX_TX_TC * EFX_MAX_CHANNELS) #define EFX_TXQ_TYPE_OUTER_CSUM 1 /* Outer checksum offload */ #define EFX_TXQ_TYPE_INNER_CSUM 2 /* Inner checksum offload */ -#define EFX_TXQ_TYPE_HIGHPRI 4 /* High-priority (for TC) */ -#define EFX_TXQ_TYPES 8 -/* HIGHPRI is Siena-only, and INNER_CSUM is EF10, so no need for both */ +#define EFX_TXQ_TYPES 4 #define EFX_MAX_TXQ_PER_CHANNEL 4 #define EFX_MAX_TX_QUEUES (EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_CHANNELS) @@ -125,26 +123,6 @@ struct efx_buffer { }; /** - * struct efx_special_buffer - DMA buffer entered into buffer table - * @buf: Standard &struct efx_buffer - * @index: Buffer index within controller;s buffer table - * @entries: Number of buffer table entries - * - * The NIC has a buffer table that maps buffers of size %EFX_BUF_SIZE. - * Event and descriptor rings are addressed via one or more buffer - * table entries (and so can be physically non-contiguous, although we - * currently do not take advantage of that). On Falcon and Siena we - * have to take care of allocating and initialising the entries - * ourselves. On later hardware this is managed by the firmware and - * @index and @entries are left as 0. - */ -struct efx_special_buffer { - struct efx_buffer buf; - unsigned int index; - unsigned int entries; -}; - -/** * struct efx_tx_buffer - buffer state for a TX descriptor * @skb: When @flags & %EFX_TX_BUF_SKB, the associated socket buffer to be * freed when descriptor completes @@ -237,7 +215,7 @@ struct efx_tx_buffer { * Normally this will equal @write_count, but as option descriptors * don't produce completion events, they won't update this. * Filled in iff @efx->type->option_descriptors; only used for PIO. - * Thus, this is written and used on EF10, and neither on farch. + * Thus, this is only written and used on EF10. * @old_read_count: The value of read_count when last checked. * This is here for performance reasons. The xmit path will * only get the up-to-date value of read_count if this @@ -270,7 +248,7 @@ struct efx_tx_queue { struct netdev_queue *core_txq; struct efx_tx_buffer *buffer; struct efx_buffer *cb_page; - struct efx_special_buffer txd; + struct efx_buffer txd; unsigned int ptr_mask; void __iomem *piobuf; unsigned int piobuf_offset; @@ -399,7 +377,7 @@ struct efx_rx_queue { struct efx_nic *efx; int core_index; struct efx_rx_buffer *buffer; - struct efx_special_buffer rxd; + struct efx_buffer rxd; unsigned int ptr_mask; bool refill_enabled; bool flush_pending; @@ -515,7 +493,7 @@ struct efx_channel { #ifdef CONFIG_NET_RX_BUSY_POLL unsigned long busy_poll_state; #endif - struct efx_special_buffer eventq; + struct efx_buffer eventq; unsigned int eventq_mask; unsigned int eventq_read_ptr; int event_test_cpu; @@ -754,18 +732,6 @@ struct efx_hw_stat_desc { u16 offset; }; -/* Number of bits used in a multicast filter hash address */ -#define EFX_MCAST_HASH_BITS 8 - -/* Number of (single-bit) entries in a multicast filter hash */ -#define EFX_MCAST_HASH_ENTRIES (1 << EFX_MCAST_HASH_BITS) - -/* An Efx multicast filter hash */ -union efx_multicast_hash { - u8 byte[EFX_MCAST_HASH_ENTRIES / 8]; - efx_oword_t oword[EFX_MCAST_HASH_ENTRIES / sizeof(efx_oword_t) / 8]; -}; - struct vfdi_status; /* The reserved RSS context value */ @@ -895,7 +861,6 @@ struct efx_mae; * @tx_dc_base: Base qword address in SRAM of TX queue descriptor caches * @rx_dc_base: Base qword address in SRAM of RX queue descriptor caches * @sram_lim_qw: Qword address limit of SRAM - * @next_buffer_table: First available buffer table id * @n_channels: Number of channels in use * @n_rx_channels: Number of channels used for RX (= number of RX queues) * @n_tx_channels: Number of channels used for TX @@ -957,10 +922,6 @@ struct efx_mae; * see &enum ethtool_fec_config_bits. * @link_state: Current state of the link * @n_link_state_changes: Number of times the link has changed state - * @unicast_filter: Flag for Falcon-arch simple unicast filter. - * Protected by @mac_lock. - * @multicast_hash: Multicast hash table for Falcon-arch. - * Protected by @mac_lock. * @wanted_fc: Wanted flow control flags * @fc_disable: When non-zero flow control is disabled. Typically used to * ensure that network back pressure doesn't delay dma queue flushes. @@ -1064,7 +1025,6 @@ struct efx_nic { unsigned tx_dc_base; unsigned rx_dc_base; unsigned sram_lim_qw; - unsigned next_buffer_table; unsigned int max_channels; unsigned int max_vis; @@ -1139,8 +1099,6 @@ struct efx_nic { struct efx_link_state link_state; unsigned int n_link_state_changes; - bool unicast_filter; - union efx_multicast_hash multicast_hash; u8 wanted_fc; unsigned fc_disable; @@ -1263,10 +1221,6 @@ struct efx_udp_tunnel { * @remove_port: Free resources allocated by probe_port() * @handle_global_event: Handle a "global" event (may be %NULL) * @fini_dmaq: Flush and finalise DMA queues (RX and TX queues) - * @prepare_flush: Prepare the hardware for flushing the DMA queues - * (for Falcon architecture) - * @finish_flush: Clean up after flushing the DMA queues (for Falcon - * architecture) * @prepare_flr: Prepare for an FLR * @finish_flr: Clean up after an FLR * @describe_stats: Describe statistics for ethtool @@ -1288,8 +1242,7 @@ struct efx_udp_tunnel { * @set_wol: Push WoL configuration to the NIC * @resume_wol: Synchronise WoL state between driver and MC (e.g. after resume) * @get_fec_stats: Get standard FEC statistics. - * @test_chip: Test registers. May use efx_farch_test_registers(), and is - * expected to reset the NIC. + * @test_chip: Test registers. This is expected to reset the NIC. * @test_nvram: Test validity of NVRAM contents * @mcdi_request: Send an MCDI request with the given header and SDU. * The SDU length may be any value from 0 up to the protocol- @@ -1414,8 +1367,6 @@ struct efx_nic_type { void (*remove_port)(struct efx_nic *efx); bool (*handle_global_event)(struct efx_channel *channel, efx_qword_t *); int (*fini_dmaq)(struct efx_nic *efx); - void (*prepare_flush)(struct efx_nic *efx); - void (*finish_flush)(struct efx_nic *efx); void (*prepare_flr)(struct efx_nic *efx); void (*finish_flr)(struct efx_nic *efx); size_t (*describe_stats)(struct efx_nic *efx, u8 *names); @@ -1531,8 +1482,6 @@ struct efx_nic_type { int (*sriov_init)(struct efx_nic *efx); void (*sriov_fini)(struct efx_nic *efx); bool (*sriov_wanted)(struct efx_nic *efx); - void (*sriov_reset)(struct efx_nic *efx); - void (*sriov_flr)(struct efx_nic *efx, unsigned vf_i); int (*sriov_set_vf_mac)(struct efx_nic *efx, int vf_i, const u8 *mac); int (*sriov_set_vf_vlan)(struct efx_nic *efx, int vf_i, u16 vlan, u8 qos); diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c index 63e2394382bb..a33ed473cc8a 100644 --- a/drivers/net/ethernet/sfc/nic.c +++ b/drivers/net/ethernet/sfc/nic.c @@ -17,7 +17,6 @@ #include "efx.h" #include "nic.h" #include "ef10_regs.h" -#include "farch_regs.h" #include "io.h" #include "workarounds.h" #include "mcdi_pcol.h" @@ -172,10 +171,6 @@ void efx_nic_fini_interrupt(struct efx_nic *efx) /* Register dump */ -#define REGISTER_REVISION_FA 1 -#define REGISTER_REVISION_FB 2 -#define REGISTER_REVISION_FC 3 -#define REGISTER_REVISION_FZ 3 /* last Falcon arch revision */ #define REGISTER_REVISION_ED 4 #define REGISTER_REVISION_EZ 4 /* latest EF10 revision */ @@ -189,117 +184,9 @@ struct efx_nic_reg { REGISTER_REVISION_ ## arch ## min_rev, \ REGISTER_REVISION_ ## arch ## max_rev \ } -#define REGISTER_AA(name) REGISTER(name, F, A, A) -#define REGISTER_AB(name) REGISTER(name, F, A, B) -#define REGISTER_AZ(name) REGISTER(name, F, A, Z) -#define REGISTER_BB(name) REGISTER(name, F, B, B) -#define REGISTER_BZ(name) REGISTER(name, F, B, Z) -#define REGISTER_CZ(name) REGISTER(name, F, C, Z) #define REGISTER_DZ(name) REGISTER(name, E, D, Z) static const struct efx_nic_reg efx_nic_regs[] = { - REGISTER_AZ(ADR_REGION), - REGISTER_AZ(INT_EN_KER), - REGISTER_BZ(INT_EN_CHAR), - REGISTER_AZ(INT_ADR_KER), - REGISTER_BZ(INT_ADR_CHAR), - /* INT_ACK_KER is WO */ - /* INT_ISR0 is RC */ - REGISTER_AZ(HW_INIT), - REGISTER_CZ(USR_EV_CFG), - REGISTER_AB(EE_SPI_HCMD), - REGISTER_AB(EE_SPI_HADR), - REGISTER_AB(EE_SPI_HDATA), - REGISTER_AB(EE_BASE_PAGE), - REGISTER_AB(EE_VPD_CFG0), - /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */ - /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */ - /* PCIE_CORE_INDIRECT is indirect */ - REGISTER_AB(NIC_STAT), - REGISTER_AB(GPIO_CTL), - REGISTER_AB(GLB_CTL), - /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */ - REGISTER_BZ(DP_CTRL), - REGISTER_AZ(MEM_STAT), - REGISTER_AZ(CS_DEBUG), - REGISTER_AZ(ALTERA_BUILD), - REGISTER_AZ(CSR_SPARE), - REGISTER_AB(PCIE_SD_CTL0123), - REGISTER_AB(PCIE_SD_CTL45), - REGISTER_AB(PCIE_PCS_CTL_STAT), - /* DEBUG_DATA_OUT is not used */ - /* DRV_EV is WO */ - REGISTER_AZ(EVQ_CTL), - REGISTER_AZ(EVQ_CNT1), - REGISTER_AZ(EVQ_CNT2), - REGISTER_AZ(BUF_TBL_CFG), - REGISTER_AZ(SRM_RX_DC_CFG), - REGISTER_AZ(SRM_TX_DC_CFG), - REGISTER_AZ(SRM_CFG), - /* BUF_TBL_UPD is WO */ - REGISTER_AZ(SRM_UPD_EVQ), - REGISTER_AZ(SRAM_PARITY), - REGISTER_AZ(RX_CFG), - REGISTER_BZ(RX_FILTER_CTL), - /* RX_FLUSH_DESCQ is WO */ - REGISTER_AZ(RX_DC_CFG), - REGISTER_AZ(RX_DC_PF_WM), - REGISTER_BZ(RX_RSS_TKEY), - /* RX_NODESC_DROP is RC */ - REGISTER_AA(RX_SELF_RST), - /* RX_DEBUG, RX_PUSH_DROP are not used */ - REGISTER_CZ(RX_RSS_IPV6_REG1), - REGISTER_CZ(RX_RSS_IPV6_REG2), - REGISTER_CZ(RX_RSS_IPV6_REG3), - /* TX_FLUSH_DESCQ is WO */ - REGISTER_AZ(TX_DC_CFG), - REGISTER_AA(TX_CHKSM_CFG), - REGISTER_AZ(TX_CFG), - /* TX_PUSH_DROP is not used */ - REGISTER_AZ(TX_RESERVED), - REGISTER_BZ(TX_PACE), - /* TX_PACE_DROP_QID is RC */ - REGISTER_BB(TX_VLAN), - REGISTER_BZ(TX_IPFIL_PORTEN), - REGISTER_AB(MD_TXD), - REGISTER_AB(MD_RXD), - REGISTER_AB(MD_CS), - REGISTER_AB(MD_PHY_ADR), - REGISTER_AB(MD_ID), - /* MD_STAT is RC */ - REGISTER_AB(MAC_STAT_DMA), - REGISTER_AB(MAC_CTRL), - REGISTER_BB(GEN_MODE), - REGISTER_AB(MAC_MC_HASH_REG0), - REGISTER_AB(MAC_MC_HASH_REG1), - REGISTER_AB(GM_CFG1), - REGISTER_AB(GM_CFG2), - /* GM_IPG and GM_HD are not used */ - REGISTER_AB(GM_MAX_FLEN), - /* GM_TEST is not used */ - REGISTER_AB(GM_ADR1), - REGISTER_AB(GM_ADR2), - REGISTER_AB(GMF_CFG0), - REGISTER_AB(GMF_CFG1), - REGISTER_AB(GMF_CFG2), - REGISTER_AB(GMF_CFG3), - REGISTER_AB(GMF_CFG4), - REGISTER_AB(GMF_CFG5), - REGISTER_BB(TX_SRC_MAC_CTL), - REGISTER_AB(XM_ADR_LO), - REGISTER_AB(XM_ADR_HI), - REGISTER_AB(XM_GLB_CFG), - REGISTER_AB(XM_TX_CFG), - REGISTER_AB(XM_RX_CFG), - REGISTER_AB(XM_MGT_INT_MASK), - REGISTER_AB(XM_FC), - REGISTER_AB(XM_PAUSE_TIME), - REGISTER_AB(XM_TX_PARAM), - REGISTER_AB(XM_RX_PARAM), - /* XM_MGT_INT_MSK (note no 'A') is RC */ - REGISTER_AB(XX_PWR_RST), - REGISTER_AB(XX_SD_CTL), - REGISTER_AB(XX_TXDRV_CTL), /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */ /* XX_CORE_STAT is partly RC */ REGISTER_DZ(BIU_HW_REV_ID), @@ -325,49 +212,9 @@ struct efx_nic_reg_table { arch, min_rev, max_rev, \ arch ## R_ ## min_rev ## max_rev ## _ ## name ## _STEP, \ arch ## R_ ## min_rev ## max_rev ## _ ## name ## _ROWS) -#define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, F, A, A) -#define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, F, A, Z) -#define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, F, B, B) -#define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, F, B, Z) -#define REGISTER_TABLE_BB_CZ(name) \ - REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, B, B, \ - FR_BZ_ ## name ## _STEP, \ - FR_BB_ ## name ## _ROWS), \ - REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, C, Z, \ - FR_BZ_ ## name ## _STEP, \ - FR_CZ_ ## name ## _ROWS) -#define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, F, C, Z) #define REGISTER_TABLE_DZ(name) REGISTER_TABLE(name, E, D, Z) static const struct efx_nic_reg_table efx_nic_reg_tables[] = { - /* DRIVER is not used */ - /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */ - REGISTER_TABLE_BB(TX_IPFIL_TBL), - REGISTER_TABLE_BB(TX_SRC_MAC_TBL), - REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER), - REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL), - REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER), - REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL), - REGISTER_TABLE_AA(EVQ_PTR_TBL_KER), - REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL), - /* We can't reasonably read all of the buffer table (up to 8MB!). - * However this driver will only use a few entries. Reading - * 1K entries allows for some expansion of queue count and - * size before we need to change the version. */ - REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER, - F, A, A, 8, 1024), - REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL, - F, B, Z, 8, 1024), - REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0), - REGISTER_TABLE_BB_CZ(TIMER_TBL), - REGISTER_TABLE_BB_CZ(TX_PACE_TBL), - REGISTER_TABLE_BZ(RX_INDIRECTION_TBL), - /* TX_FILTER_TBL0 is huge and not used by this driver */ - REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0), - REGISTER_TABLE_CZ(MC_TREG_SMEM), - /* MSIX_PBA_TABLE is not mapped */ - /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */ - REGISTER_TABLE_BZ(RX_FILTER_TBL0), REGISTER_TABLE_DZ(BIU_MC_SFT_STATUS), }; @@ -425,11 +272,6 @@ void efx_nic_get_regs(struct efx_nic *efx, void *buf) case 4: /* 32-bit SRAM */ efx_readd(efx, buf, table->offset + 4 * i); break; - case 8: /* 64-bit SRAM */ - efx_sram_readq(efx, - efx->membase + table->offset, - buf, i); - break; case 16: /* 128-bit-readable register */ efx_reado_table(efx, buf, table->offset, i); break; diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h index 251868235ae4..1db64fc6e909 100644 --- a/drivers/net/ethernet/sfc/nic.h +++ b/drivers/net/ethernet/sfc/nic.h @@ -11,8 +11,6 @@ #include "nic_common.h" #include "efx.h" -u32 efx_farch_fpga_ver(struct efx_nic *efx); - enum { PHY_TYPE_NONE = 0, PHY_TYPE_TXC43128 = 1, @@ -26,97 +24,6 @@ enum { }; enum { - SIENA_STAT_tx_bytes = GENERIC_STAT_COUNT, - SIENA_STAT_tx_good_bytes, - SIENA_STAT_tx_bad_bytes, - SIENA_STAT_tx_packets, - SIENA_STAT_tx_bad, - SIENA_STAT_tx_pause, - SIENA_STAT_tx_control, - SIENA_STAT_tx_unicast, - SIENA_STAT_tx_multicast, - SIENA_STAT_tx_broadcast, - SIENA_STAT_tx_lt64, - SIENA_STAT_tx_64, - SIENA_STAT_tx_65_to_127, - SIENA_STAT_tx_128_to_255, - SIENA_STAT_tx_256_to_511, - SIENA_STAT_tx_512_to_1023, - SIENA_STAT_tx_1024_to_15xx, - SIENA_STAT_tx_15xx_to_jumbo, - SIENA_STAT_tx_gtjumbo, - SIENA_STAT_tx_collision, - SIENA_STAT_tx_single_collision, - SIENA_STAT_tx_multiple_collision, - SIENA_STAT_tx_excessive_collision, - SIENA_STAT_tx_deferred, - SIENA_STAT_tx_late_collision, - SIENA_STAT_tx_excessive_deferred, - SIENA_STAT_tx_non_tcpudp, - SIENA_STAT_tx_mac_src_error, - SIENA_STAT_tx_ip_src_error, - SIENA_STAT_rx_bytes, - SIENA_STAT_rx_good_bytes, - SIENA_STAT_rx_bad_bytes, - SIENA_STAT_rx_packets, - SIENA_STAT_rx_good, - SIENA_STAT_rx_bad, - SIENA_STAT_rx_pause, - SIENA_STAT_rx_control, - SIENA_STAT_rx_unicast, - SIENA_STAT_rx_multicast, - SIENA_STAT_rx_broadcast, - SIENA_STAT_rx_lt64, - SIENA_STAT_rx_64, - SIENA_STAT_rx_65_to_127, - SIENA_STAT_rx_128_to_255, - SIENA_STAT_rx_256_to_511, - SIENA_STAT_rx_512_to_1023, - SIENA_STAT_rx_1024_to_15xx, - SIENA_STAT_rx_15xx_to_jumbo, - SIENA_STAT_rx_gtjumbo, - SIENA_STAT_rx_bad_gtjumbo, - SIENA_STAT_rx_overflow, - SIENA_STAT_rx_false_carrier, - SIENA_STAT_rx_symbol_error, - SIENA_STAT_rx_align_error, - SIENA_STAT_rx_length_error, - SIENA_STAT_rx_internal_error, - SIENA_STAT_rx_nodesc_drop_cnt, - SIENA_STAT_COUNT -}; - -/** - * struct siena_nic_data - Siena NIC state - * @efx: Pointer back to main interface structure - * @wol_filter_id: Wake-on-LAN packet filter id - * @stats: Hardware statistics - * @vf: Array of &struct siena_vf objects - * @vf_buftbl_base: The zeroth buffer table index used to back VF queues. - * @vfdi_status: Common VFDI status page to be dmad to VF address space. - * @local_addr_list: List of local addresses. Protected by %local_lock. - * @local_page_list: List of DMA addressable pages used to broadcast - * %local_addr_list. Protected by %local_lock. - * @local_lock: Mutex protecting %local_addr_list and %local_page_list. - * @peer_work: Work item to broadcast peer addresses to VMs. - */ -struct siena_nic_data { - struct efx_nic *efx; - int wol_filter_id; - u64 stats[SIENA_STAT_COUNT]; -#ifdef CONFIG_SFC_SRIOV - struct siena_vf *vf; - struct efx_channel *vfdi_channel; - unsigned vf_buftbl_base; - struct efx_buffer vfdi_status; - struct list_head local_addr_list; - struct list_head local_page_list; - struct mutex local_lock; - struct work_struct peer_work; -#endif -}; - -enum { EF10_STAT_port_tx_bytes = GENERIC_STAT_COUNT, EF10_STAT_port_tx_packets, EF10_STAT_port_tx_pause, @@ -304,89 +211,4 @@ int efx_ef10_tx_tso_desc(struct efx_tx_queue *tx_queue, struct sk_buff *skb, extern const struct efx_nic_type efx_hunt_a0_nic_type; extern const struct efx_nic_type efx_hunt_a0_vf_nic_type; -int falcon_probe_board(struct efx_nic *efx, u16 revision_info); - -/* Falcon/Siena queue operations */ -int efx_farch_tx_probe(struct efx_tx_queue *tx_queue); -void efx_farch_tx_init(struct efx_tx_queue *tx_queue); -void efx_farch_tx_fini(struct efx_tx_queue *tx_queue); -void efx_farch_tx_remove(struct efx_tx_queue *tx_queue); -void efx_farch_tx_write(struct efx_tx_queue *tx_queue); -unsigned int efx_farch_tx_limit_len(struct efx_tx_queue *tx_queue, - dma_addr_t dma_addr, unsigned int len); -int efx_farch_rx_probe(struct efx_rx_queue *rx_queue); -void efx_farch_rx_init(struct efx_rx_queue *rx_queue); -void efx_farch_rx_fini(struct efx_rx_queue *rx_queue); -void efx_farch_rx_remove(struct efx_rx_queue *rx_queue); -void efx_farch_rx_write(struct efx_rx_queue *rx_queue); -void efx_farch_rx_defer_refill(struct efx_rx_queue *rx_queue); -int efx_farch_ev_probe(struct efx_channel *channel); -int efx_farch_ev_init(struct efx_channel *channel); -void efx_farch_ev_fini(struct efx_channel *channel); -void efx_farch_ev_remove(struct efx_channel *channel); -int efx_farch_ev_process(struct efx_channel *channel, int quota); -void efx_farch_ev_read_ack(struct efx_channel *channel); -void efx_farch_ev_test_generate(struct efx_channel *channel); - -/* Falcon/Siena filter operations */ -int efx_farch_filter_table_probe(struct efx_nic *efx); -void efx_farch_filter_table_restore(struct efx_nic *efx); -void efx_farch_filter_table_remove(struct efx_nic *efx); -void efx_farch_filter_update_rx_scatter(struct efx_nic *efx); -s32 efx_farch_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec, - bool replace); -int efx_farch_filter_remove_safe(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 filter_id); -int efx_farch_filter_get_safe(struct efx_nic *efx, - enum efx_filter_priority priority, u32 filter_id, - struct efx_filter_spec *); -int efx_farch_filter_clear_rx(struct efx_nic *efx, - enum efx_filter_priority priority); -u32 efx_farch_filter_count_rx_used(struct efx_nic *efx, - enum efx_filter_priority priority); -u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx); -s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx, - enum efx_filter_priority priority, u32 *buf, - u32 size); -#ifdef CONFIG_RFS_ACCEL -bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, - unsigned int index); -#endif -void efx_farch_filter_sync_rx_mode(struct efx_nic *efx); - -/* Falcon/Siena interrupts */ -void efx_farch_irq_enable_master(struct efx_nic *efx); -int efx_farch_irq_test_generate(struct efx_nic *efx); -void efx_farch_irq_disable_master(struct efx_nic *efx); -irqreturn_t efx_farch_msi_interrupt(int irq, void *dev_id); -irqreturn_t efx_farch_legacy_interrupt(int irq, void *dev_id); -irqreturn_t efx_farch_fatal_interrupt(struct efx_nic *efx); - -/* Global Resources */ -void siena_prepare_flush(struct efx_nic *efx); -int efx_farch_fini_dmaq(struct efx_nic *efx); -void efx_farch_finish_flr(struct efx_nic *efx); -void siena_finish_flush(struct efx_nic *efx); -void falcon_start_nic_stats(struct efx_nic *efx); -void falcon_stop_nic_stats(struct efx_nic *efx); -int falcon_reset_xaui(struct efx_nic *efx); -void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw); -void efx_farch_init_common(struct efx_nic *efx); -void efx_farch_rx_push_indir_table(struct efx_nic *efx); -void efx_farch_rx_pull_indir_table(struct efx_nic *efx); - -/* Tests */ -struct efx_farch_register_test { - unsigned address; - efx_oword_t mask; -}; - -int efx_farch_test_registers(struct efx_nic *efx, - const struct efx_farch_register_test *regs, - size_t n_regs); - -void efx_farch_generate_event(struct efx_nic *efx, unsigned int evq, - efx_qword_t *event); - #endif /* EFX_NIC_H */ diff --git a/drivers/net/ethernet/sfc/nic_common.h b/drivers/net/ethernet/sfc/nic_common.h index 0cef35c0c559..466df5348b29 100644 --- a/drivers/net/ethernet/sfc/nic_common.h +++ b/drivers/net/ethernet/sfc/nic_common.h @@ -15,11 +15,10 @@ #include "ptp.h" enum { - /* Revisions 0-2 were Falcon A0, A1 and B0 respectively. + /* Revisions 0-3 were Falcon A0, A1, B0 and Siena respectively. * They are not supported by this driver but these revision numbers * form part of the ethtool API for register dumping. */ - EFX_REV_SIENA_A0 = 3, EFX_REV_HUNT_A0 = 4, EFX_REV_EF100 = 5, }; @@ -33,7 +32,7 @@ static inline int efx_nic_rev(struct efx_nic *efx) static inline efx_qword_t *efx_event(struct efx_channel *channel, unsigned int index) { - return ((efx_qword_t *) (channel->eventq.buf.addr)) + + return ((efx_qword_t *)(channel->eventq.addr)) + (index & channel->eventq_mask); } @@ -59,7 +58,7 @@ static inline int efx_event_present(efx_qword_t *event) static inline efx_qword_t * efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) { - return ((efx_qword_t *) (tx_queue->txd.buf.addr)) + index; + return ((efx_qword_t *)(tx_queue->txd.addr)) + index; } /* Report whether this TX queue would be empty for the given write_count. @@ -80,9 +79,7 @@ int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb, /* Decide whether to push a TX descriptor to the NIC vs merely writing * the doorbell. This can reduce latency when we are adding a single - * descriptor to an empty queue, but is otherwise pointless. Further, - * Falcon and Siena have hardware bugs (SF bug 33851) that may be - * triggered if we don't check this. + * descriptor to an empty queue, but is otherwise pointless. * We use the write_count used for the last doorbell push, to get the * NIC's view of the tx queue. */ @@ -99,7 +96,7 @@ static inline bool efx_nic_may_push_tx_desc(struct efx_tx_queue *tx_queue, static inline efx_qword_t * efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) { - return ((efx_qword_t *) (rx_queue->rxd.buf.addr)) + index; + return ((efx_qword_t *)(rx_queue->rxd.addr)) + index; } /* Alignment of PCIe DMA boundaries (4KB) */ diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c index 0c40571133cb..3eab1802f6b0 100644 --- a/drivers/net/ethernet/sfc/ptp.c +++ b/drivers/net/ethernet/sfc/ptp.c @@ -43,7 +43,6 @@ #include "mcdi.h" #include "mcdi_pcol.h" #include "io.h" -#include "farch_regs.h" #include "tx.h" #include "nic.h" /* indirectly includes ptp.h */ #include "efx_channels.h" @@ -87,9 +86,6 @@ #define PTP_V1_VERSION_LENGTH 2 #define PTP_V1_VERSION_OFFSET 28 -#define PTP_V1_UUID_LENGTH 6 -#define PTP_V1_UUID_OFFSET 50 - #define PTP_V1_SEQUENCE_LENGTH 2 #define PTP_V1_SEQUENCE_OFFSET 58 @@ -101,17 +97,6 @@ #define PTP_V2_VERSION_LENGTH 1 #define PTP_V2_VERSION_OFFSET 29 -#define PTP_V2_UUID_LENGTH 8 -#define PTP_V2_UUID_OFFSET 48 - -/* Although PTP V2 UUIDs are comprised a ClockIdentity (8) and PortNumber (2), - * the MC only captures the last six bytes of the clock identity. These values - * reflect those, not the ones used in the standard. The standard permits - * mapping of V1 UUIDs to V2 UUIDs with these same values. - */ -#define PTP_V2_MC_UUID_LENGTH 6 -#define PTP_V2_MC_UUID_OFFSET 50 - #define PTP_V2_SEQUENCE_LENGTH 2 #define PTP_V2_SEQUENCE_OFFSET 58 @@ -167,14 +152,12 @@ enum ptp_packet_state { /** * struct efx_ptp_match - Matching structure, stored in sk_buff's cb area. - * @words: UUID and (partial) sequence number * @expiry: Time after which the packet should be delivered irrespective of * event arrival. * @state: The state of the packet - whether it is ready for processing or * whether that is of no interest. */ struct efx_ptp_match { - u32 words[DIV_ROUND_UP(PTP_V1_UUID_LENGTH, 4)]; unsigned long expiry; enum ptp_packet_state state; }; @@ -236,15 +219,9 @@ struct efx_ptp_rxfilter { /** * struct efx_ptp_data - Precision Time Protocol (PTP) state * @efx: The NIC context - * @channel: The PTP channel (Siena only) - * @rx_ts_inline: Flag for whether RX timestamps are inline (else they are - * separate events) + * @channel: The PTP channel (for Medford and Medford2) * @rxq: Receive SKB queue (awaiting timestamps) * @txq: Transmit SKB queue - * @evt_list: List of MC receive events awaiting packets - * @evt_free_list: List of free events - * @evt_lock: Lock for manipulating evt_list and evt_free_list - * @rx_evts: Instantiated events (on evt_list and evt_free_list) * @workwq: Work queue for processing pending PTP operations * @work: Work task * @cleanup_work: Work task for periodic cleanup @@ -310,13 +287,8 @@ struct efx_ptp_rxfilter { struct efx_ptp_data { struct efx_nic *efx; struct efx_channel *channel; - bool rx_ts_inline; struct sk_buff_head rxq; struct sk_buff_head txq; - struct list_head evt_list; - struct list_head evt_free_list; - spinlock_t evt_lock; - struct efx_ptp_event_rx rx_evts[MAX_RECEIVE_EVENTS]; struct workqueue_struct *workwq; struct work_struct work; struct delayed_work cleanup_work; @@ -465,25 +437,6 @@ size_t efx_ptp_update_stats(struct efx_nic *efx, u64 *stats) return PTP_STAT_COUNT; } -/* For Siena platforms NIC time is s and ns */ -static void efx_ptp_ns_to_s_ns(s64 ns, u32 *nic_major, u32 *nic_minor) -{ - struct timespec64 ts = ns_to_timespec64(ns); - *nic_major = (u32)ts.tv_sec; - *nic_minor = ts.tv_nsec; -} - -static ktime_t efx_ptp_s_ns_to_ktime_correction(u32 nic_major, u32 nic_minor, - s32 correction) -{ - ktime_t kt = ktime_set(nic_major, nic_minor); - if (correction >= 0) - kt = ktime_add_ns(kt, (u64)correction); - else - kt = ktime_sub_ns(kt, (u64)-correction); - return kt; -} - /* To convert from s27 format to ns we multiply then divide by a power of 2. * For the conversion from ns to s27, the operation is also converted to a * multiply and shift. @@ -697,12 +650,6 @@ static int efx_ptp_get_attributes(struct efx_nic *efx) ptp->nic_time.minor_max = 1 << 27; ptp->nic_time.sync_event_minor_shift = 19; break; - case MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS: - ptp->ns_to_nic_time = efx_ptp_ns_to_s_ns; - ptp->nic_to_kernel_time = efx_ptp_s_ns_to_ktime_correction; - ptp->nic_time.minor_max = 1000000000; - ptp->nic_time.sync_event_minor_shift = 22; - break; case MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_QTR_NANOSECONDS: ptp->ns_to_nic_time = efx_ptp_ns_to_s_qns; ptp->nic_to_kernel_time = efx_ptp_s_qns_to_ktime_correction; @@ -1217,76 +1164,6 @@ fail: return; } -static void efx_ptp_drop_time_expired_events(struct efx_nic *efx) -{ - struct efx_ptp_data *ptp = efx->ptp_data; - struct list_head *cursor; - struct list_head *next; - - if (ptp->rx_ts_inline) - return; - - /* Drop time-expired events */ - spin_lock_bh(&ptp->evt_lock); - list_for_each_safe(cursor, next, &ptp->evt_list) { - struct efx_ptp_event_rx *evt; - - evt = list_entry(cursor, struct efx_ptp_event_rx, - link); - if (time_after(jiffies, evt->expiry)) { - list_move(&evt->link, &ptp->evt_free_list); - netif_warn(efx, hw, efx->net_dev, - "PTP rx event dropped\n"); - } - } - spin_unlock_bh(&ptp->evt_lock); -} - -static enum ptp_packet_state efx_ptp_match_rx(struct efx_nic *efx, - struct sk_buff *skb) -{ - struct efx_ptp_data *ptp = efx->ptp_data; - bool evts_waiting; - struct list_head *cursor; - struct list_head *next; - struct efx_ptp_match *match; - enum ptp_packet_state rc = PTP_PACKET_STATE_UNMATCHED; - - WARN_ON_ONCE(ptp->rx_ts_inline); - - spin_lock_bh(&ptp->evt_lock); - evts_waiting = !list_empty(&ptp->evt_list); - spin_unlock_bh(&ptp->evt_lock); - - if (!evts_waiting) - return PTP_PACKET_STATE_UNMATCHED; - - match = (struct efx_ptp_match *)skb->cb; - /* Look for a matching timestamp in the event queue */ - spin_lock_bh(&ptp->evt_lock); - list_for_each_safe(cursor, next, &ptp->evt_list) { - struct efx_ptp_event_rx *evt; - - evt = list_entry(cursor, struct efx_ptp_event_rx, link); - if ((evt->seq0 == match->words[0]) && - (evt->seq1 == match->words[1])) { - struct skb_shared_hwtstamps *timestamps; - - /* Match - add in hardware timestamp */ - timestamps = skb_hwtstamps(skb); - timestamps->hwtstamp = evt->hwtimestamp; - - match->state = PTP_PACKET_STATE_MATCHED; - rc = PTP_PACKET_STATE_MATCHED; - list_move(&evt->link, &ptp->evt_free_list); - break; - } - } - spin_unlock_bh(&ptp->evt_lock); - - return rc; -} - /* Process any queued receive events and corresponding packets * * q is returned with all the packets that are ready for delivery. @@ -1302,9 +1179,6 @@ static void efx_ptp_process_events(struct efx_nic *efx, struct sk_buff_head *q) match = (struct efx_ptp_match *)skb->cb; if (match->state == PTP_PACKET_STATE_MATCH_UNWANTED) { __skb_queue_tail(q, skb); - } else if (efx_ptp_match_rx(efx, skb) == - PTP_PACKET_STATE_MATCHED) { - __skb_queue_tail(q, skb); } else if (time_after(jiffies, match->expiry)) { match->state = PTP_PACKET_STATE_TIMED_OUT; ++ptp->rx_no_timestamp; @@ -1581,8 +1455,6 @@ fail: static int efx_ptp_stop(struct efx_nic *efx) { struct efx_ptp_data *ptp = efx->ptp_data; - struct list_head *cursor; - struct list_head *next; int rc; if (ptp == NULL) @@ -1597,13 +1469,6 @@ static int efx_ptp_stop(struct efx_nic *efx) efx_ptp_deliver_rx_queue(&efx->ptp_data->rxq); skb_queue_purge(&efx->ptp_data->txq); - /* Drop any pending receive events */ - spin_lock_bh(&efx->ptp_data->evt_lock); - list_for_each_safe(cursor, next, &efx->ptp_data->evt_list) { - list_move(cursor, &efx->ptp_data->evt_free_list); - } - spin_unlock_bh(&efx->ptp_data->evt_lock); - return rc; } @@ -1643,8 +1508,6 @@ static void efx_ptp_worker(struct work_struct *work) return; } - efx_ptp_drop_time_expired_events(efx); - __skb_queue_head_init(&tempq); efx_ptp_process_events(efx, &tempq); @@ -1693,7 +1556,6 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel) { struct efx_ptp_data *ptp; int rc = 0; - unsigned int pos; if (efx->ptp_data) { efx->ptp_data->channel = channel; @@ -1707,7 +1569,6 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel) ptp->efx = efx; ptp->channel = channel; - ptp->rx_ts_inline = efx_nic_rev(efx) >= EFX_REV_HUNT_A0; rc = efx_nic_alloc_buffer(efx, &ptp->start, sizeof(int), GFP_KERNEL); if (rc != 0) @@ -1734,12 +1595,6 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel) ptp->config.flags = 0; ptp->config.tx_type = HWTSTAMP_TX_OFF; ptp->config.rx_filter = HWTSTAMP_FILTER_NONE; - INIT_LIST_HEAD(&ptp->evt_list); - INIT_LIST_HEAD(&ptp->evt_free_list); - spin_lock_init(&ptp->evt_lock); - for (pos = 0; pos < MAX_RECEIVE_EVENTS; pos++) - list_add(&ptp->rx_evts[pos].link, &ptp->evt_free_list); - INIT_LIST_HEAD(&ptp->rxfilters_mcast); INIT_LIST_HEAD(&ptp->rxfilters_ucast); @@ -1879,7 +1734,6 @@ static bool efx_ptp_rx(struct efx_channel *channel, struct sk_buff *skb) struct efx_nic *efx = channel->efx; struct efx_ptp_data *ptp = efx->ptp_data; struct efx_ptp_match *match = (struct efx_ptp_match *)skb->cb; - u8 *match_data_012, *match_data_345; unsigned int version; u8 *data; @@ -1895,12 +1749,6 @@ static bool efx_ptp_rx(struct efx_channel *channel, struct sk_buff *skb) if (version != PTP_VERSION_V1) { return false; } - - /* PTP V1 uses all six bytes of the UUID to match the packet - * to the timestamp - */ - match_data_012 = data + PTP_V1_UUID_OFFSET; - match_data_345 = data + PTP_V1_UUID_OFFSET + 3; } else { if (!pskb_may_pull(skb, PTP_V2_MIN_LENGTH)) { return false; @@ -1910,21 +1758,6 @@ static bool efx_ptp_rx(struct efx_channel *channel, struct sk_buff *skb) if ((version & PTP_VERSION_V2_MASK) != PTP_VERSION_V2) { return false; } - - /* The original V2 implementation uses bytes 2-7 of - * the UUID to match the packet to the timestamp. This - * discards two of the bytes of the MAC address used - * to create the UUID (SF bug 33070). The PTP V2 - * enhanced mode fixes this issue and uses bytes 0-2 - * and byte 5-7 of the UUID. - */ - match_data_345 = data + PTP_V2_UUID_OFFSET + 5; - if (ptp->mode == MC_CMD_PTP_MODE_V2) { - match_data_012 = data + PTP_V2_UUID_OFFSET + 2; - } else { - match_data_012 = data + PTP_V2_UUID_OFFSET + 0; - BUG_ON(ptp->mode != MC_CMD_PTP_MODE_V2_ENHANCED); - } } /* Does this packet require timestamping? */ @@ -1936,17 +1769,6 @@ static bool efx_ptp_rx(struct efx_channel *channel, struct sk_buff *skb) */ BUILD_BUG_ON(PTP_V1_SEQUENCE_OFFSET != PTP_V2_SEQUENCE_OFFSET); BUILD_BUG_ON(PTP_V1_SEQUENCE_LENGTH != PTP_V2_SEQUENCE_LENGTH); - - /* Extract UUID/Sequence information */ - match->words[0] = (match_data_012[0] | - (match_data_012[1] << 8) | - (match_data_012[2] << 16) | - (match_data_345[0] << 24)); - match->words[1] = (match_data_345[1] | - (match_data_345[2] << 8) | - (data[PTP_V1_SEQUENCE_OFFSET + - PTP_V1_SEQUENCE_LENGTH - 1] << - 16)); } else { match->state = PTP_PACKET_STATE_MATCH_UNWANTED; } @@ -2110,50 +1932,6 @@ static void ptp_event_failure(struct efx_nic *efx, int expected_frag_len) queue_work(ptp->workwq, &ptp->work); } -/* Process a completed receive event. Put it on the event queue and - * start worker thread. This is required because event and their - * correspoding packets may come in either order. - */ -static void ptp_event_rx(struct efx_nic *efx, struct efx_ptp_data *ptp) -{ - struct efx_ptp_event_rx *evt = NULL; - - if (WARN_ON_ONCE(ptp->rx_ts_inline)) - return; - - if (ptp->evt_frag_idx != 3) { - ptp_event_failure(efx, 3); - return; - } - - spin_lock_bh(&ptp->evt_lock); - if (!list_empty(&ptp->evt_free_list)) { - evt = list_first_entry(&ptp->evt_free_list, - struct efx_ptp_event_rx, link); - list_del(&evt->link); - - evt->seq0 = EFX_QWORD_FIELD(ptp->evt_frags[2], MCDI_EVENT_DATA); - evt->seq1 = (EFX_QWORD_FIELD(ptp->evt_frags[2], - MCDI_EVENT_SRC) | - (EFX_QWORD_FIELD(ptp->evt_frags[1], - MCDI_EVENT_SRC) << 8) | - (EFX_QWORD_FIELD(ptp->evt_frags[0], - MCDI_EVENT_SRC) << 16)); - evt->hwtimestamp = efx->ptp_data->nic_to_kernel_time( - EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA), - EFX_QWORD_FIELD(ptp->evt_frags[1], MCDI_EVENT_DATA), - ptp->ts_corrections.ptp_rx); - evt->expiry = jiffies + msecs_to_jiffies(PKT_EVENT_LIFETIME_MS); - list_add_tail(&evt->link, &ptp->evt_list); - - queue_work(ptp->workwq, &ptp->work); - } else if (net_ratelimit()) { - /* Log a rate-limited warning message. */ - netif_err(efx, rx_err, efx->net_dev, "PTP event queue overflow\n"); - } - spin_unlock_bh(&ptp->evt_lock); -} - static void ptp_event_fault(struct efx_nic *efx, struct efx_ptp_data *ptp) { int code = EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA); @@ -2200,9 +1978,6 @@ void efx_ptp_event(struct efx_nic *efx, efx_qword_t *ev) if (!MCDI_EVENT_FIELD(*ev, CONT)) { /* Process resulting event */ switch (code) { - case MCDI_EVENT_CODE_PTP_RX: - ptp_event_rx(efx, ptp); - break; case MCDI_EVENT_CODE_PTP_FAULT: ptp_event_fault(efx, ptp); break; diff --git a/drivers/net/ethernet/sfc/selftest.c b/drivers/net/ethernet/sfc/selftest.c index 19a0b8584afb..e6d3bd4af044 100644 --- a/drivers/net/ethernet/sfc/selftest.c +++ b/drivers/net/ethernet/sfc/selftest.c @@ -38,8 +38,7 @@ /* * Loopback test packet structure * - * The self-test should stress every RSS vector, and unfortunately - * Falcon only performs RSS on TCP/UDP packets. + * The self-test should stress every RSS vector. */ struct efx_loopback_payload { char pad[2]; /* Ensures ip is 4-byte aligned */ @@ -584,10 +583,6 @@ efx_test_loopback(struct efx_tx_queue *tx_queue, return 0; } -/* Wait for link up. On Falcon, we would prefer to rely on efx_monitor, but - * any contention on the mac lock (via e.g. efx_mac_mcast_work) causes it - * to delay and retry. Therefore, it's safer to just poll directly. Wait - * for link up and any faults to dissipate. */ static int efx_wait_for_link(struct efx_nic *efx) { struct efx_link_state *link_state = &efx->link_state; diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c index 15ebd3973922..4dc881159246 100644 --- a/drivers/net/ethernet/sfc/tc.c +++ b/drivers/net/ethernet/sfc/tc.c @@ -201,23 +201,23 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx, } } if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_VLAN) | - BIT(FLOW_DISSECTOR_KEY_CVLAN) | - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IP) | - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | - BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_TCP) | - BIT(FLOW_DISSECTOR_KEY_IP))) { - NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#x", + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx", dissector->used_keys); return -EOPNOTSUPP; } @@ -228,12 +228,13 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx, !(match->value.eth_proto == htons(ETH_P_IP) || match->value.eth_proto == htons(ETH_P_IPV6))) if (dissector->used_keys & - (BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_IP) | - BIT(FLOW_DISSECTOR_KEY_TCP))) { - NL_SET_ERR_MSG_FMT_MOD(extack, "L3/L4 flower keys %#x require protocol ipv[46]", + (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "L3/L4 flower keys %#llx require protocol ipv[46]", dissector->used_keys); return -EINVAL; } @@ -281,9 +282,10 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx, if ((match->value.ip_proto != IPPROTO_UDP && match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) if (dissector->used_keys & - (BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_TCP))) { - NL_SET_ERR_MSG_FMT_MOD(extack, "L4 flower keys %#x require ipproto udp or tcp", + (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "L4 flower keys %#llx require ipproto udp or tcp", dissector->used_keys); return -EINVAL; } @@ -344,12 +346,13 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx, MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport); MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid); } else if (dissector->used_keys & - (BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | - BIT(FLOW_DISSECTOR_KEY_ENC_IP) | - BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))) { - NL_SET_ERR_MSG_FMT_MOD(extack, "Flower enc keys require enc_control (keys: %#x)", + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Flower enc keys require enc_control (keys: %#llx)", dissector->used_keys); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index 4ed4082836a9..fe2d476028e7 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -517,13 +517,8 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, unsigned index, type; EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); - index = skb_get_queue_mapping(skb); type = efx_tx_csum_type_skb(skb); - if (index >= efx->n_tx_channels) { - index -= efx->n_tx_channels; - type |= EFX_TXQ_TYPE_HIGHPRI; - } /* PTP "event" packet */ if (unlikely(efx_xmit_with_hwtstamp(skb)) && @@ -603,43 +598,5 @@ void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) /* Must be inverse of queue lookup in efx_hard_start_xmit() */ tx_queue->core_txq = netdev_get_tx_queue(efx->net_dev, - tx_queue->channel->channel + - ((tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ? - efx->n_tx_channels : 0)); -} - -int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, - void *type_data) -{ - struct efx_nic *efx = efx_netdev_priv(net_dev); - struct tc_mqprio_qopt *mqprio = type_data; - unsigned tc, num_tc; - - if (type != TC_SETUP_QDISC_MQPRIO) - return -EOPNOTSUPP; - - /* Only Siena supported highpri queues */ - if (efx_nic_rev(efx) > EFX_REV_SIENA_A0) - return -EOPNOTSUPP; - - num_tc = mqprio->num_tc; - - if (num_tc > EFX_MAX_TX_TC) - return -EINVAL; - - mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; - - if (num_tc == net_dev->num_tc) - return 0; - - for (tc = 0; tc < num_tc; tc++) { - net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; - net_dev->tc_to_txq[tc].count = efx->n_tx_channels; - } - - net_dev->num_tc = num_tc; - - return netif_set_real_num_tx_queues(net_dev, - max_t(int, num_tc, 1) * - efx->n_tx_channels); + tx_queue->channel->channel); } diff --git a/drivers/net/ethernet/sfc/tx_tso.c b/drivers/net/ethernet/sfc/tx_tso.c index d381d8164f07..64a6768f75ea 100644 --- a/drivers/net/ethernet/sfc/tx_tso.c +++ b/drivers/net/ethernet/sfc/tx_tso.c @@ -85,7 +85,7 @@ static inline void prefetch_ptr(struct efx_tx_queue *tx_queue) prefetch(ptr); prefetch(ptr + 0x80); - ptr = (char *) (((efx_qword_t *)tx_queue->txd.buf.addr) + insert_ptr); + ptr = (char *)(((efx_qword_t *)tx_queue->txd.addr) + insert_ptr); prefetch(ptr); prefetch(ptr + 0x80); } diff --git a/drivers/net/ethernet/sfc/vfdi.h b/drivers/net/ethernet/sfc/vfdi.h deleted file mode 100644 index 480b872eb4d1..000000000000 --- a/drivers/net/ethernet/sfc/vfdi.h +++ /dev/null @@ -1,252 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/**************************************************************************** - * Driver for Solarflare network controllers and boards - * Copyright 2010-2012 Solarflare Communications Inc. - */ -#ifndef _VFDI_H -#define _VFDI_H - -/** - * DOC: Virtual Function Driver Interface - * - * This file contains software structures used to form a two way - * communication channel between the VF driver and the PF driver, - * named Virtual Function Driver Interface (VFDI). - * - * For the purposes of VFDI, a page is a memory region with size and - * alignment of 4K. All addresses are DMA addresses to be used within - * the domain of the relevant VF. - * - * The only hardware-defined channels for a VF driver to communicate - * with the PF driver are the event mailboxes (%FR_CZ_USR_EV - * registers). Writing to these registers generates an event with - * EV_CODE = EV_CODE_USR_EV, USER_QID set to the index of the mailbox - * and USER_EV_REG_VALUE set to the value written. The PF driver may - * direct or disable delivery of these events by setting - * %FR_CZ_USR_EV_CFG. - * - * The PF driver can send arbitrary events to arbitrary event queues. - * However, for consistency, VFDI events from the PF are defined to - * follow the same form and be sent to the first event queue assigned - * to the VF while that queue is enabled by the VF driver. - * - * The general form of the variable bits of VFDI events is: - * - * 0 16 24 31 - * | DATA | TYPE | SEQ | - * - * SEQ is a sequence number which should be incremented by 1 (modulo - * 256) for each event. The sequence numbers used in each direction - * are independent. - * - * The VF submits requests of type &struct vfdi_req by sending the - * address of the request (ADDR) in a series of 4 events: - * - * 0 16 24 31 - * | ADDR[0:15] | VFDI_EV_TYPE_REQ_WORD0 | SEQ | - * | ADDR[16:31] | VFDI_EV_TYPE_REQ_WORD1 | SEQ+1 | - * | ADDR[32:47] | VFDI_EV_TYPE_REQ_WORD2 | SEQ+2 | - * | ADDR[48:63] | VFDI_EV_TYPE_REQ_WORD3 | SEQ+3 | - * - * The address must be page-aligned. After receiving such a valid - * series of events, the PF driver will attempt to read the request - * and write a response to the same address. In case of an invalid - * sequence of events or a DMA error, there will be no response. - * - * The VF driver may request that the PF driver writes status - * information into its domain asynchronously. After writing the - * status, the PF driver will send an event of the form: - * - * 0 16 24 31 - * | reserved | VFDI_EV_TYPE_STATUS | SEQ | - * - * In case the VF must be reset for any reason, the PF driver will - * send an event of the form: - * - * 0 16 24 31 - * | reserved | VFDI_EV_TYPE_RESET | SEQ | - * - * It is then the responsibility of the VF driver to request - * reinitialisation of its queues. - */ -#define VFDI_EV_SEQ_LBN 24 -#define VFDI_EV_SEQ_WIDTH 8 -#define VFDI_EV_TYPE_LBN 16 -#define VFDI_EV_TYPE_WIDTH 8 -#define VFDI_EV_TYPE_REQ_WORD0 0 -#define VFDI_EV_TYPE_REQ_WORD1 1 -#define VFDI_EV_TYPE_REQ_WORD2 2 -#define VFDI_EV_TYPE_REQ_WORD3 3 -#define VFDI_EV_TYPE_STATUS 4 -#define VFDI_EV_TYPE_RESET 5 -#define VFDI_EV_DATA_LBN 0 -#define VFDI_EV_DATA_WIDTH 16 - -struct vfdi_endpoint { - u8 mac_addr[ETH_ALEN]; - __be16 tci; -}; - -/** - * enum vfdi_op - VFDI operation enumeration - * @VFDI_OP_RESPONSE: Indicates a response to the request. - * @VFDI_OP_INIT_EVQ: Initialize SRAM entries and initialize an EVQ. - * @VFDI_OP_INIT_RXQ: Initialize SRAM entries and initialize an RXQ. - * @VFDI_OP_INIT_TXQ: Initialize SRAM entries and initialize a TXQ. - * @VFDI_OP_FINI_ALL_QUEUES: Flush all queues, finalize all queues, then - * finalize the SRAM entries. - * @VFDI_OP_INSERT_FILTER: Insert a MAC filter targeting the given RXQ. - * @VFDI_OP_REMOVE_ALL_FILTERS: Remove all filters. - * @VFDI_OP_SET_STATUS_PAGE: Set the DMA page(s) used for status updates - * from PF and write the initial status. - * @VFDI_OP_CLEAR_STATUS_PAGE: Clear the DMA page(s) used for status - * updates from PF. - */ -enum vfdi_op { - VFDI_OP_RESPONSE = 0, - VFDI_OP_INIT_EVQ = 1, - VFDI_OP_INIT_RXQ = 2, - VFDI_OP_INIT_TXQ = 3, - VFDI_OP_FINI_ALL_QUEUES = 4, - VFDI_OP_INSERT_FILTER = 5, - VFDI_OP_REMOVE_ALL_FILTERS = 6, - VFDI_OP_SET_STATUS_PAGE = 7, - VFDI_OP_CLEAR_STATUS_PAGE = 8, - VFDI_OP_LIMIT, -}; - -/* Response codes for VFDI operations. Other values may be used in future. */ -#define VFDI_RC_SUCCESS 0 -#define VFDI_RC_ENOMEM (-12) -#define VFDI_RC_EINVAL (-22) -#define VFDI_RC_EOPNOTSUPP (-95) -#define VFDI_RC_ETIMEDOUT (-110) - -/** - * struct vfdi_req - Request from VF driver to PF driver - * @op: Operation code or response indicator, taken from &enum vfdi_op. - * @rc: Response code. Set to 0 on success or a negative error code on failure. - * @u.init_evq.index: Index of event queue to create. - * @u.init_evq.buf_count: Number of 4k buffers backing event queue. - * @u.init_evq.addr: Array of length %u.init_evq.buf_count containing DMA - * address of each page backing the event queue. - * @u.init_rxq.index: Index of receive queue to create. - * @u.init_rxq.buf_count: Number of 4k buffers backing receive queue. - * @u.init_rxq.evq: Instance of event queue to target receive events at. - * @u.init_rxq.label: Label used in receive events. - * @u.init_rxq.flags: Unused. - * @u.init_rxq.addr: Array of length %u.init_rxq.buf_count containing DMA - * address of each page backing the receive queue. - * @u.init_txq.index: Index of transmit queue to create. - * @u.init_txq.buf_count: Number of 4k buffers backing transmit queue. - * @u.init_txq.evq: Instance of event queue to target transmit completion - * events at. - * @u.init_txq.label: Label used in transmit completion events. - * @u.init_txq.flags: Checksum offload flags. - * @u.init_txq.addr: Array of length %u.init_txq.buf_count containing DMA - * address of each page backing the transmit queue. - * @u.mac_filter.rxq: Insert MAC filter at VF local address/VLAN targeting - * all traffic at this receive queue. - * @u.mac_filter.flags: MAC filter flags. - * @u.set_status_page.dma_addr: Base address for the &struct vfdi_status. - * This address must be page-aligned and the PF may write up to a - * whole page (allowing for extension of the structure). - * @u.set_status_page.peer_page_count: Number of additional pages the VF - * has provided into which peer addresses may be DMAd. - * @u.set_status_page.peer_page_addr: Array of DMA addresses of pages. - * If the number of peers exceeds 256, then the VF must provide - * additional pages in this array. The PF will then DMA up to - * 512 vfdi_endpoint structures into each page. These addresses - * must be page-aligned. - */ -struct vfdi_req { - u32 op; - u32 reserved1; - s32 rc; - u32 reserved2; - union { - struct { - u32 index; - u32 buf_count; - u64 addr[]; - } init_evq; - struct { - u32 index; - u32 buf_count; - u32 evq; - u32 label; - u32 flags; -#define VFDI_RXQ_FLAG_SCATTER_EN 1 - u32 reserved; - u64 addr[]; - } init_rxq; - struct { - u32 index; - u32 buf_count; - u32 evq; - u32 label; - u32 flags; -#define VFDI_TXQ_FLAG_IP_CSUM_DIS 1 -#define VFDI_TXQ_FLAG_TCPUDP_CSUM_DIS 2 - u32 reserved; - u64 addr[]; - } init_txq; - struct { - u32 rxq; - u32 flags; -#define VFDI_MAC_FILTER_FLAG_RSS 1 -#define VFDI_MAC_FILTER_FLAG_SCATTER 2 - } mac_filter; - struct { - u64 dma_addr; - u64 peer_page_count; - u64 peer_page_addr[]; - } set_status_page; - } u; -}; - -/** - * struct vfdi_status - Status provided by PF driver to VF driver - * @generation_start: A generation count DMA'd to VF *before* the - * rest of the structure. - * @generation_end: A generation count DMA'd to VF *after* the - * rest of the structure. - * @version: Version of this structure; currently set to 1. Later - * versions must either be layout-compatible or only be sent to VFs - * that specifically request them. - * @length: Total length of this structure including embedded tables - * @vi_scale: log2 the number of VIs available on this VF. This quantity - * is used by the hardware for register decoding. - * @max_tx_channels: The maximum number of transmit queues the VF can use. - * @rss_rxq_count: The number of receive queues present in the shared RSS - * indirection table. - * @peer_count: Total number of peers in the complete peer list. If larger - * than ARRAY_SIZE(%peers), then the VF must provide sufficient - * additional pages each of which is filled with vfdi_endpoint structures. - * @local: The MAC address and outer VLAN tag of *this* VF - * @peers: Table of peer addresses. The @tci fields in these structures - * are currently unused and must be ignored. Additional peers are - * written into any additional pages provided by the VF. - * @timer_quantum_ns: Timer quantum (nominal period between timer ticks) - * for interrupt moderation timers, in nanoseconds. This member is only - * present if @length is sufficiently large. - */ -struct vfdi_status { - u32 generation_start; - u32 generation_end; - u32 version; - u32 length; - u8 vi_scale; - u8 max_tx_channels; - u8 rss_rxq_count; - u8 reserved1; - u16 peer_count; - u16 reserved2; - struct vfdi_endpoint local; - struct vfdi_endpoint peers[256]; - - /* Members below here extend version 1 of this structure */ - u32 timer_quantum_ns; -}; - -#endif diff --git a/drivers/net/ethernet/sfc/workarounds.h b/drivers/net/ethernet/sfc/workarounds.h index 815be2d20c4b..e10e7f84958d 100644 --- a/drivers/net/ethernet/sfc/workarounds.h +++ b/drivers/net/ethernet/sfc/workarounds.h @@ -12,14 +12,7 @@ * Bug numbers are from Solarflare's Bugzilla. */ -#define EFX_WORKAROUND_SIENA(efx) (efx_nic_rev(efx) == EFX_REV_SIENA_A0) #define EFX_WORKAROUND_EF10(efx) (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) -#define EFX_WORKAROUND_10G(efx) 1 - -/* Bit-bashed I2C reads cause performance drop */ -#define EFX_WORKAROUND_7884 EFX_WORKAROUND_10G -/* Legacy interrupt storm when interrupt fifo fills */ -#define EFX_WORKAROUND_17213 EFX_WORKAROUND_SIENA /* Lockup when writing event block registers at gen2/gen3 */ #define EFX_EF10_WORKAROUND_35388(efx) \ diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c index 174dc8908b72..cb590db625e8 100644 --- a/drivers/net/ethernet/smsc/smsc911x.c +++ b/drivers/net/ethernet/smsc/smsc911x.c @@ -43,7 +43,6 @@ #include <linux/smsc911x.h> #include <linux/device.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_net.h> #include <linux/acpi.h> @@ -552,7 +551,7 @@ static void smsc911x_mac_write(struct smsc911x_data *pdata, /* Get a phy register */ static int smsc911x_mii_read(struct mii_bus *bus, int phyaddr, int regidx) { - struct smsc911x_data *pdata = (struct smsc911x_data *)bus->priv; + struct smsc911x_data *pdata = bus->priv; unsigned long flags; unsigned int addr; int i, reg; @@ -591,7 +590,7 @@ out: static int smsc911x_mii_write(struct mii_bus *bus, int phyaddr, int regidx, u16 val) { - struct smsc911x_data *pdata = (struct smsc911x_data *)bus->priv; + struct smsc911x_data *pdata = bus->priv; unsigned long flags; unsigned int addr; int i, reg; diff --git a/drivers/net/ethernet/smsc/smsc9420.c b/drivers/net/ethernet/smsc/smsc9420.c index 71fbb358bb7d..3b26f1d86beb 100644 --- a/drivers/net/ethernet/smsc/smsc9420.c +++ b/drivers/net/ethernet/smsc/smsc9420.c @@ -102,7 +102,7 @@ static inline void smsc9420_pci_flush_write(struct smsc9420_pdata *pd) static int smsc9420_mii_read(struct mii_bus *bus, int phyaddr, int regidx) { - struct smsc9420_pdata *pd = (struct smsc9420_pdata *)bus->priv; + struct smsc9420_pdata *pd = bus->priv; unsigned long flags; u32 addr; int i, reg = -EIO; @@ -140,7 +140,7 @@ out: static int smsc9420_mii_write(struct mii_bus *bus, int phyaddr, int regidx, u16 val) { - struct smsc9420_pdata *pd = (struct smsc9420_pdata *)bus->priv; + struct smsc9420_pdata *pd = bus->priv; unsigned long flags; u32 addr; int i, reg = -EIO; diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index 492c39c08af1..4838d2383a43 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -15,10 +15,11 @@ #include <linux/mii.h> #include <linux/module.h> #include <linux/netdevice.h> +#include <linux/of.h> #include <linux/of_net.h> #include <linux/of_mdio.h> -#include <linux/of_platform.h> #include <linux/phy.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/reset.h> #include <linux/types.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 5583f0b055ec..06c6871f8788 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -121,17 +121,6 @@ config DWMAC_MESON the stmmac device driver. This driver is used for Meson6, Meson8, Meson8b and GXBB SoCs. -config DWMAC_OXNAS - tristate "Oxford Semiconductor OXNAS dwmac support" - default ARCH_OXNAS - depends on OF && COMMON_CLK && (ARCH_OXNAS || COMPILE_TEST) - select MFD_SYSCON - help - Support for Ethernet controller on Oxford Semiconductor OXNAS SoCs. - - This selects the Oxford Semiconductor OXNASSoC glue layer support for - the stmmac device driver. This driver is used for OX820. - config DWMAC_QCOM_ETHQOS tristate "Qualcomm ETHQOS support" default ARCH_QCOM diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index 7dd3d388068b..5b57aee19267 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -19,7 +19,6 @@ obj-$(CONFIG_DWMAC_IPQ806X) += dwmac-ipq806x.o obj-$(CONFIG_DWMAC_LPC18XX) += dwmac-lpc18xx.o obj-$(CONFIG_DWMAC_MEDIATEK) += dwmac-mediatek.o obj-$(CONFIG_DWMAC_MESON) += dwmac-meson.o dwmac-meson8b.o -obj-$(CONFIG_DWMAC_OXNAS) += dwmac-oxnas.o obj-$(CONFIG_DWMAC_QCOM_ETHQOS) += dwmac-qcom-ethqos.o obj-$(CONFIG_DWMAC_ROCKCHIP) += dwmac-rk.o obj-$(CONFIG_DWMAC_SOCFPGA) += dwmac-altr-socfpga.o diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 16e67c18b6f7..c67171975d5c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -35,6 +35,7 @@ #define DWMAC_CORE_5_10 0x51 #define DWMAC_CORE_5_20 0x52 #define DWXGMAC_CORE_2_10 0x21 +#define DWXGMAC_CORE_2_20 0x22 #define DWXLGMAC_CORE_2_00 0x20 /* Device ID */ @@ -59,13 +60,25 @@ /* #define FRAME_FILTER_DEBUG */ struct stmmac_txq_stats { - unsigned long tx_pkt_n; - unsigned long tx_normal_irq_n; + u64 tx_bytes; + u64 tx_packets; + u64 tx_pkt_n; + u64 tx_normal_irq_n; + u64 napi_poll; + u64 tx_clean; + u64 tx_set_ic_bit; + u64 tx_tso_frames; + u64 tx_tso_nfrags; + struct u64_stats_sync syncp; }; struct stmmac_rxq_stats { - unsigned long rx_pkt_n; - unsigned long rx_normal_irq_n; + u64 rx_bytes; + u64 rx_packets; + u64 rx_pkt_n; + u64 rx_normal_irq_n; + u64 napi_poll; + struct u64_stats_sync syncp; }; /* Extra statistic and debug information exposed by ethtool */ @@ -81,6 +94,7 @@ struct stmmac_extra_stats { unsigned long tx_frame_flushed; unsigned long tx_payload_error; unsigned long tx_ip_header_error; + unsigned long tx_collision; /* Receive errors */ unsigned long rx_desc; unsigned long sa_filter_fail; @@ -113,14 +127,6 @@ struct stmmac_extra_stats { /* Tx/Rx IRQ Events */ unsigned long rx_early_irq; unsigned long threshold; - unsigned long tx_pkt_n; - unsigned long rx_pkt_n; - unsigned long normal_irq_n; - unsigned long rx_normal_irq_n; - unsigned long napi_poll; - unsigned long tx_normal_irq_n; - unsigned long tx_clean; - unsigned long tx_set_ic_bit; unsigned long irq_receive_pmt_irq_n; /* MMC info */ unsigned long mmc_tx_irq_n; @@ -190,18 +196,16 @@ struct stmmac_extra_stats { unsigned long mtl_rx_fifo_ctrl_active; unsigned long mac_rx_frame_ctrl_fifo; unsigned long mac_gmii_rx_proto_engine; - /* TSO */ - unsigned long tx_tso_frames; - unsigned long tx_tso_nfrags; /* EST */ unsigned long mtl_est_cgce; unsigned long mtl_est_hlbs; unsigned long mtl_est_hlbf; unsigned long mtl_est_btre; unsigned long mtl_est_btrlm; - /* per queue statistics */ - struct stmmac_txq_stats txq_stats[MTL_MAX_TX_QUEUES]; - struct stmmac_rxq_stats rxq_stats[MTL_MAX_RX_QUEUES]; + unsigned long rx_dropped; + unsigned long rx_errors; + unsigned long tx_dropped; + unsigned long tx_errors; }; /* Safety Feature statistics exposed by ethtool */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index 9f88530c5e8c..74c10373cc39 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -14,7 +14,7 @@ #include <linux/iopoll.h> #include <linux/ioport.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_net.h> #include <linux/mfd/syscon.h> #include <linux/platform_device.h> @@ -113,7 +113,7 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev, /* dwc-qos needs GMAC4, AAL, TSO and PMT */ plat_dat->has_gmac4 = 1; plat_dat->dma_cfg->aal = 1; - plat_dat->tso_en = 1; + plat_dat->flags |= STMMAC_FLAG_TSO_EN; plat_dat->pmt = 1; return 0; @@ -359,7 +359,7 @@ bypass_clk_reset_gpio: data->fix_mac_speed = tegra_eqos_fix_speed; data->init = tegra_eqos_init; data->bsp_priv = eqos; - data->sph_disable = 1; + data->flags |= STMMAC_FLAG_SPH_DISABLE; err = tegra_eqos_init(pdev, eqos); if (err < 0) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c index b9378a63f0e8..92e06a96757a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c @@ -12,7 +12,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/phy.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c index 8063ba1c3ce8..e22ef0d6bc73 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c @@ -11,7 +11,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/phy.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index ab9f876b6df7..979c755964b1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -257,9 +257,8 @@ static void intel_speed_mode_2500(struct net_device *ndev, void *intel_data) /* Program PTP Clock Frequency for different variant of * Intel mGBE that has slightly different GPO mapping */ -static void intel_mgbe_ptp_clk_freq_config(void *npriv) +static void intel_mgbe_ptp_clk_freq_config(struct stmmac_priv *priv) { - struct stmmac_priv *priv = (struct stmmac_priv *)npriv; struct intel_priv_data *intel_priv; u32 gpio_value; @@ -326,10 +325,10 @@ static int intel_crosststamp(ktime_t *device, /* Both internal crosstimestamping and external triggered event * timestamping cannot be run concurrently. */ - if (priv->plat->ext_snapshot_en) + if (priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN) return -EBUSY; - priv->plat->int_snapshot_en = 1; + priv->plat->flags |= STMMAC_FLAG_INT_SNAPSHOT_EN; mutex_lock(&priv->aux_ts_lock); /* Enable Internal snapshot trigger */ @@ -350,7 +349,7 @@ static int intel_crosststamp(ktime_t *device, break; default: mutex_unlock(&priv->aux_ts_lock); - priv->plat->int_snapshot_en = 0; + priv->plat->flags &= ~STMMAC_FLAG_INT_SNAPSHOT_EN; return -EINVAL; } writel(acr_value, ptpaddr + PTP_ACR); @@ -376,7 +375,7 @@ static int intel_crosststamp(ktime_t *device, if (!wait_event_interruptible_timeout(priv->tstamp_busy_wait, stmmac_cross_ts_isr(priv), HZ / 100)) { - priv->plat->int_snapshot_en = 0; + priv->plat->flags &= ~STMMAC_FLAG_INT_SNAPSHOT_EN; return -ETIMEDOUT; } @@ -395,7 +394,7 @@ static int intel_crosststamp(ktime_t *device, } system->cycles *= intel_priv->crossts_adj; - priv->plat->int_snapshot_en = 0; + priv->plat->flags &= ~STMMAC_FLAG_INT_SNAPSHOT_EN; return 0; } @@ -458,8 +457,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->has_gmac = 0; plat->has_gmac4 = 1; plat->force_sf_dma_mode = 0; - plat->tso_en = 1; - plat->sph_disable = 1; + plat->flags |= (STMMAC_FLAG_TSO_EN | STMMAC_FLAG_SPH_DISABLE); /* Multiplying factor to the clk_eee_i clock time * period to make it closer to 100 ns. This value @@ -561,7 +559,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, /* Set the maxmtu to a default of JUMBO_LEN */ plat->maxmtu = JUMBO_LEN; - plat->vlan_fail_q_en = true; + plat->flags |= STMMAC_FLAG_VLAN_FAIL_Q_EN; /* Use the last Rx queue */ plat->vlan_fail_q = plat->rx_queues_to_use - 1; @@ -610,7 +608,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->ext_snapshot_num = AUX_SNAPSHOT0; plat->crosststamp = intel_crosststamp; - plat->int_snapshot_en = 0; + plat->flags &= ~STMMAC_FLAG_INT_SNAPSHOT_EN; /* Setup MSI vector offset specific to Intel mGbE controller */ plat->msi_mac_vec = 29; @@ -628,7 +626,7 @@ static int ehl_common_data(struct pci_dev *pdev, { plat->rx_queues_to_use = 8; plat->tx_queues_to_use = 8; - plat->use_phy_wol = 1; + plat->flags |= STMMAC_FLAG_USE_PHY_WOL; plat->safety_feat_cfg->tsoee = 1; plat->safety_feat_cfg->mrxpee = 1; @@ -954,7 +952,7 @@ static int stmmac_config_single_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, 0); res->wol_irq = res->irq; - plat->multi_msi_en = 0; + plat->flags &= ~STMMAC_FLAG_MULTI_MSI_EN; dev_info(&pdev->dev, "%s: Single IRQ enablement successful\n", __func__); @@ -1006,7 +1004,7 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) res->sfty_ue_irq = pci_irq_vector(pdev, plat->msi_sfty_ue_vec); - plat->multi_msi_en = 1; + plat->flags |= STMMAC_FLAG_MULTI_MSI_EN; dev_info(&pdev->dev, "%s: multi MSI enablement successful\n", __func__); return 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c index 73c1dfa7ecb1..7580077383c0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c @@ -7,8 +7,8 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/stmmac.h> @@ -588,7 +588,10 @@ static int mediatek_dwmac_common_data(struct platform_device *pdev, int i; plat->interface = priv_plat->phy_mode; - plat->use_phy_wol = priv_plat->mac_wol ? 0 : 1; + if (priv_plat->mac_wol) + plat->flags |= STMMAC_FLAG_USE_PHY_WOL; + else + plat->flags &= ~STMMAC_FLAG_USE_PHY_WOL; plat->riwt_off = 1; plat->maxmtu = ETH_DATA_LEN; plat->host_dma_width = priv_plat->variant->dma_bit_mask; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c index 92b16048f91c..0b159dc0d5f6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c @@ -13,7 +13,7 @@ #include <linux/io.h> #include <linux/ioport.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_net.h> #include <linux/mfd/syscon.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c deleted file mode 100644 index 42954020de2c..000000000000 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c +++ /dev/null @@ -1,245 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Oxford Semiconductor OXNAS DWMAC glue layer - * - * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com> - * Copyright (C) 2014 Daniel Golle <daniel@makrotopia.org> - * Copyright (C) 2013 Ma Haijun <mahaijuns@gmail.com> - * Copyright (C) 2012 John Crispin <blogic@openwrt.org> - */ - -#include <linux/device.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/platform_device.h> -#include <linux/regmap.h> -#include <linux/mfd/syscon.h> -#include <linux/stmmac.h> - -#include "stmmac_platform.h" - -/* System Control regmap offsets */ -#define OXNAS_DWMAC_CTRL_REGOFFSET 0x78 -#define OXNAS_DWMAC_DELAY_REGOFFSET 0x100 - -/* Control Register */ -#define DWMAC_CKEN_RX_IN 14 -#define DWMAC_CKEN_RXN_OUT 13 -#define DWMAC_CKEN_RX_OUT 12 -#define DWMAC_CKEN_TX_IN 10 -#define DWMAC_CKEN_TXN_OUT 9 -#define DWMAC_CKEN_TX_OUT 8 -#define DWMAC_RX_SOURCE 7 -#define DWMAC_TX_SOURCE 6 -#define DWMAC_LOW_TX_SOURCE 4 -#define DWMAC_AUTO_TX_SOURCE 3 -#define DWMAC_RGMII 2 -#define DWMAC_SIMPLE_MUX 1 -#define DWMAC_CKEN_GTX 0 - -/* Delay register */ -#define DWMAC_TX_VARDELAY_SHIFT 0 -#define DWMAC_TXN_VARDELAY_SHIFT 8 -#define DWMAC_RX_VARDELAY_SHIFT 16 -#define DWMAC_RXN_VARDELAY_SHIFT 24 -#define DWMAC_TX_VARDELAY(d) ((d) << DWMAC_TX_VARDELAY_SHIFT) -#define DWMAC_TXN_VARDELAY(d) ((d) << DWMAC_TXN_VARDELAY_SHIFT) -#define DWMAC_RX_VARDELAY(d) ((d) << DWMAC_RX_VARDELAY_SHIFT) -#define DWMAC_RXN_VARDELAY(d) ((d) << DWMAC_RXN_VARDELAY_SHIFT) - -struct oxnas_dwmac; - -struct oxnas_dwmac_data { - int (*setup)(struct oxnas_dwmac *dwmac); -}; - -struct oxnas_dwmac { - struct device *dev; - struct clk *clk; - struct regmap *regmap; - const struct oxnas_dwmac_data *data; -}; - -static int oxnas_dwmac_setup_ox810se(struct oxnas_dwmac *dwmac) -{ - unsigned int value; - int ret; - - ret = regmap_read(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, &value); - if (ret < 0) - return ret; - - /* Enable GMII_GTXCLK to follow GMII_REFCLK, required for gigabit PHY */ - value |= BIT(DWMAC_CKEN_GTX) | - /* Use simple mux for 25/125 Mhz clock switching */ - BIT(DWMAC_SIMPLE_MUX); - - regmap_write(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, value); - - return 0; -} - -static int oxnas_dwmac_setup_ox820(struct oxnas_dwmac *dwmac) -{ - unsigned int value; - int ret; - - ret = regmap_read(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, &value); - if (ret < 0) - return ret; - - /* Enable GMII_GTXCLK to follow GMII_REFCLK, required for gigabit PHY */ - value |= BIT(DWMAC_CKEN_GTX) | - /* Use simple mux for 25/125 Mhz clock switching */ - BIT(DWMAC_SIMPLE_MUX) | - /* set auto switch tx clock source */ - BIT(DWMAC_AUTO_TX_SOURCE) | - /* enable tx & rx vardelay */ - BIT(DWMAC_CKEN_TX_OUT) | - BIT(DWMAC_CKEN_TXN_OUT) | - BIT(DWMAC_CKEN_TX_IN) | - BIT(DWMAC_CKEN_RX_OUT) | - BIT(DWMAC_CKEN_RXN_OUT) | - BIT(DWMAC_CKEN_RX_IN); - regmap_write(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, value); - - /* set tx & rx vardelay */ - value = DWMAC_TX_VARDELAY(4) | - DWMAC_TXN_VARDELAY(2) | - DWMAC_RX_VARDELAY(10) | - DWMAC_RXN_VARDELAY(8); - regmap_write(dwmac->regmap, OXNAS_DWMAC_DELAY_REGOFFSET, value); - - return 0; -} - -static int oxnas_dwmac_init(struct platform_device *pdev, void *priv) -{ - struct oxnas_dwmac *dwmac = priv; - int ret; - - /* Reset HW here before changing the glue configuration */ - ret = device_reset(dwmac->dev); - if (ret) - return ret; - - ret = clk_prepare_enable(dwmac->clk); - if (ret) - return ret; - - ret = dwmac->data->setup(dwmac); - if (ret) - clk_disable_unprepare(dwmac->clk); - - return ret; -} - -static void oxnas_dwmac_exit(struct platform_device *pdev, void *priv) -{ - struct oxnas_dwmac *dwmac = priv; - - clk_disable_unprepare(dwmac->clk); -} - -static int oxnas_dwmac_probe(struct platform_device *pdev) -{ - struct plat_stmmacenet_data *plat_dat; - struct stmmac_resources stmmac_res; - struct oxnas_dwmac *dwmac; - int ret; - - ret = stmmac_get_platform_resources(pdev, &stmmac_res); - if (ret) - return ret; - - plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); - if (IS_ERR(plat_dat)) - return PTR_ERR(plat_dat); - - dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL); - if (!dwmac) { - ret = -ENOMEM; - goto err_remove_config_dt; - } - - dwmac->data = (const struct oxnas_dwmac_data *)of_device_get_match_data(&pdev->dev); - if (!dwmac->data) { - ret = -EINVAL; - goto err_remove_config_dt; - } - - dwmac->dev = &pdev->dev; - plat_dat->bsp_priv = dwmac; - plat_dat->init = oxnas_dwmac_init; - plat_dat->exit = oxnas_dwmac_exit; - - dwmac->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, - "oxsemi,sys-ctrl"); - if (IS_ERR(dwmac->regmap)) { - dev_err(&pdev->dev, "failed to have sysctrl regmap\n"); - ret = PTR_ERR(dwmac->regmap); - goto err_remove_config_dt; - } - - dwmac->clk = devm_clk_get(&pdev->dev, "gmac"); - if (IS_ERR(dwmac->clk)) { - ret = PTR_ERR(dwmac->clk); - goto err_remove_config_dt; - } - - ret = oxnas_dwmac_init(pdev, plat_dat->bsp_priv); - if (ret) - goto err_remove_config_dt; - - ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); - if (ret) - goto err_dwmac_exit; - - - return 0; - -err_dwmac_exit: - oxnas_dwmac_exit(pdev, plat_dat->bsp_priv); -err_remove_config_dt: - stmmac_remove_config_dt(pdev, plat_dat); - - return ret; -} - -static const struct oxnas_dwmac_data ox810se_dwmac_data = { - .setup = oxnas_dwmac_setup_ox810se, -}; - -static const struct oxnas_dwmac_data ox820_dwmac_data = { - .setup = oxnas_dwmac_setup_ox820, -}; - -static const struct of_device_id oxnas_dwmac_match[] = { - { - .compatible = "oxsemi,ox810se-dwmac", - .data = &ox810se_dwmac_data, - }, - { - .compatible = "oxsemi,ox820-dwmac", - .data = &ox820_dwmac_data, - }, - { } -}; -MODULE_DEVICE_TABLE(of, oxnas_dwmac_match); - -static struct platform_driver oxnas_dwmac_driver = { - .probe = oxnas_dwmac_probe, - .remove_new = stmmac_pltfr_remove, - .driver = { - .name = "oxnas-dwmac", - .pm = &stmmac_pltfr_pm_ops, - .of_match_table = oxnas_dwmac_match, - }, -}; -module_platform_driver(oxnas_dwmac_driver); - -MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); -MODULE_DESCRIPTION("Oxford Semiconductor OXNAS DWMAC glue layer"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index e62940414e54..5b3df27035ec 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -3,11 +3,10 @@ #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_net.h> #include <linux/platform_device.h> #include <linux/phy.h> #include <linux/phy/phy.h> -#include <linux/property.h> #include "stmmac.h" #include "stmmac_platform.h" @@ -104,7 +103,7 @@ struct qcom_ethqos { struct clk *link_clk; struct phy *serdes_phy; unsigned int speed; - int phy_mode; + phy_interface_t phy_mode; const struct ethqos_emac_por *por; unsigned int num_por; @@ -694,6 +693,23 @@ static void ethqos_clks_disable(void *data) ethqos_clks_config(data, false); } +static void ethqos_ptp_clk_freq_config(struct stmmac_priv *priv) +{ + struct plat_stmmacenet_data *plat_dat = priv->plat; + int err; + + if (!plat_dat->clk_ptp_ref) + return; + + /* Max the PTP ref clock out to get the best resolution possible */ + err = clk_set_rate(plat_dat->clk_ptp_ref, ULONG_MAX); + if (err) + netdev_err(priv->dev, "Failed to max out clk_ptp_ref: %d\n", err); + plat_dat->clk_ptp_rate = clk_get_rate(plat_dat->clk_ptp_ref); + + netdev_dbg(priv->dev, "PTP rate %d\n", plat_dat->clk_ptp_rate); +} + static int qcom_ethqos_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -706,12 +722,13 @@ static int qcom_ethqos_probe(struct platform_device *pdev) ret = stmmac_get_platform_resources(pdev, &stmmac_res); if (ret) - return ret; + return dev_err_probe(dev, ret, + "Failed to get platform resources\n"); plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) { - dev_err(dev, "dt configuration failed\n"); - return PTR_ERR(plat_dat); + return dev_err_probe(dev, PTR_ERR(plat_dat), + "dt configuration failed\n"); } plat_dat->clks_config = ethqos_clks_config; @@ -720,7 +737,9 @@ static int qcom_ethqos_probe(struct platform_device *pdev) if (!ethqos) return -ENOMEM; - ethqos->phy_mode = device_get_phy_mode(dev); + ret = of_get_phy_mode(np, ðqos->phy_mode); + if (ret) + return dev_err_probe(dev, ret, "Failed to get phy mode\n"); switch (ethqos->phy_mode) { case PHY_INTERFACE_MODE_RGMII: case PHY_INTERFACE_MODE_RGMII_ID: @@ -731,16 +750,17 @@ static int qcom_ethqos_probe(struct platform_device *pdev) case PHY_INTERFACE_MODE_SGMII: ethqos->configure_func = ethqos_configure_sgmii; break; - case -ENODEV: - return -ENODEV; default: + dev_err(dev, "Unsupported phy mode %s\n", + phy_modes(ethqos->phy_mode)); return -EINVAL; } ethqos->pdev = pdev; ethqos->rgmii_base = devm_platform_ioremap_resource_byname(pdev, "rgmii"); if (IS_ERR(ethqos->rgmii_base)) - return PTR_ERR(ethqos->rgmii_base); + return dev_err_probe(dev, PTR_ERR(ethqos->rgmii_base), + "Failed to map rgmii resource\n"); ethqos->mac_base = stmmac_res.addr; @@ -752,7 +772,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev) ethqos->link_clk = devm_clk_get(dev, data->link_clk_name ?: "rgmii"); if (IS_ERR(ethqos->link_clk)) - return PTR_ERR(ethqos->link_clk); + return dev_err_probe(dev, PTR_ERR(ethqos->link_clk), + "Failed to get link_clk\n"); ret = ethqos_clks_config(ethqos, true); if (ret) @@ -764,7 +785,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev) ethqos->serdes_phy = devm_phy_optional_get(dev, "serdes"); if (IS_ERR(ethqos->serdes_phy)) - return PTR_ERR(ethqos->serdes_phy); + return dev_err_probe(dev, PTR_ERR(ethqos->serdes_phy), + "Failed to get serdes phy\n"); ethqos->speed = SPEED_1000; ethqos_update_link_clk(ethqos, SPEED_1000); @@ -773,14 +795,17 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->bsp_priv = ethqos; plat_dat->fix_mac_speed = ethqos_fix_mac_speed; plat_dat->dump_debug_regs = rgmii_dump; + plat_dat->ptp_clk_freq_config = ethqos_ptp_clk_freq_config; plat_dat->has_gmac4 = 1; if (ethqos->has_emac_ge_3) plat_dat->dwmac4_addrs = &data->dwmac4_addrs; plat_dat->pmt = 1; - plat_dat->tso_en = of_property_read_bool(np, "snps,tso"); + if (of_property_read_bool(np, "snps,tso")) + plat_dat->flags |= STMMAC_FLAG_TSO_EN; if (of_device_is_compatible(np, "qcom,qcs404-ethqos")) - plat_dat->rx_clk_runs_in_lpi = 1; - plat_dat->has_integrated_pcs = data->has_integrated_pcs; + plat_dat->flags |= STMMAC_FLAG_RX_CLK_RUNS_IN_LPI; + if (data->has_integrated_pcs) + plat_dat->flags |= STMMAC_FLAG_HAS_INTEGRATED_PCS; if (ethqos->serdes_phy) { plat_dat->serdes_powerup = qcom_ethqos_serdes_powerup; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index d81591b470a2..35789cb549f7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -14,8 +14,8 @@ #include <linux/of_net.h> #include <linux/gpio.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_gpio.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/regulator/consumer.h> #include <linux/delay.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c index d3a39d2fb3a9..b1c5f2527c0d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c @@ -7,8 +7,10 @@ * */ +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/property.h> #include <linux/mfd/syscon.h> -#include <linux/of_device.h> #include <linux/regmap.h> #include "stmmac_platform.h" diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c index dcbb17c4f07a..ef03e4669491 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c @@ -17,7 +17,6 @@ #include <linux/regmap.h> #include <linux/clk.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include "stmmac_platform.h" diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c index bdb4de59a672..3a09085819dc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c @@ -11,7 +11,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_net.h> #include <linux/phy.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c index 1e714380d125..c23420863a8d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c @@ -11,9 +11,10 @@ #include <linux/mdio-mux.h> #include <linux/mfd/syscon.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_mdio.h> #include <linux/of_net.h> +#include <linux/of_platform.h> #include <linux/phy.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -440,8 +441,10 @@ static int sun8i_dwmac_dma_interrupt(struct stmmac_priv *priv, struct stmmac_extra_stats *x, u32 chan, u32 dir) { - u32 v; + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; int ret = 0; + u32 v; v = readl(ioaddr + EMAC_INT_STA); @@ -452,7 +455,9 @@ static int sun8i_dwmac_dma_interrupt(struct stmmac_priv *priv, if (v & EMAC_TX_INT) { ret |= handle_tx; - x->tx_normal_irq_n++; + u64_stats_update_begin(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_normal_irq_n++; + u64_stats_update_end(&tx_q->txq_stats.syncp); } if (v & EMAC_TX_DMA_STOP_INT) @@ -474,7 +479,9 @@ static int sun8i_dwmac_dma_interrupt(struct stmmac_priv *priv, if (v & EMAC_RX_INT) { ret |= handle_rx; - x->rx_normal_irq_n++; + u64_stats_update_begin(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_normal_irq_n++; + u64_stats_update_end(&rx_q->rxq_stats.syncp); } if (v & EMAC_RX_BUF_UA_INT) @@ -1227,7 +1234,7 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) plat_dat->interface = interface; plat_dat->rx_coe = STMMAC_RX_COE_TYPE2; plat_dat->tx_coe = 1; - plat_dat->has_sun8i = true; + plat_dat->flags |= STMMAC_FLAG_HAS_SUN8I; plat_dat->bsp_priv = gmac; plat_dat->init = sun8i_dwmac_init; plat_dat->exit = sun8i_dwmac_exit; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c index fbb0ccf84afc..e0f3cbd36852 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include <linux/platform_device.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/module.h> #include <linux/stmmac.h> #include <linux/clk.h> @@ -291,7 +291,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) } plat->has_xgmac = 1; - plat->tso_en = 1; + plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; plat->bsp_priv = mgbe; @@ -338,7 +338,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) /* Program SID */ writel(MGBE_SID, mgbe->hv + MGBE_WRAP_AXI_ASID0_CTRL); - plat->serdes_up_after_phy_linkup = 1; + plat->flags |= STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP; err = stmmac_dvr_probe(&pdev->dev, plat, &res); if (err < 0) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c index acbb284be174..a8731ce0fff0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c @@ -6,7 +6,8 @@ */ #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <linux/of_net.h> #include <linux/stmmac.h> diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c index 1c32b1788f02..dea270f60cc3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c @@ -82,29 +82,24 @@ static void dwmac100_dump_dma_regs(struct stmmac_priv *priv, } /* DMA controller has two counters to track the number of the missed frames. */ -static void dwmac100_dma_diagnostic_fr(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static void dwmac100_dma_diagnostic_fr(struct stmmac_extra_stats *x, void __iomem *ioaddr) { u32 csr8 = readl(ioaddr + DMA_MISSED_FRAME_CTR); if (unlikely(csr8)) { if (csr8 & DMA_MISSED_FRAME_OVE) { - stats->rx_over_errors += 0x800; x->rx_overflow_cntr += 0x800; } else { unsigned int ove_cntr; ove_cntr = ((csr8 & DMA_MISSED_FRAME_OVE_CNTR) >> 17); - stats->rx_over_errors += ove_cntr; x->rx_overflow_cntr += ove_cntr; } if (csr8 & DMA_MISSED_FRAME_OVE_M) { - stats->rx_missed_errors += 0xffff; x->rx_missed_cntr += 0xffff; } else { unsigned int miss_f = (csr8 & DMA_MISSED_FRAME_M_CNTR); - stats->rx_missed_errors += miss_f; x->rx_missed_cntr += miss_f; } } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index 6a011d8633e8..89a14084c611 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -13,8 +13,7 @@ #include "dwmac4.h" #include "dwmac4_descs.h" -static int dwmac4_wrback_get_tx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int dwmac4_wrback_get_tx_status(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr) { @@ -40,15 +39,13 @@ static int dwmac4_wrback_get_tx_status(struct net_device_stats *stats, x->tx_frame_flushed++; if (unlikely(tdes3 & TDES3_LOSS_CARRIER)) { x->tx_losscarrier++; - stats->tx_carrier_errors++; } if (unlikely(tdes3 & TDES3_NO_CARRIER)) { x->tx_carrier++; - stats->tx_carrier_errors++; } if (unlikely((tdes3 & TDES3_LATE_COLLISION) || (tdes3 & TDES3_EXCESSIVE_COLLISION))) - stats->collisions += + x->tx_collision += (tdes3 & TDES3_COLLISION_COUNT_MASK) >> TDES3_COLLISION_COUNT_SHIFT; @@ -73,8 +70,7 @@ static int dwmac4_wrback_get_tx_status(struct net_device_stats *stats, return ret; } -static int dwmac4_wrback_get_rx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int dwmac4_wrback_get_rx_status(struct stmmac_extra_stats *x, struct dma_desc *p) { unsigned int rdes1 = le32_to_cpu(p->des1); @@ -93,7 +89,7 @@ static int dwmac4_wrback_get_rx_status(struct net_device_stats *stats, if (unlikely(rdes3 & RDES3_ERROR_SUMMARY)) { if (unlikely(rdes3 & RDES3_GIANT_PACKET)) - stats->rx_length_errors++; + x->rx_length++; if (unlikely(rdes3 & RDES3_OVERFLOW_ERROR)) x->rx_gmac_overflow++; @@ -103,10 +99,8 @@ static int dwmac4_wrback_get_rx_status(struct net_device_stats *stats, if (unlikely(rdes3 & RDES3_RECEIVE_ERROR)) x->rx_mii++; - if (unlikely(rdes3 & RDES3_CRC_ERROR)) { + if (unlikely(rdes3 & RDES3_CRC_ERROR)) x->rx_crc_errors++; - stats->rx_crc_errors++; - } if (unlikely(rdes3 & RDES3_DRIBBLE_ERROR)) x->dribbling_bit++; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c index 03ceb6a94073..980e5f8a37ec 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c @@ -171,6 +171,8 @@ int dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr, const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs; u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, chan)); u32 intr_en = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; int ret = 0; if (dir == DMA_DIR_RX) @@ -198,18 +200,19 @@ int dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr, } } /* TX/RX NORMAL interrupts */ - if (likely(intr_status & DMA_CHAN_STATUS_NIS)) - x->normal_irq_n++; if (likely(intr_status & DMA_CHAN_STATUS_RI)) { - x->rx_normal_irq_n++; - x->rxq_stats[chan].rx_normal_irq_n++; + u64_stats_update_begin(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_normal_irq_n++; + u64_stats_update_end(&rx_q->rxq_stats.syncp); ret |= handle_rx; } if (likely(intr_status & DMA_CHAN_STATUS_TI)) { - x->tx_normal_irq_n++; - x->txq_stats[chan].tx_normal_irq_n++; + u64_stats_update_begin(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_normal_irq_n++; + u64_stats_update_end(&tx_q->txq_stats.syncp); ret |= handle_tx; } + if (unlikely(intr_status & DMA_CHAN_STATUS_TBU)) ret |= handle_tx; if (unlikely(intr_status & DMA_CHAN_STATUS_ERI)) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c index 0b6f999a8305..aaa09b16b016 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c @@ -10,6 +10,7 @@ #include <linux/iopoll.h> #include "common.h" #include "dwmac_dma.h" +#include "stmmac.h" #define GMAC_HI_REG_AE 0x80000000 @@ -161,6 +162,8 @@ static void show_rx_process_state(unsigned int status) int dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr, struct stmmac_extra_stats *x, u32 chan, u32 dir) { + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; int ret = 0; /* read the status register (CSR5) */ u32 intr_status = readl(ioaddr + DMA_STATUS); @@ -208,17 +211,20 @@ int dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr, } /* TX/RX NORMAL interrupts */ if (likely(intr_status & DMA_STATUS_NIS)) { - x->normal_irq_n++; if (likely(intr_status & DMA_STATUS_RI)) { u32 value = readl(ioaddr + DMA_INTR_ENA); /* to schedule NAPI on real RIE event. */ if (likely(value & DMA_INTR_ENA_RIE)) { - x->rx_normal_irq_n++; + u64_stats_update_begin(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_normal_irq_n++; + u64_stats_update_end(&rx_q->rxq_stats.syncp); ret |= handle_rx; } } if (likely(intr_status & DMA_STATUS_TI)) { - x->tx_normal_irq_n++; + u64_stats_update_begin(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_normal_irq_n++; + u64_stats_update_end(&tx_q->txq_stats.syncp); ret |= handle_tx; } if (unlikely(intr_status & DMA_STATUS_ERI)) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h index 1913385df685..153321fe42c3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h @@ -165,7 +165,7 @@ #define XGMAC_DCS_SHIFT 16 #define XGMAC_ADDRx_LOW(x) (0x00000304 + (x) * 0x8) #define XGMAC_L3L4_ADDR_CTRL 0x00000c00 -#define XGMAC_IDDR GENMASK(15, 8) +#define XGMAC_IDDR GENMASK(16, 8) #define XGMAC_IDDR_SHIFT 8 #define XGMAC_IDDR_FNUM 4 #define XGMAC_TT BIT(1) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c index 13c347ee8be9..fc82862a612c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c @@ -8,8 +8,7 @@ #include "common.h" #include "dwxgmac2.h" -static int dwxgmac2_get_tx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int dwxgmac2_get_tx_status(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr) { unsigned int tdes3 = le32_to_cpu(p->des3); @@ -23,8 +22,7 @@ static int dwxgmac2_get_tx_status(struct net_device_stats *stats, return ret; } -static int dwxgmac2_get_rx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int dwxgmac2_get_rx_status(struct stmmac_extra_stats *x, struct dma_desc *p) { unsigned int rdes3 = le32_to_cpu(p->des3); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c index 070bd912580b..b09395f5edcb 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c @@ -337,6 +337,8 @@ static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv, struct stmmac_extra_stats *x, u32 chan, u32 dir) { + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; u32 intr_status = readl(ioaddr + XGMAC_DMA_CH_STATUS(chan)); u32 intr_en = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan)); int ret = 0; @@ -364,16 +366,16 @@ static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv, /* TX/RX NORMAL interrupts */ if (likely(intr_status & XGMAC_NIS)) { - x->normal_irq_n++; - if (likely(intr_status & XGMAC_RI)) { - x->rx_normal_irq_n++; - x->rxq_stats[chan].rx_normal_irq_n++; + u64_stats_update_begin(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_normal_irq_n++; + u64_stats_update_end(&rx_q->rxq_stats.syncp); ret |= handle_rx; } if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) { - x->tx_normal_irq_n++; - x->txq_stats[chan].tx_normal_irq_n++; + u64_stats_update_begin(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_normal_irq_n++; + u64_stats_update_end(&tx_q->txq_stats.syncp); ret |= handle_tx; } } @@ -408,6 +410,16 @@ static int dwxgmac2_get_hw_feature(void __iomem *ioaddr, /* MAC HW feature 1 */ hw_cap = readl(ioaddr + XGMAC_HW_FEATURE1); dma_cap->l3l4fnum = (hw_cap & XGMAC_HWFEAT_L3L4FNUM) >> 27; + /* If L3L4FNUM < 8, then the number of L3L4 filters supported by + * XGMAC is equal to L3L4FNUM. From L3L4FNUM >= 8 the number of + * L3L4 filters goes on like 8, 16, 32, ... Current maximum of + * L3L4FNUM = 10. + */ + if (dma_cap->l3l4fnum >= 8 && dma_cap->l3l4fnum <= 10) + dma_cap->l3l4fnum = 8 << (dma_cap->l3l4fnum - 8); + else if (dma_cap->l3l4fnum > 10) + dma_cap->l3l4fnum = 32; + dma_cap->hash_tb_sz = (hw_cap & XGMAC_HWFEAT_HASHTBLSZ) >> 24; dma_cap->rssen = (hw_cap & XGMAC_HWFEAT_RSSEN) >> 20; dma_cap->tsoen = (hw_cap & XGMAC_HWFEAT_TSOEN) >> 18; diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c index a91d8f13a931..937b7a0466fc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c @@ -12,8 +12,7 @@ #include "common.h" #include "descs_com.h" -static int enh_desc_get_tx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int enh_desc_get_tx_status(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr) { unsigned int tdes0 = le32_to_cpu(p->des0); @@ -38,15 +37,13 @@ static int enh_desc_get_tx_status(struct net_device_stats *stats, if (unlikely(tdes0 & ETDES0_LOSS_CARRIER)) { x->tx_losscarrier++; - stats->tx_carrier_errors++; } if (unlikely(tdes0 & ETDES0_NO_CARRIER)) { x->tx_carrier++; - stats->tx_carrier_errors++; } if (unlikely((tdes0 & ETDES0_LATE_COLLISION) || (tdes0 & ETDES0_EXCESSIVE_COLLISIONS))) - stats->collisions += + x->tx_collision += (tdes0 & ETDES0_COLLISION_COUNT_MASK) >> 3; if (unlikely(tdes0 & ETDES0_EXCESSIVE_DEFERRAL)) @@ -117,8 +114,7 @@ static int enh_desc_coe_rdes0(int ipc_err, int type, int payload_err) return ret; } -static void enh_desc_get_ext_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static void enh_desc_get_ext_status(struct stmmac_extra_stats *x, struct dma_extended_desc *p) { unsigned int rdes0 = le32_to_cpu(p->basic.des0); @@ -182,8 +178,7 @@ static void enh_desc_get_ext_status(struct net_device_stats *stats, } } -static int enh_desc_get_rx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int enh_desc_get_rx_status(struct stmmac_extra_stats *x, struct dma_desc *p) { unsigned int rdes0 = le32_to_cpu(p->des0); @@ -193,14 +188,14 @@ static int enh_desc_get_rx_status(struct net_device_stats *stats, return dma_own; if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) { - stats->rx_length_errors++; + x->rx_length++; return discard_frame; } if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) { if (unlikely(rdes0 & RDES0_DESCRIPTOR_ERROR)) { x->rx_desc++; - stats->rx_length_errors++; + x->rx_length++; } if (unlikely(rdes0 & RDES0_OVERFLOW_ERROR)) x->rx_gmac_overflow++; @@ -209,7 +204,7 @@ static int enh_desc_get_rx_status(struct net_device_stats *stats, pr_err("\tIPC Csum Error/Giant frame\n"); if (unlikely(rdes0 & RDES0_COLLISION)) - stats->collisions++; + x->rx_collision++; if (unlikely(rdes0 & RDES0_RECEIVE_WATCHDOG)) x->rx_watchdog++; @@ -218,7 +213,6 @@ static int enh_desc_get_rx_status(struct net_device_stats *stats, if (unlikely(rdes0 & RDES0_CRC_ERROR)) { x->rx_crc_errors++; - stats->rx_crc_errors++; } ret = discard_frame; } diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 6ee7cf07cfd7..652af8f6e75f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -57,8 +57,7 @@ struct stmmac_desc_ops { /* Last tx segment reports the transmit status */ int (*get_tx_ls)(struct dma_desc *p); /* Return the transmit status looking at the TDES1 */ - int (*tx_status)(struct net_device_stats *stats, - struct stmmac_extra_stats *x, + int (*tx_status)(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr); /* Get the buffer size from the descriptor */ int (*get_tx_len)(struct dma_desc *p); @@ -67,11 +66,9 @@ struct stmmac_desc_ops { /* Get the receive frame size */ int (*get_rx_frame_len)(struct dma_desc *p, int rx_coe_type); /* Return the reception status looking at the RDES1 */ - int (*rx_status)(struct net_device_stats *stats, - struct stmmac_extra_stats *x, + int (*rx_status)(struct stmmac_extra_stats *x, struct dma_desc *p); - void (*rx_extended_status)(struct net_device_stats *stats, - struct stmmac_extra_stats *x, + void (*rx_extended_status)(struct stmmac_extra_stats *x, struct dma_extended_desc *p); /* Set tx timestamp enable bit */ void (*enable_tx_timestamp) (struct dma_desc *p); @@ -191,8 +188,7 @@ struct stmmac_dma_ops { void (*dma_tx_mode)(struct stmmac_priv *priv, void __iomem *ioaddr, int mode, u32 channel, int fifosz, u8 qmode); /* To track extra statistic (if supported) */ - void (*dma_diagnostic_fr)(struct net_device_stats *stats, - struct stmmac_extra_stats *x, + void (*dma_diagnostic_fr)(struct stmmac_extra_stats *x, void __iomem *ioaddr); void (*enable_dma_transmission) (void __iomem *ioaddr); void (*enable_dma_irq)(struct stmmac_priv *priv, void __iomem *ioaddr, diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c index 350e6670a576..68a7cfcb1d8f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c @@ -12,8 +12,7 @@ #include "common.h" #include "descs_com.h" -static int ndesc_get_tx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int ndesc_get_tx_status(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr) { unsigned int tdes0 = le32_to_cpu(p->des0); @@ -31,15 +30,12 @@ static int ndesc_get_tx_status(struct net_device_stats *stats, if (unlikely(tdes0 & TDES0_ERROR_SUMMARY)) { if (unlikely(tdes0 & TDES0_UNDERFLOW_ERROR)) { x->tx_underflow++; - stats->tx_fifo_errors++; } if (unlikely(tdes0 & TDES0_NO_CARRIER)) { x->tx_carrier++; - stats->tx_carrier_errors++; } if (unlikely(tdes0 & TDES0_LOSS_CARRIER)) { x->tx_losscarrier++; - stats->tx_carrier_errors++; } if (unlikely((tdes0 & TDES0_EXCESSIVE_DEFERRAL) || (tdes0 & TDES0_EXCESSIVE_COLLISIONS) || @@ -47,7 +43,7 @@ static int ndesc_get_tx_status(struct net_device_stats *stats, unsigned int collisions; collisions = (tdes0 & TDES0_COLLISION_COUNT_MASK) >> 3; - stats->collisions += collisions; + x->tx_collision += collisions; } ret = tx_err; } @@ -70,8 +66,7 @@ static int ndesc_get_tx_len(struct dma_desc *p) * and, if required, updates the multicast statistics. * In case of success, it returns good_frame because the GMAC device * is supposed to be able to compute the csum in HW. */ -static int ndesc_get_rx_status(struct net_device_stats *stats, - struct stmmac_extra_stats *x, +static int ndesc_get_rx_status(struct stmmac_extra_stats *x, struct dma_desc *p) { int ret = good_frame; @@ -81,7 +76,7 @@ static int ndesc_get_rx_status(struct net_device_stats *stats, return dma_own; if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) { - stats->rx_length_errors++; + x->rx_length++; return discard_frame; } @@ -96,11 +91,9 @@ static int ndesc_get_rx_status(struct net_device_stats *stats, x->ipc_csum_error++; if (unlikely(rdes0 & RDES0_COLLISION)) { x->rx_collision++; - stats->collisions++; } if (unlikely(rdes0 & RDES0_CRC_ERROR)) { x->rx_crc_errors++; - stats->rx_crc_errors++; } ret = discard_frame; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 07ea5ab0a60b..4ce5eaaae513 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -77,6 +77,7 @@ struct stmmac_tx_queue { dma_addr_t dma_tx_phy; dma_addr_t tx_tail_addr; u32 mss; + struct stmmac_txq_stats txq_stats; }; struct stmmac_rx_buffer { @@ -121,6 +122,7 @@ struct stmmac_rx_queue { unsigned int len; unsigned int error; } state; + struct stmmac_rxq_stats rxq_stats; }; struct stmmac_channel { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 2ae73ab842d4..b7ac7abecdd3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -89,14 +89,6 @@ static const struct stmmac_stats stmmac_gstrings_stats[] = { /* Tx/Rx IRQ Events */ STMMAC_STAT(rx_early_irq), STMMAC_STAT(threshold), - STMMAC_STAT(tx_pkt_n), - STMMAC_STAT(rx_pkt_n), - STMMAC_STAT(normal_irq_n), - STMMAC_STAT(rx_normal_irq_n), - STMMAC_STAT(napi_poll), - STMMAC_STAT(tx_normal_irq_n), - STMMAC_STAT(tx_clean), - STMMAC_STAT(tx_set_ic_bit), STMMAC_STAT(irq_receive_pmt_irq_n), /* MMC info */ STMMAC_STAT(mmc_tx_irq_n), @@ -163,9 +155,6 @@ static const struct stmmac_stats stmmac_gstrings_stats[] = { STMMAC_STAT(mtl_rx_fifo_ctrl_active), STMMAC_STAT(mac_rx_frame_ctrl_fifo), STMMAC_STAT(mac_gmii_rx_proto_engine), - /* TSO */ - STMMAC_STAT(tx_tso_frames), - STMMAC_STAT(tx_tso_nfrags), /* EST */ STMMAC_STAT(mtl_est_cgce), STMMAC_STAT(mtl_est_hlbs), @@ -175,6 +164,23 @@ static const struct stmmac_stats stmmac_gstrings_stats[] = { }; #define STMMAC_STATS_LEN ARRAY_SIZE(stmmac_gstrings_stats) +/* statistics collected in queue which will be summed up for all TX or RX + * queues, or summed up for both TX and RX queues(napi_poll, normal_irq_n). + */ +static const char stmmac_qstats_string[][ETH_GSTRING_LEN] = { + "rx_pkt_n", + "rx_normal_irq_n", + "tx_pkt_n", + "tx_normal_irq_n", + "tx_clean", + "tx_set_ic_bit", + "tx_tso_frames", + "tx_tso_nfrags", + "normal_irq_n", + "napi_poll", +}; +#define STMMAC_QSTATS ARRAY_SIZE(stmmac_qstats_string) + /* HW MAC Management counters (if supported) */ #define STMMAC_MMC_STAT(m) \ { #m, sizeof_field(struct stmmac_counters, m), \ @@ -535,23 +541,44 @@ static void stmmac_get_per_qstats(struct stmmac_priv *priv, u64 *data) { u32 tx_cnt = priv->plat->tx_queues_to_use; u32 rx_cnt = priv->plat->rx_queues_to_use; + unsigned int start; int q, stat; + u64 *pos; char *p; + pos = data; for (q = 0; q < tx_cnt; q++) { - p = (char *)priv + offsetof(struct stmmac_priv, - xstats.txq_stats[q].tx_pkt_n); + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[q]; + struct stmmac_txq_stats snapshot; + + data = pos; + do { + start = u64_stats_fetch_begin(&tx_q->txq_stats.syncp); + snapshot = tx_q->txq_stats; + } while (u64_stats_fetch_retry(&tx_q->txq_stats.syncp, start)); + + p = (char *)&snapshot + offsetof(struct stmmac_txq_stats, tx_pkt_n); for (stat = 0; stat < STMMAC_TXQ_STATS; stat++) { - *data++ = (*(unsigned long *)p); - p += sizeof(unsigned long); + *data++ += (*(u64 *)p); + p += sizeof(u64); } } + + pos = data; for (q = 0; q < rx_cnt; q++) { - p = (char *)priv + offsetof(struct stmmac_priv, - xstats.rxq_stats[q].rx_pkt_n); + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[q]; + struct stmmac_rxq_stats snapshot; + + data = pos; + do { + start = u64_stats_fetch_begin(&rx_q->rxq_stats.syncp); + snapshot = rx_q->rxq_stats; + } while (u64_stats_fetch_retry(&rx_q->rxq_stats.syncp, start)); + + p = (char *)&snapshot + offsetof(struct stmmac_rxq_stats, rx_pkt_n); for (stat = 0; stat < STMMAC_RXQ_STATS; stat++) { - *data++ = (*(unsigned long *)p); - p += sizeof(unsigned long); + *data++ += (*(u64 *)p); + p += sizeof(u64); } } } @@ -562,8 +589,10 @@ static void stmmac_get_ethtool_stats(struct net_device *dev, struct stmmac_priv *priv = netdev_priv(dev); u32 rx_queues_count = priv->plat->rx_queues_to_use; u32 tx_queues_count = priv->plat->tx_queues_to_use; + u64 napi_poll = 0, normal_irq_n = 0; + int i, j = 0, pos, ret; unsigned long count; - int i, j = 0, ret; + unsigned int start; if (priv->dma_cap.asp) { for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) { @@ -574,8 +603,7 @@ static void stmmac_get_ethtool_stats(struct net_device *dev, } /* Update the DMA HW counters for dwmac10/100 */ - ret = stmmac_dma_diagnostic_fr(priv, &dev->stats, (void *) &priv->xstats, - priv->ioaddr); + ret = stmmac_dma_diagnostic_fr(priv, &priv->xstats, priv->ioaddr); if (ret) { /* If supported, for new GMAC chips expose the MMC counters */ if (priv->dma_cap.rmon) { @@ -606,6 +634,48 @@ static void stmmac_get_ethtool_stats(struct net_device *dev, data[j++] = (stmmac_gstrings_stats[i].sizeof_stat == sizeof(u64)) ? (*(u64 *)p) : (*(u32 *)p); } + + pos = j; + for (i = 0; i < rx_queues_count; i++) { + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[i]; + struct stmmac_rxq_stats snapshot; + + j = pos; + do { + start = u64_stats_fetch_begin(&rx_q->rxq_stats.syncp); + snapshot = rx_q->rxq_stats; + } while (u64_stats_fetch_retry(&rx_q->rxq_stats.syncp, start)); + + data[j++] += snapshot.rx_pkt_n; + data[j++] += snapshot.rx_normal_irq_n; + normal_irq_n += snapshot.rx_normal_irq_n; + napi_poll += snapshot.napi_poll; + } + + pos = j; + for (i = 0; i < tx_queues_count; i++) { + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[i]; + struct stmmac_txq_stats snapshot; + + j = pos; + do { + start = u64_stats_fetch_begin(&tx_q->txq_stats.syncp); + snapshot = tx_q->txq_stats; + } while (u64_stats_fetch_retry(&tx_q->txq_stats.syncp, start)); + + data[j++] += snapshot.tx_pkt_n; + data[j++] += snapshot.tx_normal_irq_n; + normal_irq_n += snapshot.tx_normal_irq_n; + data[j++] += snapshot.tx_clean; + data[j++] += snapshot.tx_set_ic_bit; + data[j++] += snapshot.tx_tso_frames; + data[j++] += snapshot.tx_tso_nfrags; + napi_poll += snapshot.napi_poll; + } + normal_irq_n += priv->xstats.rx_early_irq; + data[j++] = normal_irq_n; + data[j++] = napi_poll; + stmmac_get_per_qstats(priv, &data[j]); } @@ -618,7 +688,7 @@ static int stmmac_get_sset_count(struct net_device *netdev, int sset) switch (sset) { case ETH_SS_STATS: - len = STMMAC_STATS_LEN + + len = STMMAC_STATS_LEN + STMMAC_QSTATS + STMMAC_TXQ_STATS * tx_cnt + STMMAC_RXQ_STATS * rx_cnt; @@ -691,8 +761,11 @@ static void stmmac_get_strings(struct net_device *dev, u32 stringset, u8 *data) p += ETH_GSTRING_LEN; } for (i = 0; i < STMMAC_STATS_LEN; i++) { - memcpy(p, stmmac_gstrings_stats[i].stat_string, - ETH_GSTRING_LEN); + memcpy(p, stmmac_gstrings_stats[i].stat_string, ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + for (i = 0; i < STMMAC_QSTATS; i++) { + memcpy(p, stmmac_qstats_string[i], ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } stmmac_get_qstats_string(priv, p); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c index 8b50f03056b7..fa2c3ba7e9fe 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c @@ -180,7 +180,7 @@ static void timestamp_interrupt(struct stmmac_priv *priv) u64 ptp_time; int i; - if (priv->plat->int_snapshot_en) { + if (priv->plat->flags & STMMAC_FLAG_INT_SNAPSHOT_EN) { wake_up(&priv->tstamp_busy_wait); return; } @@ -195,7 +195,7 @@ static void timestamp_interrupt(struct stmmac_priv *priv) */ ts_status = readl(priv->ioaddr + GMAC_TIMESTAMP_STATUS); - if (!priv->plat->ext_snapshot_en) + if (priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN) return; num_snapshot = (ts_status & GMAC_TIMESTAMP_ATSNS_MASK) >> diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 4727f7be4f86..e1f1c034d325 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -325,7 +325,7 @@ static void stmmac_clk_csr_set(struct stmmac_priv *priv) priv->clk_csr = STMMAC_CSR_250_300M; } - if (priv->plat->has_sun8i) { + if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { if (clk_rate > 160000000) priv->clk_csr = 0x03; else if (clk_rate > 80000000) @@ -421,7 +421,7 @@ static int stmmac_enable_eee_mode(struct stmmac_priv *priv) /* Check and enter in LPI mode */ if (!priv->tx_path_in_lpi_mode) stmmac_set_eee_mode(priv, priv->hw, - priv->plat->en_tx_lpi_clockgating); + priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); return 0; } @@ -991,7 +991,8 @@ static void stmmac_mac_link_up(struct phylink_config *config, struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); u32 old_ctrl, ctrl; - if (priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) + if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && + priv->plat->serdes_powerup) priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); @@ -1084,7 +1085,8 @@ static void stmmac_mac_link_up(struct phylink_config *config, stmmac_mac_set(priv, priv->ioaddr, true); if (phy && priv->dma_cap.eee) { priv->eee_active = - phy_init_eee(phy, !priv->plat->rx_clk_runs_in_lpi) >= 0; + phy_init_eee(phy, !(priv->plat->flags & + STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; priv->eee_enabled = stmmac_eee_init(priv); priv->tx_lpi_enabled = priv->eee_enabled; stmmac_set_eee_pls(priv, priv->hw, true); @@ -2432,6 +2434,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) struct dma_desc *tx_desc = NULL; struct xdp_desc xdp_desc; bool work_done = true; + u32 tx_set_ic_bit = 0; + unsigned long flags; /* Avoids TX time-out as we are sharing with slow path */ txq_trans_cond_update(nq); @@ -2492,7 +2496,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) if (set_ic) { tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, tx_desc); - priv->xstats.tx_set_ic_bit++; + tx_set_ic_bit++; } stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, @@ -2504,6 +2508,9 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); entry = tx_q->cur_tx; } + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_set_ic_bit += tx_set_ic_bit; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); if (tx_desc) { stmmac_flush_tx_descriptors(priv, queue); @@ -2545,11 +2552,11 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; unsigned int bytes_compl = 0, pkts_compl = 0; unsigned int entry, xmits = 0, count = 0; + u32 tx_packets = 0, tx_errors = 0; + unsigned long flags; __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); - priv->xstats.tx_clean++; - tx_q->xsk_frames_done = 0; entry = tx_q->dirty_tx; @@ -2580,8 +2587,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) else p = tx_q->dma_tx + entry; - status = stmmac_tx_status(priv, &priv->dev->stats, - &priv->xstats, p, priv->ioaddr); + status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); /* Check if the descriptor is owned by the DMA */ if (unlikely(status & tx_dma_own)) break; @@ -2597,13 +2603,11 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) if (likely(!(status & tx_not_ls))) { /* ... verify the status error condition */ if (unlikely(status & tx_err)) { - priv->dev->stats.tx_errors++; + tx_errors++; if (unlikely(status & tx_err_bump_tc)) stmmac_bump_dma_threshold(priv, queue); } else { - priv->dev->stats.tx_packets++; - priv->xstats.tx_pkt_n++; - priv->xstats.txq_stats[queue].tx_pkt_n++; + tx_packets++; } if (skb) stmmac_get_tx_hwtstamp(priv, p, skb); @@ -2707,6 +2711,14 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), HRTIMER_MODE_REL); + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_packets += tx_packets; + tx_q->txq_stats.tx_pkt_n += tx_packets; + tx_q->txq_stats.tx_clean++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); + + priv->xstats.tx_errors += tx_errors; + __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); /* Combine decisions from TX clean and XSK TX */ @@ -2734,7 +2746,7 @@ static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) tx_q->dma_tx_phy, chan); stmmac_start_tx_dma(priv, chan); - priv->dev->stats.tx_errors++; + priv->xstats.tx_errors++; netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); } @@ -3710,7 +3722,7 @@ static int stmmac_request_irq(struct net_device *dev) int ret; /* Request the IRQ lines */ - if (priv->plat->multi_msi_en) + if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) ret = stmmac_request_irq_multi_msi(dev); else ret = stmmac_request_irq_single(dev); @@ -3827,10 +3839,6 @@ static int __stmmac_open(struct net_device *dev, } } - /* Extra statistics */ - memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); - priv->xstats.threshold = tc; - priv->rx_copybreak = STMMAC_RX_COPYBREAK; buf_sz = dma_conf->dma_buf_sz; @@ -3838,7 +3846,8 @@ static int __stmmac_open(struct net_device *dev, stmmac_reset_queues_param(priv); - if (!priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) { + if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && + priv->plat->serdes_powerup) { ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); if (ret < 0) { netdev_err(priv->dev, "%s: Serdes powerup failed\n", @@ -4110,6 +4119,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) struct stmmac_tx_queue *tx_q; bool has_vlan, set_ic; u8 proto_hdr_len, hdr; + unsigned long flags; u32 pay_len, mss; dma_addr_t des; int i; @@ -4258,7 +4268,6 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, desc); - priv->xstats.tx_set_ic_bit++; } /* We've used all descriptors we need for this skb, however, @@ -4274,9 +4283,13 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); } - dev->stats.tx_bytes += skb->len; - priv->xstats.tx_tso_frames++; - priv->xstats.tx_tso_nfrags += nfrags; + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_bytes += skb->len; + tx_q->txq_stats.tx_tso_frames++; + tx_q->txq_stats.tx_tso_nfrags += nfrags; + if (set_ic) + tx_q->txq_stats.tx_set_ic_bit++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); if (priv->sarc_type) stmmac_set_desc_sarc(priv, first, priv->sarc_type); @@ -4326,7 +4339,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) dma_map_err: dev_err(priv->device, "Tx dma map failed\n"); dev_kfree_skb(skb); - priv->dev->stats.tx_dropped++; + priv->xstats.tx_dropped++; return NETDEV_TX_OK; } @@ -4352,6 +4365,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) struct stmmac_tx_queue *tx_q; bool has_vlan, set_ic; int entry, first_tx; + unsigned long flags; dma_addr_t des; tx_q = &priv->dma_conf.tx_queue[queue]; @@ -4480,7 +4494,6 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, desc); - priv->xstats.tx_set_ic_bit++; } /* We've used all descriptors we need for this skb, however, @@ -4507,7 +4520,11 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); } - dev->stats.tx_bytes += skb->len; + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_bytes += skb->len; + if (set_ic) + tx_q->txq_stats.tx_set_ic_bit++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); if (priv->sarc_type) stmmac_set_desc_sarc(priv, first, priv->sarc_type); @@ -4569,7 +4586,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) dma_map_err: netdev_err(priv->dev, "Tx DMA map failed\n"); dev_kfree_skb(skb); - priv->dev->stats.tx_dropped++; + priv->xstats.tx_dropped++; return NETDEV_TX_OK; } @@ -4770,9 +4787,12 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, set_ic = false; if (set_ic) { + unsigned long flags; tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, tx_desc); - priv->xstats.tx_set_ic_bit++; + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.tx_set_ic_bit++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); } stmmac_enable_dma_transmission(priv, priv->ioaddr); @@ -4917,16 +4937,18 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, struct dma_desc *p, struct dma_desc *np, struct xdp_buff *xdp) { + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; unsigned int len = xdp->data_end - xdp->data; enum pkt_hash_types hash_type; int coe = priv->hw->rx_csum; + unsigned long flags; struct sk_buff *skb; u32 hash; skb = stmmac_construct_skb_zc(ch, xdp); if (!skb) { - priv->dev->stats.rx_dropped++; + priv->xstats.rx_dropped++; return; } @@ -4945,8 +4967,10 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, skb_record_rx_queue(skb, queue); napi_gro_receive(&ch->rxtx_napi, skb); - priv->dev->stats.rx_packets++; - priv->dev->stats.rx_bytes += len; + flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_pkt_n++; + rx_q->rxq_stats.rx_bytes += len; + u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); } static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) @@ -5023,9 +5047,11 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) unsigned int count = 0, error = 0, len = 0; int dirty = stmmac_rx_dirty(priv, queue); unsigned int next_entry = rx_q->cur_rx; + u32 rx_errors = 0, rx_dropped = 0; unsigned int desc_size; struct bpf_prog *prog; bool failure = false; + unsigned long flags; int xdp_status = 0; int status = 0; @@ -5081,8 +5107,7 @@ read_again: p = rx_q->dma_rx + entry; /* read the status of the incoming frame */ - status = stmmac_rx_status(priv, &priv->dev->stats, - &priv->xstats, p); + status = stmmac_rx_status(priv, &priv->xstats, p); /* check if managed by the DMA otherwise go ahead */ if (unlikely(status & dma_own)) break; @@ -5104,8 +5129,7 @@ read_again: break; if (priv->extend_desc) - stmmac_rx_extended_status(priv, &priv->dev->stats, - &priv->xstats, + stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); if (unlikely(status == discard_frame)) { xsk_buff_free(buf->xdp); @@ -5113,7 +5137,7 @@ read_again: dirty++; error = 1; if (!priv->hwts_rx_en) - priv->dev->stats.rx_errors++; + rx_errors++; } if (unlikely(error && (status & rx_not_ls))) @@ -5161,7 +5185,7 @@ read_again: break; case STMMAC_XDP_CONSUMED: xsk_buff_free(buf->xdp); - priv->dev->stats.rx_dropped++; + rx_dropped++; break; case STMMAC_XDP_TX: case STMMAC_XDP_REDIRECT: @@ -5182,8 +5206,12 @@ read_again: stmmac_finalize_xdp_rx(priv, xdp_status); - priv->xstats.rx_pkt_n += count; - priv->xstats.rxq_stats[queue].rx_pkt_n += count; + flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_pkt_n += count; + u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); + + priv->xstats.rx_dropped += rx_dropped; + priv->xstats.rx_errors += rx_errors; if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { if (failure || stmmac_rx_dirty(priv, queue) > 0) @@ -5207,6 +5235,7 @@ read_again: */ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) { + u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; unsigned int count = 0, error = 0, len = 0; @@ -5216,6 +5245,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) unsigned int desc_size; struct sk_buff *skb = NULL; struct stmmac_xdp_buff ctx; + unsigned long flags; int xdp_status = 0; int buf_sz; @@ -5271,8 +5301,7 @@ read_again: p = rx_q->dma_rx + entry; /* read the status of the incoming frame */ - status = stmmac_rx_status(priv, &priv->dev->stats, - &priv->xstats, p); + status = stmmac_rx_status(priv, &priv->xstats, p); /* check if managed by the DMA otherwise go ahead */ if (unlikely(status & dma_own)) break; @@ -5289,14 +5318,13 @@ read_again: prefetch(np); if (priv->extend_desc) - stmmac_rx_extended_status(priv, &priv->dev->stats, - &priv->xstats, rx_q->dma_erx + entry); + stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); if (unlikely(status == discard_frame)) { page_pool_recycle_direct(rx_q->page_pool, buf->page); buf->page = NULL; error = 1; if (!priv->hwts_rx_en) - priv->dev->stats.rx_errors++; + rx_errors++; } if (unlikely(error && (status & rx_not_ls))) @@ -5364,7 +5392,7 @@ read_again: virt_to_head_page(ctx.xdp.data), sync_len, true); buf->page = NULL; - priv->dev->stats.rx_dropped++; + rx_dropped++; /* Clear skb as it was set as * status by XDP program. @@ -5393,7 +5421,7 @@ read_again: skb = napi_alloc_skb(&ch->rx_napi, buf1_len); if (!skb) { - priv->dev->stats.rx_dropped++; + rx_dropped++; count++; goto drain_data; } @@ -5413,7 +5441,7 @@ read_again: priv->dma_conf.dma_buf_sz); /* Data payload appended into SKB */ - page_pool_release_page(rx_q->page_pool, buf->page); + skb_mark_for_recycle(skb); buf->page = NULL; } @@ -5425,7 +5453,7 @@ read_again: priv->dma_conf.dma_buf_sz); /* Data payload appended into SKB */ - page_pool_release_page(rx_q->page_pool, buf->sec_page); + skb_mark_for_recycle(skb); buf->sec_page = NULL; } @@ -5453,8 +5481,8 @@ drain_data: napi_gro_receive(&ch->rx_napi, skb); skb = NULL; - priv->dev->stats.rx_packets++; - priv->dev->stats.rx_bytes += len; + rx_packets++; + rx_bytes += len; count++; } @@ -5469,8 +5497,14 @@ drain_data: stmmac_rx_refill(priv, queue); - priv->xstats.rx_pkt_n += count; - priv->xstats.rxq_stats[queue].rx_pkt_n += count; + flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.rx_packets += rx_packets; + rx_q->rxq_stats.rx_bytes += rx_bytes; + rx_q->rxq_stats.rx_pkt_n += count; + u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); + + priv->xstats.rx_dropped += rx_dropped; + priv->xstats.rx_errors += rx_errors; return count; } @@ -5480,10 +5514,15 @@ static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) struct stmmac_channel *ch = container_of(napi, struct stmmac_channel, rx_napi); struct stmmac_priv *priv = ch->priv_data; + struct stmmac_rx_queue *rx_q; u32 chan = ch->index; + unsigned long flags; int work_done; - priv->xstats.napi_poll++; + rx_q = &priv->dma_conf.rx_queue[chan]; + flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.napi_poll++; + u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); work_done = stmmac_rx(priv, budget, chan); if (work_done < budget && napi_complete_done(napi, work_done)) { @@ -5502,10 +5541,15 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) struct stmmac_channel *ch = container_of(napi, struct stmmac_channel, tx_napi); struct stmmac_priv *priv = ch->priv_data; + struct stmmac_tx_queue *tx_q; u32 chan = ch->index; + unsigned long flags; int work_done; - priv->xstats.napi_poll++; + tx_q = &priv->dma_conf.tx_queue[chan]; + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.napi_poll++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); work_done = stmmac_tx_clean(priv, budget, chan); work_done = min(work_done, budget); @@ -5527,9 +5571,20 @@ static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) container_of(napi, struct stmmac_channel, rxtx_napi); struct stmmac_priv *priv = ch->priv_data; int rx_done, tx_done, rxtx_done; + struct stmmac_rx_queue *rx_q; + struct stmmac_tx_queue *tx_q; u32 chan = ch->index; + unsigned long flags; + + rx_q = &priv->dma_conf.rx_queue[chan]; + flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); + rx_q->rxq_stats.napi_poll++; + u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); - priv->xstats.napi_poll++; + tx_q = &priv->dma_conf.tx_queue[chan]; + flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); + tx_q->txq_stats.napi_poll++; + u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); tx_done = stmmac_tx_clean(priv, budget, chan); tx_done = min(tx_done, budget); @@ -5677,7 +5732,7 @@ static netdev_features_t stmmac_fix_features(struct net_device *dev, features &= ~NETIF_F_CSUM_MASK; /* Disable tso if asked by ethtool */ - if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { + if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { if (features & NETIF_F_TSO) priv->tso = true; else @@ -5798,7 +5853,8 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) } /* PCS link status */ - if (priv->hw->pcs && !priv->plat->has_integrated_pcs) { + if (priv->hw->pcs && + !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { if (priv->xstats.pcs_link) netif_carrier_on(priv->dev); else @@ -5951,7 +6007,7 @@ static void stmmac_poll_controller(struct net_device *dev) if (test_bit(STMMAC_DOWN, &priv->state)) return; - if (priv->plat->multi_msi_en) { + if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { for (i = 0; i < priv->plat->rx_queues_to_use; i++) stmmac_msi_intr_rx(0, &priv->dma_conf.rx_queue[i]); @@ -6788,6 +6844,56 @@ int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) return 0; } +static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 rx_cnt = priv->plat->rx_queues_to_use; + unsigned int start; + int q; + + for (q = 0; q < tx_cnt; q++) { + struct stmmac_txq_stats *txq_stats = &priv->dma_conf.tx_queue[q].txq_stats; + u64 tx_packets; + u64 tx_bytes; + + do { + start = u64_stats_fetch_begin(&txq_stats->syncp); + tx_packets = txq_stats->tx_packets; + tx_bytes = txq_stats->tx_bytes; + } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); + + stats->tx_packets += tx_packets; + stats->tx_bytes += tx_bytes; + } + + for (q = 0; q < rx_cnt; q++) { + struct stmmac_rxq_stats *rxq_stats = &priv->dma_conf.rx_queue[q].rxq_stats; + u64 rx_packets; + u64 rx_bytes; + + do { + start = u64_stats_fetch_begin(&rxq_stats->syncp); + rx_packets = rxq_stats->rx_packets; + rx_bytes = rxq_stats->rx_bytes; + } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); + + stats->rx_packets += rx_packets; + stats->rx_bytes += rx_bytes; + } + + stats->rx_dropped = priv->xstats.rx_dropped; + stats->rx_errors = priv->xstats.rx_errors; + stats->tx_dropped = priv->xstats.tx_dropped; + stats->tx_errors = priv->xstats.tx_errors; + stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; + stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; + stats->rx_length_errors = priv->xstats.rx_length; + stats->rx_crc_errors = priv->xstats.rx_crc_errors; + stats->rx_over_errors = priv->xstats.rx_overflow_cntr; + stats->rx_missed_errors = priv->xstats.rx_missed_cntr; +} + static const struct net_device_ops stmmac_netdev_ops = { .ndo_open = stmmac_open, .ndo_start_xmit = stmmac_xmit, @@ -6798,6 +6904,7 @@ static const struct net_device_ops stmmac_netdev_ops = { .ndo_set_rx_mode = stmmac_set_rx_mode, .ndo_tx_timeout = stmmac_tx_timeout, .ndo_eth_ioctl = stmmac_ioctl, + .ndo_get_stats64 = stmmac_get_stats64, .ndo_setup_tc = stmmac_setup_tc, .ndo_select_queue = stmmac_select_queue, #ifdef CONFIG_NET_POLL_CONTROLLER @@ -6855,7 +6962,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv) int ret; /* dwmac-sun8i only work in chain mode */ - if (priv->plat->has_sun8i) + if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) chain_mode = 1; priv->chain_mode = chain_mode; @@ -6876,7 +6983,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv) */ priv->plat->enh_desc = priv->dma_cap.enh_desc; priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && - !priv->plat->use_phy_wol; + !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); priv->hw->pmt = priv->plat->pmt; if (priv->dma_cap.hash_tb_sz) { priv->hw->multicast_filter_bins = @@ -6920,7 +7027,8 @@ static int stmmac_hw_init(struct stmmac_priv *priv) if (priv->dma_cap.tsoen) dev_info(priv->device, "TSO supported\n"); - priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en; + priv->hw->vlan_fail_q_en = + (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; /* Run HW quirks, if any */ @@ -7160,12 +7268,18 @@ int stmmac_dvr_probe(struct device *device, priv->device = device; priv->dev = ndev; + for (i = 0; i < MTL_MAX_RX_QUEUES; i++) + u64_stats_init(&priv->dma_conf.rx_queue[i].rxq_stats.syncp); + for (i = 0; i < MTL_MAX_TX_QUEUES; i++) + u64_stats_init(&priv->dma_conf.tx_queue[i].txq_stats.syncp); + stmmac_set_ethtool_ops(ndev); priv->pause = pause; priv->plat = plat_dat; priv->ioaddr = res->addr; priv->dev->base_addr = (unsigned long)res->addr; - priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en; + priv->plat->dma_cfg->multi_msi_en = + (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; @@ -7249,7 +7363,7 @@ int stmmac_dvr_probe(struct device *device, ndev->hw_features |= NETIF_F_HW_TC; } - if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { + if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; if (priv->plat->has_gmac4) ndev->hw_features |= NETIF_F_GSO_UDP_L4; @@ -7257,7 +7371,8 @@ int stmmac_dvr_probe(struct device *device, dev_info(priv->device, "TSO feature enabled\n"); } - if (priv->dma_cap.sphen && !priv->plat->sph_disable) { + if (priv->dma_cap.sphen && + !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { ndev->hw_features |= NETIF_F_GRO; priv->sph_cap = true; priv->sph = priv->sph_cap; @@ -7315,6 +7430,8 @@ int stmmac_dvr_probe(struct device *device, #endif priv->msg_enable = netif_msg_init(debug, default_msg_level); + priv->xstats.threshold = tc; + /* Initialize RSS */ rxq = priv->plat->rx_queues_to_use; netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); @@ -7621,7 +7738,8 @@ int stmmac_resume(struct device *dev) stmmac_mdio_reset(priv->mii); } - if (!priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) { + if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && + priv->plat->serdes_powerup) { ret = priv->plat->serdes_powerup(ndev, priv->plat->bsp_priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index 3db1cb0fd160..dd9e2fec5328 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -62,11 +62,16 @@ static void stmmac_xgmac2_c45_format(struct stmmac_priv *priv, int phyaddr, static void stmmac_xgmac2_c22_format(struct stmmac_priv *priv, int phyaddr, int phyreg, u32 *hw_addr) { - u32 tmp; + u32 tmp = 0; + if (priv->synopsys_id < DWXGMAC_CORE_2_20) { + /* Until ver 2.20 XGMAC does not support C22 addr >= 4. Those + * bits above bit 3 of XGMAC_MDIO_C22P register are reserved. + */ + tmp = readl(priv->ioaddr + XGMAC_MDIO_C22P); + tmp &= ~MII_XGMAC_C22P_MASK; + } /* Set port as Clause 22 */ - tmp = readl(priv->ioaddr + XGMAC_MDIO_C22P); - tmp &= ~MII_XGMAC_C22P_MASK; tmp |= BIT(phyaddr); writel(tmp, priv->ioaddr + XGMAC_MDIO_C22P); @@ -132,8 +137,9 @@ static int stmmac_xgmac2_mdio_read_c22(struct mii_bus *bus, int phyaddr, priv = netdev_priv(ndev); - /* HW does not support C22 addr >= 4 */ - if (phyaddr > MII_XGMAC_MAX_C22ADDR) + /* Until ver 2.20 XGMAC does not support C22 addr >= 4 */ + if (priv->synopsys_id < DWXGMAC_CORE_2_20 && + phyaddr > MII_XGMAC_MAX_C22ADDR) return -ENODEV; stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr); @@ -209,8 +215,9 @@ static int stmmac_xgmac2_mdio_write_c22(struct mii_bus *bus, int phyaddr, priv = netdev_priv(ndev); - /* HW does not support C22 addr >= 4 */ - if (phyaddr > MII_XGMAC_MAX_C22ADDR) + /* Until ver 2.20 XGMAC does not support C22 addr >= 4 */ + if (priv->synopsys_id < DWXGMAC_CORE_2_20 && + phyaddr > MII_XGMAC_MAX_C22ADDR) return -ENODEV; stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr); @@ -551,13 +558,18 @@ int stmmac_mdio_register(struct net_device *ndev) new_bus->read_c45 = &stmmac_xgmac2_mdio_read_c45; new_bus->write_c45 = &stmmac_xgmac2_mdio_write_c45; - /* Right now only C22 phys are supported */ - max_addr = MII_XGMAC_MAX_C22ADDR + 1; + if (priv->synopsys_id < DWXGMAC_CORE_2_20) { + /* Right now only C22 phys are supported */ + max_addr = MII_XGMAC_MAX_C22ADDR + 1; - /* Check if DT specified an unsupported phy addr */ - if (priv->plat->phy_addr > MII_XGMAC_MAX_C22ADDR) - dev_err(dev, "Unsupported phy_addr (max=%d)\n", + /* Check if DT specified an unsupported phy addr */ + if (priv->plat->phy_addr > MII_XGMAC_MAX_C22ADDR) + dev_err(dev, "Unsupported phy_addr (max=%d)\n", MII_XGMAC_MAX_C22ADDR); + } else { + /* XGMAC version 2.20 onwards support 32 phy addr */ + max_addr = PHY_MAX_ADDR; + } } else { new_bus->read = &stmmac_mdio_read_c22; new_bus->write = &stmmac_mdio_write_c22; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 644bb54f5f02..352b01678c22 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -77,7 +77,7 @@ static int snps_gmac5_default_data(struct pci_dev *pdev, plat->clk_csr = 5; plat->has_gmac4 = 1; plat->force_sf_dma_mode = 1; - plat->tso_en = 1; + plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; /* Set default value for multicast hash bins */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 231152ee5a32..be8e79c7aa34 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -15,7 +15,6 @@ #include <linux/io.h> #include <linux/of.h> #include <linux/of_net.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include "stmmac.h" @@ -466,8 +465,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) plat->force_sf_dma_mode = of_property_read_bool(np, "snps,force_sf_dma_mode"); - plat->en_tx_lpi_clockgating = - of_property_read_bool(np, "snps,en-tx-lpi-clockgating"); + if (of_property_read_bool(np, "snps,en-tx-lpi-clockgating")) + plat->flags |= STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; /* Set the maxmtu to a default of JUMBO_LEN in case the * parameter is not present in the device tree. @@ -525,7 +524,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) plat->has_gmac4 = 1; plat->has_gmac = 0; plat->pmt = 1; - plat->tso_en = of_property_read_bool(np, "snps,tso"); + if (of_property_read_bool(np, "snps,tso")) + plat->flags |= STMMAC_FLAG_TSO_EN; } if (of_device_is_compatible(np, "snps,dwmac-3.610") || @@ -538,7 +538,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) if (of_device_is_compatible(np, "snps,dwxgmac")) { plat->has_xgmac = 1; plat->pmt = 1; - plat->tso_en = of_property_read_bool(np, "snps,tso"); + if (of_property_read_bool(np, "snps,tso")) + plat->flags |= STMMAC_FLAG_TSO_EN; } dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*dma_cfg), diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index b4388ca8d211..3d7825cb30bb 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -192,7 +192,10 @@ static int stmmac_enable(struct ptp_clock_info *ptp, write_unlock_irqrestore(&priv->ptp_lock, flags); break; case PTP_CLK_REQ_EXTTS: - priv->plat->ext_snapshot_en = on; + if (on) + priv->plat->flags |= STMMAC_FLAG_EXT_SNAPSHOT_EN; + else + priv->plat->flags &= ~STMMAC_FLAG_EXT_SNAPSHOT_EN; mutex_lock(&priv->aux_ts_lock); acr_value = readl(ptpaddr + PTP_ACR); acr_value &= ~PTP_ACR_MASK; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index 687f43cd466c..f9e43fc32ee8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -1355,7 +1355,7 @@ static int __stmmac_test_l3filt(struct stmmac_priv *priv, u32 dst, u32 src, goto cleanup_rss; } - dissector->used_keys |= (1 << FLOW_DISSECTOR_KEY_IPV4_ADDRS); + dissector->used_keys |= (1ULL << FLOW_DISSECTOR_KEY_IPV4_ADDRS); dissector->offset[FLOW_DISSECTOR_KEY_IPV4_ADDRS] = 0; cls = kzalloc(sizeof(*cls), GFP_KERNEL); @@ -1481,8 +1481,8 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, goto cleanup_rss; } - dissector->used_keys |= (1 << FLOW_DISSECTOR_KEY_BASIC); - dissector->used_keys |= (1 << FLOW_DISSECTOR_KEY_PORTS); + dissector->used_keys |= (1ULL << FLOW_DISSECTOR_KEY_BASIC); + dissector->used_keys |= (1ULL << FLOW_DISSECTOR_KEY_PORTS); dissector->offset[FLOW_DISSECTOR_KEY_BASIC] = 0; dissector->offset[FLOW_DISSECTOR_KEY_PORTS] = offsetof(typeof(keys), key); diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c index 7a2e76776297..011d74087f86 100644 --- a/drivers/net/ethernet/sun/niu.c +++ b/drivers/net/ethernet/sun/niu.c @@ -31,7 +31,7 @@ #include <linux/slab.h> #include <linux/io.h> -#include <linux/of_device.h> +#include <linux/of.h> #include "niu.h" diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c index 34b94153bf0c..cc34d92d2e3d 100644 --- a/drivers/net/ethernet/sun/sunbmac.c +++ b/drivers/net/ethernet/sun/sunbmac.c @@ -25,7 +25,7 @@ #include <linux/bitops.h> #include <linux/dma-mapping.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/gfp.h> #include <asm/auxio.h> diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c index 4154e68639ac..9bd1df8308d2 100644 --- a/drivers/net/ethernet/sun/sungem.c +++ b/drivers/net/ethernet/sun/sungem.c @@ -40,6 +40,7 @@ #include <linux/bitops.h> #include <linux/mm.h> #include <linux/gfp.h> +#include <linux/of.h> #include <asm/io.h> #include <asm/byteorder.h> diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index b93613cd1994..b983b9c23be6 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -32,9 +32,10 @@ #include <linux/mm.h> #include <linux/module.h> #include <linux/netdevice.h> -#include <linux/of_device.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/pci.h> +#include <linux/platform_device.h> #include <linux/random.h> #include <linux/skbuff.h> #include <linux/slab.h> diff --git a/drivers/net/ethernet/sun/sunqe.c b/drivers/net/ethernet/sun/sunqe.c index 6418fcc3139f..b37360f44972 100644 --- a/drivers/net/ethernet/sun/sunqe.c +++ b/drivers/net/ethernet/sun/sunqe.c @@ -27,8 +27,8 @@ #include <linux/bitops.h> #include <linux/dma-mapping.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/pgtable.h> +#include <linux/platform_device.h> #include <asm/io.h> #include <asm/dma.h> diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index fce06663e1e1..63e510b6860f 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -183,4 +183,17 @@ config CPMAC help TI AR7 CPMAC Ethernet support +config TI_ICSSG_PRUETH + tristate "TI Gigabit PRU Ethernet driver" + select PHYLIB + depends on PRU_REMOTEPROC + depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER + help + Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem. + This subsystem is available starting with the AM65 platform. + + This driver requires firmware binaries which will run on the PRUs + to support the Ethernet operation. Currently, it supports Ethernet + with 1G and 100M link speed. + endif # NET_VENDOR_TI diff --git a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile index 75f761efbea7..9176d79c36e1 100644 --- a/drivers/net/ethernet/ti/Makefile +++ b/drivers/net/ethernet/ti/Makefile @@ -28,3 +28,13 @@ obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o cpsw_ale.o k3-cppi-desc-pool.o am65-cpsw-qos.o ti-am65-cpsw-nuss-$(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV) += am65-cpsw-switchdev.o obj-$(CONFIG_TI_K3_AM65_CPTS) += am65-cpts.o + +obj-$(CONFIG_TI_ICSSG_PRUETH) += icssg-prueth.o +icssg-prueth-y := k3-cppi-desc-pool.o \ + icssg/icssg_prueth.o \ + icssg/icssg_classifier.o \ + icssg/icssg_queues.o \ + icssg/icssg_config.o \ + icssg/icssg_mii_cfg.o \ + icssg/icssg_stats.o \ + icssg/icssg_ethtool.o diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index bebcfd5e6b57..bea6fc0f324c 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -19,6 +19,7 @@ #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/of_device.h> +#include <linux/of_platform.h> #include <linux/phylink.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/ti/am65-cpsw-qos.c b/drivers/net/ethernet/ti/am65-cpsw-qos.c index eced87fa261c..9ac2ff05d501 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-qos.c +++ b/drivers/net/ethernet/ti/am65-cpsw-qos.c @@ -624,9 +624,9 @@ static int am65_cpsw_qos_clsflower_add_policer(struct am65_cpsw_port *port, int ret; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used"); return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/ti/cpsw-common.c b/drivers/net/ethernet/ti/cpsw-common.c index bfa81bbfce3f..26dc906eae90 100644 --- a/drivers/net/ethernet/ti/cpsw-common.c +++ b/drivers/net/ethernet/ti/cpsw-common.c @@ -3,7 +3,6 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/regmap.h> #include <linux/mfd/syscon.h> diff --git a/drivers/net/ethernet/ti/cpsw-phy-sel.c b/drivers/net/ethernet/ti/cpsw-phy-sel.c index 25e707d7b87c..4edb7963f856 100644 --- a/drivers/net/ethernet/ti/cpsw-phy-sel.c +++ b/drivers/net/ethernet/ti/cpsw-phy-sel.c @@ -12,7 +12,6 @@ #include <linux/netdevice.h> #include <linux/phy.h> #include <linux/of.h> -#include <linux/of_device.h> #include "cpsw.h" diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index e966dd47e2db..ae52cdbcf8cc 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -1396,9 +1396,9 @@ static int cpsw_qos_clsflower_add_policer(struct cpsw_priv *priv, int ret; if (dissector->used_keys & - ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { + ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used"); return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c index 23169e36a3d4..89b6d23e9937 100644 --- a/drivers/net/ethernet/ti/davinci_mdio.c +++ b/drivers/net/ethernet/ti/davinci_mdio.c @@ -23,7 +23,6 @@ #include <linux/pm_runtime.h> #include <linux/davinci_emac.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/pinctrl/consumer.h> #include <linux/mdio-bitbang.h> diff --git a/drivers/net/ethernet/ti/icssg/icssg_classifier.c b/drivers/net/ethernet/ti/icssg/icssg_classifier.c new file mode 100644 index 000000000000..6df53ab17fbc --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_classifier.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Texas Instruments ICSSG Ethernet Driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#include <linux/etherdevice.h> +#include <linux/types.h> +#include <linux/regmap.h> + +#include "icssg_prueth.h" + +#define ICSSG_NUM_CLASSIFIERS 16 +#define ICSSG_NUM_FT1_SLOTS 8 +#define ICSSG_NUM_FT3_SLOTS 16 + +#define ICSSG_NUM_CLASSIFIERS_IN_USE 5 + +/* Filter 1 - FT1 */ +#define FT1_NUM_SLOTS 8 +#define FT1_SLOT_SIZE 0x10 /* bytes */ + +/* offsets from FT1 slot base i.e. slot 1 start */ +#define FT1_DA0 0x0 +#define FT1_DA1 0x4 +#define FT1_DA0_MASK 0x8 +#define FT1_DA1_MASK 0xc + +#define FT1_N_REG(slize, n, reg) \ + (offs[slice].ft1_slot_base + FT1_SLOT_SIZE * (n) + (reg)) + +#define FT1_LEN_MASK GENMASK(19, 16) +#define FT1_LEN_SHIFT 16 +#define FT1_LEN(len) (((len) << FT1_LEN_SHIFT) & FT1_LEN_MASK) +#define FT1_START_MASK GENMASK(14, 0) +#define FT1_START(start) ((start) & FT1_START_MASK) +#define FT1_MATCH_SLOT(n) (GENMASK(23, 16) & (BIT(n) << 16)) + +/* FT1 config type */ +enum ft1_cfg_type { + FT1_CFG_TYPE_DISABLED = 0, + FT1_CFG_TYPE_EQ, + FT1_CFG_TYPE_GT, + FT1_CFG_TYPE_LT, +}; + +#define FT1_CFG_SHIFT(n) (2 * (n)) +#define FT1_CFG_MASK(n) (0x3 << FT1_CFG_SHIFT((n))) + +/* Filter 3 - FT3 */ +#define FT3_NUM_SLOTS 16 +#define FT3_SLOT_SIZE 0x20 /* bytes */ + +/* offsets from FT3 slot n's base */ +#define FT3_START 0 +#define FT3_START_AUTO 0x4 +#define FT3_START_OFFSET 0x8 +#define FT3_JUMP_OFFSET 0xc +#define FT3_LEN 0x10 +#define FT3_CFG 0x14 +#define FT3_T 0x18 +#define FT3_T_MASK 0x1c + +#define FT3_N_REG(slize, n, reg) \ + (offs[slice].ft3_slot_base + FT3_SLOT_SIZE * (n) + (reg)) + +/* offsets from rx_class n's base */ +#define RX_CLASS_AND_EN 0 +#define RX_CLASS_OR_EN 0x4 +#define RX_CLASS_NUM_SLOTS 16 +#define RX_CLASS_EN_SIZE 0x8 /* bytes */ + +#define RX_CLASS_N_REG(slice, n, reg) \ + (offs[slice].rx_class_base + RX_CLASS_EN_SIZE * (n) + (reg)) + +/* RX Class Gates */ +#define RX_CLASS_GATES_SIZE 0x4 /* bytes */ + +#define RX_CLASS_GATES_N_REG(slice, n) \ + (offs[slice].rx_class_gates_base + RX_CLASS_GATES_SIZE * (n)) + +#define RX_CLASS_GATES_ALLOW_MASK BIT(6) +#define RX_CLASS_GATES_RAW_MASK BIT(5) +#define RX_CLASS_GATES_PHASE_MASK BIT(4) + +/* RX Class traffic data matching bits */ +#define RX_CLASS_FT_UC BIT(31) +#define RX_CLASS_FT_MC BIT(30) +#define RX_CLASS_FT_BC BIT(29) +#define RX_CLASS_FT_FW BIT(28) +#define RX_CLASS_FT_RCV BIT(27) +#define RX_CLASS_FT_VLAN BIT(26) +#define RX_CLASS_FT_DA_P BIT(25) +#define RX_CLASS_FT_DA_I BIT(24) +#define RX_CLASS_FT_FT1_MATCH_MASK GENMASK(23, 16) +#define RX_CLASS_FT_FT1_MATCH_SHIFT 16 +#define RX_CLASS_FT_FT3_MATCH_MASK GENMASK(15, 0) +#define RX_CLASS_FT_FT3_MATCH_SHIFT 0 + +#define RX_CLASS_FT_FT1_MATCH(slot) \ + ((BIT(slot) << RX_CLASS_FT_FT1_MATCH_SHIFT) & \ + RX_CLASS_FT_FT1_MATCH_MASK) + +/* RX class type */ +enum rx_class_sel_type { + RX_CLASS_SEL_TYPE_OR = 0, + RX_CLASS_SEL_TYPE_AND = 1, + RX_CLASS_SEL_TYPE_OR_AND_AND = 2, + RX_CLASS_SEL_TYPE_OR_OR_AND = 3, +}; + +#define FT1_CFG_SHIFT(n) (2 * (n)) +#define FT1_CFG_MASK(n) (0x3 << FT1_CFG_SHIFT((n))) + +#define RX_CLASS_SEL_SHIFT(n) (2 * (n)) +#define RX_CLASS_SEL_MASK(n) (0x3 << RX_CLASS_SEL_SHIFT((n))) + +#define ICSSG_CFG_OFFSET 0 +#define MAC_INTERFACE_0 0x18 +#define MAC_INTERFACE_1 0x1c + +#define ICSSG_CFG_RX_L2_G_EN BIT(2) + +/* These are register offsets per PRU */ +struct miig_rt_offsets { + u32 mac0; + u32 mac1; + u32 ft1_start_len; + u32 ft1_cfg; + u32 ft1_slot_base; + u32 ft3_slot_base; + u32 ft3_p_base; + u32 ft_rx_ptr; + u32 rx_class_base; + u32 rx_class_cfg1; + u32 rx_class_cfg2; + u32 rx_class_gates_base; + u32 rx_green; + u32 rx_rate_cfg_base; + u32 rx_rate_src_sel0; + u32 rx_rate_src_sel1; + u32 tx_rate_cfg_base; + u32 stat_base; + u32 tx_hsr_tag; + u32 tx_hsr_seq; + u32 tx_vlan_type; + u32 tx_vlan_ins; +}; + +/* These are the offset values for miig_rt_offsets registers */ +static const struct miig_rt_offsets offs[] = { + /* PRU0 */ + { + 0x8, + 0xc, + 0x80, + 0x84, + 0x88, + 0x108, + 0x308, + 0x408, + 0x40c, + 0x48c, + 0x490, + 0x494, + 0x4d4, + 0x4e4, + 0x504, + 0x508, + 0x50c, + 0x54c, + 0x63c, + 0x640, + 0x644, + 0x648, + }, + /* PRU1 */ + { + 0x10, + 0x14, + 0x64c, + 0x650, + 0x654, + 0x6d4, + 0x8d4, + 0x9d4, + 0x9d8, + 0xa58, + 0xa5c, + 0xa60, + 0xaa0, + 0xab0, + 0xad0, + 0xad4, + 0xad8, + 0xb18, + 0xc08, + 0xc0c, + 0xc10, + 0xc14, + }, +}; + +static void rx_class_ft1_set_start_len(struct regmap *miig_rt, int slice, + u16 start, u8 len) +{ + u32 offset, val; + + offset = offs[slice].ft1_start_len; + val = FT1_LEN(len) | FT1_START(start); + regmap_write(miig_rt, offset, val); +} + +static void rx_class_ft1_set_da(struct regmap *miig_rt, int slice, + int n, const u8 *addr) +{ + u32 offset; + + offset = FT1_N_REG(slice, n, FT1_DA0); + regmap_write(miig_rt, offset, (u32)(addr[0] | addr[1] << 8 | + addr[2] << 16 | addr[3] << 24)); + offset = FT1_N_REG(slice, n, FT1_DA1); + regmap_write(miig_rt, offset, (u32)(addr[4] | addr[5] << 8)); +} + +static void rx_class_ft1_set_da_mask(struct regmap *miig_rt, int slice, + int n, const u8 *addr) +{ + u32 offset; + + offset = FT1_N_REG(slice, n, FT1_DA0_MASK); + regmap_write(miig_rt, offset, (u32)(addr[0] | addr[1] << 8 | + addr[2] << 16 | addr[3] << 24)); + offset = FT1_N_REG(slice, n, FT1_DA1_MASK); + regmap_write(miig_rt, offset, (u32)(addr[4] | addr[5] << 8)); +} + +static void rx_class_ft1_cfg_set_type(struct regmap *miig_rt, int slice, int n, + enum ft1_cfg_type type) +{ + u32 offset; + + offset = offs[slice].ft1_cfg; + regmap_update_bits(miig_rt, offset, FT1_CFG_MASK(n), + type << FT1_CFG_SHIFT(n)); +} + +static void rx_class_sel_set_type(struct regmap *miig_rt, int slice, int n, + enum rx_class_sel_type type) +{ + u32 offset; + + offset = offs[slice].rx_class_cfg1; + regmap_update_bits(miig_rt, offset, RX_CLASS_SEL_MASK(n), + type << RX_CLASS_SEL_SHIFT(n)); +} + +static void rx_class_set_and(struct regmap *miig_rt, int slice, int n, + u32 data) +{ + u32 offset; + + offset = RX_CLASS_N_REG(slice, n, RX_CLASS_AND_EN); + regmap_write(miig_rt, offset, data); +} + +static void rx_class_set_or(struct regmap *miig_rt, int slice, int n, + u32 data) +{ + u32 offset; + + offset = RX_CLASS_N_REG(slice, n, RX_CLASS_OR_EN); + regmap_write(miig_rt, offset, data); +} + +void icssg_class_set_host_mac_addr(struct regmap *miig_rt, const u8 *mac) +{ + regmap_write(miig_rt, MAC_INTERFACE_0, (u32)(mac[0] | mac[1] << 8 | + mac[2] << 16 | mac[3] << 24)); + regmap_write(miig_rt, MAC_INTERFACE_1, (u32)(mac[4] | mac[5] << 8)); +} + +void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac) +{ + regmap_write(miig_rt, offs[slice].mac0, (u32)(mac[0] | mac[1] << 8 | + mac[2] << 16 | mac[3] << 24)); + regmap_write(miig_rt, offs[slice].mac1, (u32)(mac[4] | mac[5] << 8)); +} + +/* disable all RX traffic */ +void icssg_class_disable(struct regmap *miig_rt, int slice) +{ + u32 data, offset; + int n; + + /* Enable RX_L2_G */ + regmap_update_bits(miig_rt, ICSSG_CFG_OFFSET, ICSSG_CFG_RX_L2_G_EN, + ICSSG_CFG_RX_L2_G_EN); + + for (n = 0; n < ICSSG_NUM_CLASSIFIERS; n++) { + /* AND_EN = 0 */ + rx_class_set_and(miig_rt, slice, n, 0); + /* OR_EN = 0 */ + rx_class_set_or(miig_rt, slice, n, 0); + + /* set CFG1 to OR */ + rx_class_sel_set_type(miig_rt, slice, n, RX_CLASS_SEL_TYPE_OR); + + /* configure gate */ + offset = RX_CLASS_GATES_N_REG(slice, n); + regmap_read(miig_rt, offset, &data); + /* clear class_raw so we go through filters */ + data &= ~RX_CLASS_GATES_RAW_MASK; + /* set allow and phase mask */ + data |= RX_CLASS_GATES_ALLOW_MASK | RX_CLASS_GATES_PHASE_MASK; + regmap_write(miig_rt, offset, data); + } + + /* FT1 Disabled */ + for (n = 0; n < ICSSG_NUM_FT1_SLOTS; n++) { + const u8 addr[] = { 0, 0, 0, 0, 0, 0, }; + + rx_class_ft1_cfg_set_type(miig_rt, slice, n, + FT1_CFG_TYPE_DISABLED); + rx_class_ft1_set_da(miig_rt, slice, n, addr); + rx_class_ft1_set_da_mask(miig_rt, slice, n, addr); + } + + /* clear CFG2 */ + regmap_write(miig_rt, offs[slice].rx_class_cfg2, 0); +} + +void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti) +{ + u32 data; + + /* defaults */ + icssg_class_disable(miig_rt, slice); + + /* Setup Classifier */ + /* match on Broadcast or MAC_PRU address */ + data = RX_CLASS_FT_BC | RX_CLASS_FT_DA_P; + + /* multicast */ + if (allmulti) + data |= RX_CLASS_FT_MC; + + rx_class_set_or(miig_rt, slice, 0, data); + + /* set CFG1 for OR_OR_AND for classifier */ + rx_class_sel_set_type(miig_rt, slice, 0, RX_CLASS_SEL_TYPE_OR_OR_AND); + + /* clear CFG2 */ + regmap_write(miig_rt, offs[slice].rx_class_cfg2, 0); +} + +/* required for SAV check */ +void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr) +{ + const u8 mask_addr[] = { 0, 0, 0, 0, 0, 0, }; + + rx_class_ft1_set_start_len(miig_rt, slice, 0, 6); + rx_class_ft1_set_da(miig_rt, slice, 0, mac_addr); + rx_class_ft1_set_da_mask(miig_rt, slice, 0, mask_addr); + rx_class_ft1_cfg_set_type(miig_rt, slice, 0, FT1_CFG_TYPE_EQ); +} diff --git a/drivers/net/ethernet/ti/icssg/icssg_config.c b/drivers/net/ethernet/ti/icssg/icssg_config.c new file mode 100644 index 000000000000..ab648d3efe85 --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_config.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ICSSG Ethernet driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/iopoll.h> +#include <linux/regmap.h> +#include <uapi/linux/if_ether.h> +#include "icssg_config.h" +#include "icssg_prueth.h" +#include "icssg_switch_map.h" +#include "icssg_mii_rt.h" + +/* TX IPG Values to be set for 100M link speed. These values are + * in ocp_clk cycles. So need change if ocp_clk is changed for a specific + * h/w design. + */ + +/* IPG is in core_clk cycles */ +#define MII_RT_TX_IPG_100M 0x17 +#define MII_RT_TX_IPG_1G 0xb + +#define ICSSG_QUEUES_MAX 64 +#define ICSSG_QUEUE_OFFSET 0xd00 +#define ICSSG_QUEUE_PEEK_OFFSET 0xe00 +#define ICSSG_QUEUE_CNT_OFFSET 0xe40 +#define ICSSG_QUEUE_RESET_OFFSET 0xf40 + +#define ICSSG_NUM_TX_QUEUES 8 + +#define RECYCLE_Q_SLICE0 16 +#define RECYCLE_Q_SLICE1 17 + +#define ICSSG_NUM_OTHER_QUEUES 5 /* port, host and special queues */ + +#define PORT_HI_Q_SLICE0 32 +#define PORT_LO_Q_SLICE0 33 +#define HOST_HI_Q_SLICE0 34 +#define HOST_LO_Q_SLICE0 35 +#define HOST_SPL_Q_SLICE0 40 /* Special Queue */ + +#define PORT_HI_Q_SLICE1 36 +#define PORT_LO_Q_SLICE1 37 +#define HOST_HI_Q_SLICE1 38 +#define HOST_LO_Q_SLICE1 39 +#define HOST_SPL_Q_SLICE1 41 /* Special Queue */ + +#define MII_RXCFG_DEFAULT (PRUSS_MII_RT_RXCFG_RX_ENABLE | \ + PRUSS_MII_RT_RXCFG_RX_DATA_RDY_MODE_DIS | \ + PRUSS_MII_RT_RXCFG_RX_L2_EN | \ + PRUSS_MII_RT_RXCFG_RX_L2_EOF_SCLR_DIS) + +#define MII_TXCFG_DEFAULT (PRUSS_MII_RT_TXCFG_TX_ENABLE | \ + PRUSS_MII_RT_TXCFG_TX_AUTO_PREAMBLE | \ + PRUSS_MII_RT_TXCFG_TX_32_MODE_EN | \ + PRUSS_MII_RT_TXCFG_TX_IPG_WIRE_CLK_EN) + +#define ICSSG_CFG_DEFAULT (ICSSG_CFG_TX_L1_EN | \ + ICSSG_CFG_TX_L2_EN | ICSSG_CFG_RX_L2_G_EN | \ + ICSSG_CFG_TX_PRU_EN | \ + ICSSG_CFG_SGMII_MODE) + +#define FDB_GEN_CFG1 0x60 +#define SMEM_VLAN_OFFSET 8 +#define SMEM_VLAN_OFFSET_MASK GENMASK(25, 8) + +#define FDB_GEN_CFG2 0x64 +#define FDB_VLAN_EN BIT(6) +#define FDB_HOST_EN BIT(2) +#define FDB_PRU1_EN BIT(1) +#define FDB_PRU0_EN BIT(0) +#define FDB_EN_ALL (FDB_PRU0_EN | FDB_PRU1_EN | \ + FDB_HOST_EN | FDB_VLAN_EN) + +/** + * struct map - ICSSG Queue Map + * @queue: Queue number + * @pd_addr_start: Packet descriptor queue reserved memory + * @flags: Flags + * @special: Indicates whether this queue is a special queue or not + */ +struct map { + int queue; + u32 pd_addr_start; + u32 flags; + bool special; +}; + +/* Hardware queue map for ICSSG */ +static const struct map hwq_map[2][ICSSG_NUM_OTHER_QUEUES] = { + { + { PORT_HI_Q_SLICE0, PORT_DESC0_HI, 0x200000, 0 }, + { PORT_LO_Q_SLICE0, PORT_DESC0_LO, 0, 0 }, + { HOST_HI_Q_SLICE0, HOST_DESC0_HI, 0x200000, 0 }, + { HOST_LO_Q_SLICE0, HOST_DESC0_LO, 0, 0 }, + { HOST_SPL_Q_SLICE0, HOST_SPPD0, 0x400000, 1 }, + }, + { + { PORT_HI_Q_SLICE1, PORT_DESC1_HI, 0xa00000, 0 }, + { PORT_LO_Q_SLICE1, PORT_DESC1_LO, 0x800000, 0 }, + { HOST_HI_Q_SLICE1, HOST_DESC1_HI, 0xa00000, 0 }, + { HOST_LO_Q_SLICE1, HOST_DESC1_LO, 0x800000, 0 }, + { HOST_SPL_Q_SLICE1, HOST_SPPD1, 0xc00000, 1 }, + }, +}; + +static void icssg_config_mii_init(struct prueth_emac *emac) +{ + u32 rxcfg, txcfg, rxcfg_reg, txcfg_reg, pcnt_reg; + struct prueth *prueth = emac->prueth; + int slice = prueth_emac_slice(emac); + struct regmap *mii_rt; + + mii_rt = prueth->mii_rt; + + rxcfg_reg = (slice == ICSS_MII0) ? PRUSS_MII_RT_RXCFG0 : + PRUSS_MII_RT_RXCFG1; + txcfg_reg = (slice == ICSS_MII0) ? PRUSS_MII_RT_TXCFG0 : + PRUSS_MII_RT_TXCFG1; + pcnt_reg = (slice == ICSS_MII0) ? PRUSS_MII_RT_RX_PCNT0 : + PRUSS_MII_RT_RX_PCNT1; + + rxcfg = MII_RXCFG_DEFAULT; + txcfg = MII_TXCFG_DEFAULT; + + if (slice == ICSS_MII1) + rxcfg |= PRUSS_MII_RT_RXCFG_RX_MUX_SEL; + + /* In MII mode TX lines swapped inside ICSSG, so TX_MUX_SEL cfg need + * to be swapped also comparing to RGMII mode. + */ + if (emac->phy_if == PHY_INTERFACE_MODE_MII && slice == ICSS_MII0) + txcfg |= PRUSS_MII_RT_TXCFG_TX_MUX_SEL; + else if (emac->phy_if != PHY_INTERFACE_MODE_MII && slice == ICSS_MII1) + txcfg |= PRUSS_MII_RT_TXCFG_TX_MUX_SEL; + + regmap_write(mii_rt, rxcfg_reg, rxcfg); + regmap_write(mii_rt, txcfg_reg, txcfg); + regmap_write(mii_rt, pcnt_reg, 0x1); +} + +static void icssg_miig_queues_init(struct prueth *prueth, int slice) +{ + struct regmap *miig_rt = prueth->miig_rt; + void __iomem *smem = prueth->shram.va; + u8 pd[ICSSG_SPECIAL_PD_SIZE]; + int queue = 0, i, j; + u32 *pdword; + + /* reset hwqueues */ + if (slice) + queue = ICSSG_NUM_TX_QUEUES; + + for (i = 0; i < ICSSG_NUM_TX_QUEUES; i++) { + regmap_write(miig_rt, ICSSG_QUEUE_RESET_OFFSET, queue); + queue++; + } + + queue = slice ? RECYCLE_Q_SLICE1 : RECYCLE_Q_SLICE0; + regmap_write(miig_rt, ICSSG_QUEUE_RESET_OFFSET, queue); + + for (i = 0; i < ICSSG_NUM_OTHER_QUEUES; i++) { + regmap_write(miig_rt, ICSSG_QUEUE_RESET_OFFSET, + hwq_map[slice][i].queue); + } + + /* initialize packet descriptors in SMEM */ + /* push pakcet descriptors to hwqueues */ + + pdword = (u32 *)pd; + for (j = 0; j < ICSSG_NUM_OTHER_QUEUES; j++) { + const struct map *mp; + int pd_size, num_pds; + u32 pdaddr; + + mp = &hwq_map[slice][j]; + if (mp->special) { + pd_size = ICSSG_SPECIAL_PD_SIZE; + num_pds = ICSSG_NUM_SPECIAL_PDS; + } else { + pd_size = ICSSG_NORMAL_PD_SIZE; + num_pds = ICSSG_NUM_NORMAL_PDS; + } + + for (i = 0; i < num_pds; i++) { + memset(pd, 0, pd_size); + + pdword[0] &= ICSSG_FLAG_MASK; + pdword[0] |= mp->flags; + pdaddr = mp->pd_addr_start + i * pd_size; + + memcpy_toio(smem + pdaddr, pd, pd_size); + queue = mp->queue; + regmap_write(miig_rt, ICSSG_QUEUE_OFFSET + 4 * queue, + pdaddr); + } + } +} + +void icssg_config_ipg(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + int slice = prueth_emac_slice(emac); + + switch (emac->speed) { + case SPEED_1000: + icssg_mii_update_ipg(prueth->mii_rt, slice, MII_RT_TX_IPG_1G); + break; + case SPEED_100: + icssg_mii_update_ipg(prueth->mii_rt, slice, MII_RT_TX_IPG_100M); + break; + default: + /* Other links speeds not supported */ + netdev_err(emac->ndev, "Unsupported link speed\n"); + return; + } +} + +static void emac_r30_cmd_init(struct prueth_emac *emac) +{ + struct icssg_r30_cmd __iomem *p; + int i; + + p = emac->dram.va + MGR_R30_CMD_OFFSET; + + for (i = 0; i < 4; i++) + writel(EMAC_NONE, &p->cmd[i]); +} + +static int emac_r30_is_done(struct prueth_emac *emac) +{ + const struct icssg_r30_cmd __iomem *p; + u32 cmd; + int i; + + p = emac->dram.va + MGR_R30_CMD_OFFSET; + + for (i = 0; i < 4; i++) { + cmd = readl(&p->cmd[i]); + if (cmd != EMAC_NONE) + return 0; + } + + return 1; +} + +static int prueth_emac_buffer_setup(struct prueth_emac *emac) +{ + struct icssg_buffer_pool_cfg __iomem *bpool_cfg; + struct icssg_rxq_ctx __iomem *rxq_ctx; + struct prueth *prueth = emac->prueth; + int slice = prueth_emac_slice(emac); + u32 addr; + int i; + + /* Layout to have 64KB aligned buffer pool + * |BPOOL0|BPOOL1|RX_CTX0|RX_CTX1| + */ + + addr = lower_32_bits(prueth->msmcram.pa); + if (slice) + addr += PRUETH_NUM_BUF_POOLS * PRUETH_EMAC_BUF_POOL_SIZE; + + if (addr % SZ_64K) { + dev_warn(prueth->dev, "buffer pool needs to be 64KB aligned\n"); + return -EINVAL; + } + + bpool_cfg = emac->dram.va + BUFFER_POOL_0_ADDR_OFFSET; + /* workaround for f/w bug. bpool 0 needs to be initilalized */ + writel(addr, &bpool_cfg[0].addr); + writel(0, &bpool_cfg[0].len); + + for (i = PRUETH_EMAC_BUF_POOL_START; + i < PRUETH_EMAC_BUF_POOL_START + PRUETH_NUM_BUF_POOLS; + i++) { + writel(addr, &bpool_cfg[i].addr); + writel(PRUETH_EMAC_BUF_POOL_SIZE, &bpool_cfg[i].len); + addr += PRUETH_EMAC_BUF_POOL_SIZE; + } + + if (!slice) + addr += PRUETH_NUM_BUF_POOLS * PRUETH_EMAC_BUF_POOL_SIZE; + else + addr += PRUETH_EMAC_RX_CTX_BUF_SIZE * 2; + + /* Pre-emptible RX buffer queue */ + rxq_ctx = emac->dram.va + HOST_RX_Q_PRE_CONTEXT_OFFSET; + for (i = 0; i < 3; i++) + writel(addr, &rxq_ctx->start[i]); + + addr += PRUETH_EMAC_RX_CTX_BUF_SIZE; + writel(addr, &rxq_ctx->end); + + /* Express RX buffer queue */ + rxq_ctx = emac->dram.va + HOST_RX_Q_EXP_CONTEXT_OFFSET; + for (i = 0; i < 3; i++) + writel(addr, &rxq_ctx->start[i]); + + addr += PRUETH_EMAC_RX_CTX_BUF_SIZE; + writel(addr, &rxq_ctx->end); + + return 0; +} + +static void icssg_init_emac_mode(struct prueth *prueth) +{ + /* When the device is configured as a bridge and it is being brought + * back to the emac mode, the host mac address has to be set as 0. + */ + u8 mac[ETH_ALEN] = { 0 }; + + if (prueth->emacs_initialized) + return; + + regmap_update_bits(prueth->miig_rt, FDB_GEN_CFG1, + SMEM_VLAN_OFFSET_MASK, 0); + regmap_write(prueth->miig_rt, FDB_GEN_CFG2, 0); + /* Clear host MAC address */ + icssg_class_set_host_mac_addr(prueth->miig_rt, mac); +} + +int icssg_config(struct prueth *prueth, struct prueth_emac *emac, int slice) +{ + void __iomem *config = emac->dram.va + ICSSG_CONFIG_OFFSET; + struct icssg_flow_cfg __iomem *flow_cfg; + int ret; + + icssg_init_emac_mode(prueth); + + memset_io(config, 0, TAS_GATE_MASK_LIST0); + icssg_miig_queues_init(prueth, slice); + + emac->speed = SPEED_1000; + emac->duplex = DUPLEX_FULL; + if (!phy_interface_mode_is_rgmii(emac->phy_if)) { + emac->speed = SPEED_100; + emac->duplex = DUPLEX_FULL; + } + regmap_update_bits(prueth->miig_rt, ICSSG_CFG_OFFSET, + ICSSG_CFG_DEFAULT, ICSSG_CFG_DEFAULT); + icssg_miig_set_interface_mode(prueth->miig_rt, slice, emac->phy_if); + icssg_config_mii_init(emac); + icssg_config_ipg(emac); + icssg_update_rgmii_cfg(prueth->miig_rt, emac); + + /* set GPI mode */ + pruss_cfg_gpimode(prueth->pruss, prueth->pru_id[slice], + PRUSS_GPI_MODE_MII); + + /* enable XFR shift for PRU and RTU */ + pruss_cfg_xfr_enable(prueth->pruss, PRU_TYPE_PRU, true); + pruss_cfg_xfr_enable(prueth->pruss, PRU_TYPE_RTU, true); + + /* set C28 to 0x100 */ + pru_rproc_set_ctable(prueth->pru[slice], PRU_C28, 0x100 << 8); + pru_rproc_set_ctable(prueth->rtu[slice], PRU_C28, 0x100 << 8); + pru_rproc_set_ctable(prueth->txpru[slice], PRU_C28, 0x100 << 8); + + flow_cfg = config + PSI_L_REGULAR_FLOW_ID_BASE_OFFSET; + writew(emac->rx_flow_id_base, &flow_cfg->rx_base_flow); + writew(0, &flow_cfg->mgm_base_flow); + writeb(0, config + SPL_PKT_DEFAULT_PRIORITY); + writeb(0, config + QUEUE_NUM_UNTAGGED); + + ret = prueth_emac_buffer_setup(emac); + if (ret) + return ret; + + emac_r30_cmd_init(emac); + + return 0; +} + +/* Bitmask for ICSSG r30 commands */ +static const struct icssg_r30_cmd emac_r32_bitmask[] = { + {{0xffff0004, 0xffff0100, 0xffff0100, EMAC_NONE}}, /* EMAC_PORT_DISABLE */ + {{0xfffb0040, 0xfeff0200, 0xfeff0200, EMAC_NONE}}, /* EMAC_PORT_BLOCK */ + {{0xffbb0000, 0xfcff0000, 0xdcff0000, EMAC_NONE}}, /* EMAC_PORT_FORWARD */ + {{0xffbb0000, 0xfcff0000, 0xfcff2000, EMAC_NONE}}, /* EMAC_PORT_FORWARD_WO_LEARNING */ + {{0xffff0001, EMAC_NONE, EMAC_NONE, EMAC_NONE}}, /* ACCEPT ALL */ + {{0xfffe0002, EMAC_NONE, EMAC_NONE, EMAC_NONE}}, /* ACCEPT TAGGED */ + {{0xfffc0000, EMAC_NONE, EMAC_NONE, EMAC_NONE}}, /* ACCEPT UNTAGGED and PRIO */ + {{EMAC_NONE, 0xffff0020, EMAC_NONE, EMAC_NONE}}, /* TAS Trigger List change */ + {{EMAC_NONE, 0xdfff1000, EMAC_NONE, EMAC_NONE}}, /* TAS set state ENABLE*/ + {{EMAC_NONE, 0xefff2000, EMAC_NONE, EMAC_NONE}}, /* TAS set state RESET*/ + {{EMAC_NONE, 0xcfff0000, EMAC_NONE, EMAC_NONE}}, /* TAS set state DISABLE*/ + {{EMAC_NONE, EMAC_NONE, 0xffff0400, EMAC_NONE}}, /* UC flooding ENABLE*/ + {{EMAC_NONE, EMAC_NONE, 0xfbff0000, EMAC_NONE}}, /* UC flooding DISABLE*/ + {{EMAC_NONE, EMAC_NONE, 0xffff0800, EMAC_NONE}}, /* MC flooding ENABLE*/ + {{EMAC_NONE, EMAC_NONE, 0xf7ff0000, EMAC_NONE}}, /* MC flooding DISABLE*/ + {{EMAC_NONE, 0xffff4000, EMAC_NONE, EMAC_NONE}}, /* Preemption on Tx ENABLE*/ + {{EMAC_NONE, 0xbfff0000, EMAC_NONE, EMAC_NONE}}, /* Preemption on Tx DISABLE*/ + {{0xffff0010, EMAC_NONE, 0xffff0010, EMAC_NONE}}, /* VLAN AWARE*/ + {{0xffef0000, EMAC_NONE, 0xffef0000, EMAC_NONE}} /* VLAN UNWARE*/ +}; + +int emac_set_port_state(struct prueth_emac *emac, + enum icssg_port_state_cmd cmd) +{ + struct icssg_r30_cmd __iomem *p; + int ret = -ETIMEDOUT; + int done = 0; + int i; + + p = emac->dram.va + MGR_R30_CMD_OFFSET; + + if (cmd >= ICSSG_EMAC_PORT_MAX_COMMANDS) { + netdev_err(emac->ndev, "invalid port command\n"); + return -EINVAL; + } + + /* only one command at a time allowed to firmware */ + mutex_lock(&emac->cmd_lock); + + for (i = 0; i < 4; i++) + writel(emac_r32_bitmask[cmd].cmd[i], &p->cmd[i]); + + /* wait for done */ + ret = read_poll_timeout(emac_r30_is_done, done, done == 1, + 1000, 10000, false, emac); + + if (ret == -ETIMEDOUT) + netdev_err(emac->ndev, "timeout waiting for command done\n"); + + mutex_unlock(&emac->cmd_lock); + + return ret; +} + +void icssg_config_set_speed(struct prueth_emac *emac) +{ + u8 fw_speed; + + switch (emac->speed) { + case SPEED_1000: + fw_speed = FW_LINK_SPEED_1G; + break; + case SPEED_100: + fw_speed = FW_LINK_SPEED_100M; + break; + default: + /* Other links speeds not supported */ + netdev_err(emac->ndev, "Unsupported link speed\n"); + return; + } + + writeb(fw_speed, emac->dram.va + PORT_LINK_SPEED_OFFSET); +} diff --git a/drivers/net/ethernet/ti/icssg/icssg_config.h b/drivers/net/ethernet/ti/icssg/icssg_config.h new file mode 100644 index 000000000000..43eb0922172a --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_config.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#ifndef __NET_TI_ICSSG_CONFIG_H +#define __NET_TI_ICSSG_CONFIG_H + +struct icssg_buffer_pool_cfg { + __le32 addr; + __le32 len; +} __packed; + +struct icssg_flow_cfg { + __le16 rx_base_flow; + __le16 mgm_base_flow; +} __packed; + +#define PRUETH_PKT_TYPE_CMD 0x10 +#define PRUETH_NAV_PS_DATA_SIZE 16 /* Protocol specific data size */ +#define PRUETH_NAV_SW_DATA_SIZE 16 /* SW related data size */ +#define PRUETH_MAX_TX_DESC 512 +#define PRUETH_MAX_RX_DESC 512 +#define PRUETH_MAX_RX_FLOWS 1 /* excluding default flow */ +#define PRUETH_RX_FLOW_DATA 0 + +#define PRUETH_EMAC_BUF_POOL_SIZE SZ_8K +#define PRUETH_EMAC_POOLS_PER_SLICE 24 +#define PRUETH_EMAC_BUF_POOL_START 8 +#define PRUETH_NUM_BUF_POOLS 8 +#define PRUETH_EMAC_RX_CTX_BUF_SIZE SZ_16K /* per slice */ +#define MSMC_RAM_SIZE \ + (2 * (PRUETH_EMAC_BUF_POOL_SIZE * PRUETH_NUM_BUF_POOLS + \ + PRUETH_EMAC_RX_CTX_BUF_SIZE * 2)) + +struct icssg_rxq_ctx { + __le32 start[3]; + __le32 end; +} __packed; + +/* Load time Fiwmware Configuration */ + +#define ICSSG_FW_MGMT_CMD_HEADER 0x81 +#define ICSSG_FW_MGMT_FDB_CMD_TYPE 0x03 +#define ICSSG_FW_MGMT_CMD_TYPE 0x04 +#define ICSSG_FW_MGMT_PKT 0x80000000 + +struct icssg_r30_cmd { + u32 cmd[4]; +} __packed; + +enum icssg_port_state_cmd { + ICSSG_EMAC_PORT_DISABLE = 0, + ICSSG_EMAC_PORT_BLOCK, + ICSSG_EMAC_PORT_FORWARD, + ICSSG_EMAC_PORT_FORWARD_WO_LEARNING, + ICSSG_EMAC_PORT_ACCEPT_ALL, + ICSSG_EMAC_PORT_ACCEPT_TAGGED, + ICSSG_EMAC_PORT_ACCEPT_UNTAGGED_N_PRIO, + ICSSG_EMAC_PORT_TAS_TRIGGER, + ICSSG_EMAC_PORT_TAS_ENABLE, + ICSSG_EMAC_PORT_TAS_RESET, + ICSSG_EMAC_PORT_TAS_DISABLE, + ICSSG_EMAC_PORT_UC_FLOODING_ENABLE, + ICSSG_EMAC_PORT_UC_FLOODING_DISABLE, + ICSSG_EMAC_PORT_MC_FLOODING_ENABLE, + ICSSG_EMAC_PORT_MC_FLOODING_DISABLE, + ICSSG_EMAC_PORT_PREMPT_TX_ENABLE, + ICSSG_EMAC_PORT_PREMPT_TX_DISABLE, + ICSSG_EMAC_PORT_VLAN_AWARE_ENABLE, + ICSSG_EMAC_PORT_VLAN_AWARE_DISABLE, + ICSSG_EMAC_PORT_MAX_COMMANDS +}; + +#define EMAC_NONE 0xffff0000 +#define EMAC_PRU0_P_DI 0xffff0004 +#define EMAC_PRU1_P_DI 0xffff0040 +#define EMAC_TX_P_DI 0xffff0100 + +#define EMAC_PRU0_P_EN 0xfffb0000 +#define EMAC_PRU1_P_EN 0xffbf0000 +#define EMAC_TX_P_EN 0xfeff0000 + +#define EMAC_P_BLOCK 0xffff0040 +#define EMAC_TX_P_BLOCK 0xffff0200 +#define EMAC_P_UNBLOCK 0xffbf0000 +#define EMAC_TX_P_UNBLOCK 0xfdff0000 +#define EMAC_LEAN_EN 0xfff70000 +#define EMAC_LEAN_DI 0xffff0008 + +#define EMAC_ACCEPT_ALL 0xffff0001 +#define EMAC_ACCEPT_TAG 0xfffe0002 +#define EMAC_ACCEPT_PRIOR 0xfffc0000 + +/* Config area lies in DRAM */ +#define ICSSG_CONFIG_OFFSET 0x0 + +/* Config area lies in shared RAM */ +#define ICSSG_CONFIG_OFFSET_SLICE0 0 +#define ICSSG_CONFIG_OFFSET_SLICE1 0x8000 + +#define ICSSG_NUM_NORMAL_PDS 64 +#define ICSSG_NUM_SPECIAL_PDS 16 + +#define ICSSG_NORMAL_PD_SIZE 8 +#define ICSSG_SPECIAL_PD_SIZE 20 + +#define ICSSG_FLAG_MASK 0xff00ffff + +struct icssg_setclock_desc { + u8 request; + u8 restore; + u8 acknowledgment; + u8 cmp_status; + u32 margin; + u32 cyclecounter0_set; + u32 cyclecounter1_set; + u32 iepcount_set; + u32 rsvd1; + u32 rsvd2; + u32 CMP0_current; + u32 iepcount_current; + u32 difference; + u32 cyclecounter0_new; + u32 cyclecounter1_new; + u32 CMP0_new; +} __packed; + +#define ICSSG_CMD_POP_SLICE0 56 +#define ICSSG_CMD_POP_SLICE1 60 + +#define ICSSG_CMD_PUSH_SLICE0 57 +#define ICSSG_CMD_PUSH_SLICE1 61 + +#define ICSSG_RSP_POP_SLICE0 58 +#define ICSSG_RSP_POP_SLICE1 62 + +#define ICSSG_RSP_PUSH_SLICE0 56 +#define ICSSG_RSP_PUSH_SLICE1 60 + +#define ICSSG_TS_POP_SLICE0 59 +#define ICSSG_TS_POP_SLICE1 63 + +#define ICSSG_TS_PUSH_SLICE0 40 +#define ICSSG_TS_PUSH_SLICE1 41 + +/* FDB FID_C2 flag definitions */ +/* Indicates host port membership.*/ +#define ICSSG_FDB_ENTRY_P0_MEMBERSHIP BIT(0) +/* Indicates that MAC ID is connected to physical port 1 */ +#define ICSSG_FDB_ENTRY_P1_MEMBERSHIP BIT(1) +/* Indicates that MAC ID is connected to physical port 2 */ +#define ICSSG_FDB_ENTRY_P2_MEMBERSHIP BIT(2) +/* Ageable bit is set for learned entries and cleared for static entries */ +#define ICSSG_FDB_ENTRY_AGEABLE BIT(3) +/* If set for DA then packet is determined to be a special packet */ +#define ICSSG_FDB_ENTRY_BLOCK BIT(4) +/* If set for DA then the SA from the packet is not learned */ +#define ICSSG_FDB_ENTRY_SECURE BIT(5) +/* If set, it means packet has been seen recently with source address + FID + * matching MAC address/FID of entry + */ +#define ICSSG_FDB_ENTRY_TOUCHED BIT(6) +/* Set if entry is valid */ +#define ICSSG_FDB_ENTRY_VALID BIT(7) + +/** + * struct prueth_vlan_tbl - VLAN table entries struct in ICSSG SMEM + * @fid_c1: membership and forwarding rules flag to this table. See + * above to defines for bit definitions + * @fid: FDB index for this VID (there is 1-1 mapping b/w VID and FID) + */ +struct prueth_vlan_tbl { + u8 fid_c1; + u8 fid; +} __packed; + +/** + * struct prueth_fdb_slot - Result of FDB slot lookup + * @mac: MAC address + * @fid: fid to be associated with MAC + * @fid_c2: FID_C2 entry for this MAC + */ +struct prueth_fdb_slot { + u8 mac[ETH_ALEN]; + u8 fid; + u8 fid_c2; +} __packed; + +enum icssg_ietfpe_verify_states { + ICSSG_IETFPE_STATE_UNKNOWN = 0, + ICSSG_IETFPE_STATE_INITIAL, + ICSSG_IETFPE_STATE_VERIFYING, + ICSSG_IETFPE_STATE_SUCCEEDED, + ICSSG_IETFPE_STATE_FAILED, + ICSSG_IETFPE_STATE_DISABLED +}; +#endif /* __NET_TI_ICSSG_CONFIG_H */ diff --git a/drivers/net/ethernet/ti/icssg/icssg_ethtool.c b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c new file mode 100644 index 000000000000..02c312f01d10 --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#include "icssg_prueth.h" +#include "icssg_stats.h" + +static void emac_get_drvinfo(struct net_device *ndev, + struct ethtool_drvinfo *info) +{ + struct prueth_emac *emac = netdev_priv(ndev); + struct prueth *prueth = emac->prueth; + + strscpy(info->driver, dev_driver_string(prueth->dev), + sizeof(info->driver)); + strscpy(info->bus_info, dev_name(prueth->dev), sizeof(info->bus_info)); +} + +static u32 emac_get_msglevel(struct net_device *ndev) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + return emac->msg_enable; +} + +static void emac_set_msglevel(struct net_device *ndev, u32 value) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + emac->msg_enable = value; +} + +static int emac_get_link_ksettings(struct net_device *ndev, + struct ethtool_link_ksettings *ecmd) +{ + return phy_ethtool_get_link_ksettings(ndev, ecmd); +} + +static int emac_set_link_ksettings(struct net_device *ndev, + const struct ethtool_link_ksettings *ecmd) +{ + return phy_ethtool_set_link_ksettings(ndev, ecmd); +} + +static int emac_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +{ + if (!ndev->phydev) + return -EOPNOTSUPP; + + return phy_ethtool_get_eee(ndev->phydev, edata); +} + +static int emac_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +{ + if (!ndev->phydev) + return -EOPNOTSUPP; + + return phy_ethtool_set_eee(ndev->phydev, edata); +} + +static int emac_nway_reset(struct net_device *ndev) +{ + return phy_ethtool_nway_reset(ndev); +} + +static int emac_get_sset_count(struct net_device *ndev, int stringset) +{ + switch (stringset) { + case ETH_SS_STATS: + return ICSSG_NUM_ETHTOOL_STATS; + default: + return -EOPNOTSUPP; + } +} + +static void emac_get_strings(struct net_device *ndev, u32 stringset, u8 *data) +{ + u8 *p = data; + int i; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < ARRAY_SIZE(icssg_all_stats); i++) { + if (!icssg_all_stats[i].standard_stats) { + memcpy(p, icssg_all_stats[i].name, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + } + break; + default: + break; + } +} + +static void emac_get_ethtool_stats(struct net_device *ndev, + struct ethtool_stats *stats, u64 *data) +{ + struct prueth_emac *emac = netdev_priv(ndev); + int i; + + emac_update_hardware_stats(emac); + + for (i = 0; i < ARRAY_SIZE(icssg_all_stats); i++) + if (!icssg_all_stats[i].standard_stats) + *(data++) = emac->stats[i]; +} + +static int emac_set_channels(struct net_device *ndev, + struct ethtool_channels *ch) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + /* Check if interface is up. Can change the num queues when + * the interface is down. + */ + if (netif_running(emac->ndev)) + return -EBUSY; + + emac->tx_ch_num = ch->tx_count; + + return 0; +} + +static void emac_get_channels(struct net_device *ndev, + struct ethtool_channels *ch) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + ch->max_rx = 1; + ch->max_tx = PRUETH_MAX_TX_QUEUES; + ch->rx_count = 1; + ch->tx_count = emac->tx_ch_num; +} + +static const struct ethtool_rmon_hist_range emac_rmon_ranges[] = { + { 0, 64}, + { 65, 128}, + { 129, 256}, + { 257, 512}, + { 513, PRUETH_MAX_PKT_SIZE}, + {} +}; + +static void emac_get_rmon_stats(struct net_device *ndev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + *ranges = emac_rmon_ranges; + + rmon_stats->undersize_pkts = emac_get_stat_by_name(emac, "rx_bucket1_frames") - + emac_get_stat_by_name(emac, "rx_64B_frames"); + + rmon_stats->hist[0] = emac_get_stat_by_name(emac, "rx_bucket1_frames"); + rmon_stats->hist[1] = emac_get_stat_by_name(emac, "rx_bucket2_frames"); + rmon_stats->hist[2] = emac_get_stat_by_name(emac, "rx_bucket3_frames"); + rmon_stats->hist[3] = emac_get_stat_by_name(emac, "rx_bucket4_frames"); + rmon_stats->hist[4] = emac_get_stat_by_name(emac, "rx_bucket5_frames"); + + rmon_stats->hist_tx[0] = emac_get_stat_by_name(emac, "tx_bucket1_frames"); + rmon_stats->hist_tx[1] = emac_get_stat_by_name(emac, "tx_bucket2_frames"); + rmon_stats->hist_tx[2] = emac_get_stat_by_name(emac, "tx_bucket3_frames"); + rmon_stats->hist_tx[3] = emac_get_stat_by_name(emac, "tx_bucket4_frames"); + rmon_stats->hist_tx[4] = emac_get_stat_by_name(emac, "tx_bucket5_frames"); +} + +const struct ethtool_ops icssg_ethtool_ops = { + .get_drvinfo = emac_get_drvinfo, + .get_msglevel = emac_get_msglevel, + .set_msglevel = emac_set_msglevel, + .get_sset_count = emac_get_sset_count, + .get_ethtool_stats = emac_get_ethtool_stats, + .get_strings = emac_get_strings, + .get_channels = emac_get_channels, + .set_channels = emac_set_channels, + .get_link_ksettings = emac_get_link_ksettings, + .set_link_ksettings = emac_set_link_ksettings, + .get_link = ethtool_op_get_link, + .get_eee = emac_get_eee, + .set_eee = emac_set_eee, + .nway_reset = emac_nway_reset, + .get_rmon_stats = emac_get_rmon_stats, +}; diff --git a/drivers/net/ethernet/ti/icssg/icssg_mii_cfg.c b/drivers/net/ethernet/ti/icssg/icssg_mii_cfg.c new file mode 100644 index 000000000000..92718ae40d7e --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_mii_cfg.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Texas Instruments ICSSG Ethernet Driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#include <linux/etherdevice.h> +#include <linux/regmap.h> +#include <linux/types.h> + +#include "icssg_mii_rt.h" +#include "icssg_prueth.h" + +void icssg_mii_update_ipg(struct regmap *mii_rt, int mii, u32 ipg) +{ + u32 val; + + if (mii == ICSS_MII0) { + regmap_write(mii_rt, PRUSS_MII_RT_TX_IPG0, ipg); + } else { + regmap_read(mii_rt, PRUSS_MII_RT_TX_IPG0, &val); + regmap_write(mii_rt, PRUSS_MII_RT_TX_IPG1, ipg); + regmap_write(mii_rt, PRUSS_MII_RT_TX_IPG0, val); + } +} + +void icssg_mii_update_mtu(struct regmap *mii_rt, int mii, int mtu) +{ + mtu += (ETH_HLEN + ETH_FCS_LEN); + if (mii == ICSS_MII0) { + regmap_update_bits(mii_rt, + PRUSS_MII_RT_RX_FRMS0, + PRUSS_MII_RT_RX_FRMS_MAX_FRM_MASK, + (mtu - 1) << PRUSS_MII_RT_RX_FRMS_MAX_FRM_SHIFT); + } else { + regmap_update_bits(mii_rt, + PRUSS_MII_RT_RX_FRMS1, + PRUSS_MII_RT_RX_FRMS_MAX_FRM_MASK, + (mtu - 1) << PRUSS_MII_RT_RX_FRMS_MAX_FRM_SHIFT); + } +} + +void icssg_update_rgmii_cfg(struct regmap *miig_rt, struct prueth_emac *emac) +{ + u32 gig_en_mask, gig_val = 0, full_duplex_mask, full_duplex_val = 0; + int slice = prueth_emac_slice(emac); + u32 inband_en_mask, inband_val = 0; + + gig_en_mask = (slice == ICSS_MII0) ? RGMII_CFG_GIG_EN_MII0 : + RGMII_CFG_GIG_EN_MII1; + if (emac->speed == SPEED_1000) + gig_val = gig_en_mask; + regmap_update_bits(miig_rt, RGMII_CFG_OFFSET, gig_en_mask, gig_val); + + inband_en_mask = (slice == ICSS_MII0) ? RGMII_CFG_INBAND_EN_MII0 : + RGMII_CFG_INBAND_EN_MII1; + if (emac->speed == SPEED_10 && phy_interface_mode_is_rgmii(emac->phy_if)) + inband_val = inband_en_mask; + regmap_update_bits(miig_rt, RGMII_CFG_OFFSET, inband_en_mask, inband_val); + + full_duplex_mask = (slice == ICSS_MII0) ? RGMII_CFG_FULL_DUPLEX_MII0 : + RGMII_CFG_FULL_DUPLEX_MII1; + if (emac->duplex == DUPLEX_FULL) + full_duplex_val = full_duplex_mask; + regmap_update_bits(miig_rt, RGMII_CFG_OFFSET, full_duplex_mask, + full_duplex_val); +} + +void icssg_miig_set_interface_mode(struct regmap *miig_rt, int mii, phy_interface_t phy_if) +{ + u32 val, mask, shift; + + mask = mii == ICSS_MII0 ? ICSSG_CFG_MII0_MODE : ICSSG_CFG_MII1_MODE; + shift = mii == ICSS_MII0 ? ICSSG_CFG_MII0_MODE_SHIFT : ICSSG_CFG_MII1_MODE_SHIFT; + + val = MII_MODE_RGMII; + if (phy_if == PHY_INTERFACE_MODE_MII) + val = MII_MODE_MII; + + val <<= shift; + regmap_update_bits(miig_rt, ICSSG_CFG_OFFSET, mask, val); + regmap_read(miig_rt, ICSSG_CFG_OFFSET, &val); +} + +u32 icssg_rgmii_cfg_get_bitfield(struct regmap *miig_rt, u32 mask, u32 shift) +{ + u32 val; + + regmap_read(miig_rt, RGMII_CFG_OFFSET, &val); + val &= mask; + val >>= shift; + + return val; +} + +u32 icssg_rgmii_get_speed(struct regmap *miig_rt, int mii) +{ + u32 shift = RGMII_CFG_SPEED_MII0_SHIFT, mask = RGMII_CFG_SPEED_MII0; + + if (mii == ICSS_MII1) { + shift = RGMII_CFG_SPEED_MII1_SHIFT; + mask = RGMII_CFG_SPEED_MII1; + } + + return icssg_rgmii_cfg_get_bitfield(miig_rt, mask, shift); +} + +u32 icssg_rgmii_get_fullduplex(struct regmap *miig_rt, int mii) +{ + u32 shift = RGMII_CFG_FULLDUPLEX_MII0_SHIFT; + u32 mask = RGMII_CFG_FULLDUPLEX_MII0; + + if (mii == ICSS_MII1) { + shift = RGMII_CFG_FULLDUPLEX_MII1_SHIFT; + mask = RGMII_CFG_FULLDUPLEX_MII1; + } + + return icssg_rgmii_cfg_get_bitfield(miig_rt, mask, shift); +} diff --git a/drivers/net/ethernet/ti/icssg/icssg_mii_rt.h b/drivers/net/ethernet/ti/icssg/icssg_mii_rt.h new file mode 100644 index 000000000000..55a59bf5299c --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_mii_rt.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* PRU-ICSS MII_RT register definitions + * + * Copyright (C) 2015-2022 Texas Instruments Incorporated - https://www.ti.com + */ + +#ifndef __NET_PRUSS_MII_RT_H__ +#define __NET_PRUSS_MII_RT_H__ + +#include <linux/if_ether.h> +#include <linux/phy.h> + +/* PRUSS_MII_RT Registers */ +#define PRUSS_MII_RT_RXCFG0 0x0 +#define PRUSS_MII_RT_RXCFG1 0x4 +#define PRUSS_MII_RT_TXCFG0 0x10 +#define PRUSS_MII_RT_TXCFG1 0x14 +#define PRUSS_MII_RT_TX_CRC0 0x20 +#define PRUSS_MII_RT_TX_CRC1 0x24 +#define PRUSS_MII_RT_TX_IPG0 0x30 +#define PRUSS_MII_RT_TX_IPG1 0x34 +#define PRUSS_MII_RT_PRS0 0x38 +#define PRUSS_MII_RT_PRS1 0x3c +#define PRUSS_MII_RT_RX_FRMS0 0x40 +#define PRUSS_MII_RT_RX_FRMS1 0x44 +#define PRUSS_MII_RT_RX_PCNT0 0x48 +#define PRUSS_MII_RT_RX_PCNT1 0x4c +#define PRUSS_MII_RT_RX_ERR0 0x50 +#define PRUSS_MII_RT_RX_ERR1 0x54 + +/* PRUSS_MII_RT_RXCFG0/1 bits */ +#define PRUSS_MII_RT_RXCFG_RX_ENABLE BIT(0) +#define PRUSS_MII_RT_RXCFG_RX_DATA_RDY_MODE_DIS BIT(1) +#define PRUSS_MII_RT_RXCFG_RX_CUT_PREAMBLE BIT(2) +#define PRUSS_MII_RT_RXCFG_RX_MUX_SEL BIT(3) +#define PRUSS_MII_RT_RXCFG_RX_L2_EN BIT(4) +#define PRUSS_MII_RT_RXCFG_RX_BYTE_SWAP BIT(5) +#define PRUSS_MII_RT_RXCFG_RX_AUTO_FWD_PRE BIT(6) +#define PRUSS_MII_RT_RXCFG_RX_L2_EOF_SCLR_DIS BIT(9) + +/* PRUSS_MII_RT_TXCFG0/1 bits */ +#define PRUSS_MII_RT_TXCFG_TX_ENABLE BIT(0) +#define PRUSS_MII_RT_TXCFG_TX_AUTO_PREAMBLE BIT(1) +#define PRUSS_MII_RT_TXCFG_TX_EN_MODE BIT(2) +#define PRUSS_MII_RT_TXCFG_TX_BYTE_SWAP BIT(3) +#define PRUSS_MII_RT_TXCFG_TX_MUX_SEL BIT(8) +#define PRUSS_MII_RT_TXCFG_PRE_TX_AUTO_SEQUENCE BIT(9) +#define PRUSS_MII_RT_TXCFG_PRE_TX_AUTO_ESC_ERR BIT(10) +#define PRUSS_MII_RT_TXCFG_TX_32_MODE_EN BIT(11) +#define PRUSS_MII_RT_TXCFG_TX_IPG_WIRE_CLK_EN BIT(12) /* SR2.0 onwards */ + +#define PRUSS_MII_RT_TXCFG_TX_START_DELAY_SHIFT 16 +#define PRUSS_MII_RT_TXCFG_TX_START_DELAY_MASK GENMASK(25, 16) + +#define PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_SHIFT 28 +#define PRUSS_MII_RT_TXCFG_TX_CLK_DELAY_MASK GENMASK(30, 28) + +/* PRUSS_MII_RT_TX_IPG0/1 bits */ +#define PRUSS_MII_RT_TX_IPG_IPG_SHIFT 0 +#define PRUSS_MII_RT_TX_IPG_IPG_MASK GENMASK(9, 0) + +/* PRUSS_MII_RT_PRS0/1 bits */ +#define PRUSS_MII_RT_PRS_COL BIT(0) +#define PRUSS_MII_RT_PRS_CRS BIT(1) + +/* PRUSS_MII_RT_RX_FRMS0/1 bits */ +#define PRUSS_MII_RT_RX_FRMS_MIN_FRM_SHIFT 0 +#define PRUSS_MII_RT_RX_FRMS_MIN_FRM_MASK GENMASK(15, 0) + +#define PRUSS_MII_RT_RX_FRMS_MAX_FRM_SHIFT 16 +#define PRUSS_MII_RT_RX_FRMS_MAX_FRM_MASK GENMASK(31, 16) + +/* Min/Max in MII_RT_RX_FRMS */ +/* For EMAC and Switch */ +#define PRUSS_MII_RT_RX_FRMS_MAX (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) +#define PRUSS_MII_RT_RX_FRMS_MIN_FRM (64) + +/* for HSR and PRP */ +#define PRUSS_MII_RT_RX_FRMS_MAX_FRM_LRE (PRUSS_MII_RT_RX_FRMS_MAX + \ + ICSS_LRE_TAG_RCT_SIZE) +/* PRUSS_MII_RT_RX_PCNT0/1 bits */ +#define PRUSS_MII_RT_RX_PCNT_MIN_PCNT_SHIFT 0 +#define PRUSS_MII_RT_RX_PCNT_MIN_PCNT_MASK GENMASK(3, 0) + +#define PRUSS_MII_RT_RX_PCNT_MAX_PCNT_SHIFT 4 +#define PRUSS_MII_RT_RX_PCNT_MAX_PCNT_MASK GENMASK(7, 4) + +/* PRUSS_MII_RT_RX_ERR0/1 bits */ +#define PRUSS_MII_RT_RX_ERR_MIN_PCNT_ERR BIT(0) +#define PRUSS_MII_RT_RX_ERR_MAX_PCNT_ERR BIT(1) +#define PRUSS_MII_RT_RX_ERR_MIN_FRM_ERR BIT(2) +#define PRUSS_MII_RT_RX_ERR_MAX_FRM_ERR BIT(3) + +#define ICSSG_CFG_OFFSET 0 +#define RGMII_CFG_OFFSET 4 + +/* Constant to choose between MII0 and MII1 */ +#define ICSS_MII0 0 +#define ICSS_MII1 1 + +/* ICSSG_CFG Register bits */ +#define ICSSG_CFG_SGMII_MODE BIT(16) +#define ICSSG_CFG_TX_PRU_EN BIT(11) +#define ICSSG_CFG_RX_SFD_TX_SOF_EN BIT(10) +#define ICSSG_CFG_RTU_PRU_PSI_SHARE_EN BIT(9) +#define ICSSG_CFG_IEP1_TX_EN BIT(8) +#define ICSSG_CFG_MII1_MODE GENMASK(6, 5) +#define ICSSG_CFG_MII1_MODE_SHIFT 5 +#define ICSSG_CFG_MII0_MODE GENMASK(4, 3) +#define ICSSG_CFG_MII0_MODE_SHIFT 3 +#define ICSSG_CFG_RX_L2_G_EN BIT(2) +#define ICSSG_CFG_TX_L2_EN BIT(1) +#define ICSSG_CFG_TX_L1_EN BIT(0) + +enum mii_mode { + MII_MODE_MII = 0, + MII_MODE_RGMII +}; + +/* RGMII CFG Register bits */ +#define RGMII_CFG_INBAND_EN_MII0 BIT(16) +#define RGMII_CFG_GIG_EN_MII0 BIT(17) +#define RGMII_CFG_INBAND_EN_MII1 BIT(20) +#define RGMII_CFG_GIG_EN_MII1 BIT(21) +#define RGMII_CFG_FULL_DUPLEX_MII0 BIT(18) +#define RGMII_CFG_FULL_DUPLEX_MII1 BIT(22) +#define RGMII_CFG_SPEED_MII0 GENMASK(2, 1) +#define RGMII_CFG_SPEED_MII1 GENMASK(6, 5) +#define RGMII_CFG_SPEED_MII0_SHIFT 1 +#define RGMII_CFG_SPEED_MII1_SHIFT 5 +#define RGMII_CFG_FULLDUPLEX_MII0 BIT(3) +#define RGMII_CFG_FULLDUPLEX_MII1 BIT(7) +#define RGMII_CFG_FULLDUPLEX_MII0_SHIFT 3 +#define RGMII_CFG_FULLDUPLEX_MII1_SHIFT 7 +#define RGMII_CFG_SPEED_10M 0 +#define RGMII_CFG_SPEED_100M 1 +#define RGMII_CFG_SPEED_1G 2 + +struct regmap; +struct prueth_emac; + +void icssg_mii_update_ipg(struct regmap *mii_rt, int mii, u32 ipg); +void icssg_mii_update_mtu(struct regmap *mii_rt, int mii, int mtu); +void icssg_update_rgmii_cfg(struct regmap *miig_rt, struct prueth_emac *emac); +u32 icssg_rgmii_cfg_get_bitfield(struct regmap *miig_rt, u32 mask, u32 shift); +u32 icssg_rgmii_get_speed(struct regmap *miig_rt, int mii); +u32 icssg_rgmii_get_fullduplex(struct regmap *miig_rt, int mii); +void icssg_miig_set_interface_mode(struct regmap *miig_rt, int mii, phy_interface_t phy_if); + +#endif /* __NET_PRUSS_MII_RT_H__ */ diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c new file mode 100644 index 000000000000..47b941fb0198 --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -0,0 +1,1897 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Texas Instruments ICSSG Ethernet Driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/dma/ti-cppi5.h> +#include <linux/etherdevice.h> +#include <linux/genalloc.h> +#include <linux/if_vlan.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_mdio.h> +#include <linux/of_net.h> +#include <linux/of_platform.h> +#include <linux/phy.h> +#include <linux/remoteproc/pruss.h> +#include <linux/regmap.h> +#include <linux/remoteproc.h> + +#include "icssg_prueth.h" +#include "icssg_mii_rt.h" +#include "../k3-cppi-desc-pool.h" + +#define PRUETH_MODULE_DESCRIPTION "PRUSS ICSSG Ethernet driver" + +/* Netif debug messages possible */ +#define PRUETH_EMAC_DEBUG (NETIF_MSG_DRV | \ + NETIF_MSG_PROBE | \ + NETIF_MSG_LINK | \ + NETIF_MSG_TIMER | \ + NETIF_MSG_IFDOWN | \ + NETIF_MSG_IFUP | \ + NETIF_MSG_RX_ERR | \ + NETIF_MSG_TX_ERR | \ + NETIF_MSG_TX_QUEUED | \ + NETIF_MSG_INTR | \ + NETIF_MSG_TX_DONE | \ + NETIF_MSG_RX_STATUS | \ + NETIF_MSG_PKTDATA | \ + NETIF_MSG_HW | \ + NETIF_MSG_WOL) + +#define prueth_napi_to_emac(napi) container_of(napi, struct prueth_emac, napi_rx) + +/* CTRLMMR_ICSSG_RGMII_CTRL register bits */ +#define ICSSG_CTRL_RGMII_ID_MODE BIT(24) + +static void prueth_cleanup_rx_chns(struct prueth_emac *emac, + struct prueth_rx_chn *rx_chn, + int max_rflows) +{ + if (rx_chn->desc_pool) + k3_cppi_desc_pool_destroy(rx_chn->desc_pool); + + if (rx_chn->rx_chn) + k3_udma_glue_release_rx_chn(rx_chn->rx_chn); +} + +static void prueth_cleanup_tx_chns(struct prueth_emac *emac) +{ + int i; + + for (i = 0; i < emac->tx_ch_num; i++) { + struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; + + if (tx_chn->desc_pool) + k3_cppi_desc_pool_destroy(tx_chn->desc_pool); + + if (tx_chn->tx_chn) + k3_udma_glue_release_tx_chn(tx_chn->tx_chn); + + /* Assume prueth_cleanup_tx_chns() is called at the + * end after all channel resources are freed + */ + memset(tx_chn, 0, sizeof(*tx_chn)); + } +} + +static void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num) +{ + int i; + + for (i = 0; i < num; i++) { + struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; + + if (tx_chn->irq) + free_irq(tx_chn->irq, tx_chn); + netif_napi_del(&tx_chn->napi_tx); + } +} + +static void prueth_xmit_free(struct prueth_tx_chn *tx_chn, + struct cppi5_host_desc_t *desc) +{ + struct cppi5_host_desc_t *first_desc, *next_desc; + dma_addr_t buf_dma, next_desc_dma; + u32 buf_dma_len; + + first_desc = desc; + next_desc = first_desc; + + cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len); + k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); + + dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, + DMA_TO_DEVICE); + + next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc); + k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); + while (next_desc_dma) { + next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, + next_desc_dma); + cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len); + k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); + + dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len, + DMA_TO_DEVICE); + + next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc); + k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); + + k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); + } + + k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc); +} + +static int emac_tx_complete_packets(struct prueth_emac *emac, int chn, + int budget) +{ + struct net_device *ndev = emac->ndev; + struct cppi5_host_desc_t *desc_tx; + struct netdev_queue *netif_txq; + struct prueth_tx_chn *tx_chn; + unsigned int total_bytes = 0; + struct sk_buff *skb; + dma_addr_t desc_dma; + int res, num_tx = 0; + void **swdata; + + tx_chn = &emac->tx_chns[chn]; + + while (true) { + res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); + if (res == -ENODATA) + break; + + /* teardown completion */ + if (cppi5_desc_is_tdcm(desc_dma)) { + if (atomic_dec_and_test(&emac->tdown_cnt)) + complete(&emac->tdown_complete); + break; + } + + desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, + desc_dma); + swdata = cppi5_hdesc_get_swdata(desc_tx); + + skb = *(swdata); + prueth_xmit_free(tx_chn, desc_tx); + + ndev = skb->dev; + ndev->stats.tx_packets++; + ndev->stats.tx_bytes += skb->len; + total_bytes += skb->len; + napi_consume_skb(skb, budget); + num_tx++; + } + + if (!num_tx) + return 0; + + netif_txq = netdev_get_tx_queue(ndev, chn); + netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); + + if (netif_tx_queue_stopped(netif_txq)) { + /* If the TX queue was stopped, wake it now + * if we have enough room. + */ + __netif_tx_lock(netif_txq, smp_processor_id()); + if (netif_running(ndev) && + (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= + MAX_SKB_FRAGS)) + netif_tx_wake_queue(netif_txq); + __netif_tx_unlock(netif_txq); + } + + return num_tx; +} + +static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget) +{ + struct prueth_tx_chn *tx_chn = prueth_napi_to_tx_chn(napi_tx); + struct prueth_emac *emac = tx_chn->emac; + int num_tx_packets; + + num_tx_packets = emac_tx_complete_packets(emac, tx_chn->id, budget); + + if (num_tx_packets >= budget) + return budget; + + if (napi_complete_done(napi_tx, num_tx_packets)) + enable_irq(tx_chn->irq); + + return num_tx_packets; +} + +static irqreturn_t prueth_tx_irq(int irq, void *dev_id) +{ + struct prueth_tx_chn *tx_chn = dev_id; + + disable_irq_nosync(irq); + napi_schedule(&tx_chn->napi_tx); + + return IRQ_HANDLED; +} + +static int prueth_ndev_add_tx_napi(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + int i, ret; + + for (i = 0; i < emac->tx_ch_num; i++) { + struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; + + netif_napi_add_tx(emac->ndev, &tx_chn->napi_tx, emac_napi_tx_poll); + ret = request_irq(tx_chn->irq, prueth_tx_irq, + IRQF_TRIGGER_HIGH, tx_chn->name, + tx_chn); + if (ret) { + netif_napi_del(&tx_chn->napi_tx); + dev_err(prueth->dev, "unable to request TX IRQ %d\n", + tx_chn->irq); + goto fail; + } + } + + return 0; +fail: + prueth_ndev_del_tx_napi(emac, i); + return ret; +} + +static int prueth_init_tx_chns(struct prueth_emac *emac) +{ + static const struct k3_ring_cfg ring_cfg = { + .elm_size = K3_RINGACC_RING_ELSIZE_8, + .mode = K3_RINGACC_RING_MODE_RING, + .flags = 0, + .size = PRUETH_MAX_TX_DESC, + }; + struct k3_udma_glue_tx_channel_cfg tx_cfg; + struct device *dev = emac->prueth->dev; + struct net_device *ndev = emac->ndev; + int ret, slice, i; + u32 hdesc_size; + + slice = prueth_emac_slice(emac); + if (slice < 0) + return slice; + + init_completion(&emac->tdown_complete); + + hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE, + PRUETH_NAV_SW_DATA_SIZE); + memset(&tx_cfg, 0, sizeof(tx_cfg)); + tx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE; + tx_cfg.tx_cfg = ring_cfg; + tx_cfg.txcq_cfg = ring_cfg; + + for (i = 0; i < emac->tx_ch_num; i++) { + struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; + + /* To differentiate channels for SLICE0 vs SLICE1 */ + snprintf(tx_chn->name, sizeof(tx_chn->name), + "tx%d-%d", slice, i); + + tx_chn->emac = emac; + tx_chn->id = i; + tx_chn->descs_num = PRUETH_MAX_TX_DESC; + + tx_chn->tx_chn = + k3_udma_glue_request_tx_chn(dev, tx_chn->name, + &tx_cfg); + if (IS_ERR(tx_chn->tx_chn)) { + ret = PTR_ERR(tx_chn->tx_chn); + tx_chn->tx_chn = NULL; + netdev_err(ndev, + "Failed to request tx dma ch: %d\n", ret); + goto fail; + } + + tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn); + tx_chn->desc_pool = + k3_cppi_desc_pool_create_name(tx_chn->dma_dev, + tx_chn->descs_num, + hdesc_size, + tx_chn->name); + if (IS_ERR(tx_chn->desc_pool)) { + ret = PTR_ERR(tx_chn->desc_pool); + tx_chn->desc_pool = NULL; + netdev_err(ndev, "Failed to create tx pool: %d\n", ret); + goto fail; + } + + tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); + if (tx_chn->irq <= 0) { + ret = -EINVAL; + netdev_err(ndev, "failed to get tx irq\n"); + goto fail; + } + + snprintf(tx_chn->name, sizeof(tx_chn->name), "%s-tx%d", + dev_name(dev), tx_chn->id); + } + + return 0; + +fail: + prueth_cleanup_tx_chns(emac); + return ret; +} + +static int prueth_init_rx_chns(struct prueth_emac *emac, + struct prueth_rx_chn *rx_chn, + char *name, u32 max_rflows, + u32 max_desc_num) +{ + struct k3_udma_glue_rx_channel_cfg rx_cfg; + struct device *dev = emac->prueth->dev; + struct net_device *ndev = emac->ndev; + u32 fdqring_id, hdesc_size; + int i, ret = 0, slice; + + slice = prueth_emac_slice(emac); + if (slice < 0) + return slice; + + /* To differentiate channels for SLICE0 vs SLICE1 */ + snprintf(rx_chn->name, sizeof(rx_chn->name), "%s%d", name, slice); + + hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE, + PRUETH_NAV_SW_DATA_SIZE); + memset(&rx_cfg, 0, sizeof(rx_cfg)); + rx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE; + rx_cfg.flow_id_num = max_rflows; + rx_cfg.flow_id_base = -1; /* udmax will auto select flow id base */ + + /* init all flows */ + rx_chn->dev = dev; + rx_chn->descs_num = max_desc_num; + + rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, rx_chn->name, + &rx_cfg); + if (IS_ERR(rx_chn->rx_chn)) { + ret = PTR_ERR(rx_chn->rx_chn); + rx_chn->rx_chn = NULL; + netdev_err(ndev, "Failed to request rx dma ch: %d\n", ret); + goto fail; + } + + rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn); + rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev, + rx_chn->descs_num, + hdesc_size, + rx_chn->name); + if (IS_ERR(rx_chn->desc_pool)) { + ret = PTR_ERR(rx_chn->desc_pool); + rx_chn->desc_pool = NULL; + netdev_err(ndev, "Failed to create rx pool: %d\n", ret); + goto fail; + } + + emac->rx_flow_id_base = k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn); + netdev_dbg(ndev, "flow id base = %d\n", emac->rx_flow_id_base); + + fdqring_id = K3_RINGACC_RING_ID_ANY; + for (i = 0; i < rx_cfg.flow_id_num; i++) { + struct k3_ring_cfg rxring_cfg = { + .elm_size = K3_RINGACC_RING_ELSIZE_8, + .mode = K3_RINGACC_RING_MODE_RING, + .flags = 0, + }; + struct k3_ring_cfg fdqring_cfg = { + .elm_size = K3_RINGACC_RING_ELSIZE_8, + .flags = K3_RINGACC_RING_SHARED, + }; + struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = { + .rx_cfg = rxring_cfg, + .rxfdq_cfg = fdqring_cfg, + .ring_rxq_id = K3_RINGACC_RING_ID_ANY, + .src_tag_lo_sel = + K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG, + }; + + rx_flow_cfg.ring_rxfdq0_id = fdqring_id; + rx_flow_cfg.rx_cfg.size = max_desc_num; + rx_flow_cfg.rxfdq_cfg.size = max_desc_num; + rx_flow_cfg.rxfdq_cfg.mode = emac->prueth->pdata.fdqring_mode; + + ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn, + i, &rx_flow_cfg); + if (ret) { + netdev_err(ndev, "Failed to init rx flow%d %d\n", + i, ret); + goto fail; + } + if (!i) + fdqring_id = k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn, + i); + rx_chn->irq[i] = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i); + if (rx_chn->irq[i] <= 0) { + ret = rx_chn->irq[i]; + netdev_err(ndev, "Failed to get rx dma irq"); + goto fail; + } + } + + return 0; + +fail: + prueth_cleanup_rx_chns(emac, rx_chn, max_rflows); + return ret; +} + +static int prueth_dma_rx_push(struct prueth_emac *emac, + struct sk_buff *skb, + struct prueth_rx_chn *rx_chn) +{ + struct net_device *ndev = emac->ndev; + struct cppi5_host_desc_t *desc_rx; + u32 pkt_len = skb_tailroom(skb); + dma_addr_t desc_dma; + dma_addr_t buf_dma; + void **swdata; + + desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool); + if (!desc_rx) { + netdev_err(ndev, "rx push: failed to allocate descriptor\n"); + return -ENOMEM; + } + desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); + + buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); + netdev_err(ndev, "rx push: failed to map rx pkt buffer\n"); + return -EINVAL; + } + + cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT, + PRUETH_NAV_PS_DATA_SIZE); + k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); + cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb)); + + swdata = cppi5_hdesc_get_swdata(desc_rx); + *swdata = skb; + + return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, + desc_rx, desc_dma); +} + +static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id) +{ + struct prueth_rx_chn *rx_chn = &emac->rx_chns; + u32 buf_dma_len, pkt_len, port_id = 0; + struct net_device *ndev = emac->ndev; + struct cppi5_host_desc_t *desc_rx; + struct sk_buff *skb, *new_skb; + dma_addr_t desc_dma, buf_dma; + void **swdata; + int ret; + + ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma); + if (ret) { + if (ret != -ENODATA) + netdev_err(ndev, "rx pop: failed: %d\n", ret); + return ret; + } + + if (cppi5_desc_is_tdcm(desc_dma)) /* Teardown ? */ + return 0; + + desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); + + swdata = cppi5_hdesc_get_swdata(desc_rx); + skb = *swdata; + + cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); + k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); + pkt_len = cppi5_hdesc_get_pktlen(desc_rx); + /* firmware adds 4 CRC bytes, strip them */ + pkt_len -= 4; + cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); + + dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); + + skb->dev = ndev; + new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE); + /* if allocation fails we drop the packet but push the + * descriptor back to the ring with old skb to prevent a stall + */ + if (!new_skb) { + ndev->stats.rx_dropped++; + new_skb = skb; + } else { + /* send the filled skb up the n/w stack */ + skb_put(skb, pkt_len); + skb->protocol = eth_type_trans(skb, ndev); + napi_gro_receive(&emac->napi_rx, skb); + ndev->stats.rx_bytes += pkt_len; + ndev->stats.rx_packets++; + } + + /* queue another RX DMA */ + ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns); + if (WARN_ON(ret < 0)) { + dev_kfree_skb_any(new_skb); + ndev->stats.rx_errors++; + ndev->stats.rx_dropped++; + } + + return ret; +} + +static void prueth_rx_cleanup(void *data, dma_addr_t desc_dma) +{ + struct prueth_rx_chn *rx_chn = data; + struct cppi5_host_desc_t *desc_rx; + struct sk_buff *skb; + dma_addr_t buf_dma; + u32 buf_dma_len; + void **swdata; + + desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); + swdata = cppi5_hdesc_get_swdata(desc_rx); + skb = *swdata; + cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); + k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); + + dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, + DMA_FROM_DEVICE); + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); + + dev_kfree_skb_any(skb); +} + +/** + * emac_ndo_start_xmit - EMAC Transmit function + * @skb: SKB pointer + * @ndev: EMAC network adapter + * + * Called by the system to transmit a packet - we queue the packet in + * EMAC hardware transmit queue + * Doesn't wait for completion we'll check for TX completion in + * emac_tx_complete_packets(). + * + * Return: enum netdev_tx + */ +static enum netdev_tx emac_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc; + struct prueth_emac *emac = netdev_priv(ndev); + struct netdev_queue *netif_txq; + struct prueth_tx_chn *tx_chn; + dma_addr_t desc_dma, buf_dma; + int i, ret = 0, q_idx; + void **swdata; + u32 pkt_len; + u32 *epib; + + pkt_len = skb_headlen(skb); + q_idx = skb_get_queue_mapping(skb); + + tx_chn = &emac->tx_chns[q_idx]; + netif_txq = netdev_get_tx_queue(ndev, q_idx); + + /* Map the linear buffer */ + buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, DMA_TO_DEVICE); + if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) { + netdev_err(ndev, "tx: failed to map skb buffer\n"); + ret = NETDEV_TX_OK; + goto drop_free_skb; + } + + first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); + if (!first_desc) { + netdev_dbg(ndev, "tx: failed to allocate descriptor\n"); + dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, DMA_TO_DEVICE); + goto drop_stop_q_busy; + } + + cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, + PRUETH_NAV_PS_DATA_SIZE); + cppi5_hdesc_set_pkttype(first_desc, 0); + epib = first_desc->epib; + epib[0] = 0; + epib[1] = 0; + + /* set dst tag to indicate internal qid at the firmware which is at + * bit8..bit15. bit0..bit7 indicates port num for directed + * packets in case of switch mode operation + */ + cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8))); + k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); + cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); + swdata = cppi5_hdesc_get_swdata(first_desc); + *swdata = skb; + + /* Handle the case where skb is fragmented in pages */ + cur_desc = first_desc; + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + u32 frag_size = skb_frag_size(frag); + + next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); + if (!next_desc) { + netdev_err(ndev, + "tx: failed to allocate frag. descriptor\n"); + goto free_desc_stop_q_busy; + } + + buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size, + DMA_TO_DEVICE); + if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) { + netdev_err(ndev, "tx: Failed to map skb page\n"); + k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); + ret = NETDEV_TX_OK; + goto drop_free_descs; + } + + cppi5_hdesc_reset_hbdesc(next_desc); + k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); + cppi5_hdesc_attach_buf(next_desc, + buf_dma, frag_size, buf_dma, frag_size); + + desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, + next_desc); + k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma); + cppi5_hdesc_link_hbdesc(cur_desc, desc_dma); + + pkt_len += frag_size; + cur_desc = next_desc; + } + WARN_ON_ONCE(pkt_len != skb->len); + + /* report bql before sending packet */ + netdev_tx_sent_queue(netif_txq, pkt_len); + + cppi5_hdesc_set_pktlen(first_desc, pkt_len); + desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); + /* cppi5_desc_dump(first_desc, 64); */ + + skb_tx_timestamp(skb); /* SW timestamp if SKBTX_IN_PROGRESS not set */ + ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); + if (ret) { + netdev_err(ndev, "tx: push failed: %d\n", ret); + goto drop_free_descs; + } + + if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) { + netif_tx_stop_queue(netif_txq); + /* Barrier, so that stop_queue visible to other cpus */ + smp_mb__after_atomic(); + + if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= + MAX_SKB_FRAGS) + netif_tx_wake_queue(netif_txq); + } + + return NETDEV_TX_OK; + +drop_free_descs: + prueth_xmit_free(tx_chn, first_desc); + +drop_free_skb: + dev_kfree_skb_any(skb); + + /* error */ + ndev->stats.tx_dropped++; + netdev_err(ndev, "tx: error: %d\n", ret); + + return ret; + +free_desc_stop_q_busy: + prueth_xmit_free(tx_chn, first_desc); + +drop_stop_q_busy: + netif_tx_stop_queue(netif_txq); + return NETDEV_TX_BUSY; +} + +static void prueth_tx_cleanup(void *data, dma_addr_t desc_dma) +{ + struct prueth_tx_chn *tx_chn = data; + struct cppi5_host_desc_t *desc_tx; + struct sk_buff *skb; + void **swdata; + + desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); + swdata = cppi5_hdesc_get_swdata(desc_tx); + skb = *(swdata); + prueth_xmit_free(tx_chn, desc_tx); + + dev_kfree_skb_any(skb); +} + +static irqreturn_t prueth_rx_irq(int irq, void *dev_id) +{ + struct prueth_emac *emac = dev_id; + + disable_irq_nosync(irq); + napi_schedule(&emac->napi_rx); + + return IRQ_HANDLED; +} + +struct icssg_firmwares { + char *pru; + char *rtu; + char *txpru; +}; + +static struct icssg_firmwares icssg_emac_firmwares[] = { + { + .pru = "ti-pruss/am65x-sr2-pru0-prueth-fw.elf", + .rtu = "ti-pruss/am65x-sr2-rtu0-prueth-fw.elf", + .txpru = "ti-pruss/am65x-sr2-txpru0-prueth-fw.elf", + }, + { + .pru = "ti-pruss/am65x-sr2-pru1-prueth-fw.elf", + .rtu = "ti-pruss/am65x-sr2-rtu1-prueth-fw.elf", + .txpru = "ti-pruss/am65x-sr2-txpru1-prueth-fw.elf", + } +}; + +static int prueth_emac_start(struct prueth *prueth, struct prueth_emac *emac) +{ + struct icssg_firmwares *firmwares; + struct device *dev = prueth->dev; + int slice, ret; + + firmwares = icssg_emac_firmwares; + + slice = prueth_emac_slice(emac); + if (slice < 0) { + netdev_err(emac->ndev, "invalid port\n"); + return -EINVAL; + } + + ret = icssg_config(prueth, emac, slice); + if (ret) + return ret; + + ret = rproc_set_firmware(prueth->pru[slice], firmwares[slice].pru); + ret = rproc_boot(prueth->pru[slice]); + if (ret) { + dev_err(dev, "failed to boot PRU%d: %d\n", slice, ret); + return -EINVAL; + } + + ret = rproc_set_firmware(prueth->rtu[slice], firmwares[slice].rtu); + ret = rproc_boot(prueth->rtu[slice]); + if (ret) { + dev_err(dev, "failed to boot RTU%d: %d\n", slice, ret); + goto halt_pru; + } + + ret = rproc_set_firmware(prueth->txpru[slice], firmwares[slice].txpru); + ret = rproc_boot(prueth->txpru[slice]); + if (ret) { + dev_err(dev, "failed to boot TX_PRU%d: %d\n", slice, ret); + goto halt_rtu; + } + + emac->fw_running = 1; + return 0; + +halt_rtu: + rproc_shutdown(prueth->rtu[slice]); + +halt_pru: + rproc_shutdown(prueth->pru[slice]); + + return ret; +} + +static void prueth_emac_stop(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + int slice; + + switch (emac->port_id) { + case PRUETH_PORT_MII0: + slice = ICSS_SLICE0; + break; + case PRUETH_PORT_MII1: + slice = ICSS_SLICE1; + break; + default: + netdev_err(emac->ndev, "invalid port\n"); + return; + } + + emac->fw_running = 0; + rproc_shutdown(prueth->txpru[slice]); + rproc_shutdown(prueth->rtu[slice]); + rproc_shutdown(prueth->pru[slice]); +} + +/* called back by PHY layer if there is change in link state of hw port*/ +static void emac_adjust_link(struct net_device *ndev) +{ + struct prueth_emac *emac = netdev_priv(ndev); + struct phy_device *phydev = ndev->phydev; + struct prueth *prueth = emac->prueth; + bool new_state = false; + unsigned long flags; + + if (phydev->link) { + /* check the mode of operation - full/half duplex */ + if (phydev->duplex != emac->duplex) { + new_state = true; + emac->duplex = phydev->duplex; + } + if (phydev->speed != emac->speed) { + new_state = true; + emac->speed = phydev->speed; + } + if (!emac->link) { + new_state = true; + emac->link = 1; + } + } else if (emac->link) { + new_state = true; + emac->link = 0; + + /* f/w should support 100 & 1000 */ + emac->speed = SPEED_1000; + + /* half duplex may not be supported by f/w */ + emac->duplex = DUPLEX_FULL; + } + + if (new_state) { + phy_print_status(phydev); + + /* update RGMII and MII configuration based on PHY negotiated + * values + */ + if (emac->link) { + /* Set the RGMII cfg for gig en and full duplex */ + icssg_update_rgmii_cfg(prueth->miig_rt, emac); + + /* update the Tx IPG based on 100M/1G speed */ + spin_lock_irqsave(&emac->lock, flags); + icssg_config_ipg(emac); + spin_unlock_irqrestore(&emac->lock, flags); + icssg_config_set_speed(emac); + emac_set_port_state(emac, ICSSG_EMAC_PORT_FORWARD); + + } else { + emac_set_port_state(emac, ICSSG_EMAC_PORT_DISABLE); + } + } + + if (emac->link) { + /* reactivate the transmit queue */ + netif_tx_wake_all_queues(ndev); + } else { + netif_tx_stop_all_queues(ndev); + } +} + +static int emac_napi_rx_poll(struct napi_struct *napi_rx, int budget) +{ + struct prueth_emac *emac = prueth_napi_to_emac(napi_rx); + int rx_flow = PRUETH_RX_FLOW_DATA; + int flow = PRUETH_MAX_RX_FLOWS; + int num_rx = 0; + int cur_budget; + int ret; + + while (flow--) { + cur_budget = budget - num_rx; + + while (cur_budget--) { + ret = emac_rx_packet(emac, flow); + if (ret) + break; + num_rx++; + } + + if (num_rx >= budget) + break; + } + + if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) + enable_irq(emac->rx_chns.irq[rx_flow]); + + return num_rx; +} + +static int prueth_prepare_rx_chan(struct prueth_emac *emac, + struct prueth_rx_chn *chn, + int buf_size) +{ + struct sk_buff *skb; + int i, ret; + + for (i = 0; i < chn->descs_num; i++) { + skb = __netdev_alloc_skb_ip_align(NULL, buf_size, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + ret = prueth_dma_rx_push(emac, skb, chn); + if (ret < 0) { + netdev_err(emac->ndev, + "cannot submit skb for rx chan %s ret %d\n", + chn->name, ret); + kfree_skb(skb); + return ret; + } + } + + return 0; +} + +static void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num, + bool free_skb) +{ + int i; + + for (i = 0; i < ch_num; i++) { + if (free_skb) + k3_udma_glue_reset_tx_chn(emac->tx_chns[i].tx_chn, + &emac->tx_chns[i], + prueth_tx_cleanup); + k3_udma_glue_disable_tx_chn(emac->tx_chns[i].tx_chn); + } +} + +static void prueth_reset_rx_chan(struct prueth_rx_chn *chn, + int num_flows, bool disable) +{ + int i; + + for (i = 0; i < num_flows; i++) + k3_udma_glue_reset_rx_chn(chn->rx_chn, i, chn, + prueth_rx_cleanup, !!i); + if (disable) + k3_udma_glue_disable_rx_chn(chn->rx_chn); +} + +static int emac_phy_connect(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + struct net_device *ndev = emac->ndev; + /* connect PHY */ + ndev->phydev = of_phy_connect(emac->ndev, emac->phy_node, + &emac_adjust_link, 0, + emac->phy_if); + if (!ndev->phydev) { + dev_err(prueth->dev, "couldn't connect to phy %s\n", + emac->phy_node->full_name); + return -ENODEV; + } + + /* remove unsupported modes */ + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_Pause_BIT); + phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_Asym_Pause_BIT); + + if (emac->phy_if == PHY_INTERFACE_MODE_MII) + phy_set_max_speed(ndev->phydev, SPEED_100); + + return 0; +} + +/** + * emac_ndo_open - EMAC device open + * @ndev: network adapter device + * + * Called when system wants to start the interface. + * + * Return: 0 for a successful open, or appropriate error code + */ +static int emac_ndo_open(struct net_device *ndev) +{ + struct prueth_emac *emac = netdev_priv(ndev); + int ret, i, num_data_chn = emac->tx_ch_num; + struct prueth *prueth = emac->prueth; + int slice = prueth_emac_slice(emac); + struct device *dev = prueth->dev; + int max_rx_flows; + int rx_flow; + + /* clear SMEM and MSMC settings for all slices */ + if (!prueth->emacs_initialized) { + memset_io(prueth->msmcram.va, 0, prueth->msmcram.size); + memset_io(prueth->shram.va, 0, ICSSG_CONFIG_OFFSET_SLICE1 * PRUETH_NUM_MACS); + } + + /* set h/w MAC as user might have re-configured */ + ether_addr_copy(emac->mac_addr, ndev->dev_addr); + + icssg_class_set_mac_addr(prueth->miig_rt, slice, emac->mac_addr); + icssg_ft1_set_mac_addr(prueth->miig_rt, slice, emac->mac_addr); + + icssg_class_default(prueth->miig_rt, slice, 0); + + /* Notify the stack of the actual queue counts. */ + ret = netif_set_real_num_tx_queues(ndev, num_data_chn); + if (ret) { + dev_err(dev, "cannot set real number of tx queues\n"); + return ret; + } + + init_completion(&emac->cmd_complete); + ret = prueth_init_tx_chns(emac); + if (ret) { + dev_err(dev, "failed to init tx channel: %d\n", ret); + return ret; + } + + max_rx_flows = PRUETH_MAX_RX_FLOWS; + ret = prueth_init_rx_chns(emac, &emac->rx_chns, "rx", + max_rx_flows, PRUETH_MAX_RX_DESC); + if (ret) { + dev_err(dev, "failed to init rx channel: %d\n", ret); + goto cleanup_tx; + } + + ret = prueth_ndev_add_tx_napi(emac); + if (ret) + goto cleanup_rx; + + /* we use only the highest priority flow for now i.e. @irq[3] */ + rx_flow = PRUETH_RX_FLOW_DATA; + ret = request_irq(emac->rx_chns.irq[rx_flow], prueth_rx_irq, + IRQF_TRIGGER_HIGH, dev_name(dev), emac); + if (ret) { + dev_err(dev, "unable to request RX IRQ\n"); + goto cleanup_napi; + } + + /* reset and start PRU firmware */ + ret = prueth_emac_start(prueth, emac); + if (ret) + goto free_rx_irq; + + icssg_mii_update_mtu(prueth->mii_rt, slice, ndev->max_mtu); + + /* Prepare RX */ + ret = prueth_prepare_rx_chan(emac, &emac->rx_chns, PRUETH_MAX_PKT_SIZE); + if (ret) + goto stop; + + ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn); + if (ret) + goto reset_rx_chn; + + for (i = 0; i < emac->tx_ch_num; i++) { + ret = k3_udma_glue_enable_tx_chn(emac->tx_chns[i].tx_chn); + if (ret) + goto reset_tx_chan; + } + + /* Enable NAPI in Tx and Rx direction */ + for (i = 0; i < emac->tx_ch_num; i++) + napi_enable(&emac->tx_chns[i].napi_tx); + napi_enable(&emac->napi_rx); + + /* start PHY */ + phy_start(ndev->phydev); + + prueth->emacs_initialized++; + + queue_work(system_long_wq, &emac->stats_work.work); + + return 0; + +reset_tx_chan: + /* Since interface is not yet up, there is wouldn't be + * any SKB for completion. So set false to free_skb + */ + prueth_reset_tx_chan(emac, i, false); +reset_rx_chn: + prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, false); +stop: + prueth_emac_stop(emac); +free_rx_irq: + free_irq(emac->rx_chns.irq[rx_flow], emac); +cleanup_napi: + prueth_ndev_del_tx_napi(emac, emac->tx_ch_num); +cleanup_rx: + prueth_cleanup_rx_chns(emac, &emac->rx_chns, max_rx_flows); +cleanup_tx: + prueth_cleanup_tx_chns(emac); + + return ret; +} + +/** + * emac_ndo_stop - EMAC device stop + * @ndev: network adapter device + * + * Called when system wants to stop or down the interface. + * + * Return: Always 0 (Success) + */ +static int emac_ndo_stop(struct net_device *ndev) +{ + struct prueth_emac *emac = netdev_priv(ndev); + struct prueth *prueth = emac->prueth; + int rx_flow = PRUETH_RX_FLOW_DATA; + int max_rx_flows; + int ret, i; + + /* inform the upper layers. */ + netif_tx_stop_all_queues(ndev); + + /* block packets from wire */ + if (ndev->phydev) + phy_stop(ndev->phydev); + + icssg_class_disable(prueth->miig_rt, prueth_emac_slice(emac)); + + atomic_set(&emac->tdown_cnt, emac->tx_ch_num); + /* ensure new tdown_cnt value is visible */ + smp_mb__after_atomic(); + /* tear down and disable UDMA channels */ + reinit_completion(&emac->tdown_complete); + for (i = 0; i < emac->tx_ch_num; i++) + k3_udma_glue_tdown_tx_chn(emac->tx_chns[i].tx_chn, false); + + ret = wait_for_completion_timeout(&emac->tdown_complete, + msecs_to_jiffies(1000)); + if (!ret) + netdev_err(ndev, "tx teardown timeout\n"); + + prueth_reset_tx_chan(emac, emac->tx_ch_num, true); + for (i = 0; i < emac->tx_ch_num; i++) + napi_disable(&emac->tx_chns[i].napi_tx); + + max_rx_flows = PRUETH_MAX_RX_FLOWS; + k3_udma_glue_tdown_rx_chn(emac->rx_chns.rx_chn, true); + + prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, true); + + napi_disable(&emac->napi_rx); + + cancel_work_sync(&emac->rx_mode_work); + + /* Destroying the queued work in ndo_stop() */ + cancel_delayed_work_sync(&emac->stats_work); + + /* stop PRUs */ + prueth_emac_stop(emac); + + free_irq(emac->rx_chns.irq[rx_flow], emac); + prueth_ndev_del_tx_napi(emac, emac->tx_ch_num); + prueth_cleanup_tx_chns(emac); + + prueth_cleanup_rx_chns(emac, &emac->rx_chns, max_rx_flows); + prueth_cleanup_tx_chns(emac); + + prueth->emacs_initialized--; + + return 0; +} + +static void emac_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) +{ + ndev->stats.tx_errors++; +} + +static void emac_ndo_set_rx_mode_work(struct work_struct *work) +{ + struct prueth_emac *emac = container_of(work, struct prueth_emac, rx_mode_work); + struct net_device *ndev = emac->ndev; + bool promisc, allmulti; + + if (!netif_running(ndev)) + return; + + promisc = ndev->flags & IFF_PROMISC; + allmulti = ndev->flags & IFF_ALLMULTI; + emac_set_port_state(emac, ICSSG_EMAC_PORT_UC_FLOODING_DISABLE); + emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_DISABLE); + + if (promisc) { + emac_set_port_state(emac, ICSSG_EMAC_PORT_UC_FLOODING_ENABLE); + emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE); + return; + } + + if (allmulti) { + emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE); + return; + } + + if (!netdev_mc_empty(ndev)) { + emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE); + return; + } +} + +/** + * emac_ndo_set_rx_mode - EMAC set receive mode function + * @ndev: The EMAC network adapter + * + * Called when system wants to set the receive mode of the device. + * + */ +static void emac_ndo_set_rx_mode(struct net_device *ndev) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + queue_work(emac->cmd_wq, &emac->rx_mode_work); +} + +static int emac_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd) +{ + return phy_do_ioctl(ndev, ifr, cmd); +} + +static void emac_ndo_get_stats64(struct net_device *ndev, + struct rtnl_link_stats64 *stats) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + emac_update_hardware_stats(emac); + + stats->rx_packets = emac_get_stat_by_name(emac, "rx_packets"); + stats->rx_bytes = emac_get_stat_by_name(emac, "rx_bytes"); + stats->tx_packets = emac_get_stat_by_name(emac, "tx_packets"); + stats->tx_bytes = emac_get_stat_by_name(emac, "tx_bytes"); + stats->rx_crc_errors = emac_get_stat_by_name(emac, "rx_crc_errors"); + stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors"); + stats->multicast = emac_get_stat_by_name(emac, "rx_multicast_frames"); + + stats->rx_errors = ndev->stats.rx_errors; + stats->rx_dropped = ndev->stats.rx_dropped; + stats->tx_errors = ndev->stats.tx_errors; + stats->tx_dropped = ndev->stats.tx_dropped; +} + +static const struct net_device_ops emac_netdev_ops = { + .ndo_open = emac_ndo_open, + .ndo_stop = emac_ndo_stop, + .ndo_start_xmit = emac_ndo_start_xmit, + .ndo_set_mac_address = eth_mac_addr, + .ndo_validate_addr = eth_validate_addr, + .ndo_tx_timeout = emac_ndo_tx_timeout, + .ndo_set_rx_mode = emac_ndo_set_rx_mode, + .ndo_eth_ioctl = emac_ndo_ioctl, + .ndo_get_stats64 = emac_ndo_get_stats64, +}; + +/* get emac_port corresponding to eth_node name */ +static int prueth_node_port(struct device_node *eth_node) +{ + u32 port_id; + int ret; + + ret = of_property_read_u32(eth_node, "reg", &port_id); + if (ret) + return ret; + + if (port_id == 0) + return PRUETH_PORT_MII0; + else if (port_id == 1) + return PRUETH_PORT_MII1; + else + return PRUETH_PORT_INVALID; +} + +/* get MAC instance corresponding to eth_node name */ +static int prueth_node_mac(struct device_node *eth_node) +{ + u32 port_id; + int ret; + + ret = of_property_read_u32(eth_node, "reg", &port_id); + if (ret) + return ret; + + if (port_id == 0) + return PRUETH_MAC0; + else if (port_id == 1) + return PRUETH_MAC1; + else + return PRUETH_MAC_INVALID; +} + +static int prueth_netdev_init(struct prueth *prueth, + struct device_node *eth_node) +{ + int ret, num_tx_chn = PRUETH_MAX_TX_QUEUES; + struct prueth_emac *emac; + struct net_device *ndev; + enum prueth_port port; + enum prueth_mac mac; + + port = prueth_node_port(eth_node); + if (port == PRUETH_PORT_INVALID) + return -EINVAL; + + mac = prueth_node_mac(eth_node); + if (mac == PRUETH_MAC_INVALID) + return -EINVAL; + + ndev = alloc_etherdev_mq(sizeof(*emac), num_tx_chn); + if (!ndev) + return -ENOMEM; + + emac = netdev_priv(ndev); + emac->prueth = prueth; + emac->ndev = ndev; + emac->port_id = port; + emac->cmd_wq = create_singlethread_workqueue("icssg_cmd_wq"); + if (!emac->cmd_wq) { + ret = -ENOMEM; + goto free_ndev; + } + INIT_WORK(&emac->rx_mode_work, emac_ndo_set_rx_mode_work); + + INIT_DELAYED_WORK(&emac->stats_work, emac_stats_work_handler); + + ret = pruss_request_mem_region(prueth->pruss, + port == PRUETH_PORT_MII0 ? + PRUSS_MEM_DRAM0 : PRUSS_MEM_DRAM1, + &emac->dram); + if (ret) { + dev_err(prueth->dev, "unable to get DRAM: %d\n", ret); + ret = -ENOMEM; + goto free_wq; + } + + emac->tx_ch_num = 1; + + SET_NETDEV_DEV(ndev, prueth->dev); + spin_lock_init(&emac->lock); + mutex_init(&emac->cmd_lock); + + emac->phy_node = of_parse_phandle(eth_node, "phy-handle", 0); + if (!emac->phy_node && !of_phy_is_fixed_link(eth_node)) { + dev_err(prueth->dev, "couldn't find phy-handle\n"); + ret = -ENODEV; + goto free; + } else if (of_phy_is_fixed_link(eth_node)) { + ret = of_phy_register_fixed_link(eth_node); + if (ret) { + ret = dev_err_probe(prueth->dev, ret, + "failed to register fixed-link phy\n"); + goto free; + } + + emac->phy_node = eth_node; + } + + ret = of_get_phy_mode(eth_node, &emac->phy_if); + if (ret) { + dev_err(prueth->dev, "could not get phy-mode property\n"); + goto free; + } + + if (emac->phy_if != PHY_INTERFACE_MODE_MII && + !phy_interface_mode_is_rgmii(emac->phy_if)) { + dev_err(prueth->dev, "PHY mode unsupported %s\n", phy_modes(emac->phy_if)); + ret = -EINVAL; + goto free; + } + + /* AM65 SR2.0 has TX Internal delay always enabled by hardware + * and it is not possible to disable TX Internal delay. The below + * switch case block describes how we handle different phy modes + * based on hardware restriction. + */ + switch (emac->phy_if) { + case PHY_INTERFACE_MODE_RGMII_ID: + emac->phy_if = PHY_INTERFACE_MODE_RGMII_RXID; + break; + case PHY_INTERFACE_MODE_RGMII_TXID: + emac->phy_if = PHY_INTERFACE_MODE_RGMII; + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_RXID: + dev_err(prueth->dev, "RGMII mode without TX delay is not supported"); + ret = -EINVAL; + goto free; + default: + break; + } + + /* get mac address from DT and set private and netdev addr */ + ret = of_get_ethdev_address(eth_node, ndev); + if (!is_valid_ether_addr(ndev->dev_addr)) { + eth_hw_addr_random(ndev); + dev_warn(prueth->dev, "port %d: using random MAC addr: %pM\n", + port, ndev->dev_addr); + } + ether_addr_copy(emac->mac_addr, ndev->dev_addr); + + ndev->min_mtu = PRUETH_MIN_PKT_SIZE; + ndev->max_mtu = PRUETH_MAX_MTU; + ndev->netdev_ops = &emac_netdev_ops; + ndev->ethtool_ops = &icssg_ethtool_ops; + ndev->hw_features = NETIF_F_SG; + ndev->features = ndev->hw_features; + + netif_napi_add(ndev, &emac->napi_rx, emac_napi_rx_poll); + prueth->emac[mac] = emac; + + return 0; + +free: + pruss_release_mem_region(prueth->pruss, &emac->dram); +free_wq: + destroy_workqueue(emac->cmd_wq); +free_ndev: + emac->ndev = NULL; + prueth->emac[mac] = NULL; + free_netdev(ndev); + + return ret; +} + +static void prueth_netdev_exit(struct prueth *prueth, + struct device_node *eth_node) +{ + struct prueth_emac *emac; + enum prueth_mac mac; + + mac = prueth_node_mac(eth_node); + if (mac == PRUETH_MAC_INVALID) + return; + + emac = prueth->emac[mac]; + if (!emac) + return; + + if (of_phy_is_fixed_link(emac->phy_node)) + of_phy_deregister_fixed_link(emac->phy_node); + + netif_napi_del(&emac->napi_rx); + + pruss_release_mem_region(prueth->pruss, &emac->dram); + destroy_workqueue(emac->cmd_wq); + free_netdev(emac->ndev); + prueth->emac[mac] = NULL; +} + +static int prueth_get_cores(struct prueth *prueth, int slice) +{ + struct device *dev = prueth->dev; + enum pruss_pru_id pruss_id; + struct device_node *np; + int idx = -1, ret; + + np = dev->of_node; + + switch (slice) { + case ICSS_SLICE0: + idx = 0; + break; + case ICSS_SLICE1: + idx = 3; + break; + default: + return -EINVAL; + } + + prueth->pru[slice] = pru_rproc_get(np, idx, &pruss_id); + if (IS_ERR(prueth->pru[slice])) { + ret = PTR_ERR(prueth->pru[slice]); + prueth->pru[slice] = NULL; + return dev_err_probe(dev, ret, "unable to get PRU%d\n", slice); + } + prueth->pru_id[slice] = pruss_id; + + idx++; + prueth->rtu[slice] = pru_rproc_get(np, idx, NULL); + if (IS_ERR(prueth->rtu[slice])) { + ret = PTR_ERR(prueth->rtu[slice]); + prueth->rtu[slice] = NULL; + return dev_err_probe(dev, ret, "unable to get RTU%d\n", slice); + } + + idx++; + prueth->txpru[slice] = pru_rproc_get(np, idx, NULL); + if (IS_ERR(prueth->txpru[slice])) { + ret = PTR_ERR(prueth->txpru[slice]); + prueth->txpru[slice] = NULL; + return dev_err_probe(dev, ret, "unable to get TX_PRU%d\n", slice); + } + + return 0; +} + +static void prueth_put_cores(struct prueth *prueth, int slice) +{ + if (prueth->txpru[slice]) + pru_rproc_put(prueth->txpru[slice]); + + if (prueth->rtu[slice]) + pru_rproc_put(prueth->rtu[slice]); + + if (prueth->pru[slice]) + pru_rproc_put(prueth->pru[slice]); +} + +static const struct of_device_id prueth_dt_match[]; + +static int prueth_probe(struct platform_device *pdev) +{ + struct device_node *eth_node, *eth_ports_node; + struct device_node *eth0_node = NULL; + struct device_node *eth1_node = NULL; + struct genpool_data_align gp_data = { + .align = SZ_64K, + }; + const struct of_device_id *match; + struct device *dev = &pdev->dev; + struct device_node *np; + struct prueth *prueth; + struct pruss *pruss; + u32 msmc_ram_size; + int i, ret; + + np = dev->of_node; + + match = of_match_device(prueth_dt_match, dev); + if (!match) + return -ENODEV; + + prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL); + if (!prueth) + return -ENOMEM; + + dev_set_drvdata(dev, prueth); + prueth->pdev = pdev; + prueth->pdata = *(const struct prueth_pdata *)match->data; + + prueth->dev = dev; + eth_ports_node = of_get_child_by_name(np, "ethernet-ports"); + if (!eth_ports_node) + return -ENOENT; + + for_each_child_of_node(eth_ports_node, eth_node) { + u32 reg; + + if (strcmp(eth_node->name, "port")) + continue; + ret = of_property_read_u32(eth_node, "reg", ®); + if (ret < 0) { + dev_err(dev, "%pOF error reading port_id %d\n", + eth_node, ret); + } + + of_node_get(eth_node); + + if (reg == 0) { + eth0_node = eth_node; + if (!of_device_is_available(eth0_node)) { + of_node_put(eth0_node); + eth0_node = NULL; + } + } else if (reg == 1) { + eth1_node = eth_node; + if (!of_device_is_available(eth1_node)) { + of_node_put(eth1_node); + eth1_node = NULL; + } + } else { + dev_err(dev, "port reg should be 0 or 1\n"); + } + } + + of_node_put(eth_ports_node); + + /* At least one node must be present and available else we fail */ + if (!eth0_node && !eth1_node) { + dev_err(dev, "neither port0 nor port1 node available\n"); + return -ENODEV; + } + + if (eth0_node == eth1_node) { + dev_err(dev, "port0 and port1 can't have same reg\n"); + of_node_put(eth0_node); + return -ENODEV; + } + + prueth->eth_node[PRUETH_MAC0] = eth0_node; + prueth->eth_node[PRUETH_MAC1] = eth1_node; + + prueth->miig_rt = syscon_regmap_lookup_by_phandle(np, "ti,mii-g-rt"); + if (IS_ERR(prueth->miig_rt)) { + dev_err(dev, "couldn't get ti,mii-g-rt syscon regmap\n"); + return -ENODEV; + } + + prueth->mii_rt = syscon_regmap_lookup_by_phandle(np, "ti,mii-rt"); + if (IS_ERR(prueth->mii_rt)) { + dev_err(dev, "couldn't get ti,mii-rt syscon regmap\n"); + return -ENODEV; + } + + if (eth0_node) { + ret = prueth_get_cores(prueth, ICSS_SLICE0); + if (ret) + goto put_cores; + } + + if (eth1_node) { + ret = prueth_get_cores(prueth, ICSS_SLICE1); + if (ret) + goto put_cores; + } + + pruss = pruss_get(eth0_node ? + prueth->pru[ICSS_SLICE0] : prueth->pru[ICSS_SLICE1]); + if (IS_ERR(pruss)) { + ret = PTR_ERR(pruss); + dev_err(dev, "unable to get pruss handle\n"); + goto put_cores; + } + + prueth->pruss = pruss; + + ret = pruss_request_mem_region(pruss, PRUSS_MEM_SHRD_RAM2, + &prueth->shram); + if (ret) { + dev_err(dev, "unable to get PRUSS SHRD RAM2: %d\n", ret); + pruss_put(prueth->pruss); + } + + prueth->sram_pool = of_gen_pool_get(np, "sram", 0); + if (!prueth->sram_pool) { + dev_err(dev, "unable to get SRAM pool\n"); + ret = -ENODEV; + + goto put_mem; + } + + msmc_ram_size = MSMC_RAM_SIZE; + + /* NOTE: FW bug needs buffer base to be 64KB aligned */ + prueth->msmcram.va = + (void __iomem *)gen_pool_alloc_algo(prueth->sram_pool, + msmc_ram_size, + gen_pool_first_fit_align, + &gp_data); + + if (!prueth->msmcram.va) { + ret = -ENOMEM; + dev_err(dev, "unable to allocate MSMC resource\n"); + goto put_mem; + } + prueth->msmcram.pa = gen_pool_virt_to_phys(prueth->sram_pool, + (unsigned long)prueth->msmcram.va); + prueth->msmcram.size = msmc_ram_size; + memset_io(prueth->msmcram.va, 0, msmc_ram_size); + dev_dbg(dev, "sram: pa %llx va %p size %zx\n", prueth->msmcram.pa, + prueth->msmcram.va, prueth->msmcram.size); + + /* setup netdev interfaces */ + if (eth0_node) { + ret = prueth_netdev_init(prueth, eth0_node); + if (ret) { + dev_err_probe(dev, ret, "netdev init %s failed\n", + eth0_node->name); + goto netdev_exit; + } + } + + if (eth1_node) { + ret = prueth_netdev_init(prueth, eth1_node); + if (ret) { + dev_err_probe(dev, ret, "netdev init %s failed\n", + eth1_node->name); + goto netdev_exit; + } + } + + /* register the network devices */ + if (eth0_node) { + ret = register_netdev(prueth->emac[PRUETH_MAC0]->ndev); + if (ret) { + dev_err(dev, "can't register netdev for port MII0"); + goto netdev_exit; + } + + prueth->registered_netdevs[PRUETH_MAC0] = prueth->emac[PRUETH_MAC0]->ndev; + + emac_phy_connect(prueth->emac[PRUETH_MAC0]); + phy_attached_info(prueth->emac[PRUETH_MAC0]->ndev->phydev); + } + + if (eth1_node) { + ret = register_netdev(prueth->emac[PRUETH_MAC1]->ndev); + if (ret) { + dev_err(dev, "can't register netdev for port MII1"); + goto netdev_unregister; + } + + prueth->registered_netdevs[PRUETH_MAC1] = prueth->emac[PRUETH_MAC1]->ndev; + emac_phy_connect(prueth->emac[PRUETH_MAC1]); + phy_attached_info(prueth->emac[PRUETH_MAC1]->ndev->phydev); + } + + dev_info(dev, "TI PRU ethernet driver initialized: %s EMAC mode\n", + (!eth0_node || !eth1_node) ? "single" : "dual"); + + if (eth1_node) + of_node_put(eth1_node); + if (eth0_node) + of_node_put(eth0_node); + return 0; + +netdev_unregister: + for (i = 0; i < PRUETH_NUM_MACS; i++) { + if (!prueth->registered_netdevs[i]) + continue; + if (prueth->emac[i]->ndev->phydev) { + phy_disconnect(prueth->emac[i]->ndev->phydev); + prueth->emac[i]->ndev->phydev = NULL; + } + unregister_netdev(prueth->registered_netdevs[i]); + } + +netdev_exit: + for (i = 0; i < PRUETH_NUM_MACS; i++) { + eth_node = prueth->eth_node[i]; + if (!eth_node) + continue; + + prueth_netdev_exit(prueth, eth_node); + } + + gen_pool_free(prueth->sram_pool, + (unsigned long)prueth->msmcram.va, msmc_ram_size); + +put_mem: + pruss_release_mem_region(prueth->pruss, &prueth->shram); + pruss_put(prueth->pruss); + +put_cores: + if (eth1_node) { + prueth_put_cores(prueth, ICSS_SLICE1); + of_node_put(eth1_node); + } + + if (eth0_node) { + prueth_put_cores(prueth, ICSS_SLICE0); + of_node_put(eth0_node); + } + + return ret; +} + +static void prueth_remove(struct platform_device *pdev) +{ + struct prueth *prueth = platform_get_drvdata(pdev); + struct device_node *eth_node; + int i; + + for (i = 0; i < PRUETH_NUM_MACS; i++) { + if (!prueth->registered_netdevs[i]) + continue; + phy_stop(prueth->emac[i]->ndev->phydev); + phy_disconnect(prueth->emac[i]->ndev->phydev); + prueth->emac[i]->ndev->phydev = NULL; + unregister_netdev(prueth->registered_netdevs[i]); + } + + for (i = 0; i < PRUETH_NUM_MACS; i++) { + eth_node = prueth->eth_node[i]; + if (!eth_node) + continue; + + prueth_netdev_exit(prueth, eth_node); + } + + gen_pool_free(prueth->sram_pool, + (unsigned long)prueth->msmcram.va, + MSMC_RAM_SIZE); + + pruss_release_mem_region(prueth->pruss, &prueth->shram); + + pruss_put(prueth->pruss); + + if (prueth->eth_node[PRUETH_MAC1]) + prueth_put_cores(prueth, ICSS_SLICE1); + + if (prueth->eth_node[PRUETH_MAC0]) + prueth_put_cores(prueth, ICSS_SLICE0); +} + +#ifdef CONFIG_PM_SLEEP +static int prueth_suspend(struct device *dev) +{ + struct prueth *prueth = dev_get_drvdata(dev); + struct net_device *ndev; + int i, ret; + + for (i = 0; i < PRUETH_NUM_MACS; i++) { + ndev = prueth->registered_netdevs[i]; + + if (!ndev) + continue; + + if (netif_running(ndev)) { + netif_device_detach(ndev); + ret = emac_ndo_stop(ndev); + if (ret < 0) { + netdev_err(ndev, "failed to stop: %d", ret); + return ret; + } + } + } + + return 0; +} + +static int prueth_resume(struct device *dev) +{ + struct prueth *prueth = dev_get_drvdata(dev); + struct net_device *ndev; + int i, ret; + + for (i = 0; i < PRUETH_NUM_MACS; i++) { + ndev = prueth->registered_netdevs[i]; + + if (!ndev) + continue; + + if (netif_running(ndev)) { + ret = emac_ndo_open(ndev); + if (ret < 0) { + netdev_err(ndev, "failed to start: %d", ret); + return ret; + } + netif_device_attach(ndev); + } + } + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static const struct dev_pm_ops prueth_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume) +}; + +static const struct prueth_pdata am654_icssg_pdata = { + .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE, + .quirk_10m_link_issue = 1, +}; + +static const struct of_device_id prueth_dt_match[] = { + { .compatible = "ti,am654-icssg-prueth", .data = &am654_icssg_pdata }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, prueth_dt_match); + +static struct platform_driver prueth_driver = { + .probe = prueth_probe, + .remove_new = prueth_remove, + .driver = { + .name = "icssg-prueth", + .of_match_table = prueth_dt_match, + .pm = &prueth_dev_pm_ops, + }, +}; +module_platform_driver(prueth_driver); + +MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>"); +MODULE_AUTHOR("Md Danish Anwar <danishanwar@ti.com>"); +MODULE_DESCRIPTION("PRUSS ICSSG Ethernet Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.h b/drivers/net/ethernet/ti/icssg/icssg_prueth.h new file mode 100644 index 000000000000..a8ce4d01ef16 --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.h @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#ifndef __NET_TI_ICSSG_PRUETH_H +#define __NET_TI_ICSSG_PRUETH_H + +#include <linux/etherdevice.h> +#include <linux/genalloc.h> +#include <linux/if_vlan.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/net_tstamp.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_mdio.h> +#include <linux/of_net.h> +#include <linux/of_platform.h> +#include <linux/phy.h> +#include <linux/remoteproc/pruss.h> +#include <linux/pruss_driver.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/remoteproc.h> + +#include <linux/dma-mapping.h> +#include <linux/dma/ti-cppi5.h> +#include <linux/dma/k3-udma-glue.h> + +#include <net/devlink.h> + +#include "icssg_config.h" +#include "icssg_switch_map.h" + +#define PRUETH_MAX_MTU (2000 - ETH_HLEN - ETH_FCS_LEN) +#define PRUETH_MIN_PKT_SIZE (VLAN_ETH_ZLEN) +#define PRUETH_MAX_PKT_SIZE (PRUETH_MAX_MTU + ETH_HLEN + ETH_FCS_LEN) + +#define ICSS_SLICE0 0 +#define ICSS_SLICE1 1 + +#define ICSS_FW_PRU 0 +#define ICSS_FW_RTU 1 + +#define ICSSG_MAX_RFLOWS 8 /* per slice */ + +/* Number of ICSSG related stats */ +#define ICSSG_NUM_STATS 60 +#define ICSSG_NUM_STANDARD_STATS 31 +#define ICSSG_NUM_ETHTOOL_STATS (ICSSG_NUM_STATS - ICSSG_NUM_STANDARD_STATS) + +/* Firmware status codes */ +#define ICSS_HS_FW_READY 0x55555555 +#define ICSS_HS_FW_DEAD 0xDEAD0000 /* lower 16 bits contain error code */ + +/* Firmware command codes */ +#define ICSS_HS_CMD_BUSY 0x40000000 +#define ICSS_HS_CMD_DONE 0x80000000 +#define ICSS_HS_CMD_CANCEL 0x10000000 + +/* Firmware commands */ +#define ICSS_CMD_SPAD 0x20 +#define ICSS_CMD_RXTX 0x10 +#define ICSS_CMD_ADD_FDB 0x1 +#define ICSS_CMD_DEL_FDB 0x2 +#define ICSS_CMD_SET_RUN 0x4 +#define ICSS_CMD_GET_FDB_SLOT 0x5 +#define ICSS_CMD_ENABLE_VLAN 0x5 +#define ICSS_CMD_DISABLE_VLAN 0x6 +#define ICSS_CMD_ADD_FILTER 0x7 +#define ICSS_CMD_ADD_MAC 0x8 + +/* In switch mode there are 3 real ports i.e. 3 mac addrs. + * however Linux sees only the host side port. The other 2 ports + * are the switch ports. + * In emac mode there are 2 real ports i.e. 2 mac addrs. + * Linux sees both the ports. + */ +enum prueth_port { + PRUETH_PORT_HOST = 0, /* host side port */ + PRUETH_PORT_MII0, /* physical port RG/SG MII 0 */ + PRUETH_PORT_MII1, /* physical port RG/SG MII 1 */ + PRUETH_PORT_INVALID, /* Invalid prueth port */ +}; + +enum prueth_mac { + PRUETH_MAC0 = 0, + PRUETH_MAC1, + PRUETH_NUM_MACS, + PRUETH_MAC_INVALID, +}; + +struct prueth_tx_chn { + struct device *dma_dev; + struct napi_struct napi_tx; + struct k3_cppi_desc_pool *desc_pool; + struct k3_udma_glue_tx_channel *tx_chn; + struct prueth_emac *emac; + u32 id; + u32 descs_num; + unsigned int irq; + char name[32]; +}; + +struct prueth_rx_chn { + struct device *dev; + struct device *dma_dev; + struct k3_cppi_desc_pool *desc_pool; + struct k3_udma_glue_rx_channel *rx_chn; + u32 descs_num; + unsigned int irq[ICSSG_MAX_RFLOWS]; /* separate irq per flow */ + char name[32]; +}; + +/* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3) + * and lower three are lower priority channels or threads. + */ +#define PRUETH_MAX_TX_QUEUES 4 + +/* data for each emac port */ +struct prueth_emac { + bool fw_running; + struct prueth *prueth; + struct net_device *ndev; + u8 mac_addr[6]; + struct napi_struct napi_rx; + u32 msg_enable; + + int link; + int speed; + int duplex; + + const char *phy_id; + struct device_node *phy_node; + phy_interface_t phy_if; + enum prueth_port port_id; + + /* DMA related */ + struct prueth_tx_chn tx_chns[PRUETH_MAX_TX_QUEUES]; + struct completion tdown_complete; + atomic_t tdown_cnt; + struct prueth_rx_chn rx_chns; + int rx_flow_id_base; + int tx_ch_num; + + spinlock_t lock; /* serialize access */ + + unsigned long state; + struct completion cmd_complete; + /* Mutex to serialize access to firmware command interface */ + struct mutex cmd_lock; + struct work_struct rx_mode_work; + struct workqueue_struct *cmd_wq; + + struct pruss_mem_region dram; + + struct delayed_work stats_work; + u64 stats[ICSSG_NUM_STATS]; +}; + +/** + * struct prueth_pdata - PRUeth platform data + * @fdqring_mode: Free desc queue mode + * @quirk_10m_link_issue: 10M link detect errata + */ +struct prueth_pdata { + enum k3_ring_mode fdqring_mode; + u32 quirk_10m_link_issue:1; +}; + +/** + * struct prueth - PRUeth structure + * @dev: device + * @pruss: pruss handle + * @pru: rproc instances of PRUs + * @rtu: rproc instances of RTUs + * @txpru: rproc instances of TX_PRUs + * @shram: PRUSS shared RAM region + * @sram_pool: MSMC RAM pool for buffers + * @msmcram: MSMC RAM region + * @eth_node: DT node for the port + * @emac: private EMAC data structure + * @registered_netdevs: list of registered netdevs + * @miig_rt: regmap to mii_g_rt block + * @mii_rt: regmap to mii_rt block + * @pru_id: ID for each of the PRUs + * @pdev: pointer to ICSSG platform device + * @pdata: pointer to platform data for ICSSG driver + * @icssg_hwcmdseq: seq counter or HWQ messages + * @emacs_initialized: num of EMACs/ext ports that are up/running + */ +struct prueth { + struct device *dev; + struct pruss *pruss; + struct rproc *pru[PRUSS_NUM_PRUS]; + struct rproc *rtu[PRUSS_NUM_PRUS]; + struct rproc *txpru[PRUSS_NUM_PRUS]; + struct pruss_mem_region shram; + struct gen_pool *sram_pool; + struct pruss_mem_region msmcram; + + struct device_node *eth_node[PRUETH_NUM_MACS]; + struct prueth_emac *emac[PRUETH_NUM_MACS]; + struct net_device *registered_netdevs[PRUETH_NUM_MACS]; + struct regmap *miig_rt; + struct regmap *mii_rt; + + enum pruss_pru_id pru_id[PRUSS_NUM_PRUS]; + struct platform_device *pdev; + struct prueth_pdata pdata; + u8 icssg_hwcmdseq; + + int emacs_initialized; +}; + +/* get PRUSS SLICE number from prueth_emac */ +static inline int prueth_emac_slice(struct prueth_emac *emac) +{ + switch (emac->port_id) { + case PRUETH_PORT_MII0: + return ICSS_SLICE0; + case PRUETH_PORT_MII1: + return ICSS_SLICE1; + default: + return -EINVAL; + } +} + +extern const struct ethtool_ops icssg_ethtool_ops; + +/* Classifier helpers */ +void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac); +void icssg_class_set_host_mac_addr(struct regmap *miig_rt, const u8 *mac); +void icssg_class_disable(struct regmap *miig_rt, int slice); +void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti); +void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr); + +/* config helpers */ +void icssg_config_ipg(struct prueth_emac *emac); +int icssg_config(struct prueth *prueth, struct prueth_emac *emac, + int slice); +int emac_set_port_state(struct prueth_emac *emac, + enum icssg_port_state_cmd state); +void icssg_config_set_speed(struct prueth_emac *emac); + +/* Buffer queue helpers */ +int icssg_queue_pop(struct prueth *prueth, u8 queue); +void icssg_queue_push(struct prueth *prueth, int queue, u16 addr); +u32 icssg_queue_level(struct prueth *prueth, int queue); + +#define prueth_napi_to_tx_chn(pnapi) \ + container_of(pnapi, struct prueth_tx_chn, napi_tx) + +void emac_stats_work_handler(struct work_struct *work); +void emac_update_hardware_stats(struct prueth_emac *emac); +int emac_get_stat_by_name(struct prueth_emac *emac, char *stat_name); +#endif /* __NET_TI_ICSSG_PRUETH_H */ diff --git a/drivers/net/ethernet/ti/icssg/icssg_queues.c b/drivers/net/ethernet/ti/icssg/icssg_queues.c new file mode 100644 index 000000000000..3c34f61ad40b --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_queues.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ICSSG Buffer queue helpers + * + * Copyright (C) 2021 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/regmap.h> +#include "icssg_prueth.h" + +#define ICSSG_QUEUES_MAX 64 +#define ICSSG_QUEUE_OFFSET 0xd00 +#define ICSSG_QUEUE_PEEK_OFFSET 0xe00 +#define ICSSG_QUEUE_CNT_OFFSET 0xe40 +#define ICSSG_QUEUE_RESET_OFFSET 0xf40 + +int icssg_queue_pop(struct prueth *prueth, u8 queue) +{ + u32 val, cnt; + + if (queue >= ICSSG_QUEUES_MAX) + return -EINVAL; + + regmap_read(prueth->miig_rt, ICSSG_QUEUE_CNT_OFFSET + 4 * queue, &cnt); + if (!cnt) + return -EINVAL; + + regmap_read(prueth->miig_rt, ICSSG_QUEUE_OFFSET + 4 * queue, &val); + + return val; +} + +void icssg_queue_push(struct prueth *prueth, int queue, u16 addr) +{ + if (queue >= ICSSG_QUEUES_MAX) + return; + + regmap_write(prueth->miig_rt, ICSSG_QUEUE_OFFSET + 4 * queue, addr); +} + +u32 icssg_queue_level(struct prueth *prueth, int queue) +{ + u32 reg; + + if (queue >= ICSSG_QUEUES_MAX) + return 0; + + regmap_read(prueth->miig_rt, ICSSG_QUEUE_CNT_OFFSET + 4 * queue, ®); + + return reg; +} diff --git a/drivers/net/ethernet/ti/icssg/icssg_stats.c b/drivers/net/ethernet/ti/icssg/icssg_stats.c new file mode 100644 index 000000000000..bb0b33927e3b --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_stats.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2018-2021 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#include "icssg_prueth.h" +#include "icssg_stats.h" +#include <linux/regmap.h> + +static u32 stats_base[] = { 0x54c, /* Slice 0 stats start */ + 0xb18, /* Slice 1 stats start */ +}; + +void emac_update_hardware_stats(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + int slice = prueth_emac_slice(emac); + u32 base = stats_base[slice]; + u32 val; + int i; + + for (i = 0; i < ARRAY_SIZE(icssg_all_stats); i++) { + regmap_read(prueth->miig_rt, + base + icssg_all_stats[i].offset, + &val); + regmap_write(prueth->miig_rt, + base + icssg_all_stats[i].offset, + val); + + emac->stats[i] += val; + } +} + +void emac_stats_work_handler(struct work_struct *work) +{ + struct prueth_emac *emac = container_of(work, struct prueth_emac, + stats_work.work); + emac_update_hardware_stats(emac); + + queue_delayed_work(system_long_wq, &emac->stats_work, + msecs_to_jiffies((STATS_TIME_LIMIT_1G_MS * 1000) / emac->speed)); +} + +int emac_get_stat_by_name(struct prueth_emac *emac, char *stat_name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(icssg_all_stats); i++) { + if (!strcmp(icssg_all_stats[i].name, stat_name)) + return emac->stats[icssg_all_stats[i].offset / sizeof(u32)]; + } + + netdev_err(emac->ndev, "Invalid stats %s\n", stat_name); + return -EINVAL; +} diff --git a/drivers/net/ethernet/ti/icssg/icssg_stats.h b/drivers/net/ethernet/ti/icssg/icssg_stats.h new file mode 100644 index 000000000000..999a4a91276c --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_stats.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#ifndef __NET_TI_ICSSG_STATS_H +#define __NET_TI_ICSSG_STATS_H + +#include "icssg_prueth.h" + +#define STATS_TIME_LIMIT_1G_MS 25000 /* 25 seconds @ 1G */ + +struct miig_stats_regs { + /* Rx */ + u32 rx_packets; + u32 rx_broadcast_frames; + u32 rx_multicast_frames; + u32 rx_crc_errors; + u32 rx_mii_error_frames; + u32 rx_odd_nibble_frames; + u32 rx_frame_max_size; + u32 rx_max_size_error_frames; + u32 rx_frame_min_size; + u32 rx_min_size_error_frames; + u32 rx_over_errors; + u32 rx_class0_hits; + u32 rx_class1_hits; + u32 rx_class2_hits; + u32 rx_class3_hits; + u32 rx_class4_hits; + u32 rx_class5_hits; + u32 rx_class6_hits; + u32 rx_class7_hits; + u32 rx_class8_hits; + u32 rx_class9_hits; + u32 rx_class10_hits; + u32 rx_class11_hits; + u32 rx_class12_hits; + u32 rx_class13_hits; + u32 rx_class14_hits; + u32 rx_class15_hits; + u32 rx_smd_frags; + u32 rx_bucket1_size; + u32 rx_bucket2_size; + u32 rx_bucket3_size; + u32 rx_bucket4_size; + u32 rx_64B_frames; + u32 rx_bucket1_frames; + u32 rx_bucket2_frames; + u32 rx_bucket3_frames; + u32 rx_bucket4_frames; + u32 rx_bucket5_frames; + u32 rx_bytes; + u32 rx_tx_total_bytes; + /* Tx */ + u32 tx_packets; + u32 tx_broadcast_frames; + u32 tx_multicast_frames; + u32 tx_odd_nibble_frames; + u32 tx_underflow_errors; + u32 tx_frame_max_size; + u32 tx_max_size_error_frames; + u32 tx_frame_min_size; + u32 tx_min_size_error_frames; + u32 tx_bucket1_size; + u32 tx_bucket2_size; + u32 tx_bucket3_size; + u32 tx_bucket4_size; + u32 tx_64B_frames; + u32 tx_bucket1_frames; + u32 tx_bucket2_frames; + u32 tx_bucket3_frames; + u32 tx_bucket4_frames; + u32 tx_bucket5_frames; + u32 tx_bytes; +}; + +#define ICSSG_STATS(field, stats_type) \ +{ \ + #field, \ + offsetof(struct miig_stats_regs, field), \ + stats_type \ +} + +struct icssg_stats { + char name[ETH_GSTRING_LEN]; + u32 offset; + bool standard_stats; +}; + +static const struct icssg_stats icssg_all_stats[] = { + /* Rx */ + ICSSG_STATS(rx_packets, true), + ICSSG_STATS(rx_broadcast_frames, false), + ICSSG_STATS(rx_multicast_frames, true), + ICSSG_STATS(rx_crc_errors, true), + ICSSG_STATS(rx_mii_error_frames, false), + ICSSG_STATS(rx_odd_nibble_frames, false), + ICSSG_STATS(rx_frame_max_size, true), + ICSSG_STATS(rx_max_size_error_frames, false), + ICSSG_STATS(rx_frame_min_size, true), + ICSSG_STATS(rx_min_size_error_frames, false), + ICSSG_STATS(rx_over_errors, true), + ICSSG_STATS(rx_class0_hits, false), + ICSSG_STATS(rx_class1_hits, false), + ICSSG_STATS(rx_class2_hits, false), + ICSSG_STATS(rx_class3_hits, false), + ICSSG_STATS(rx_class4_hits, false), + ICSSG_STATS(rx_class5_hits, false), + ICSSG_STATS(rx_class6_hits, false), + ICSSG_STATS(rx_class7_hits, false), + ICSSG_STATS(rx_class8_hits, false), + ICSSG_STATS(rx_class9_hits, false), + ICSSG_STATS(rx_class10_hits, false), + ICSSG_STATS(rx_class11_hits, false), + ICSSG_STATS(rx_class12_hits, false), + ICSSG_STATS(rx_class13_hits, false), + ICSSG_STATS(rx_class14_hits, false), + ICSSG_STATS(rx_class15_hits, false), + ICSSG_STATS(rx_smd_frags, false), + ICSSG_STATS(rx_bucket1_size, true), + ICSSG_STATS(rx_bucket2_size, true), + ICSSG_STATS(rx_bucket3_size, true), + ICSSG_STATS(rx_bucket4_size, true), + ICSSG_STATS(rx_64B_frames, true), + ICSSG_STATS(rx_bucket1_frames, true), + ICSSG_STATS(rx_bucket2_frames, true), + ICSSG_STATS(rx_bucket3_frames, true), + ICSSG_STATS(rx_bucket4_frames, true), + ICSSG_STATS(rx_bucket5_frames, true), + ICSSG_STATS(rx_bytes, true), + ICSSG_STATS(rx_tx_total_bytes, false), + /* Tx */ + ICSSG_STATS(tx_packets, true), + ICSSG_STATS(tx_broadcast_frames, false), + ICSSG_STATS(tx_multicast_frames, false), + ICSSG_STATS(tx_odd_nibble_frames, false), + ICSSG_STATS(tx_underflow_errors, false), + ICSSG_STATS(tx_frame_max_size, true), + ICSSG_STATS(tx_max_size_error_frames, false), + ICSSG_STATS(tx_frame_min_size, true), + ICSSG_STATS(tx_min_size_error_frames, false), + ICSSG_STATS(tx_bucket1_size, true), + ICSSG_STATS(tx_bucket2_size, true), + ICSSG_STATS(tx_bucket3_size, true), + ICSSG_STATS(tx_bucket4_size, true), + ICSSG_STATS(tx_64B_frames, true), + ICSSG_STATS(tx_bucket1_frames, true), + ICSSG_STATS(tx_bucket2_frames, true), + ICSSG_STATS(tx_bucket3_frames, true), + ICSSG_STATS(tx_bucket4_frames, true), + ICSSG_STATS(tx_bucket5_frames, true), + ICSSG_STATS(tx_bytes, true), +}; + +#endif /* __NET_TI_ICSSG_STATS_H */ diff --git a/drivers/net/ethernet/ti/icssg/icssg_switch_map.h b/drivers/net/ethernet/ti/icssg/icssg_switch_map.h new file mode 100644 index 000000000000..424a7e945ea8 --- /dev/null +++ b/drivers/net/ethernet/ti/icssg/icssg_switch_map.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Texas Instruments ICSSG Ethernet driver + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + * + */ + +#ifndef __NET_TI_ICSSG_SWITCH_MAP_H +#define __NET_TI_ICSSG_SWITCH_MAP_H + +/************************* Ethernet Switch Constants *********************/ + +/* if bucket size is changed in firmware then this too should be changed + * because it directly impacts FDB ageing calculation + */ +#define NUMBER_OF_FDB_BUCKET_ENTRIES (4) + +/* This is fixed in ICSSG */ +#define SIZE_OF_FDB (2048) + +#define FW_LINK_SPEED_1G (0x00) +#define FW_LINK_SPEED_100M (0x01) +#define FW_LINK_SPEED_10M (0x02) +#define FW_LINK_SPEED_HD (0x80) + +/* Time after which FDB entries are checked for aged out values. + * Values are in nanoseconds + */ +#define FDB_AGEING_TIMEOUT_OFFSET 0x0014 + +/* Default VLAN tag for Host Port */ +#define HOST_PORT_DF_VLAN_OFFSET 0x001C + +/* Same as HOST_PORT_DF_VLAN_OFFSET */ +#define EMAC_ICSSG_SWITCH_PORT0_DEFAULT_VLAN_OFFSET HOST_PORT_DF_VLAN_OFFSET + +/* Default VLAN tag for P1 Port */ +#define P1_PORT_DF_VLAN_OFFSET 0x0020 + +/* Same as P1_PORT_DF_VLAN_OFFSET */ +#define EMAC_ICSSG_SWITCH_PORT1_DEFAULT_VLAN_OFFSET P1_PORT_DF_VLAN_OFFSET + +/* default VLAN tag for P2 Port */ +#define P2_PORT_DF_VLAN_OFFSET 0x0024 + +/* Same as P2_PORT_DF_VLAN_OFFSET */ +#define EMAC_ICSSG_SWITCH_PORT2_DEFAULT_VLAN_OFFSET P2_PORT_DF_VLAN_OFFSET + +/* VLAN-FID Table offset. 4096 VIDs. 2B per VID = 8KB = 0x2000 */ +#define VLAN_STATIC_REG_TABLE_OFFSET 0x0100 + +/* VLAN-FID Table offset for EMAC */ +#define EMAC_ICSSG_SWITCH_DEFAULT_VLAN_TABLE_OFFSET VLAN_STATIC_REG_TABLE_OFFSET + +/* Packet descriptor Q reserved memory */ +#define PORT_DESC0_HI 0x2104 + +/* Packet descriptor Q reserved memory */ +#define PORT_DESC0_LO 0x2F6C + +/* Packet descriptor Q reserved memory */ +#define PORT_DESC1_HI 0x3DD4 + +/* Packet descriptor Q reserved memory */ +#define PORT_DESC1_LO 0x4C3C + +/* Packet descriptor Q reserved memory */ +#define HOST_DESC0_HI 0x5AA4 + +/* Packet descriptor Q reserved memory */ +#define HOST_DESC0_LO 0x5F0C + +/* Packet descriptor Q reserved memory */ +#define HOST_DESC1_HI 0x6374 + +/* Packet descriptor Q reserved memory */ +#define HOST_DESC1_LO 0x67DC + +/* Special packet descriptor Q reserved memory */ +#define HOST_SPPD0 0x7AAC + +/* Special acket descriptor Q reserved memory */ +#define HOST_SPPD1 0x7EAC + +/* IEP count cycle counter*/ +#define TIMESYNC_FW_WC_CYCLECOUNT_OFFSET 0x83EC + +/* IEP count hi roll over count */ +#define TIMESYNC_FW_WC_HI_ROLLOVER_COUNT_OFFSET 0x83F4 + +/* IEP count hi sw counter */ +#define TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET 0x83F8 + +/* Set clock descriptor */ +#define TIMESYNC_FW_WC_SETCLOCK_DESC_OFFSET 0x83FC + +/* IEP count syncout reduction factor */ +#define TIMESYNC_FW_WC_SYNCOUT_REDUCTION_FACTOR_OFFSET 0x843C + +/* IEP count syncout reduction counter */ +#define TIMESYNC_FW_WC_SYNCOUT_REDUCTION_COUNT_OFFSET 0x8440 + +/* IEP count syncout start time cycle counter */ +#define TIMESYNC_FW_WC_SYNCOUT_START_TIME_CYCLECOUNT_OFFSET 0x8444 + +/* Control variable to generate SYNC1 */ +#define TIMESYNC_FW_WC_ISOM_PIN_SIGNAL_EN_OFFSET 0x844C + +/* SystemTime Sync0 periodicity */ +#define TIMESYNC_FW_ST_SYNCOUT_PERIOD_OFFSET 0x8450 + +/* pktTxDelay for P1 = link speed dependent p1 mac delay + p1 phy delay */ +#define TIMESYNC_FW_WC_PKTTXDELAY_P1_OFFSET 0x8454 + +/* pktTxDelay for P2 = link speed dependent p2 mac delay + p2 phy delay */ +#define TIMESYNC_FW_WC_PKTTXDELAY_P2_OFFSET 0x8458 + +/* Set clock operation done signal for next task */ +#define TIMESYNC_FW_SIG_PNFW_OFFSET 0x845C + +/* Set clock operation done signal for next task */ +#define TIMESYNC_FW_SIG_TIMESYNCFW_OFFSET 0x8460 + +/* New list is copied at this time */ +#define TAS_CONFIG_CHANGE_TIME 0x000C + +/* config change error counter */ +#define TAS_CONFIG_CHANGE_ERROR_COUNTER 0x0014 + +/* TAS List update pending flag */ +#define TAS_CONFIG_PENDING 0x0018 + +/* TAS list update trigger flag */ +#define TAS_CONFIG_CHANGE 0x0019 + +/* List length for new TAS schedule */ +#define TAS_ADMIN_LIST_LENGTH 0x001A + +/* Currently active TAS list index */ +#define TAS_ACTIVE_LIST_INDEX 0x001B + +/* Cycle time for the new TAS schedule */ +#define TAS_ADMIN_CYCLE_TIME 0x001C + +/* Cycle counts remaining till the TAS list update */ +#define TAS_CONFIG_CHANGE_CYCLE_COUNT 0x0020 + +/* Base Flow ID for sending Packets to Host for Slice0 */ +#define PSI_L_REGULAR_FLOW_ID_BASE_OFFSET 0x0024 + +/* Same as PSI_L_REGULAR_FLOW_ID_BASE_OFFSET */ +#define EMAC_ICSSG_SWITCH_PSI_L_REGULAR_FLOW_ID_BASE_OFFSET PSI_L_REGULAR_FLOW_ID_BASE_OFFSET + +/* Base Flow ID for sending mgmt and Tx TS to Host for Slice0 */ +#define PSI_L_MGMT_FLOW_ID_OFFSET 0x0026 + +/* Same as PSI_L_MGMT_FLOW_ID_OFFSET */ +#define EMAC_ICSSG_SWITCH_PSI_L_MGMT_FLOW_ID_BASE_OFFSET PSI_L_MGMT_FLOW_ID_OFFSET + +/* Queue number for Special Packets written here */ +#define SPL_PKT_DEFAULT_PRIORITY 0x0028 + +/* Express Preemptible Queue Mask */ +#define EXPRESS_PRE_EMPTIVE_Q_MASK 0x0029 + +/* Port1/Port2 Default Queue number for untagged Packets, only 1B is used */ +#define QUEUE_NUM_UNTAGGED 0x002A + +/* Stores the table used for priority regeneration. 1B per PCP/Queue */ +#define PORT_Q_PRIORITY_REGEN_OFFSET 0x002C + +/* For marking Packet as priority/express (this feature is disabled) or + * cut-through/S&F. + */ +#define EXPRESS_PRE_EMPTIVE_Q_MAP 0x0034 + +/* Stores the table used for priority mapping. 1B per PCP/Queue */ +#define PORT_Q_PRIORITY_MAPPING_OFFSET 0x003C + +/* Used to notify the FW of the current link speed */ +#define PORT_LINK_SPEED_OFFSET 0x00A8 + +/* TAS gate mask for windows list0 */ +#define TAS_GATE_MASK_LIST0 0x0100 + +/* TAS gate mask for windows list1 */ +#define TAS_GATE_MASK_LIST1 0x0350 + +/* Memory to Enable/Disable Preemption on TX side */ +#define PRE_EMPTION_ENABLE_TX 0x05A0 + +/* Active State of Preemption on TX side */ +#define PRE_EMPTION_ACTIVE_TX 0x05A1 + +/* Memory to Enable/Disable Verify State Machine Preemption */ +#define PRE_EMPTION_ENABLE_VERIFY 0x05A2 + +/* Verify Status of State Machine */ +#define PRE_EMPTION_VERIFY_STATUS 0x05A3 + +/* Non Final Fragment Size supported by Link Partner */ +#define PRE_EMPTION_ADD_FRAG_SIZE_REMOTE 0x05A4 + +/* Non Final Fragment Size supported by Firmware */ +#define PRE_EMPTION_ADD_FRAG_SIZE_LOCAL 0x05A6 + +/* Time in ms the State machine waits for respond Packet */ +#define PRE_EMPTION_VERIFY_TIME 0x05A8 + +/* Memory used for R30 related management commands */ +#define MGR_R30_CMD_OFFSET 0x05AC + +/* HW Buffer Pool0 base address */ +#define BUFFER_POOL_0_ADDR_OFFSET 0x05BC + +/* 16B for Host Egress MSMC Q (Pre-emptible) context */ +#define HOST_RX_Q_PRE_CONTEXT_OFFSET 0x0684 + +/* Buffer for 8 FDB entries to be added by 'Add Multiple FDB entries IOCTL' */ +#define FDB_CMD_BUFFER 0x0894 + +/* TAS queue max sdu length list */ +#define TAS_QUEUE_MAX_SDU_LIST 0x08FA + +/* Used by FW to generate random number with the SEED value */ +#define HD_RAND_SEED_OFFSET 0x0934 + +/* 16B for Host Egress MSMC Q (Express) context */ +#define HOST_RX_Q_EXP_CONTEXT_OFFSET 0x0940 + +/* Start of 32 bits PA_STAT counters */ +#define PA_STAT_32b_START_OFFSET 0x0080 + +#endif /* __NET_TI_ICSSG_SWITCH_MAP_H */ diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c index d716e6fe26e1..3e09e5036490 100644 --- a/drivers/net/ethernet/via/via-rhine.c +++ b/drivers/net/ethernet/via/via-rhine.c @@ -94,7 +94,7 @@ static const int multicast_filter_limit = 32; #include <linux/ioport.h> #include <linux/interrupt.h> #include <linux/pci.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_irq.h> #include <linux/platform_device.h> #include <linux/dma-mapping.h> diff --git a/drivers/net/ethernet/via/via-velocity.c b/drivers/net/ethernet/via/via-velocity.c index 86f7843b4591..731f689412e6 100644 --- a/drivers/net/ethernet/via/via-velocity.c +++ b/drivers/net/ethernet/via/via-velocity.c @@ -57,8 +57,8 @@ #include <linux/if.h> #include <linux/uaccess.h> #include <linux/proc_fs.h> +#include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/inetdevice.h> #include <linux/platform_device.h> diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index 6321178fc814..85dc16faca54 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -432,71 +432,6 @@ out: EXPORT_SYMBOL(wx_read_ee_hostif_buffer); /** - * wx_calculate_checksum - Calculate checksum for buffer - * @buffer: pointer to EEPROM - * @length: size of EEPROM to calculate a checksum for - * Calculates the checksum for some buffer on a specified length. The - * checksum calculated is returned. - **/ -static u8 wx_calculate_checksum(u8 *buffer, u32 length) -{ - u8 sum = 0; - u32 i; - - if (!buffer) - return 0; - - for (i = 0; i < length; i++) - sum += buffer[i]; - - return (u8)(0 - sum); -} - -/** - * wx_reset_hostif - send reset cmd to fw - * @wx: pointer to hardware structure - * - * Sends reset cmd to firmware through the manageability - * block. - **/ -int wx_reset_hostif(struct wx *wx) -{ - struct wx_hic_reset reset_cmd; - int ret_val = 0; - int i; - - reset_cmd.hdr.cmd = FW_RESET_CMD; - reset_cmd.hdr.buf_len = FW_RESET_LEN; - reset_cmd.hdr.cmd_or_resp.cmd_resv = FW_CEM_CMD_RESERVED; - reset_cmd.lan_id = wx->bus.func; - reset_cmd.reset_type = (u16)wx->reset_type; - reset_cmd.hdr.checksum = 0; - reset_cmd.hdr.checksum = wx_calculate_checksum((u8 *)&reset_cmd, - (FW_CEM_HDR_LEN + - reset_cmd.hdr.buf_len)); - - for (i = 0; i <= FW_CEM_MAX_RETRIES; i++) { - ret_val = wx_host_interface_command(wx, (u32 *)&reset_cmd, - sizeof(reset_cmd), - WX_HI_COMMAND_TIMEOUT, - true); - if (ret_val != 0) - continue; - - if (reset_cmd.hdr.cmd_or_resp.ret_status == - FW_CEM_RESP_STATUS_SUCCESS) - ret_val = 0; - else - ret_val = -EFAULT; - - break; - } - - return ret_val; -} -EXPORT_SYMBOL(wx_reset_hostif); - -/** * wx_init_eeprom_params - Initialize EEPROM params * @wx: pointer to hardware structure * @@ -1501,7 +1436,7 @@ static void wx_restore_vlan(struct wx *wx) * * Configure the Rx unit of the MAC after a reset. **/ -static void wx_configure_rx(struct wx *wx) +void wx_configure_rx(struct wx *wx) { u32 psrtype, i; int ret; @@ -1544,6 +1479,7 @@ static void wx_configure_rx(struct wx *wx) wx_enable_rx(wx); wx_enable_sec_rx_path(wx); } +EXPORT_SYMBOL(wx_configure_rx); static void wx_configure_isb(struct wx *wx) { diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.h b/drivers/net/ethernet/wangxun/libwx/wx_hw.h index 1f93ca32c921..0b3447bc6f2f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.h @@ -14,7 +14,6 @@ int wx_host_interface_command(struct wx *wx, u32 *buffer, int wx_read_ee_hostif(struct wx *wx, u16 offset, u16 *data); int wx_read_ee_hostif_buffer(struct wx *wx, u16 offset, u16 words, u16 *data); -int wx_reset_hostif(struct wx *wx); void wx_init_eeprom_params(struct wx *wx); void wx_get_mac_addr(struct wx *wx, u8 *mac_addr); void wx_init_rx_addrs(struct wx *wx); @@ -25,6 +24,7 @@ void wx_disable_rx(struct wx *wx); void wx_set_rx_mode(struct net_device *netdev); int wx_change_mtu(struct net_device *netdev, int new_mtu); void wx_disable_rx_queue(struct wx *wx, struct wx_ring *ring); +void wx_configure_rx(struct wx *wx); void wx_configure(struct wx *wx); void wx_start_hw(struct wx *wx); int wx_disable_pcie_master(struct wx *wx); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 29dfb561887d..1de88a33a698 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -160,6 +160,10 @@ #define WX_PSR_LAN_FLEX_DW_H(_i) (0x15C04 + ((_i) * 16)) #define WX_PSR_LAN_FLEX_MSK(_i) (0x15C08 + ((_i) * 16)) +#define WX_PSR_WKUP_CTL 0x15B80 +/* Wake Up Filter Control Bit */ +#define WX_PSR_WKUP_CTL_MAG BIT(1) /* Magic Packet Wakeup Enable */ + /* vlan tbl */ #define WX_PSR_VLAN_TBL(_i) (0x16000 + ((_i) * 4)) @@ -846,7 +850,7 @@ struct wx { int duplex; struct phy_device *phydev; - bool wol_enabled; + bool wol_hw_supported; bool ncsi_enabled; bool gpio_ctrl; raw_spinlock_t gpio_lock; diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c index 5b25834baf38..ec0e869e9aac 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c @@ -6,14 +6,49 @@ #include <linux/netdevice.h> #include "../libwx/wx_ethtool.h" +#include "../libwx/wx_type.h" #include "ngbe_ethtool.h" +static void ngbe_get_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct wx *wx = netdev_priv(netdev); + + if (!wx->wol_hw_supported) + return; + wol->supported = WAKE_MAGIC; + wol->wolopts = 0; + if (wx->wol & WX_PSR_WKUP_CTL_MAG) + wol->wolopts |= WAKE_MAGIC; +} + +static int ngbe_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct wx *wx = netdev_priv(netdev); + struct pci_dev *pdev = wx->pdev; + + if (!wx->wol_hw_supported) + return -EOPNOTSUPP; + + wx->wol = 0; + if (wol->wolopts & WAKE_MAGIC) + wx->wol = WX_PSR_WKUP_CTL_MAG; + netdev->wol_enabled = !!(wx->wol); + wr32(wx, WX_PSR_WKUP_CTL, wx->wol); + device_set_wakeup_enable(&pdev->dev, netdev->wol_enabled); + + return 0; +} + static const struct ethtool_ops ngbe_ethtool_ops = { .get_drvinfo = wx_get_drvinfo, .get_link = ethtool_op_get_link, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, .nway_reset = phy_ethtool_nway_reset, + .get_wol = ngbe_get_wol, + .set_wol = ngbe_set_wol, }; void ngbe_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c index c99a5d3de72e..2b431db6085a 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -62,7 +62,7 @@ static void ngbe_init_type_code(struct wx *wx) em_mac_type_rgmii : em_mac_type_mdi; - wx->wol_enabled = (wol_mask == NGBE_WOL_SUP) ? 1 : 0; + wx->wol_hw_supported = (wol_mask == NGBE_WOL_SUP) ? 1 : 0; wx->ncsi_enabled = (ncsi_mask == NGBE_NCSI_MASK || type_mask == NGBE_SUBID_OCP_CARD) ? 1 : 0; @@ -440,14 +440,26 @@ static void ngbe_dev_shutdown(struct pci_dev *pdev, bool *enable_wake) { struct wx *wx = pci_get_drvdata(pdev); struct net_device *netdev; + u32 wufc = wx->wol; netdev = wx->netdev; + rtnl_lock(); netif_device_detach(netdev); - rtnl_lock(); if (netif_running(netdev)) - ngbe_down(wx); + ngbe_close(netdev); + wx_clear_interrupt_scheme(wx); rtnl_unlock(); + + if (wufc) { + wx_set_rx_mode(netdev); + wx_configure_rx(wx); + wr32(wx, NGBE_PSR_WKUP_CTL, wufc); + } else { + wr32(wx, NGBE_PSR_WKUP_CTL, 0); + } + pci_wake_from_d3(pdev, !!wufc); + *enable_wake = !!wufc; wx_control_hw(wx, false); pci_disable_device(pdev); @@ -621,12 +633,11 @@ static int ngbe_probe(struct pci_dev *pdev, } wx->wol = 0; - if (wx->wol_enabled) + if (wx->wol_hw_supported) wx->wol = NGBE_PSR_WKUP_CTL_MAG; - wx->wol_enabled = !!(wx->wol); + netdev->wol_enabled = !!(wx->wol); wr32(wx, NGBE_PSR_WKUP_CTL, wx->wol); - device_set_wakeup_enable(&pdev->dev, wx->wol); /* Save off EEPROM version number and Option Rom version which @@ -712,11 +723,52 @@ static void ngbe_remove(struct pci_dev *pdev) pci_disable_device(pdev); } +static int ngbe_suspend(struct pci_dev *pdev, pm_message_t state) +{ + bool wake; + + ngbe_dev_shutdown(pdev, &wake); + device_set_wakeup_enable(&pdev->dev, wake); + + return 0; +} + +static int ngbe_resume(struct pci_dev *pdev) +{ + struct net_device *netdev; + struct wx *wx; + u32 err; + + wx = pci_get_drvdata(pdev); + netdev = wx->netdev; + + err = pci_enable_device_mem(pdev); + if (err) { + wx_err(wx, "Cannot enable PCI device from suspend\n"); + return err; + } + pci_set_master(pdev); + device_wakeup_disable(&pdev->dev); + + ngbe_reset_hw(wx); + rtnl_lock(); + err = wx_init_interrupt_scheme(wx); + if (!err && netif_running(netdev)) + err = ngbe_open(netdev); + if (!err) + netif_device_attach(netdev); + rtnl_unlock(); + + return 0; +} + static struct pci_driver ngbe_driver = { .name = ngbe_driver_name, .id_table = ngbe_pci_tbl, .probe = ngbe_probe, .remove = ngbe_remove, + .suspend = ngbe_suspend, + .resume = ngbe_resume, .shutdown = ngbe_shutdown, }; diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c index c9ddbbc3fa4f..cc2f325a52f7 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c @@ -236,6 +236,7 @@ static void ngbe_phy_fixup(struct wx *wx) phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); + phydev->mac_managed_pm = true; if (wx->mac_type != em_mac_type_mdi) return; /* disable EEE, internal phy does not support eee */ diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_hw.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_hw.c index 0772eb14eabf..6e130d1f7a7b 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_hw.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_hw.c @@ -257,16 +257,16 @@ static void txgbe_reset_misc(struct wx *wx) int txgbe_reset_hw(struct wx *wx) { int status; + u32 val; /* Call adapter stop to disable tx/rx and clear interrupts */ status = wx_stop_adapter(wx); if (status != 0) return status; - if (!(((wx->subsystem_device_id & WX_NCSI_MASK) == WX_NCSI_SUP) || - ((wx->subsystem_device_id & WX_WOL_MASK) == WX_WOL_SUP))) - wx_reset_hostif(wx); - + val = WX_MIS_RST_LAN_RST(wx->bus.func); + wr32(wx, WX_MIS_RST, val | rd32(wx, WX_MIS_RST)); + WX_WRITE_FLUSH(wx); usleep_range(10, 100); status = wx_check_flash_load(wx, TXGBE_SPI_ILDR_STATUS_LAN_SW_RST(wx->bus.func)); diff --git a/drivers/net/ethernet/xilinx/ll_temac.h b/drivers/net/ethernet/xilinx/ll_temac.h index 6668d1b760d8..90d122d5475c 100644 --- a/drivers/net/ethernet/xilinx/ll_temac.h +++ b/drivers/net/ethernet/xilinx/ll_temac.h @@ -5,6 +5,7 @@ #include <linux/netdevice.h> #include <linux/of.h> +#include <linux/platform_device.h> #include <linux/spinlock.h> #ifdef CONFIG_PPC_DCR diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index 49f303353ecb..1444b855e7aa 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -35,12 +35,10 @@ #include <linux/netdevice.h> #include <linux/if_ether.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> -#include <linux/of_address.h> +#include <linux/platform_device.h> #include <linux/skbuff.h> #include <linux/spinlock.h> #include <linux/tcp.h> /* needed for sizeof(tcphdr) */ diff --git a/drivers/net/ethernet/xilinx/ll_temac_mdio.c b/drivers/net/ethernet/xilinx/ll_temac_mdio.c index 2371c072b53f..07a9fb49eda1 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_mdio.c +++ b/drivers/net/ethernet/xilinx/ll_temac_mdio.c @@ -10,8 +10,8 @@ #include <linux/mutex.h> #include <linux/phy.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/of_mdio.h> #include <linux/platform_data/xilinx-ll-temac.h> diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 8e32dc50a408..b7ec4dafae90 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -27,11 +27,12 @@ #include <linux/etherdevice.h> #include <linux/module.h> #include <linux/netdevice.h> +#include <linux/of.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/of_platform.h> #include <linux/of_irq.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <linux/skbuff.h> #include <linux/math64.h> #include <linux/phy.h> diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c index ad2c30d9a482..b358ecc67227 100644 --- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c +++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c @@ -8,6 +8,7 @@ */ #include <linux/module.h> +#include <linux/platform_device.h> #include <linux/uaccess.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> @@ -15,9 +16,8 @@ #include <linux/ethtool.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> -#include <linux/of_platform.h> #include <linux/of_mdio.h> #include <linux/of_net.h> #include <linux/phy.h> diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index acb20ad4e37e..144ec626230d 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -243,7 +243,8 @@ static struct rtable *ip4_route_output_gtp(struct flowi4 *fl4, fl4->flowi4_oif = sk->sk_bound_dev_if; fl4->daddr = daddr; fl4->saddr = saddr; - fl4->flowi4_tos = RT_CONN_FLAGS(sk); + fl4->flowi4_tos = ip_sock_rt_tos(sk); + fl4->flowi4_scope = ip_sock_rt_scope(sk); fl4->flowi4_proto = sk->sk_protocol; return ip_route_output_key(sock_net(sk), fl4); diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index f9b10e84de06..aebb19f1b3a4 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -57,7 +57,6 @@ #include <linux/io.h> #include <linux/kfifo.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/module.h> #include <linux/mutex.h> diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 6a2f2fc2f501..da853353a5c7 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -13,8 +13,8 @@ #include <linux/firmware.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_address.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/firmware/qcom/qcom_scm.h> #include <linux/soc/qcom/mdt_loader.h> diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index ed908165a8b4..02bd201bc7e5 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -868,31 +868,24 @@ static int macvlan_change_mtu(struct net_device *dev, int new_mtu) return 0; } -static int macvlan_eth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) +static int macvlan_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) { struct net_device *real_dev = macvlan_dev_real_dev(dev); - const struct net_device_ops *ops = real_dev->netdev_ops; - struct ifreq ifrr; - int err = -EOPNOTSUPP; - strscpy(ifrr.ifr_name, real_dev->name, IFNAMSIZ); - ifrr.ifr_ifru = ifr->ifr_ifru; + return generic_hwtstamp_get_lower(real_dev, cfg); +} - switch (cmd) { - case SIOCSHWTSTAMP: - if (!net_eq(dev_net(dev), &init_net)) - break; - fallthrough; - case SIOCGHWTSTAMP: - if (netif_device_present(real_dev) && ops->ndo_eth_ioctl) - err = ops->ndo_eth_ioctl(real_dev, &ifrr, cmd); - break; - } +static int macvlan_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + struct net_device *real_dev = macvlan_dev_real_dev(dev); - if (!err) - ifr->ifr_ifru = ifrr.ifr_ifru; + if (!net_eq(dev_net(dev), &init_net)) + return -EOPNOTSUPP; - return err; + return generic_hwtstamp_set_lower(real_dev, cfg, extack); } /* @@ -1193,7 +1186,6 @@ static const struct net_device_ops macvlan_netdev_ops = { .ndo_stop = macvlan_stop, .ndo_start_xmit = macvlan_start_xmit, .ndo_change_mtu = macvlan_change_mtu, - .ndo_eth_ioctl = macvlan_eth_ioctl, .ndo_fix_features = macvlan_fix_features, .ndo_change_rx_flags = macvlan_change_rx_flags, .ndo_set_mac_address = macvlan_set_mac_address, @@ -1212,6 +1204,8 @@ static const struct net_device_ops macvlan_netdev_ops = { #endif .ndo_get_iflink = macvlan_dev_get_iflink, .ndo_features_check = passthru_features_check, + .ndo_hwtstamp_get = macvlan_hwtstamp_get, + .ndo_hwtstamp_set = macvlan_hwtstamp_set, }; static void macvlan_dev_free(struct net_device *dev) diff --git a/drivers/net/mdio/mdio-bcm-unimac.c b/drivers/net/mdio/mdio-bcm-unimac.c index bfc9be23c973..6b26a0803696 100644 --- a/drivers/net/mdio/mdio-bcm-unimac.c +++ b/drivers/net/mdio/mdio-bcm-unimac.c @@ -334,6 +334,8 @@ static SIMPLE_DEV_PM_OPS(unimac_mdio_pm_ops, unimac_mdio_suspend, unimac_mdio_resume); static const struct of_device_id unimac_mdio_ids[] = { + { .compatible = "brcm,asp-v2.1-mdio", }, + { .compatible = "brcm,asp-v2.0-mdio", }, { .compatible = "brcm,genet-mdio-v5", }, { .compatible = "brcm,genet-mdio-v4", }, { .compatible = "brcm,genet-mdio-v3", }, diff --git a/drivers/net/mdio/mdio-xgene.c b/drivers/net/mdio/mdio-xgene.c index 7aafc221b5cf..683e8f8319ab 100644 --- a/drivers/net/mdio/mdio-xgene.c +++ b/drivers/net/mdio/mdio-xgene.c @@ -79,7 +79,7 @@ EXPORT_SYMBOL(xgene_mdio_wr_mac); int xgene_mdio_rgmii_read(struct mii_bus *bus, int phy_id, int reg) { - struct xgene_mdio_pdata *pdata = (struct xgene_mdio_pdata *)bus->priv; + struct xgene_mdio_pdata *pdata = bus->priv; u32 data, done; u8 wait = 10; @@ -105,7 +105,7 @@ EXPORT_SYMBOL(xgene_mdio_rgmii_read); int xgene_mdio_rgmii_write(struct mii_bus *bus, int phy_id, int reg, u16 data) { - struct xgene_mdio_pdata *pdata = (struct xgene_mdio_pdata *)bus->priv; + struct xgene_mdio_pdata *pdata = bus->priv; u32 val, done; u8 wait = 10; diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 4f4f79532c6c..87f18aedd3bd 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -36,6 +36,7 @@ #include <linux/inet.h> #include <linux/configfs.h> #include <linux/etherdevice.h> +#include <linux/utsname.h> MODULE_AUTHOR("Maintainer: Matt Mackall <mpm@selenic.com>"); MODULE_DESCRIPTION("Console driver for network interfaces"); @@ -84,6 +85,8 @@ static struct console netconsole_ext; * Also, other parameters of a target may be modified at * runtime only when it is disabled (enabled == 0). * @extended: Denotes whether console is extended or not. + * @release: Denotes whether kernel release version should be prepended + * to the message. Depends on extended console. * @np: The netpoll structure for this target. * Contains the other userspace visible parameters: * dev_name (read-write) @@ -101,6 +104,7 @@ struct netconsole_target { #endif bool enabled; bool extended; + bool release; struct netpoll np; }; @@ -188,6 +192,15 @@ static struct netconsole_target *alloc_param_target(char *target_config) target_config++; } + if (*target_config == 'r') { + if (!nt->extended) { + pr_err("Netconsole configuration error. Release feature requires extended log message"); + goto fail; + } + nt->release = true; + target_config++; + } + /* Parse parameters and setup netpoll */ err = netpoll_parse_options(&nt->np, target_config); if (err) @@ -222,6 +235,7 @@ static void free_param_target(struct netconsole_target *nt) * | * <target>/ * | enabled + * | release * | dev_name * | local_port * | remote_port @@ -246,27 +260,32 @@ static struct netconsole_target *to_target(struct config_item *item) static ssize_t enabled_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%d\n", to_target(item)->enabled); + return sysfs_emit(buf, "%d\n", to_target(item)->enabled); } static ssize_t extended_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%d\n", to_target(item)->extended); + return sysfs_emit(buf, "%d\n", to_target(item)->extended); +} + +static ssize_t release_show(struct config_item *item, char *buf) +{ + return sysfs_emit(buf, "%d\n", to_target(item)->release); } static ssize_t dev_name_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%s\n", to_target(item)->np.dev_name); + return sysfs_emit(buf, "%s\n", to_target(item)->np.dev_name); } static ssize_t local_port_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%d\n", to_target(item)->np.local_port); + return sysfs_emit(buf, "%d\n", to_target(item)->np.local_port); } static ssize_t remote_port_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%d\n", to_target(item)->np.remote_port); + return sysfs_emit(buf, "%d\n", to_target(item)->np.remote_port); } static ssize_t local_ip_show(struct config_item *item, char *buf) @@ -274,9 +293,9 @@ static ssize_t local_ip_show(struct config_item *item, char *buf) struct netconsole_target *nt = to_target(item); if (nt->np.ipv6) - return snprintf(buf, PAGE_SIZE, "%pI6c\n", &nt->np.local_ip.in6); + return sysfs_emit(buf, "%pI6c\n", &nt->np.local_ip.in6); else - return snprintf(buf, PAGE_SIZE, "%pI4\n", &nt->np.local_ip); + return sysfs_emit(buf, "%pI4\n", &nt->np.local_ip); } static ssize_t remote_ip_show(struct config_item *item, char *buf) @@ -284,9 +303,9 @@ static ssize_t remote_ip_show(struct config_item *item, char *buf) struct netconsole_target *nt = to_target(item); if (nt->np.ipv6) - return snprintf(buf, PAGE_SIZE, "%pI6c\n", &nt->np.remote_ip.in6); + return sysfs_emit(buf, "%pI6c\n", &nt->np.remote_ip.in6); else - return snprintf(buf, PAGE_SIZE, "%pI4\n", &nt->np.remote_ip); + return sysfs_emit(buf, "%pI4\n", &nt->np.remote_ip); } static ssize_t local_mac_show(struct config_item *item, char *buf) @@ -294,12 +313,12 @@ static ssize_t local_mac_show(struct config_item *item, char *buf) struct net_device *dev = to_target(item)->np.dev; static const u8 bcast[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - return snprintf(buf, PAGE_SIZE, "%pM\n", dev ? dev->dev_addr : bcast); + return sysfs_emit(buf, "%pM\n", dev ? dev->dev_addr : bcast); } static ssize_t remote_mac_show(struct config_item *item, char *buf) { - return snprintf(buf, PAGE_SIZE, "%pM\n", to_target(item)->np.remote_mac); + return sysfs_emit(buf, "%pM\n", to_target(item)->np.remote_mac); } /* @@ -314,17 +333,15 @@ static ssize_t enabled_store(struct config_item *item, { struct netconsole_target *nt = to_target(item); unsigned long flags; - int enabled; + bool enabled; int err; mutex_lock(&dynamic_netconsole_mutex); - err = kstrtoint(buf, 10, &enabled); - if (err < 0) + err = kstrtobool(buf, &enabled); + if (err) goto out_unlock; err = -EINVAL; - if (enabled < 0 || enabled > 1) - goto out_unlock; if ((bool)enabled == nt->enabled) { pr_info("network logging has already %s\n", nt->enabled ? "started" : "stopped"); @@ -332,6 +349,11 @@ static ssize_t enabled_store(struct config_item *item, } if (enabled) { /* true */ + if (nt->release && !nt->extended) { + pr_err("Not enabling netconsole. Release feature requires extended log message"); + goto out_unlock; + } + if (nt->extended && !console_is_registered(&netconsole_ext)) register_console(&netconsole_ext); @@ -366,11 +388,11 @@ out_unlock: return err; } -static ssize_t extended_store(struct config_item *item, const char *buf, - size_t count) +static ssize_t release_store(struct config_item *item, const char *buf, + size_t count) { struct netconsole_target *nt = to_target(item); - int extended; + bool release; int err; mutex_lock(&dynamic_netconsole_mutex); @@ -381,14 +403,38 @@ static ssize_t extended_store(struct config_item *item, const char *buf, goto out_unlock; } - err = kstrtoint(buf, 10, &extended); - if (err < 0) + err = kstrtobool(buf, &release); + if (err) goto out_unlock; - if (extended < 0 || extended > 1) { + + nt->release = release; + + mutex_unlock(&dynamic_netconsole_mutex); + return strnlen(buf, count); +out_unlock: + mutex_unlock(&dynamic_netconsole_mutex); + return err; +} + +static ssize_t extended_store(struct config_item *item, const char *buf, + size_t count) +{ + struct netconsole_target *nt = to_target(item); + bool extended; + int err; + + mutex_lock(&dynamic_netconsole_mutex); + if (nt->enabled) { + pr_err("target (%s) is enabled, disable to update parameters\n", + config_item_name(&nt->item)); err = -EINVAL; goto out_unlock; } + err = kstrtobool(buf, &extended); + if (err) + goto out_unlock; + nt->extended = extended; mutex_unlock(&dynamic_netconsole_mutex); @@ -576,10 +622,12 @@ CONFIGFS_ATTR(, local_ip); CONFIGFS_ATTR(, remote_ip); CONFIGFS_ATTR_RO(, local_mac); CONFIGFS_ATTR(, remote_mac); +CONFIGFS_ATTR(, release); static struct configfs_attribute *netconsole_target_attrs[] = { &attr_enabled, &attr_extended, + &attr_release, &attr_dev_name, &attr_local_port, &attr_remote_port, @@ -772,9 +820,23 @@ static void send_ext_msg_udp(struct netconsole_target *nt, const char *msg, const char *header, *body; int offset = 0; int header_len, body_len; + const char *msg_ready = msg; + const char *release; + int release_len = 0; - if (msg_len <= MAX_PRINT_CHUNK) { - netpoll_send_udp(&nt->np, msg, msg_len); + if (nt->release) { + release = init_utsname()->release; + release_len = strlen(release) + 1; + } + + if (msg_len + release_len <= MAX_PRINT_CHUNK) { + /* No fragmentation needed */ + if (nt->release) { + scnprintf(buf, MAX_PRINT_CHUNK, "%s,%s", release, msg); + msg_len += release_len; + msg_ready = buf; + } + netpoll_send_udp(&nt->np, msg_ready, msg_len); return; } @@ -792,7 +854,10 @@ static void send_ext_msg_udp(struct netconsole_target *nt, const char *msg, * Transfer multiple chunks with the following extra header. * "ncfrag=<byte-offset>/<total-bytes>" */ - memcpy(buf, header, header_len); + if (nt->release) + scnprintf(buf, MAX_PRINT_CHUNK, "%s,", release); + memcpy(buf + release_len, header, header_len); + header_len += release_len; while (offset < body_len) { int this_header = header_len; diff --git a/drivers/net/netdevsim/Makefile b/drivers/net/netdevsim/Makefile index 5735e5b1a2cb..f8de93bc5f5b 100644 --- a/drivers/net/netdevsim/Makefile +++ b/drivers/net/netdevsim/Makefile @@ -17,3 +17,7 @@ endif ifneq ($(CONFIG_PSAMPLE),) netdevsim-objs += psample.o endif + +ifneq ($(CONFIG_MACSEC),) +netdevsim-objs += macsec.o +endif diff --git a/drivers/net/netdevsim/macsec.c b/drivers/net/netdevsim/macsec.c new file mode 100644 index 000000000000..0d5f50430dd3 --- /dev/null +++ b/drivers/net/netdevsim/macsec.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <net/macsec.h> +#include "netdevsim.h" + +static inline u64 sci_to_cpu(sci_t sci) +{ + return be64_to_cpu((__force __be64)sci); +} + +static int nsim_macsec_find_secy(struct netdevsim *ns, sci_t sci) +{ + int i; + + for (i = 0; i < NSIM_MACSEC_MAX_SECY_COUNT; i++) { + if (ns->macsec.nsim_secy[i].sci == sci) + return i; + } + + return -1; +} + +static int nsim_macsec_find_rxsc(struct nsim_secy *ns_secy, sci_t sci) +{ + int i; + + for (i = 0; i < NSIM_MACSEC_MAX_RXSC_COUNT; i++) { + if (ns_secy->nsim_rxsc[i].sci == sci) + return i; + } + + return -1; +} + +static int nsim_macsec_add_secy(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + if (ns->macsec.nsim_secy_count == NSIM_MACSEC_MAX_SECY_COUNT) + return -ENOSPC; + + for (idx = 0; idx < NSIM_MACSEC_MAX_SECY_COUNT; idx++) { + if (!ns->macsec.nsim_secy[idx].used) + break; + } + + if (idx == NSIM_MACSEC_MAX_SECY_COUNT) { + netdev_err(ctx->netdev, "%s: nsim_secy_count not full but all SecYs used\n", + __func__); + return -ENOSPC; + } + + netdev_dbg(ctx->netdev, "%s: adding new secy with sci %08llx at index %d\n", + __func__, sci_to_cpu(ctx->secy->sci), idx); + ns->macsec.nsim_secy[idx].used = true; + ns->macsec.nsim_secy[idx].nsim_rxsc_count = 0; + ns->macsec.nsim_secy[idx].sci = ctx->secy->sci; + ns->macsec.nsim_secy_count++; + + return 0; +} + +static int nsim_macsec_upd_secy(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: updating secy with sci %08llx at index %d\n", + __func__, sci_to_cpu(ctx->secy->sci), idx); + + return 0; +} + +static int nsim_macsec_del_secy(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: removing SecY with SCI %08llx at index %d\n", + __func__, sci_to_cpu(ctx->secy->sci), idx); + + ns->macsec.nsim_secy[idx].used = false; + memset(&ns->macsec.nsim_secy[idx], 0, sizeof(ns->macsec.nsim_secy[idx])); + ns->macsec.nsim_secy_count--; + + return 0; +} + +static int nsim_macsec_add_rxsc(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + if (secy->nsim_rxsc_count == NSIM_MACSEC_MAX_RXSC_COUNT) + return -ENOSPC; + + for (idx = 0; idx < NSIM_MACSEC_MAX_RXSC_COUNT; idx++) { + if (!secy->nsim_rxsc[idx].used) + break; + } + + if (idx == NSIM_MACSEC_MAX_RXSC_COUNT) + netdev_err(ctx->netdev, "%s: nsim_rxsc_count not full but all RXSCs used\n", + __func__); + + netdev_dbg(ctx->netdev, "%s: adding new rxsc with sci %08llx at index %d\n", + __func__, sci_to_cpu(ctx->rx_sc->sci), idx); + secy->nsim_rxsc[idx].used = true; + secy->nsim_rxsc[idx].sci = ctx->rx_sc->sci; + secy->nsim_rxsc_count++; + + return 0; +} + +static int nsim_macsec_upd_rxsc(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + idx = nsim_macsec_find_rxsc(secy, ctx->rx_sc->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in RXSC table\n", + __func__, sci_to_cpu(ctx->rx_sc->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: updating RXSC with sci %08llx at index %d\n", + __func__, sci_to_cpu(ctx->rx_sc->sci), idx); + + return 0; +} + +static int nsim_macsec_del_rxsc(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + idx = nsim_macsec_find_rxsc(secy, ctx->rx_sc->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in RXSC table\n", + __func__, sci_to_cpu(ctx->rx_sc->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: removing RXSC with sci %08llx at index %d\n", + __func__, sci_to_cpu(ctx->rx_sc->sci), idx); + + secy->nsim_rxsc[idx].used = false; + memset(&secy->nsim_rxsc[idx], 0, sizeof(secy->nsim_rxsc[idx])); + secy->nsim_rxsc_count--; + + return 0; +} + +static int nsim_macsec_add_rxsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + idx = nsim_macsec_find_rxsc(secy, ctx->sa.rx_sa->sc->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in RXSC table\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: RXSC with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci), ctx->sa.assoc_num); + + return 0; +} + +static int nsim_macsec_upd_rxsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + idx = nsim_macsec_find_rxsc(secy, ctx->sa.rx_sa->sc->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in RXSC table\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: RXSC with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci), ctx->sa.assoc_num); + + return 0; +} + +static int nsim_macsec_del_rxsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + struct nsim_secy *secy; + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + secy = &ns->macsec.nsim_secy[idx]; + + idx = nsim_macsec_find_rxsc(secy, ctx->sa.rx_sa->sc->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in RXSC table\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: RXSC with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->sa.rx_sa->sc->sci), ctx->sa.assoc_num); + + return 0; +} + +static int nsim_macsec_add_txsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: SECY with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->secy->sci), ctx->sa.assoc_num); + + return 0; +} + +static int nsim_macsec_upd_txsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: SECY with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->secy->sci), ctx->sa.assoc_num); + + return 0; +} + +static int nsim_macsec_del_txsa(struct macsec_context *ctx) +{ + struct netdevsim *ns = netdev_priv(ctx->netdev); + int idx; + + idx = nsim_macsec_find_secy(ns, ctx->secy->sci); + if (idx < 0) { + netdev_err(ctx->netdev, "%s: sci %08llx not found in secy table\n", + __func__, sci_to_cpu(ctx->secy->sci)); + return -ENOENT; + } + + netdev_dbg(ctx->netdev, "%s: SECY with sci %08llx, AN %u\n", + __func__, sci_to_cpu(ctx->secy->sci), ctx->sa.assoc_num); + + return 0; +} + +static const struct macsec_ops nsim_macsec_ops = { + .mdo_add_secy = nsim_macsec_add_secy, + .mdo_upd_secy = nsim_macsec_upd_secy, + .mdo_del_secy = nsim_macsec_del_secy, + .mdo_add_rxsc = nsim_macsec_add_rxsc, + .mdo_upd_rxsc = nsim_macsec_upd_rxsc, + .mdo_del_rxsc = nsim_macsec_del_rxsc, + .mdo_add_rxsa = nsim_macsec_add_rxsa, + .mdo_upd_rxsa = nsim_macsec_upd_rxsa, + .mdo_del_rxsa = nsim_macsec_del_rxsa, + .mdo_add_txsa = nsim_macsec_add_txsa, + .mdo_upd_txsa = nsim_macsec_upd_txsa, + .mdo_del_txsa = nsim_macsec_del_txsa, +}; + +void nsim_macsec_init(struct netdevsim *ns) +{ + ns->netdev->macsec_ops = &nsim_macsec_ops; + ns->netdev->features |= NETIF_F_HW_MACSEC; + memset(&ns->macsec, 0, sizeof(ns->macsec)); +} + +void nsim_macsec_teardown(struct netdevsim *ns) +{ +} diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 35fa1ca98671..0c8daeb0d62b 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -304,6 +304,7 @@ static int nsim_init_netdevsim(struct netdevsim *ns) if (err) goto err_utn_destroy; + nsim_macsec_init(ns); nsim_ipsec_init(ns); err = register_netdevice(ns->netdev); @@ -314,6 +315,7 @@ static int nsim_init_netdevsim(struct netdevsim *ns) err_ipsec_teardown: nsim_ipsec_teardown(ns); + nsim_macsec_teardown(ns); nsim_bpf_uninit(ns); err_utn_destroy: rtnl_unlock(); @@ -374,6 +376,7 @@ void nsim_destroy(struct netdevsim *ns) rtnl_lock(); unregister_netdevice(dev); if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { + nsim_macsec_teardown(ns); nsim_ipsec_teardown(ns); nsim_bpf_uninit(ns); } diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h index 7d8ed8d8df5c..7be98b7dcca9 100644 --- a/drivers/net/netdevsim/netdevsim.h +++ b/drivers/net/netdevsim/netdevsim.h @@ -23,6 +23,7 @@ #include <net/devlink.h> #include <net/udp_tunnel.h> #include <net/xdp.h> +#include <net/macsec.h> #define DRV_NAME "netdevsim" @@ -52,6 +53,25 @@ struct nsim_ipsec { u32 ok; }; +#define NSIM_MACSEC_MAX_SECY_COUNT 3 +#define NSIM_MACSEC_MAX_RXSC_COUNT 1 +struct nsim_rxsc { + sci_t sci; + bool used; +}; + +struct nsim_secy { + sci_t sci; + struct nsim_rxsc nsim_rxsc[NSIM_MACSEC_MAX_RXSC_COUNT]; + u8 nsim_rxsc_count; + bool used; +}; + +struct nsim_macsec { + struct nsim_secy nsim_secy[NSIM_MACSEC_MAX_SECY_COUNT]; + u8 nsim_secy_count; +}; + struct nsim_ethtool_pauseparam { bool rx; bool tx; @@ -93,6 +113,7 @@ struct netdevsim { bool bpf_map_accept; struct nsim_ipsec ipsec; + struct nsim_macsec macsec; struct { u32 inject_error; u32 sleep; @@ -366,6 +387,19 @@ static inline bool nsim_ipsec_tx(struct netdevsim *ns, struct sk_buff *skb) } #endif +#if IS_ENABLED(CONFIG_MACSEC) +void nsim_macsec_init(struct netdevsim *ns); +void nsim_macsec_teardown(struct netdevsim *ns); +#else +static inline void nsim_macsec_init(struct netdevsim *ns) +{ +} + +static inline void nsim_macsec_teardown(struct netdevsim *ns) +{ +} +#endif + struct nsim_bus_dev { struct device dev; struct list_head list; diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c index 323bec5e57f8..e5d642c67a2c 100644 --- a/drivers/net/pcs/pcs-rzn1-miic.c +++ b/drivers/net/pcs/pcs-rzn1-miic.c @@ -12,6 +12,7 @@ #include <linux/of_platform.h> #include <linux/pcs-rzn1-miic.h> #include <linux/phylink.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <dt-bindings/net/pcs-rzn1-miic.h> diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig index 78e6981650d9..107880d13d21 100644 --- a/drivers/net/phy/Kconfig +++ b/drivers/net/phy/Kconfig @@ -217,6 +217,12 @@ config MARVELL_10G_PHY help Support for the Marvell Alaska MV88X3310 and compatible PHYs. +config MARVELL_88Q2XXX_PHY + tristate "Marvell 88Q2XXX PHY" + help + Support for the Marvell 88Q2XXX 100/1000BASE-T1 Automotive Ethernet + PHYs. + config MARVELL_88X2222_PHY tristate "Marvell 88X2222 PHY" help @@ -300,7 +306,7 @@ config NXP_C45_TJA11XX_PHY depends on PTP_1588_CLOCK_OPTIONAL help Enable support for NXP C45 TJA11XX PHYs. - Currently supports only the TJA1103 PHY. + Currently supports the TJA1103 and TJA1120 PHYs. config NXP_TJA11XX_PHY tristate "NXP TJA11xx PHYs support" @@ -344,6 +350,7 @@ config ROCKCHIP_PHY config SMSC_PHY tristate "SMSC PHYs" + select CRC16 help Currently supports the LAN83C185, LAN8187 and LAN8700 PHYs diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile index 2fe51ea83bab..c945ed9bd14b 100644 --- a/drivers/net/phy/Makefile +++ b/drivers/net/phy/Makefile @@ -14,6 +14,8 @@ endif # dedicated loadable module, so we bundle them all together into libphy.ko ifdef CONFIG_PHYLIB libphy-y += $(mdio-bus-y) +# the stubs are built-in whenever PHYLIB is built-in or module +obj-y += stubs.o else obj-$(CONFIG_MDIO_DEVICE) += mdio-bus.o endif @@ -66,6 +68,7 @@ obj-$(CONFIG_LSI_ET1011C_PHY) += et1011c.o obj-$(CONFIG_LXT_PHY) += lxt.o obj-$(CONFIG_MARVELL_10G_PHY) += marvell10g.o obj-$(CONFIG_MARVELL_PHY) += marvell.o +obj-$(CONFIG_MARVELL_88Q2XXX_PHY) += marvell-88q2xxx.o obj-$(CONFIG_MARVELL_88X2222_PHY) += marvell-88x2222.o obj-$(CONFIG_MAXLINEAR_GPHY) += mxl-gpy.o obj-$(CONFIG_MEDIATEK_GE_PHY) += mediatek-ge.o diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index c1f307d90518..13c4121fa309 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -272,6 +272,13 @@ #define QCA808X_CDT_STATUS_STAT_OPEN 2 #define QCA808X_CDT_STATUS_STAT_SHORT 3 +/* QCA808X 1G chip type */ +#define QCA808X_PHY_MMD7_CHIP_TYPE 0x901d +#define QCA808X_PHY_CHIP_TYPE_1G BIT(0) + +#define QCA8081_PHY_SERDES_MMD1_FIFO_CTRL 0x9072 +#define QCA8081_PHY_FIFO_RSTN BIT(11) + MODULE_DESCRIPTION("Qualcomm Atheros AR803x and QCA808X PHY driver"); MODULE_AUTHOR("Matus Ujhelyi"); MODULE_LICENSE("GPL"); @@ -897,15 +904,6 @@ static int at803x_get_features(struct phy_device *phydev) if (err) return err; - if (phydev->drv->phy_id == QCA8081_PHY_ID) { - err = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_NG_EXTABLE); - if (err < 0) - return err; - - linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, phydev->supported, - err & MDIO_PMA_NG_EXTABLE_2_5GBT); - } - if (phydev->drv->phy_id != ATH8031_PHY_ID) return 0; @@ -1734,24 +1732,30 @@ static int qca808x_phy_fast_retrain_config(struct phy_device *phydev) return 0; } -static int qca808x_phy_ms_random_seed_set(struct phy_device *phydev) +static int qca808x_phy_ms_seed_enable(struct phy_device *phydev, bool enable) { - u16 seed_value = get_random_u32_below(QCA808X_MASTER_SLAVE_SEED_RANGE); + u16 seed_value; + + if (!enable) + return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_LOCAL_SEED, + QCA808X_MASTER_SLAVE_SEED_ENABLE, 0); + seed_value = get_random_u32_below(QCA808X_MASTER_SLAVE_SEED_RANGE); return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_LOCAL_SEED, - QCA808X_MASTER_SLAVE_SEED_CFG, - FIELD_PREP(QCA808X_MASTER_SLAVE_SEED_CFG, seed_value)); + QCA808X_MASTER_SLAVE_SEED_CFG | QCA808X_MASTER_SLAVE_SEED_ENABLE, + FIELD_PREP(QCA808X_MASTER_SLAVE_SEED_CFG, seed_value) | + QCA808X_MASTER_SLAVE_SEED_ENABLE); } -static int qca808x_phy_ms_seed_enable(struct phy_device *phydev, bool enable) +static bool qca808x_is_prefer_master(struct phy_device *phydev) { - u16 seed_enable = 0; - - if (enable) - seed_enable = QCA808X_MASTER_SLAVE_SEED_ENABLE; + return (phydev->master_slave_get == MASTER_SLAVE_CFG_MASTER_FORCE) || + (phydev->master_slave_get == MASTER_SLAVE_CFG_MASTER_PREFERRED); +} - return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_LOCAL_SEED, - QCA808X_MASTER_SLAVE_SEED_ENABLE, seed_enable); +static bool qca808x_has_fast_retrain_or_slave_seed(struct phy_device *phydev) +{ + return linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, phydev->supported); } static int qca808x_config_init(struct phy_device *phydev) @@ -1770,20 +1774,25 @@ static int qca808x_config_init(struct phy_device *phydev) if (ret) return ret; - /* Config the fast retrain for the link 2500M */ - ret = qca808x_phy_fast_retrain_config(phydev); - if (ret) - return ret; + if (qca808x_has_fast_retrain_or_slave_seed(phydev)) { + /* Config the fast retrain for the link 2500M */ + ret = qca808x_phy_fast_retrain_config(phydev); + if (ret) + return ret; - /* Configure lower ramdom seed to make phy linked as slave mode */ - ret = qca808x_phy_ms_random_seed_set(phydev); - if (ret) - return ret; + ret = genphy_read_master_slave(phydev); + if (ret < 0) + return ret; - /* Enable seed */ - ret = qca808x_phy_ms_seed_enable(phydev, true); - if (ret) - return ret; + if (!qca808x_is_prefer_master(phydev)) { + /* Enable seed and configure lower ramdom seed to make phy + * linked as slave mode. + */ + ret = qca808x_phy_ms_seed_enable(phydev, true); + if (ret) + return ret; + } + } /* Configure adc threshold as 100mv for the link 10M */ return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_ADC_THRESHOLD, @@ -1816,17 +1825,21 @@ static int qca808x_read_status(struct phy_device *phydev) phydev->interface = PHY_INTERFACE_MODE_SGMII; } else { /* generate seed as a lower random value to make PHY linked as SLAVE easily, - * except for master/slave configuration fault detected. + * except for master/slave configuration fault detected or the master mode + * preferred. + * * the reason for not putting this code into the function link_change_notify is * the corner case where the link partner is also the qca8081 PHY and the seed * value is configured as the same value, the link can't be up and no link change * occurs. */ - if (phydev->master_slave_state == MASTER_SLAVE_STATE_ERR) { - qca808x_phy_ms_seed_enable(phydev, false); - } else { - qca808x_phy_ms_random_seed_set(phydev); - qca808x_phy_ms_seed_enable(phydev, true); + if (qca808x_has_fast_retrain_or_slave_seed(phydev)) { + if (phydev->master_slave_state == MASTER_SLAVE_STATE_ERR || + qca808x_is_prefer_master(phydev)) { + qca808x_phy_ms_seed_enable(phydev, false); + } else { + qca808x_phy_ms_seed_enable(phydev, true); + } } } @@ -1841,7 +1854,10 @@ static int qca808x_soft_reset(struct phy_device *phydev) if (ret < 0) return ret; - return qca808x_phy_ms_seed_enable(phydev, true); + if (qca808x_has_fast_retrain_or_slave_seed(phydev)) + ret = qca808x_phy_ms_seed_enable(phydev, true); + + return ret; } static bool qca808x_cdt_fault_length_valid(int cdt_code) @@ -1991,6 +2007,44 @@ static int qca808x_cable_test_get_status(struct phy_device *phydev, bool *finish return 0; } +static int qca808x_get_features(struct phy_device *phydev) +{ + int ret; + + ret = genphy_c45_pma_read_abilities(phydev); + if (ret) + return ret; + + /* The autoneg ability is not existed in bit3 of MMD7.1, + * but it is supported by qca808x PHY, so we add it here + * manually. + */ + linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, phydev->supported); + + /* As for the qca8081 1G version chip, the 2500baseT ability is also + * existed in the bit0 of MMD1.21, we need to remove it manually if + * it is the qca8081 1G chip according to the bit0 of MMD7.0x901d. + */ + ret = phy_read_mmd(phydev, MDIO_MMD_AN, QCA808X_PHY_MMD7_CHIP_TYPE); + if (ret < 0) + return ret; + + if (QCA808X_PHY_CHIP_TYPE_1G & ret) + linkmode_clear_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, phydev->supported); + + return 0; +} + +static void qca808x_link_change_notify(struct phy_device *phydev) +{ + /* Assert interface sgmii fifo on link down, deassert it on link up, + * the interface device address is always phy address added by 1. + */ + mdiobus_c45_modify_changed(phydev->mdio.bus, phydev->mdio.addr + 1, + MDIO_MMD_PMAPMD, QCA8081_PHY_SERDES_MMD1_FIFO_CTRL, + QCA8081_PHY_FIFO_RSTN, phydev->link ? QCA8081_PHY_FIFO_RSTN : 0); +} + static struct phy_driver at803x_driver[] = { { /* Qualcomm Atheros AR8035 */ @@ -2160,7 +2214,7 @@ static struct phy_driver at803x_driver[] = { .set_tunable = at803x_set_tunable, .set_wol = at803x_set_wol, .get_wol = at803x_get_wol, - .get_features = at803x_get_features, + .get_features = qca808x_get_features, .config_aneg = at803x_config_aneg, .suspend = genphy_suspend, .resume = genphy_resume, @@ -2169,6 +2223,7 @@ static struct phy_driver at803x_driver[] = { .soft_reset = qca808x_soft_reset, .cable_test_start = qca808x_cable_test_start, .cable_test_get_status = qca808x_cable_test_get_status, + .link_change_notify = qca808x_link_change_notify, }, }; module_phy_driver(at803x_driver); diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c index f8c17a253f8b..8478b081c058 100644 --- a/drivers/net/phy/bcm7xxx.c +++ b/drivers/net/phy/bcm7xxx.c @@ -913,6 +913,7 @@ static struct phy_driver bcm7xxx_driver[] = { BCM7XXX_28NM_GPHY(PHY_ID_BCM7278, "Broadcom BCM7278"), BCM7XXX_28NM_GPHY(PHY_ID_BCM7364, "Broadcom BCM7364"), BCM7XXX_28NM_GPHY(PHY_ID_BCM7366, "Broadcom BCM7366"), + BCM7XXX_16NM_EPHY(PHY_ID_BCM74165, "Broadcom BCM74165"), BCM7XXX_28NM_GPHY(PHY_ID_BCM74371, "Broadcom BCM74371"), BCM7XXX_28NM_GPHY(PHY_ID_BCM7439, "Broadcom BCM7439"), BCM7XXX_28NM_GPHY(PHY_ID_BCM7439_2, "Broadcom BCM7439 (2)"), diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c new file mode 100644 index 000000000000..1c3ff77de56b --- /dev/null +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Marvell 88Q2XXX automotive 100BASE-T1/1000BASE-T1 PHY driver + */ +#include <linux/ethtool_netlink.h> +#include <linux/marvell_phy.h> +#include <linux/phy.h> + +#define MDIO_MMD_AN_MV_STAT 32769 +#define MDIO_MMD_AN_MV_STAT_ANEG 0x0100 +#define MDIO_MMD_AN_MV_STAT_LOCAL_RX 0x1000 +#define MDIO_MMD_AN_MV_STAT_REMOTE_RX 0x2000 +#define MDIO_MMD_AN_MV_STAT_LOCAL_MASTER 0x4000 +#define MDIO_MMD_AN_MV_STAT_MS_CONF_FAULT 0x8000 + +#define MDIO_MMD_PCS_MV_100BT1_STAT1 33032 +#define MDIO_MMD_PCS_MV_100BT1_STAT1_IDLE_ERROR 0x00FF +#define MDIO_MMD_PCS_MV_100BT1_STAT1_JABBER 0x0100 +#define MDIO_MMD_PCS_MV_100BT1_STAT1_LINK 0x0200 +#define MDIO_MMD_PCS_MV_100BT1_STAT1_LOCAL_RX 0x1000 +#define MDIO_MMD_PCS_MV_100BT1_STAT1_REMOTE_RX 0x2000 +#define MDIO_MMD_PCS_MV_100BT1_STAT1_LOCAL_MASTER 0x4000 + +#define MDIO_MMD_PCS_MV_100BT1_STAT2 33033 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_JABBER 0x0001 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_POL 0x0002 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_LINK 0x0004 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_ANGE 0x0008 + +static int mv88q2xxx_soft_reset(struct phy_device *phydev) +{ + int ret; + int val; + + ret = phy_write_mmd(phydev, MDIO_MMD_PCS, + MDIO_PCS_1000BT1_CTRL, MDIO_PCS_1000BT1_CTRL_RESET); + if (ret < 0) + return ret; + + return phy_read_mmd_poll_timeout(phydev, MDIO_MMD_PCS, + MDIO_PCS_1000BT1_CTRL, val, + !(val & MDIO_PCS_1000BT1_CTRL_RESET), + 50000, 600000, true); +} + +static int mv88q2xxx_read_link_gbit(struct phy_device *phydev) +{ + int ret; + bool link = false; + + /* Read vendor specific Auto-Negotiation status register to get local + * and remote receiver status according to software initialization + * guide. + */ + ret = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_MMD_AN_MV_STAT); + if (ret < 0) { + return ret; + } else if ((ret & MDIO_MMD_AN_MV_STAT_LOCAL_RX) && + (ret & MDIO_MMD_AN_MV_STAT_REMOTE_RX)) { + /* The link state is latched low so that momentary link + * drops can be detected. Do not double-read the status + * in polling mode to detect such short link drops except + * the link was already down. + */ + if (!phy_polling_mode(phydev) || !phydev->link) { + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_1000BT1_STAT); + if (ret < 0) + return ret; + else if (ret & MDIO_PCS_1000BT1_STAT_LINK) + link = true; + } + + if (!link) { + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_1000BT1_STAT); + if (ret < 0) + return ret; + else if (ret & MDIO_PCS_1000BT1_STAT_LINK) + link = true; + } + } + + phydev->link = link; + + return 0; +} + +static int mv88q2xxx_read_link_100m(struct phy_device *phydev) +{ + int ret; + + /* The link state is latched low so that momentary link + * drops can be detected. Do not double-read the status + * in polling mode to detect such short link drops except + * the link was already down. In case we are not polling, + * we always read the realtime status. + */ + if (!phy_polling_mode(phydev) || !phydev->link) { + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_MMD_PCS_MV_100BT1_STAT1); + if (ret < 0) + return ret; + else if (ret & MDIO_MMD_PCS_MV_100BT1_STAT1_LINK) + goto out; + } + + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_MMD_PCS_MV_100BT1_STAT1); + if (ret < 0) + return ret; + +out: + /* Check if we have link and if the remote and local receiver are ok */ + if ((ret & MDIO_MMD_PCS_MV_100BT1_STAT1_LINK) && + (ret & MDIO_MMD_PCS_MV_100BT1_STAT1_LOCAL_RX) && + (ret & MDIO_MMD_PCS_MV_100BT1_STAT1_REMOTE_RX)) + phydev->link = true; + else + phydev->link = false; + + return 0; +} + +static int mv88q2xxx_read_link(struct phy_device *phydev) +{ + int ret; + + /* The 88Q2XXX PHYs do not have the PMA/PMD status register available, + * therefore we need to read the link status from the vendor specific + * registers depending on the speed. + */ + if (phydev->speed == SPEED_1000) + ret = mv88q2xxx_read_link_gbit(phydev); + else + ret = mv88q2xxx_read_link_100m(phydev); + + return ret; +} + +static int mv88q2xxx_read_status(struct phy_device *phydev) +{ + int ret; + + ret = mv88q2xxx_read_link(phydev); + if (ret < 0) + return ret; + + return genphy_c45_read_pma(phydev); +} + +static int mv88q2xxx_get_features(struct phy_device *phydev) +{ + int ret; + + ret = genphy_c45_pma_read_abilities(phydev); + if (ret) + return ret; + + /* We need to read the baset1 extended abilities manually because the + * PHY does not signalize it has the extended abilities register + * available. + */ + ret = genphy_c45_pma_baset1_read_abilities(phydev); + if (ret) + return ret; + + /* The PHY signalizes it supports autonegotiation. Unfortunately, so + * far it was not possible to get a link even when following the init + * sequence provided by Marvell. Disable it for now until a proper + * workaround is found or a new PHY revision is released. + */ + linkmode_clear_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, phydev->supported); + + return 0; +} + +static int mv88q2xxx_config_aneg(struct phy_device *phydev) +{ + int ret; + + ret = genphy_c45_config_aneg(phydev); + if (ret) + return ret; + + return mv88q2xxx_soft_reset(phydev); +} + +static int mv88q2xxx_config_init(struct phy_device *phydev) +{ + int ret; + + /* The 88Q2XXX PHYs do have the extended ability register available, but + * register MDIO_PMA_EXTABLE where they should signalize it does not + * work according to specification. Therefore, we force it here. + */ + phydev->pma_extable = MDIO_PMA_EXTABLE_BT1; + + /* Read the current PHY configuration */ + ret = genphy_c45_read_pma(phydev); + if (ret) + return ret; + + return mv88q2xxx_config_aneg(phydev); +} + +static int mv88q2xxxx_get_sqi(struct phy_device *phydev) +{ + int ret; + + if (phydev->speed == SPEED_100) { + /* Read the SQI from the vendor specific receiver status + * register + */ + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, 0x8230); + if (ret < 0) + return ret; + + ret = ret >> 12; + } else { + /* Read from vendor specific registers, they are not documented + * but can be found in the Software Initialization Guide. Only + * revisions >= A0 are supported. + */ + ret = phy_modify_mmd(phydev, MDIO_MMD_PCS, 0xFC5D, 0x00FF, 0x00AC); + if (ret < 0) + return ret; + + ret = phy_read_mmd(phydev, MDIO_MMD_PCS, 0xfc88); + if (ret < 0) + return ret; + } + + return ret & 0x0F; +} + +static int mv88q2xxxx_get_sqi_max(struct phy_device *phydev) +{ + return 15; +} + +static struct phy_driver mv88q2xxx_driver[] = { + { + .phy_id = MARVELL_PHY_ID_88Q2110, + .phy_id_mask = MARVELL_PHY_ID_MASK, + .name = "mv88q2110", + .get_features = mv88q2xxx_get_features, + .config_aneg = mv88q2xxx_config_aneg, + .config_init = mv88q2xxx_config_init, + .read_status = mv88q2xxx_read_status, + .soft_reset = mv88q2xxx_soft_reset, + .set_loopback = genphy_c45_loopback, + .get_sqi = mv88q2xxxx_get_sqi, + .get_sqi_max = mv88q2xxxx_get_sqi_max, + }, +}; + +module_phy_driver(mv88q2xxx_driver); + +static struct mdio_device_id __maybe_unused mv88q2xxx_tbl[] = { + { MARVELL_PHY_ID_88Q2110, MARVELL_PHY_ID_MASK }, + { /*sentinel*/ } +}; +MODULE_DEVICE_TABLE(mdio, mv88q2xxx_tbl); + +MODULE_DESCRIPTION("Marvell 88Q2XXX 100/1000BASE-T1 Automotive Ethernet PHY driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c index f83cae64585d..e3aa30dad2e6 100644 --- a/drivers/net/phy/marvell-88x2222.c +++ b/drivers/net/phy/marvell-88x2222.c @@ -14,7 +14,6 @@ #include <linux/mdio.h> #include <linux/marvell_phy.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/sfp.h> #include <linux/netdevice.h> diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 8b3618d3da4a..25dcaa49ab8b 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -107,16 +107,21 @@ int mdiobus_unregister_device(struct mdio_device *mdiodev) } EXPORT_SYMBOL(mdiobus_unregister_device); -struct phy_device *mdiobus_get_phy(struct mii_bus *bus, int addr) +static struct mdio_device *mdiobus_find_device(struct mii_bus *bus, int addr) { bool addr_valid = addr >= 0 && addr < ARRAY_SIZE(bus->mdio_map); - struct mdio_device *mdiodev; if (WARN_ONCE(!addr_valid, "addr %d out of range\n", addr)) return NULL; - mdiodev = bus->mdio_map[addr]; + return bus->mdio_map[addr]; +} + +struct phy_device *mdiobus_get_phy(struct mii_bus *bus, int addr) +{ + struct mdio_device *mdiodev; + mdiodev = mdiobus_find_device(bus, addr); if (!mdiodev) return NULL; @@ -129,7 +134,7 @@ EXPORT_SYMBOL(mdiobus_get_phy); bool mdiobus_is_registered_device(struct mii_bus *bus, int addr) { - return bus->mdio_map[addr]; + return mdiobus_find_device(bus, addr) != NULL; } EXPORT_SYMBOL(mdiobus_is_registered_device); @@ -1210,6 +1215,26 @@ int mdiobus_c45_write_nested(struct mii_bus *bus, int addr, int devad, } EXPORT_SYMBOL(mdiobus_c45_write_nested); +/* + * __mdiobus_modify - Convenience function for modifying a given mdio device + * register + * @bus: the mii_bus struct + * @addr: the phy address + * @regnum: register number to write + * @mask: bit mask of bits to clear + * @set: bit mask of bits to set + */ +int __mdiobus_modify(struct mii_bus *bus, int addr, u32 regnum, u16 mask, + u16 set) +{ + int err; + + err = __mdiobus_modify_changed(bus, addr, regnum, mask, set); + + return err < 0 ? err : 0; +} +EXPORT_SYMBOL_GPL(__mdiobus_modify); + /** * mdiobus_modify - Convenience function for modifying a given mdio device * register @@ -1224,10 +1249,10 @@ int mdiobus_modify(struct mii_bus *bus, int addr, u32 regnum, u16 mask, u16 set) int err; mutex_lock(&bus->mdio_lock); - err = __mdiobus_modify_changed(bus, addr, regnum, mask, set); + err = __mdiobus_modify(bus, addr, regnum, mask, set); mutex_unlock(&bus->mdio_lock); - return err < 0 ? err : 0; + return err; } EXPORT_SYMBOL_GPL(mdiobus_modify); diff --git a/drivers/net/phy/mediatek-ge-soc.c b/drivers/net/phy/mediatek-ge-soc.c index 95369171a7ba..da512fab0eb0 100644 --- a/drivers/net/phy/mediatek-ge-soc.c +++ b/drivers/net/phy/mediatek-ge-soc.c @@ -2,8 +2,6 @@ #include <linux/bitfield.h> #include <linux/module.h> #include <linux/nvmem-consumer.h> -#include <linux/of_address.h> -#include <linux/of_platform.h> #include <linux/pinctrl/consumer.h> #include <linux/phy.h> diff --git a/drivers/net/phy/motorcomm.c b/drivers/net/phy/motorcomm.c index 2fa5a90e073b..7a11fdb687cc 100644 --- a/drivers/net/phy/motorcomm.c +++ b/drivers/net/phy/motorcomm.c @@ -163,6 +163,10 @@ #define YT8521_CHIP_CONFIG_REG 0xA001 #define YT8521_CCR_SW_RST BIT(15) +#define YT8531_RGMII_LDO_VOL_MASK GENMASK(5, 4) +#define YT8531_LDO_VOL_3V3 0x0 +#define YT8531_LDO_VOL_1V8 0x2 + /* 1b0 disable 1.9ns rxc clock delay *default* * 1b1 enable 1.9ns rxc clock delay */ @@ -236,6 +240,12 @@ */ #define YTPHY_WCR_TYPE_PULSE BIT(0) +#define YTPHY_PAD_DRIVE_STRENGTH_REG 0xA010 +#define YT8531_RGMII_RXC_DS_MASK GENMASK(15, 13) +#define YT8531_RGMII_RXD_DS_HI_MASK BIT(12) /* Bit 2 of rxd_ds */ +#define YT8531_RGMII_RXD_DS_LOW_MASK GENMASK(5, 4) /* Bit 1/0 of rxd_ds */ +#define YT8531_RGMII_RX_DS_DEFAULT 0x3 + #define YTPHY_SYNCE_CFG_REG 0xA012 #define YT8521_SCR_SYNCE_ENABLE BIT(5) /* 1b0 output 25m clock @@ -835,6 +845,110 @@ static int ytphy_rgmii_clk_delay_config_with_lock(struct phy_device *phydev) } /** + * struct ytphy_ldo_vol_map - map a current value to a register value + * @vol: ldo voltage + * @ds: value in the register + * @cur: value in device configuration + */ +struct ytphy_ldo_vol_map { + u32 vol; + u32 ds; + u32 cur; +}; + +static const struct ytphy_ldo_vol_map yt8531_ldo_vol[] = { + {.vol = YT8531_LDO_VOL_1V8, .ds = 0, .cur = 1200}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 1, .cur = 2100}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 2, .cur = 2700}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 3, .cur = 2910}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 4, .cur = 3110}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 5, .cur = 3600}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 6, .cur = 3970}, + {.vol = YT8531_LDO_VOL_1V8, .ds = 7, .cur = 4350}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 0, .cur = 3070}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 1, .cur = 4080}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 2, .cur = 4370}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 3, .cur = 4680}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 4, .cur = 5020}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 5, .cur = 5450}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 6, .cur = 5740}, + {.vol = YT8531_LDO_VOL_3V3, .ds = 7, .cur = 6140}, +}; + +static u32 yt8531_get_ldo_vol(struct phy_device *phydev) +{ + u32 val; + + val = ytphy_read_ext_with_lock(phydev, YT8521_CHIP_CONFIG_REG); + val = FIELD_GET(YT8531_RGMII_LDO_VOL_MASK, val); + + return val <= YT8531_LDO_VOL_1V8 ? val : YT8531_LDO_VOL_1V8; +} + +static int yt8531_get_ds_map(struct phy_device *phydev, u32 cur) +{ + u32 vol; + int i; + + vol = yt8531_get_ldo_vol(phydev); + for (i = 0; i < ARRAY_SIZE(yt8531_ldo_vol); i++) { + if (yt8531_ldo_vol[i].vol == vol && yt8531_ldo_vol[i].cur == cur) + return yt8531_ldo_vol[i].ds; + } + + return -EINVAL; +} + +static int yt8531_set_ds(struct phy_device *phydev) +{ + struct device_node *node = phydev->mdio.dev.of_node; + u32 ds_field_low, ds_field_hi, val; + int ret, ds; + + /* set rgmii rx clk driver strength */ + if (!of_property_read_u32(node, "motorcomm,rx-clk-drv-microamp", &val)) { + ds = yt8531_get_ds_map(phydev, val); + if (ds < 0) + return dev_err_probe(&phydev->mdio.dev, ds, + "No matching current value was found.\n"); + } else { + ds = YT8531_RGMII_RX_DS_DEFAULT; + } + + ret = ytphy_modify_ext_with_lock(phydev, + YTPHY_PAD_DRIVE_STRENGTH_REG, + YT8531_RGMII_RXC_DS_MASK, + FIELD_PREP(YT8531_RGMII_RXC_DS_MASK, ds)); + if (ret < 0) + return ret; + + /* set rgmii rx data driver strength */ + if (!of_property_read_u32(node, "motorcomm,rx-data-drv-microamp", &val)) { + ds = yt8531_get_ds_map(phydev, val); + if (ds < 0) + return dev_err_probe(&phydev->mdio.dev, ds, + "No matching current value was found.\n"); + } else { + ds = YT8531_RGMII_RX_DS_DEFAULT; + } + + ds_field_hi = FIELD_GET(BIT(2), ds); + ds_field_hi = FIELD_PREP(YT8531_RGMII_RXD_DS_HI_MASK, ds_field_hi); + + ds_field_low = FIELD_GET(GENMASK(1, 0), ds); + ds_field_low = FIELD_PREP(YT8531_RGMII_RXD_DS_LOW_MASK, ds_field_low); + + ret = ytphy_modify_ext_with_lock(phydev, + YTPHY_PAD_DRIVE_STRENGTH_REG, + YT8531_RGMII_RXD_DS_LOW_MASK | YT8531_RGMII_RXD_DS_HI_MASK, + ds_field_low | ds_field_hi); + if (ret < 0) + return ret; + + return 0; +} + +/** * yt8521_probe() - read chip config then set suitable polling_mode * @phydev: a pointer to a &struct phy_device * @@ -1518,6 +1632,10 @@ static int yt8531_config_init(struct phy_device *phydev) return ret; } + ret = yt8531_set_ds(phydev); + if (ret < 0) + return ret; + return 0; } diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c index 029875a59ff8..7ab080ff02df 100644 --- a/drivers/net/phy/nxp-c45-tja11xx.c +++ b/drivers/net/phy/nxp-c45-tja11xx.c @@ -18,24 +18,37 @@ #include <linux/net_tstamp.h> #define PHY_ID_TJA_1103 0x001BB010 - -#define PMAPMD_B100T1_PMAPMD_CTL 0x0834 -#define B100T1_PMAPMD_CONFIG_EN BIT(15) -#define B100T1_PMAPMD_MASTER BIT(14) -#define MASTER_MODE (B100T1_PMAPMD_CONFIG_EN | \ - B100T1_PMAPMD_MASTER) -#define SLAVE_MODE (B100T1_PMAPMD_CONFIG_EN) +#define PHY_ID_TJA_1120 0x001BB031 #define VEND1_DEVICE_CONTROL 0x0040 #define DEVICE_CONTROL_RESET BIT(15) #define DEVICE_CONTROL_CONFIG_GLOBAL_EN BIT(14) #define DEVICE_CONTROL_CONFIG_ALL_EN BIT(13) +#define VEND1_DEVICE_CONFIG 0x0048 + +#define TJA1120_VEND1_EXT_TS_MODE 0x1012 + +#define TJA1120_GLOBAL_INFRA_IRQ_ACK 0x2C08 +#define TJA1120_GLOBAL_INFRA_IRQ_EN 0x2C0A +#define TJA1120_GLOBAL_INFRA_IRQ_STATUS 0x2C0C +#define TJA1120_DEV_BOOT_DONE BIT(1) + +#define TJA1120_VEND1_PTP_TRIG_DATA_S 0x1070 + +#define TJA1120_EGRESS_TS_DATA_S 0x9060 +#define TJA1120_EGRESS_TS_END 0x9067 +#define TJA1120_TS_VALID BIT(0) +#define TJA1120_MORE_TS BIT(15) + #define VEND1_PHY_IRQ_ACK 0x80A0 #define VEND1_PHY_IRQ_EN 0x80A1 #define VEND1_PHY_IRQ_STATUS 0x80A2 #define PHY_IRQ_LINK_EVENT BIT(1) +#define VEND1_ALWAYS_ACCESSIBLE 0x801F +#define FUSA_PASS BIT(4) + #define VEND1_PHY_CONTROL 0x8100 #define PHY_CONFIG_EN BIT(14) #define PHY_START_OP BIT(0) @@ -43,15 +56,16 @@ #define VEND1_PHY_CONFIG 0x8108 #define PHY_CONFIG_AUTO BIT(0) +#define TJA1120_EPHY_RESETS 0x810A +#define EPHY_PCS_RESET BIT(3) + #define VEND1_SIGNAL_QUALITY 0x8320 #define SQI_VALID BIT(14) #define SQI_MASK GENMASK(2, 0) #define MAX_SQI SQI_MASK -#define VEND1_CABLE_TEST 0x8330 #define CABLE_TEST_ENABLE BIT(15) #define CABLE_TEST_START BIT(14) -#define CABLE_TEST_VALID BIT(13) #define CABLE_TEST_OK 0x00 #define CABLE_TEST_SHORTED 0x01 #define CABLE_TEST_OPEN 0x02 @@ -63,6 +77,12 @@ #define VEND1_PORT_ABILITIES 0x8046 #define PTP_ABILITY BIT(3) +#define VEND1_PORT_FUNC_IRQ_EN 0x807A +#define PTP_IRQS BIT(3) + +#define VEND1_PTP_IRQ_ACK 0x9008 +#define EGR_TS_IRQ BIT(1) + #define VEND1_PORT_INFRA_CONTROL 0xAC00 #define PORT_INFRA_CONTROL_EN BIT(14) @@ -85,12 +105,17 @@ #define MII_BASIC_CONFIG_RMII 0x5 #define MII_BASIC_CONFIG_MII 0x4 +#define VEND1_SYMBOL_ERROR_CNT_XTD 0x8351 +#define EXTENDED_CNT_EN BIT(15) +#define VEND1_MONITOR_STATUS 0xAC80 +#define MONITOR_RESET BIT(15) +#define VEND1_MONITOR_CONFIG 0xAC86 +#define LOST_FRAMES_CNT_EN BIT(9) +#define ALL_FRAMES_CNT_EN BIT(8) + #define VEND1_SYMBOL_ERROR_COUNTER 0x8350 #define VEND1_LINK_DROP_COUNTER 0x8352 #define VEND1_LINK_LOSSES_AND_FAILURES 0x8353 -#define VEND1_R_GOOD_FRAME_CNT 0xA950 -#define VEND1_R_BAD_FRAME_CNT 0xA952 -#define VEND1_R_RXER_FRAME_CNT 0xA954 #define VEND1_RX_PREAMBLE_COUNT 0xAFCE #define VEND1_TX_PREAMBLE_COUNT 0xAFCF #define VEND1_RX_IPG_LENGTH 0xAFD0 @@ -99,81 +124,43 @@ #define VEND1_PTP_CONFIG 0x1102 #define EXT_TRG_EDGE BIT(1) -#define PPS_OUT_POL BIT(2) -#define PPS_OUT_EN BIT(3) -#define VEND1_LTC_LOAD_CTRL 0x1105 -#define READ_LTC BIT(2) -#define LOAD_LTC BIT(0) +#define TJA1120_SYNC_TRIG_FILTER 0x1010 +#define PTP_TRIG_RISE_TS BIT(3) +#define PTP_TRIG_FALLING_TS BIT(2) -#define VEND1_LTC_WR_NSEC_0 0x1106 -#define VEND1_LTC_WR_NSEC_1 0x1107 -#define VEND1_LTC_WR_SEC_0 0x1108 -#define VEND1_LTC_WR_SEC_1 0x1109 - -#define VEND1_LTC_RD_NSEC_0 0x110A -#define VEND1_LTC_RD_NSEC_1 0x110B -#define VEND1_LTC_RD_SEC_0 0x110C -#define VEND1_LTC_RD_SEC_1 0x110D - -#define VEND1_RATE_ADJ_SUBNS_0 0x110F -#define VEND1_RATE_ADJ_SUBNS_1 0x1110 #define CLK_RATE_ADJ_LD BIT(15) #define CLK_RATE_ADJ_DIR BIT(14) -#define VEND1_HW_LTC_LOCK_CTRL 0x1115 -#define HW_LTC_LOCK_EN BIT(0) - -#define VEND1_PTP_IRQ_EN 0x1131 -#define VEND1_PTP_IRQ_STATUS 0x1132 -#define PTP_IRQ_EGR_TS BIT(0) - #define VEND1_RX_TS_INSRT_CTRL 0x114D -#define RX_TS_INSRT_MODE2 0x02 +#define TJA1103_RX_TS_INSRT_MODE2 0x02 + +#define TJA1120_RX_TS_INSRT_CTRL 0x9012 +#define TJA1120_RX_TS_INSRT_EN BIT(15) +#define TJA1120_TS_INSRT_MODE BIT(4) #define VEND1_EGR_RING_DATA_0 0x114E -#define VEND1_EGR_RING_DATA_1_SEQ_ID 0x114F -#define VEND1_EGR_RING_DATA_2_NSEC_15_0 0x1150 -#define VEND1_EGR_RING_DATA_3 0x1151 #define VEND1_EGR_RING_CTRL 0x1154 -#define VEND1_EXT_TRG_TS_DATA_0 0x1121 -#define VEND1_EXT_TRG_TS_DATA_1 0x1122 -#define VEND1_EXT_TRG_TS_DATA_2 0x1123 -#define VEND1_EXT_TRG_TS_DATA_3 0x1124 -#define VEND1_EXT_TRG_TS_DATA_4 0x1125 -#define VEND1_EXT_TRG_TS_CTRL 0x1126 - -#define RING_DATA_0_DOMAIN_NUMBER GENMASK(7, 0) -#define RING_DATA_0_MSG_TYPE GENMASK(11, 8) -#define RING_DATA_0_SEC_4_2 GENMASK(14, 2) #define RING_DATA_0_TS_VALID BIT(15) -#define RING_DATA_3_NSEC_29_16 GENMASK(13, 0) -#define RING_DATA_3_SEC_1_0 GENMASK(15, 14) -#define RING_DATA_5_SEC_16_5 GENMASK(15, 4) #define RING_DONE BIT(0) #define TS_SEC_MASK GENMASK(1, 0) #define VEND1_PORT_FUNC_ENABLES 0x8048 #define PTP_ENABLE BIT(3) +#define PHY_TEST_ENABLE BIT(0) #define VEND1_PORT_PTP_CONTROL 0x9000 #define PORT_PTP_CONTROL_BYPASS BIT(11) -#define VEND1_PTP_CLK_PERIOD 0x1104 #define PTP_CLK_PERIOD_100BT1 15ULL +#define PTP_CLK_PERIOD_1000BT1 8ULL -#define VEND1_EVENT_MSG_FILT 0x1148 #define EVENT_MSG_FILT_ALL 0x0F #define EVENT_MSG_FILT_NONE 0x00 -#define VEND1_TX_PIPE_DLY_NS 0x1149 -#define VEND1_TX_PIPEDLY_SUBNS 0x114A -#define VEND1_RX_PIPE_DLY_NS 0x114B -#define VEND1_RX_PIPEDLY_SUBNS 0x114C - #define VEND1_GPIO_FUNC_CONFIG_BASE 0x2C40 #define GPIO_FUNC_EN BIT(15) #define GPIO_FUNC_PTP BIT(6) @@ -191,16 +178,33 @@ #define MAX_ID_PS 2260U #define DEFAULT_ID_PS 2000U -#define PPM_TO_SUBNS_INC(ppb) div_u64(GENMASK_ULL(31, 0) * (ppb) * \ - PTP_CLK_PERIOD_100BT1, NSEC_PER_SEC) +#define PPM_TO_SUBNS_INC(ppb, ptp_clk_period) div_u64(GENMASK_ULL(31, 0) * \ + (ppb) * (ptp_clk_period), NSEC_PER_SEC) #define NXP_C45_SKB_CB(skb) ((struct nxp_c45_skb_cb *)(skb)->cb) +struct nxp_c45_phy; + struct nxp_c45_skb_cb { struct ptp_header *header; unsigned int type; }; +#define NXP_C45_REG_FIELD(_reg, _devad, _offset, _size) \ + ((struct nxp_c45_reg_field) { \ + .reg = _reg, \ + .devad = _devad, \ + .offset = _offset, \ + .size = _size, \ + }) + +struct nxp_c45_reg_field { + u16 reg; + u8 devad; + u8 offset; + u8 size; +}; + struct nxp_c45_hwts { u32 nsec; u32 sec; @@ -209,7 +213,76 @@ struct nxp_c45_hwts { u8 msg_type; }; +struct nxp_c45_regmap { + /* PTP config regs. */ + u16 vend1_ptp_clk_period; + u16 vend1_event_msg_filt; + + /* LTC bits and regs. */ + struct nxp_c45_reg_field ltc_read; + struct nxp_c45_reg_field ltc_write; + struct nxp_c45_reg_field ltc_lock_ctrl; + u16 vend1_ltc_wr_nsec_0; + u16 vend1_ltc_wr_nsec_1; + u16 vend1_ltc_wr_sec_0; + u16 vend1_ltc_wr_sec_1; + u16 vend1_ltc_rd_nsec_0; + u16 vend1_ltc_rd_nsec_1; + u16 vend1_ltc_rd_sec_0; + u16 vend1_ltc_rd_sec_1; + u16 vend1_rate_adj_subns_0; + u16 vend1_rate_adj_subns_1; + + /* External trigger reg fields. */ + struct nxp_c45_reg_field irq_egr_ts_en; + struct nxp_c45_reg_field irq_egr_ts_status; + struct nxp_c45_reg_field domain_number; + struct nxp_c45_reg_field msg_type; + struct nxp_c45_reg_field sequence_id; + struct nxp_c45_reg_field sec_1_0; + struct nxp_c45_reg_field sec_4_2; + struct nxp_c45_reg_field nsec_15_0; + struct nxp_c45_reg_field nsec_29_16; + + /* PPS and EXT Trigger bits and regs. */ + struct nxp_c45_reg_field pps_enable; + struct nxp_c45_reg_field pps_polarity; + u16 vend1_ext_trg_data_0; + u16 vend1_ext_trg_data_1; + u16 vend1_ext_trg_data_2; + u16 vend1_ext_trg_data_3; + u16 vend1_ext_trg_ctrl; + + /* Cable test reg fields. */ + u16 cable_test; + struct nxp_c45_reg_field cable_test_valid; + struct nxp_c45_reg_field cable_test_result; +}; + +struct nxp_c45_phy_stats { + const char *name; + const struct nxp_c45_reg_field counter; +}; + +struct nxp_c45_phy_data { + const struct nxp_c45_regmap *regmap; + const struct nxp_c45_phy_stats *stats; + int n_stats; + u8 ptp_clk_period; + bool ext_ts_both_edges; + bool ack_ptp_irq; + void (*counters_enable)(struct phy_device *phydev); + bool (*get_egressts)(struct nxp_c45_phy *priv, + struct nxp_c45_hwts *hwts); + bool (*get_extts)(struct nxp_c45_phy *priv, struct timespec64 *extts); + void (*ptp_init)(struct phy_device *phydev); + void (*ptp_enable)(struct phy_device *phydev, bool enable); + void (*nmi_handler)(struct phy_device *phydev, + irqreturn_t *irq_status); +}; + struct nxp_c45_phy { + const struct nxp_c45_phy_data *phy_data; struct phy_device *phydev; struct mii_timestamper mii_ts; struct ptp_clock *ptp_clock; @@ -227,13 +300,86 @@ struct nxp_c45_phy { bool extts; }; -struct nxp_c45_phy_stats { - const char *name; - u8 mmd; - u16 reg; - u8 off; - u16 mask; -}; +static const +struct nxp_c45_phy_data *nxp_c45_get_data(struct phy_device *phydev) +{ + return phydev->drv->driver_data; +} + +static const +struct nxp_c45_regmap *nxp_c45_get_regmap(struct phy_device *phydev) +{ + const struct nxp_c45_phy_data *phy_data = nxp_c45_get_data(phydev); + + return phy_data->regmap; +} + +static int nxp_c45_read_reg_field(struct phy_device *phydev, + const struct nxp_c45_reg_field *reg_field) +{ + u16 mask; + int ret; + + if (reg_field->size == 0) { + phydev_err(phydev, "Trying to read a reg field of size 0.\n"); + return -EINVAL; + } + + ret = phy_read_mmd(phydev, reg_field->devad, reg_field->reg); + if (ret < 0) + return ret; + + mask = reg_field->size == 1 ? BIT(reg_field->offset) : + GENMASK(reg_field->offset + reg_field->size - 1, + reg_field->offset); + ret &= mask; + ret >>= reg_field->offset; + + return ret; +} + +static int nxp_c45_write_reg_field(struct phy_device *phydev, + const struct nxp_c45_reg_field *reg_field, + u16 val) +{ + u16 mask; + u16 set; + + if (reg_field->size == 0) { + phydev_err(phydev, "Trying to write a reg field of size 0.\n"); + return -EINVAL; + } + + mask = reg_field->size == 1 ? BIT(reg_field->offset) : + GENMASK(reg_field->offset + reg_field->size - 1, + reg_field->offset); + set = val << reg_field->offset; + + return phy_modify_mmd_changed(phydev, reg_field->devad, + reg_field->reg, mask, set); +} + +static int nxp_c45_set_reg_field(struct phy_device *phydev, + const struct nxp_c45_reg_field *reg_field) +{ + if (reg_field->size != 1) { + phydev_err(phydev, "Trying to set a reg field of size different than 1.\n"); + return -EINVAL; + } + + return nxp_c45_write_reg_field(phydev, reg_field, 1); +} + +static int nxp_c45_clear_reg_field(struct phy_device *phydev, + const struct nxp_c45_reg_field *reg_field) +{ + if (reg_field->size != 1) { + phydev_err(phydev, "Trying to set a reg field of size different than 1.\n"); + return -EINVAL; + } + + return nxp_c45_write_reg_field(phydev, reg_field, 0); +} static bool nxp_c45_poll_txts(struct phy_device *phydev) { @@ -245,17 +391,17 @@ static int _nxp_c45_ptp_gettimex64(struct ptp_clock_info *ptp, struct ptp_system_timestamp *sts) { struct nxp_c45_phy *priv = container_of(ptp, struct nxp_c45_phy, caps); + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_LOAD_CTRL, - READ_LTC); + nxp_c45_set_reg_field(priv->phydev, ®map->ltc_read); ts->tv_nsec = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_LTC_RD_NSEC_0); + regmap->vend1_ltc_rd_nsec_0); ts->tv_nsec |= phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_LTC_RD_NSEC_1) << 16; + regmap->vend1_ltc_rd_nsec_1) << 16; ts->tv_sec = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_LTC_RD_SEC_0); + regmap->vend1_ltc_rd_sec_0); ts->tv_sec |= phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_LTC_RD_SEC_1) << 16; + regmap->vend1_ltc_rd_sec_1) << 16; return 0; } @@ -277,17 +423,17 @@ static int _nxp_c45_ptp_settime64(struct ptp_clock_info *ptp, const struct timespec64 *ts) { struct nxp_c45_phy *priv = container_of(ptp, struct nxp_c45_phy, caps); + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_WR_NSEC_0, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, regmap->vend1_ltc_wr_nsec_0, ts->tv_nsec); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_WR_NSEC_1, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, regmap->vend1_ltc_wr_nsec_1, ts->tv_nsec >> 16); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_WR_SEC_0, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, regmap->vend1_ltc_wr_sec_0, ts->tv_sec); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_WR_SEC_1, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, regmap->vend1_ltc_wr_sec_1, ts->tv_sec >> 16); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_LTC_LOAD_CTRL, - LOAD_LTC); + nxp_c45_set_reg_field(priv->phydev, ®map->ltc_write); return 0; } @@ -307,6 +453,8 @@ static int nxp_c45_ptp_settime64(struct ptp_clock_info *ptp, static int nxp_c45_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { struct nxp_c45_phy *priv = container_of(ptp, struct nxp_c45_phy, caps); + const struct nxp_c45_phy_data *data = nxp_c45_get_data(priv->phydev); + const struct nxp_c45_regmap *regmap = data->regmap; s32 ppb = scaled_ppm_to_ppb(scaled_ppm); u64 subns_inc_val; bool inc; @@ -315,16 +463,18 @@ static int nxp_c45_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) inc = ppb >= 0; ppb = abs(ppb); - subns_inc_val = PPM_TO_SUBNS_INC(ppb); + subns_inc_val = PPM_TO_SUBNS_INC(ppb, data->ptp_clk_period); - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_RATE_ADJ_SUBNS_0, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, + regmap->vend1_rate_adj_subns_0, subns_inc_val); subns_inc_val >>= 16; subns_inc_val |= CLK_RATE_ADJ_LD; if (inc) subns_inc_val |= CLK_RATE_ADJ_DIR; - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_RATE_ADJ_SUBNS_1, + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, + regmap->vend1_rate_adj_subns_1, subns_inc_val); mutex_unlock(&priv->ptp_lock); @@ -365,19 +515,88 @@ static bool nxp_c45_match_ts(struct ptp_header *header, header->domain_number == hwts->domain_number; } -static void nxp_c45_get_extts(struct nxp_c45_phy *priv, +static bool nxp_c45_get_extts(struct nxp_c45_phy *priv, struct timespec64 *extts) { + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); + extts->tv_nsec = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EXT_TRG_TS_DATA_0); + regmap->vend1_ext_trg_data_0); extts->tv_nsec |= phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EXT_TRG_TS_DATA_1) << 16; + regmap->vend1_ext_trg_data_1) << 16; extts->tv_sec = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EXT_TRG_TS_DATA_2); + regmap->vend1_ext_trg_data_2); extts->tv_sec |= phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EXT_TRG_TS_DATA_3) << 16; - phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_EXT_TRG_TS_CTRL, - RING_DONE); + regmap->vend1_ext_trg_data_3) << 16; + phy_write_mmd(priv->phydev, MDIO_MMD_VEND1, + regmap->vend1_ext_trg_ctrl, RING_DONE); + + return true; +} + +static bool tja1120_extts_is_valid(struct phy_device *phydev) +{ + bool valid; + int reg; + + reg = phy_read_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_VEND1_PTP_TRIG_DATA_S); + valid = !!(reg & TJA1120_TS_VALID); + + return valid; +} + +static bool tja1120_get_extts(struct nxp_c45_phy *priv, + struct timespec64 *extts) +{ + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); + struct phy_device *phydev = priv->phydev; + bool more_ts; + bool valid; + u16 reg; + + reg = phy_read_mmd(phydev, MDIO_MMD_VEND1, + regmap->vend1_ext_trg_ctrl); + more_ts = !!(reg & TJA1120_MORE_TS); + + valid = tja1120_extts_is_valid(phydev); + if (!valid) { + if (!more_ts) + goto tja1120_get_extts_out; + + /* Bug workaround for TJA1120 engineering samples: move the new + * timestamp from the FIFO to the buffer. + */ + phy_write_mmd(phydev, MDIO_MMD_VEND1, + regmap->vend1_ext_trg_ctrl, RING_DONE); + valid = tja1120_extts_is_valid(phydev); + if (!valid) + goto tja1120_get_extts_out; + } + + nxp_c45_get_extts(priv, extts); +tja1120_get_extts_out: + return valid; +} + +static void nxp_c45_read_egress_ts(struct nxp_c45_phy *priv, + struct nxp_c45_hwts *hwts) +{ + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); + struct phy_device *phydev = priv->phydev; + + hwts->domain_number = + nxp_c45_read_reg_field(phydev, ®map->domain_number); + hwts->msg_type = + nxp_c45_read_reg_field(phydev, ®map->msg_type); + hwts->sequence_id = + nxp_c45_read_reg_field(phydev, ®map->sequence_id); + hwts->nsec = + nxp_c45_read_reg_field(phydev, ®map->nsec_15_0); + hwts->nsec |= + nxp_c45_read_reg_field(phydev, ®map->nsec_29_16) << 16; + hwts->sec = nxp_c45_read_reg_field(phydev, ®map->sec_1_0); + hwts->sec |= nxp_c45_read_reg_field(phydev, ®map->sec_4_2) << 2; } static bool nxp_c45_get_hwtxts(struct nxp_c45_phy *priv, @@ -394,22 +613,56 @@ static bool nxp_c45_get_hwtxts(struct nxp_c45_phy *priv, if (!valid) goto nxp_c45_get_hwtxts_out; - hwts->domain_number = reg; - hwts->msg_type = (reg & RING_DATA_0_MSG_TYPE) >> 8; - hwts->sec = (reg & RING_DATA_0_SEC_4_2) >> 10; - hwts->sequence_id = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EGR_RING_DATA_1_SEQ_ID); - hwts->nsec = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_EGR_RING_DATA_2_NSEC_15_0); - reg = phy_read_mmd(priv->phydev, MDIO_MMD_VEND1, VEND1_EGR_RING_DATA_3); - hwts->nsec |= (reg & RING_DATA_3_NSEC_29_16) << 16; - hwts->sec |= (reg & RING_DATA_3_SEC_1_0) >> 14; - + nxp_c45_read_egress_ts(priv, hwts); nxp_c45_get_hwtxts_out: mutex_unlock(&priv->ptp_lock); return valid; } +static bool tja1120_egress_ts_is_valid(struct phy_device *phydev) +{ + bool valid; + u16 reg; + + reg = phy_read_mmd(phydev, MDIO_MMD_VEND1, TJA1120_EGRESS_TS_DATA_S); + valid = !!(reg & TJA1120_TS_VALID); + + return valid; +} + +static bool tja1120_get_hwtxts(struct nxp_c45_phy *priv, + struct nxp_c45_hwts *hwts) +{ + struct phy_device *phydev = priv->phydev; + bool more_ts; + bool valid; + u16 reg; + + mutex_lock(&priv->ptp_lock); + reg = phy_read_mmd(phydev, MDIO_MMD_VEND1, TJA1120_EGRESS_TS_END); + more_ts = !!(reg & TJA1120_MORE_TS); + valid = tja1120_egress_ts_is_valid(phydev); + if (!valid) { + if (!more_ts) + goto tja1120_get_hwtxts_out; + + /* Bug workaround for TJA1120 engineering samples: move the + * new timestamp from the FIFO to the buffer. + */ + phy_write_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_EGRESS_TS_END, TJA1120_TS_VALID); + valid = tja1120_egress_ts_is_valid(phydev); + if (!valid) + goto tja1120_get_hwtxts_out; + } + nxp_c45_read_egress_ts(priv, hwts); + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, TJA1120_EGRESS_TS_DATA_S, + TJA1120_TS_VALID); +tja1120_get_hwtxts_out: + mutex_unlock(&priv->ptp_lock); + return valid; +} + static void nxp_c45_process_txts(struct nxp_c45_phy *priv, struct nxp_c45_hwts *txts) { @@ -448,6 +701,7 @@ static void nxp_c45_process_txts(struct nxp_c45_phy *priv, static long nxp_c45_do_aux_work(struct ptp_clock_info *ptp) { struct nxp_c45_phy *priv = container_of(ptp, struct nxp_c45_phy, caps); + const struct nxp_c45_phy_data *data = nxp_c45_get_data(priv->phydev); bool poll_txts = nxp_c45_poll_txts(priv->phydev); struct skb_shared_hwtstamps *shhwtstamps_rx; struct ptp_clock_event event; @@ -455,12 +709,12 @@ static long nxp_c45_do_aux_work(struct ptp_clock_info *ptp) bool reschedule = false; struct timespec64 ts; struct sk_buff *skb; - bool txts_valid; + bool ts_valid; u32 ts_raw; while (!skb_queue_empty_lockless(&priv->tx_queue) && poll_txts) { - txts_valid = nxp_c45_get_hwtxts(priv, &hwts); - if (unlikely(!txts_valid)) { + ts_valid = data->get_egressts(priv, &hwts); + if (unlikely(!ts_valid)) { /* Still more skbs in the queue */ reschedule = true; break; @@ -482,8 +736,8 @@ static long nxp_c45_do_aux_work(struct ptp_clock_info *ptp) } if (priv->extts) { - nxp_c45_get_extts(priv, &ts); - if (timespec64_compare(&ts, &priv->extts_ts) != 0) { + ts_valid = data->get_extts(priv, &ts); + if (ts_valid && timespec64_compare(&ts, &priv->extts_ts) != 0) { priv->extts_ts = ts; event.index = priv->extts_index; event.type = PTP_CLOCK_EXTTS; @@ -508,6 +762,7 @@ static void nxp_c45_gpio_config(struct nxp_c45_phy *priv, static int nxp_c45_perout_enable(struct nxp_c45_phy *priv, struct ptp_perout_request *perout, int on) { + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(priv->phydev); struct phy_device *phydev = priv->phydev; int pin; @@ -519,10 +774,10 @@ static int nxp_c45_perout_enable(struct nxp_c45_phy *priv, return pin; if (!on) { - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PTP_CONFIG, - PPS_OUT_EN); - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PTP_CONFIG, - PPS_OUT_POL); + nxp_c45_clear_reg_field(priv->phydev, + ®map->pps_enable); + nxp_c45_clear_reg_field(priv->phydev, + ®map->pps_polarity); nxp_c45_gpio_config(priv, pin, GPIO_DISABLE); @@ -551,23 +806,62 @@ static int nxp_c45_perout_enable(struct nxp_c45_phy *priv, } if (perout->phase.nsec == 0) - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, - VEND1_PTP_CONFIG, PPS_OUT_POL); + nxp_c45_clear_reg_field(priv->phydev, + ®map->pps_polarity); else - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, - VEND1_PTP_CONFIG, PPS_OUT_POL); + nxp_c45_set_reg_field(priv->phydev, + ®map->pps_polarity); } nxp_c45_gpio_config(priv, pin, GPIO_PPS_OUT_CFG); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PTP_CONFIG, PPS_OUT_EN); + nxp_c45_set_reg_field(priv->phydev, ®map->pps_enable); return 0; } +static void nxp_c45_set_rising_or_falling(struct phy_device *phydev, + struct ptp_extts_request *extts) +{ + if (extts->flags & PTP_RISING_EDGE) + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PTP_CONFIG, EXT_TRG_EDGE); + + if (extts->flags & PTP_FALLING_EDGE) + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PTP_CONFIG, EXT_TRG_EDGE); +} + +static void nxp_c45_set_rising_and_falling(struct phy_device *phydev, + struct ptp_extts_request *extts) +{ + /* PTP_EXTTS_REQUEST may have only the PTP_ENABLE_FEATURE flag set. In + * this case external ts will be enabled on rising edge. + */ + if (extts->flags & PTP_RISING_EDGE || + extts->flags == PTP_ENABLE_FEATURE) + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_SYNC_TRIG_FILTER, + PTP_TRIG_RISE_TS); + else + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_SYNC_TRIG_FILTER, + PTP_TRIG_RISE_TS); + + if (extts->flags & PTP_FALLING_EDGE) + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_SYNC_TRIG_FILTER, + PTP_TRIG_FALLING_TS); + else + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_SYNC_TRIG_FILTER, + PTP_TRIG_FALLING_TS); +} + static int nxp_c45_extts_enable(struct nxp_c45_phy *priv, struct ptp_extts_request *extts, int on) { + const struct nxp_c45_phy_data *data = nxp_c45_get_data(priv->phydev); int pin; if (extts->flags & ~(PTP_ENABLE_FEATURE | @@ -578,7 +872,8 @@ static int nxp_c45_extts_enable(struct nxp_c45_phy *priv, /* Sampling on both edges is not supported */ if ((extts->flags & PTP_RISING_EDGE) && - (extts->flags & PTP_FALLING_EDGE)) + (extts->flags & PTP_FALLING_EDGE) && + !data->ext_ts_both_edges) return -EOPNOTSUPP; pin = ptp_find_pin(priv->ptp_clock, PTP_PF_EXTTS, extts->index); @@ -592,13 +887,10 @@ static int nxp_c45_extts_enable(struct nxp_c45_phy *priv, return 0; } - if (extts->flags & PTP_RISING_EDGE) - phy_clear_bits_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_PTP_CONFIG, EXT_TRG_EDGE); - - if (extts->flags & PTP_FALLING_EDGE) - phy_set_bits_mmd(priv->phydev, MDIO_MMD_VEND1, - VEND1_PTP_CONFIG, EXT_TRG_EDGE); + if (data->ext_ts_both_edges) + nxp_c45_set_rising_and_falling(priv->phydev, extts); + else + nxp_c45_set_rising_or_falling(priv->phydev, extts); nxp_c45_gpio_config(priv, pin, GPIO_EXTTS_OUT_CFG); priv->extts = true; @@ -735,6 +1027,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, struct nxp_c45_phy *priv = container_of(mii_ts, struct nxp_c45_phy, mii_ts); struct phy_device *phydev = priv->phydev; + const struct nxp_c45_phy_data *data; struct hwtstamp_config cfg; if (copy_from_user(&cfg, ifreq->ifr_data, sizeof(cfg))) @@ -743,6 +1036,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, if (cfg.tx_type < 0 || cfg.tx_type > HWTSTAMP_TX_ON) return -ERANGE; + data = nxp_c45_get_data(phydev); priv->hwts_tx = cfg.tx_type; switch (cfg.rx_filter) { @@ -760,27 +1054,24 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, } if (priv->hwts_rx || priv->hwts_tx) { - phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_EVENT_MSG_FILT, + phy_write_mmd(phydev, MDIO_MMD_VEND1, + data->regmap->vend1_event_msg_filt, EVENT_MSG_FILT_ALL); - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, - VEND1_PORT_PTP_CONTROL, - PORT_PTP_CONTROL_BYPASS); + data->ptp_enable(phydev, true); } else { - phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_EVENT_MSG_FILT, + phy_write_mmd(phydev, MDIO_MMD_VEND1, + data->regmap->vend1_event_msg_filt, EVENT_MSG_FILT_NONE); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PORT_PTP_CONTROL, - PORT_PTP_CONTROL_BYPASS); + data->ptp_enable(phydev, false); } if (nxp_c45_poll_txts(priv->phydev)) goto nxp_c45_no_ptp_irq; if (priv->hwts_tx) - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, - VEND1_PTP_IRQ_EN, PTP_IRQ_EGR_TS); + nxp_c45_set_reg_field(phydev, &data->regmap->irq_egr_ts_en); else - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, - VEND1_PTP_IRQ_EN, PTP_IRQ_EGR_TS); + nxp_c45_clear_reg_field(phydev, &data->regmap->irq_egr_ts_en); nxp_c45_no_ptp_irq: return copy_to_user(ifreq->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; @@ -805,63 +1096,100 @@ static int nxp_c45_ts_info(struct mii_timestamper *mii_ts, return 0; } -static const struct nxp_c45_phy_stats nxp_c45_hw_stats[] = { - { "phy_symbol_error_cnt", MDIO_MMD_VEND1, - VEND1_SYMBOL_ERROR_COUNTER, 0, GENMASK(15, 0) }, - { "phy_link_status_drop_cnt", MDIO_MMD_VEND1, - VEND1_LINK_DROP_COUNTER, 8, GENMASK(13, 8) }, - { "phy_link_availability_drop_cnt", MDIO_MMD_VEND1, - VEND1_LINK_DROP_COUNTER, 0, GENMASK(5, 0) }, - { "phy_link_loss_cnt", MDIO_MMD_VEND1, - VEND1_LINK_LOSSES_AND_FAILURES, 10, GENMASK(15, 10) }, - { "phy_link_failure_cnt", MDIO_MMD_VEND1, - VEND1_LINK_LOSSES_AND_FAILURES, 0, GENMASK(9, 0) }, - { "r_good_frame_cnt", MDIO_MMD_VEND1, - VEND1_R_GOOD_FRAME_CNT, 0, GENMASK(15, 0) }, - { "r_bad_frame_cnt", MDIO_MMD_VEND1, - VEND1_R_BAD_FRAME_CNT, 0, GENMASK(15, 0) }, - { "r_rxer_frame_cnt", MDIO_MMD_VEND1, - VEND1_R_RXER_FRAME_CNT, 0, GENMASK(15, 0) }, - { "rx_preamble_count", MDIO_MMD_VEND1, - VEND1_RX_PREAMBLE_COUNT, 0, GENMASK(5, 0) }, - { "tx_preamble_count", MDIO_MMD_VEND1, - VEND1_TX_PREAMBLE_COUNT, 0, GENMASK(5, 0) }, - { "rx_ipg_length", MDIO_MMD_VEND1, - VEND1_RX_IPG_LENGTH, 0, GENMASK(8, 0) }, - { "tx_ipg_length", MDIO_MMD_VEND1, - VEND1_TX_IPG_LENGTH, 0, GENMASK(8, 0) }, +static const struct nxp_c45_phy_stats common_hw_stats[] = { + { "phy_link_status_drop_cnt", + NXP_C45_REG_FIELD(0x8352, MDIO_MMD_VEND1, 8, 6), }, + { "phy_link_availability_drop_cnt", + NXP_C45_REG_FIELD(0x8352, MDIO_MMD_VEND1, 0, 6), }, + { "phy_link_loss_cnt", + NXP_C45_REG_FIELD(0x8353, MDIO_MMD_VEND1, 10, 6), }, + { "phy_link_failure_cnt", + NXP_C45_REG_FIELD(0x8353, MDIO_MMD_VEND1, 0, 10), }, + { "phy_symbol_error_cnt", + NXP_C45_REG_FIELD(0x8350, MDIO_MMD_VEND1, 0, 16) }, +}; + +static const struct nxp_c45_phy_stats tja1103_hw_stats[] = { + { "rx_preamble_count", + NXP_C45_REG_FIELD(0xAFCE, MDIO_MMD_VEND1, 0, 6), }, + { "tx_preamble_count", + NXP_C45_REG_FIELD(0xAFCF, MDIO_MMD_VEND1, 0, 6), }, + { "rx_ipg_length", + NXP_C45_REG_FIELD(0xAFD0, MDIO_MMD_VEND1, 0, 9), }, + { "tx_ipg_length", + NXP_C45_REG_FIELD(0xAFD1, MDIO_MMD_VEND1, 0, 9), }, +}; + +static const struct nxp_c45_phy_stats tja1120_hw_stats[] = { + { "phy_symbol_error_cnt_ext", + NXP_C45_REG_FIELD(0x8351, MDIO_MMD_VEND1, 0, 14) }, + { "tx_frames_xtd", + NXP_C45_REG_FIELD(0xACA1, MDIO_MMD_VEND1, 0, 8), }, + { "tx_frames", + NXP_C45_REG_FIELD(0xACA0, MDIO_MMD_VEND1, 0, 16), }, + { "rx_frames_xtd", + NXP_C45_REG_FIELD(0xACA3, MDIO_MMD_VEND1, 0, 8), }, + { "rx_frames", + NXP_C45_REG_FIELD(0xACA2, MDIO_MMD_VEND1, 0, 16), }, + { "tx_lost_frames_xtd", + NXP_C45_REG_FIELD(0xACA5, MDIO_MMD_VEND1, 0, 8), }, + { "tx_lost_frames", + NXP_C45_REG_FIELD(0xACA4, MDIO_MMD_VEND1, 0, 16), }, + { "rx_lost_frames_xtd", + NXP_C45_REG_FIELD(0xACA7, MDIO_MMD_VEND1, 0, 8), }, + { "rx_lost_frames", + NXP_C45_REG_FIELD(0xACA6, MDIO_MMD_VEND1, 0, 16), }, }; static int nxp_c45_get_sset_count(struct phy_device *phydev) { - return ARRAY_SIZE(nxp_c45_hw_stats); + const struct nxp_c45_phy_data *phy_data = nxp_c45_get_data(phydev); + + return ARRAY_SIZE(common_hw_stats) + (phy_data ? phy_data->n_stats : 0); } static void nxp_c45_get_strings(struct phy_device *phydev, u8 *data) { + const struct nxp_c45_phy_data *phy_data = nxp_c45_get_data(phydev); + size_t count = nxp_c45_get_sset_count(phydev); + size_t idx; size_t i; - for (i = 0; i < ARRAY_SIZE(nxp_c45_hw_stats); i++) { - strncpy(data + i * ETH_GSTRING_LEN, - nxp_c45_hw_stats[i].name, ETH_GSTRING_LEN); + for (i = 0; i < count; i++) { + if (i < ARRAY_SIZE(common_hw_stats)) { + strscpy(data + i * ETH_GSTRING_LEN, + common_hw_stats[i].name, ETH_GSTRING_LEN); + continue; + } + idx = i - ARRAY_SIZE(common_hw_stats); + strscpy(data + i * ETH_GSTRING_LEN, + phy_data->stats[idx].name, ETH_GSTRING_LEN); } } static void nxp_c45_get_stats(struct phy_device *phydev, struct ethtool_stats *stats, u64 *data) { + const struct nxp_c45_phy_data *phy_data = nxp_c45_get_data(phydev); + size_t count = nxp_c45_get_sset_count(phydev); + const struct nxp_c45_reg_field *reg_field; + size_t idx; size_t i; int ret; - for (i = 0; i < ARRAY_SIZE(nxp_c45_hw_stats); i++) { - ret = phy_read_mmd(phydev, nxp_c45_hw_stats[i].mmd, - nxp_c45_hw_stats[i].reg); - if (ret < 0) { - data[i] = U64_MAX; + for (i = 0; i < count; i++) { + if (i < ARRAY_SIZE(common_hw_stats)) { + reg_field = &common_hw_stats[i].counter; } else { - data[i] = ret & nxp_c45_hw_stats[i].mask; - data[i] >>= nxp_c45_hw_stats[i].off; + idx = i - ARRAY_SIZE(common_hw_stats); + reg_field = &phy_data->stats[idx].counter; } + + ret = nxp_c45_read_reg_field(phydev, reg_field); + if (ret < 0) + data[i] = U64_MAX; + else + data[i] = ret; } } @@ -898,8 +1226,40 @@ static int nxp_c45_config_intr(struct phy_device *phydev) VEND1_PHY_IRQ_EN, PHY_IRQ_LINK_EVENT); } +static int tja1103_config_intr(struct phy_device *phydev) +{ + int ret; + + /* We can't disable the FUSA IRQ for TJA1103, but we can clean it up. */ + ret = phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_ALWAYS_ACCESSIBLE, + FUSA_PASS); + if (ret) + return ret; + + return nxp_c45_config_intr(phydev); +} + +static int tja1120_config_intr(struct phy_device *phydev) +{ + int ret; + + if (phydev->interrupts == PHY_INTERRUPT_ENABLED) + ret = phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_GLOBAL_INFRA_IRQ_EN, + TJA1120_DEV_BOOT_DONE); + else + ret = phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_GLOBAL_INFRA_IRQ_EN, + TJA1120_DEV_BOOT_DONE); + if (ret) + return ret; + + return nxp_c45_config_intr(phydev); +} + static irqreturn_t nxp_c45_handle_interrupt(struct phy_device *phydev) { + const struct nxp_c45_phy_data *data = nxp_c45_get_data(phydev); struct nxp_c45_phy *priv = phydev->priv; irqreturn_t ret = IRQ_NONE; struct nxp_c45_hwts hwts; @@ -913,18 +1273,23 @@ static irqreturn_t nxp_c45_handle_interrupt(struct phy_device *phydev) ret = IRQ_HANDLED; } - /* There is no need for ACK. - * The irq signal will be asserted until the EGR TS FIFO will be - * emptied. - */ - irq = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_PTP_IRQ_STATUS); - if (irq & PTP_IRQ_EGR_TS) { - while (nxp_c45_get_hwtxts(priv, &hwts)) + irq = nxp_c45_read_reg_field(phydev, &data->regmap->irq_egr_ts_status); + if (irq) { + /* If ack_ptp_irq is false, the IRQ bit is self-clear and will + * be cleared when the EGR TS FIFO is empty. Otherwise, the + * IRQ bit should be cleared before reading the timestamp, + */ + if (data->ack_ptp_irq) + phy_write_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PTP_IRQ_ACK, EGR_TS_IRQ); + while (data->get_egressts(priv, &hwts)) nxp_c45_process_txts(priv, &hwts); ret = IRQ_HANDLED; } + data->nmi_handler(phydev, &ret); + return ret; } @@ -945,24 +1310,30 @@ static int nxp_c45_soft_reset(struct phy_device *phydev) static int nxp_c45_cable_test_start(struct phy_device *phydev) { - return phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_CABLE_TEST, - CABLE_TEST_ENABLE | CABLE_TEST_START); + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(phydev); + + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_FUNC_ENABLES, PHY_TEST_ENABLE); + return phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, regmap->cable_test, + CABLE_TEST_ENABLE | CABLE_TEST_START); } static int nxp_c45_cable_test_get_status(struct phy_device *phydev, bool *finished) { + const struct nxp_c45_regmap *regmap = nxp_c45_get_regmap(phydev); int ret; u8 cable_test_result; - ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_CABLE_TEST); - if (!(ret & CABLE_TEST_VALID)) { + ret = nxp_c45_read_reg_field(phydev, ®map->cable_test_valid); + if (!ret) { *finished = false; return 0; } *finished = true; - cable_test_result = ret & GENMASK(2, 0); + cable_test_result = nxp_c45_read_reg_field(phydev, + ®map->cable_test_result); switch (cable_test_result) { case CABLE_TEST_OK: @@ -982,78 +1353,14 @@ static int nxp_c45_cable_test_get_status(struct phy_device *phydev, ETHTOOL_A_CABLE_RESULT_CODE_UNSPEC); } - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_CABLE_TEST, + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, regmap->cable_test, CABLE_TEST_ENABLE); + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_FUNC_ENABLES, PHY_TEST_ENABLE); return nxp_c45_start_op(phydev); } -static int nxp_c45_setup_master_slave(struct phy_device *phydev) -{ - switch (phydev->master_slave_set) { - case MASTER_SLAVE_CFG_MASTER_FORCE: - case MASTER_SLAVE_CFG_MASTER_PREFERRED: - phy_write_mmd(phydev, MDIO_MMD_PMAPMD, PMAPMD_B100T1_PMAPMD_CTL, - MASTER_MODE); - break; - case MASTER_SLAVE_CFG_SLAVE_PREFERRED: - case MASTER_SLAVE_CFG_SLAVE_FORCE: - phy_write_mmd(phydev, MDIO_MMD_PMAPMD, PMAPMD_B100T1_PMAPMD_CTL, - SLAVE_MODE); - break; - case MASTER_SLAVE_CFG_UNKNOWN: - case MASTER_SLAVE_CFG_UNSUPPORTED: - return 0; - default: - phydev_warn(phydev, "Unsupported Master/Slave mode\n"); - return -EOPNOTSUPP; - } - - return 0; -} - -static int nxp_c45_read_master_slave(struct phy_device *phydev) -{ - int reg; - - phydev->master_slave_get = MASTER_SLAVE_CFG_UNKNOWN; - phydev->master_slave_state = MASTER_SLAVE_STATE_UNKNOWN; - - reg = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, PMAPMD_B100T1_PMAPMD_CTL); - if (reg < 0) - return reg; - - if (reg & B100T1_PMAPMD_MASTER) { - phydev->master_slave_get = MASTER_SLAVE_CFG_MASTER_FORCE; - phydev->master_slave_state = MASTER_SLAVE_STATE_MASTER; - } else { - phydev->master_slave_get = MASTER_SLAVE_CFG_SLAVE_FORCE; - phydev->master_slave_state = MASTER_SLAVE_STATE_SLAVE; - } - - return 0; -} - -static int nxp_c45_config_aneg(struct phy_device *phydev) -{ - return nxp_c45_setup_master_slave(phydev); -} - -static int nxp_c45_read_status(struct phy_device *phydev) -{ - int ret; - - ret = genphy_c45_read_status(phydev); - if (ret) - return ret; - - ret = nxp_c45_read_master_slave(phydev); - if (ret) - return ret; - - return 0; -} - static int nxp_c45_get_sqi(struct phy_device *phydev) { int reg; @@ -1067,6 +1374,19 @@ static int nxp_c45_get_sqi(struct phy_device *phydev) return reg; } +static void tja1120_link_change_notify(struct phy_device *phydev) +{ + /* Bug workaround for TJA1120 enegineering samples: fix egress + * timestamps lost after link recovery. + */ + if (phydev->state == PHY_NOLINK) { + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_EPHY_RESETS, EPHY_PCS_RESET); + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_EPHY_RESETS, EPHY_PCS_RESET); + } +} + static int nxp_c45_get_sqi_max(struct phy_device *phydev) { return MAX_SQI; @@ -1087,6 +1407,28 @@ static int nxp_c45_check_delay(struct phy_device *phydev, u32 delay) return 0; } +static void nxp_c45_counters_enable(struct phy_device *phydev) +{ + const struct nxp_c45_phy_data *data = nxp_c45_get_data(phydev); + + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_LINK_DROP_COUNTER, + COUNTER_EN); + + data->counters_enable(phydev); +} + +static void nxp_c45_ptp_init(struct phy_device *phydev) +{ + const struct nxp_c45_phy_data *data = nxp_c45_get_data(phydev); + + phy_write_mmd(phydev, MDIO_MMD_VEND1, + data->regmap->vend1_ptp_clk_period, + data->ptp_clk_period); + nxp_c45_clear_reg_field(phydev, &data->regmap->ltc_lock_ctrl); + + data->ptp_init(phydev); +} + static u64 nxp_c45_get_phase_shift(u64 phase_offset_raw) { /* The delay in degree phase is 73.8 + phase_offset_raw * 0.9. @@ -1264,35 +1606,26 @@ static int nxp_c45_config_init(struct phy_device *phydev) phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PHY_CONFIG, PHY_CONFIG_AUTO); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_LINK_DROP_COUNTER, - COUNTER_EN); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_PREAMBLE_COUNT, - COUNTER_EN); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_TX_PREAMBLE_COUNT, - COUNTER_EN); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_IPG_LENGTH, - COUNTER_EN); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_TX_IPG_LENGTH, - COUNTER_EN); - ret = nxp_c45_set_phy_mode(phydev); if (ret) return ret; phydev->autoneg = AUTONEG_DISABLE; - phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_PTP_CLK_PERIOD, - PTP_CLK_PERIOD_100BT1); - phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_HW_LTC_LOCK_CTRL, - HW_LTC_LOCK_EN); - phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_TS_INSRT_CTRL, - RX_TS_INSRT_MODE2); - phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PORT_FUNC_ENABLES, - PTP_ENABLE); + nxp_c45_counters_enable(phydev); + nxp_c45_ptp_init(phydev); return nxp_c45_start_op(phydev); } +static int nxp_c45_get_features(struct phy_device *phydev) +{ + linkmode_set_bit(ETHTOOL_LINK_MODE_TP_BIT, phydev->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_MII_BIT, phydev->supported); + + return genphy_c45_pma_read_abilities(phydev); +} + static int nxp_c45_probe(struct phy_device *phydev) { struct nxp_c45_phy *priv; @@ -1348,18 +1681,274 @@ static void nxp_c45_remove(struct phy_device *phydev) skb_queue_purge(&priv->rx_queue); } +static void tja1103_counters_enable(struct phy_device *phydev) +{ + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_PREAMBLE_COUNT, + COUNTER_EN); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_TX_PREAMBLE_COUNT, + COUNTER_EN); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_IPG_LENGTH, + COUNTER_EN); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_TX_IPG_LENGTH, + COUNTER_EN); +} + +static void tja1103_ptp_init(struct phy_device *phydev) +{ + phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_RX_TS_INSRT_CTRL, + TJA1103_RX_TS_INSRT_MODE2); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_PORT_FUNC_ENABLES, + PTP_ENABLE); +} + +static void tja1103_ptp_enable(struct phy_device *phydev, bool enable) +{ + if (enable) + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_PTP_CONTROL, + PORT_PTP_CONTROL_BYPASS); + else + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_PTP_CONTROL, + PORT_PTP_CONTROL_BYPASS); +} + +static void tja1103_nmi_handler(struct phy_device *phydev, + irqreturn_t *irq_status) +{ + int ret; + + ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, + VEND1_ALWAYS_ACCESSIBLE); + if (ret & FUSA_PASS) { + phy_write_mmd(phydev, MDIO_MMD_VEND1, + VEND1_ALWAYS_ACCESSIBLE, + FUSA_PASS); + *irq_status = IRQ_HANDLED; + } +} + +static const struct nxp_c45_regmap tja1103_regmap = { + .vend1_ptp_clk_period = 0x1104, + .vend1_event_msg_filt = 0x1148, + .pps_enable = + NXP_C45_REG_FIELD(0x1102, MDIO_MMD_VEND1, 3, 1), + .pps_polarity = + NXP_C45_REG_FIELD(0x1102, MDIO_MMD_VEND1, 2, 1), + .ltc_lock_ctrl = + NXP_C45_REG_FIELD(0x1115, MDIO_MMD_VEND1, 0, 1), + .ltc_read = + NXP_C45_REG_FIELD(0x1105, MDIO_MMD_VEND1, 2, 1), + .ltc_write = + NXP_C45_REG_FIELD(0x1105, MDIO_MMD_VEND1, 0, 1), + .vend1_ltc_wr_nsec_0 = 0x1106, + .vend1_ltc_wr_nsec_1 = 0x1107, + .vend1_ltc_wr_sec_0 = 0x1108, + .vend1_ltc_wr_sec_1 = 0x1109, + .vend1_ltc_rd_nsec_0 = 0x110A, + .vend1_ltc_rd_nsec_1 = 0x110B, + .vend1_ltc_rd_sec_0 = 0x110C, + .vend1_ltc_rd_sec_1 = 0x110D, + .vend1_rate_adj_subns_0 = 0x110F, + .vend1_rate_adj_subns_1 = 0x1110, + .irq_egr_ts_en = + NXP_C45_REG_FIELD(0x1131, MDIO_MMD_VEND1, 0, 1), + .irq_egr_ts_status = + NXP_C45_REG_FIELD(0x1132, MDIO_MMD_VEND1, 0, 1), + .domain_number = + NXP_C45_REG_FIELD(0x114E, MDIO_MMD_VEND1, 0, 8), + .msg_type = + NXP_C45_REG_FIELD(0x114E, MDIO_MMD_VEND1, 8, 4), + .sequence_id = + NXP_C45_REG_FIELD(0x114F, MDIO_MMD_VEND1, 0, 16), + .sec_1_0 = + NXP_C45_REG_FIELD(0x1151, MDIO_MMD_VEND1, 14, 2), + .sec_4_2 = + NXP_C45_REG_FIELD(0x114E, MDIO_MMD_VEND1, 12, 3), + .nsec_15_0 = + NXP_C45_REG_FIELD(0x1150, MDIO_MMD_VEND1, 0, 16), + .nsec_29_16 = + NXP_C45_REG_FIELD(0x1151, MDIO_MMD_VEND1, 0, 14), + .vend1_ext_trg_data_0 = 0x1121, + .vend1_ext_trg_data_1 = 0x1122, + .vend1_ext_trg_data_2 = 0x1123, + .vend1_ext_trg_data_3 = 0x1124, + .vend1_ext_trg_ctrl = 0x1126, + .cable_test = 0x8330, + .cable_test_valid = + NXP_C45_REG_FIELD(0x8330, MDIO_MMD_VEND1, 13, 1), + .cable_test_result = + NXP_C45_REG_FIELD(0x8330, MDIO_MMD_VEND1, 0, 3), +}; + +static const struct nxp_c45_phy_data tja1103_phy_data = { + .regmap = &tja1103_regmap, + .stats = tja1103_hw_stats, + .n_stats = ARRAY_SIZE(tja1103_hw_stats), + .ptp_clk_period = PTP_CLK_PERIOD_100BT1, + .ext_ts_both_edges = false, + .ack_ptp_irq = false, + .counters_enable = tja1103_counters_enable, + .get_egressts = nxp_c45_get_hwtxts, + .get_extts = nxp_c45_get_extts, + .ptp_init = tja1103_ptp_init, + .ptp_enable = tja1103_ptp_enable, + .nmi_handler = tja1103_nmi_handler, +}; + +static void tja1120_counters_enable(struct phy_device *phydev) +{ + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_SYMBOL_ERROR_CNT_XTD, + EXTENDED_CNT_EN); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_MONITOR_STATUS, + MONITOR_RESET); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_MONITOR_CONFIG, + ALL_FRAMES_CNT_EN | LOST_FRAMES_CNT_EN); +} + +static void tja1120_ptp_init(struct phy_device *phydev) +{ + phy_write_mmd(phydev, MDIO_MMD_VEND1, TJA1120_RX_TS_INSRT_CTRL, + TJA1120_RX_TS_INSRT_EN | TJA1120_TS_INSRT_MODE); + phy_write_mmd(phydev, MDIO_MMD_VEND1, TJA1120_VEND1_EXT_TS_MODE, + TJA1120_TS_INSRT_MODE); + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_DEVICE_CONFIG, + PTP_ENABLE); +} + +static void tja1120_ptp_enable(struct phy_device *phydev, bool enable) +{ + if (enable) + phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_FUNC_ENABLES, + PTP_ENABLE); + else + phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, + VEND1_PORT_FUNC_ENABLES, + PTP_ENABLE); +} + +static void tja1120_nmi_handler(struct phy_device *phydev, + irqreturn_t *irq_status) +{ + int ret; + + ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_GLOBAL_INFRA_IRQ_STATUS); + if (ret & TJA1120_DEV_BOOT_DONE) { + phy_write_mmd(phydev, MDIO_MMD_VEND1, + TJA1120_GLOBAL_INFRA_IRQ_ACK, + TJA1120_DEV_BOOT_DONE); + *irq_status = IRQ_HANDLED; + } +} + +static const struct nxp_c45_regmap tja1120_regmap = { + .vend1_ptp_clk_period = 0x1020, + .vend1_event_msg_filt = 0x9010, + .pps_enable = + NXP_C45_REG_FIELD(0x1006, MDIO_MMD_VEND1, 4, 1), + .pps_polarity = + NXP_C45_REG_FIELD(0x1006, MDIO_MMD_VEND1, 5, 1), + .ltc_lock_ctrl = + NXP_C45_REG_FIELD(0x1006, MDIO_MMD_VEND1, 2, 1), + .ltc_read = + NXP_C45_REG_FIELD(0x1000, MDIO_MMD_VEND1, 1, 1), + .ltc_write = + NXP_C45_REG_FIELD(0x1000, MDIO_MMD_VEND1, 2, 1), + .vend1_ltc_wr_nsec_0 = 0x1040, + .vend1_ltc_wr_nsec_1 = 0x1041, + .vend1_ltc_wr_sec_0 = 0x1042, + .vend1_ltc_wr_sec_1 = 0x1043, + .vend1_ltc_rd_nsec_0 = 0x1048, + .vend1_ltc_rd_nsec_1 = 0x1049, + .vend1_ltc_rd_sec_0 = 0x104A, + .vend1_ltc_rd_sec_1 = 0x104B, + .vend1_rate_adj_subns_0 = 0x1030, + .vend1_rate_adj_subns_1 = 0x1031, + .irq_egr_ts_en = + NXP_C45_REG_FIELD(0x900A, MDIO_MMD_VEND1, 1, 1), + .irq_egr_ts_status = + NXP_C45_REG_FIELD(0x900C, MDIO_MMD_VEND1, 1, 1), + .domain_number = + NXP_C45_REG_FIELD(0x9061, MDIO_MMD_VEND1, 8, 8), + .msg_type = + NXP_C45_REG_FIELD(0x9061, MDIO_MMD_VEND1, 4, 4), + .sequence_id = + NXP_C45_REG_FIELD(0x9062, MDIO_MMD_VEND1, 0, 16), + .sec_1_0 = + NXP_C45_REG_FIELD(0x9065, MDIO_MMD_VEND1, 0, 2), + .sec_4_2 = + NXP_C45_REG_FIELD(0x9065, MDIO_MMD_VEND1, 2, 3), + .nsec_15_0 = + NXP_C45_REG_FIELD(0x9063, MDIO_MMD_VEND1, 0, 16), + .nsec_29_16 = + NXP_C45_REG_FIELD(0x9064, MDIO_MMD_VEND1, 0, 14), + .vend1_ext_trg_data_0 = 0x1071, + .vend1_ext_trg_data_1 = 0x1072, + .vend1_ext_trg_data_2 = 0x1073, + .vend1_ext_trg_data_3 = 0x1074, + .vend1_ext_trg_ctrl = 0x1075, + .cable_test = 0x8360, + .cable_test_valid = + NXP_C45_REG_FIELD(0x8361, MDIO_MMD_VEND1, 15, 1), + .cable_test_result = + NXP_C45_REG_FIELD(0x8361, MDIO_MMD_VEND1, 0, 3), +}; + +static const struct nxp_c45_phy_data tja1120_phy_data = { + .regmap = &tja1120_regmap, + .stats = tja1120_hw_stats, + .n_stats = ARRAY_SIZE(tja1120_hw_stats), + .ptp_clk_period = PTP_CLK_PERIOD_1000BT1, + .ext_ts_both_edges = true, + .ack_ptp_irq = true, + .counters_enable = tja1120_counters_enable, + .get_egressts = tja1120_get_hwtxts, + .get_extts = tja1120_get_extts, + .ptp_init = tja1120_ptp_init, + .ptp_enable = tja1120_ptp_enable, + .nmi_handler = tja1120_nmi_handler, +}; + static struct phy_driver nxp_c45_driver[] = { { PHY_ID_MATCH_MODEL(PHY_ID_TJA_1103), .name = "NXP C45 TJA1103", - .features = PHY_BASIC_T1_FEATURES, + .get_features = nxp_c45_get_features, + .driver_data = &tja1103_phy_data, + .probe = nxp_c45_probe, + .soft_reset = nxp_c45_soft_reset, + .config_aneg = genphy_c45_config_aneg, + .config_init = nxp_c45_config_init, + .config_intr = tja1103_config_intr, + .handle_interrupt = nxp_c45_handle_interrupt, + .read_status = genphy_c45_read_status, + .suspend = genphy_c45_pma_suspend, + .resume = genphy_c45_pma_resume, + .get_sset_count = nxp_c45_get_sset_count, + .get_strings = nxp_c45_get_strings, + .get_stats = nxp_c45_get_stats, + .cable_test_start = nxp_c45_cable_test_start, + .cable_test_get_status = nxp_c45_cable_test_get_status, + .set_loopback = genphy_c45_loopback, + .get_sqi = nxp_c45_get_sqi, + .get_sqi_max = nxp_c45_get_sqi_max, + .remove = nxp_c45_remove, + }, + { + PHY_ID_MATCH_MODEL(PHY_ID_TJA_1120), + .name = "NXP C45 TJA1120", + .get_features = nxp_c45_get_features, + .driver_data = &tja1120_phy_data, .probe = nxp_c45_probe, .soft_reset = nxp_c45_soft_reset, - .config_aneg = nxp_c45_config_aneg, + .config_aneg = genphy_c45_config_aneg, .config_init = nxp_c45_config_init, - .config_intr = nxp_c45_config_intr, + .config_intr = tja1120_config_intr, .handle_interrupt = nxp_c45_handle_interrupt, - .read_status = nxp_c45_read_status, + .read_status = genphy_c45_read_status, + .link_change_notify = tja1120_link_change_notify, .suspend = genphy_c45_pma_suspend, .resume = genphy_c45_pma_resume, .get_sset_count = nxp_c45_get_sset_count, @@ -1378,6 +1967,7 @@ module_phy_driver(nxp_c45_driver); static struct mdio_device_id __maybe_unused nxp_c45_tbl[] = { { PHY_ID_MATCH_MODEL(PHY_ID_TJA_1103) }, + { PHY_ID_MATCH_MODEL(PHY_ID_TJA_1120) }, { /*sentinel*/ }, }; diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c index 93ed07223377..8e6fd4962c48 100644 --- a/drivers/net/phy/phy-c45.c +++ b/drivers/net/phy/phy-c45.c @@ -108,7 +108,7 @@ EXPORT_SYMBOL_GPL(genphy_c45_pma_baset1_setup_master_slave); */ int genphy_c45_pma_setup_forced(struct phy_device *phydev) { - int ctrl1, ctrl2, ret; + int bt1_ctrl, ctrl1, ctrl2, ret; /* Half duplex is not supported */ if (phydev->duplex != DUPLEX_FULL) @@ -176,6 +176,15 @@ int genphy_c45_pma_setup_forced(struct phy_device *phydev) ret = genphy_c45_pma_baset1_setup_master_slave(phydev); if (ret < 0) return ret; + + bt1_ctrl = 0; + if (phydev->speed == SPEED_1000) + bt1_ctrl = MDIO_PMA_PMD_BT1_CTRL_STRAP_B1000; + + ret = phy_modify_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_PMD_BT1_CTRL, + MDIO_PMA_PMD_BT1_CTRL_STRAP, bt1_ctrl); + if (ret < 0) + return ret; } return genphy_c45_an_disable_aneg(phydev); @@ -873,6 +882,44 @@ int genphy_c45_an_config_eee_aneg(struct phy_device *phydev) } /** + * genphy_c45_pma_baset1_read_abilities - read supported baset1 link modes from PMA + * @phydev: target phy_device struct + * + * Read the supported link modes from the extended BASE-T1 ability register + */ +int genphy_c45_pma_baset1_read_abilities(struct phy_device *phydev) +{ + int val; + + val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_PMD_BT1); + if (val < 0) + return val; + + linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT1L_Full_BIT, + phydev->supported, + val & MDIO_PMA_PMD_BT1_B10L_ABLE); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT1_Full_BIT, + phydev->supported, + val & MDIO_PMA_PMD_BT1_B100_ABLE); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT1_Full_BIT, + phydev->supported, + val & MDIO_PMA_PMD_BT1_B1000_ABLE); + + val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_T1_STAT); + if (val < 0) + return val; + + linkmode_mod_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + phydev->supported, + val & MDIO_AN_STAT1_ABLE); + + return 0; +} +EXPORT_SYMBOL_GPL(genphy_c45_pma_baset1_read_abilities); + +/** * genphy_c45_pma_read_abilities - read supported link modes from PMA * @phydev: target phy_device struct * @@ -968,21 +1015,9 @@ int genphy_c45_pma_read_abilities(struct phy_device *phydev) } if (val & MDIO_PMA_EXTABLE_BT1) { - val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_PMD_BT1); + val = genphy_c45_pma_baset1_read_abilities(phydev); if (val < 0) return val; - - linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT1L_Full_BIT, - phydev->supported, - val & MDIO_PMA_PMD_BT1_B10L_ABLE); - - val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_T1_STAT); - if (val < 0) - return val; - - linkmode_mod_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, - phydev->supported, - val & MDIO_AN_STAT1_ABLE); } } diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index bdf00b2b2c1d..8aec8e83038c 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -456,6 +456,40 @@ int phy_do_ioctl_running(struct net_device *dev, struct ifreq *ifr, int cmd) EXPORT_SYMBOL(phy_do_ioctl_running); /** + * __phy_hwtstamp_get - Get hardware timestamping configuration from PHY + * + * @phydev: the PHY device structure + * @config: structure holding the timestamping configuration + * + * Query the PHY device for its current hardware timestamping configuration. + */ +int __phy_hwtstamp_get(struct phy_device *phydev, + struct kernel_hwtstamp_config *config) +{ + if (!phydev) + return -ENODEV; + + return phy_mii_ioctl(phydev, config->ifr, SIOCGHWTSTAMP); +} + +/** + * __phy_hwtstamp_set - Modify PHY hardware timestamping configuration + * + * @phydev: the PHY device structure + * @config: structure holding the timestamping configuration + * @extack: netlink extended ack structure, for error reporting + */ +int __phy_hwtstamp_set(struct phy_device *phydev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + if (!phydev) + return -ENODEV; + + return phy_mii_ioctl(phydev, config->ifr, SIOCSHWTSTAMP); +} + +/** * phy_queue_state_machine - Trigger the state machine to run soon * * @phydev: the phy_device struct diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 61921d4dbb13..e19c4fee8d22 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -27,9 +27,11 @@ #include <linux/of.h> #include <linux/netdevice.h> #include <linux/phy.h> +#include <linux/phylib_stubs.h> #include <linux/phy_led_triggers.h> #include <linux/pse-pd/pse.h> #include <linux/property.h> +#include <linux/rtnetlink.h> #include <linux/sfp.h> #include <linux/skbuff.h> #include <linux/slab.h> @@ -3447,11 +3449,29 @@ static const struct ethtool_phy_ops phy_ethtool_phy_ops = { .start_cable_test_tdr = phy_start_cable_test_tdr, }; +static const struct phylib_stubs __phylib_stubs = { + .hwtstamp_get = __phy_hwtstamp_get, + .hwtstamp_set = __phy_hwtstamp_set, +}; + +static void phylib_register_stubs(void) +{ + phylib_stubs = &__phylib_stubs; +} + +static void phylib_unregister_stubs(void) +{ + phylib_stubs = NULL; +} + static int __init phy_init(void) { int rc; + rtnl_lock(); ethtool_set_ethtool_phy_ops(&phy_ethtool_phy_ops); + phylib_register_stubs(); + rtnl_unlock(); rc = mdio_bus_init(); if (rc) @@ -3474,7 +3494,10 @@ err_c45: err_mdio_bus: mdio_bus_exit(); err_ethtool_phy_ops: + rtnl_lock(); + phylib_unregister_stubs(); ethtool_set_ethtool_phy_ops(NULL); + rtnl_unlock(); return rc; } @@ -3484,7 +3507,10 @@ static void __exit phy_exit(void) phy_driver_unregister(&genphy_c45_driver); phy_driver_unregister(&genphy_driver); mdio_bus_exit(); + rtnl_lock(); + phylib_unregister_stubs(); ethtool_set_ethtool_phy_ops(NULL); + rtnl_unlock(); } subsys_initcall(phy_init); diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index d0aaa5cad853..4f1c8bb199e9 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -34,6 +34,10 @@ enum { PHYLINK_DISABLE_STOPPED, PHYLINK_DISABLE_LINK, PHYLINK_DISABLE_MAC_WOL, + + PCS_STATE_DOWN = 0, + PCS_STATE_STARTING, + PCS_STATE_STARTED, }; /** @@ -72,6 +76,7 @@ struct phylink { struct phylink_link_state phy_state; struct work_struct resolve; unsigned int pcs_neg_mode; + unsigned int pcs_state; bool mac_link_dropped; bool using_mac_select_pcs; @@ -993,6 +998,40 @@ static void phylink_resolve_an_pause(struct phylink_link_state *state) } } +static void phylink_pcs_pre_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + if (pcs && pcs->ops->pcs_pre_config) + pcs->ops->pcs_pre_config(pcs, interface); +} + +static int phylink_pcs_post_config(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + int err = 0; + + if (pcs && pcs->ops->pcs_post_config) + err = pcs->ops->pcs_post_config(pcs, interface); + + return err; +} + +static void phylink_pcs_disable(struct phylink_pcs *pcs) +{ + if (pcs && pcs->ops->pcs_disable) + pcs->ops->pcs_disable(pcs); +} + +static int phylink_pcs_enable(struct phylink_pcs *pcs) +{ + int err = 0; + + if (pcs && pcs->ops->pcs_enable) + err = pcs->ops->pcs_enable(pcs); + + return err; +} + static int phylink_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, const struct phylink_link_state *state, bool permit_pause_to_mac) @@ -1027,30 +1066,33 @@ static void phylink_pcs_poll_start(struct phylink *pl) static void phylink_mac_config(struct phylink *pl, const struct phylink_link_state *state) { + struct phylink_link_state st = *state; + + /* Stop drivers incorrectly using these */ + linkmode_zero(st.lp_advertising); + st.speed = SPEED_UNKNOWN; + st.duplex = DUPLEX_UNKNOWN; + st.an_complete = false; + st.link = false; + phylink_dbg(pl, - "%s: mode=%s/%s/%s/%s/%s adv=%*pb pause=%02x link=%u\n", + "%s: mode=%s/%s/%s adv=%*pb pause=%02x\n", __func__, phylink_an_mode_str(pl->cur_link_an_mode), - phy_modes(state->interface), - phy_speed_to_str(state->speed), - phy_duplex_to_str(state->duplex), - phy_rate_matching_to_str(state->rate_matching), - __ETHTOOL_LINK_MODE_MASK_NBITS, state->advertising, - state->pause, state->link); + phy_modes(st.interface), + phy_rate_matching_to_str(st.rate_matching), + __ETHTOOL_LINK_MODE_MASK_NBITS, st.advertising, + st.pause); - pl->mac_ops->mac_config(pl->config, pl->cur_link_an_mode, state); + pl->mac_ops->mac_config(pl->config, pl->cur_link_an_mode, &st); } -static void phylink_mac_pcs_an_restart(struct phylink *pl) +static void phylink_pcs_an_restart(struct phylink *pl) { - if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, - pl->link_config.advertising) && + if (pl->pcs && linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + pl->link_config.advertising) && phy_interface_mode_is_8023z(pl->link_config.interface) && - phylink_autoneg_inband(pl->cur_link_an_mode)) { - if (pl->pcs) - pl->pcs->ops->pcs_an_restart(pl->pcs); - else if (pl->config->legacy_pre_march2020) - pl->mac_ops->mac_an_restart(pl->config); - } + phylink_autoneg_inband(pl->cur_link_an_mode)) + pl->pcs->ops->pcs_an_restart(pl->pcs); } static void phylink_major_config(struct phylink *pl, bool restart, @@ -1095,11 +1137,28 @@ static void phylink_major_config(struct phylink *pl, bool restart, /* If we have a new PCS, switch to the new PCS after preparing the MAC * for the change. */ - if (pcs_changed) + if (pcs_changed) { + phylink_pcs_disable(pl->pcs); + + if (pl->pcs) + pl->pcs->phylink = NULL; + + pcs->phylink = pl; + pl->pcs = pcs; + } + + if (pl->pcs) + phylink_pcs_pre_config(pl->pcs, state->interface); phylink_mac_config(pl, state); + if (pl->pcs) + phylink_pcs_post_config(pl->pcs, state->interface); + + if (pl->pcs_state == PCS_STATE_STARTING || pcs_changed) + phylink_pcs_enable(pl->pcs); + neg_mode = pl->cur_link_an_mode; if (pl->pcs && pl->pcs->neg_mode) neg_mode = pl->pcs_neg_mode; @@ -1113,7 +1172,7 @@ static void phylink_major_config(struct phylink *pl, bool restart, restart = true; if (restart) - phylink_mac_pcs_an_restart(pl); + phylink_pcs_an_restart(pl); if (pl->mac_ops->mac_finish) { err = pl->mac_ops->mac_finish(pl->config, pl->cur_link_an_mode, @@ -1146,13 +1205,6 @@ static int phylink_change_inband_advert(struct phylink *pl) if (test_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state)) return 0; - if (!pl->pcs && pl->config->legacy_pre_march2020) { - /* Legacy method */ - phylink_mac_config(pl, &pl->link_config); - phylink_mac_pcs_an_restart(pl); - return 0; - } - phylink_dbg(pl, "%s: mode=%s/%s adv=%*pb pause=%02x\n", __func__, phylink_an_mode_str(pl->cur_link_an_mode), phy_modes(pl->link_config.interface), @@ -1178,7 +1230,7 @@ static int phylink_change_inband_advert(struct phylink *pl) return ret; if (ret > 0) - phylink_mac_pcs_an_restart(pl); + phylink_pcs_an_restart(pl); return 0; } @@ -1205,9 +1257,6 @@ static void phylink_mac_pcs_get_state(struct phylink *pl, if (pl->pcs) pl->pcs->ops->pcs_get_state(pl->pcs, state); - else if (pl->mac_ops->mac_pcs_get_state && - pl->config->legacy_pre_march2020) - pl->mac_ops->mac_pcs_get_state(pl->config, state); else state->link = 0; } @@ -1440,13 +1489,6 @@ static void phylink_resolve(struct work_struct *w) } phylink_major_config(pl, false, &link_state); pl->link_config.interface = link_state.interface; - } else if (!pl->pcs && pl->config->legacy_pre_march2020) { - /* The interface remains unchanged, only the speed, - * duplex or pause settings have changed. Call the - * old mac_config() method to configure the MAC/PCS - * only if we do not have a legacy MAC driver. - */ - phylink_mac_config(pl, &link_state); } } @@ -1586,6 +1628,7 @@ struct phylink *phylink_create(struct phylink_config *config, pl->link_config.pause = MLO_PAUSE_AN; pl->link_config.speed = SPEED_UNKNOWN; pl->link_config.duplex = DUPLEX_UNKNOWN; + pl->pcs_state = PCS_STATE_DOWN; pl->mac_ops = mac_ops; __set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state); timer_setup(&pl->link_poll, phylink_fixed_poll, 0); @@ -1939,6 +1982,14 @@ void phylink_disconnect_phy(struct phylink *pl) } EXPORT_SYMBOL_GPL(phylink_disconnect_phy); +static void phylink_link_changed(struct phylink *pl, bool up, const char *what) +{ + if (!up) + pl->mac_link_dropped = true; + phylink_run_resolve(pl); + phylink_dbg(pl, "%s link %s\n", what, up ? "up" : "down"); +} + /** * phylink_mac_change() - notify phylink of a change in MAC state * @pl: a pointer to a &struct phylink returned from phylink_create() @@ -1949,13 +2000,30 @@ EXPORT_SYMBOL_GPL(phylink_disconnect_phy); */ void phylink_mac_change(struct phylink *pl, bool up) { - if (!up) - pl->mac_link_dropped = true; - phylink_run_resolve(pl); - phylink_dbg(pl, "mac link %s\n", up ? "up" : "down"); + phylink_link_changed(pl, up, "mac"); } EXPORT_SYMBOL_GPL(phylink_mac_change); +/** + * phylink_pcs_change() - notify phylink of a change to PCS link state + * @pcs: pointer to &struct phylink_pcs + * @up: indicates whether the link is currently up. + * + * The PCS driver should call this when the state of its link changes + * (e.g. link failure, new negotiation results, etc.) Note: it should + * not determine "up" by reading the BMSR. If in doubt about the link + * state at interrupt time, then pass true if pcs_get_state() returns + * the latched link-down state, otherwise pass false. + */ +void phylink_pcs_change(struct phylink_pcs *pcs, bool up) +{ + struct phylink *pl = pcs->phylink; + + if (pl) + phylink_link_changed(pl, up, "pcs"); +} +EXPORT_SYMBOL_GPL(phylink_pcs_change); + static irqreturn_t phylink_link_handler(int irq, void *data) { struct phylink *pl = data; @@ -1987,6 +2055,8 @@ void phylink_start(struct phylink *pl) if (pl->netdev) netif_carrier_off(pl->netdev); + pl->pcs_state = PCS_STATE_STARTING; + /* Apply the link configuration to the MAC when starting. This allows * a fixed-link to start with the correct parameters, and also * ensures that we set the appropriate advertisement for Serdes links. @@ -1997,6 +2067,8 @@ void phylink_start(struct phylink *pl) */ phylink_mac_initial_config(pl, true); + pl->pcs_state = PCS_STATE_STARTED; + phylink_enable_and_run_resolve(pl, PHYLINK_DISABLE_STOPPED); if (pl->cfg_link_an_mode == MLO_AN_FIXED && pl->link_gpio) { @@ -2015,15 +2087,9 @@ void phylink_start(struct phylink *pl) poll = true; } - switch (pl->cfg_link_an_mode) { - case MLO_AN_FIXED: + if (pl->cfg_link_an_mode == MLO_AN_FIXED) poll |= pl->config->poll_fixed_state; - break; - case MLO_AN_INBAND: - if (pl->pcs) - poll |= pl->pcs->poll; - break; - } + if (poll) mod_timer(&pl->link_poll, jiffies + HZ); if (pl->phydev) @@ -2060,6 +2126,10 @@ void phylink_stop(struct phylink *pl) } phylink_run_resolve_and_disable(pl, PHYLINK_DISABLE_STOPPED); + + pl->pcs_state = PCS_STATE_DOWN; + + phylink_pcs_disable(pl->pcs); } EXPORT_SYMBOL_GPL(phylink_stop); @@ -2449,7 +2519,7 @@ int phylink_ethtool_nway_reset(struct phylink *pl) if (pl->phydev) ret = phy_restart_aneg(pl->phydev); - phylink_mac_pcs_an_restart(pl); + phylink_pcs_an_restart(pl); return ret; } @@ -3433,7 +3503,7 @@ static void phylink_decode_usgmii_word(struct phylink_link_state *state, * * Parse the Clause 37 or Cisco SGMII link partner negotiation word into * the phylink @state structure. This is suitable to be used for implementing - * the mac_pcs_get_state() member of the struct phylink_mac_ops structure if + * the pcs_get_state() member of the struct phylink_pcs_ops structure if * accessing @bmsr and @lpa cannot be done with MDIO directly. */ void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, @@ -3483,7 +3553,7 @@ EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_decode_state); * Read the MAC PCS state from the MII device configured in @config and * parse the Clause 37 or Cisco SGMII link partner negotiation word into * the phylink @state structure. This is suitable to be directly plugged - * into the mac_pcs_get_state() member of the struct phylink_mac_ops + * into the pcs_get_state() member of the struct phylink_pcs_ops * structure. */ void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, @@ -3594,8 +3664,8 @@ EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_config); * clause 37 negotiation. * * Restart the clause 37 negotiation with the link partner. This is - * suitable to be directly plugged into the mac_pcs_get_state() member - * of the struct phylink_mac_ops structure. + * suitable to be directly plugged into the pcs_get_state() member + * of the struct phylink_pcs_ops structure. */ void phylink_mii_c22_pcs_an_restart(struct mdio_device *pcs) { diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c index 692930750215..c88edb19d2e7 100644 --- a/drivers/net/phy/smsc.c +++ b/drivers/net/phy/smsc.c @@ -20,6 +20,8 @@ #include <linux/of.h> #include <linux/phy.h> #include <linux/netdevice.h> +#include <linux/crc16.h> +#include <linux/etherdevice.h> #include <linux/smscphy.h> /* Vendor-specific PHY Definitions */ @@ -51,6 +53,7 @@ struct smsc_phy_priv { unsigned int edpd_enable:1; unsigned int edpd_mode_set_by_user:1; unsigned int edpd_max_wait_ms; + bool wol_arp; }; static int smsc_phy_ack_interrupt(struct phy_device *phydev) @@ -258,6 +261,243 @@ int lan87xx_read_status(struct phy_device *phydev) } EXPORT_SYMBOL_GPL(lan87xx_read_status); +static int lan874x_phy_config_init(struct phy_device *phydev) +{ + u16 val; + int rc; + + /* Setup LED2/nINT/nPME pin to function as nPME. May need user option + * to use LED1/nINT/nPME. + */ + val = MII_LAN874X_PHY_PME2_SET; + + /* The bits MII_LAN874X_PHY_WOL_PFDA_FR, MII_LAN874X_PHY_WOL_WUFR, + * MII_LAN874X_PHY_WOL_MPR, and MII_LAN874X_PHY_WOL_BCAST_FR need to + * be cleared to de-assert PME signal after a WoL event happens, but + * using PME auto clear gets around that. + */ + val |= MII_LAN874X_PHY_PME_SELF_CLEAR; + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, MII_LAN874X_PHY_MMD_WOL_WUCSR, + val); + if (rc < 0) + return rc; + + /* set nPME self clear delay time */ + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, MII_LAN874X_PHY_MMD_MCFGR, + MII_LAN874X_PHY_PME_SELF_CLEAR_DELAY); + if (rc < 0) + return rc; + + return smsc_phy_config_init(phydev); +} + +static void lan874x_get_wol(struct phy_device *phydev, + struct ethtool_wolinfo *wol) +{ + struct smsc_phy_priv *priv = phydev->priv; + int rc; + + wol->supported = (WAKE_UCAST | WAKE_BCAST | WAKE_MAGIC | + WAKE_ARP | WAKE_MCAST); + wol->wolopts = 0; + + rc = phy_read_mmd(phydev, MDIO_MMD_PCS, MII_LAN874X_PHY_MMD_WOL_WUCSR); + if (rc < 0) + return; + + if (rc & MII_LAN874X_PHY_WOL_PFDAEN) + wol->wolopts |= WAKE_UCAST; + + if (rc & MII_LAN874X_PHY_WOL_BCSTEN) + wol->wolopts |= WAKE_BCAST; + + if (rc & MII_LAN874X_PHY_WOL_MPEN) + wol->wolopts |= WAKE_MAGIC; + + if (rc & MII_LAN874X_PHY_WOL_WUEN) { + if (priv->wol_arp) + wol->wolopts |= WAKE_ARP; + else + wol->wolopts |= WAKE_MCAST; + } +} + +static u16 smsc_crc16(const u8 *buffer, size_t len) +{ + return bitrev16(crc16(0xFFFF, buffer, len)); +} + +static int lan874x_chk_wol_pattern(const u8 pattern[], const u16 *mask, + u8 len, u8 *data, u8 *datalen) +{ + size_t i, j, k; + int ret = 0; + u16 bits; + + /* Pattern filtering can match up to 128 bytes of frame data. There + * are 8 registers to program the 16-bit masks, where each bit means + * the byte will be compared. The frame data will then go through a + * CRC16 calculation for hardware comparison. This helper function + * makes sure only relevant frame data are included in this + * calculation. It provides a warning when the masks and expected + * data size do not match. + */ + i = 0; + k = 0; + while (len > 0) { + bits = *mask; + for (j = 0; j < 16; j++, i++, len--) { + /* No more pattern. */ + if (!len) { + /* The rest of bitmap is not empty. */ + if (bits) + ret = i + 1; + break; + } + if (bits & 1) + data[k++] = pattern[i]; + bits >>= 1; + } + mask++; + } + *datalen = k; + return ret; +} + +static int lan874x_set_wol_pattern(struct phy_device *phydev, u16 val, + const u8 data[], u8 datalen, + const u16 *mask, u8 masklen) +{ + u16 crc, reg; + int rc; + + /* Starting pattern offset is set before calling this function. */ + val |= MII_LAN874X_PHY_WOL_FILTER_EN; + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, + MII_LAN874X_PHY_MMD_WOL_WUF_CFGA, val); + if (rc < 0) + return rc; + + crc = smsc_crc16(data, datalen); + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, + MII_LAN874X_PHY_MMD_WOL_WUF_CFGB, crc); + if (rc < 0) + return rc; + + masklen = (masklen + 15) & ~0xf; + reg = MII_LAN874X_PHY_MMD_WOL_WUF_MASK7; + while (masklen >= 16) { + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, reg, *mask); + if (rc < 0) + return rc; + reg--; + mask++; + masklen -= 16; + } + + /* Clear out the rest of mask registers. */ + while (reg != MII_LAN874X_PHY_MMD_WOL_WUF_MASK0) { + phy_write_mmd(phydev, MDIO_MMD_PCS, reg, 0); + reg--; + } + return rc; +} + +static int lan874x_set_wol(struct phy_device *phydev, + struct ethtool_wolinfo *wol) +{ + struct net_device *ndev = phydev->attached_dev; + struct smsc_phy_priv *priv = phydev->priv; + u16 val, val_wucsr; + u8 data[128]; + u8 datalen; + int rc; + + /* lan874x has only one WoL filter pattern */ + if ((wol->wolopts & (WAKE_ARP | WAKE_MCAST)) == + (WAKE_ARP | WAKE_MCAST)) { + phydev_info(phydev, + "lan874x WoL supports one of ARP|MCAST at a time\n"); + return -EOPNOTSUPP; + } + + rc = phy_read_mmd(phydev, MDIO_MMD_PCS, MII_LAN874X_PHY_MMD_WOL_WUCSR); + if (rc < 0) + return rc; + + val_wucsr = rc; + + if (wol->wolopts & WAKE_UCAST) + val_wucsr |= MII_LAN874X_PHY_WOL_PFDAEN; + else + val_wucsr &= ~MII_LAN874X_PHY_WOL_PFDAEN; + + if (wol->wolopts & WAKE_BCAST) + val_wucsr |= MII_LAN874X_PHY_WOL_BCSTEN; + else + val_wucsr &= ~MII_LAN874X_PHY_WOL_BCSTEN; + + if (wol->wolopts & WAKE_MAGIC) + val_wucsr |= MII_LAN874X_PHY_WOL_MPEN; + else + val_wucsr &= ~MII_LAN874X_PHY_WOL_MPEN; + + /* Need to use pattern matching */ + if (wol->wolopts & (WAKE_ARP | WAKE_MCAST)) + val_wucsr |= MII_LAN874X_PHY_WOL_WUEN; + else + val_wucsr &= ~MII_LAN874X_PHY_WOL_WUEN; + + if (wol->wolopts & WAKE_ARP) { + const u8 pattern[2] = { 0x08, 0x06 }; + const u16 mask[1] = { 0x0003 }; + + rc = lan874x_chk_wol_pattern(pattern, mask, 2, data, + &datalen); + if (rc) + phydev_dbg(phydev, "pattern not valid at %d\n", rc); + + /* Need to match broadcast destination address and provided + * data pattern at offset 12. + */ + val = 12 | MII_LAN874X_PHY_WOL_FILTER_BCSTEN; + rc = lan874x_set_wol_pattern(phydev, val, data, datalen, mask, + 2); + if (rc < 0) + return rc; + priv->wol_arp = true; + } + + if (wol->wolopts & WAKE_MCAST) { + /* Need to match multicast destination address. */ + val = MII_LAN874X_PHY_WOL_FILTER_MCASTTEN; + rc = lan874x_set_wol_pattern(phydev, val, data, 0, NULL, 0); + if (rc < 0) + return rc; + priv->wol_arp = false; + } + + if (wol->wolopts & (WAKE_MAGIC | WAKE_UCAST)) { + const u8 *mac = (const u8 *)ndev->dev_addr; + int i, reg; + + reg = MII_LAN874X_PHY_MMD_WOL_RX_ADDRC; + for (i = 0; i < 6; i += 2, reg--) { + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, reg, + ((mac[i + 1] << 8) | mac[i])); + if (rc < 0) + return rc; + } + } + + rc = phy_write_mmd(phydev, MDIO_MMD_PCS, MII_LAN874X_PHY_MMD_WOL_WUCSR, + val_wucsr); + if (rc < 0) + return rc; + + return 0; +} + static int smsc_get_sset_count(struct phy_device *phydev) { return ARRAY_SIZE(smsc_hw_stats); @@ -533,7 +773,7 @@ static struct phy_driver smsc_phy_driver[] = { /* basic functions */ .read_status = lan87xx_read_status, - .config_init = smsc_phy_config_init, + .config_init = lan874x_phy_config_init, .soft_reset = smsc_phy_reset, /* IRQ related */ @@ -548,6 +788,10 @@ static struct phy_driver smsc_phy_driver[] = { .get_tunable = smsc_phy_get_tunable, .set_tunable = smsc_phy_set_tunable, + /* WoL */ + .set_wol = lan874x_set_wol, + .get_wol = lan874x_get_wol, + .suspend = genphy_suspend, .resume = genphy_resume, }, { @@ -566,7 +810,7 @@ static struct phy_driver smsc_phy_driver[] = { /* basic functions */ .read_status = lan87xx_read_status, - .config_init = smsc_phy_config_init, + .config_init = lan874x_phy_config_init, .soft_reset = smsc_phy_reset, /* IRQ related */ @@ -581,6 +825,10 @@ static struct phy_driver smsc_phy_driver[] = { .get_tunable = smsc_phy_get_tunable, .set_tunable = smsc_phy_set_tunable, + /* WoL */ + .set_wol = lan874x_set_wol, + .get_wol = lan874x_get_wol, + .suspend = genphy_suspend, .resume = genphy_resume, } }; diff --git a/drivers/net/phy/stubs.c b/drivers/net/phy/stubs.c new file mode 100644 index 000000000000..cfb9f275eb18 --- /dev/null +++ b/drivers/net/phy/stubs.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Stubs for PHY library functionality called by the core network stack. + * These are necessary because CONFIG_PHYLIB can be a module, and built-in + * code cannot directly call symbols exported by modules. + */ +#include <linux/phylib_stubs.h> + +const struct phylib_stubs *phylib_stubs; +EXPORT_SYMBOL_GPL(phylib_stubs); diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c index 3b79c603b936..ba8b6bd8233c 100644 --- a/drivers/net/ppp/pppoe.c +++ b/drivers/net/ppp/pppoe.c @@ -968,7 +968,7 @@ abort: ***********************************************************************/ static int pppoe_xmit(struct ppp_channel *chan, struct sk_buff *skb) { - struct sock *sk = (struct sock *)chan->private; + struct sock *sk = chan->private; return __pppoe_xmit(sk, skb); } @@ -976,7 +976,7 @@ static int pppoe_fill_forward_path(struct net_device_path_ctx *ctx, struct net_device_path *path, const struct ppp_channel *chan) { - struct sock *sk = (struct sock *)chan->private; + struct sock *sk = chan->private; struct pppox_sock *po = pppox_sk(sk); struct net_device *dev = po->pppoe_dev; diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c index 32183f24e63f..6833ef0c7930 100644 --- a/drivers/net/ppp/pptp.c +++ b/drivers/net/ppp/pptp.c @@ -129,10 +129,10 @@ static void del_chan(struct pppox_sock *sock) spin_unlock(&chan_lock); } -static struct rtable *pptp_route_output(struct pppox_sock *po, +static struct rtable *pptp_route_output(const struct pppox_sock *po, struct flowi4 *fl4) { - struct sock *sk = &po->sk; + const struct sock *sk = &po->sk; struct net *net; net = sock_net(sk); @@ -148,7 +148,7 @@ static struct rtable *pptp_route_output(struct pppox_sock *po, static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) { - struct sock *sk = (struct sock *) chan->private; + struct sock *sk = chan->private; struct pppox_sock *po = pppox_sk(sk); struct net *net = sock_net(sk); struct pptp_opt *opt = &po->proto.pptp; @@ -575,7 +575,7 @@ out: static int pptp_ppp_ioctl(struct ppp_channel *chan, unsigned int cmd, unsigned long arg) { - struct sock *sk = (struct sock *) chan->private; + struct sock *sk = chan->private; struct pppox_sock *po = pppox_sk(sk); struct pptp_opt *opt = &po->proto.pptp; void __user *argp = (void __user *)arg; diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 49d1d6acf95e..5b0f2abf454d 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -614,8 +614,10 @@ static inline struct sk_buff *tap_alloc_skb(struct sock *sk, size_t prepad, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - err, 0); + err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return NULL; diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index d3dc22509ea5..bc50fc3f6913 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c @@ -2321,8 +2321,7 @@ static struct team *team_nl_team_get(struct genl_info *info) ifindex = nla_get_u32(info->attrs[TEAM_ATTR_TEAM_IFINDEX]); dev = dev_get_by_index(net, ifindex); if (!dev || dev->netdev_ops != &team_netdev_ops) { - if (dev) - dev_put(dev); + dev_put(dev); return NULL; } diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 25f0191df00b..973b2fc74de3 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1526,8 +1526,10 @@ static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - &err, 0); + &err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return ERR_PTR(err); diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 0738baa5b82e..db9897e825b4 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -1314,16 +1314,24 @@ static int generic_ocp_write(struct r8152 *tp, u16 index, u16 byteen, byteen_end = byteen & BYTE_EN_END_MASK; byen = byteen_start | (byteen_start << 4); - ret = set_registers(tp, index, type | byen, 4, data); - if (ret < 0) - goto error1; - index += 4; - data += 4; - size -= 4; + /* Split the first DWORD if the byte_en is not 0xff */ + if (byen != BYTE_EN_DWORD) { + ret = set_registers(tp, index, type | byen, 4, data); + if (ret < 0) + goto error1; - if (size) { + index += 4; + data += 4; size -= 4; + } + + if (size) { + byen = byteen_end | (byteen_end >> 4); + + /* Split the last DWORD if the byte_en is not 0xff */ + if (byen != BYTE_EN_DWORD) + size -= 4; while (size) { if (size > limit) { @@ -1350,10 +1358,9 @@ static int generic_ocp_write(struct r8152 *tp, u16 index, u16 byteen, } } - byen = byteen_end | (byteen_end >> 4); - ret = set_registers(tp, index, type | byen, 4, data); - if (ret < 0) - goto error1; + /* Set the last DWORD */ + if (byen != BYTE_EN_DWORD) + ret = set_registers(tp, index, type | byen, 4, data); } error1: @@ -3971,29 +3978,10 @@ static void rtl_reset_bmu(struct r8152 *tp) /* Clear the bp to stop the firmware before loading a new one */ static void rtl_clear_bp(struct r8152 *tp, u16 type) { - switch (tp->version) { - case RTL_VER_01: - case RTL_VER_02: - case RTL_VER_07: - break; - case RTL_VER_03: - case RTL_VER_04: - case RTL_VER_05: - case RTL_VER_06: - ocp_write_byte(tp, type, PLA_BP_EN, 0); - break; - case RTL_VER_14: - ocp_write_word(tp, type, USB_BP2_EN, 0); + u16 bp[16] = {0}; + u16 bp_num; - ocp_write_word(tp, type, USB_BP_8, 0); - ocp_write_word(tp, type, USB_BP_9, 0); - ocp_write_word(tp, type, USB_BP_10, 0); - ocp_write_word(tp, type, USB_BP_11, 0); - ocp_write_word(tp, type, USB_BP_12, 0); - ocp_write_word(tp, type, USB_BP_13, 0); - ocp_write_word(tp, type, USB_BP_14, 0); - ocp_write_word(tp, type, USB_BP_15, 0); - break; + switch (tp->version) { case RTL_VER_08: case RTL_VER_09: case RTL_VER_10: @@ -4001,32 +3989,31 @@ static void rtl_clear_bp(struct r8152 *tp, u16 type) case RTL_VER_12: case RTL_VER_13: case RTL_VER_15: - default: if (type == MCU_TYPE_USB) { ocp_write_word(tp, MCU_TYPE_USB, USB_BP2_EN, 0); - - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_8, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_9, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_10, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_11, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_12, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_13, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_14, 0); - ocp_write_word(tp, MCU_TYPE_USB, USB_BP_15, 0); - } else { - ocp_write_byte(tp, MCU_TYPE_PLA, PLA_BP_EN, 0); + bp_num = 16; + break; } + fallthrough; + case RTL_VER_03: + case RTL_VER_04: + case RTL_VER_05: + case RTL_VER_06: + ocp_write_byte(tp, type, PLA_BP_EN, 0); + fallthrough; + case RTL_VER_01: + case RTL_VER_02: + case RTL_VER_07: + bp_num = 8; + break; + case RTL_VER_14: + default: + ocp_write_word(tp, type, USB_BP2_EN, 0); + bp_num = 16; break; } - ocp_write_word(tp, type, PLA_BP_0, 0); - ocp_write_word(tp, type, PLA_BP_1, 0); - ocp_write_word(tp, type, PLA_BP_2, 0); - ocp_write_word(tp, type, PLA_BP_3, 0); - ocp_write_word(tp, type, PLA_BP_4, 0); - ocp_write_word(tp, type, PLA_BP_5, 0); - ocp_write_word(tp, type, PLA_BP_6, 0); - ocp_write_word(tp, type, PLA_BP_7, 0); + generic_ocp_write(tp, PLA_BP_0, BYTE_EN_DWORD, bp_num << 1, bp, type); /* wait 3 ms to make sure the firmware is stopped */ usleep_range(3000, 6000); @@ -5000,10 +4987,9 @@ static void rtl8152_fw_phy_nc_apply(struct r8152 *tp, struct fw_phy_nc *phy) static void rtl8152_fw_mac_apply(struct r8152 *tp, struct fw_mac *mac) { - u16 bp_en_addr, bp_index, type, bp_num, fw_ver_reg; + u16 bp_en_addr, type, fw_ver_reg; u32 length; u8 *data; - int i; switch (__le32_to_cpu(mac->blk_hdr.type)) { case RTL_FW_PLA: @@ -5045,12 +5031,8 @@ static void rtl8152_fw_mac_apply(struct r8152 *tp, struct fw_mac *mac) ocp_write_word(tp, type, __le16_to_cpu(mac->bp_ba_addr), __le16_to_cpu(mac->bp_ba_value)); - bp_index = __le16_to_cpu(mac->bp_start); - bp_num = __le16_to_cpu(mac->bp_num); - for (i = 0; i < bp_num; i++) { - ocp_write_word(tp, type, bp_index, __le16_to_cpu(mac->bp[i])); - bp_index += 2; - } + generic_ocp_write(tp, __le16_to_cpu(mac->bp_start), BYTE_EN_DWORD, + __le16_to_cpu(mac->bp_num) << 1, mac->bp, type); bp_en_addr = __le16_to_cpu(mac->bp_en_addr); if (bp_en_addr) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 1270c8d23463..c615bbf24c0d 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -126,6 +126,11 @@ static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc) #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc) +struct virtnet_interrupt_coalesce { + u32 max_packets; + u32 max_usecs; +}; + /* Internal representation of a send virtqueue */ struct send_queue { /* Virtqueue associated with this send _queue */ @@ -139,6 +144,8 @@ struct send_queue { struct virtnet_sq_stats stats; + struct virtnet_interrupt_coalesce intr_coal; + struct napi_struct napi; /* Record whether sq is in reset state. */ @@ -156,6 +163,8 @@ struct receive_queue { struct virtnet_rq_stats stats; + struct virtnet_interrupt_coalesce intr_coal; + /* Chain pages by the private ptr. */ struct page *pages; @@ -207,6 +216,7 @@ struct control_buf { struct virtio_net_ctrl_rss rss; struct virtio_net_ctrl_coal_tx coal_tx; struct virtio_net_ctrl_coal_rx coal_rx; + struct virtio_net_ctrl_coal_vq coal_vq; }; struct virtnet_info { @@ -281,10 +291,8 @@ struct virtnet_info { u32 speed; /* Interrupt coalescing settings */ - u32 tx_usecs; - u32 rx_usecs; - u32 tx_max_packets; - u32 rx_max_packets; + struct virtnet_interrupt_coalesce intr_coal_tx; + struct virtnet_interrupt_coalesce intr_coal_rx; unsigned long guest_offloads; unsigned long guest_offloads_capable; @@ -3056,8 +3064,8 @@ static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, return -EINVAL; /* Save parameters */ - vi->tx_usecs = ec->tx_coalesce_usecs; - vi->tx_max_packets = ec->tx_max_coalesced_frames; + vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs; + vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames; vi->ctrl->coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); vi->ctrl->coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); @@ -3069,8 +3077,57 @@ static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, return -EINVAL; /* Save parameters */ - vi->rx_usecs = ec->rx_coalesce_usecs; - vi->rx_max_packets = ec->rx_max_coalesced_frames; + vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs; + vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames; + + return 0; +} + +static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, + u16 vqn, u32 max_usecs, u32 max_packets) +{ + struct scatterlist sgs; + + vi->ctrl->coal_vq.vqn = cpu_to_le16(vqn); + vi->ctrl->coal_vq.coal.max_usecs = cpu_to_le32(max_usecs); + vi->ctrl->coal_vq.coal.max_packets = cpu_to_le32(max_packets); + sg_init_one(&sgs, &vi->ctrl->coal_vq, sizeof(vi->ctrl->coal_vq)); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, + VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET, + &sgs)) + return -EINVAL; + + return 0; +} + +static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, + struct ethtool_coalesce *ec, + u16 queue) +{ + int err; + + if (ec->rx_coalesce_usecs || ec->rx_max_coalesced_frames) { + err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue), + ec->rx_coalesce_usecs, + ec->rx_max_coalesced_frames); + if (err) + return err; + /* Save parameters */ + vi->rq[queue].intr_coal.max_usecs = ec->rx_coalesce_usecs; + vi->rq[queue].intr_coal.max_packets = ec->rx_max_coalesced_frames; + } + + if (ec->tx_coalesce_usecs || ec->tx_max_coalesced_frames) { + err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue), + ec->tx_coalesce_usecs, + ec->tx_max_coalesced_frames); + if (err) + return err; + /* Save parameters */ + vi->sq[queue].intr_coal.max_usecs = ec->tx_coalesce_usecs; + vi->sq[queue].intr_coal.max_packets = ec->tx_max_coalesced_frames; + } return 0; } @@ -3090,22 +3147,42 @@ static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) return 0; } +static int virtnet_should_update_vq_weight(int dev_flags, int weight, + int vq_weight, bool *should_update) +{ + if (weight ^ vq_weight) { + if (dev_flags & IFF_UP) + return -EBUSY; + *should_update = true; + } + + return 0; +} + static int virtnet_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); - int ret, i, napi_weight; + int ret, queue_number, napi_weight; bool update_napi = false; /* Can't change NAPI weight if the link is up */ napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; - if (napi_weight ^ vi->sq[0].napi.weight) { - if (dev->flags & IFF_UP) - return -EBUSY; - else - update_napi = true; + for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) { + ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, + vi->sq[queue_number].napi.weight, + &update_napi); + if (ret) + return ret; + + if (update_napi) { + /* All queues that belong to [queue_number, vi->max_queue_pairs] will be + * updated for the sake of simplicity, which might not be necessary + */ + break; + } } if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) @@ -3117,8 +3194,8 @@ static int virtnet_set_coalesce(struct net_device *dev, return ret; if (update_napi) { - for (i = 0; i < vi->max_queue_pairs; i++) - vi->sq[i].napi.weight = napi_weight; + for (; queue_number < vi->max_queue_pairs; queue_number++) + vi->sq[queue_number].napi.weight = napi_weight; } return ret; @@ -3132,10 +3209,67 @@ static int virtnet_get_coalesce(struct net_device *dev, struct virtnet_info *vi = netdev_priv(dev); if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { - ec->rx_coalesce_usecs = vi->rx_usecs; - ec->tx_coalesce_usecs = vi->tx_usecs; - ec->tx_max_coalesced_frames = vi->tx_max_packets; - ec->rx_max_coalesced_frames = vi->rx_max_packets; + ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs; + ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs; + ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets; + ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets; + } else { + ec->rx_max_coalesced_frames = 1; + + if (vi->sq[0].napi.weight) + ec->tx_max_coalesced_frames = 1; + } + + return 0; +} + +static int virtnet_set_per_queue_coalesce(struct net_device *dev, + u32 queue, + struct ethtool_coalesce *ec) +{ + struct virtnet_info *vi = netdev_priv(dev); + int ret, napi_weight; + bool update_napi = false; + + if (queue >= vi->max_queue_pairs) + return -EINVAL; + + /* Can't change NAPI weight if the link is up */ + napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; + ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, + vi->sq[queue].napi.weight, + &update_napi); + if (ret) + return ret; + + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) + ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue); + else + ret = virtnet_coal_params_supported(ec); + + if (ret) + return ret; + + if (update_napi) + vi->sq[queue].napi.weight = napi_weight; + + return 0; +} + +static int virtnet_get_per_queue_coalesce(struct net_device *dev, + u32 queue, + struct ethtool_coalesce *ec) +{ + struct virtnet_info *vi = netdev_priv(dev); + + if (queue >= vi->max_queue_pairs) + return -EINVAL; + + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { + ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs; + ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs; + ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets; + ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets; } else { ec->rx_max_coalesced_frames = 1; @@ -3276,6 +3410,8 @@ static const struct ethtool_ops virtnet_ethtool_ops = { .set_link_ksettings = virtnet_set_link_ksettings, .set_coalesce = virtnet_set_coalesce, .get_coalesce = virtnet_get_coalesce, + .set_per_queue_coalesce = virtnet_set_per_queue_coalesce, + .get_per_queue_coalesce = virtnet_get_per_queue_coalesce, .get_rxfh_key_size = virtnet_get_rxfh_key_size, .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, .get_rxfh = virtnet_get_rxfh, @@ -3952,6 +4088,8 @@ static bool virtnet_validate_features(struct virtio_device *vdev) VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, "VIRTIO_NET_F_CTRL_VQ") || VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, + "VIRTIO_NET_F_CTRL_VQ") || + VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL, "VIRTIO_NET_F_CTRL_VQ"))) { return false; } @@ -4119,10 +4257,10 @@ static int virtnet_probe(struct virtio_device *vdev) } if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { - vi->rx_usecs = 0; - vi->tx_usecs = 0; - vi->tx_max_packets = 0; - vi->rx_max_packets = 0; + vi->intr_coal_rx.max_usecs = 0; + vi->intr_coal_tx.max_usecs = 0; + vi->intr_coal_tx.max_packets = 0; + vi->intr_coal_rx.max_packets = 0; } if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) @@ -4376,6 +4514,7 @@ static struct virtio_device_id id_table[] = { VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \ + VIRTIO_NET_F_VQ_NOTF_COAL, \ VIRTIO_NET_F_GUEST_HDRLEN static unsigned int features[] = { diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index c9a9373733c0..2bddcdf482a7 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -2720,6 +2720,45 @@ drop: dev_kfree_skb(skb); } +static netdev_tx_t vxlan_xmit_nhid(struct sk_buff *skb, struct net_device *dev, + u32 nhid, __be32 vni) +{ + struct vxlan_dev *vxlan = netdev_priv(dev); + struct vxlan_rdst nh_rdst; + struct nexthop *nh; + bool do_xmit; + u32 hash; + + memset(&nh_rdst, 0, sizeof(struct vxlan_rdst)); + hash = skb_get_hash(skb); + + rcu_read_lock(); + nh = nexthop_find_by_id(dev_net(dev), nhid); + if (unlikely(!nh || !nexthop_is_fdb(nh) || !nexthop_is_multipath(nh))) { + rcu_read_unlock(); + goto drop; + } + do_xmit = vxlan_fdb_nh_path_select(nh, hash, &nh_rdst); + rcu_read_unlock(); + + if (vxlan->cfg.saddr.sa.sa_family != nh_rdst.remote_ip.sa.sa_family) + goto drop; + + if (likely(do_xmit)) + vxlan_xmit_one(skb, dev, vni, &nh_rdst, false); + else + goto drop; + + return NETDEV_TX_OK; + +drop: + dev->stats.tx_dropped++; + vxlan_vnifilter_count(netdev_priv(dev), vni, NULL, + VXLAN_VNI_STATS_TX_DROPS, 0); + dev_kfree_skb(skb); + return NETDEV_TX_OK; +} + /* Transmit local packets over Vxlan * * Outer IP header inherits ECN and DF from inner header. @@ -2735,6 +2774,7 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev) struct vxlan_fdb *f; struct ethhdr *eth; __be32 vni = 0; + u32 nhid = 0; info = skb_tunnel_info(skb); @@ -2744,6 +2784,7 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev) if (info && info->mode & IP_TUNNEL_INFO_BRIDGE && info->mode & IP_TUNNEL_INFO_TX) { vni = tunnel_id_to_key32(info->key.tun_id); + nhid = info->key.nhid; } else { if (info && info->mode & IP_TUNNEL_INFO_TX) vxlan_xmit_one(skb, dev, vni, NULL, false); @@ -2771,6 +2812,9 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev) #endif } + if (nhid) + return vxlan_xmit_nhid(skb, dev, nhid, vni); + if (vxlan->cfg.flags & VXLAN_F_MDB) { struct vxlan_mdb_entry *mdb_entry; diff --git a/drivers/net/wwan/t7xx/t7xx_hif_cldma.c b/drivers/net/wwan/t7xx/t7xx_hif_cldma.c index 7162bf38a8c9..cc70360364b7 100644 --- a/drivers/net/wwan/t7xx/t7xx_hif_cldma.c +++ b/drivers/net/wwan/t7xx/t7xx_hif_cldma.c @@ -1066,13 +1066,18 @@ static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl) struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; u32 phy_ao_base, phy_pd_base; - if (md_ctrl->hif_id != CLDMA_ID_MD) - return; - - phy_ao_base = CLDMA1_AO_BASE; - phy_pd_base = CLDMA1_PD_BASE; - hw_info->phy_interrupt_id = CLDMA1_INT; hw_info->hw_mode = MODE_BIT_64; + + if (md_ctrl->hif_id == CLDMA_ID_MD) { + phy_ao_base = CLDMA1_AO_BASE; + phy_pd_base = CLDMA1_PD_BASE; + hw_info->phy_interrupt_id = CLDMA1_INT; + } else { + phy_ao_base = CLDMA0_AO_BASE; + phy_pd_base = CLDMA0_PD_BASE; + hw_info->phy_interrupt_id = CLDMA0_INT; + } + hw_info->ap_ao_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base, pbase->pcie_dev_reg_trsl_addr, phy_ao_base); hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base, diff --git a/drivers/net/wwan/t7xx/t7xx_hif_cldma.h b/drivers/net/wwan/t7xx/t7xx_hif_cldma.h index 47a35e552da7..4410bac6993a 100644 --- a/drivers/net/wwan/t7xx/t7xx_hif_cldma.h +++ b/drivers/net/wwan/t7xx/t7xx_hif_cldma.h @@ -34,7 +34,7 @@ /** * enum cldma_id - Identifiers for CLDMA HW units. * @CLDMA_ID_MD: Modem control channel. - * @CLDMA_ID_AP: Application Processor control channel (not used at the moment). + * @CLDMA_ID_AP: Application Processor control channel. * @CLDMA_NUM: Number of CLDMA HW units available. */ enum cldma_id { diff --git a/drivers/net/wwan/t7xx/t7xx_mhccif.h b/drivers/net/wwan/t7xx/t7xx_mhccif.h index 209b386bc088..20c50dce9fc3 100644 --- a/drivers/net/wwan/t7xx/t7xx_mhccif.h +++ b/drivers/net/wwan/t7xx/t7xx_mhccif.h @@ -25,6 +25,7 @@ D2H_INT_EXCEPTION_CLEARQ_DONE | \ D2H_INT_EXCEPTION_ALLQ_RESET | \ D2H_INT_PORT_ENUM | \ + D2H_INT_ASYNC_AP_HK | \ D2H_INT_ASYNC_MD_HK) void t7xx_mhccif_mask_set(struct t7xx_pci_dev *t7xx_dev, u32 val); diff --git a/drivers/net/wwan/t7xx/t7xx_modem_ops.c b/drivers/net/wwan/t7xx/t7xx_modem_ops.c index 7d0f5e4f0a78..24e7d491468e 100644 --- a/drivers/net/wwan/t7xx/t7xx_modem_ops.c +++ b/drivers/net/wwan/t7xx/t7xx_modem_ops.c @@ -44,6 +44,7 @@ #include "t7xx_state_monitor.h" #define RT_ID_MD_PORT_ENUM 0 +#define RT_ID_AP_PORT_ENUM 1 /* Modem feature query identification code - "ICCC" */ #define MD_FEATURE_QUERY_ID 0x49434343 @@ -298,6 +299,7 @@ static void t7xx_md_exception(struct t7xx_modem *md, enum hif_ex_stage stage) } t7xx_cldma_exception(md->md_ctrl[CLDMA_ID_MD], stage); + t7xx_cldma_exception(md->md_ctrl[CLDMA_ID_AP], stage); if (stage == HIF_EX_INIT) t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_EXCEPTION_ACK); @@ -426,7 +428,7 @@ static int t7xx_parse_host_rt_data(struct t7xx_fsm_ctl *ctl, struct t7xx_sys_inf if (ft_spt_st != MTK_FEATURE_MUST_BE_SUPPORTED) return -EINVAL; - if (i == RT_ID_MD_PORT_ENUM) + if (i == RT_ID_MD_PORT_ENUM || i == RT_ID_AP_PORT_ENUM) t7xx_port_enum_msg_handler(ctl->md, rt_feature->data); } @@ -456,12 +458,12 @@ static int t7xx_core_reset(struct t7xx_modem *md) return 0; } -static void t7xx_core_hk_handler(struct t7xx_modem *md, struct t7xx_fsm_ctl *ctl, +static void t7xx_core_hk_handler(struct t7xx_modem *md, struct t7xx_sys_info *core_info, + struct t7xx_fsm_ctl *ctl, enum t7xx_fsm_event_state event_id, enum t7xx_fsm_event_state err_detect) { struct t7xx_fsm_event *event = NULL, *event_next; - struct t7xx_sys_info *core_info = &md->core_md; struct device *dev = &md->t7xx_dev->pdev->dev; unsigned long flags; int ret; @@ -531,19 +533,33 @@ static void t7xx_md_hk_wq(struct work_struct *work) t7xx_cldma_start(md->md_ctrl[CLDMA_ID_MD]); t7xx_fsm_broadcast_state(ctl, MD_STATE_WAITING_FOR_HS2); md->core_md.handshake_ongoing = true; - t7xx_core_hk_handler(md, ctl, FSM_EVENT_MD_HS2, FSM_EVENT_MD_HS2_EXIT); + t7xx_core_hk_handler(md, &md->core_md, ctl, FSM_EVENT_MD_HS2, FSM_EVENT_MD_HS2_EXIT); +} + +static void t7xx_ap_hk_wq(struct work_struct *work) +{ + struct t7xx_modem *md = container_of(work, struct t7xx_modem, ap_handshake_work); + struct t7xx_fsm_ctl *ctl = md->fsm_ctl; + + /* Clear the HS2 EXIT event appended in t7xx_core_reset(). */ + t7xx_fsm_clr_event(ctl, FSM_EVENT_AP_HS2_EXIT); + t7xx_cldma_stop(md->md_ctrl[CLDMA_ID_AP]); + t7xx_cldma_switch_cfg(md->md_ctrl[CLDMA_ID_AP]); + t7xx_cldma_start(md->md_ctrl[CLDMA_ID_AP]); + md->core_ap.handshake_ongoing = true; + t7xx_core_hk_handler(md, &md->core_ap, ctl, FSM_EVENT_AP_HS2, FSM_EVENT_AP_HS2_EXIT); } void t7xx_md_event_notify(struct t7xx_modem *md, enum md_event_id evt_id) { struct t7xx_fsm_ctl *ctl = md->fsm_ctl; - void __iomem *mhccif_base; unsigned int int_sta; unsigned long flags; switch (evt_id) { case FSM_PRE_START: - t7xx_mhccif_mask_clr(md->t7xx_dev, D2H_INT_PORT_ENUM); + t7xx_mhccif_mask_clr(md->t7xx_dev, D2H_INT_PORT_ENUM | D2H_INT_ASYNC_MD_HK | + D2H_INT_ASYNC_AP_HK); break; case FSM_START: @@ -556,16 +572,26 @@ void t7xx_md_event_notify(struct t7xx_modem *md, enum md_event_id evt_id) ctl->exp_flg = true; md->exp_id &= ~D2H_INT_EXCEPTION_INIT; md->exp_id &= ~D2H_INT_ASYNC_MD_HK; + md->exp_id &= ~D2H_INT_ASYNC_AP_HK; } else if (ctl->exp_flg) { md->exp_id &= ~D2H_INT_ASYNC_MD_HK; - } else if (md->exp_id & D2H_INT_ASYNC_MD_HK) { - queue_work(md->handshake_wq, &md->handshake_work); - md->exp_id &= ~D2H_INT_ASYNC_MD_HK; - mhccif_base = md->t7xx_dev->base_addr.mhccif_rc_base; - iowrite32(D2H_INT_ASYNC_MD_HK, mhccif_base + REG_EP2RC_SW_INT_ACK); - t7xx_mhccif_mask_set(md->t7xx_dev, D2H_INT_ASYNC_MD_HK); + md->exp_id &= ~D2H_INT_ASYNC_AP_HK; } else { - t7xx_mhccif_mask_clr(md->t7xx_dev, D2H_INT_ASYNC_MD_HK); + void __iomem *mhccif_base = md->t7xx_dev->base_addr.mhccif_rc_base; + + if (md->exp_id & D2H_INT_ASYNC_MD_HK) { + queue_work(md->handshake_wq, &md->handshake_work); + md->exp_id &= ~D2H_INT_ASYNC_MD_HK; + iowrite32(D2H_INT_ASYNC_MD_HK, mhccif_base + REG_EP2RC_SW_INT_ACK); + t7xx_mhccif_mask_set(md->t7xx_dev, D2H_INT_ASYNC_MD_HK); + } + + if (md->exp_id & D2H_INT_ASYNC_AP_HK) { + queue_work(md->handshake_wq, &md->ap_handshake_work); + md->exp_id &= ~D2H_INT_ASYNC_AP_HK; + iowrite32(D2H_INT_ASYNC_AP_HK, mhccif_base + REG_EP2RC_SW_INT_ACK); + t7xx_mhccif_mask_set(md->t7xx_dev, D2H_INT_ASYNC_AP_HK); + } } spin_unlock_irqrestore(&md->exp_lock, flags); @@ -578,6 +604,7 @@ void t7xx_md_event_notify(struct t7xx_modem *md, enum md_event_id evt_id) case FSM_READY: t7xx_mhccif_mask_set(md->t7xx_dev, D2H_INT_ASYNC_MD_HK); + t7xx_mhccif_mask_set(md->t7xx_dev, D2H_INT_ASYNC_AP_HK); break; default: @@ -629,6 +656,12 @@ static struct t7xx_modem *t7xx_md_alloc(struct t7xx_pci_dev *t7xx_dev) md->core_md.feature_set[RT_ID_MD_PORT_ENUM] &= ~FEATURE_MSK; md->core_md.feature_set[RT_ID_MD_PORT_ENUM] |= FIELD_PREP(FEATURE_MSK, MTK_FEATURE_MUST_BE_SUPPORTED); + + INIT_WORK(&md->ap_handshake_work, t7xx_ap_hk_wq); + md->core_ap.feature_set[RT_ID_AP_PORT_ENUM] &= ~FEATURE_MSK; + md->core_ap.feature_set[RT_ID_AP_PORT_ENUM] |= + FIELD_PREP(FEATURE_MSK, MTK_FEATURE_MUST_BE_SUPPORTED); + return md; } @@ -640,6 +673,7 @@ int t7xx_md_reset(struct t7xx_pci_dev *t7xx_dev) md->exp_id = 0; t7xx_fsm_reset(md); t7xx_cldma_reset(md->md_ctrl[CLDMA_ID_MD]); + t7xx_cldma_reset(md->md_ctrl[CLDMA_ID_AP]); t7xx_port_proxy_reset(md->port_prox); md->md_init_finish = true; return t7xx_core_reset(md); @@ -669,6 +703,10 @@ int t7xx_md_init(struct t7xx_pci_dev *t7xx_dev) if (ret) goto err_destroy_hswq; + ret = t7xx_cldma_alloc(CLDMA_ID_AP, t7xx_dev); + if (ret) + goto err_destroy_hswq; + ret = t7xx_fsm_init(md); if (ret) goto err_destroy_hswq; @@ -681,12 +719,16 @@ int t7xx_md_init(struct t7xx_pci_dev *t7xx_dev) if (ret) goto err_uninit_ccmni; - ret = t7xx_port_proxy_init(md); + ret = t7xx_cldma_init(md->md_ctrl[CLDMA_ID_AP]); if (ret) goto err_uninit_md_cldma; + ret = t7xx_port_proxy_init(md); + if (ret) + goto err_uninit_ap_cldma; + ret = t7xx_fsm_append_cmd(md->fsm_ctl, FSM_CMD_START, 0); - if (ret) /* fsm_uninit flushes cmd queue */ + if (ret) /* t7xx_fsm_uninit() flushes cmd queue */ goto err_uninit_proxy; t7xx_md_sys_sw_init(t7xx_dev); @@ -696,6 +738,9 @@ int t7xx_md_init(struct t7xx_pci_dev *t7xx_dev) err_uninit_proxy: t7xx_port_proxy_uninit(md->port_prox); +err_uninit_ap_cldma: + t7xx_cldma_exit(md->md_ctrl[CLDMA_ID_AP]); + err_uninit_md_cldma: t7xx_cldma_exit(md->md_ctrl[CLDMA_ID_MD]); @@ -722,6 +767,7 @@ void t7xx_md_exit(struct t7xx_pci_dev *t7xx_dev) t7xx_fsm_append_cmd(md->fsm_ctl, FSM_CMD_PRE_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION); t7xx_port_proxy_uninit(md->port_prox); + t7xx_cldma_exit(md->md_ctrl[CLDMA_ID_AP]); t7xx_cldma_exit(md->md_ctrl[CLDMA_ID_MD]); t7xx_ccmni_exit(t7xx_dev); t7xx_fsm_uninit(md); diff --git a/drivers/net/wwan/t7xx/t7xx_modem_ops.h b/drivers/net/wwan/t7xx/t7xx_modem_ops.h index 7469ed636ae8..abe633cf7adc 100644 --- a/drivers/net/wwan/t7xx/t7xx_modem_ops.h +++ b/drivers/net/wwan/t7xx/t7xx_modem_ops.h @@ -66,10 +66,12 @@ struct t7xx_modem { struct cldma_ctrl *md_ctrl[CLDMA_NUM]; struct t7xx_pci_dev *t7xx_dev; struct t7xx_sys_info core_md; + struct t7xx_sys_info core_ap; bool md_init_finish; bool rgu_irq_asserted; struct workqueue_struct *handshake_wq; struct work_struct handshake_work; + struct work_struct ap_handshake_work; struct t7xx_fsm_ctl *fsm_ctl; struct port_proxy *port_prox; unsigned int exp_id; diff --git a/drivers/net/wwan/t7xx/t7xx_port.h b/drivers/net/wwan/t7xx/t7xx_port.h index 8ea9079af997..4ae8a00a8532 100644 --- a/drivers/net/wwan/t7xx/t7xx_port.h +++ b/drivers/net/wwan/t7xx/t7xx_port.h @@ -36,9 +36,13 @@ /* Channel ID and Message ID definitions. * The channel number consists of peer_id(15:12) , channel_id(11:0) * peer_id: - * 0:reserved, 1: to sAP, 2: to MD + * 0:reserved, 1: to AP, 2: to MD */ enum port_ch { + /* to AP */ + PORT_CH_AP_CONTROL_RX = 0x1000, + PORT_CH_AP_CONTROL_TX = 0x1001, + /* to MD */ PORT_CH_CONTROL_RX = 0x2000, PORT_CH_CONTROL_TX = 0x2001, diff --git a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c index 68430b130a67..ae632ef96698 100644 --- a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c +++ b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c @@ -167,8 +167,12 @@ static int control_msg_handler(struct t7xx_port *port, struct sk_buff *skb) case CTL_ID_HS2_MSG: skb_pull(skb, sizeof(*ctrl_msg_h)); - if (port_conf->rx_ch == PORT_CH_CONTROL_RX) { - ret = t7xx_fsm_append_event(ctl, FSM_EVENT_MD_HS2, skb->data, + if (port_conf->rx_ch == PORT_CH_CONTROL_RX || + port_conf->rx_ch == PORT_CH_AP_CONTROL_RX) { + int event = port_conf->rx_ch == PORT_CH_CONTROL_RX ? + FSM_EVENT_MD_HS2 : FSM_EVENT_AP_HS2; + + ret = t7xx_fsm_append_event(ctl, event, skb->data, le32_to_cpu(ctrl_msg_h->data_length)); if (ret) dev_err(port->dev, "Failed to append Handshake 2 event"); diff --git a/drivers/net/wwan/t7xx/t7xx_port_proxy.c b/drivers/net/wwan/t7xx/t7xx_port_proxy.c index 894b1d11b2c9..274846d39fbf 100644 --- a/drivers/net/wwan/t7xx/t7xx_port_proxy.c +++ b/drivers/net/wwan/t7xx/t7xx_port_proxy.c @@ -48,7 +48,7 @@ i < (proxy)->port_count; \ i++, (p) = &(proxy)->ports[i]) -static const struct t7xx_port_conf t7xx_md_port_conf[] = { +static const struct t7xx_port_conf t7xx_port_conf[] = { { .tx_ch = PORT_CH_UART2_TX, .rx_ch = PORT_CH_UART2_RX, @@ -89,6 +89,14 @@ static const struct t7xx_port_conf t7xx_md_port_conf[] = { .path_id = CLDMA_ID_MD, .ops = &ctl_port_ops, .name = "t7xx_ctrl", + }, { + .tx_ch = PORT_CH_AP_CONTROL_TX, + .rx_ch = PORT_CH_AP_CONTROL_RX, + .txq_index = Q_IDX_CTRL, + .rxq_index = Q_IDX_CTRL, + .path_id = CLDMA_ID_AP, + .ops = &ctl_port_ops, + .name = "t7xx_ap_ctrl", }, }; @@ -428,6 +436,9 @@ static void t7xx_proxy_init_all_ports(struct t7xx_modem *md) if (port_conf->tx_ch == PORT_CH_CONTROL_TX) md->core_md.ctl_port = port; + if (port_conf->tx_ch == PORT_CH_AP_CONTROL_TX) + md->core_ap.ctl_port = port; + port->t7xx_dev = md->t7xx_dev; port->dev = &md->t7xx_dev->pdev->dev; spin_lock_init(&port->port_update_lock); @@ -442,7 +453,7 @@ static void t7xx_proxy_init_all_ports(struct t7xx_modem *md) static int t7xx_proxy_alloc(struct t7xx_modem *md) { - unsigned int port_count = ARRAY_SIZE(t7xx_md_port_conf); + unsigned int port_count = ARRAY_SIZE(t7xx_port_conf); struct device *dev = &md->t7xx_dev->pdev->dev; struct port_proxy *port_prox; int i; @@ -456,7 +467,7 @@ static int t7xx_proxy_alloc(struct t7xx_modem *md) port_prox->dev = dev; for (i = 0; i < port_count; i++) - port_prox->ports[i].port_conf = &t7xx_md_port_conf[i]; + port_prox->ports[i].port_conf = &t7xx_port_conf[i]; port_prox->port_count = port_count; t7xx_proxy_init_all_ports(md); @@ -481,6 +492,7 @@ int t7xx_port_proxy_init(struct t7xx_modem *md) if (ret) return ret; + t7xx_cldma_set_recv_skb(md->md_ctrl[CLDMA_ID_AP], t7xx_port_proxy_recv_skb); t7xx_cldma_set_recv_skb(md->md_ctrl[CLDMA_ID_MD], t7xx_port_proxy_recv_skb); return 0; } diff --git a/drivers/net/wwan/t7xx/t7xx_reg.h b/drivers/net/wwan/t7xx/t7xx_reg.h index 7c1b81091a0f..c41d7d094c08 100644 --- a/drivers/net/wwan/t7xx/t7xx_reg.h +++ b/drivers/net/wwan/t7xx/t7xx_reg.h @@ -56,7 +56,7 @@ #define D2H_INT_RESUME_ACK BIT(12) #define D2H_INT_SUSPEND_ACK_AP BIT(13) #define D2H_INT_RESUME_ACK_AP BIT(14) -#define D2H_INT_ASYNC_SAP_HK BIT(15) +#define D2H_INT_ASYNC_AP_HK BIT(15) #define D2H_INT_ASYNC_MD_HK BIT(16) /* Register base */ diff --git a/drivers/net/wwan/t7xx/t7xx_state_monitor.c b/drivers/net/wwan/t7xx/t7xx_state_monitor.c index 0bcca08ff2bd..80edb8e75a6a 100644 --- a/drivers/net/wwan/t7xx/t7xx_state_monitor.c +++ b/drivers/net/wwan/t7xx/t7xx_state_monitor.c @@ -285,8 +285,9 @@ static int fsm_routine_starting(struct t7xx_fsm_ctl *ctl) t7xx_fsm_broadcast_state(ctl, MD_STATE_WAITING_FOR_HS1); t7xx_md_event_notify(md, FSM_START); - wait_event_interruptible_timeout(ctl->async_hk_wq, md->core_md.ready || ctl->exp_flg, - HZ * 60); + wait_event_interruptible_timeout(ctl->async_hk_wq, + (md->core_md.ready && md->core_ap.ready) || + ctl->exp_flg, HZ * 60); dev = &md->t7xx_dev->pdev->dev; if (ctl->exp_flg) @@ -299,6 +300,13 @@ static int fsm_routine_starting(struct t7xx_fsm_ctl *ctl) fsm_routine_exception(ctl, NULL, EXCEPTION_HS_TIMEOUT); return -ETIMEDOUT; + } else if (!md->core_ap.ready) { + dev_err(dev, "AP handshake timeout\n"); + if (md->core_ap.handshake_ongoing) + t7xx_fsm_append_event(ctl, FSM_EVENT_AP_HS2_EXIT, NULL, 0); + + fsm_routine_exception(ctl, NULL, EXCEPTION_HS_TIMEOUT); + return -ETIMEDOUT; } t7xx_pci_pm_init_late(md->t7xx_dev); @@ -335,6 +343,7 @@ static void fsm_routine_start(struct t7xx_fsm_ctl *ctl, struct t7xx_fsm_command return; } + t7xx_cldma_hif_hw_init(md->md_ctrl[CLDMA_ID_AP]); t7xx_cldma_hif_hw_init(md->md_ctrl[CLDMA_ID_MD]); fsm_finish_command(ctl, cmd, fsm_routine_starting(ctl)); } diff --git a/drivers/net/wwan/t7xx/t7xx_state_monitor.h b/drivers/net/wwan/t7xx/t7xx_state_monitor.h index b1af0259d4c5..b6e76f3903c8 100644 --- a/drivers/net/wwan/t7xx/t7xx_state_monitor.h +++ b/drivers/net/wwan/t7xx/t7xx_state_monitor.h @@ -38,10 +38,12 @@ enum t7xx_fsm_state { enum t7xx_fsm_event_state { FSM_EVENT_INVALID, FSM_EVENT_MD_HS2, + FSM_EVENT_AP_HS2, FSM_EVENT_MD_EX, FSM_EVENT_MD_EX_REC_OK, FSM_EVENT_MD_EX_PASS, FSM_EVENT_MD_HS2_EXIT, + FSM_EVENT_AP_HS2_EXIT, FSM_EVENT_MAX }; diff --git a/drivers/ptp/ptp_qoriq.c b/drivers/ptp/ptp_qoriq.c index 350154e4c2b5..a52859d024f0 100644 --- a/drivers/ptp/ptp_qoriq.c +++ b/drivers/ptp/ptp_qoriq.c @@ -12,7 +12,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/timex.h> #include <linux/slab.h> #include <linux/clk.h> diff --git a/drivers/s390/net/Kconfig b/drivers/s390/net/Kconfig index 9c67b97faba2..74760c1a163b 100644 --- a/drivers/s390/net/Kconfig +++ b/drivers/s390/net/Kconfig @@ -5,12 +5,11 @@ menu "S/390 network device drivers" config LCS def_tristate m prompt "Lan Channel Station Interface" - depends on CCW && NETDEVICES && (ETHERNET || FDDI) + depends on CCW && NETDEVICES && ETHERNET help Select this option if you want to use LCS networking on IBM System z. - This device driver supports FDDI (IEEE 802.7) and Ethernet. To compile as a module, choose M. The module name is lcs. - If you do not know what it is, it's safe to choose Y. + If you do not use LCS, choose N. config CTCM def_tristate m diff --git a/drivers/s390/net/lcs.c b/drivers/s390/net/lcs.c index 9fd8e6f07a03..a1f2acd6fb8f 100644 --- a/drivers/s390/net/lcs.c +++ b/drivers/s390/net/lcs.c @@ -17,7 +17,6 @@ #include <linux/if.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> -#include <linux/fddidevice.h> #include <linux/inetdevice.h> #include <linux/in.h> #include <linux/igmp.h> @@ -36,10 +35,6 @@ #include "lcs.h" -#if !defined(CONFIG_ETHERNET) && !defined(CONFIG_FDDI) -#error Cannot compile lcs.c without some net devices switched on. -#endif - /* * initialization string for output */ @@ -1601,19 +1596,11 @@ lcs_startlan_auto(struct lcs_card *card) int rc; LCS_DBF_TEXT(2, trace, "strtauto"); -#ifdef CONFIG_ETHERNET card->lan_type = LCS_FRAME_TYPE_ENET; rc = lcs_send_startlan(card, LCS_INITIATOR_TCPIP); if (rc == 0) return 0; -#endif -#ifdef CONFIG_FDDI - card->lan_type = LCS_FRAME_TYPE_FDDI; - rc = lcs_send_startlan(card, LCS_INITIATOR_TCPIP); - if (rc == 0) - return 0; -#endif return -EIO; } @@ -1806,22 +1793,16 @@ lcs_get_frames_cb(struct lcs_channel *channel, struct lcs_buffer *buffer) card->stats.rx_errors++; return; } - /* What kind of frame is it? */ - if (lcs_hdr->type == LCS_FRAME_TYPE_CONTROL) { - /* Control frame. */ + if (lcs_hdr->type == LCS_FRAME_TYPE_CONTROL) lcs_get_control(card, (struct lcs_cmd *) lcs_hdr); - } else if (lcs_hdr->type == LCS_FRAME_TYPE_ENET || - lcs_hdr->type == LCS_FRAME_TYPE_TR || - lcs_hdr->type == LCS_FRAME_TYPE_FDDI) { - /* Normal network packet. */ + else if (lcs_hdr->type == LCS_FRAME_TYPE_ENET) lcs_get_skb(card, (char *)(lcs_hdr + 1), lcs_hdr->offset - offset - sizeof(struct lcs_header)); - } else { - /* Unknown frame type. */ - ; // FIXME: error message ? - } - /* Proceed to next frame. */ + else + dev_info_once(&card->dev->dev, + "Unknown frame type %d\n", + lcs_hdr->type); offset = lcs_hdr->offset; lcs_hdr->offset = LCS_ILLEGAL_OFFSET; lcs_hdr = (struct lcs_header *) (buffer->data + offset); @@ -2140,18 +2121,10 @@ lcs_new_device(struct ccwgroup_device *ccwgdev) goto netdev_out; } switch (card->lan_type) { -#ifdef CONFIG_ETHERNET case LCS_FRAME_TYPE_ENET: card->lan_type_trans = eth_type_trans; dev = alloc_etherdev(0); break; -#endif -#ifdef CONFIG_FDDI - case LCS_FRAME_TYPE_FDDI: - card->lan_type_trans = fddi_type_trans; - dev = alloc_fddidev(0); - break; -#endif default: LCS_DBF_TEXT(3, setup, "errinit"); pr_err(" Initialization failed\n"); diff --git a/drivers/w1/w1_netlink.c b/drivers/w1/w1_netlink.c index db110cc442b1..691978cddab7 100644 --- a/drivers/w1/w1_netlink.c +++ b/drivers/w1/w1_netlink.c @@ -65,7 +65,8 @@ static void w1_unref_block(struct w1_cb_block *block) u16 len = w1_reply_len(block); if (len) { cn_netlink_send_mult(block->first_cn, len, - block->portid, 0, GFP_KERNEL); + block->portid, 0, + GFP_KERNEL, NULL, NULL); } kfree(block); } @@ -83,7 +84,8 @@ static void w1_reply_make_space(struct w1_cb_block *block, u16 space) { u16 len = w1_reply_len(block); if (len + space >= block->maxlen) { - cn_netlink_send_mult(block->first_cn, len, block->portid, 0, GFP_KERNEL); + cn_netlink_send_mult(block->first_cn, len, block->portid, + 0, GFP_KERNEL, NULL, NULL); block->first_cn->len = 0; block->cn = NULL; block->msg = NULL; diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 57e9e109257e..8506690dbb9c 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -199,9 +199,9 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \ + if (cgroup_bpf_enabled(CGROUP_INET_EGRESS) && sk) { \ typeof(sk) __sk = sk_to_full_sk(sk); \ - if (sk_fullsock(__sk) && \ + if (sk_fullsock(__sk) && __sk == skb_to_full_sk(skb) && \ cgroup_bpf_sock_enabled(__sk, CGROUP_INET_EGRESS)) \ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \ CGROUP_INET_EGRESS); \ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f58895830ada..ceaa8c23287f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -228,6 +228,18 @@ struct btf_record { struct btf_field fields[]; }; +/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */ +struct bpf_rb_node_kern { + struct rb_node rb_node; + void *owner; +} __attribute__((aligned(8))); + +/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */ +struct bpf_list_node_kern { + struct list_head list_head; + void *owner; +} __attribute__((aligned(8))); + struct bpf_map { /* The first two cachelines with read-mostly members of which some * are also accessed in fast-path (e.g. ops, max_entries). @@ -275,6 +287,7 @@ struct bpf_map { } owner; bool bypass_spec_v1; bool frozen; /* write-once; write-protected by freeze_mutex */ + s64 __percpu *elem_count; }; static inline const char *btf_field_type_name(enum btf_field_type type) @@ -2040,6 +2053,35 @@ bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, } #endif +static inline int +bpf_map_init_elem_count(struct bpf_map *map) +{ + size_t size = sizeof(*map->elem_count), align = size; + gfp_t flags = GFP_USER | __GFP_NOWARN; + + map->elem_count = bpf_map_alloc_percpu(map, size, align, flags); + if (!map->elem_count) + return -ENOMEM; + + return 0; +} + +static inline void +bpf_map_free_elem_count(struct bpf_map *map) +{ + free_percpu(map->elem_count); +} + +static inline void bpf_map_inc_elem_count(struct bpf_map *map) +{ + this_cpu_inc(*map->elem_count); +} + +static inline void bpf_map_dec_elem_count(struct bpf_map *map) +{ + this_cpu_dec(*map->elem_count); +} + extern int sysctl_unprivileged_bpf_disabled; static inline bool bpf_allow_ptr_leaks(void) diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h index 3929be5743f4..d644bbb298af 100644 --- a/include/linux/bpf_mem_alloc.h +++ b/include/linux/bpf_mem_alloc.h @@ -27,10 +27,12 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma); /* kmalloc/kfree equivalent: */ void *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size); void bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr); +void bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr); /* kmem_cache_alloc/free equivalent: */ void *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma); void bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr); +void bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr); void bpf_mem_cache_raw_free(void *ptr); void *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags); diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h new file mode 100644 index 000000000000..929225f7b095 --- /dev/null +++ b/include/linux/bpf_mprog.h @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Isovalent */ +#ifndef __BPF_MPROG_H +#define __BPF_MPROG_H + +#include <linux/bpf.h> + +/* bpf_mprog framework: + * + * bpf_mprog is a generic layer for multi-program attachment. In-kernel users + * of the bpf_mprog don't need to care about the dependency resolution + * internals, they can just consume it with few API calls. Currently available + * dependency directives are BPF_F_{BEFORE,AFTER} which enable insertion of + * a BPF program or BPF link relative to an existing BPF program or BPF link + * inside the multi-program array as well as prepend and append behavior if + * no relative object was specified, see corresponding selftests for concrete + * examples (e.g. tc_links and tc_opts test cases of test_progs). + * + * Usage of bpf_mprog_{attach,detach,query}() core APIs with pseudo code: + * + * Attach case: + * + * struct bpf_mprog_entry *entry, *entry_new; + * int ret; + * + * // bpf_mprog user-side lock + * // fetch active @entry from attach location + * [...] + * ret = bpf_mprog_attach(entry, &entry_new, [...]); + * if (!ret) { + * if (entry != entry_new) { + * // swap @entry to @entry_new at attach location + * // ensure there are no inflight users of @entry: + * synchronize_rcu(); + * } + * bpf_mprog_commit(entry); + * } else { + * // error path, bail out, propagate @ret + * } + * // bpf_mprog user-side unlock + * + * Detach case: + * + * struct bpf_mprog_entry *entry, *entry_new; + * int ret; + * + * // bpf_mprog user-side lock + * // fetch active @entry from attach location + * [...] + * ret = bpf_mprog_detach(entry, &entry_new, [...]); + * if (!ret) { + * // all (*) marked is optional and depends on the use-case + * // whether bpf_mprog_bundle should be freed or not + * if (!bpf_mprog_total(entry_new)) (*) + * entry_new = NULL (*) + * // swap @entry to @entry_new at attach location + * // ensure there are no inflight users of @entry: + * synchronize_rcu(); + * bpf_mprog_commit(entry); + * if (!entry_new) (*) + * // free bpf_mprog_bundle (*) + * } else { + * // error path, bail out, propagate @ret + * } + * // bpf_mprog user-side unlock + * + * Query case: + * + * struct bpf_mprog_entry *entry; + * int ret; + * + * // bpf_mprog user-side lock + * // fetch active @entry from attach location + * [...] + * ret = bpf_mprog_query(attr, uattr, entry); + * // bpf_mprog user-side unlock + * + * Data/fast path: + * + * struct bpf_mprog_entry *entry; + * struct bpf_mprog_fp *fp; + * struct bpf_prog *prog; + * int ret = [...]; + * + * rcu_read_lock(); + * // fetch active @entry from attach location + * [...] + * bpf_mprog_foreach_prog(entry, fp, prog) { + * ret = bpf_prog_run(prog, [...]); + * // process @ret from program + * } + * [...] + * rcu_read_unlock(); + * + * bpf_mprog locking considerations: + * + * bpf_mprog_{attach,detach,query}() must be protected by an external lock + * (like RTNL in case of tcx). + * + * bpf_mprog_entry pointer can be an __rcu annotated pointer (in case of tcx + * the netdevice has tcx_ingress and tcx_egress __rcu pointer) which gets + * updated via rcu_assign_pointer() pointing to the active bpf_mprog_entry of + * the bpf_mprog_bundle. + * + * Fast path accesses the active bpf_mprog_entry within RCU critical section + * (in case of tcx it runs in NAPI which provides RCU protection there, + * other users might need explicit rcu_read_lock()). The bpf_mprog_commit() + * assumes that for the old bpf_mprog_entry there are no inflight users + * anymore. + * + * The READ_ONCE()/WRITE_ONCE() pairing for bpf_mprog_fp's prog access is for + * the replacement case where we don't swap the bpf_mprog_entry. + */ + +#define bpf_mprog_foreach_tuple(entry, fp, cp, t) \ + for (fp = &entry->fp_items[0], cp = &entry->parent->cp_items[0];\ + ({ \ + t.prog = READ_ONCE(fp->prog); \ + t.link = cp->link; \ + t.prog; \ + }); \ + fp++, cp++) + +#define bpf_mprog_foreach_prog(entry, fp, p) \ + for (fp = &entry->fp_items[0]; \ + (p = READ_ONCE(fp->prog)); \ + fp++) + +#define BPF_MPROG_MAX 64 + +struct bpf_mprog_fp { + struct bpf_prog *prog; +}; + +struct bpf_mprog_cp { + struct bpf_link *link; +}; + +struct bpf_mprog_entry { + struct bpf_mprog_fp fp_items[BPF_MPROG_MAX]; + struct bpf_mprog_bundle *parent; +}; + +struct bpf_mprog_bundle { + struct bpf_mprog_entry a; + struct bpf_mprog_entry b; + struct bpf_mprog_cp cp_items[BPF_MPROG_MAX]; + struct bpf_prog *ref; + atomic64_t revision; + u32 count; +}; + +struct bpf_tuple { + struct bpf_prog *prog; + struct bpf_link *link; +}; + +static inline struct bpf_mprog_entry * +bpf_mprog_peer(const struct bpf_mprog_entry *entry) +{ + if (entry == &entry->parent->a) + return &entry->parent->b; + else + return &entry->parent->a; +} + +static inline void bpf_mprog_bundle_init(struct bpf_mprog_bundle *bundle) +{ + BUILD_BUG_ON(sizeof(bundle->a.fp_items[0]) > sizeof(u64)); + BUILD_BUG_ON(ARRAY_SIZE(bundle->a.fp_items) != + ARRAY_SIZE(bundle->cp_items)); + + memset(bundle, 0, sizeof(*bundle)); + atomic64_set(&bundle->revision, 1); + bundle->a.parent = bundle; + bundle->b.parent = bundle; +} + +static inline void bpf_mprog_inc(struct bpf_mprog_entry *entry) +{ + entry->parent->count++; +} + +static inline void bpf_mprog_dec(struct bpf_mprog_entry *entry) +{ + entry->parent->count--; +} + +static inline int bpf_mprog_max(void) +{ + return ARRAY_SIZE(((struct bpf_mprog_entry *)NULL)->fp_items) - 1; +} + +static inline int bpf_mprog_total(struct bpf_mprog_entry *entry) +{ + int total = entry->parent->count; + + WARN_ON_ONCE(total > bpf_mprog_max()); + return total; +} + +static inline bool bpf_mprog_exists(struct bpf_mprog_entry *entry, + struct bpf_prog *prog) +{ + const struct bpf_mprog_fp *fp; + const struct bpf_prog *tmp; + + bpf_mprog_foreach_prog(entry, fp, tmp) { + if (tmp == prog) + return true; + } + return false; +} + +static inline void bpf_mprog_mark_for_release(struct bpf_mprog_entry *entry, + struct bpf_tuple *tuple) +{ + WARN_ON_ONCE(entry->parent->ref); + if (!tuple->link) + entry->parent->ref = tuple->prog; +} + +static inline void bpf_mprog_complete_release(struct bpf_mprog_entry *entry) +{ + /* In the non-link case prog deletions can only drop the reference + * to the prog after the bpf_mprog_entry got swapped and the + * bpf_mprog ensured that there are no inflight users anymore. + * + * Paired with bpf_mprog_mark_for_release(). + */ + if (entry->parent->ref) { + bpf_prog_put(entry->parent->ref); + entry->parent->ref = NULL; + } +} + +static inline void bpf_mprog_revision_new(struct bpf_mprog_entry *entry) +{ + atomic64_inc(&entry->parent->revision); +} + +static inline void bpf_mprog_commit(struct bpf_mprog_entry *entry) +{ + bpf_mprog_complete_release(entry); + bpf_mprog_revision_new(entry); +} + +static inline u64 bpf_mprog_revision(struct bpf_mprog_entry *entry) +{ + return atomic64_read(&entry->parent->revision); +} + +static inline void bpf_mprog_entry_copy(struct bpf_mprog_entry *dst, + struct bpf_mprog_entry *src) +{ + memcpy(dst->fp_items, src->fp_items, sizeof(src->fp_items)); +} + +static inline void bpf_mprog_entry_clear(struct bpf_mprog_entry *dst) +{ + memset(dst->fp_items, 0, sizeof(dst->fp_items)); +} + +static inline void bpf_mprog_clear_all(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new) +{ + struct bpf_mprog_entry *peer; + + peer = bpf_mprog_peer(entry); + bpf_mprog_entry_clear(peer); + peer->parent->count = 0; + *entry_new = peer; +} + +static inline void bpf_mprog_entry_grow(struct bpf_mprog_entry *entry, int idx) +{ + int total = bpf_mprog_total(entry); + + memmove(entry->fp_items + idx + 1, + entry->fp_items + idx, + (total - idx) * sizeof(struct bpf_mprog_fp)); + + memmove(entry->parent->cp_items + idx + 1, + entry->parent->cp_items + idx, + (total - idx) * sizeof(struct bpf_mprog_cp)); +} + +static inline void bpf_mprog_entry_shrink(struct bpf_mprog_entry *entry, int idx) +{ + /* Total array size is needed in this case to enure the NULL + * entry is copied at the end. + */ + int total = ARRAY_SIZE(entry->fp_items); + + memmove(entry->fp_items + idx, + entry->fp_items + idx + 1, + (total - idx - 1) * sizeof(struct bpf_mprog_fp)); + + memmove(entry->parent->cp_items + idx, + entry->parent->cp_items + idx + 1, + (total - idx - 1) * sizeof(struct bpf_mprog_cp)); +} + +static inline void bpf_mprog_read(struct bpf_mprog_entry *entry, u32 idx, + struct bpf_mprog_fp **fp, + struct bpf_mprog_cp **cp) +{ + *fp = &entry->fp_items[idx]; + *cp = &entry->parent->cp_items[idx]; +} + +static inline void bpf_mprog_write(struct bpf_mprog_fp *fp, + struct bpf_mprog_cp *cp, + struct bpf_tuple *tuple) +{ + WRITE_ONCE(fp->prog, tuple->prog); + cp->link = tuple->link; +} + +int bpf_mprog_attach(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_prog *prog_new, struct bpf_link *link, + struct bpf_prog *prog_old, + u32 flags, u32 id_or_fd, u64 revision); + +int bpf_mprog_detach(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_prog *prog, struct bpf_link *link, + u32 flags, u32 id_or_fd, u64 revision); + +int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr, + struct bpf_mprog_entry *entry); + +static inline bool bpf_mprog_supported(enum bpf_prog_type type) +{ + switch (type) { + case BPF_PROG_TYPE_SCHED_CLS: + return true; + default: + return false; + } +} +#endif /* __BPF_MPROG_H */ diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 5d732f48f787..c55810a43541 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -44,6 +44,7 @@ #define PHY_ID_BCM7366 0x600d8490 #define PHY_ID_BCM7346 0x600d8650 #define PHY_ID_BCM7362 0x600d84b0 +#define PHY_ID_BCM74165 0x359052c0 #define PHY_ID_BCM7425 0x600d86b0 #define PHY_ID_BCM7429 0x600d8730 #define PHY_ID_BCM7435 0x600d8750 diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 00950cc03bff..a3462a9b8e18 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -267,5 +267,6 @@ MAX_BTF_TRACING_TYPE, extern u32 btf_tracing_ids[]; extern u32 bpf_cgroup_btf_id[]; extern u32 bpf_local_storage_map_btf_id[]; +extern u32 btf_bpf_map_id[]; #endif diff --git a/include/linux/can/rx-offload.h b/include/linux/can/rx-offload.h index c205c51d79c9..d29bb4521947 100644 --- a/include/linux/can/rx-offload.h +++ b/include/linux/can/rx-offload.h @@ -3,7 +3,7 @@ * linux/can/rx-offload.h * * Copyright (c) 2014 David Jander, Protonic Holland - * Copyright (c) 2014-2017 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de> + * Copyright (c) 2014-2017, 2023 Pengutronix, Marc Kleine-Budde <kernel@pengutronix.de> */ #ifndef _CAN_RX_OFFLOAD_H @@ -44,11 +44,14 @@ int can_rx_offload_irq_offload_timestamp(struct can_rx_offload *offload, int can_rx_offload_irq_offload_fifo(struct can_rx_offload *offload); int can_rx_offload_queue_timestamp(struct can_rx_offload *offload, struct sk_buff *skb, u32 timestamp); -unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, - unsigned int idx, u32 timestamp, - unsigned int *frame_len_ptr); +unsigned int can_rx_offload_get_echo_skb_queue_timestamp(struct can_rx_offload *offload, + unsigned int idx, u32 timestamp, + unsigned int *frame_len_ptr); int can_rx_offload_queue_tail(struct can_rx_offload *offload, struct sk_buff *skb); +unsigned int can_rx_offload_get_echo_skb_queue_tail(struct can_rx_offload *offload, + unsigned int idx, + unsigned int *frame_len_ptr); void can_rx_offload_irq_finish(struct can_rx_offload *offload); void can_rx_offload_threaded_irq_finish(struct can_rx_offload *offload); void can_rx_offload_del(struct can_rx_offload *offload); diff --git a/include/linux/connector.h b/include/linux/connector.h index 487350bb19c3..cec2d99ae902 100644 --- a/include/linux/connector.h +++ b/include/linux/connector.h @@ -90,13 +90,19 @@ void cn_del_callback(const struct cb_id *id); * If @group is not zero, then message will be delivered * to the specified group. * @gfp_mask: GFP mask. + * @filter: Filter function to be used at netlink layer. + * @filter_data:Filter data to be supplied to the filter function * * It can be safely called from softirq context, but may silently * fail under strong memory pressure. * * If there are no listeners for given group %-ESRCH can be returned. */ -int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 group, gfp_t gfp_mask); +int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, + u32 group, gfp_t gfp_mask, + int (*filter)(struct sock *dsk, struct sk_buff *skb, + void *data), + void *filter_data); /** * cn_netlink_send - Sends message to the specified groups. diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index db0f4fcfdaf4..e3b3b0fa2a8f 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -85,12 +85,10 @@ extern void icmpv6_param_prob_reason(struct sk_buff *skb, struct flowi6; struct in6_addr; -extern void icmpv6_flow_init(struct sock *sk, - struct flowi6 *fl6, - u8 type, - const struct in6_addr *saddr, - const struct in6_addr *daddr, - int oif); + +void icmpv6_flow_init(const struct sock *sk, struct flowi6 *fl6, u8 type, + const struct in6_addr *saddr, + const struct in6_addr *daddr, int oif); static inline void icmpv6_param_prob(struct sk_buff *skb, u8 code, int pos) { diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 839247a4f48e..5883551b1ee8 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -33,6 +33,7 @@ struct ipv6_devconf { __s32 accept_ra_defrtr; __u32 ra_defrtr_metric; __s32 accept_ra_min_hop_limit; + __s32 accept_ra_min_lft; __s32 accept_ra_pinfo; __s32 ignore_routes_with_linkdown; #ifdef CONFIG_IPV6_ROUTER_PREF @@ -199,14 +200,7 @@ struct inet6_cork { u8 tclass; }; -/** - * struct ipv6_pinfo - ipv6 private area - * - * In the struct sock hierarchy (tcp6_sock, upd6_sock, etc) - * this _must_ be the last member, so that inet6_sk_generic - * is able to calculate its offset from the base struct sock - * by using the struct proto->slab_obj_size member. -acme - */ +/* struct ipv6_pinfo - ipv6 private area */ struct ipv6_pinfo { struct in6_addr saddr; struct in6_pktinfo sticky_pktinfo; @@ -306,19 +300,19 @@ struct raw6_sock { __u32 offset; /* checksum offset */ struct icmp6_filter filter; __u32 ip6mr_table; - /* ipv6_pinfo has to be the last member of raw6_sock, see inet6_sk_generic */ + struct ipv6_pinfo inet6; }; struct udp6_sock { struct udp_sock udp; - /* ipv6_pinfo has to be the last member of udp6_sock, see inet6_sk_generic */ + struct ipv6_pinfo inet6; }; struct tcp6_sock { struct tcp_sock tcp; - /* ipv6_pinfo has to be the last member of tcp6_sock, see inet6_sk_generic */ + struct ipv6_pinfo inet6; }; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7308a1a7599b..4f2621e87634 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -316,7 +316,7 @@ LSM_HOOK(int, 0, sk_alloc_security, struct sock *sk, int family, gfp_t priority) LSM_HOOK(void, LSM_RET_VOID, sk_free_security, struct sock *sk) LSM_HOOK(void, LSM_RET_VOID, sk_clone_security, const struct sock *sk, struct sock *newsk) -LSM_HOOK(void, LSM_RET_VOID, sk_getsecid, struct sock *sk, u32 *secid) +LSM_HOOK(void, LSM_RET_VOID, sk_getsecid, const struct sock *sk, u32 *secid) LSM_HOOK(void, LSM_RET_VOID, sock_graft, struct sock *sk, struct socket *parent) LSM_HOOK(int, 0, inet_conn_request, const struct sock *sk, struct sk_buff *skb, struct request_sock *req) diff --git a/include/linux/marvell_phy.h b/include/linux/marvell_phy.h index 0f06c2287b52..9b54c4f0677f 100644 --- a/include/linux/marvell_phy.h +++ b/include/linux/marvell_phy.h @@ -25,6 +25,7 @@ #define MARVELL_PHY_ID_88X3310 0x002b09a0 #define MARVELL_PHY_ID_88E2110 0x002b09b0 #define MARVELL_PHY_ID_88X2222 0x01410f10 +#define MARVELL_PHY_ID_88Q2110 0x002b0980 /* Marvel 88E1111 in Finisar SFP module with modified PHY ID */ #define MARVELL_PHY_ID_88E1111_FINISAR 0x01ff0cc0 diff --git a/include/linux/mdio.h b/include/linux/mdio.h index c1b7008826e5..8fa23bdcedbf 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -537,6 +537,8 @@ static inline void mii_c73_mod_linkmode(unsigned long *adv, u16 *lpa) int __mdiobus_read(struct mii_bus *bus, int addr, u32 regnum); int __mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val); +int __mdiobus_modify(struct mii_bus *bus, int addr, u32 regnum, u16 mask, + u16 set); int __mdiobus_modify_changed(struct mii_bus *bus, int addr, u32 regnum, u16 mask, u16 set); @@ -564,6 +566,30 @@ int mdiobus_c45_modify(struct mii_bus *bus, int addr, int devad, u32 regnum, int mdiobus_c45_modify_changed(struct mii_bus *bus, int addr, int devad, u32 regnum, u16 mask, u16 set); +static inline int __mdiodev_read(struct mdio_device *mdiodev, u32 regnum) +{ + return __mdiobus_read(mdiodev->bus, mdiodev->addr, regnum); +} + +static inline int __mdiodev_write(struct mdio_device *mdiodev, u32 regnum, + u16 val) +{ + return __mdiobus_write(mdiodev->bus, mdiodev->addr, regnum, val); +} + +static inline int __mdiodev_modify(struct mdio_device *mdiodev, u32 regnum, + u16 mask, u16 set) +{ + return __mdiobus_modify(mdiodev->bus, mdiodev->addr, regnum, mask, set); +} + +static inline int __mdiodev_modify_changed(struct mdio_device *mdiodev, + u32 regnum, u16 mask, u16 set) +{ + return __mdiobus_modify_changed(mdiodev->bus, mdiodev->addr, regnum, + mask, set); +} + static inline int mdiodev_read(struct mdio_device *mdiodev, u32 regnum) { return mdiobus_read(mdiodev->bus, mdiodev->addr, regnum); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 25d0528f9219..fa70c25423b2 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -287,18 +287,23 @@ struct mlx5_cmd_stats { struct mlx5_cmd { struct mlx5_nb nb; + /* members which needs to be queried or reinitialized each reload */ + struct { + u16 cmdif_rev; + u8 log_sz; + u8 log_stride; + int max_reg_cmds; + unsigned long bitmask; + struct semaphore sem; + struct semaphore pages_sem; + struct semaphore throttle_sem; + } vars; enum mlx5_cmdif_state state; void *cmd_alloc_buf; dma_addr_t alloc_dma; int alloc_size; void *cmd_buf; dma_addr_t dma; - u16 cmdif_rev; - u8 log_sz; - u8 log_stride; - int max_reg_cmds; - int events; - u32 __iomem *vector; /* protect command queue allocations */ @@ -308,12 +313,8 @@ struct mlx5_cmd { */ spinlock_t token_lock; u8 token; - unsigned long bitmask; char wq_name[MLX5_CMD_WQ_MAX_NAME]; struct workqueue_struct *wq; - struct semaphore sem; - struct semaphore pages_sem; - struct semaphore throttle_sem; int mode; u16 allowed_opcode; struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS]; @@ -321,7 +322,7 @@ struct mlx5_cmd { struct mlx5_cmd_debug dbg; struct cmd_msg_cache cache[MLX5_NUM_COMMAND_CACHES]; int checksum_disabled; - struct mlx5_cmd_stats stats[MLX5_CMD_OP_MAX]; + struct xarray stats; }; struct mlx5_cmd_mailbox { @@ -501,7 +502,7 @@ struct mlx5_events; struct mlx5_mpfs; struct mlx5_eswitch; struct mlx5_lag; -struct mlx5_devcom; +struct mlx5_devcom_dev; struct mlx5_fw_reset; struct mlx5_eq_table; struct mlx5_irq_table; @@ -618,7 +619,7 @@ struct mlx5_priv { struct mlx5_core_sriov sriov; struct mlx5_lag *lag; u32 flags; - struct mlx5_devcom *devcom; + struct mlx5_devcom_dev *devc; struct mlx5_fw_reset *fw_reset; struct mlx5_core_roce roce; struct mlx5_fc_stats fc_stats; @@ -805,6 +806,8 @@ struct mlx5_core_dev { u32 vsc_addr; struct mlx5_hv_vhca *hv_vhca; struct mlx5_thermal *thermal; + u64 num_block_tc; + u64 num_block_ipsec; }; struct mlx5_db { diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index e2701ed0200e..950d2431a53c 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -144,6 +144,9 @@ u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, GENMASK(31 - ESW_TUN_ID_BITS - ESW_RESERVED_BITS, \ ESW_TUN_OPTS_OFFSET + 1) +/* reuse tun_opts for the mapped ipsec obj id when tun_id is 0 (invalid) */ +#define ESW_IPSEC_RX_MAPPED_ID_MASK GENMASK(ESW_TUN_OPTS_BITS - 1, 0) + u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev); u16 mlx5_eswitch_get_total_vports(const struct mlx5_core_dev *dev); struct mlx5_core_dev *mlx5_eswitch_get_core_dev(struct mlx5_eswitch *esw); diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 2cb404c7ea13..c302ec34255b 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -109,11 +109,13 @@ enum mlx5_flow_namespace_type { enum { FDB_BYPASS_PATH, + FDB_CRYPTO_INGRESS, FDB_TC_OFFLOAD, FDB_FT_OFFLOAD, FDB_TC_MISS, FDB_BR_OFFLOAD, FDB_SLOW_PATH, + FDB_CRYPTO_EGRESS, FDB_PER_VPORT, }; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 33344a71c3e3..b3ad6b9852ec 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -464,10 +464,10 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 reformat_add_esp_trasport[0x1]; u8 reformat_l2_to_l3_esp_tunnel[0x1]; - u8 reserved_at_42[0x1]; + u8 reformat_add_esp_transport_over_udp[0x1]; u8 reformat_del_esp_trasport[0x1]; u8 reformat_l3_esp_tunnel_to_l2[0x1]; - u8 reserved_at_45[0x1]; + u8 reformat_del_esp_transport_over_udp[0x1]; u8 execute_aso[0x1]; u8 reserved_at_47[0x19]; @@ -6665,9 +6665,12 @@ enum mlx5_reformat_ctx_type { MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4, MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5, MLX5_REFORMAT_TYPE_L2_TO_L3_ESP_TUNNEL = 0x6, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4 = 0x7, MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8, MLX5_REFORMAT_TYPE_L3_ESP_TUNNEL_TO_L2 = 0x9, + MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP = 0xa, MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6 = 0xc, MLX5_REFORMAT_TYPE_INSERT_HDR = 0xf, MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10, MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11, diff --git a/include/linux/net_tstamp.h b/include/linux/net_tstamp.h index fd67f3cc0c4b..eb01c37e71e0 100644 --- a/include/linux/net_tstamp.h +++ b/include/linux/net_tstamp.h @@ -5,12 +5,23 @@ #include <uapi/linux/net_tstamp.h> +enum hwtstamp_source { + HWTSTAMP_SOURCE_NETDEV, + HWTSTAMP_SOURCE_PHYLIB, +}; + /** * struct kernel_hwtstamp_config - Kernel copy of struct hwtstamp_config * * @flags: see struct hwtstamp_config * @tx_type: see struct hwtstamp_config * @rx_filter: see struct hwtstamp_config + * @ifr: pointer to ifreq structure from the original ioctl request, to pass to + * a legacy implementation of a lower driver + * @copied_to_user: request was passed to a legacy implementation which already + * copied the ioctl request back to user space + * @source: indication whether timestamps should come from the netdev or from + * an attached phylib PHY * * Prefer using this structure for in-kernel processing of hardware * timestamping configuration, over the inextensible struct hwtstamp_config @@ -20,6 +31,9 @@ struct kernel_hwtstamp_config { int flags; int tx_type; int rx_filter; + struct ifreq *ifr; + bool copied_to_user; + enum hwtstamp_source source; }; static inline void hwtstamp_config_to_kernel(struct kernel_hwtstamp_config *kernel_cfg, @@ -30,4 +44,20 @@ static inline void hwtstamp_config_to_kernel(struct kernel_hwtstamp_config *kern kernel_cfg->rx_filter = cfg->rx_filter; } +static inline void hwtstamp_config_from_kernel(struct hwtstamp_config *cfg, + const struct kernel_hwtstamp_config *kernel_cfg) +{ + cfg->flags = kernel_cfg->flags; + cfg->tx_type = kernel_cfg->tx_type; + cfg->rx_filter = kernel_cfg->rx_filter; +} + +static inline bool kernel_hwtstamp_config_changed(const struct kernel_hwtstamp_config *a, + const struct kernel_hwtstamp_config *b) +{ + return a->flags != b->flags || + a->tx_type != b->tx_type || + a->rx_filter != b->rx_filter; +} + #endif /* _LINUX_NET_TIMESTAMPING_H_ */ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index b828c7a75be2..4176a738177b 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -57,6 +57,7 @@ struct netpoll_info; struct device; struct ethtool_ops; +struct kernel_hwtstamp_config; struct phy_device; struct dsa_port; struct ip_tunnel_parm; @@ -1418,6 +1419,16 @@ struct netdev_net_notifier { * Get hardware timestamp based on normal/adjustable time or free running * cycle counter. This function is required if physical clock supports a * free running cycle counter. + * + * int (*ndo_hwtstamp_get)(struct net_device *dev, + * struct kernel_hwtstamp_config *kernel_config); + * Get the currently configured hardware timestamping parameters for the + * NIC device. + * + * int (*ndo_hwtstamp_set)(struct net_device *dev, + * struct kernel_hwtstamp_config *kernel_config, + * struct netlink_ext_ack *extack); + * Change the hardware timestamping parameters for NIC device. */ struct net_device_ops { int (*ndo_init)(struct net_device *dev); @@ -1652,6 +1663,11 @@ struct net_device_ops { ktime_t (*ndo_get_tstamp)(struct net_device *dev, const struct skb_shared_hwtstamps *hwtstamps, bool cycles); + int (*ndo_hwtstamp_get)(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_config); + int (*ndo_hwtstamp_set)(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_config, + struct netlink_ext_ack *extack); }; struct xdp_metadata_ops { @@ -1708,6 +1724,9 @@ struct xdp_metadata_ops { * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with * skb_headlen(skb) == 0 (data starts from frag0) * @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN + * @IFF_SEE_ALL_HWTSTAMP_REQUESTS: device wants to see calls to + * ndo_hwtstamp_set() for all timestamp requests regardless of source, + * even if those aren't HWTSTAMP_SOURCE_NETDEV. */ enum netdev_priv_flags { IFF_802_1Q_VLAN = 1<<0, @@ -1743,6 +1762,7 @@ enum netdev_priv_flags { IFF_NO_ADDRCONF = BIT_ULL(30), IFF_TX_SKB_NO_LINEAR = BIT_ULL(31), IFF_CHANGE_PROTO_DOWN = BIT_ULL(32), + IFF_SEE_ALL_HWTSTAMP_REQUESTS = BIT_ULL(33), }; #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN @@ -1930,8 +1950,7 @@ enum netdev_ml_priv_type { * * @rx_handler: handler for received packets * @rx_handler_data: XXX: need comments on this one - * @miniq_ingress: ingress/clsact qdisc specific data for - * ingress processing + * @tcx_ingress: BPF & clsact qdisc specific data for ingress processing * @ingress_queue: XXX: need comments on this one * @nf_hooks_ingress: netfilter hooks executed for ingress packets * @broadcast: hw bcast address @@ -1952,8 +1971,7 @@ enum netdev_ml_priv_type { * @xps_maps: all CPUs/RXQs maps for XPS device * * @xps_maps: XXX: need comments on this one - * @miniq_egress: clsact qdisc specific data for - * egress processing + * @tcx_egress: BPF & clsact qdisc specific data for egress processing * @nf_hooks_egress: netfilter hooks executed for egress packets * @qdisc_hash: qdisc hash table * @watchdog_timeo: Represents the timeout that is used by @@ -2045,6 +2063,8 @@ enum netdev_ml_priv_type { * receive offload (GRO) * @gro_ipv4_max_size: Maximum size of aggregated packet in generic * receive offload (GRO), for IPv4. + * @xdp_zc_max_segs: Maximum number of segments supported by AF_XDP + * zero copy driver * * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. * @linkwatch_dev_tracker: refcount tracker used by linkwatch. @@ -2250,11 +2270,11 @@ struct net_device { #define GRO_MAX_SIZE (8 * 65535u) unsigned int gro_max_size; unsigned int gro_ipv4_max_size; + unsigned int xdp_zc_max_segs; rx_handler_func_t __rcu *rx_handler; void __rcu *rx_handler_data; - -#ifdef CONFIG_NET_CLS_ACT - struct mini_Qdisc __rcu *miniq_ingress; +#ifdef CONFIG_NET_XGRESS + struct bpf_mprog_entry __rcu *tcx_ingress; #endif struct netdev_queue __rcu *ingress_queue; #ifdef CONFIG_NETFILTER_INGRESS @@ -2282,8 +2302,8 @@ struct net_device { #ifdef CONFIG_XPS struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX]; #endif -#ifdef CONFIG_NET_CLS_ACT - struct mini_Qdisc __rcu *miniq_egress; +#ifdef CONFIG_NET_XGRESS + struct bpf_mprog_entry __rcu *tcx_egress; #endif #ifdef CONFIG_NETFILTER_EGRESS struct nf_hook_entries __rcu *nf_hooks_egress; @@ -3016,6 +3036,9 @@ extern rwlock_t dev_base_lock; /* Device list lock */ if (netdev_master_upper_dev_get_rcu(slave) == (bond)) #define net_device_entry(lh) list_entry(lh, struct net_device, dev_list) +#define for_each_netdev_dump(net, d, ifindex) \ + xa_for_each_start(&(net)->dev_by_index, (ifindex), (d), (ifindex)) + static inline struct net_device *next_net_device(struct net_device *dev) { struct list_head *lh; @@ -3130,8 +3153,6 @@ struct net_device *netdev_get_by_name(struct net *net, const char *name, netdevice_tracker *tracker, gfp_t gfp); struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); struct net_device *dev_get_by_napi_id(unsigned int napi_id); -int dev_restart(struct net_device *dev); - static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, @@ -3933,6 +3954,11 @@ int put_user_ifreq(struct ifreq *ifr, void __user *arg); int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr, void __user *data, bool *need_copyout); int dev_ifconf(struct net *net, struct ifconf __user *ifc); +int generic_hwtstamp_get_lower(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_cfg); +int generic_hwtstamp_set_lower(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_cfg, + struct netlink_ext_ack *extack); int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata); unsigned int dev_get_flags(const struct net_device *); int __dev_change_flags(struct net_device *dev, unsigned int flags, @@ -5102,6 +5128,11 @@ static inline bool netif_is_ovs_port(const struct net_device *dev) return dev->priv_flags & IFF_OVS_DATAPATH; } +static inline bool netif_is_any_bridge_master(const struct net_device *dev) +{ + return netif_is_bridge_master(dev) || netif_is_ovs_master(dev); +} + static inline bool netif_is_any_bridge_port(const struct net_device *dev) { return netif_is_bridge_port(dev) || netif_is_ovs_port(dev); diff --git a/include/linux/netlink.h b/include/linux/netlink.h index 9eec3f4f5351..75d7de34c908 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -50,6 +50,7 @@ struct netlink_kernel_cfg { struct mutex *cb_mutex; int (*bind)(struct net *net, int group); void (*unbind)(struct net *net, int group); + void (*release) (struct sock *sk, unsigned long *groups); }; struct sock *__netlink_kernel_create(struct net *net, int unit, @@ -227,6 +228,11 @@ bool netlink_strict_get_check(struct sk_buff *skb); int netlink_unicast(struct sock *ssk, struct sk_buff *skb, __u32 portid, int nonblock); int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, __u32 portid, __u32 group, gfp_t allocation); +int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb, + __u32 portid, __u32 group, gfp_t allocation, + int (*filter)(struct sock *dsk, + struct sk_buff *skb, void *data), + void *filter_data); int netlink_set_err(struct sock *ssk, __u32 portid, __u32 group, int code); int netlink_register_notifier(struct notifier_block *nb); int netlink_unregister_notifier(struct notifier_block *nb); diff --git a/include/linux/phy.h b/include/linux/phy.h index 11c1e91563d4..ba08b0e60279 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -298,6 +298,7 @@ static inline const char *phy_modes(phy_interface_t interface) #define MII_BUS_ID_SIZE 61 struct device; +struct kernel_hwtstamp_config; struct phylink; struct sfp_bus; struct sfp_upstream_ops; @@ -1826,6 +1827,7 @@ int genphy_c45_an_config_aneg(struct phy_device *phydev); int genphy_c45_an_disable_aneg(struct phy_device *phydev); int genphy_c45_read_mdix(struct phy_device *phydev); int genphy_c45_pma_read_abilities(struct phy_device *phydev); +int genphy_c45_pma_baset1_read_abilities(struct phy_device *phydev); int genphy_c45_read_eee_abilities(struct phy_device *phydev); int genphy_c45_pma_baset1_read_master_slave(struct phy_device *phydev); int genphy_c45_read_status(struct phy_device *phydev); @@ -1954,6 +1956,12 @@ int phy_ethtool_set_plca_cfg(struct phy_device *phydev, int phy_ethtool_get_plca_status(struct phy_device *phydev, struct phy_plca_status *plca_st); +int __phy_hwtstamp_get(struct phy_device *phydev, + struct kernel_hwtstamp_config *config); +int __phy_hwtstamp_set(struct phy_device *phydev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack); + static inline int phy_package_read(struct phy_device *phydev, u32 regnum) { struct phy_package_shared *shared = phydev->shared; diff --git a/include/linux/phylib_stubs.h b/include/linux/phylib_stubs.h new file mode 100644 index 000000000000..1279f48c8a70 --- /dev/null +++ b/include/linux/phylib_stubs.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Stubs for the Network PHY library + */ + +#include <linux/rtnetlink.h> + +struct kernel_hwtstamp_config; +struct netlink_ext_ack; +struct phy_device; + +#if IS_ENABLED(CONFIG_PHYLIB) + +extern const struct phylib_stubs *phylib_stubs; + +struct phylib_stubs { + int (*hwtstamp_get)(struct phy_device *phydev, + struct kernel_hwtstamp_config *config); + int (*hwtstamp_set)(struct phy_device *phydev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack); +}; + +static inline int phy_hwtstamp_get(struct phy_device *phydev, + struct kernel_hwtstamp_config *config) +{ + /* phylib_register_stubs() and phylib_unregister_stubs() + * also run under rtnl_lock(). + */ + ASSERT_RTNL(); + + if (!phylib_stubs) + return -EOPNOTSUPP; + + return phylib_stubs->hwtstamp_get(phydev, config); +} + +static inline int phy_hwtstamp_set(struct phy_device *phydev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + /* phylib_register_stubs() and phylib_unregister_stubs() + * also run under rtnl_lock(). + */ + ASSERT_RTNL(); + + if (!phylib_stubs) + return -EOPNOTSUPP; + + return phylib_stubs->hwtstamp_set(phydev, config, extack); +} + +#else + +static inline int phy_hwtstamp_get(struct phy_device *phydev, + struct kernel_hwtstamp_config *config) +{ + return -EOPNOTSUPP; +} + +static inline int phy_hwtstamp_set(struct phy_device *phydev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + +#endif diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 1817940a3418..789c516c6b4a 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -9,6 +9,7 @@ struct device_node; struct ethtool_cmd; struct fwnode_handle; struct net_device; +struct phylink; enum { MLO_PAUSE_NONE, @@ -200,8 +201,6 @@ enum phylink_op_type { * struct phylink_config - PHYLINK configuration structure * @dev: a pointer to a struct device associated with the MAC * @type: operation type of PHYLINK instance - * @legacy_pre_march2020: driver has not been updated for March 2020 updates - * (See commit 7cceb599d15d ("net: phylink: avoid mac_config calls") * @poll_fixed_state: if true, starts link_poll, * if MAC link is at %MLO_AN_FIXED mode. * @mac_managed_pm: if true, indicate the MAC driver is responsible for PHY PM. @@ -215,7 +214,6 @@ enum phylink_op_type { struct phylink_config { struct device *dev; enum phylink_op_type type; - bool legacy_pre_march2020; bool poll_fixed_state; bool mac_managed_pm; bool ovr_an_inband; @@ -229,11 +227,9 @@ struct phylink_config { * struct phylink_mac_ops - MAC operations structure. * @validate: Validate and update the link configuration. * @mac_select_pcs: Select a PCS for the interface mode. - * @mac_pcs_get_state: Read the current link state from the hardware. * @mac_prepare: prepare for a major reconfiguration of the interface. * @mac_config: configure the MAC for the selected mode and state. * @mac_finish: finish a major reconfiguration of the interface. - * @mac_an_restart: restart 802.3z BaseX autonegotiation. * @mac_link_down: take the link down. * @mac_link_up: allow the link to come up. * @@ -245,15 +241,12 @@ struct phylink_mac_ops { struct phylink_link_state *state); struct phylink_pcs *(*mac_select_pcs)(struct phylink_config *config, phy_interface_t interface); - void (*mac_pcs_get_state)(struct phylink_config *config, - struct phylink_link_state *state); int (*mac_prepare)(struct phylink_config *config, unsigned int mode, phy_interface_t iface); void (*mac_config)(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state); int (*mac_finish)(struct phylink_config *config, unsigned int mode, phy_interface_t iface); - void (*mac_an_restart)(struct phylink_config *config); void (*mac_link_down)(struct phylink_config *config, unsigned int mode, phy_interface_t interface); void (*mac_link_up)(struct phylink_config *config, @@ -314,25 +307,6 @@ struct phylink_pcs *mac_select_pcs(struct phylink_config *config, phy_interface_t interface); /** - * mac_pcs_get_state() - Read the current inband link state from the hardware - * @config: a pointer to a &struct phylink_config. - * @state: a pointer to a &struct phylink_link_state. - * - * Read the current inband link state from the MAC PCS, reporting the - * current speed in @state->speed, duplex mode in @state->duplex, pause - * mode in @state->pause using the %MLO_PAUSE_RX and %MLO_PAUSE_TX bits, - * negotiation completion state in @state->an_complete, and link up state - * in @state->link. If possible, @state->lp_advertising should also be - * populated. - * - * Note: This is a legacy method. This function will not be called unless - * legacy_pre_march2020 is set in &struct phylink_config and there is no - * PCS attached. - */ -void mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state); - -/** * mac_prepare() - prepare to change the PHY interface mode * @config: a pointer to a &struct phylink_config. * @mode: one of %MLO_AN_FIXED, %MLO_AN_PHY, %MLO_AN_INBAND. @@ -368,17 +342,9 @@ int mac_prepare(struct phylink_config *config, unsigned int mode, * guaranteed to be correct, and so any mac_config() implementation must * never reference these fields. * - * Note: For legacy March 2020 drivers (drivers with legacy_pre_march2020 set - * in their &phylnk_config and which don't have a PCS), this function will be - * called on each link up event, and to also change the in-band advert. For - * non-legacy drivers, it will only be called to reconfigure the MAC for a - * "major" change in e.g. interface mode. It will not be called for changes - * in speed, duplex or pause modes or to change the in-band advertisement. - * In any case, it is strongly preferred that speed, duplex and pause settings - * are handled in the mac_link_up() method and not in this method. - * - * (this requires a rewrite - please refer to mac_link_up() for situations - * where the PCS and MAC are not tightly integrated.) + * This will only be called to reconfigure the MAC for a "major" change in + * e.g. interface mode. It will not be called for changes in speed, duplex + * or pause modes or to change the in-band advertisement. * * In all negotiation modes, as defined by @mode, @state->pause indicates the * pause settings which should be applied as follows. If %MLO_PAUSE_AN is not @@ -410,7 +376,7 @@ int mac_prepare(struct phylink_config *config, unsigned int mode, * 1000base-X or Cisco SGMII mode depending on the @state->interface * mode). In both cases, link state management (whether the link * is up or not) is performed by the MAC, and reported via the - * mac_pcs_get_state() callback. Changes in link state must be made + * pcs_get_state() callback. Changes in link state must be made * by calling phylink_mac_change(). * * Interface mode specific details are mentioned below. @@ -459,16 +425,6 @@ int mac_finish(struct phylink_config *config, unsigned int mode, phy_interface_t iface); /** - * mac_an_restart() - restart 802.3z BaseX autonegotiation - * @config: a pointer to a &struct phylink_config. - * - * Note: This is a legacy method. This function will not be called unless - * legacy_pre_march2020 is set in &struct phylink_config and there is no - * PCS attached. - */ -void mac_an_restart(struct phylink_config *config); - -/** * mac_link_down() - take the link down * @config: a pointer to a &struct phylink_config. * @mode: link autonegotiation mode @@ -520,14 +476,19 @@ struct phylink_pcs_ops; /** * struct phylink_pcs - PHYLINK PCS instance * @ops: a pointer to the &struct phylink_pcs_ops structure + * @phylink: pointer to &struct phylink_config * @neg_mode: provide PCS neg mode via "mode" argument * @poll: poll the PCS for link changes * * This structure is designed to be embedded within the PCS private data, * and will be passed between phylink and the PCS. + * + * The @phylink member is private to phylink and must not be touched by + * the PCS driver. */ struct phylink_pcs { const struct phylink_pcs_ops *ops; + struct phylink *phylink; bool neg_mode; bool poll; }; @@ -535,6 +496,10 @@ struct phylink_pcs { /** * struct phylink_pcs_ops - MAC PCS operations structure. * @pcs_validate: validate the link configuration. + * @pcs_enable: enable the PCS. + * @pcs_disable: disable the PCS. + * @pcs_pre_config: pre-mac_config method (for errata) + * @pcs_post_config: post-mac_config method (for arrata) * @pcs_get_state: read the current MAC PCS link state from the hardware. * @pcs_config: configure the MAC PCS for the selected mode and state. * @pcs_an_restart: restart 802.3z BaseX autonegotiation. @@ -544,6 +509,12 @@ struct phylink_pcs { struct phylink_pcs_ops { int (*pcs_validate)(struct phylink_pcs *pcs, unsigned long *supported, const struct phylink_link_state *state); + int (*pcs_enable)(struct phylink_pcs *pcs); + void (*pcs_disable)(struct phylink_pcs *pcs); + void (*pcs_pre_config)(struct phylink_pcs *pcs, + phy_interface_t interface); + int (*pcs_post_config)(struct phylink_pcs *pcs, + phy_interface_t interface); void (*pcs_get_state)(struct phylink_pcs *pcs, struct phylink_link_state *state); int (*pcs_config)(struct phylink_pcs *pcs, unsigned int neg_mode, @@ -574,6 +545,18 @@ int pcs_validate(struct phylink_pcs *pcs, unsigned long *supported, const struct phylink_link_state *state); /** + * pcs_enable() - enable the PCS. + * @pcs: a pointer to a &struct phylink_pcs. + */ +int pcs_enable(struct phylink_pcs *pcs); + +/** + * pcs_disable() - disable the PCS. + * @pcs: a pointer to a &struct phylink_pcs. + */ +void pcs_disable(struct phylink_pcs *pcs); + +/** * pcs_get_state() - Read the current inband link state from the hardware * @pcs: a pointer to a &struct phylink_pcs. * @state: a pointer to a &struct phylink_link_state. @@ -585,8 +568,8 @@ int pcs_validate(struct phylink_pcs *pcs, unsigned long *supported, * in @state->link. If possible, @state->lp_advertising should also be * populated. * - * When present, this overrides mac_pcs_get_state() in &struct - * phylink_mac_ops. + * When present, this overrides pcs_get_state() in &struct + * phylink_pcs_ops. */ void pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state); @@ -677,6 +660,7 @@ int phylink_fwnode_phy_connect(struct phylink *pl, void phylink_disconnect_phy(struct phylink *); void phylink_mac_change(struct phylink *, bool up); +void phylink_pcs_change(struct phylink_pcs *, bool up); void phylink_start(struct phylink *); void phylink_stop(struct phylink *); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 7f17acf29dda..7b949292908a 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -138,6 +138,8 @@ static inline int rcu_needs_cpu(void) return 0; } +static inline void rcu_request_urgent_qs_task(struct task_struct *t) { } + /* * Take advantage of the fact that there is only one CPU, which * allows us to ignore virtualization-based context switches. diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 56bccb5a8fde..126f6b418f6a 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -21,6 +21,7 @@ void rcu_softirq_qs(void); void rcu_note_context_switch(bool preempt); int rcu_needs_cpu(void); void rcu_cpu_stall_reset(void); +void rcu_request_urgent_qs_task(struct task_struct *t); /* * Note a virtualization-based context switch. This is simply a diff --git a/include/linux/security.h b/include/linux/security.h index 32828502f09e..994cf099d9ac 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1439,7 +1439,8 @@ int security_socket_getpeersec_dgram(struct socket *sock, struct sk_buff *skb, u int security_sk_alloc(struct sock *sk, int family, gfp_t priority); void security_sk_free(struct sock *sk); void security_sk_clone(const struct sock *sk, struct sock *newsk); -void security_sk_classify_flow(struct sock *sk, struct flowi_common *flic); +void security_sk_classify_flow(const struct sock *sk, + struct flowi_common *flic); void security_req_classify_flow(const struct request_sock *req, struct flowi_common *flic); void security_sock_graft(struct sock*sk, struct socket *parent); @@ -1597,7 +1598,7 @@ static inline void security_sk_clone(const struct sock *sk, struct sock *newsk) { } -static inline void security_sk_classify_flow(struct sock *sk, +static inline void security_sk_classify_flow(const struct sock *sk, struct flowi_common *flic) { } diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 91ed66952580..16a49ba534e4 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -441,8 +441,6 @@ static inline bool skb_frag_must_loop(struct page *p) copied += p_len, p++, p_off = 0, \ p_len = min_t(u32, f_len - copied, PAGE_SIZE)) \ -#define HAVE_HW_TIME_STAMP - /** * struct skb_shared_hwtstamps - hardware time stamps * @hwtstamp: hardware time stamp transformed into duration @@ -944,7 +942,7 @@ struct sk_buff { __u8 __mono_tc_offset[0]; /* public: */ __u8 mono_delivery_time:1; /* See SKB_MONO_DELIVERY_TIME_MASK */ -#ifdef CONFIG_NET_CLS_ACT +#ifdef CONFIG_NET_XGRESS __u8 tc_at_ingress:1; /* See TC_AT_INGRESS_MASK */ __u8 tc_skip_classify:1; #endif @@ -993,7 +991,7 @@ struct sk_buff { __u8 csum_not_inet:1; #endif -#ifdef CONFIG_NET_SCHED +#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS) __u16 tc_index; /* traffic control index */ #endif @@ -4023,7 +4021,7 @@ __skb_header_pointer(const struct sk_buff *skb, int offset, int len, if (likely(hlen - offset >= len)) return (void *)data + offset; - if (!skb || !buffer || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) + if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) return NULL; return buffer; @@ -4036,6 +4034,14 @@ skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) skb_headlen(skb), buffer); } +static inline void * __must_check +skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len) +{ + if (likely(skb_headlen(skb) - offset >= len)) + return skb->data + offset; + return NULL; +} + /** * skb_needs_linearize - check if we need to linearize a given skb * depending on the given device features. diff --git a/include/linux/smscphy.h b/include/linux/smscphy.h index e1c88627755a..1a6a851d2cf8 100644 --- a/include/linux/smscphy.h +++ b/include/linux/smscphy.h @@ -38,4 +38,38 @@ int smsc_phy_set_tunable(struct phy_device *phydev, struct ethtool_tunable *tuna, const void *data); int smsc_phy_probe(struct phy_device *phydev); +#define MII_LAN874X_PHY_MMD_WOL_WUCSR 0x8010 +#define MII_LAN874X_PHY_MMD_WOL_WUF_CFGA 0x8011 +#define MII_LAN874X_PHY_MMD_WOL_WUF_CFGB 0x8012 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK0 0x8021 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK1 0x8022 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK2 0x8023 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK3 0x8024 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK4 0x8025 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK5 0x8026 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK6 0x8027 +#define MII_LAN874X_PHY_MMD_WOL_WUF_MASK7 0x8028 +#define MII_LAN874X_PHY_MMD_WOL_RX_ADDRA 0x8061 +#define MII_LAN874X_PHY_MMD_WOL_RX_ADDRB 0x8062 +#define MII_LAN874X_PHY_MMD_WOL_RX_ADDRC 0x8063 +#define MII_LAN874X_PHY_MMD_MCFGR 0x8064 + +#define MII_LAN874X_PHY_PME1_SET (2 << 13) +#define MII_LAN874X_PHY_PME2_SET (2 << 11) +#define MII_LAN874X_PHY_PME_SELF_CLEAR BIT(9) +#define MII_LAN874X_PHY_WOL_PFDA_FR BIT(7) +#define MII_LAN874X_PHY_WOL_WUFR BIT(6) +#define MII_LAN874X_PHY_WOL_MPR BIT(5) +#define MII_LAN874X_PHY_WOL_BCAST_FR BIT(4) +#define MII_LAN874X_PHY_WOL_PFDAEN BIT(3) +#define MII_LAN874X_PHY_WOL_WUEN BIT(2) +#define MII_LAN874X_PHY_WOL_MPEN BIT(1) +#define MII_LAN874X_PHY_WOL_BCSTEN BIT(0) + +#define MII_LAN874X_PHY_WOL_FILTER_EN BIT(15) +#define MII_LAN874X_PHY_WOL_FILTER_MCASTTEN BIT(9) +#define MII_LAN874X_PHY_WOL_FILTER_BCSTEN BIT(8) + +#define MII_LAN874X_PHY_PME_SELF_CLEAR_DELAY 0x1000 /* 81 milliseconds */ + #endif /* __LINUX_SMSCPHY_H__ */ diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 06090538fe2d..3d0702510224 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -76,6 +76,8 @@ | DMA_AXI_BLEN_32 | DMA_AXI_BLEN_64 \ | DMA_AXI_BLEN_128 | DMA_AXI_BLEN_256) +struct stmmac_priv; + /* Platfrom data for platform device structure's platform_data field */ struct stmmac_mdio_bus_data { @@ -204,6 +206,19 @@ struct dwmac4_addrs { u32 mtl_low_cred_offset; }; +#define STMMAC_FLAG_HAS_INTEGRATED_PCS BIT(0) +#define STMMAC_FLAG_SPH_DISABLE BIT(1) +#define STMMAC_FLAG_USE_PHY_WOL BIT(2) +#define STMMAC_FLAG_HAS_SUN8I BIT(3) +#define STMMAC_FLAG_TSO_EN BIT(4) +#define STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP BIT(5) +#define STMMAC_FLAG_VLAN_FAIL_Q_EN BIT(6) +#define STMMAC_FLAG_MULTI_MSI_EN BIT(7) +#define STMMAC_FLAG_EXT_SNAPSHOT_EN BIT(8) +#define STMMAC_FLAG_INT_SNAPSHOT_EN BIT(9) +#define STMMAC_FLAG_RX_CLK_RUNS_IN_LPI BIT(10) +#define STMMAC_FLAG_EN_TX_LPI_CLOCKGATING BIT(11) + struct plat_stmmacenet_data { int bus_id; int phy_addr; @@ -245,7 +260,7 @@ struct plat_stmmacenet_data { int (*serdes_powerup)(struct net_device *ndev, void *priv); void (*serdes_powerdown)(struct net_device *ndev, void *priv); void (*speed_mode_2500)(struct net_device *ndev, void *priv); - void (*ptp_clk_freq_config)(void *priv); + void (*ptp_clk_freq_config)(struct stmmac_priv *priv); int (*init)(struct platform_device *pdev, void *priv); void (*exit)(struct platform_device *pdev, void *priv); struct mac_device_info *(*setup)(void *priv); @@ -266,22 +281,14 @@ struct plat_stmmacenet_data { struct reset_control *stmmac_ahb_rst; struct stmmac_axi *axi; int has_gmac4; - bool has_sun8i; - bool tso_en; int rss_en; int mac_port_sel_speed; - bool en_tx_lpi_clockgating; - bool rx_clk_runs_in_lpi; int has_xgmac; - bool vlan_fail_q_en; u8 vlan_fail_q; unsigned int eee_usecs_rate; struct pci_dev *pdev; int int_snapshot_num; int ext_snapshot_num; - bool int_snapshot_en; - bool ext_snapshot_en; - bool multi_msi_en; int msi_mac_vec; int msi_wol_vec; int msi_lpi_vec; @@ -289,10 +296,7 @@ struct plat_stmmacenet_data { int msi_sfty_ue_vec; int msi_rx_base_vec; int msi_tx_base_vec; - bool use_phy_wol; - bool sph_disable; - bool serdes_up_after_phy_linkup; const struct dwmac4_addrs *dwmac4_addrs; - bool has_integrated_pcs; + unsigned int flags; }; #endif diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 91a37c99ba66..d16abdb3541a 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -172,6 +172,8 @@ static inline struct tcp_request_sock *tcp_rsk(const struct request_sock *req) return (struct tcp_request_sock *)req; } +#define TCP_RMEM_TO_WIN_SCALE 8 + struct tcp_sock { /* inet_connection_sock has to be the first member of tcp_sock */ struct inet_connection_sock inet_conn; @@ -238,7 +240,7 @@ struct tcp_sock { u32 window_clamp; /* Maximal window to advertise */ u32 rcv_ssthresh; /* Current window clamp */ - + u8 scaling_ratio; /* see tcp_win_from_space() */ /* Information of the most recently (s)acked skb */ struct tcp_rack { u64 mstamp; /* (Re)sent time of the skb */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 3930e676436c..e66d04dbe56a 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -867,7 +867,8 @@ extern int perf_uprobe_init(struct perf_event *event, extern void perf_uprobe_destroy(struct perf_event *event); extern int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, const char **filename, - u64 *probe_offset, bool perf_type_tracepoint); + u64 *probe_offset, u64 *probe_addr, + bool perf_type_tracepoint); #endif extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, char *filter_str); diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index a016f275cb01..c5e57c6bd873 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -301,7 +301,6 @@ int __bond_3ad_get_active_agg_info(struct bonding *bond, int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond, struct slave *slave); int bond_3ad_set_carrier(struct bonding *bond); -void bond_3ad_update_lacp_active(struct bonding *bond); void bond_3ad_update_lacp_rate(struct bonding *bond); void bond_3ad_update_ad_actor_settings(struct bonding *bond); int bond_3ad_stats_fill(struct sk_buff *skb, struct bond_3ad_stats *stats); diff --git a/include/net/datalink.h b/include/net/datalink.h index c837ffc7ebf8..6c529a40e00d 100644 --- a/include/net/datalink.h +++ b/include/net/datalink.h @@ -23,6 +23,4 @@ struct datalink_proto { struct list_head node; }; -struct datalink_proto *make_EII_client(void); -void destroy_EII_client(struct datalink_proto *dl); #endif diff --git a/include/net/devlink.h b/include/net/devlink.h index 0cdb4b16e5b5..a1a8e1b6e7df 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -1790,8 +1790,6 @@ devlink_port_region_create(struct devlink_port *port, u32 region_max_snapshots, u64 region_size); void devl_region_destroy(struct devlink_region *region); void devlink_region_destroy(struct devlink_region *region); -void devlink_port_region_destroy(struct devlink_region *region); - int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id); void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id); int devlink_region_snapshot_create(struct devlink_region *region, diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h index a2b953b57689..f291a3b0f9e5 100644 --- a/include/net/dropreason-core.h +++ b/include/net/dropreason-core.h @@ -30,6 +30,7 @@ FN(TCP_OVERWINDOW) \ FN(TCP_OFOMERGE) \ FN(TCP_RFC7323_PAWS) \ + FN(TCP_OLD_SEQUENCE) \ FN(TCP_INVALID_SEQUENCE) \ FN(TCP_RESET) \ FN(TCP_INVALID_SYN) \ @@ -188,6 +189,8 @@ enum skb_drop_reason { * LINUX_MIB_PAWSESTABREJECTED */ SKB_DROP_REASON_TCP_RFC7323_PAWS, + /** @SKB_DROP_REASON_TCP_OLD_SEQUENCE: Old SEQ field (duplicate packet) */ + SKB_DROP_REASON_TCP_OLD_SEQUENCE, /** @SKB_DROP_REASON_TCP_INVALID_SEQUENCE: Not acceptable SEQ field */ SKB_DROP_REASON_TCP_INVALID_SEQUENCE, /** @SKB_DROP_REASON_TCP_RESET: Invalid RST packet */ diff --git a/include/net/dsa.h b/include/net/dsa.h index d309ee7ed04b..0b9c6aa27047 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -873,8 +873,6 @@ struct dsa_switch_ops { struct phylink_pcs *(*phylink_mac_select_pcs)(struct dsa_switch *ds, int port, phy_interface_t iface); - int (*phylink_mac_link_state)(struct dsa_switch *ds, int port, - struct phylink_link_state *state); int (*phylink_mac_prepare)(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface); @@ -884,7 +882,6 @@ struct dsa_switch_ops { int (*phylink_mac_finish)(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface); - void (*phylink_mac_an_restart)(struct dsa_switch *ds, int port); void (*phylink_mac_link_down)(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface); diff --git a/include/net/flow_dissector.h b/include/net/flow_dissector.h index 8664ed4fbbdf..1a7131d6cb0e 100644 --- a/include/net/flow_dissector.h +++ b/include/net/flow_dissector.h @@ -302,6 +302,14 @@ struct flow_dissector_key_l2tpv3 { }; /** + * struct flow_dissector_key_ipsec: + * @spi: identifier for a ipsec connection + */ +struct flow_dissector_key_ipsec { + __be32 spi; +}; + +/** * struct flow_dissector_key_cfm * @mdl_ver: maintenance domain level (mdl) and cfm protocol version * @opcode: code specifying a type of cfm protocol packet @@ -354,6 +362,7 @@ enum flow_dissector_key_id { FLOW_DISSECTOR_KEY_PPPOE, /* struct flow_dissector_key_pppoe */ FLOW_DISSECTOR_KEY_L2TPV3, /* struct flow_dissector_key_l2tpv3 */ FLOW_DISSECTOR_KEY_CFM, /* struct flow_dissector_key_cfm */ + FLOW_DISSECTOR_KEY_IPSEC, /* struct flow_dissector_key_ipsec */ FLOW_DISSECTOR_KEY_MAX, }; @@ -370,7 +379,8 @@ struct flow_dissector_key { }; struct flow_dissector { - unsigned int used_keys; /* each bit repesents presence of one key id */ + unsigned long long used_keys; + /* each bit represents presence of one key id */ unsigned short int offset[FLOW_DISSECTOR_KEY_MAX]; }; @@ -430,7 +440,7 @@ void skb_flow_get_icmp_tci(const struct sk_buff *skb, static inline bool dissector_uses_key(const struct flow_dissector *flow_dissector, enum flow_dissector_key_id key_id) { - return flow_dissector->used_keys & (1 << key_id); + return flow_dissector->used_keys & (1ULL << key_id); } static inline void *skb_flow_dissector_target(struct flow_dissector *flow_dissector, diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h index 118082eae48c..9efa9a59e81f 100644 --- a/include/net/flow_offload.h +++ b/include/net/flow_offload.h @@ -64,6 +64,10 @@ struct flow_match_tcp { struct flow_dissector_key_tcp *key, *mask; }; +struct flow_match_ipsec { + struct flow_dissector_key_ipsec *key, *mask; +}; + struct flow_match_mpls { struct flow_dissector_key_mpls *key, *mask; }; @@ -116,6 +120,8 @@ void flow_rule_match_ports_range(const struct flow_rule *rule, struct flow_match_ports_range *out); void flow_rule_match_tcp(const struct flow_rule *rule, struct flow_match_tcp *out); +void flow_rule_match_ipsec(const struct flow_rule *rule, + struct flow_match_ipsec *out); void flow_rule_match_icmp(const struct flow_rule *rule, struct flow_match_icmp *out); void flow_rule_match_mpls(const struct flow_rule *rule, diff --git a/include/net/handshake.h b/include/net/handshake.h index 2e26e436e85f..8ebd4f9ed26e 100644 --- a/include/net/handshake.h +++ b/include/net/handshake.h @@ -40,5 +40,10 @@ int tls_server_hello_x509(const struct tls_handshake_args *args, gfp_t flags); int tls_server_hello_psk(const struct tls_handshake_args *args, gfp_t flags); bool tls_handshake_cancel(struct sock *sk); +void tls_handshake_close(struct socket *sock); + +u8 tls_get_record_type(const struct sock *sk, const struct cmsghdr *msg); +void tls_alert_recv(const struct sock *sk, const struct msghdr *msg, + u8 *level, u8 *description); #endif /* _NET_HANDSHAKE_H */ diff --git a/include/net/ila.h b/include/net/ila.h deleted file mode 100644 index 73ebe5eab272..000000000000 --- a/include/net/ila.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * ILA kernel interface - * - * Copyright (c) 2015 Tom Herbert <tom@herbertland.com> - */ - -#ifndef _NET_ILA_H -#define _NET_ILA_H - -struct sk_buff; - -int ila_xlat_outgoing(struct sk_buff *skb); -int ila_xlat_incoming(struct sk_buff *skb); - -#endif /* _NET_ILA_H */ diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 3556595ce59a..b32539bb0fb0 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -156,7 +156,7 @@ void fib6_force_start_gc(struct net *net); struct fib6_info *addrconf_f6i_alloc(struct net *net, struct inet6_dev *idev, const struct in6_addr *addr, bool anycast, - gfp_t gfp_flags); + gfp_t gfp_flags, struct netlink_ext_ack *extack); struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, int flags); diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index ed4b6ad3fcac..e8750b4ef7e1 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -52,6 +52,7 @@ struct ip_tunnel_key { u8 tos; /* TOS for IPv4, TC for IPv6 */ u8 ttl; /* TTL for IPv4, HL for IPv6 */ __be32 label; /* Flow Label for IPv6 */ + u32 nhid; __be16 tp_src; __be16 tp_dst; __u8 flow_flags; diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index 78beaa765c73..9f6add96de2d 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -42,6 +42,7 @@ #include <linux/idr.h> #include <linux/skbuff.h> #include <linux/notifier.h> +#include <linux/xarray.h> struct user_namespace; struct proc_dir_entry; @@ -69,7 +70,7 @@ struct net { atomic_t dev_unreg_count; unsigned int dev_base_seq; /* protected by rtnl_mutex */ - int ifindex; + u32 ifindex; spinlock_t nsid_lock; atomic_t fnhe_genid; @@ -110,6 +111,7 @@ struct net { struct hlist_head *dev_name_head; struct hlist_head *dev_index_head; + struct xarray dev_by_index; struct raw_notifier_head netdev_chain; /* Note that @hash_mix can be read millions times per second, diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index cf0d81be5a96..165e7a03b8e9 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -100,7 +100,7 @@ nf_ct_expect_find_get(struct net *net, struct nf_conntrack_expect * nf_ct_find_expectation(struct net *net, const struct nf_conntrack_zone *zone, - const struct nf_conntrack_tuple *tuple); + const struct nf_conntrack_tuple *tuple, bool unlink); void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, u32 portid, int report); diff --git a/include/net/netlink.h b/include/net/netlink.h index b12cd957abb4..8a7cd1170e1f 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -375,12 +375,11 @@ struct nla_policy { #define NLA_POLICY_BITFIELD32(valid) \ { .type = NLA_BITFIELD32, .bitfield32_valid = valid } -#define __NLA_IS_UINT_TYPE(tp) \ - (tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 || tp == NLA_U64) +#define __NLA_IS_UINT_TYPE(tp) \ + (tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 || \ + tp == NLA_U64 || tp == NLA_BE16 || tp == NLA_BE32) #define __NLA_IS_SINT_TYPE(tp) \ (tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64) -#define __NLA_IS_BEINT_TYPE(tp) \ - (tp == NLA_BE16 || tp == NLA_BE32) #define __NLA_ENSURE(condition) BUILD_BUG_ON_ZERO(!(condition)) #define NLA_ENSURE_UINT_TYPE(tp) \ @@ -394,7 +393,6 @@ struct nla_policy { #define NLA_ENSURE_INT_OR_BINARY_TYPE(tp) \ (__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) || \ __NLA_IS_SINT_TYPE(tp) || \ - __NLA_IS_BEINT_TYPE(tp) || \ tp == NLA_MSECS || \ tp == NLA_BINARY) + tp) #define NLA_ENSURE_NO_VALIDATION_PTR(tp) \ @@ -402,8 +400,6 @@ struct nla_policy { tp != NLA_REJECT && \ tp != NLA_NESTED && \ tp != NLA_NESTED_ARRAY) + tp) -#define NLA_ENSURE_BEINT_TYPE(tp) \ - (__NLA_ENSURE(__NLA_IS_BEINT_TYPE(tp)) + tp) #define NLA_POLICY_RANGE(tp, _min, _max) { \ .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \ diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index f00374718159..7a41c4791536 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -152,7 +152,7 @@ struct netns_ipv4 { u8 sysctl_tcp_abort_on_overflow; u8 sysctl_tcp_fack; /* obsolete */ int sysctl_tcp_max_reordering; - int sysctl_tcp_adv_win_scale; + int sysctl_tcp_adv_win_scale; /* obsolete */ u8 sysctl_tcp_dsack; u8 sysctl_tcp_app_win; u8 sysctl_tcp_frto; diff --git a/include/net/page_pool.h b/include/net/page_pool.h index 126f9e294389..73d4f786418d 100644 --- a/include/net/page_pool.h +++ b/include/net/page_pool.h @@ -18,9 +18,8 @@ * * API keeps track of in-flight pages, in-order to let API user know * when it is safe to dealloactor page_pool object. Thus, API users - * must make sure to call page_pool_release_page() when a page is - * "leaving" the page_pool. Or call page_pool_put_page() where - * appropiate. For maintaining correct accounting. + * must call page_pool_put_page() where appropriate and only attach + * the page to a page_pool-aware objects, like skbs marked for recycling. * * API user must only call page_pool_put_page() once on a page, as it * will either recycle the page, or in case of elevated refcnt, it @@ -71,47 +70,76 @@ struct pp_alloc_cache { struct page *cache[PP_ALLOC_CACHE_SIZE]; }; +/** + * struct page_pool_params - page pool parameters + * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_PAGE_FRAG + * @order: 2^order pages on allocation + * @pool_size: size of the ptr_ring + * @nid: NUMA node id to allocate from pages from + * @dev: device, for DMA pre-mapping purposes + * @napi: NAPI which is the sole consumer of pages, otherwise NULL + * @dma_dir: DMA mapping direction + * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV + * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV + */ struct page_pool_params { unsigned int flags; unsigned int order; unsigned int pool_size; - int nid; /* Numa node id to allocate from pages from */ - struct device *dev; /* device, for DMA pre-mapping purposes */ - struct napi_struct *napi; /* Sole consumer of pages, otherwise NULL */ - enum dma_data_direction dma_dir; /* DMA mapping direction */ - unsigned int max_len; /* max DMA sync memory size */ - unsigned int offset; /* DMA addr offset */ + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; +/* private: used by test code only */ void (*init_callback)(struct page *page, void *arg); void *init_arg; }; #ifdef CONFIG_PAGE_POOL_STATS +/** + * struct page_pool_alloc_stats - allocation statistics + * @fast: successful fast path allocations + * @slow: slow path order-0 allocations + * @slow_high_order: slow path high order allocations + * @empty: ptr ring is empty, so a slow path allocation was forced + * @refill: an allocation which triggered a refill of the cache + * @waive: pages obtained from the ptr ring that cannot be added to + * the cache due to a NUMA mismatch + */ struct page_pool_alloc_stats { - u64 fast; /* fast path allocations */ - u64 slow; /* slow-path order 0 allocations */ - u64 slow_high_order; /* slow-path high order allocations */ - u64 empty; /* failed refills due to empty ptr ring, forcing - * slow path allocation - */ - u64 refill; /* allocations via successful refill */ - u64 waive; /* failed refills due to numa zone mismatch */ + u64 fast; + u64 slow; + u64 slow_high_order; + u64 empty; + u64 refill; + u64 waive; }; +/** + * struct page_pool_recycle_stats - recycling (freeing) statistics + * @cached: recycling placed page in the page pool cache + * @cache_full: page pool cache was full + * @ring: page placed into the ptr ring + * @ring_full: page released from page pool because the ptr ring was full + * @released_refcnt: page released (and not recycled) because refcnt > 1 + */ struct page_pool_recycle_stats { - u64 cached; /* recycling placed page in the cache. */ - u64 cache_full; /* cache was full */ - u64 ring; /* recycling placed page back into ptr ring */ - u64 ring_full; /* page was released from page-pool because - * PTR ring was full. - */ - u64 released_refcnt; /* page released because of elevated - * refcnt - */ + u64 cached; + u64 cache_full; + u64 ring; + u64 ring_full; + u64 released_refcnt; }; -/* This struct wraps the above stats structs so users of the - * page_pool_get_stats API can pass a single argument when requesting the - * stats for the page pool. +/** + * struct page_pool_stats - combined page pool use statistics + * @alloc_stats: see struct page_pool_alloc_stats + * @recycle_stats: see struct page_pool_recycle_stats + * + * Wrapper struct for combining page pool stats with different storage + * requirements. */ struct page_pool_stats { struct page_pool_alloc_stats alloc_stats; @@ -212,6 +240,12 @@ struct page_pool { struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); +/** + * page_pool_dev_alloc_pages() - allocate a page. + * @pool: pool from which to allocate + * + * Get a page from the page allocator or page_pool caches. + */ static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) { gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); @@ -231,8 +265,12 @@ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, return page_pool_alloc_frag(pool, offset, size, gfp); } -/* get the stored dma direction. A driver might decide to treat this locally and - * avoid the extra cache line from page_pool to determine the direction +/** + * page_pool_get_dma_dir() - Retrieve the stored DMA direction. + * @pool: pool from which page was allocated + * + * Get the stored dma direction. A driver might decide to store this locally + * and avoid the extra cache line from page_pool to determine the direction. */ static inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) @@ -251,7 +289,6 @@ void page_pool_unlink_napi(struct page_pool *pool); void page_pool_destroy(struct page_pool *pool); void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), struct xdp_mem_info *mem); -void page_pool_release_page(struct page_pool *pool, struct page *page); void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count); #else @@ -268,10 +305,6 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool, struct xdp_mem_info *mem) { } -static inline void page_pool_release_page(struct page_pool *pool, - struct page *page) -{ -} static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count) @@ -327,6 +360,19 @@ static inline bool page_pool_is_last_frag(struct page_pool *pool, (page_pool_defrag_page(page, 1) == 0); } +/** + * page_pool_put_page() - release a reference to a page pool page + * @pool: pool from which page was allocated + * @page: page to release a reference on + * @dma_sync_size: how much of the page may have been touched by the device + * @allow_direct: released by the consumer, allow lockless caching + * + * The outcome of this depends on the page refcnt. If the driver bumps + * the refcnt > 1 this will unmap the page. If the page refcnt is 1 + * the allocator owns the page and will try to recycle it in one of the pool + * caches. If PP_FLAG_DMA_SYNC_DEV is set, the page will be synced for_device + * using dma_sync_single_range_for_device(). + */ static inline void page_pool_put_page(struct page_pool *pool, struct page *page, unsigned int dma_sync_size, @@ -343,14 +389,29 @@ static inline void page_pool_put_page(struct page_pool *pool, #endif } -/* Same as above but will try to sync the entire area pool->max_len */ +/** + * page_pool_put_full_page() - release a reference on a page pool page + * @pool: pool from which page was allocated + * @page: page to release a reference on + * @allow_direct: released by the consumer, allow lockless caching + * + * Similar to page_pool_put_page(), but will DMA sync the entire memory area + * as configured in &page_pool_params.max_len. + */ static inline void page_pool_put_full_page(struct page_pool *pool, struct page *page, bool allow_direct) { page_pool_put_page(pool, page, -1, allow_direct); } -/* Same as above but the caller must guarantee safe context. e.g NAPI */ +/** + * page_pool_recycle_direct() - release a reference on a page pool page + * @pool: pool from which page was allocated + * @page: page to release a reference on + * + * Similar to page_pool_put_full_page() but caller must guarantee safe context + * (e.g NAPI), since it will recycle the page directly into the pool fast cache. + */ static inline void page_pool_recycle_direct(struct page_pool *pool, struct page *page) { @@ -360,6 +421,13 @@ static inline void page_pool_recycle_direct(struct page_pool *pool, #define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \ (sizeof(dma_addr_t) > sizeof(unsigned long)) +/** + * page_pool_get_dma_addr() - Retrieve the stored DMA address. + * @page: page allocated from a page pool + * + * Fetch the DMA address of the page. The page pool to which the page belongs + * must had been created with PP_FLAG_DMA_MAP. + */ static inline dma_addr_t page_pool_get_dma_addr(struct page *page) { dma_addr_t ret = page->dma_addr; diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index a2ea45c7b53e..139cd09828af 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -866,6 +866,7 @@ struct tc_htb_qopt_offload { u32 parent_classid; u16 classid; u16 qid; + u32 quantum; u64 rate; u64 ceil; u8 prio; diff --git a/include/net/route.h b/include/net/route.h index 8c2a8e7d8f8e..d9ca98d2366f 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -163,7 +163,7 @@ static inline struct rtable *ip_route_output(struct net *net, __be32 daddr, } static inline struct rtable *ip_route_output_ports(struct net *net, struct flowi4 *fl4, - struct sock *sk, + const struct sock *sk, __be32 daddr, __be32 saddr, __be16 dport, __be16 sport, __u8 proto, __u8 tos, int oif) @@ -309,7 +309,7 @@ static inline void ip_route_connect_init(struct flowi4 *fl4, __be32 dst, static inline struct rtable *ip_route_connect(struct flowi4 *fl4, __be32 dst, __be32 src, int oif, u8 protocol, __be16 sport, __be16 dport, - struct sock *sk) + const struct sock *sk) { struct net *net = sock_net(sk); struct rtable *rt; @@ -330,7 +330,7 @@ static inline struct rtable *ip_route_connect(struct flowi4 *fl4, __be32 dst, static inline struct rtable *ip_route_newports(struct flowi4 *fl4, struct rtable *rt, __be16 orig_sport, __be16 orig_dport, __be16 sport, __be16 dport, - struct sock *sk) + const struct sock *sk) { if (sport != orig_sport || dport != orig_dport) { fl4->fl4_dport = dport; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index e92f73bb3198..f232512505f8 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -599,6 +599,7 @@ get_default_qdisc_ops(const struct net_device *dev, int ntx) struct Qdisc_class_common { u32 classid; + unsigned int filter_cnt; struct hlist_node hnode; }; @@ -633,6 +634,31 @@ qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) return NULL; } +static inline bool qdisc_class_in_use(const struct Qdisc_class_common *cl) +{ + return cl->filter_cnt > 0; +} + +static inline void qdisc_class_get(struct Qdisc_class_common *cl) +{ + unsigned int res; + + if (check_add_overflow(cl->filter_cnt, 1, &res)) + WARN(1, "Qdisc class overflow"); + + cl->filter_cnt = res; +} + +static inline void qdisc_class_put(struct Qdisc_class_common *cl) +{ + unsigned int res; + + if (check_sub_overflow(cl->filter_cnt, 1, &res)) + WARN(1, "Qdisc class underflow"); + + cl->filter_cnt = res; +} + static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) { u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; @@ -703,7 +729,7 @@ int skb_do_redirect(struct sk_buff *); static inline bool skb_at_tc_ingress(const struct sk_buff *skb) { -#ifdef CONFIG_NET_CLS_ACT +#ifdef CONFIG_NET_XGRESS return skb->tc_at_ingress; #else return false; diff --git a/include/net/sctp/sm.h b/include/net/sctp/sm.h index f37c7a558d6d..64c42bd56bb2 100644 --- a/include/net/sctp/sm.h +++ b/include/net/sctp/sm.h @@ -156,7 +156,6 @@ sctp_state_fn_t sctp_sf_do_6_2_sack; sctp_state_fn_t sctp_sf_autoclose_timer_expire; /* Prototypes for utility support functions. */ -__u8 sctp_get_chunk_type(struct sctp_chunk *chunk); const struct sctp_sm_table_entry *sctp_sm_lookup_event( struct net *net, enum sctp_event_type event_type, @@ -166,8 +165,6 @@ int sctp_chunk_iif(const struct sctp_chunk *); struct sctp_association *sctp_make_temp_asoc(const struct sctp_endpoint *, struct sctp_chunk *, gfp_t gfp); -__u32 sctp_generate_verification_tag(void); -void sctp_populate_tie_tags(__u8 *cookie, __u32 curTag, __u32 hisTag); /* Prototypes for chunk-building functions. */ struct sctp_chunk *sctp_make_init(const struct sctp_association *asoc, diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 5c72d1864dd6..5a24d6d8522a 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -1122,8 +1122,6 @@ void sctp_outq_free(struct sctp_outq*); void sctp_outq_tail(struct sctp_outq *, struct sctp_chunk *chunk, gfp_t); int sctp_outq_sack(struct sctp_outq *, struct sctp_chunk *); int sctp_outq_is_empty(const struct sctp_outq *); -void sctp_outq_restart(struct sctp_outq *); - void sctp_retransmit(struct sctp_outq *q, struct sctp_transport *transport, enum sctp_retransmit_reason reason); void sctp_retransmit_mark(struct sctp_outq *, struct sctp_transport *, __u8); diff --git a/include/net/sock.h b/include/net/sock.h index 2eb916d1ff64..7ae44bf866af 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1339,6 +1339,7 @@ struct proto { struct kmem_cache *slab; unsigned int obj_size; + unsigned int ipv6_pinfo_offset; slab_flags_t slab_flags; unsigned int useroffset; /* Usercopy region offset */ unsigned int usersize; /* Usercopy region size */ diff --git a/include/net/switchdev.h b/include/net/switchdev.h index ca0312b78294..0294cfec9c37 100644 --- a/include/net/switchdev.h +++ b/include/net/switchdev.h @@ -201,8 +201,6 @@ struct switchdev_obj_in_state_mrp { #define SWITCHDEV_OBJ_IN_STATE_MRP(OBJ) \ container_of((OBJ), struct switchdev_obj_in_state_mrp, obj) -typedef int switchdev_obj_dump_cb_t(struct switchdev_obj *obj); - struct switchdev_brport { struct net_device *dev; const void *ctx; @@ -231,6 +229,7 @@ enum switchdev_notifier_type { SWITCHDEV_BRPORT_OFFLOADED, SWITCHDEV_BRPORT_UNOFFLOADED, + SWITCHDEV_BRPORT_REPLAY, }; struct switchdev_notifier_info { @@ -299,6 +298,11 @@ void switchdev_bridge_port_unoffload(struct net_device *brport_dev, const void *ctx, struct notifier_block *atomic_nb, struct notifier_block *blocking_nb); +int switchdev_bridge_port_replay(struct net_device *brport_dev, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack); void switchdev_deferred_process(void); int switchdev_port_attr_set(struct net_device *dev, diff --git a/include/net/tcp.h b/include/net/tcp.h index 0ca972ebd3dd..6d77c08d83b7 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -323,7 +323,6 @@ int tcp_v4_early_demux(struct sk_buff *skb); int tcp_v4_rcv(struct sk_buff *skb); void tcp_remove_empty_skb(struct sock *sk); -int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, @@ -350,7 +349,6 @@ ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, bool force_schedule); -void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks); static inline void tcp_dec_quickack_mode(struct sock *sk, const unsigned int pkts) { @@ -606,7 +604,6 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, unsigned int mss_now, gfp_t gfp); void tcp_send_probe0(struct sock *); -void tcp_send_partial(struct sock *); int tcp_write_wakeup(struct sock *, int mib); void tcp_send_fin(struct sock *sk); void tcp_send_active_reset(struct sock *sk, gfp_t priority); @@ -624,7 +621,6 @@ void tcp_skb_collapse_tstamp(struct sk_buff *skb, void tcp_rearm_rto(struct sock *sk); void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); void tcp_reset(struct sock *sk, struct sk_buff *skb); -void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb); void tcp_fin(struct sock *sk); void tcp_check_space(struct sock *sk); void tcp_sack_compress_send_ack(struct sock *sk); @@ -1432,13 +1428,39 @@ void tcp_select_initial_window(const struct sock *sk, int __space, __u32 *window_clamp, int wscale_ok, __u8 *rcv_wscale, __u32 init_rcv_wnd); +static inline int __tcp_win_from_space(u8 scaling_ratio, int space) +{ + s64 scaled_space = (s64)space * scaling_ratio; + + return scaled_space >> TCP_RMEM_TO_WIN_SCALE; +} + static inline int tcp_win_from_space(const struct sock *sk, int space) { - int tcp_adv_win_scale = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale); + return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); +} + +/* inverse of __tcp_win_from_space() */ +static inline int __tcp_space_from_win(u8 scaling_ratio, int win) +{ + u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; - return tcp_adv_win_scale <= 0 ? - (space>>(-tcp_adv_win_scale)) : - space - (space>>tcp_adv_win_scale); + do_div(val, scaling_ratio); + return val; +} + +static inline int tcp_space_from_win(const struct sock *sk, int win) +{ + return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); +} + +static inline void tcp_scaling_ratio_init(struct sock *sk) +{ + /* Assume a conservative default of 1200 bytes of payload per 4K page. + * This may be adjusted later in tcp_measure_rcv_mss(). + */ + tcp_sk(sk)->scaling_ratio = (1200 << TCP_RMEM_TO_WIN_SCALE) / + SKB_TRUESIZE(4096); } /* Note: caller must be prepared to deal with negative returns */ @@ -2336,7 +2358,6 @@ struct sk_msg; struct sk_psock; #ifdef CONFIG_BPF_SYSCALL -struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock); int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); #endif /* CONFIG_BPF_SYSCALL */ diff --git a/include/net/tcx.h b/include/net/tcx.h new file mode 100644 index 000000000000..264f147953ba --- /dev/null +++ b/include/net/tcx.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Isovalent */ +#ifndef __NET_TCX_H +#define __NET_TCX_H + +#include <linux/bpf.h> +#include <linux/bpf_mprog.h> + +#include <net/sch_generic.h> + +struct mini_Qdisc; + +struct tcx_entry { + struct mini_Qdisc __rcu *miniq; + struct bpf_mprog_bundle bundle; + bool miniq_active; + struct rcu_head rcu; +}; + +struct tcx_link { + struct bpf_link link; + struct net_device *dev; + u32 location; +}; + +static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress) +{ +#ifdef CONFIG_NET_XGRESS + skb->tc_at_ingress = ingress; +#endif +} + +#ifdef CONFIG_NET_XGRESS +static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry) +{ + struct bpf_mprog_bundle *bundle = entry->parent; + + return container_of(bundle, struct tcx_entry, bundle); +} + +static inline struct tcx_link *tcx_link(struct bpf_link *link) +{ + return container_of(link, struct tcx_link, link); +} + +static inline const struct tcx_link *tcx_link_const(const struct bpf_link *link) +{ + return tcx_link((struct bpf_link *)link); +} + +void tcx_inc(void); +void tcx_dec(void); + +static inline void tcx_entry_sync(void) +{ + /* bpf_mprog_entry got a/b swapped, therefore ensure that + * there are no inflight users on the old one anymore. + */ + synchronize_rcu(); +} + +static inline void +tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry, + bool ingress) +{ + ASSERT_RTNL(); + if (ingress) + rcu_assign_pointer(dev->tcx_ingress, entry); + else + rcu_assign_pointer(dev->tcx_egress, entry); +} + +static inline struct bpf_mprog_entry * +tcx_entry_fetch(struct net_device *dev, bool ingress) +{ + ASSERT_RTNL(); + if (ingress) + return rcu_dereference_rtnl(dev->tcx_ingress); + else + return rcu_dereference_rtnl(dev->tcx_egress); +} + +static inline struct bpf_mprog_entry *tcx_entry_create(void) +{ + struct tcx_entry *tcx = kzalloc(sizeof(*tcx), GFP_KERNEL); + + if (tcx) { + bpf_mprog_bundle_init(&tcx->bundle); + return &tcx->bundle.a; + } + return NULL; +} + +static inline void tcx_entry_free(struct bpf_mprog_entry *entry) +{ + kfree_rcu(tcx_entry(entry), rcu); +} + +static inline struct bpf_mprog_entry * +tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created) +{ + struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress); + + *created = false; + if (!entry) { + entry = tcx_entry_create(); + if (!entry) + return NULL; + *created = true; + } + return entry; +} + +static inline void tcx_skeys_inc(bool ingress) +{ + tcx_inc(); + if (ingress) + net_inc_ingress_queue(); + else + net_inc_egress_queue(); +} + +static inline void tcx_skeys_dec(bool ingress) +{ + if (ingress) + net_dec_ingress_queue(); + else + net_dec_egress_queue(); + tcx_dec(); +} + +static inline void tcx_miniq_set_active(struct bpf_mprog_entry *entry, + const bool active) +{ + ASSERT_RTNL(); + tcx_entry(entry)->miniq_active = active; +} + +static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry) +{ + ASSERT_RTNL(); + return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active; +} + +static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb, + int code) +{ + switch (code) { + case TCX_PASS: + skb->tc_index = qdisc_skb_cb(skb)->tc_classid; + fallthrough; + case TCX_DROP: + case TCX_REDIRECT: + return code; + case TCX_NEXT: + default: + return TCX_NEXT; + } +} +#endif /* CONFIG_NET_XGRESS */ + +#if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL) +int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog); +int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); +int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog); +void tcx_uninstall(struct net_device *dev, bool ingress); + +int tcx_prog_query(const union bpf_attr *attr, + union bpf_attr __user *uattr); + +static inline void dev_tcx_uninstall(struct net_device *dev) +{ + ASSERT_RTNL(); + tcx_uninstall(dev, true); + tcx_uninstall(dev, false); +} +#else +static inline int tcx_prog_attach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + return -EINVAL; +} + +static inline int tcx_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + return -EINVAL; +} + +static inline int tcx_prog_detach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + return -EINVAL; +} + +static inline int tcx_prog_query(const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + return -EINVAL; +} + +static inline void dev_tcx_uninstall(struct net_device *dev) +{ +} +#endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */ +#endif /* __NET_TCX_H */ diff --git a/include/net/tls.h b/include/net/tls.h index 5e71dd3df8ca..06fca9160346 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -69,10 +69,6 @@ extern const struct tls_cipher_size_desc tls_cipher_size_desc[]; #define TLS_CRYPTO_INFO_READY(info) ((info)->cipher_type) -#define TLS_RECORD_TYPE_ALERT 0x15 -#define TLS_RECORD_TYPE_HANDSHAKE 0x16 -#define TLS_RECORD_TYPE_DATA 0x17 - #define TLS_AAD_SPACE_SIZE 13 #define MAX_IV_SIZE 16 diff --git a/include/net/tls_prot.h b/include/net/tls_prot.h new file mode 100644 index 000000000000..68a40756440b --- /dev/null +++ b/include/net/tls_prot.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. + * + * TLS Protocol definitions + * + * From https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml + */ + +#ifndef _TLS_PROT_H +#define _TLS_PROT_H + +/* + * TLS Record protocol: ContentType + */ +enum { + TLS_RECORD_TYPE_CHANGE_CIPHER_SPEC = 20, + TLS_RECORD_TYPE_ALERT = 21, + TLS_RECORD_TYPE_HANDSHAKE = 22, + TLS_RECORD_TYPE_DATA = 23, + TLS_RECORD_TYPE_HEARTBEAT = 24, + TLS_RECORD_TYPE_TLS12_CID = 25, + TLS_RECORD_TYPE_ACK = 26, +}; + +/* + * TLS Alert protocol: AlertLevel + */ +enum { + TLS_ALERT_LEVEL_WARNING = 1, + TLS_ALERT_LEVEL_FATAL = 2, +}; + +/* + * TLS Alert protocol: AlertDescription + */ +enum { + TLS_ALERT_DESC_CLOSE_NOTIFY = 0, + TLS_ALERT_DESC_UNEXPECTED_MESSAGE = 10, + TLS_ALERT_DESC_BAD_RECORD_MAC = 20, + TLS_ALERT_DESC_RECORD_OVERFLOW = 22, + TLS_ALERT_DESC_HANDSHAKE_FAILURE = 40, + TLS_ALERT_DESC_BAD_CERTIFICATE = 42, + TLS_ALERT_DESC_UNSUPPORTED_CERTIFICATE = 43, + TLS_ALERT_DESC_CERTIFICATE_REVOKED = 44, + TLS_ALERT_DESC_CERTIFICATE_EXPIRED = 45, + TLS_ALERT_DESC_CERTIFICATE_UNKNOWN = 46, + TLS_ALERT_DESC_ILLEGAL_PARAMETER = 47, + TLS_ALERT_DESC_UNKNOWN_CA = 48, + TLS_ALERT_DESC_ACCESS_DENIED = 49, + TLS_ALERT_DESC_DECODE_ERROR = 50, + TLS_ALERT_DESC_DECRYPT_ERROR = 51, + TLS_ALERT_DESC_TOO_MANY_CIDS_REQUESTED = 52, + TLS_ALERT_DESC_PROTOCOL_VERSION = 70, + TLS_ALERT_DESC_INSUFFICIENT_SECURITY = 71, + TLS_ALERT_DESC_INTERNAL_ERROR = 80, + TLS_ALERT_DESC_INAPPROPRIATE_FALLBACK = 86, + TLS_ALERT_DESC_USER_CANCELED = 90, + TLS_ALERT_DESC_MISSING_EXTENSION = 109, + TLS_ALERT_DESC_UNSUPPORTED_EXTENSION = 110, + TLS_ALERT_DESC_UNRECOGNIZED_NAME = 112, + TLS_ALERT_DESC_BAD_CERTIFICATE_STATUS_RESPONSE = 113, + TLS_ALERT_DESC_UNKNOWN_PSK_IDENTITY = 115, + TLS_ALERT_DESC_CERTIFICATE_REQUIRED = 116, + TLS_ALERT_DESC_NO_APPLICATION_PROTOCOL = 120, +}; + +#endif /* _TLS_PROT_H */ diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h index d27b1caf3753..1a97e3f32029 100644 --- a/include/net/transp_v6.h +++ b/include/net/transp_v6.h @@ -33,8 +33,6 @@ void udplitev6_exit(void); int tcpv6_init(void); void tcpv6_exit(void); -int udpv6_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); - /* this does all the common and the specific ctl work */ void ip6_datagram_recv_ctl(struct sock *sk, struct msghdr *msg, struct sk_buff *skb); diff --git a/include/net/udp.h b/include/net/udp.h index 4d13424f8f72..5a8421cd9083 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -529,7 +529,6 @@ static inline void udp_post_segment_fix_csum(struct sk_buff *skb) #ifdef CONFIG_BPF_SYSCALL struct sk_psock; -struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock); int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); #endif diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index e96a1151ec75..1617af380162 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -52,6 +52,7 @@ struct xdp_sock { struct xsk_buff_pool *pool; u16 queue_id; bool zc; + bool sg; enum { XSK_READY = 0, XSK_BOUND, @@ -67,6 +68,12 @@ struct xdp_sock { u64 rx_dropped; u64 rx_queue_full; + /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current + * packet, the partially built skb is saved here so that packet building can resume in next + * call of __xsk_generic_xmit(). + */ + struct sk_buff *skb; + struct list_head map_list; /* Protects map_list */ spinlock_t map_list_lock; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index c243f906ebed..1f6fc8c7a84c 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -89,6 +89,11 @@ static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) return xp_alloc(pool); } +static inline bool xsk_is_eop_desc(struct xdp_desc *desc) +{ + return !xp_mb_desc(desc); +} + /* Returns as many entries as possible up to max. 0 <= N <= max. */ static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) { @@ -103,10 +108,45 @@ static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count) static inline void xsk_buff_free(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); + struct list_head *xskb_list = &xskb->pool->xskb_list; + struct xdp_buff_xsk *pos, *tmp; + + if (likely(!xdp_buff_has_frags(xdp))) + goto out; + list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) { + list_del(&pos->xskb_list_node); + xp_free(pos); + } + + xdp_get_shared_info_from_buff(xdp)->nr_frags = 0; +out: xp_free(xskb); } +static inline void xsk_buff_add_frag(struct xdp_buff *xdp) +{ + struct xdp_buff_xsk *frag = container_of(xdp, struct xdp_buff_xsk, xdp); + + list_add_tail(&frag->xskb_list_node, &frag->pool->xskb_list); +} + +static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) +{ + struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); + struct xdp_buff *ret = NULL; + struct xdp_buff_xsk *frag; + + frag = list_first_entry_or_null(&xskb->pool->xskb_list, + struct xdp_buff_xsk, xskb_list_node); + if (frag) { + list_del(&frag->xskb_list_node); + ret = &frag->xdp; + } + + return ret; +} + static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) { xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; @@ -241,6 +281,11 @@ static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) return NULL; } +static inline bool xsk_is_eop_desc(struct xdp_desc *desc) +{ + return false; +} + static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) { return 0; @@ -255,6 +300,15 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) { } +static inline void xsk_buff_add_frag(struct xdp_buff *xdp) +{ +} + +static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) +{ + return NULL; +} + static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) { } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index a8d7b8a3688a..b0bdff26fc88 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -29,6 +29,7 @@ struct xdp_buff_xsk { struct xsk_buff_pool *pool; u64 orig_addr; struct list_head free_list_node; + struct list_head xskb_list_node; }; #define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb)) @@ -54,6 +55,7 @@ struct xsk_buff_pool { struct xdp_umem *umem; struct work_struct work; struct list_head free_list; + struct list_head xskb_list; u32 heads_cnt; u16 queue_id; @@ -184,6 +186,11 @@ static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool, !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK); } +static inline bool xp_mb_desc(struct xdp_desc *desc) +{ + return desc->options & XDP_PKT_CONTD; +} + static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr) { return addr & pool->chunk_mask; diff --git a/include/trace/events/handshake.h b/include/trace/events/handshake.h index 8dadcab5f12a..bdd8a03cf5ba 100644 --- a/include/trace/events/handshake.h +++ b/include/trace/events/handshake.h @@ -6,7 +6,86 @@ #define _TRACE_HANDSHAKE_H #include <linux/net.h> +#include <net/tls_prot.h> #include <linux/tracepoint.h> +#include <trace/events/net_probe_common.h> + +#define TLS_RECORD_TYPE_LIST \ + record_type(CHANGE_CIPHER_SPEC) \ + record_type(ALERT) \ + record_type(HANDSHAKE) \ + record_type(DATA) \ + record_type(HEARTBEAT) \ + record_type(TLS12_CID) \ + record_type_end(ACK) + +#undef record_type +#undef record_type_end +#define record_type(x) TRACE_DEFINE_ENUM(TLS_RECORD_TYPE_##x); +#define record_type_end(x) TRACE_DEFINE_ENUM(TLS_RECORD_TYPE_##x); + +TLS_RECORD_TYPE_LIST + +#undef record_type +#undef record_type_end +#define record_type(x) { TLS_RECORD_TYPE_##x, #x }, +#define record_type_end(x) { TLS_RECORD_TYPE_##x, #x } + +#define show_tls_content_type(type) \ + __print_symbolic(type, TLS_RECORD_TYPE_LIST) + +TRACE_DEFINE_ENUM(TLS_ALERT_LEVEL_WARNING); +TRACE_DEFINE_ENUM(TLS_ALERT_LEVEL_FATAL); + +#define show_tls_alert_level(level) \ + __print_symbolic(level, \ + { TLS_ALERT_LEVEL_WARNING, "Warning" }, \ + { TLS_ALERT_LEVEL_FATAL, "Fatal" }) + +#define TLS_ALERT_DESCRIPTION_LIST \ + alert_description(CLOSE_NOTIFY) \ + alert_description(UNEXPECTED_MESSAGE) \ + alert_description(BAD_RECORD_MAC) \ + alert_description(RECORD_OVERFLOW) \ + alert_description(HANDSHAKE_FAILURE) \ + alert_description(BAD_CERTIFICATE) \ + alert_description(UNSUPPORTED_CERTIFICATE) \ + alert_description(CERTIFICATE_REVOKED) \ + alert_description(CERTIFICATE_EXPIRED) \ + alert_description(CERTIFICATE_UNKNOWN) \ + alert_description(ILLEGAL_PARAMETER) \ + alert_description(UNKNOWN_CA) \ + alert_description(ACCESS_DENIED) \ + alert_description(DECODE_ERROR) \ + alert_description(DECRYPT_ERROR) \ + alert_description(TOO_MANY_CIDS_REQUESTED) \ + alert_description(PROTOCOL_VERSION) \ + alert_description(INSUFFICIENT_SECURITY) \ + alert_description(INTERNAL_ERROR) \ + alert_description(INAPPROPRIATE_FALLBACK) \ + alert_description(USER_CANCELED) \ + alert_description(MISSING_EXTENSION) \ + alert_description(UNSUPPORTED_EXTENSION) \ + alert_description(UNRECOGNIZED_NAME) \ + alert_description(BAD_CERTIFICATE_STATUS_RESPONSE) \ + alert_description(UNKNOWN_PSK_IDENTITY) \ + alert_description(CERTIFICATE_REQUIRED) \ + alert_description_end(NO_APPLICATION_PROTOCOL) + +#undef alert_description +#undef alert_description_end +#define alert_description(x) TRACE_DEFINE_ENUM(TLS_ALERT_DESC_##x); +#define alert_description_end(x) TRACE_DEFINE_ENUM(TLS_ALERT_DESC_##x); + +TLS_ALERT_DESCRIPTION_LIST + +#undef alert_description +#undef alert_description_end +#define alert_description(x) { TLS_ALERT_DESC_##x, #x }, +#define alert_description_end(x) { TLS_ALERT_DESC_##x, #x } + +#define show_tls_alert_description(desc) \ + __print_symbolic(desc, TLS_ALERT_DESCRIPTION_LIST) DECLARE_EVENT_CLASS(handshake_event_class, TP_PROTO( @@ -106,6 +185,47 @@ DECLARE_EVENT_CLASS(handshake_error_class, ), \ TP_ARGS(net, req, sk, err)) +DECLARE_EVENT_CLASS(handshake_alert_class, + TP_PROTO( + const struct sock *sk, + unsigned char level, + unsigned char description + ), + TP_ARGS(sk, level, description), + TP_STRUCT__entry( + /* sockaddr_in6 is always bigger than sockaddr_in */ + __array(__u8, saddr, sizeof(struct sockaddr_in6)) + __array(__u8, daddr, sizeof(struct sockaddr_in6)) + __field(unsigned int, netns_ino) + __field(unsigned long, level) + __field(unsigned long, description) + ), + TP_fast_assign( + const struct inet_sock *inet = inet_sk(sk); + + memset(__entry->saddr, 0, sizeof(struct sockaddr_in6)); + memset(__entry->daddr, 0, sizeof(struct sockaddr_in6)); + TP_STORE_ADDR_PORTS(__entry, inet, sk); + + __entry->netns_ino = sock_net(sk)->ns.inum; + __entry->level = level; + __entry->description = description; + ), + TP_printk("src=%pISpc dest=%pISpc %s: %s", + __entry->saddr, __entry->daddr, + show_tls_alert_level(__entry->level), + show_tls_alert_description(__entry->description) + ) +); +#define DEFINE_HANDSHAKE_ALERT(name) \ + DEFINE_EVENT(handshake_alert_class, name, \ + TP_PROTO( \ + const struct sock *sk, \ + unsigned char level, \ + unsigned char description \ + ), \ + TP_ARGS(sk, level, description)) + /* * Request lifetime events @@ -154,6 +274,46 @@ DEFINE_HANDSHAKE_ERROR(handshake_cmd_accept_err); DEFINE_HANDSHAKE_FD_EVENT(handshake_cmd_done); DEFINE_HANDSHAKE_ERROR(handshake_cmd_done_err); +/* + * TLS Record events + */ + +TRACE_EVENT(tls_contenttype, + TP_PROTO( + const struct sock *sk, + unsigned char type + ), + TP_ARGS(sk, type), + TP_STRUCT__entry( + /* sockaddr_in6 is always bigger than sockaddr_in */ + __array(__u8, saddr, sizeof(struct sockaddr_in6)) + __array(__u8, daddr, sizeof(struct sockaddr_in6)) + __field(unsigned int, netns_ino) + __field(unsigned long, type) + ), + TP_fast_assign( + const struct inet_sock *inet = inet_sk(sk); + + memset(__entry->saddr, 0, sizeof(struct sockaddr_in6)); + memset(__entry->daddr, 0, sizeof(struct sockaddr_in6)); + TP_STORE_ADDR_PORTS(__entry, inet, sk); + + __entry->netns_ino = sock_net(sk)->ns.inum; + __entry->type = type; + ), + TP_printk("src=%pISpc dest=%pISpc %s", + __entry->saddr, __entry->daddr, + show_tls_content_type(__entry->type) + ) +); + +/* + * TLS Alert events + */ + +DEFINE_HANDSHAKE_ALERT(tls_alert_send); +DEFINE_HANDSHAKE_ALERT(tls_alert_recv); + #endif /* _TRACE_HANDSHAKE_H */ #include <trace/define_trace.h> diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 60a9d59beeab..739c15906a65 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1036,6 +1036,8 @@ enum bpf_attach_type { BPF_LSM_CGROUP, BPF_STRUCT_OPS, BPF_NETFILTER, + BPF_TCX_INGRESS, + BPF_TCX_EGRESS, __MAX_BPF_ATTACH_TYPE }; @@ -1053,10 +1055,20 @@ enum bpf_link_type { BPF_LINK_TYPE_KPROBE_MULTI = 8, BPF_LINK_TYPE_STRUCT_OPS = 9, BPF_LINK_TYPE_NETFILTER = 10, - + BPF_LINK_TYPE_TCX = 11, MAX_BPF_LINK_TYPE, }; +enum bpf_perf_event_type { + BPF_PERF_EVENT_UNSPEC = 0, + BPF_PERF_EVENT_UPROBE = 1, + BPF_PERF_EVENT_URETPROBE = 2, + BPF_PERF_EVENT_KPROBE = 3, + BPF_PERF_EVENT_KRETPROBE = 4, + BPF_PERF_EVENT_TRACEPOINT = 5, + BPF_PERF_EVENT_EVENT = 6, +}; + /* cgroup-bpf attach flags used in BPF_PROG_ATTACH command * * NONE(default): No further bpf programs allowed in the subtree. @@ -1103,7 +1115,12 @@ enum bpf_link_type { */ #define BPF_F_ALLOW_OVERRIDE (1U << 0) #define BPF_F_ALLOW_MULTI (1U << 1) +/* Generic attachment flags. */ #define BPF_F_REPLACE (1U << 2) +#define BPF_F_BEFORE (1U << 3) +#define BPF_F_AFTER (1U << 4) +#define BPF_F_ID (1U << 5) +#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */ /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the * verifier will perform strict alignment checking as if the kernel @@ -1434,14 +1451,19 @@ union bpf_attr { }; struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ - __u32 target_fd; /* container object to attach to */ - __u32 attach_bpf_fd; /* eBPF program to attach */ + union { + __u32 target_fd; /* target object to attach to or ... */ + __u32 target_ifindex; /* target ifindex */ + }; + __u32 attach_bpf_fd; __u32 attach_type; __u32 attach_flags; - __u32 replace_bpf_fd; /* previously attached eBPF - * program to replace if - * BPF_F_REPLACE is used - */ + __u32 replace_bpf_fd; + union { + __u32 relative_fd; + __u32 relative_id; + }; + __u64 expected_revision; }; struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ @@ -1487,16 +1509,26 @@ union bpf_attr { } info; struct { /* anonymous struct used by BPF_PROG_QUERY command */ - __u32 target_fd; /* container object to query */ + union { + __u32 target_fd; /* target object to query or ... */ + __u32 target_ifindex; /* target ifindex */ + }; __u32 attach_type; __u32 query_flags; __u32 attach_flags; __aligned_u64 prog_ids; - __u32 prog_cnt; + union { + __u32 prog_cnt; + __u32 count; + }; + __u32 :32; /* output: per-program attach_flags. * not allowed to be set during effective query. */ __aligned_u64 prog_attach_flags; + __aligned_u64 link_ids; + __aligned_u64 link_attach_flags; + __u64 revision; } query; struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ @@ -1539,13 +1571,13 @@ union bpf_attr { __u32 map_fd; /* struct_ops to attach */ }; union { - __u32 target_fd; /* object to attach to */ - __u32 target_ifindex; /* target ifindex */ + __u32 target_fd; /* target object to attach to or ... */ + __u32 target_ifindex; /* target ifindex */ }; __u32 attach_type; /* attach type */ __u32 flags; /* extra flags */ union { - __u32 target_btf_id; /* btf_id of target to attach to */ + __u32 target_btf_id; /* btf_id of target to attach to */ struct { __aligned_u64 iter_info; /* extra bpf_iter_link_info */ __u32 iter_info_len; /* iter_info length */ @@ -1579,6 +1611,13 @@ union bpf_attr { __s32 priority; __u32 flags; } netfilter; + struct { + union { + __u32 relative_fd; + __u32 relative_id; + }; + __u64 expected_revision; + } tcx; }; } link_create; @@ -6187,6 +6226,19 @@ struct bpf_sock_tuple { }; }; +/* (Simplified) user return codes for tcx prog type. + * A valid tcx program must return one of these defined values. All other + * return codes are reserved for future use. Must remain compatible with + * their TC_ACT_* counter-parts. For compatibility in behavior, unknown + * return codes are mapped to TCX_NEXT. + */ +enum tcx_action_base { + TCX_NEXT = -1, + TCX_PASS = 0, + TCX_DROP = 2, + TCX_REDIRECT = 7, +}; + struct bpf_xdp_sock { __u32 queue_id; }; @@ -6439,6 +6491,40 @@ struct bpf_link_info { __s32 priority; __u32 flags; } netfilter; + struct { + __aligned_u64 addrs; + __u32 count; /* in/out: kprobe_multi function count */ + __u32 flags; + } kprobe_multi; + struct { + __u32 type; /* enum bpf_perf_event_type */ + __u32 :32; + union { + struct { + __aligned_u64 file_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from file_name */ + } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */ + struct { + __aligned_u64 func_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from func_name */ + __u64 addr; + } kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */ + struct { + __aligned_u64 tp_name; /* in/out */ + __u32 name_len; + } tracepoint; /* BPF_PERF_EVENT_TRACEPOINT */ + struct { + __u64 config; + __u32 type; + } event; /* BPF_PERF_EVENT_EVENT */ + }; + } perf_event; + struct { + __u32 ifindex; + __u32 attach_type; + } tcx; }; } __attribute__((aligned(8))); @@ -7012,6 +7098,7 @@ struct bpf_list_head { struct bpf_list_node { __u64 :64; __u64 :64; + __u64 :64; } __attribute__((aligned(8))); struct bpf_rb_root { @@ -7023,6 +7110,7 @@ struct bpf_rb_node { __u64 :64; __u64 :64; __u64 :64; + __u64 :64; } __attribute__((aligned(8))); struct bpf_refcount { diff --git a/include/uapi/linux/cn_proc.h b/include/uapi/linux/cn_proc.h index db210625cee8..f2afb7cc4926 100644 --- a/include/uapi/linux/cn_proc.h +++ b/include/uapi/linux/cn_proc.h @@ -30,6 +30,49 @@ enum proc_cn_mcast_op { PROC_CN_MCAST_IGNORE = 2 }; +#define PROC_EVENT_ALL (PROC_EVENT_FORK | PROC_EVENT_EXEC | PROC_EVENT_UID | \ + PROC_EVENT_GID | PROC_EVENT_SID | PROC_EVENT_PTRACE | \ + PROC_EVENT_COMM | PROC_EVENT_NONZERO_EXIT | \ + PROC_EVENT_COREDUMP | PROC_EVENT_EXIT) + +/* + * If you add an entry in proc_cn_event, make sure you add it in + * PROC_EVENT_ALL above as well. + */ +enum proc_cn_event { + /* Use successive bits so the enums can be used to record + * sets of events as well + */ + PROC_EVENT_NONE = 0x00000000, + PROC_EVENT_FORK = 0x00000001, + PROC_EVENT_EXEC = 0x00000002, + PROC_EVENT_UID = 0x00000004, + PROC_EVENT_GID = 0x00000040, + PROC_EVENT_SID = 0x00000080, + PROC_EVENT_PTRACE = 0x00000100, + PROC_EVENT_COMM = 0x00000200, + /* "next" should be 0x00000400 */ + /* "last" is the last process event: exit, + * while "next to last" is coredumping event + * before that is report only if process dies + * with non-zero exit status + */ + PROC_EVENT_NONZERO_EXIT = 0x20000000, + PROC_EVENT_COREDUMP = 0x40000000, + PROC_EVENT_EXIT = 0x80000000 +}; + +struct proc_input { + enum proc_cn_mcast_op mcast_op; + enum proc_cn_event event_type; +}; + +static inline enum proc_cn_event valid_event(enum proc_cn_event ev_type) +{ + ev_type &= PROC_EVENT_ALL; + return ev_type; +} + /* * From the user's point of view, the process * ID is the thread group ID and thread ID is the internal @@ -44,24 +87,7 @@ enum proc_cn_mcast_op { */ struct proc_event { - enum what { - /* Use successive bits so the enums can be used to record - * sets of events as well - */ - PROC_EVENT_NONE = 0x00000000, - PROC_EVENT_FORK = 0x00000001, - PROC_EVENT_EXEC = 0x00000002, - PROC_EVENT_UID = 0x00000004, - PROC_EVENT_GID = 0x00000040, - PROC_EVENT_SID = 0x00000080, - PROC_EVENT_PTRACE = 0x00000100, - PROC_EVENT_COMM = 0x00000200, - /* "next" should be 0x00000400 */ - /* "last" is the last process event: exit, - * while "next to last" is coredumping event */ - PROC_EVENT_COREDUMP = 0x40000000, - PROC_EVENT_EXIT = 0x80000000 - } what; + enum proc_cn_event what; __u32 cpu; __u64 __attribute__((aligned(8))) timestamp_ns; /* Number of nano seconds since system boot */ diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 0f6a0fe09bdb..ce3117df9cec 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -570,6 +570,7 @@ enum { IFLA_BRPORT_MCAST_N_GROUPS, IFLA_BRPORT_MCAST_MAX_GROUPS, IFLA_BRPORT_NEIGH_VLAN_SUPPRESS, + IFLA_BRPORT_BACKUP_NHID, __IFLA_BRPORT_MAX }; #define IFLA_BRPORT_MAX (__IFLA_BRPORT_MAX - 1) diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h index a78a8096f4ce..8d48863472b9 100644 --- a/include/uapi/linux/if_xdp.h +++ b/include/uapi/linux/if_xdp.h @@ -25,6 +25,12 @@ * application. */ #define XDP_USE_NEED_WAKEUP (1 << 3) +/* By setting this option, userspace application indicates that it can + * handle multiple descriptors per packet thus enabling AF_XDP to split + * multi-buffer XDP frames into multiple Rx descriptors. Without this set + * such frames will be dropped. + */ +#define XDP_USE_SG (1 << 4) /* Flags for xsk_umem_config flags */ #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) @@ -108,4 +114,11 @@ struct xdp_desc { /* UMEM descriptor is __u64 */ +/* Flag indicating that the packet continues with the buffer pointed out by the + * next frame in the ring. The end of the packet is signalled by setting this + * bit to zero. For single buffer packets, every descriptor has 'options' set + * to 0 and this maintains backward compatibility. + */ +#define XDP_PKT_CONTD (1 << 0) + #endif /* _LINUX_IF_XDP_H */ diff --git a/include/uapi/linux/ipv6.h b/include/uapi/linux/ipv6.h index ac56605fe9bc..cf592d7b630f 100644 --- a/include/uapi/linux/ipv6.h +++ b/include/uapi/linux/ipv6.h @@ -198,6 +198,7 @@ enum { DEVCONF_IOAM6_ID_WIDE, DEVCONF_NDISC_EVICT_NOCARRIER, DEVCONF_ACCEPT_UNTRACKED_NA, + DEVCONF_ACCEPT_RA_MIN_LFT, DEVCONF_MAX }; diff --git a/include/uapi/linux/mdio.h b/include/uapi/linux/mdio.h index b826598d1e94..d03863da180e 100644 --- a/include/uapi/linux/mdio.h +++ b/include/uapi/linux/mdio.h @@ -82,6 +82,8 @@ #define MDIO_AN_10BT1_AN_CTRL 526 /* 10BASE-T1 AN control register */ #define MDIO_AN_10BT1_AN_STAT 527 /* 10BASE-T1 AN status register */ #define MDIO_PMA_PMD_BT1_CTRL 2100 /* BASE-T1 PMA/PMD control register */ +#define MDIO_PCS_1000BT1_CTRL 2304 /* 1000BASE-T1 PCS control register */ +#define MDIO_PCS_1000BT1_STAT 2305 /* 1000BASE-T1 PCS status register */ /* LASI (Link Alarm Status Interrupt) registers, defined by XENPAK MSA. */ #define MDIO_PMA_LASI_RXCTRL 0x9000 /* RX_ALARM control */ @@ -332,6 +334,8 @@ #define MDIO_PCS_10T1L_CTRL_RESET 0x8000 /* PCS reset */ /* BASE-T1 PMA/PMD extended ability register. */ +#define MDIO_PMA_PMD_BT1_B100_ABLE 0x0001 /* 100BASE-T1 Ability */ +#define MDIO_PMA_PMD_BT1_B1000_ABLE 0x0002 /* 1000BASE-T1 Ability */ #define MDIO_PMA_PMD_BT1_B10L_ABLE 0x0004 /* 10BASE-T1L Ability */ /* BASE-T1 auto-negotiation advertisement register [15:0] */ @@ -373,7 +377,19 @@ #define MDIO_AN_10BT1_AN_STAT_LPA_EEE_T1L 0x4000 /* 10BASE-T1L LP EEE ability advertisement */ /* BASE-T1 PMA/PMD control register */ -#define MDIO_PMA_PMD_BT1_CTRL_CFG_MST 0x4000 /* MASTER-SLAVE config value */ +#define MDIO_PMA_PMD_BT1_CTRL_STRAP 0x000F /* Type selection (Strap) */ +#define MDIO_PMA_PMD_BT1_CTRL_STRAP_B1000 0x0001 /* Select 1000BASE-T1 */ +#define MDIO_PMA_PMD_BT1_CTRL_CFG_MST 0x4000 /* MASTER-SLAVE config value */ + +/* 1000BASE-T1 PCS control register */ +#define MDIO_PCS_1000BT1_CTRL_LOW_POWER 0x0800 /* Low power mode */ +#define MDIO_PCS_1000BT1_CTRL_DISABLE_TX 0x4000 /* Global PMA transmit disable */ +#define MDIO_PCS_1000BT1_CTRL_RESET 0x8000 /* Software reset value */ + +/* 1000BASE-T1 PCS status register */ +#define MDIO_PCS_1000BT1_STAT_LINK 0x0004 /* PCS Link is up */ +#define MDIO_PCS_1000BT1_STAT_FAULT 0x0080 /* There is a fault condition */ + /* EEE Supported/Advertisement/LP Advertisement registers. * diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h index 639524b59930..c1634b95c223 100644 --- a/include/uapi/linux/netdev.h +++ b/include/uapi/linux/netdev.h @@ -11,7 +11,7 @@ /** * enum netdev_xdp_act - * @NETDEV_XDP_ACT_BASIC: XDP feautues set supported by all drivers + * @NETDEV_XDP_ACT_BASIC: XDP features set supported by all drivers * (XDP_ABORTED, XDP_DROP, XDP_PASS, XDP_TX) * @NETDEV_XDP_ACT_REDIRECT: The netdev supports XDP_REDIRECT * @NETDEV_XDP_ACT_NDO_XMIT: This feature informs if netdev implements @@ -34,6 +34,7 @@ enum netdev_xdp_act { NETDEV_XDP_ACT_RX_SG = 32, NETDEV_XDP_ACT_NDO_XMIT_SG = 64, + /* private: */ NETDEV_XDP_ACT_MASK = 127, }; @@ -41,6 +42,7 @@ enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, NETDEV_A_DEV_XDP_FEATURES, + NETDEV_A_DEV_XDP_ZC_MAX_SEGS, __NETDEV_A_DEV_MAX, NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) diff --git a/include/uapi/linux/pkt_cls.h b/include/uapi/linux/pkt_cls.h index 4f3932bb712d..c7082cc60d21 100644 --- a/include/uapi/linux/pkt_cls.h +++ b/include/uapi/linux/pkt_cls.h @@ -598,6 +598,9 @@ enum { TCA_FLOWER_KEY_CFM, /* nested */ + TCA_FLOWER_KEY_SPI, /* be32 */ + TCA_FLOWER_KEY_SPI_MASK, /* be32 */ + __TCA_FLOWER_MAX, }; diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index 12c1c9699935..cc65ef0f3c3e 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -56,6 +56,7 @@ #define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ +#define VIRTIO_NET_F_VQ_NOTF_COAL 52 /* Device supports virtqueue notification coalescing */ #define VIRTIO_NET_F_NOTF_COAL 53 /* Device supports notifications coalescing */ #define VIRTIO_NET_F_GUEST_USO4 54 /* Guest can handle USOv4 in. */ #define VIRTIO_NET_F_GUEST_USO6 55 /* Guest can handle USOv6 in. */ @@ -391,5 +392,18 @@ struct virtio_net_ctrl_coal_rx { }; #define VIRTIO_NET_CTRL_NOTF_COAL_RX_SET 1 +#define VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET 2 +#define VIRTIO_NET_CTRL_NOTF_COAL_VQ_GET 3 + +struct virtio_net_ctrl_coal { + __le32 max_packets; + __le32 max_usecs; +}; + +struct virtio_net_ctrl_coal_vq { + __le16 vqn; + __le16 reserved; + struct virtio_net_ctrl_coal coal; +}; #endif /* _UAPI_LINUX_VIRTIO_NET_H */ diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index 2dfe1079f772..6a906ff93006 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -31,6 +31,7 @@ config BPF_SYSCALL select TASKS_TRACE_RCU select BINARY_PRINTF select NET_SOCK_MSG if NET + select NET_XGRESS if NET select PAGE_POOL if NET default n help diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 1d3892168d32..f526b7573e97 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o -obj-$(CONFIG_BPF_SYSCALL) += disasm.o +obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o obj-$(CONFIG_BPF_JIT) += trampoline.o obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o obj-$(CONFIG_BPF_JIT) += dispatcher.o @@ -21,6 +21,7 @@ obj-$(CONFIG_BPF_SYSCALL) += devmap.o obj-$(CONFIG_BPF_SYSCALL) += cpumap.o obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o +obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 817204d53372..ef9581a580e2 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6133,8 +6133,9 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const char *tname, *mname, *tag_value; u32 vlen, elem_id, mid; - *flag = 0; again: + if (btf_type_is_modifier(t)) + t = btf_type_skip_modifiers(btf, t->type, NULL); tname = __btf_name_by_offset(btf, t->name_off); if (!btf_type_is_struct(t)) { bpf_log(log, "Type '%s' is not a struct\n", tname); @@ -6142,6 +6143,14 @@ again: } vlen = btf_type_vlen(t); + if (BTF_INFO_KIND(t->info) == BTF_KIND_UNION && vlen != 1 && !(*flag & PTR_UNTRUSTED)) + /* + * walking unions yields untrusted pointers + * with exception of __bpf_md_ptr and other + * unions with a single member + */ + *flag |= PTR_UNTRUSTED; + if (off + size > t->size) { /* If the last element is a variable size array, we may * need to relax the rule. @@ -6302,15 +6311,6 @@ error: * of this field or inside of this struct */ if (btf_type_is_struct(mtype)) { - if (BTF_INFO_KIND(mtype->info) == BTF_KIND_UNION && - btf_type_vlen(mtype) != 1) - /* - * walking unions yields untrusted pointers - * with exception of __bpf_md_ptr and other - * unions with a single member - */ - *flag |= PTR_UNTRUSTED; - /* our field must be inside that union or struct */ t = mtype; @@ -6368,7 +6368,7 @@ error: * that also allows using an array of int as a scratch * space. e.g. skb->cb[]. */ - if (off + size > mtrue_end) { + if (off + size > mtrue_end && !(*flag & PTR_UNTRUSTED)) { bpf_log(log, "access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n", mname, mtrue_end, tname, off, size); @@ -6476,7 +6476,7 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log, bool strict) { const struct btf_type *type; - enum bpf_type_flag flag; + enum bpf_type_flag flag = 0; int err; /* Are we already done? */ diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index 938a60ff4295..6983af8e093c 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -9,7 +9,6 @@ /** * struct bpf_cpumask - refcounted BPF cpumask wrapper structure * @cpumask: The actual cpumask embedded in the struct. - * @rcu: The RCU head used to free the cpumask with RCU safety. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. @@ -25,7 +24,6 @@ */ struct bpf_cpumask { cpumask_t cpumask; - struct rcu_head rcu; refcount_t usage; }; @@ -82,16 +80,6 @@ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) return cpumask; } -static void cpumask_free_cb(struct rcu_head *head) -{ - struct bpf_cpumask *cpumask; - - cpumask = container_of(head, struct bpf_cpumask, rcu); - migrate_disable(); - bpf_mem_cache_free(&bpf_cpumask_ma, cpumask); - migrate_enable(); -} - /** * bpf_cpumask_release() - Release a previously acquired BPF cpumask. * @cpumask: The cpumask being released. @@ -102,8 +90,12 @@ static void cpumask_free_cb(struct rcu_head *head) */ __bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask) { - if (refcount_dec_and_test(&cpumask->usage)) - call_rcu(&cpumask->rcu, cpumask_free_cb); + if (!refcount_dec_and_test(&cpumask->usage)) + return; + + migrate_disable(); + bpf_mem_cache_free_rcu(&bpf_cpumask_ma, cpumask); + migrate_enable(); } /** diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 56d3da7d0bc6..a8c7e1c5abfa 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -302,6 +302,7 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, struct htab_elem *l; if (node) { + bpf_map_inc_elem_count(&htab->map); l = container_of(node, struct htab_elem, lru_node); memcpy(l->key, key, htab->map.key_size); return l; @@ -510,12 +511,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab->n_buckets > U32_MAX / sizeof(struct bucket)) goto free_htab; + err = bpf_map_init_elem_count(&htab->map); + if (err) + goto free_htab; + err = -ENOMEM; htab->buckets = bpf_map_area_alloc(htab->n_buckets * sizeof(struct bucket), htab->map.numa_node); if (!htab->buckets) - goto free_htab; + goto free_elem_count; for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++) { htab->map_locked[i] = bpf_map_alloc_percpu(&htab->map, @@ -593,6 +598,8 @@ free_map_locked: bpf_map_area_free(htab->buckets); bpf_mem_alloc_destroy(&htab->pcpu_ma); bpf_mem_alloc_destroy(&htab->ma); +free_elem_count: + bpf_map_free_elem_count(&htab->map); free_htab: lockdep_unregister_key(&htab->lockdep_key); bpf_map_area_free(htab); @@ -804,6 +811,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) if (l == tgt_l) { hlist_nulls_del_rcu(&l->hash_node); check_and_free_fields(htab, l); + bpf_map_dec_elem_count(&htab->map); break; } @@ -900,6 +908,8 @@ static bool is_map_full(struct bpf_htab *htab) static void inc_elem_count(struct bpf_htab *htab) { + bpf_map_inc_elem_count(&htab->map); + if (htab->use_percpu_counter) percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); else @@ -908,6 +918,8 @@ static void inc_elem_count(struct bpf_htab *htab) static void dec_elem_count(struct bpf_htab *htab) { + bpf_map_dec_elem_count(&htab->map); + if (htab->use_percpu_counter) percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); else @@ -920,6 +932,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) htab_put_fd_value(htab, l); if (htab_is_prealloc(htab)) { + bpf_map_dec_elem_count(&htab->map); check_and_free_fields(htab, l); __pcpu_freelist_push(&htab->freelist, &l->fnode); } else { @@ -1000,6 +1013,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (!l) return ERR_PTR(-E2BIG); l_new = container_of(l, struct htab_elem, fnode); + bpf_map_inc_elem_count(&htab->map); } } else { if (is_map_full(htab)) @@ -1168,6 +1182,7 @@ err: static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { check_and_free_fields(htab, elem); + bpf_map_dec_elem_count(&htab->map); bpf_lru_push_free(&htab->lru, &elem->lru_node); } @@ -1357,8 +1372,10 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, err: htab_unlock_bucket(htab, b, hash, flags); err_lock_bucket: - if (l_new) + if (l_new) { + bpf_map_dec_elem_count(&htab->map); bpf_lru_push_free(&htab->lru, &l_new->lru_node); + } return ret; } @@ -1523,6 +1540,7 @@ static void htab_map_free(struct bpf_map *map) prealloc_destroy(htab); } + bpf_map_free_elem_count(map); free_percpu(htab->extra_elems); bpf_map_area_free(htab->buckets); bpf_mem_alloc_destroy(&htab->pcpu_ma); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 9e80efa59a5d..56ce5008aedd 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1942,23 +1942,29 @@ __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta return (void *)p__refcounted_kptr; } -static int __bpf_list_add(struct bpf_list_node *node, struct bpf_list_head *head, +static int __bpf_list_add(struct bpf_list_node_kern *node, + struct bpf_list_head *head, bool tail, struct btf_record *rec, u64 off) { - struct list_head *n = (void *)node, *h = (void *)head; + struct list_head *n = &node->list_head, *h = (void *)head; /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't * called on its fields, so init here */ if (unlikely(!h->next)) INIT_LIST_HEAD(h); - if (!list_empty(n)) { + + /* node->owner != NULL implies !list_empty(n), no need to separately + * check the latter + */ + if (cmpxchg(&node->owner, NULL, BPF_PTR_POISON)) { /* Only called from BPF prog, no need to migrate_disable */ __bpf_obj_drop_impl((void *)n - off, rec); return -EINVAL; } tail ? list_add_tail(n, h) : list_add(n, h); + WRITE_ONCE(node->owner, head); return 0; } @@ -1967,25 +1973,26 @@ __bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head, struct bpf_list_node *node, void *meta__ign, u64 off) { + struct bpf_list_node_kern *n = (void *)node; struct btf_struct_meta *meta = meta__ign; - return __bpf_list_add(node, head, false, - meta ? meta->record : NULL, off); + return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); } __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, struct bpf_list_node *node, void *meta__ign, u64 off) { + struct bpf_list_node_kern *n = (void *)node; struct btf_struct_meta *meta = meta__ign; - return __bpf_list_add(node, head, true, - meta ? meta->record : NULL, off); + return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); } static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) { struct list_head *n, *h = (void *)head; + struct bpf_list_node_kern *node; /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't * called on its fields, so init here @@ -1994,8 +2001,14 @@ static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tai INIT_LIST_HEAD(h); if (list_empty(h)) return NULL; + n = tail ? h->prev : h->next; + node = container_of(n, struct bpf_list_node_kern, list_head); + if (WARN_ON_ONCE(READ_ONCE(node->owner) != head)) + return NULL; + list_del_init(n); + WRITE_ONCE(node->owner, NULL); return (struct bpf_list_node *)n; } @@ -2012,29 +2025,38 @@ __bpf_kfunc struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, struct bpf_rb_node *node) { + struct bpf_rb_node_kern *node_internal = (struct bpf_rb_node_kern *)node; struct rb_root_cached *r = (struct rb_root_cached *)root; - struct rb_node *n = (struct rb_node *)node; + struct rb_node *n = &node_internal->rb_node; - if (RB_EMPTY_NODE(n)) + /* node_internal->owner != root implies either RB_EMPTY_NODE(n) or + * n is owned by some other tree. No need to check RB_EMPTY_NODE(n) + */ + if (READ_ONCE(node_internal->owner) != root) return NULL; rb_erase_cached(n, r); RB_CLEAR_NODE(n); + WRITE_ONCE(node_internal->owner, NULL); return (struct bpf_rb_node *)n; } /* Need to copy rbtree_add_cached's logic here because our 'less' is a BPF * program */ -static int __bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, +static int __bpf_rbtree_add(struct bpf_rb_root *root, + struct bpf_rb_node_kern *node, void *less, struct btf_record *rec, u64 off) { struct rb_node **link = &((struct rb_root_cached *)root)->rb_root.rb_node; - struct rb_node *parent = NULL, *n = (struct rb_node *)node; + struct rb_node *parent = NULL, *n = &node->rb_node; bpf_callback_t cb = (bpf_callback_t)less; bool leftmost = true; - if (!RB_EMPTY_NODE(n)) { + /* node->owner != NULL implies !RB_EMPTY_NODE(n), no need to separately + * check the latter + */ + if (cmpxchg(&node->owner, NULL, BPF_PTR_POISON)) { /* Only called from BPF prog, no need to migrate_disable */ __bpf_obj_drop_impl((void *)n - off, rec); return -EINVAL; @@ -2052,6 +2074,7 @@ static int __bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, rb_link_node(n, parent, link); rb_insert_color_cached(n, (struct rb_root_cached *)root, leftmost); + WRITE_ONCE(node->owner, root); return 0; } @@ -2060,8 +2083,9 @@ __bpf_kfunc int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node void *meta__ign, u64 off) { struct btf_struct_meta *meta = meta__ign; + struct bpf_rb_node_kern *n = (void *)node; - return __bpf_rbtree_add(root, node, (void *)less, meta ? meta->record : NULL, off); + return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off); } __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) @@ -2239,7 +2263,10 @@ __bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr_kern *ptr, u32 offset case BPF_DYNPTR_TYPE_RINGBUF: return ptr->data + ptr->offset + offset; case BPF_DYNPTR_TYPE_SKB: - return skb_header_pointer(ptr->data, ptr->offset + offset, len, buffer__opt); + if (buffer__opt) + return skb_header_pointer(ptr->data, ptr->offset + offset, len, buffer__opt); + else + return skb_pointer_if_linear(ptr->data, ptr->offset + offset, len); case BPF_DYNPTR_TYPE_XDP: { void *xdp_ptr = bpf_xdp_pointer(ptr->data, ptr->offset + offset, len); diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index b0fa190b0979..6fc9dae9edc8 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -78,8 +78,7 @@ static const struct seq_operations bpf_map_seq_ops = { .show = bpf_map_seq_show, }; -BTF_ID_LIST(btf_bpf_map_id) -BTF_ID(struct, bpf_map) +BTF_ID_LIST_GLOBAL_SINGLE(btf_bpf_map_id, struct, bpf_map) static const struct bpf_iter_seq_info bpf_map_seq_info = { .seq_ops = &bpf_map_seq_ops, @@ -93,7 +92,7 @@ static struct bpf_iter_reg bpf_map_reg_info = { .ctx_arg_info_size = 1, .ctx_arg_info = { { offsetof(struct bpf_iter__bpf_map, map), - PTR_TO_BTF_ID_OR_NULL }, + PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, }, .seq_info = &bpf_map_seq_info, }; @@ -193,3 +192,40 @@ static int __init bpf_map_iter_init(void) } late_initcall(bpf_map_iter_init); + +__diag_push(); +__diag_ignore_all("-Wmissing-prototypes", + "Global functions as their definitions will be in vmlinux BTF"); + +__bpf_kfunc s64 bpf_map_sum_elem_count(const struct bpf_map *map) +{ + s64 *pcount; + s64 ret = 0; + int cpu; + + if (!map || !map->elem_count) + return 0; + + for_each_possible_cpu(cpu) { + pcount = per_cpu_ptr(map->elem_count, cpu); + ret += READ_ONCE(*pcount); + } + return ret; +} + +__diag_pop(); + +BTF_SET8_START(bpf_map_iter_kfunc_ids) +BTF_ID_FLAGS(func, bpf_map_sum_elem_count, KF_TRUSTED_ARGS) +BTF_SET8_END(bpf_map_iter_kfunc_ids) + +static const struct btf_kfunc_id_set bpf_map_iter_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_map_iter_kfunc_ids, +}; + +static int init_subsystem(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_map_iter_kfunc_set); +} +late_initcall(init_subsystem); diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 0668bcd7c926..51d6389e5152 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -98,11 +98,23 @@ struct bpf_mem_cache { int free_cnt; int low_watermark, high_watermark, batch; int percpu_size; + bool draining; + struct bpf_mem_cache *tgt; - struct rcu_head rcu; + /* list of objects to be freed after RCU GP */ struct llist_head free_by_rcu; + struct llist_node *free_by_rcu_tail; struct llist_head waiting_for_gp; + struct llist_node *waiting_for_gp_tail; + struct rcu_head rcu; atomic_t call_rcu_in_progress; + struct llist_head free_llist_extra_rcu; + + /* list of objects to be freed after RCU tasks trace GP */ + struct llist_head free_by_rcu_ttrace; + struct llist_head waiting_for_gp_ttrace; + struct rcu_head rcu_ttrace; + atomic_t call_rcu_ttrace_in_progress; }; struct bpf_mem_caches { @@ -153,59 +165,83 @@ static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c) #endif } +static void inc_active(struct bpf_mem_cache *c, unsigned long *flags) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + /* In RT irq_work runs in per-cpu kthread, so disable + * interrupts to avoid preemption and interrupts and + * reduce the chance of bpf prog executing on this cpu + * when active counter is busy. + */ + local_irq_save(*flags); + /* alloc_bulk runs from irq_work which will not preempt a bpf + * program that does unit_alloc/unit_free since IRQs are + * disabled there. There is no race to increment 'active' + * counter. It protects free_llist from corruption in case NMI + * bpf prog preempted this loop. + */ + WARN_ON_ONCE(local_inc_return(&c->active) != 1); +} + +static void dec_active(struct bpf_mem_cache *c, unsigned long flags) +{ + local_dec(&c->active); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_restore(flags); +} + +static void add_obj_to_free_list(struct bpf_mem_cache *c, void *obj) +{ + unsigned long flags; + + inc_active(c, &flags); + __llist_add(obj, &c->free_llist); + c->free_cnt++; + dec_active(c, flags); +} + /* Mostly runs from irq_work except __init phase. */ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node) { struct mem_cgroup *memcg = NULL, *old_memcg; - unsigned long flags; void *obj; int i; - memcg = get_memcg(c); - old_memcg = set_active_memcg(memcg); for (i = 0; i < cnt; i++) { /* - * free_by_rcu is only manipulated by irq work refill_work(). - * IRQ works on the same CPU are called sequentially, so it is - * safe to use __llist_del_first() here. If alloc_bulk() is - * invoked by the initial prefill, there will be no running - * refill_work(), so __llist_del_first() is fine as well. - * - * In most cases, objects on free_by_rcu are from the same CPU. - * If some objects come from other CPUs, it doesn't incur any - * harm because NUMA_NO_NODE means the preference for current - * numa node and it is not a guarantee. + * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is + * done only by one CPU == current CPU. Other CPUs might + * llist_add() and llist_del_all() in parallel. */ - obj = __llist_del_first(&c->free_by_rcu); - if (!obj) { - /* Allocate, but don't deplete atomic reserves that typical - * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc - * will allocate from the current numa node which is what we - * want here. - */ - obj = __alloc(c, node, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); - if (!obj) - break; - } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - /* In RT irq_work runs in per-cpu kthread, so disable - * interrupts to avoid preemption and interrupts and - * reduce the chance of bpf prog executing on this cpu - * when active counter is busy. - */ - local_irq_save(flags); - /* alloc_bulk runs from irq_work which will not preempt a bpf - * program that does unit_alloc/unit_free since IRQs are - * disabled there. There is no race to increment 'active' - * counter. It protects free_llist from corruption in case NMI - * bpf prog preempted this loop. + obj = llist_del_first(&c->free_by_rcu_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } + if (i >= cnt) + return; + + for (; i < cnt; i++) { + obj = llist_del_first(&c->waiting_for_gp_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } + if (i >= cnt) + return; + + memcg = get_memcg(c); + old_memcg = set_active_memcg(memcg); + for (; i < cnt; i++) { + /* Allocate, but don't deplete atomic reserves that typical + * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc + * will allocate from the current numa node which is what we + * want here. */ - WARN_ON_ONCE(local_inc_return(&c->active) != 1); - __llist_add(obj, &c->free_llist); - c->free_cnt++; - local_dec(&c->active); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - local_irq_restore(flags); + obj = __alloc(c, node, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!obj) + break; + add_obj_to_free_list(c, obj); } set_active_memcg(old_memcg); mem_cgroup_put(memcg); @@ -222,20 +258,24 @@ static void free_one(void *obj, bool percpu) kfree(obj); } -static void free_all(struct llist_node *llnode, bool percpu) +static int free_all(struct llist_node *llnode, bool percpu) { struct llist_node *pos, *t; + int cnt = 0; - llist_for_each_safe(pos, t, llnode) + llist_for_each_safe(pos, t, llnode) { free_one(pos, percpu); + cnt++; + } + return cnt; } static void __free_rcu(struct rcu_head *head) { - struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu); + struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); - free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size); - atomic_set(&c->call_rcu_in_progress, 0); + free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); + atomic_set(&c->call_rcu_ttrace_in_progress, 0); } static void __free_rcu_tasks_trace(struct rcu_head *head) @@ -254,60 +294,128 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj) struct llist_node *llnode = obj; /* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work. - * Nothing races to add to free_by_rcu list. + * Nothing races to add to free_by_rcu_ttrace list. */ - __llist_add(llnode, &c->free_by_rcu); + llist_add(llnode, &c->free_by_rcu_ttrace); } -static void do_call_rcu(struct bpf_mem_cache *c) +static void do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; - if (atomic_xchg(&c->call_rcu_in_progress, 1)) + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { + if (unlikely(READ_ONCE(c->draining))) { + llnode = llist_del_all(&c->free_by_rcu_ttrace); + free_all(llnode, !!c->percpu_size); + } return; + } + + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); + llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) + llist_add(llnode, &c->waiting_for_gp_ttrace); + + if (unlikely(READ_ONCE(c->draining))) { + __free_rcu(&c->rcu_ttrace); + return; + } - WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); - llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu)) - /* There is no concurrent __llist_add(waiting_for_gp) access. - * It doesn't race with llist_del_all either. - * But there could be two concurrent llist_del_all(waiting_for_gp): - * from __free_rcu() and from drain_mem_cache(). - */ - __llist_add(llnode, &c->waiting_for_gp); /* Use call_rcu_tasks_trace() to wait for sleepable progs to finish. * If RCU Tasks Trace grace period implies RCU grace period, free * these elements directly, else use call_rcu() to wait for normal * progs to finish and finally do free_one() on each element. */ - call_rcu_tasks_trace(&c->rcu, __free_rcu_tasks_trace); + call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace); } static void free_bulk(struct bpf_mem_cache *c) { + struct bpf_mem_cache *tgt = c->tgt; struct llist_node *llnode, *t; unsigned long flags; int cnt; + WARN_ON_ONCE(tgt->unit_size != c->unit_size); + do { - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - local_irq_save(flags); - WARN_ON_ONCE(local_inc_return(&c->active) != 1); + inc_active(c, &flags); llnode = __llist_del_first(&c->free_llist); if (llnode) cnt = --c->free_cnt; else cnt = 0; - local_dec(&c->active); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - local_irq_restore(flags); + dec_active(c, flags); if (llnode) - enque_to_free(c, llnode); + enque_to_free(tgt, llnode); } while (cnt > (c->high_watermark + c->low_watermark) / 2); /* and drain free_llist_extra */ llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) - enque_to_free(c, llnode); - do_call_rcu(c); + enque_to_free(tgt, llnode); + do_call_rcu_ttrace(tgt); +} + +static void __free_by_rcu(struct rcu_head *head) +{ + struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu); + struct bpf_mem_cache *tgt = c->tgt; + struct llist_node *llnode; + + llnode = llist_del_all(&c->waiting_for_gp); + if (!llnode) + goto out; + + llist_add_batch(llnode, c->waiting_for_gp_tail, &tgt->free_by_rcu_ttrace); + + /* Objects went through regular RCU GP. Send them to RCU tasks trace */ + do_call_rcu_ttrace(tgt); +out: + atomic_set(&c->call_rcu_in_progress, 0); +} + +static void check_free_by_rcu(struct bpf_mem_cache *c) +{ + struct llist_node *llnode, *t; + unsigned long flags; + + /* drain free_llist_extra_rcu */ + if (unlikely(!llist_empty(&c->free_llist_extra_rcu))) { + inc_active(c, &flags); + llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra_rcu)) + if (__llist_add(llnode, &c->free_by_rcu)) + c->free_by_rcu_tail = llnode; + dec_active(c, flags); + } + + if (llist_empty(&c->free_by_rcu)) + return; + + if (atomic_xchg(&c->call_rcu_in_progress, 1)) { + /* + * Instead of kmalloc-ing new rcu_head and triggering 10k + * call_rcu() to hit rcutree.qhimark and force RCU to notice + * the overload just ask RCU to hurry up. There could be many + * objects in free_by_rcu list. + * This hint reduces memory consumption for an artificial + * benchmark from 2 Gbyte to 150 Mbyte. + */ + rcu_request_urgent_qs_task(current); + return; + } + + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); + + inc_active(c, &flags); + WRITE_ONCE(c->waiting_for_gp.first, __llist_del_all(&c->free_by_rcu)); + c->waiting_for_gp_tail = c->free_by_rcu_tail; + dec_active(c, flags); + + if (unlikely(READ_ONCE(c->draining))) { + free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size); + atomic_set(&c->call_rcu_in_progress, 0); + } else { + call_rcu_hurry(&c->rcu, __free_by_rcu); + } } static void bpf_mem_refill(struct irq_work *work) @@ -324,6 +432,8 @@ static void bpf_mem_refill(struct irq_work *work) alloc_bulk(c, c->batch, NUMA_NO_NODE); else if (cnt > c->high_watermark) free_bulk(c); + + check_free_by_rcu(c); } static void notrace irq_work_raise(struct bpf_mem_cache *c) @@ -406,6 +516,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->unit_size = unit_size; c->objcg = objcg; c->percpu_size = percpu_size; + c->tgt = c; prefill_mem_cache(c, cpu); } ma->cache = pc; @@ -428,6 +539,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c = &cc->cache[i]; c->unit_size = sizes[i]; c->objcg = objcg; + c->tgt = c; prefill_mem_cache(c, cpu); } } @@ -441,19 +553,57 @@ static void drain_mem_cache(struct bpf_mem_cache *c) /* No progs are using this bpf_mem_cache, but htab_map_free() called * bpf_mem_cache_free() for all remaining elements and they can be in - * free_by_rcu or in waiting_for_gp lists, so drain those lists now. + * free_by_rcu_ttrace or in waiting_for_gp_ttrace lists, so drain those lists now. * - * Except for waiting_for_gp list, there are no concurrent operations + * Except for waiting_for_gp_ttrace list, there are no concurrent operations * on these lists, so it is safe to use __llist_del_all(). */ - free_all(__llist_del_all(&c->free_by_rcu), percpu); - free_all(llist_del_all(&c->waiting_for_gp), percpu); + free_all(llist_del_all(&c->free_by_rcu_ttrace), percpu); + free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu); free_all(__llist_del_all(&c->free_llist), percpu); free_all(__llist_del_all(&c->free_llist_extra), percpu); + free_all(__llist_del_all(&c->free_by_rcu), percpu); + free_all(__llist_del_all(&c->free_llist_extra_rcu), percpu); + free_all(llist_del_all(&c->waiting_for_gp), percpu); +} + +static void check_mem_cache(struct bpf_mem_cache *c) +{ + WARN_ON_ONCE(!llist_empty(&c->free_by_rcu_ttrace)); + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); + WARN_ON_ONCE(!llist_empty(&c->free_llist)); + WARN_ON_ONCE(!llist_empty(&c->free_llist_extra)); + WARN_ON_ONCE(!llist_empty(&c->free_by_rcu)); + WARN_ON_ONCE(!llist_empty(&c->free_llist_extra_rcu)); + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); +} + +static void check_leaked_objs(struct bpf_mem_alloc *ma) +{ + struct bpf_mem_caches *cc; + struct bpf_mem_cache *c; + int cpu, i; + + if (ma->cache) { + for_each_possible_cpu(cpu) { + c = per_cpu_ptr(ma->cache, cpu); + check_mem_cache(c); + } + } + if (ma->caches) { + for_each_possible_cpu(cpu) { + cc = per_cpu_ptr(ma->caches, cpu); + for (i = 0; i < NUM_CACHES; i++) { + c = &cc->cache[i]; + check_mem_cache(c); + } + } + } } static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma) { + check_leaked_objs(ma); free_percpu(ma->cache); free_percpu(ma->caches); ma->cache = NULL; @@ -462,8 +612,8 @@ static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma) static void free_mem_alloc(struct bpf_mem_alloc *ma) { - /* waiting_for_gp lists was drained, but __free_rcu might - * still execute. Wait for it now before we freeing percpu caches. + /* waiting_for_gp[_ttrace] lists were drained, but RCU callbacks + * might still execute. Wait for them. * * rcu_barrier_tasks_trace() doesn't imply synchronize_rcu_tasks_trace(), * but rcu_barrier_tasks_trace() and rcu_barrier() below are only used @@ -472,7 +622,8 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma) * rcu_trace_implies_rcu_gp(), it will be OK to skip rcu_barrier() by * using rcu_trace_implies_rcu_gp() as well. */ - rcu_barrier_tasks_trace(); + rcu_barrier(); /* wait for __free_by_rcu */ + rcu_barrier_tasks_trace(); /* wait for __free_rcu */ if (!rcu_trace_implies_rcu_gp()) rcu_barrier(); free_mem_alloc_no_barrier(ma); @@ -498,7 +649,7 @@ static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress) return; } - copy = kmalloc(sizeof(*ma), GFP_KERNEL); + copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL); if (!copy) { /* Slow path with inline barrier-s */ free_mem_alloc(ma); @@ -506,10 +657,7 @@ static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress) } /* Defer barriers into worker to let the rest of map memory to be freed */ - copy->cache = ma->cache; - ma->cache = NULL; - copy->caches = ma->caches; - ma->caches = NULL; + memset(ma, 0, sizeof(*ma)); INIT_WORK(©->work, free_mem_alloc_deferred); queue_work(system_unbound_wq, ©->work); } @@ -524,17 +672,10 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress = 0; for_each_possible_cpu(cpu) { c = per_cpu_ptr(ma->cache, cpu); - /* - * refill_work may be unfinished for PREEMPT_RT kernel - * in which irq work is invoked in a per-CPU RT thread. - * It is also possible for kernel with - * arch_irq_work_has_interrupt() being false and irq - * work is invoked in timer interrupt. So waiting for - * the completion of irq work to ease the handling of - * concurrency. - */ + WRITE_ONCE(c->draining, true); irq_work_sync(&c->refill_work); drain_mem_cache(c); + rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } /* objcg is the same across cpus */ @@ -548,8 +689,10 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) cc = per_cpu_ptr(ma->caches, cpu); for (i = 0; i < NUM_CACHES; i++) { c = &cc->cache[i]; + WRITE_ONCE(c->draining, true); irq_work_sync(&c->refill_work); drain_mem_cache(c); + rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } } @@ -581,8 +724,10 @@ static void notrace *unit_alloc(struct bpf_mem_cache *c) local_irq_save(flags); if (local_inc_return(&c->active) == 1) { llnode = __llist_del_first(&c->free_llist); - if (llnode) + if (llnode) { cnt = --c->free_cnt; + *(struct bpf_mem_cache **)llnode = c; + } } local_dec(&c->active); local_irq_restore(flags); @@ -606,6 +751,12 @@ static void notrace unit_free(struct bpf_mem_cache *c, void *ptr) BUILD_BUG_ON(LLIST_NODE_SZ > 8); + /* + * Remember bpf_mem_cache that allocated this object. + * The hint is not accurate. + */ + c->tgt = *(struct bpf_mem_cache **)llnode; + local_irq_save(flags); if (local_inc_return(&c->active) == 1) { __llist_add(llnode, &c->free_llist); @@ -627,6 +778,27 @@ static void notrace unit_free(struct bpf_mem_cache *c, void *ptr) irq_work_raise(c); } +static void notrace unit_free_rcu(struct bpf_mem_cache *c, void *ptr) +{ + struct llist_node *llnode = ptr - LLIST_NODE_SZ; + unsigned long flags; + + c->tgt = *(struct bpf_mem_cache **)llnode; + + local_irq_save(flags); + if (local_inc_return(&c->active) == 1) { + if (__llist_add(llnode, &c->free_by_rcu)) + c->free_by_rcu_tail = llnode; + } else { + llist_add(llnode, &c->free_llist_extra_rcu); + } + local_dec(&c->active); + local_irq_restore(flags); + + if (!atomic_read(&c->call_rcu_in_progress)) + irq_work_raise(c); +} + /* Called from BPF program or from sys_bpf syscall. * In both cases migration is disabled. */ @@ -660,6 +832,20 @@ void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr) unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr); } +void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr) +{ + int idx; + + if (!ptr) + return; + + idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ)); + if (idx < 0) + return; + + unit_free_rcu(this_cpu_ptr(ma->caches)->cache + idx, ptr); +} + void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma) { void *ret; @@ -676,6 +862,14 @@ void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr) unit_free(this_cpu_ptr(ma->cache), ptr); } +void notrace bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr) +{ + if (!ptr) + return; + + unit_free_rcu(this_cpu_ptr(ma->cache), ptr); +} + /* Directly does a kfree() without putting 'ptr' back to the free_llist * for reuse and without waiting for a rcu_tasks_trace gp. * The caller must first go through the rcu_tasks_trace gp for 'ptr' diff --git a/kernel/bpf/mprog.c b/kernel/bpf/mprog.c new file mode 100644 index 000000000000..f7816d2bc3e4 --- /dev/null +++ b/kernel/bpf/mprog.c @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include <linux/bpf.h> +#include <linux/bpf_mprog.h> + +static int bpf_mprog_link(struct bpf_tuple *tuple, + u32 id_or_fd, u32 flags, + enum bpf_prog_type type) +{ + struct bpf_link *link = ERR_PTR(-EINVAL); + bool id = flags & BPF_F_ID; + + if (id) + link = bpf_link_by_id(id_or_fd); + else if (id_or_fd) + link = bpf_link_get_from_fd(id_or_fd); + if (IS_ERR(link)) + return PTR_ERR(link); + if (type && link->prog->type != type) { + bpf_link_put(link); + return -EINVAL; + } + + tuple->link = link; + tuple->prog = link->prog; + return 0; +} + +static int bpf_mprog_prog(struct bpf_tuple *tuple, + u32 id_or_fd, u32 flags, + enum bpf_prog_type type) +{ + struct bpf_prog *prog = ERR_PTR(-EINVAL); + bool id = flags & BPF_F_ID; + + if (id) + prog = bpf_prog_by_id(id_or_fd); + else if (id_or_fd) + prog = bpf_prog_get(id_or_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + if (type && prog->type != type) { + bpf_prog_put(prog); + return -EINVAL; + } + + tuple->link = NULL; + tuple->prog = prog; + return 0; +} + +static int bpf_mprog_tuple_relative(struct bpf_tuple *tuple, + u32 id_or_fd, u32 flags, + enum bpf_prog_type type) +{ + bool link = flags & BPF_F_LINK; + bool id = flags & BPF_F_ID; + + memset(tuple, 0, sizeof(*tuple)); + if (link) + return bpf_mprog_link(tuple, id_or_fd, flags, type); + /* If no relevant flag is set and no id_or_fd was passed, then + * tuple link/prog is just NULLed. This is the case when before/ + * after selects first/last position without passing fd. + */ + if (!id && !id_or_fd) + return 0; + return bpf_mprog_prog(tuple, id_or_fd, flags, type); +} + +static void bpf_mprog_tuple_put(struct bpf_tuple *tuple) +{ + if (tuple->link) + bpf_link_put(tuple->link); + else if (tuple->prog) + bpf_prog_put(tuple->prog); +} + +/* The bpf_mprog_{replace,delete}() operate on exact idx position with the + * one exception that for deletion we support delete from front/back. In + * case of front idx is -1, in case of back idx is bpf_mprog_total(entry). + * Adjustment to first and last entry is trivial. The bpf_mprog_insert() + * we have to deal with the following cases: + * + * idx + before: + * + * Insert P4 before P3: idx for old array is 1, idx for new array is 2, + * hence we adjust target idx for the new array, so that memmove copies + * P1 and P2 to the new entry, and we insert P4 into idx 2. Inserting + * before P1 would have old idx -1 and new idx 0. + * + * +--+--+--+ +--+--+--+--+ +--+--+--+--+ + * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| + * +--+--+--+ +--+--+--+--+ +--+--+--+--+ + * + * idx + after: + * + * Insert P4 after P2: idx for old array is 2, idx for new array is 2. + * Again, memmove copies P1 and P2 to the new entry, and we insert P4 + * into idx 2. Inserting after P3 would have both old/new idx at 4 aka + * bpf_mprog_total(entry). + * + * +--+--+--+ +--+--+--+--+ +--+--+--+--+ + * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| + * +--+--+--+ +--+--+--+--+ +--+--+--+--+ + */ +static int bpf_mprog_replace(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_tuple *ntuple, int idx) +{ + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + struct bpf_prog *oprog; + + bpf_mprog_read(entry, idx, &fp, &cp); + oprog = READ_ONCE(fp->prog); + bpf_mprog_write(fp, cp, ntuple); + if (!ntuple->link) { + WARN_ON_ONCE(cp->link); + bpf_prog_put(oprog); + } + *entry_new = entry; + return 0; +} + +static int bpf_mprog_insert(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_tuple *ntuple, int idx, u32 flags) +{ + int total = bpf_mprog_total(entry); + struct bpf_mprog_entry *peer; + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + + peer = bpf_mprog_peer(entry); + bpf_mprog_entry_copy(peer, entry); + if (idx == total) + goto insert; + else if (flags & BPF_F_BEFORE) + idx += 1; + bpf_mprog_entry_grow(peer, idx); +insert: + bpf_mprog_read(peer, idx, &fp, &cp); + bpf_mprog_write(fp, cp, ntuple); + bpf_mprog_inc(peer); + *entry_new = peer; + return 0; +} + +static int bpf_mprog_delete(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_tuple *dtuple, int idx) +{ + int total = bpf_mprog_total(entry); + struct bpf_mprog_entry *peer; + + peer = bpf_mprog_peer(entry); + bpf_mprog_entry_copy(peer, entry); + if (idx == -1) + idx = 0; + else if (idx == total) + idx = total - 1; + bpf_mprog_entry_shrink(peer, idx); + bpf_mprog_dec(peer); + bpf_mprog_mark_for_release(peer, dtuple); + *entry_new = peer; + return 0; +} + +/* In bpf_mprog_pos_*() we evaluate the target position for the BPF + * program/link that needs to be replaced, inserted or deleted for + * each "rule" independently. If all rules agree on that position + * or existing element, then enact replacement, addition or deletion. + * If this is not the case, then the request cannot be satisfied and + * we bail out with an error. + */ +static int bpf_mprog_pos_exact(struct bpf_mprog_entry *entry, + struct bpf_tuple *tuple) +{ + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + int i; + + for (i = 0; i < bpf_mprog_total(entry); i++) { + bpf_mprog_read(entry, i, &fp, &cp); + if (tuple->prog == READ_ONCE(fp->prog)) + return tuple->link == cp->link ? i : -EBUSY; + } + return -ENOENT; +} + +static int bpf_mprog_pos_before(struct bpf_mprog_entry *entry, + struct bpf_tuple *tuple) +{ + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + int i; + + for (i = 0; i < bpf_mprog_total(entry); i++) { + bpf_mprog_read(entry, i, &fp, &cp); + if (tuple->prog == READ_ONCE(fp->prog) && + (!tuple->link || tuple->link == cp->link)) + return i - 1; + } + return tuple->prog ? -ENOENT : -1; +} + +static int bpf_mprog_pos_after(struct bpf_mprog_entry *entry, + struct bpf_tuple *tuple) +{ + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + int i; + + for (i = 0; i < bpf_mprog_total(entry); i++) { + bpf_mprog_read(entry, i, &fp, &cp); + if (tuple->prog == READ_ONCE(fp->prog) && + (!tuple->link || tuple->link == cp->link)) + return i + 1; + } + return tuple->prog ? -ENOENT : bpf_mprog_total(entry); +} + +int bpf_mprog_attach(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_prog *prog_new, struct bpf_link *link, + struct bpf_prog *prog_old, + u32 flags, u32 id_or_fd, u64 revision) +{ + struct bpf_tuple rtuple, ntuple = { + .prog = prog_new, + .link = link, + }, otuple = { + .prog = prog_old, + .link = link, + }; + int ret, idx = -ERANGE, tidx; + + if (revision && revision != bpf_mprog_revision(entry)) + return -ESTALE; + if (bpf_mprog_exists(entry, prog_new)) + return -EEXIST; + ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, + flags & ~BPF_F_REPLACE, + prog_new->type); + if (ret) + return ret; + if (flags & BPF_F_REPLACE) { + tidx = bpf_mprog_pos_exact(entry, &otuple); + if (tidx < 0) { + ret = tidx; + goto out; + } + idx = tidx; + } + if (flags & BPF_F_BEFORE) { + tidx = bpf_mprog_pos_before(entry, &rtuple); + if (tidx < -1 || (idx >= -1 && tidx != idx)) { + ret = tidx < -1 ? tidx : -ERANGE; + goto out; + } + idx = tidx; + } + if (flags & BPF_F_AFTER) { + tidx = bpf_mprog_pos_after(entry, &rtuple); + if (tidx < -1 || (idx >= -1 && tidx != idx)) { + ret = tidx < 0 ? tidx : -ERANGE; + goto out; + } + idx = tidx; + } + if (idx < -1) { + if (rtuple.prog || flags) { + ret = -EINVAL; + goto out; + } + idx = bpf_mprog_total(entry); + flags = BPF_F_AFTER; + } + if (idx >= bpf_mprog_max()) { + ret = -ERANGE; + goto out; + } + if (flags & BPF_F_REPLACE) + ret = bpf_mprog_replace(entry, entry_new, &ntuple, idx); + else + ret = bpf_mprog_insert(entry, entry_new, &ntuple, idx, flags); +out: + bpf_mprog_tuple_put(&rtuple); + return ret; +} + +static int bpf_mprog_fetch(struct bpf_mprog_entry *entry, + struct bpf_tuple *tuple, int idx) +{ + int total = bpf_mprog_total(entry); + struct bpf_mprog_cp *cp; + struct bpf_mprog_fp *fp; + struct bpf_prog *prog; + struct bpf_link *link; + + if (idx == -1) + idx = 0; + else if (idx == total) + idx = total - 1; + bpf_mprog_read(entry, idx, &fp, &cp); + prog = READ_ONCE(fp->prog); + link = cp->link; + /* The deletion request can either be without filled tuple in which + * case it gets populated here based on idx, or with filled tuple + * where the only thing we end up doing is the WARN_ON_ONCE() assert. + * If we hit a BPF link at the given index, it must not be removed + * from opts path. + */ + if (link && !tuple->link) + return -EBUSY; + WARN_ON_ONCE(tuple->prog && tuple->prog != prog); + WARN_ON_ONCE(tuple->link && tuple->link != link); + tuple->prog = prog; + tuple->link = link; + return 0; +} + +int bpf_mprog_detach(struct bpf_mprog_entry *entry, + struct bpf_mprog_entry **entry_new, + struct bpf_prog *prog, struct bpf_link *link, + u32 flags, u32 id_or_fd, u64 revision) +{ + struct bpf_tuple rtuple, dtuple = { + .prog = prog, + .link = link, + }; + int ret, idx = -ERANGE, tidx; + + if (flags & BPF_F_REPLACE) + return -EINVAL; + if (revision && revision != bpf_mprog_revision(entry)) + return -ESTALE; + ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, flags, + prog ? prog->type : + BPF_PROG_TYPE_UNSPEC); + if (ret) + return ret; + if (dtuple.prog) { + tidx = bpf_mprog_pos_exact(entry, &dtuple); + if (tidx < 0) { + ret = tidx; + goto out; + } + idx = tidx; + } + if (flags & BPF_F_BEFORE) { + tidx = bpf_mprog_pos_before(entry, &rtuple); + if (tidx < -1 || (idx >= -1 && tidx != idx)) { + ret = tidx < -1 ? tidx : -ERANGE; + goto out; + } + idx = tidx; + } + if (flags & BPF_F_AFTER) { + tidx = bpf_mprog_pos_after(entry, &rtuple); + if (tidx < -1 || (idx >= -1 && tidx != idx)) { + ret = tidx < 0 ? tidx : -ERANGE; + goto out; + } + idx = tidx; + } + if (idx < -1) { + if (rtuple.prog || flags) { + ret = -EINVAL; + goto out; + } + idx = bpf_mprog_total(entry); + flags = BPF_F_AFTER; + } + if (idx >= bpf_mprog_max()) { + ret = -ERANGE; + goto out; + } + ret = bpf_mprog_fetch(entry, &dtuple, idx); + if (ret) + goto out; + ret = bpf_mprog_delete(entry, entry_new, &dtuple, idx); +out: + bpf_mprog_tuple_put(&rtuple); + return ret; +} + +int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr, + struct bpf_mprog_entry *entry) +{ + u32 __user *uprog_flags, *ulink_flags; + u32 __user *uprog_id, *ulink_id; + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + struct bpf_prog *prog; + const u32 flags = 0; + int i, ret = 0; + u32 id, count; + u64 revision; + + if (attr->query.query_flags || attr->query.attach_flags) + return -EINVAL; + revision = bpf_mprog_revision(entry); + count = bpf_mprog_total(entry); + if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) + return -EFAULT; + if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) + return -EFAULT; + if (copy_to_user(&uattr->query.count, &count, sizeof(count))) + return -EFAULT; + uprog_id = u64_to_user_ptr(attr->query.prog_ids); + uprog_flags = u64_to_user_ptr(attr->query.prog_attach_flags); + ulink_id = u64_to_user_ptr(attr->query.link_ids); + ulink_flags = u64_to_user_ptr(attr->query.link_attach_flags); + if (attr->query.count == 0 || !uprog_id || !count) + return 0; + if (attr->query.count < count) { + count = attr->query.count; + ret = -ENOSPC; + } + for (i = 0; i < bpf_mprog_max(); i++) { + bpf_mprog_read(entry, i, &fp, &cp); + prog = READ_ONCE(fp->prog); + if (!prog) + break; + id = prog->aux->id; + if (copy_to_user(uprog_id + i, &id, sizeof(id))) + return -EFAULT; + if (uprog_flags && + copy_to_user(uprog_flags + i, &flags, sizeof(flags))) + return -EFAULT; + id = cp->link ? cp->link->id : 0; + if (ulink_id && + copy_to_user(ulink_id + i, &id, sizeof(id))) + return -EFAULT; + if (ulink_flags && + copy_to_user(ulink_flags + i, &flags, sizeof(flags))) + return -EFAULT; + if (i + 1 == count) + break; + } + return ret; +} diff --git a/kernel/bpf/preload/iterators/Makefile b/kernel/bpf/preload/iterators/Makefile index 8937dc6bc8d0..b83c2f5e9be1 100644 --- a/kernel/bpf/preload/iterators/Makefile +++ b/kernel/bpf/preload/iterators/Makefile @@ -50,7 +50,7 @@ iterators.lskel-%.h: $(OUTPUT)/%/iterators.bpf.o | $(BPFTOOL) $(OUTPUT)/%/iterators.bpf.o: iterators.bpf.c $(BPFOBJ) | $(OUTPUT) $(call msg,BPF,$@) $(Q)mkdir -p $(@D) - $(Q)$(CLANG) -g -O2 -target bpf -m$* $(INCLUDES) \ + $(Q)$(CLANG) -g -O2 --target=bpf -m$* $(INCLUDES) \ -c $(filter %.c,$^) -o $@ && \ $(LLVM_STRIP) -g $@ diff --git a/kernel/bpf/preload/iterators/iterators.bpf.c b/kernel/bpf/preload/iterators/iterators.bpf.c index 03af863314ea..b78968b63fab 100644 --- a/kernel/bpf/preload/iterators/iterators.bpf.c +++ b/kernel/bpf/preload/iterators/iterators.bpf.c @@ -73,6 +73,8 @@ static const char *get_name(struct btf *btf, long btf_id, const char *fallback) return str + name_off; } +__s64 bpf_map_sum_elem_count(struct bpf_map *map) __ksym; + SEC("iter/bpf_map") int dump_bpf_map(struct bpf_iter__bpf_map *ctx) { @@ -84,9 +86,12 @@ int dump_bpf_map(struct bpf_iter__bpf_map *ctx) return 0; if (seq_num == 0) - BPF_SEQ_PRINTF(seq, " id name max_entries\n"); + BPF_SEQ_PRINTF(seq, " id name max_entries cur_entries\n"); + + BPF_SEQ_PRINTF(seq, "%4u %-16s %10d %10lld\n", + map->id, map->name, map->max_entries, + bpf_map_sum_elem_count(map)); - BPF_SEQ_PRINTF(seq, "%4u %-16s%6d\n", map->id, map->name, map->max_entries); return 0; } diff --git a/kernel/bpf/preload/iterators/iterators.lskel-little-endian.h b/kernel/bpf/preload/iterators/iterators.lskel-little-endian.h index 70f236a82fe1..5b98ab02025e 100644 --- a/kernel/bpf/preload/iterators/iterators.lskel-little-endian.h +++ b/kernel/bpf/preload/iterators/iterators.lskel-little-endian.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* THIS FILE IS AUTOGENERATED! */ +/* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ #ifndef __ITERATORS_BPF_SKEL_H__ #define __ITERATORS_BPF_SKEL_H__ @@ -18,8 +18,6 @@ struct iterators_bpf { int dump_bpf_map_fd; int dump_bpf_prog_fd; } links; - struct iterators_bpf__rodata { - } *rodata; }; static inline int @@ -68,7 +66,6 @@ iterators_bpf__destroy(struct iterators_bpf *skel) iterators_bpf__detach(skel); skel_closenz(skel->progs.dump_bpf_map.prog_fd); skel_closenz(skel->progs.dump_bpf_prog.prog_fd); - skel_free_map_data(skel->rodata, skel->maps.rodata.initial_value, 4096); skel_closenz(skel->maps.rodata.map_fd); skel_free(skel); } @@ -81,15 +78,6 @@ iterators_bpf__open(void) if (!skel) goto cleanup; skel->ctx.sz = (void *)&skel->links - (void *)skel; - skel->rodata = skel_prep_map_data((void *)"\ -\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ -\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x0a\0\x25\x34\x75\x20\ -\x25\x2d\x31\x36\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\ -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\ -\x64\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0", 4096, 98); - if (!skel->rodata) - goto cleanup; - skel->maps.rodata.initial_value = (__u64) (long) skel->rodata; return skel; cleanup: iterators_bpf__destroy(skel); @@ -103,7 +91,7 @@ iterators_bpf__load(struct iterators_bpf *skel) int err; opts.ctx = (struct bpf_loader_ctx *)skel; - opts.data_sz = 6056; + opts.data_sz = 6208; opts.data = (void *)"\ \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ @@ -138,190 +126,197 @@ iterators_bpf__load(struct iterators_bpf *skel) \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9f\xeb\x01\0\ -\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\xf9\x04\0\0\0\0\0\0\0\0\0\x02\x02\0\ +\x18\0\0\0\0\0\0\0\x80\x04\0\0\x80\x04\0\0\x31\x05\0\0\0\0\0\0\0\0\0\x02\x02\0\ \0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\0\0\0\x04\ \0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\0\0\0\0\0\ \0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\0\0\0\x20\ -\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xa3\0\0\0\x03\0\0\x04\x18\0\0\0\xb1\0\ -\0\0\x09\0\0\0\0\0\0\0\xb5\0\0\0\x0b\0\0\0\x40\0\0\0\xc0\0\0\0\x0b\0\0\0\x80\0\ -\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xc8\0\0\0\0\0\0\x07\0\0\0\0\xd1\0\0\0\0\0\0\ -\x08\x0c\0\0\0\xd7\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\x94\x01\0\0\x03\0\0\x04\ -\x18\0\0\0\x9c\x01\0\0\x0e\0\0\0\0\0\0\0\x9f\x01\0\0\x11\0\0\0\x20\0\0\0\xa4\ -\x01\0\0\x0e\0\0\0\xa0\0\0\0\xb0\x01\0\0\0\0\0\x08\x0f\0\0\0\xb6\x01\0\0\0\0\0\ -\x01\x04\0\0\0\x20\0\0\0\xc3\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\0\0\0\ -\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xc8\x01\0\0\0\0\0\x01\x04\0\0\0\ -\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x2c\x02\0\0\x02\0\0\x04\x10\0\0\0\x13\0\ -\0\0\x03\0\0\0\0\0\0\0\x3f\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x18\0\ -\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x44\x02\0\0\x01\0\0\x0c\ -\x16\0\0\0\x90\x02\0\0\x01\0\0\x04\x08\0\0\0\x99\x02\0\0\x19\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\x02\x1a\0\0\0\xea\x02\0\0\x06\0\0\x04\x38\0\0\0\x9c\x01\0\0\x0e\0\0\ -\0\0\0\0\0\x9f\x01\0\0\x11\0\0\0\x20\0\0\0\xf7\x02\0\0\x1b\0\0\0\xc0\0\0\0\x08\ -\x03\0\0\x15\0\0\0\0\x01\0\0\x11\x03\0\0\x1d\0\0\0\x40\x01\0\0\x1b\x03\0\0\x1e\ -\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\0\0\0\0\ -\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x65\x03\0\0\x02\0\0\x04\ -\x08\0\0\0\x73\x03\0\0\x0e\0\0\0\0\0\0\0\x7c\x03\0\0\x0e\0\0\0\x20\0\0\0\x1b\ -\x03\0\0\x03\0\0\x04\x18\0\0\0\x86\x03\0\0\x1b\0\0\0\0\0\0\0\x8e\x03\0\0\x21\0\ -\0\0\x40\0\0\0\x94\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\0\0\0\ -\0\0\0\0\0\x02\x24\0\0\0\x98\x03\0\0\x01\0\0\x04\x04\0\0\0\xa3\x03\0\0\x0e\0\0\ -\0\0\0\0\0\x0c\x04\0\0\x01\0\0\x04\x04\0\0\0\x15\x04\0\0\x0e\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x8b\x04\0\0\0\0\0\x0e\x25\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\x9f\x04\ -\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\ -\x20\0\0\0\xb5\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\ -\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xca\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xe1\x04\0\0\0\0\0\x0e\x2d\0\0\ -\0\x01\0\0\0\xe9\x04\0\0\x04\0\0\x0f\x62\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\0\x28\ -\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\0\0\ -\x11\0\0\0\xf1\x04\0\0\x01\0\0\x0f\x04\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\x62\ -\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\x74\ -\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\x70\ -\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\x30\ -\x3a\x30\0\x2f\x77\x2f\x6e\x65\x74\x2d\x6e\x65\x78\x74\x2f\x6b\x65\x72\x6e\x65\ -\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\x61\ -\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\ -\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\x2a\ -\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\ -\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\x71\0\ -\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\x73\ -\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\x6e\ -\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\x67\0\x30\x3a\x31\0\x09\x73\x74\ -\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\x61\x70\x20\x3d\ -\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\x21\x6d\x61\x70\ -\x29\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x20\x63\ -\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x5f\x6e\x75\x6d\x3b\0\x30\ -\x3a\x32\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x3d\x20\x30\ -\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\ -\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\ -\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\ -\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\ -\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\ -\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\ -\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\ -\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\ -\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\x2c\x20\x6d\x61\x70\ -\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\x65\x2c\x20\x6d\x61\ -\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x29\x3b\0\x7d\0\x62\ -\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x70\x72\ -\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x69\x74\x65\ -\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\ -\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\x20\x3d\x20\x63\x74\x78\ -\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\x70\x72\x6f\x67\x29\0\ -\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\x75\x78\x20\x3d\x20\ -\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\ -\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\ -\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\ -\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x70\x72\x6f\x67\ -\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\x63\x5f\x6e\x61\x6d\ -\x65\0\x64\x73\x74\x5f\x70\x72\x6f\x67\0\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\ -\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\ -\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\ -\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\ -\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\x74\x66\x29\0\x62\x70\x66\x5f\x66\ -\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\ -\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\x6e\x67\x73\0\x74\x79\x70\x65\x73\ -\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\x64\x65\x72\0\x73\x74\x72\x5f\x6c\ -\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\ -\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\ -\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\ -\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\ -\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\ -\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\ -\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\ -\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\ -\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\ -\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\ -\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\ -\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ -\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ -\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\ -\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\ -\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\ -\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2d\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\0\0\ -\0\x04\0\0\0\x62\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\0\0\0\0\x69\x74\x65\x72\ -\x61\x74\x6f\x72\x2e\x72\x6f\x64\x61\x74\x61\0\0\0\0\0\0\0\0\0\0\0\0\0\x2f\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\ -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\ -\x73\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x0a\0\x20\x20\x69\ -\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ -\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\ -\x25\x73\x20\x25\x73\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\ -\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\x1b\0\0\ -\0\0\0\x79\x11\0\0\0\0\0\0\x79\x11\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\ -\0\0\0\0\0\0\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x62\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\x23\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\ -\0\x61\x71\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\xb7\x01\0\0\x04\0\0\0\xbf\x72\0\ -\0\0\0\0\0\x0f\x12\0\0\0\0\0\0\x7b\x2a\xf0\xff\0\0\0\0\x61\x71\x14\0\0\0\0\0\ -\x7b\x1a\xf8\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xe8\xff\xff\xff\xbf\ -\x61\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x23\0\0\0\xb7\x03\0\0\x0e\0\0\0\ -\xb7\x05\0\0\x18\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\ -\0\0\0\0\x07\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x3c\x01\0\x01\0\0\0\x42\0\0\ -\0\x7b\0\0\0\x24\x3c\x01\0\x02\0\0\0\x42\0\0\0\xee\0\0\0\x1d\x44\x01\0\x03\0\0\ -\0\x42\0\0\0\x0f\x01\0\0\x06\x4c\x01\0\x04\0\0\0\x42\0\0\0\x1a\x01\0\0\x17\x40\ -\x01\0\x05\0\0\0\x42\0\0\0\x1a\x01\0\0\x1d\x40\x01\0\x06\0\0\0\x42\0\0\0\x43\ -\x01\0\0\x06\x58\x01\0\x08\0\0\0\x42\0\0\0\x56\x01\0\0\x03\x5c\x01\0\x0f\0\0\0\ -\x42\0\0\0\xdc\x01\0\0\x02\x64\x01\0\x1f\0\0\0\x42\0\0\0\x2a\x02\0\0\x01\x6c\ -\x01\0\0\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\ -\0\x10\0\0\0\x02\0\0\0\xea\0\0\0\0\0\0\0\x20\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\ -\x28\0\0\0\x08\0\0\0\x3f\x01\0\0\0\0\0\0\x78\0\0\0\x0d\0\0\0\x3e\0\0\0\0\0\0\0\ -\x88\0\0\0\x0d\0\0\0\xea\0\0\0\0\0\0\0\xa8\0\0\0\x0d\0\0\0\x3f\x01\0\0\0\0\0\0\ -\x1a\0\0\0\x21\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\0\0\0\ -\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x10\0\0\0\0\0\0\ -\0\0\0\0\0\x0a\0\0\0\x01\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\x10\0\0\0\0\0\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x62\x70\x66\x5f\x6d\ -\x61\x70\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\ -\0\0\0\0\x79\x12\x08\0\0\0\0\0\x15\x02\x3c\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x79\ -\x27\0\0\0\0\0\0\x79\x11\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\ -\0\x07\x04\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\ -\x31\0\0\0\xb7\x03\0\0\x20\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x7b\ -\x6a\xc8\xff\0\0\0\0\x61\x71\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xb7\x03\0\0\ -\x04\0\0\0\xbf\x79\0\0\0\0\0\0\x0f\x39\0\0\0\0\0\0\x79\x71\x28\0\0\0\0\0\x79\ -\x78\x30\0\0\0\0\0\x15\x08\x18\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x0f\x21\0\0\0\0\0\ -\0\x61\x11\x04\0\0\0\0\0\x79\x83\x08\0\0\0\0\0\x67\x01\0\0\x03\0\0\0\x0f\x13\0\ -\0\0\0\0\0\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\ -\xb7\x02\0\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\ -\0\0\0\0\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\ -\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\ -\xff\0\0\0\0\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\ -\xbf\x69\0\0\0\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\ -\xff\0\0\0\0\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\ -\x1a\xe8\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xd0\xff\xff\xff\x79\xa1\ -\xc8\xff\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x51\0\0\0\xb7\x03\0\0\x11\0\0\0\ -\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\ -\0\0\0\0\x17\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x80\x01\0\x01\0\0\0\x42\0\0\ -\0\x7b\0\0\0\x24\x80\x01\0\x02\0\0\0\x42\0\0\0\x60\x02\0\0\x1f\x88\x01\0\x03\0\ -\0\0\x42\0\0\0\x84\x02\0\0\x06\x94\x01\0\x04\0\0\0\x42\0\0\0\x1a\x01\0\0\x17\ -\x84\x01\0\x05\0\0\0\x42\0\0\0\x9d\x02\0\0\x0e\xa0\x01\0\x06\0\0\0\x42\0\0\0\ -\x1a\x01\0\0\x1d\x84\x01\0\x07\0\0\0\x42\0\0\0\x43\x01\0\0\x06\xa4\x01\0\x09\0\ -\0\0\x42\0\0\0\xaf\x02\0\0\x03\xa8\x01\0\x11\0\0\0\x42\0\0\0\x1f\x03\0\0\x02\ -\xb0\x01\0\x18\0\0\0\x42\0\0\0\x5a\x03\0\0\x06\x04\x01\0\x1b\0\0\0\x42\0\0\0\0\ -\0\0\0\0\0\0\0\x1c\0\0\0\x42\0\0\0\xab\x03\0\0\x0f\x10\x01\0\x1d\0\0\0\x42\0\0\ -\0\xc0\x03\0\0\x2d\x14\x01\0\x1f\0\0\0\x42\0\0\0\xf7\x03\0\0\x0d\x0c\x01\0\x21\ -\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x22\0\0\0\x42\0\0\0\xc0\x03\0\0\x02\x14\x01\0\ -\x25\0\0\0\x42\0\0\0\x1e\x04\0\0\x0d\x18\x01\0\x28\0\0\0\x42\0\0\0\0\0\0\0\0\0\ -\0\0\x29\0\0\0\x42\0\0\0\x1e\x04\0\0\x0d\x18\x01\0\x2c\0\0\0\x42\0\0\0\x1e\x04\ -\0\0\x0d\x18\x01\0\x2d\0\0\0\x42\0\0\0\x4c\x04\0\0\x1b\x1c\x01\0\x2e\0\0\0\x42\ -\0\0\0\x4c\x04\0\0\x06\x1c\x01\0\x2f\0\0\0\x42\0\0\0\x6f\x04\0\0\x0d\x24\x01\0\ -\x31\0\0\0\x42\0\0\0\x1f\x03\0\0\x02\xb0\x01\0\x40\0\0\0\x42\0\0\0\x2a\x02\0\0\ -\x01\xc0\x01\0\0\0\0\0\x14\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\ -\0\0\0\0\0\x10\0\0\0\x14\0\0\0\xea\0\0\0\0\0\0\0\x20\0\0\0\x14\0\0\0\x3e\0\0\0\ -\0\0\0\0\x28\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x30\0\0\0\x08\0\0\0\x3f\x01\0\0\ -\0\0\0\0\x88\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x98\0\0\0\x1a\0\0\0\xea\0\0\0\0\ -\0\0\0\xb0\0\0\0\x1a\0\0\0\x52\x03\0\0\0\0\0\0\xb8\0\0\0\x1a\0\0\0\x56\x03\0\0\ -\0\0\0\0\xc8\0\0\0\x1f\0\0\0\x84\x03\0\0\0\0\0\0\xe0\0\0\0\x20\0\0\0\xea\0\0\0\ -\0\0\0\0\xf8\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x20\x01\0\0\x24\0\0\0\x3e\0\0\0\ -\0\0\0\0\x58\x01\0\0\x1a\0\0\0\xea\0\0\0\0\0\0\0\x68\x01\0\0\x20\0\0\0\x46\x04\ -\0\0\0\0\0\0\x90\x01\0\0\x1a\0\0\0\x3f\x01\0\0\0\0\0\0\xa0\x01\0\0\x1a\0\0\0\ -\x87\x04\0\0\0\0\0\0\xa8\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x1a\0\0\0\x42\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0\x1c\0\0\ -\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x1a\0\ -\0\0\x01\0\0\0\0\0\0\0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\0\ -\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\ -\0\0\0\0"; - opts.insns_sz = 2216; +\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xb0\0\0\0\x03\0\0\x04\x18\0\0\0\xbe\0\ +\0\0\x09\0\0\0\0\0\0\0\xc2\0\0\0\x0b\0\0\0\x40\0\0\0\xcd\0\0\0\x0b\0\0\0\x80\0\ +\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xd5\0\0\0\0\0\0\x07\0\0\0\0\xde\0\0\0\0\0\0\ +\x08\x0c\0\0\0\xe4\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\xae\x01\0\0\x03\0\0\x04\ +\x18\0\0\0\xb6\x01\0\0\x0e\0\0\0\0\0\0\0\xb9\x01\0\0\x11\0\0\0\x20\0\0\0\xbe\ +\x01\0\0\x0e\0\0\0\xa0\0\0\0\xca\x01\0\0\0\0\0\x08\x0f\0\0\0\xd0\x01\0\0\0\0\0\ +\x01\x04\0\0\0\x20\0\0\0\xdd\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\0\0\0\ +\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xe2\x01\0\0\0\0\0\x01\x04\0\0\0\ +\x20\0\0\0\0\0\0\0\x01\0\0\x0d\x14\0\0\0\x26\x05\0\0\x04\0\0\0\x2b\x02\0\0\0\0\ +\0\x08\x15\0\0\0\x31\x02\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\x01\x3b\x02\0\0\x01\0\ +\0\x0c\x13\0\0\0\0\0\0\0\0\0\0\x02\x18\0\0\0\x52\x02\0\0\x02\0\0\x04\x10\0\0\0\ +\x13\0\0\0\x03\0\0\0\0\0\0\0\x65\x02\0\0\x19\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\ +\x1c\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x17\0\0\0\x6a\x02\0\0\x01\0\ +\0\x0c\x1a\0\0\0\xb6\x02\0\0\x01\0\0\x04\x08\0\0\0\xbf\x02\0\0\x1d\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x02\x1e\0\0\0\x10\x03\0\0\x06\0\0\x04\x38\0\0\0\xb6\x01\0\0\ +\x0e\0\0\0\0\0\0\0\xb9\x01\0\0\x11\0\0\0\x20\0\0\0\x1d\x03\0\0\x1f\0\0\0\xc0\0\ +\0\0\x2e\x03\0\0\x19\0\0\0\0\x01\0\0\x37\x03\0\0\x21\0\0\0\x40\x01\0\0\x41\x03\ +\0\0\x22\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\ +\0\0\0\0\0\0\0\0\0\x02\x23\0\0\0\0\0\0\0\0\0\0\x02\x24\0\0\0\x8b\x03\0\0\x02\0\ +\0\x04\x08\0\0\0\x99\x03\0\0\x0e\0\0\0\0\0\0\0\xa2\x03\0\0\x0e\0\0\0\x20\0\0\0\ +\x41\x03\0\0\x03\0\0\x04\x18\0\0\0\xac\x03\0\0\x1f\0\0\0\0\0\0\0\xb4\x03\0\0\ +\x25\0\0\0\x40\0\0\0\xba\x03\0\0\x27\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x26\0\0\ +\0\0\0\0\0\0\0\0\x02\x28\0\0\0\xbe\x03\0\0\x01\0\0\x04\x04\0\0\0\xc9\x03\0\0\ +\x0e\0\0\0\0\0\0\0\x32\x04\0\0\x01\0\0\x04\x04\0\0\0\x3b\x04\0\0\x0e\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x20\0\0\0\x12\0\0\0\x30\0\0\0\xb1\x04\0\0\0\0\0\ +\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x20\0\0\0\x12\0\0\0\x1a\0\0\0\ +\xc5\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x20\0\0\0\ +\x12\0\0\0\x20\0\0\0\xdb\x04\0\0\0\0\0\x0e\x2d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\ +\0\0\0\0\x20\0\0\0\x12\0\0\0\x11\0\0\0\xf0\x04\0\0\0\0\0\x0e\x2f\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\x07\x05\0\0\0\0\0\x0e\ +\x31\0\0\0\x01\0\0\0\x0f\x05\0\0\x01\0\0\x0f\x04\0\0\0\x36\0\0\0\0\0\0\0\x04\0\ +\0\0\x16\x05\0\0\x04\0\0\x0f\x7b\0\0\0\x2a\0\0\0\0\0\0\0\x30\0\0\0\x2c\0\0\0\ +\x30\0\0\0\x1a\0\0\0\x2e\0\0\0\x4a\0\0\0\x20\0\0\0\x30\0\0\0\x6a\0\0\0\x11\0\0\ +\0\x1e\x05\0\0\x01\0\0\x0f\x04\0\0\0\x32\0\0\0\0\0\0\0\x04\0\0\0\x26\x05\0\0\0\ +\0\0\x0e\x06\0\0\0\x01\0\0\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\ +\x66\x5f\x6d\x61\x70\0\x6d\x65\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\ +\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\ +\x62\x70\x66\x5f\x6d\x61\x70\0\x30\x3a\x30\0\x2f\x68\x6f\x6d\x65\x2f\x61\x73\ +\x70\x73\x6b\x2f\x73\x72\x63\x2f\x62\x70\x66\x2d\x6e\x65\x78\x74\x2f\x6b\x65\ +\x72\x6e\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\ +\x65\x72\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\ +\x70\x66\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\ +\x65\x20\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\ +\x3e\x73\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\ +\x73\x65\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\ +\x75\x6d\0\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\ +\x69\x67\x6e\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\x67\0\x30\x3a\x31\0\ +\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\x61\ +\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\x21\ +\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\x5f\ +\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\ +\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\x20\ +\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\ +\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\ +\x72\x69\x65\x73\x20\x20\x63\x75\x72\x5f\x65\x6e\x74\x72\x69\x65\x73\x5c\x6e\ +\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\0\x6d\ +\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\x73\ +\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\x52\ +\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\x5f\ +\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\ +\x75\x20\x25\x2d\x31\x36\x73\x20\x20\x25\x31\x30\x64\x20\x20\x20\x25\x31\x30\ +\x6c\x6c\x64\x5c\x6e\x22\x2c\0\x7d\0\x5f\x5f\x73\x36\x34\0\x6c\x6f\x6e\x67\x20\ +\x6c\x6f\x6e\x67\0\x62\x70\x66\x5f\x6d\x61\x70\x5f\x73\x75\x6d\x5f\x65\x6c\x65\ +\x6d\x5f\x63\x6f\x75\x6e\x74\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\ +\x66\x5f\x70\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\ +\x5f\x70\x72\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\ +\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\ +\x72\x6f\x67\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\ +\x20\x28\x21\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\ +\x78\0\x09\x61\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\ +\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\ +\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\ +\x3b\0\x62\x70\x66\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\ +\x68\x5f\x66\x75\x6e\x63\x5f\x6e\x61\x6d\x65\0\x64\x73\x74\x5f\x70\x72\x6f\x67\ +\0\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\ +\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\ +\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\ +\x78\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\ +\x62\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\ +\x6e\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\ +\x69\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\ +\x61\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\ +\x3d\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\ +\x72\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\ +\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\ +\x2b\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\ +\x66\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\ +\0\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\ +\x3d\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\ +\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\ +\x20\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\ +\x68\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\ +\x20\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\ +\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\ +\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\ +\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\ +\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\ +\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x6b\x73\x79\x6d\ +\x73\0\x2e\x72\x6f\x64\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x64\x75\x6d\ +\x6d\x79\x5f\x6b\x73\x79\x6d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\xc9\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x7b\0\0\0\x01\0\0\0\ +\x80\0\0\0\0\0\0\0\0\0\0\0\x69\x74\x65\x72\x61\x74\x6f\x72\x2e\x72\x6f\x64\x61\ +\x74\x61\0\0\0\0\0\0\0\0\0\0\0\0\0\x34\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x20\ +\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x20\x20\x63\x75\x72\x5f\x65\ +\x6e\x74\x72\x69\x65\x73\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x20\x25\ +\x31\x30\x64\x20\x20\x20\x25\x31\x30\x6c\x6c\x64\x0a\0\x20\x20\x69\x64\x20\x6e\ +\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\ +\x61\x63\x68\x65\x64\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\ +\x25\x73\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\x79\x12\0\0\0\ +\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\x1d\0\0\0\0\0\x79\x21\ +\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xe0\xff\ +\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x03\0\0\ +\x30\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x7e\0\0\0\x61\x71\0\0\0\0\0\0\x7b\x1a\ +\xe0\xff\0\0\0\0\xb7\x01\0\0\x04\0\0\0\xbf\x72\0\0\0\0\0\0\x0f\x12\0\0\0\0\0\0\ +\x7b\x2a\xe8\xff\0\0\0\0\x61\x71\x14\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\xbf\x71\ +\0\0\0\0\0\0\x85\x20\0\0\0\0\0\0\x7b\x0a\xf8\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\ +\x07\x04\0\0\xe0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\ +\x30\0\0\0\xb7\x03\0\0\x1a\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\ +\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x42\0\0\0\x88\0\0\0\ +\x1e\x44\x01\0\x01\0\0\0\x42\0\0\0\x88\0\0\0\x24\x44\x01\0\x02\0\0\0\x42\0\0\0\ +\xfb\0\0\0\x1d\x4c\x01\0\x03\0\0\0\x42\0\0\0\x1c\x01\0\0\x06\x54\x01\0\x04\0\0\ +\0\x42\0\0\0\x2b\x01\0\0\x1d\x48\x01\0\x05\0\0\0\x42\0\0\0\x50\x01\0\0\x06\x60\ +\x01\0\x07\0\0\0\x42\0\0\0\x63\x01\0\0\x03\x64\x01\0\x0e\0\0\0\x42\0\0\0\xf6\ +\x01\0\0\x02\x6c\x01\0\x21\0\0\0\x42\0\0\0\x29\x02\0\0\x01\x80\x01\0\0\0\0\0\ +\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\ +\x02\0\0\0\xf7\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x27\x01\0\0\0\0\0\0\x70\0\0\0\ +\x0d\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xf7\0\0\0\0\0\0\0\xa0\0\0\0\ +\x0d\0\0\0\x27\x01\0\0\0\0\0\0\x1a\0\0\0\x23\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\x62\ +\x70\x66\x5f\x6d\x61\x70\0\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\ +\0\0\0\x01\0\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x09\0\0\0\x01\0\0\0\0\0\0\0\x07\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x62\x70\x66\x5f\x69\x74\ +\x65\x72\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\0\0\0\0\0\0\0\x62\x70\x66\x5f\x6d\ +\x61\x70\x5f\x73\x75\x6d\x5f\x65\x6c\x65\x6d\x5f\x63\x6f\x75\x6e\x74\0\0\x47\ +\x50\x4c\0\0\0\0\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x11\x08\0\0\0\0\ +\0\x15\x01\x3b\0\0\0\0\0\x79\x17\0\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\ +\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\ +\x18\x62\0\0\0\0\0\0\0\0\0\0\x4a\0\0\0\xb7\x03\0\0\x20\0\0\0\xb7\x05\0\0\0\0\0\ +\0\x85\0\0\0\x7e\0\0\0\x7b\x6a\xc8\xff\0\0\0\0\x61\x71\0\0\0\0\0\0\x7b\x1a\xd0\ +\xff\0\0\0\0\xb7\x03\0\0\x04\0\0\0\xbf\x79\0\0\0\0\0\0\x0f\x39\0\0\0\0\0\0\x79\ +\x71\x28\0\0\0\0\0\x79\x78\x30\0\0\0\0\0\x15\x08\x18\0\0\0\0\0\xb7\x02\0\0\0\0\ +\0\0\x0f\x21\0\0\0\0\0\0\x61\x11\x04\0\0\0\0\0\x79\x83\x08\0\0\0\0\0\x67\x01\0\ +\0\x03\0\0\0\x0f\x13\0\0\0\0\0\0\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\ +\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\ +\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\ +\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x03\0\0\ +\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\ +\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\ +\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\ +\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\xd0\ +\xff\xff\xff\x79\xa1\xc8\xff\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x6a\0\0\0\xb7\ +\x03\0\0\x11\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x7e\0\0\0\xb7\0\0\0\0\0\0\0\ +\x95\0\0\0\0\0\0\0\0\0\0\0\x1b\0\0\0\0\0\0\0\x42\0\0\0\x88\0\0\0\x1e\x94\x01\0\ +\x01\0\0\0\x42\0\0\0\x88\0\0\0\x24\x94\x01\0\x02\0\0\0\x42\0\0\0\x86\x02\0\0\ +\x1f\x9c\x01\0\x03\0\0\0\x42\0\0\0\xaa\x02\0\0\x06\xa8\x01\0\x04\0\0\0\x42\0\0\ +\0\xc3\x02\0\0\x0e\xb4\x01\0\x05\0\0\0\x42\0\0\0\x2b\x01\0\0\x1d\x98\x01\0\x06\ +\0\0\0\x42\0\0\0\x50\x01\0\0\x06\xb8\x01\0\x08\0\0\0\x42\0\0\0\xd5\x02\0\0\x03\ +\xbc\x01\0\x10\0\0\0\x42\0\0\0\x45\x03\0\0\x02\xc4\x01\0\x17\0\0\0\x42\0\0\0\ +\x80\x03\0\0\x06\x04\x01\0\x1a\0\0\0\x42\0\0\0\x45\x03\0\0\x02\xc4\x01\0\x1b\0\ +\0\0\x42\0\0\0\xd1\x03\0\0\x0f\x10\x01\0\x1c\0\0\0\x42\0\0\0\xe6\x03\0\0\x2d\ +\x14\x01\0\x1e\0\0\0\x42\0\0\0\x1d\x04\0\0\x0d\x0c\x01\0\x20\0\0\0\x42\0\0\0\ +\x45\x03\0\0\x02\xc4\x01\0\x21\0\0\0\x42\0\0\0\xe6\x03\0\0\x02\x14\x01\0\x24\0\ +\0\0\x42\0\0\0\x44\x04\0\0\x0d\x18\x01\0\x27\0\0\0\x42\0\0\0\x45\x03\0\0\x02\ +\xc4\x01\0\x28\0\0\0\x42\0\0\0\x44\x04\0\0\x0d\x18\x01\0\x2b\0\0\0\x42\0\0\0\ +\x44\x04\0\0\x0d\x18\x01\0\x2c\0\0\0\x42\0\0\0\x72\x04\0\0\x1b\x1c\x01\0\x2d\0\ +\0\0\x42\0\0\0\x72\x04\0\0\x06\x1c\x01\0\x2e\0\0\0\x42\0\0\0\x95\x04\0\0\x0d\ +\x24\x01\0\x30\0\0\0\x42\0\0\0\x45\x03\0\0\x02\xc4\x01\0\x3f\0\0\0\x42\0\0\0\ +\x29\x02\0\0\x01\xd4\x01\0\0\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\ +\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x18\0\0\0\xf7\0\0\0\0\0\0\0\x20\0\0\0\x1c\0\0\ +\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\0\x27\x01\0\0\0\0\0\0\x80\0\0\0\x1e\0\0\ +\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1e\0\0\0\xf7\0\0\0\0\0\0\0\xa8\0\0\0\x1e\0\0\0\ +\x78\x03\0\0\0\0\0\0\xb0\0\0\0\x1e\0\0\0\x7c\x03\0\0\0\0\0\0\xc0\0\0\0\x23\0\0\ +\0\xaa\x03\0\0\0\0\0\0\xd8\0\0\0\x24\0\0\0\xf7\0\0\0\0\0\0\0\xf0\0\0\0\x24\0\0\ +\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x28\0\0\0\x3e\0\0\0\0\0\0\0\x50\x01\0\0\x1e\0\ +\0\0\xf7\0\0\0\0\0\0\0\x60\x01\0\0\x24\0\0\0\x6c\x04\0\0\0\0\0\0\x88\x01\0\0\ +\x1e\0\0\0\x27\x01\0\0\0\0\0\0\x98\x01\0\0\x1e\0\0\0\xad\x04\0\0\0\0\0\0\xa0\ +\x01\0\0\x1c\0\0\0\x3e\0\0\0\0\0\0\0\x1a\0\0\0\x41\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\ +\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\ +\0\0\0\0\0\0\x01\0\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x19\0\0\0\x01\0\0\0\0\0\0\0\ +\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x62\x70\x66\x5f\ +\x69\x74\x65\x72\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0"; + opts.insns_sz = 2456; opts.insns = (void *)"\ \xbf\x16\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x78\xff\xff\xff\xb7\x02\0\ \0\x88\0\0\0\xb7\x03\0\0\0\0\0\0\x85\0\0\0\x71\0\0\0\x05\0\x14\0\0\0\0\0\x61\ @@ -331,79 +326,83 @@ iterators_bpf__load(struct iterators_bpf *skel) \0\0\0\x85\0\0\0\xa8\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\x01\0\0\0\0\ \0\0\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\xbf\x70\0\0\ \0\0\0\0\x95\0\0\0\0\0\0\0\x61\x60\x08\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\ -\x48\x0e\0\0\x63\x01\0\0\0\0\0\0\x61\x60\x0c\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\ -\0\0\x44\x0e\0\0\x63\x01\0\0\0\0\0\0\x79\x60\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\ -\0\0\0\0\x38\x0e\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\x05\0\0\ -\x18\x61\0\0\0\0\0\0\0\0\0\0\x30\x0e\0\0\x7b\x01\0\0\0\0\0\0\xb7\x01\0\0\x12\0\ -\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x30\x0e\0\0\xb7\x03\0\0\x1c\0\0\0\x85\0\0\0\ +\xe8\x0e\0\0\x63\x01\0\0\0\0\0\0\x61\x60\x0c\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\ +\0\0\xe4\x0e\0\0\x63\x01\0\0\0\0\0\0\x79\x60\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\ +\0\0\0\0\xd8\x0e\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\x05\0\0\ +\x18\x61\0\0\0\0\0\0\0\0\0\0\xd0\x0e\0\0\x7b\x01\0\0\0\0\0\0\xb7\x01\0\0\x12\0\ +\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\xd0\x0e\0\0\xb7\x03\0\0\x1c\0\0\0\x85\0\0\0\ \xa6\0\0\0\xbf\x07\0\0\0\0\0\0\xc5\x07\xd4\xff\0\0\0\0\x63\x7a\x78\xff\0\0\0\0\ -\x61\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x80\x0e\0\0\x63\x01\0\0\0\ +\x61\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x20\x0f\0\0\x63\x01\0\0\0\ \0\0\0\x61\x60\x1c\0\0\0\0\0\x15\0\x03\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\ -\x5c\x0e\0\0\x63\x01\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\ -\0\x50\x0e\0\0\xb7\x03\0\0\x48\0\0\0\x85\0\0\0\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\ +\xfc\x0e\0\0\x63\x01\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\ +\0\xf0\x0e\0\0\xb7\x03\0\0\x48\0\0\0\x85\0\0\0\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\ \xc5\x07\xc3\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x63\x71\0\0\0\0\0\ -\0\x79\x63\x20\0\0\0\0\0\x15\x03\x08\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x98\ -\x0e\0\0\xb7\x02\0\0\x62\0\0\0\x61\x60\x04\0\0\0\0\0\x45\0\x02\0\x01\0\0\0\x85\ +\0\x79\x63\x20\0\0\0\0\0\x15\x03\x08\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x38\ +\x0f\0\0\xb7\x02\0\0\x7b\0\0\0\x61\x60\x04\0\0\0\0\0\x45\0\x02\0\x01\0\0\0\x85\ \0\0\0\x94\0\0\0\x05\0\x01\0\0\0\0\0\x85\0\0\0\x71\0\0\0\x18\x62\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\x61\x20\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x08\x0f\0\0\x63\ -\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\x0f\0\0\x18\x61\0\0\0\0\0\0\0\0\ -\0\0\x10\x0f\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x98\x0e\0\0\ -\x18\x61\0\0\0\0\0\0\0\0\0\0\x18\x0f\0\0\x7b\x01\0\0\0\0\0\0\xb7\x01\0\0\x02\0\ -\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x08\x0f\0\0\xb7\x03\0\0\x20\0\0\0\x85\0\0\0\ +\0\0\0\0\0\0\0\x61\x20\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xc0\x0f\0\0\x63\ +\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xb8\x0f\0\0\x18\x61\0\0\0\0\0\0\0\ +\0\0\0\xc8\x0f\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x38\x0f\0\0\ +\x18\x61\0\0\0\0\0\0\0\0\0\0\xd0\x0f\0\0\x7b\x01\0\0\0\0\0\0\xb7\x01\0\0\x02\0\ +\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\xc0\x0f\0\0\xb7\x03\0\0\x20\0\0\0\x85\0\0\0\ \xa6\0\0\0\xbf\x07\0\0\0\0\0\0\xc5\x07\x9f\xff\0\0\0\0\x18\x62\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\x61\x20\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x28\x0f\0\0\x63\ -\x01\0\0\0\0\0\0\xb7\x01\0\0\x16\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x28\x0f\0\0\ +\0\0\0\0\0\0\x61\x20\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xe0\x0f\0\0\x63\ +\x01\0\0\0\0\0\0\xb7\x01\0\0\x16\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\xe0\x0f\0\0\ \xb7\x03\0\0\x04\0\0\0\x85\0\0\0\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\xc5\x07\x92\xff\ -\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x30\x0f\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\ -\x78\x11\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x38\x0f\0\0\x18\ -\x61\0\0\0\0\0\0\0\0\0\0\x70\x11\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\ -\0\0\0\x40\x10\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb8\x11\0\0\x7b\x01\0\0\0\0\0\0\ -\x18\x60\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xc8\x11\0\ -\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xe8\x10\0\0\x18\x61\0\0\0\0\ -\0\0\0\0\0\0\xe8\x11\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xe0\x11\0\0\x7b\x01\0\0\0\0\0\0\x61\x60\x08\0\0\ -\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x80\x11\0\0\x63\x01\0\0\0\0\0\0\x61\x60\x0c\ -\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x84\x11\0\0\x63\x01\0\0\0\0\0\0\x79\x60\ -\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x88\x11\0\0\x7b\x01\0\0\0\0\0\0\x61\ -\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb0\x11\0\0\x63\x01\0\0\0\0\0\ -\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xf8\x11\0\0\xb7\x02\0\0\x11\0\0\0\xb7\x03\0\0\ +\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xe8\x0f\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\ +\x20\x12\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xf0\x0f\0\0\x18\ +\x61\0\0\0\0\0\0\0\0\0\0\x18\x12\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\ +\0\0\0\x08\x11\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x60\x12\0\0\x7b\x01\0\0\0\0\0\0\ +\x18\x60\0\0\0\0\0\0\0\0\0\0\x10\x11\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x70\x12\0\ +\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xa0\x11\0\0\x18\x61\0\0\0\0\ +\0\0\0\0\0\0\x90\x12\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x88\x12\0\0\x7b\x01\0\0\0\0\0\0\x61\x60\x08\0\0\ +\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x28\x12\0\0\x63\x01\0\0\0\0\0\0\x61\x60\x0c\ +\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x2c\x12\0\0\x63\x01\0\0\0\0\0\0\x79\x60\ +\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x30\x12\0\0\x7b\x01\0\0\0\0\0\0\x61\ +\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x58\x12\0\0\x63\x01\0\0\0\0\0\ +\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xa0\x12\0\0\xb7\x02\0\0\x11\0\0\0\xb7\x03\0\0\ \x0c\0\0\0\xb7\x04\0\0\0\0\0\0\x85\0\0\0\xa7\0\0\0\xbf\x07\0\0\0\0\0\0\xc5\x07\ -\x5c\xff\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x68\x11\0\0\x63\x70\x6c\0\0\0\0\0\ -\x77\x07\0\0\x20\0\0\0\x63\x70\x70\0\0\0\0\0\xb7\x01\0\0\x05\0\0\0\x18\x62\0\0\ -\0\0\0\0\0\0\0\0\x68\x11\0\0\xb7\x03\0\0\x8c\0\0\0\x85\0\0\0\xa6\0\0\0\xbf\x07\ -\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xd8\x11\0\0\x61\x01\0\0\0\0\0\0\xd5\ -\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\xc5\x07\x4a\xff\0\0\ -\0\0\x63\x7a\x80\xff\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x10\x12\0\0\x18\x61\0\ -\0\0\0\0\0\0\0\0\0\x10\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\ -\x18\x12\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x08\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\ -\x60\0\0\0\0\0\0\0\0\0\0\x28\x14\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x50\x17\0\0\ -\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x30\x14\0\0\x18\x61\0\0\0\0\0\ -\0\0\0\0\0\x60\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xd0\x15\ -\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x80\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x78\x17\0\0\x7b\x01\0\0\0\0\ -\0\0\x61\x60\x08\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x18\x17\0\0\x63\x01\0\0\ -\0\0\0\0\x61\x60\x0c\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x1c\x17\0\0\x63\x01\ -\0\0\0\0\0\0\x79\x60\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x20\x17\0\0\x7b\ -\x01\0\0\0\0\0\0\x61\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x48\x17\0\ -\0\x63\x01\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x90\x17\0\0\xb7\x02\0\0\x12\ -\0\0\0\xb7\x03\0\0\x0c\0\0\0\xb7\x04\0\0\0\0\0\0\x85\0\0\0\xa7\0\0\0\xbf\x07\0\ -\0\0\0\0\0\xc5\x07\x13\xff\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\x63\ -\x70\x6c\0\0\0\0\0\x77\x07\0\0\x20\0\0\0\x63\x70\x70\0\0\0\0\0\xb7\x01\0\0\x05\ -\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\xb7\x03\0\0\x8c\0\0\0\x85\0\0\0\ -\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x70\x17\0\0\x61\x01\ -\0\0\0\0\0\0\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\xc5\ -\x07\x01\xff\0\0\0\0\x63\x7a\x84\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\xd5\x01\ -\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\x61\xa0\x80\xff\0\0\0\0\ -\x63\x06\x28\0\0\0\0\0\x61\xa0\x84\xff\0\0\0\0\x63\x06\x2c\0\0\0\0\0\x18\x61\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\x10\0\0\0\0\0\0\x63\x06\x18\0\0\0\0\0\xb7\0\0\0\ -\0\0\0\0\x95\0\0\0\0\0\0\0"; +\x5c\xff\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x10\x12\0\0\x63\x70\x6c\0\0\0\0\0\ +\x77\x07\0\0\x20\0\0\0\x63\x70\x70\0\0\0\0\0\x18\x68\0\0\0\0\0\0\0\0\0\0\xa8\ +\x10\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb8\x12\0\0\xb7\x02\0\0\x17\0\0\0\xb7\x03\ +\0\0\x0c\0\0\0\xb7\x04\0\0\0\0\0\0\x85\0\0\0\xa7\0\0\0\xbf\x07\0\0\0\0\0\0\xc5\ +\x07\x4d\xff\0\0\0\0\x75\x07\x03\0\0\0\0\0\x62\x08\x04\0\0\0\0\0\x6a\x08\x02\0\ +\0\0\0\0\x05\0\x0a\0\0\0\0\0\x63\x78\x04\0\0\0\0\0\xbf\x79\0\0\0\0\0\0\x77\x09\ +\0\0\x20\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\x63\x90\0\0\0\0\0\0\x55\ +\x09\x02\0\0\0\0\0\x6a\x08\x02\0\0\0\0\0\x05\0\x01\0\0\0\0\0\x6a\x08\x02\0\x40\ +\0\0\0\xb7\x01\0\0\x05\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x10\x12\0\0\xb7\x03\0\ +\0\x8c\0\0\0\x85\0\0\0\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\ +\0\0\x01\0\0\x61\x01\0\0\0\0\0\0\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\ +\0\0\0\xa8\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x80\x12\0\0\x61\x01\0\0\0\0\0\0\ +\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\xc5\x07\x2c\xff\ +\0\0\0\0\x63\x7a\x80\xff\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xd0\x12\0\0\x18\ +\x61\0\0\0\0\0\0\0\0\0\0\xa8\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\ +\0\0\0\xd8\x12\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xa0\x17\0\0\x7b\x01\0\0\0\0\0\0\ +\x18\x60\0\0\0\0\0\0\0\0\0\0\xe0\x14\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xe8\x17\0\ +\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\xe8\x14\0\0\x18\x61\0\0\0\0\ +\0\0\0\0\0\0\xf8\x17\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x78\ +\x16\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x18\x18\0\0\x7b\x01\0\0\0\0\0\0\x18\x60\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x10\x18\0\0\x7b\x01\0\0\ +\0\0\0\0\x61\x60\x08\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb0\x17\0\0\x63\x01\ +\0\0\0\0\0\0\x61\x60\x0c\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb4\x17\0\0\x63\ +\x01\0\0\0\0\0\0\x79\x60\x10\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xb8\x17\0\0\ +\x7b\x01\0\0\0\0\0\0\x61\xa0\x78\xff\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\xe0\ +\x17\0\0\x63\x01\0\0\0\0\0\0\x18\x61\0\0\0\0\0\0\0\0\0\0\x28\x18\0\0\xb7\x02\0\ +\0\x12\0\0\0\xb7\x03\0\0\x0c\0\0\0\xb7\x04\0\0\0\0\0\0\x85\0\0\0\xa7\0\0\0\xbf\ +\x07\0\0\0\0\0\0\xc5\x07\xf5\xfe\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x98\x17\0\ +\0\x63\x70\x6c\0\0\0\0\0\x77\x07\0\0\x20\0\0\0\x63\x70\x70\0\0\0\0\0\xb7\x01\0\ +\0\x05\0\0\0\x18\x62\0\0\0\0\0\0\0\0\0\0\x98\x17\0\0\xb7\x03\0\0\x8c\0\0\0\x85\ +\0\0\0\xa6\0\0\0\xbf\x07\0\0\0\0\0\0\x18\x60\0\0\0\0\0\0\0\0\0\0\x08\x18\0\0\ +\x61\x01\0\0\0\0\0\0\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\ +\0\0\xc5\x07\xe3\xfe\0\0\0\0\x63\x7a\x84\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\ +\xd5\x01\x02\0\0\0\0\0\xbf\x19\0\0\0\0\0\0\x85\0\0\0\xa8\0\0\0\x61\xa0\x80\xff\ +\0\0\0\0\x63\x06\x28\0\0\0\0\0\x61\xa0\x84\xff\0\0\0\0\x63\x06\x2c\0\0\0\0\0\ +\x18\x61\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\x10\0\0\0\0\0\0\x63\x06\x18\0\0\0\0\0\ +\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0"; err = bpf_load_and_run(&opts); if (err < 0) return err; - skel->rodata = skel_finalize_map_data(&skel->maps.rodata.initial_value, - 4096, PROT_READ, skel->maps.rodata.map_fd); - if (!skel->rodata) - return -ENOMEM; return 0; } @@ -422,4 +421,15 @@ iterators_bpf__open_and_load(void) return skel; } +__attribute__((unused)) static void +iterators_bpf__assert(struct iterators_bpf *s __attribute__((unused))) +{ +#ifdef __cplusplus +#define _Static_assert static_assert +#endif +#ifdef __cplusplus +#undef _Static_assert +#endif +} + #endif /* __ITERATORS_BPF_SKEL_H__ */ diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 875ac9b698d9..f045fde632e5 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -23,15 +23,6 @@ #define RINGBUF_MAX_RECORD_SZ (UINT_MAX/4) -/* Maximum size of ring buffer area is limited by 32-bit page offset within - * record header, counted in pages. Reserve 8 bits for extensibility, and take - * into account few extra pages for consumer/producer pages and - * non-mmap()'able parts. This gives 64GB limit, which seems plenty for single - * ring buffer. - */ -#define RINGBUF_MAX_DATA_SZ \ - (((1ULL << 24) - RINGBUF_POS_PAGES - RINGBUF_PGOFF) * PAGE_SIZE) - struct bpf_ringbuf { wait_queue_head_t waitq; struct irq_work work; @@ -161,6 +152,17 @@ static void bpf_ringbuf_notify(struct irq_work *work) wake_up_all(&rb->waitq); } +/* Maximum size of ring buffer area is limited by 32-bit page offset within + * record header, counted in pages. Reserve 8 bits for extensibility, and + * take into account few extra pages for consumer/producer pages and + * non-mmap()'able parts, the current maximum size would be: + * + * (((1ULL << 24) - RINGBUF_POS_PAGES - RINGBUF_PGOFF) * PAGE_SIZE) + * + * This gives 64GB limit, which seems plenty for single ring buffer. Now + * considering that the maximum value of data_sz is (4GB - 1), there + * will be no overflow, so just note the size limit in the comments. + */ static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node) { struct bpf_ringbuf *rb; @@ -193,12 +195,6 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr) !PAGE_ALIGNED(attr->max_entries)) return ERR_PTR(-EINVAL); -#ifdef CONFIG_64BIT - /* on 32-bit arch, it's impossible to overflow record's hdr->pgoff */ - if (attr->max_entries > RINGBUF_MAX_DATA_SZ) - return ERR_PTR(-E2BIG); -#endif - rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE); if (!rb_map) return ERR_PTR(-ENOMEM); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a2aef900519c..7f4e8c357a6a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -37,6 +37,8 @@ #include <linux/trace_events.h> #include <net/netfilter/nf_bpf_link.h> +#include <net/tcx.h> + #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) @@ -3295,6 +3297,25 @@ static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link, raw_tp_link->btp->tp->name); } +static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen, + u32 len) +{ + if (ulen >= len + 1) { + if (copy_to_user(ubuf, buf, len + 1)) + return -EFAULT; + } else { + char zero = '\0'; + + if (copy_to_user(ubuf, buf, ulen - 1)) + return -EFAULT; + if (put_user(zero, ubuf + ulen - 1)) + return -EFAULT; + return -ENOSPC; + } + + return 0; +} + static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, struct bpf_link_info *info) { @@ -3313,20 +3334,7 @@ static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, if (!ubuf) return 0; - if (ulen >= tp_len + 1) { - if (copy_to_user(ubuf, tp_name, tp_len + 1)) - return -EFAULT; - } else { - char zero = '\0'; - - if (copy_to_user(ubuf, tp_name, ulen - 1)) - return -EFAULT; - if (put_user(zero, ubuf + ulen - 1)) - return -EFAULT; - return -ENOSPC; - } - - return 0; + return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len); } static const struct bpf_link_ops bpf_raw_tp_link_lops = { @@ -3358,9 +3366,155 @@ static void bpf_perf_link_dealloc(struct bpf_link *link) kfree(perf_link); } +static int bpf_perf_link_fill_common(const struct perf_event *event, + char __user *uname, u32 ulen, + u64 *probe_offset, u64 *probe_addr, + u32 *fd_type) +{ + const char *buf; + u32 prog_id; + size_t len; + int err; + + if (!ulen ^ !uname) + return -EINVAL; + if (!uname) + return 0; + + err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf, + probe_offset, probe_addr); + if (err) + return err; + + if (buf) { + len = strlen(buf); + err = bpf_copy_to_user(uname, buf, ulen, len); + if (err) + return err; + } else { + char zero = '\0'; + + if (put_user(zero, uname)) + return -EFAULT; + } + return 0; +} + +#ifdef CONFIG_KPROBE_EVENTS +static int bpf_perf_link_fill_kprobe(const struct perf_event *event, + struct bpf_link_info *info) +{ + char __user *uname; + u64 addr, offset; + u32 ulen, type; + int err; + + uname = u64_to_user_ptr(info->perf_event.kprobe.func_name); + ulen = info->perf_event.kprobe.name_len; + err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, + &type); + if (err) + return err; + if (type == BPF_FD_TYPE_KRETPROBE) + info->perf_event.type = BPF_PERF_EVENT_KRETPROBE; + else + info->perf_event.type = BPF_PERF_EVENT_KPROBE; + + info->perf_event.kprobe.offset = offset; + if (!kallsyms_show_value(current_cred())) + addr = 0; + info->perf_event.kprobe.addr = addr; + return 0; +} +#endif + +#ifdef CONFIG_UPROBE_EVENTS +static int bpf_perf_link_fill_uprobe(const struct perf_event *event, + struct bpf_link_info *info) +{ + char __user *uname; + u64 addr, offset; + u32 ulen, type; + int err; + + uname = u64_to_user_ptr(info->perf_event.uprobe.file_name); + ulen = info->perf_event.uprobe.name_len; + err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr, + &type); + if (err) + return err; + + if (type == BPF_FD_TYPE_URETPROBE) + info->perf_event.type = BPF_PERF_EVENT_URETPROBE; + else + info->perf_event.type = BPF_PERF_EVENT_UPROBE; + info->perf_event.uprobe.offset = offset; + return 0; +} +#endif + +static int bpf_perf_link_fill_probe(const struct perf_event *event, + struct bpf_link_info *info) +{ +#ifdef CONFIG_KPROBE_EVENTS + if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE) + return bpf_perf_link_fill_kprobe(event, info); +#endif +#ifdef CONFIG_UPROBE_EVENTS + if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE) + return bpf_perf_link_fill_uprobe(event, info); +#endif + return -EOPNOTSUPP; +} + +static int bpf_perf_link_fill_tracepoint(const struct perf_event *event, + struct bpf_link_info *info) +{ + char __user *uname; + u32 ulen; + + uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name); + ulen = info->perf_event.tracepoint.name_len; + info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT; + return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL); +} + +static int bpf_perf_link_fill_perf_event(const struct perf_event *event, + struct bpf_link_info *info) +{ + info->perf_event.event.type = event->attr.type; + info->perf_event.event.config = event->attr.config; + info->perf_event.type = BPF_PERF_EVENT_EVENT; + return 0; +} + +static int bpf_perf_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + struct bpf_perf_link *perf_link; + const struct perf_event *event; + + perf_link = container_of(link, struct bpf_perf_link, link); + event = perf_get_event(perf_link->perf_file); + if (IS_ERR(event)) + return PTR_ERR(event); + + switch (event->prog->type) { + case BPF_PROG_TYPE_PERF_EVENT: + return bpf_perf_link_fill_perf_event(event, info); + case BPF_PROG_TYPE_TRACEPOINT: + return bpf_perf_link_fill_tracepoint(event, info); + case BPF_PROG_TYPE_KPROBE: + return bpf_perf_link_fill_probe(event, info); + default: + return -EOPNOTSUPP; + } +} + static const struct bpf_link_ops bpf_perf_link_lops = { .release = bpf_perf_link_release, .dealloc = bpf_perf_link_dealloc, + .fill_link_info = bpf_perf_link_fill_link_info, }; static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) @@ -3588,31 +3742,45 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_XDP; case BPF_LSM_CGROUP: return BPF_PROG_TYPE_LSM; + case BPF_TCX_INGRESS: + case BPF_TCX_EGRESS: + return BPF_PROG_TYPE_SCHED_CLS; default: return BPF_PROG_TYPE_UNSPEC; } } -#define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd +#define BPF_PROG_ATTACH_LAST_FIELD expected_revision -#define BPF_F_ATTACH_MASK \ - (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE) +#define BPF_F_ATTACH_MASK_BASE \ + (BPF_F_ALLOW_OVERRIDE | \ + BPF_F_ALLOW_MULTI | \ + BPF_F_REPLACE) + +#define BPF_F_ATTACH_MASK_MPROG \ + (BPF_F_REPLACE | \ + BPF_F_BEFORE | \ + BPF_F_AFTER | \ + BPF_F_ID | \ + BPF_F_LINK) static int bpf_prog_attach(const union bpf_attr *attr) { enum bpf_prog_type ptype; struct bpf_prog *prog; + u32 mask; int ret; if (CHECK_ATTR(BPF_PROG_ATTACH)) return -EINVAL; - if (attr->attach_flags & ~BPF_F_ATTACH_MASK) - return -EINVAL; - ptype = attach_type_to_prog_type(attr->attach_type); if (ptype == BPF_PROG_TYPE_UNSPEC) return -EINVAL; + mask = bpf_mprog_supported(ptype) ? + BPF_F_ATTACH_MASK_MPROG : BPF_F_ATTACH_MASK_BASE; + if (attr->attach_flags & ~mask) + return -EINVAL; prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); if (IS_ERR(prog)) @@ -3648,6 +3816,9 @@ static int bpf_prog_attach(const union bpf_attr *attr) else ret = cgroup_bpf_prog_attach(attr, ptype, prog); break; + case BPF_PROG_TYPE_SCHED_CLS: + ret = tcx_prog_attach(attr, prog); + break; default: ret = -EINVAL; } @@ -3657,25 +3828,41 @@ static int bpf_prog_attach(const union bpf_attr *attr) return ret; } -#define BPF_PROG_DETACH_LAST_FIELD attach_type +#define BPF_PROG_DETACH_LAST_FIELD expected_revision static int bpf_prog_detach(const union bpf_attr *attr) { + struct bpf_prog *prog = NULL; enum bpf_prog_type ptype; + int ret; if (CHECK_ATTR(BPF_PROG_DETACH)) return -EINVAL; ptype = attach_type_to_prog_type(attr->attach_type); + if (bpf_mprog_supported(ptype)) { + if (ptype == BPF_PROG_TYPE_UNSPEC) + return -EINVAL; + if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG) + return -EINVAL; + if (attr->attach_bpf_fd) { + prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); + if (IS_ERR(prog)) + return PTR_ERR(prog); + } + } switch (ptype) { case BPF_PROG_TYPE_SK_MSG: case BPF_PROG_TYPE_SK_SKB: - return sock_map_prog_detach(attr, ptype); + ret = sock_map_prog_detach(attr, ptype); + break; case BPF_PROG_TYPE_LIRC_MODE2: - return lirc_prog_detach(attr); + ret = lirc_prog_detach(attr); + break; case BPF_PROG_TYPE_FLOW_DISSECTOR: - return netns_bpf_prog_detach(attr, ptype); + ret = netns_bpf_prog_detach(attr, ptype); + break; case BPF_PROG_TYPE_CGROUP_DEVICE: case BPF_PROG_TYPE_CGROUP_SKB: case BPF_PROG_TYPE_CGROUP_SOCK: @@ -3684,13 +3871,21 @@ static int bpf_prog_detach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: case BPF_PROG_TYPE_LSM: - return cgroup_bpf_prog_detach(attr, ptype); + ret = cgroup_bpf_prog_detach(attr, ptype); + break; + case BPF_PROG_TYPE_SCHED_CLS: + ret = tcx_prog_detach(attr, prog); + break; default: - return -EINVAL; + ret = -EINVAL; } + + if (prog) + bpf_prog_put(prog); + return ret; } -#define BPF_PROG_QUERY_LAST_FIELD query.prog_attach_flags +#define BPF_PROG_QUERY_LAST_FIELD query.link_attach_flags static int bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) @@ -3738,6 +3933,9 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_SK_MSG_VERDICT: case BPF_SK_SKB_VERDICT: return sock_map_bpf_prog_query(attr, uattr); + case BPF_TCX_INGRESS: + case BPF_TCX_EGRESS: + return tcx_prog_query(attr, uattr); default: return -EINVAL; } @@ -4700,6 +4898,13 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) goto out; } break; + case BPF_PROG_TYPE_SCHED_CLS: + if (attr->link_create.attach_type != BPF_TCX_INGRESS && + attr->link_create.attach_type != BPF_TCX_EGRESS) { + ret = -EINVAL; + goto out; + } + break; default: ptype = attach_type_to_prog_type(attr->link_create.attach_type); if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) { @@ -4751,6 +4956,9 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) case BPF_PROG_TYPE_XDP: ret = bpf_xdp_link_attach(attr, prog); break; + case BPF_PROG_TYPE_SCHED_CLS: + ret = tcx_link_attach(attr, prog); + break; case BPF_PROG_TYPE_NETFILTER: ret = bpf_nf_link_attach(attr, prog); break; diff --git a/kernel/bpf/tcx.c b/kernel/bpf/tcx.c new file mode 100644 index 000000000000..13f0b5dc8262 --- /dev/null +++ b/kernel/bpf/tcx.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include <linux/bpf.h> +#include <linux/bpf_mprog.h> +#include <linux/netdevice.h> + +#include <net/tcx.h> + +int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + bool created, ingress = attr->attach_type == BPF_TCX_INGRESS; + struct net *net = current->nsproxy->net_ns; + struct bpf_mprog_entry *entry, *entry_new; + struct bpf_prog *replace_prog = NULL; + struct net_device *dev; + int ret; + + rtnl_lock(); + dev = __dev_get_by_index(net, attr->target_ifindex); + if (!dev) { + ret = -ENODEV; + goto out; + } + if (attr->attach_flags & BPF_F_REPLACE) { + replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, + prog->type); + if (IS_ERR(replace_prog)) { + ret = PTR_ERR(replace_prog); + replace_prog = NULL; + goto out; + } + } + entry = tcx_entry_fetch_or_create(dev, ingress, &created); + if (!entry) { + ret = -ENOMEM; + goto out; + } + ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, + attr->attach_flags, attr->relative_fd, + attr->expected_revision); + if (!ret) { + if (entry != entry_new) { + tcx_entry_update(dev, entry_new, ingress); + tcx_entry_sync(); + tcx_skeys_inc(ingress); + } + bpf_mprog_commit(entry); + } else if (created) { + tcx_entry_free(entry); + } +out: + if (replace_prog) + bpf_prog_put(replace_prog); + rtnl_unlock(); + return ret; +} + +int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + bool ingress = attr->attach_type == BPF_TCX_INGRESS; + struct net *net = current->nsproxy->net_ns; + struct bpf_mprog_entry *entry, *entry_new; + struct net_device *dev; + int ret; + + rtnl_lock(); + dev = __dev_get_by_index(net, attr->target_ifindex); + if (!dev) { + ret = -ENODEV; + goto out; + } + entry = tcx_entry_fetch(dev, ingress); + if (!entry) { + ret = -ENOENT; + goto out; + } + ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, + attr->relative_fd, attr->expected_revision); + if (!ret) { + if (!tcx_entry_is_active(entry_new)) + entry_new = NULL; + tcx_entry_update(dev, entry_new, ingress); + tcx_entry_sync(); + tcx_skeys_dec(ingress); + bpf_mprog_commit(entry); + if (!entry_new) + tcx_entry_free(entry); + } +out: + rtnl_unlock(); + return ret; +} + +void tcx_uninstall(struct net_device *dev, bool ingress) +{ + struct bpf_mprog_entry *entry, *entry_new = NULL; + struct bpf_tuple tuple = {}; + struct bpf_mprog_fp *fp; + struct bpf_mprog_cp *cp; + bool active; + + entry = tcx_entry_fetch(dev, ingress); + if (!entry) + return; + active = tcx_entry(entry)->miniq_active; + if (active) + bpf_mprog_clear_all(entry, &entry_new); + tcx_entry_update(dev, entry_new, ingress); + tcx_entry_sync(); + bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { + if (tuple.link) + tcx_link(tuple.link)->dev = NULL; + else + bpf_prog_put(tuple.prog); + tcx_skeys_dec(ingress); + } + if (!active) + tcx_entry_free(entry); +} + +int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) +{ + bool ingress = attr->query.attach_type == BPF_TCX_INGRESS; + struct net *net = current->nsproxy->net_ns; + struct bpf_mprog_entry *entry; + struct net_device *dev; + int ret; + + rtnl_lock(); + dev = __dev_get_by_index(net, attr->query.target_ifindex); + if (!dev) { + ret = -ENODEV; + goto out; + } + entry = tcx_entry_fetch(dev, ingress); + if (!entry) { + ret = -ENOENT; + goto out; + } + ret = bpf_mprog_query(attr, uattr, entry); +out: + rtnl_unlock(); + return ret; +} + +static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, + u64 revision) +{ + struct tcx_link *tcx = tcx_link(link); + bool created, ingress = tcx->location == BPF_TCX_INGRESS; + struct bpf_mprog_entry *entry, *entry_new; + struct net_device *dev = tcx->dev; + int ret; + + ASSERT_RTNL(); + entry = tcx_entry_fetch_or_create(dev, ingress, &created); + if (!entry) + return -ENOMEM; + ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, + id_or_fd, revision); + if (!ret) { + if (entry != entry_new) { + tcx_entry_update(dev, entry_new, ingress); + tcx_entry_sync(); + tcx_skeys_inc(ingress); + } + bpf_mprog_commit(entry); + } else if (created) { + tcx_entry_free(entry); + } + return ret; +} + +static void tcx_link_release(struct bpf_link *link) +{ + struct tcx_link *tcx = tcx_link(link); + bool ingress = tcx->location == BPF_TCX_INGRESS; + struct bpf_mprog_entry *entry, *entry_new; + struct net_device *dev; + int ret = 0; + + rtnl_lock(); + dev = tcx->dev; + if (!dev) + goto out; + entry = tcx_entry_fetch(dev, ingress); + if (!entry) { + ret = -ENOENT; + goto out; + } + ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); + if (!ret) { + if (!tcx_entry_is_active(entry_new)) + entry_new = NULL; + tcx_entry_update(dev, entry_new, ingress); + tcx_entry_sync(); + tcx_skeys_dec(ingress); + bpf_mprog_commit(entry); + if (!entry_new) + tcx_entry_free(entry); + tcx->dev = NULL; + } +out: + WARN_ON_ONCE(ret); + rtnl_unlock(); +} + +static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog, + struct bpf_prog *oprog) +{ + struct tcx_link *tcx = tcx_link(link); + bool ingress = tcx->location == BPF_TCX_INGRESS; + struct bpf_mprog_entry *entry, *entry_new; + struct net_device *dev; + int ret = 0; + + rtnl_lock(); + dev = tcx->dev; + if (!dev) { + ret = -ENOLINK; + goto out; + } + if (oprog && link->prog != oprog) { + ret = -EPERM; + goto out; + } + oprog = link->prog; + if (oprog == nprog) { + bpf_prog_put(nprog); + goto out; + } + entry = tcx_entry_fetch(dev, ingress); + if (!entry) { + ret = -ENOENT; + goto out; + } + ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, + BPF_F_REPLACE | BPF_F_ID, + link->prog->aux->id, 0); + if (!ret) { + WARN_ON_ONCE(entry != entry_new); + oprog = xchg(&link->prog, nprog); + bpf_prog_put(oprog); + bpf_mprog_commit(entry); + } +out: + rtnl_unlock(); + return ret; +} + +static void tcx_link_dealloc(struct bpf_link *link) +{ + kfree(tcx_link(link)); +} + +static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) +{ + const struct tcx_link *tcx = tcx_link_const(link); + u32 ifindex = 0; + + rtnl_lock(); + if (tcx->dev) + ifindex = tcx->dev->ifindex; + rtnl_unlock(); + + seq_printf(seq, "ifindex:\t%u\n", ifindex); + seq_printf(seq, "attach_type:\t%u (%s)\n", + tcx->location, + tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress"); +} + +static int tcx_link_fill_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + const struct tcx_link *tcx = tcx_link_const(link); + u32 ifindex = 0; + + rtnl_lock(); + if (tcx->dev) + ifindex = tcx->dev->ifindex; + rtnl_unlock(); + + info->tcx.ifindex = ifindex; + info->tcx.attach_type = tcx->location; + return 0; +} + +static int tcx_link_detach(struct bpf_link *link) +{ + tcx_link_release(link); + return 0; +} + +static const struct bpf_link_ops tcx_link_lops = { + .release = tcx_link_release, + .detach = tcx_link_detach, + .dealloc = tcx_link_dealloc, + .update_prog = tcx_link_update, + .show_fdinfo = tcx_link_fdinfo, + .fill_link_info = tcx_link_fill_info, +}; + +static int tcx_link_init(struct tcx_link *tcx, + struct bpf_link_primer *link_primer, + const union bpf_attr *attr, + struct net_device *dev, + struct bpf_prog *prog) +{ + bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog); + tcx->location = attr->link_create.attach_type; + tcx->dev = dev; + return bpf_link_prime(&tcx->link, link_primer); +} + +int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + struct net *net = current->nsproxy->net_ns; + struct bpf_link_primer link_primer; + struct net_device *dev; + struct tcx_link *tcx; + int ret; + + rtnl_lock(); + dev = __dev_get_by_index(net, attr->link_create.target_ifindex); + if (!dev) { + ret = -ENODEV; + goto out; + } + tcx = kzalloc(sizeof(*tcx), GFP_USER); + if (!tcx) { + ret = -ENOMEM; + goto out; + } + ret = tcx_link_init(tcx, &link_primer, attr, dev, prog); + if (ret) { + kfree(tcx); + goto out; + } + ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags, + attr->link_create.tcx.relative_fd, + attr->link_create.tcx.expected_revision); + if (ret) { + tcx->dev = NULL; + bpf_link_cleanup(&link_primer); + goto out; + } + ret = bpf_link_settle(&link_primer); +out: + rtnl_unlock(); + return ret; +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 02a021c524ab..71473c19093d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -25,6 +25,7 @@ #include <linux/btf_ids.h> #include <linux/poison.h> #include <linux/module.h> +#include <linux/cpumask.h> #include "disasm.h" @@ -5412,12 +5413,25 @@ static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_FLOW_KEYS; } +static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { +#ifdef CONFIG_NET + [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], + [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], +#endif + [CONST_PTR_TO_MAP] = btf_bpf_map_id, +}; + static bool is_trusted_reg(const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ if (reg->ref_obj_id) return true; + /* Types listed in the reg2btf_ids are always trusted */ + if (reg2btf_ids[base_type(reg->type)]) + return true; + /* If a register is not referenced, it is trusted if it has the * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the * other type modifiers may be safe, but we elect to take an opt-in @@ -6085,6 +6099,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, type_is_rcu_or_null(env, reg, field_name, btf_id)) { /* __rcu tagged pointers can be NULL */ flag |= MEM_RCU | PTR_MAYBE_NULL; + + /* We always trust them */ + if (type_is_rcu_or_null(env, reg, field_name, btf_id) && + flag & PTR_UNTRUSTED) + flag &= ~PTR_UNTRUSTED; } else if (flag & (MEM_PERCPU | MEM_USER)) { /* keep as-is */ } else { @@ -9135,19 +9154,33 @@ static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, { struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; - if (ret_type != RET_INTEGER || - (func_id != BPF_FUNC_get_stack && - func_id != BPF_FUNC_get_task_stack && - func_id != BPF_FUNC_probe_read_str && - func_id != BPF_FUNC_probe_read_kernel_str && - func_id != BPF_FUNC_probe_read_user_str)) + if (ret_type != RET_INTEGER) return; - ret_reg->smax_value = meta->msize_max_value; - ret_reg->s32_max_value = meta->msize_max_value; - ret_reg->smin_value = -MAX_ERRNO; - ret_reg->s32_min_value = -MAX_ERRNO; - reg_bounds_sync(ret_reg); + switch (func_id) { + case BPF_FUNC_get_stack: + case BPF_FUNC_get_task_stack: + case BPF_FUNC_probe_read_str: + case BPF_FUNC_probe_read_kernel_str: + case BPF_FUNC_probe_read_user_str: + ret_reg->smax_value = meta->msize_max_value; + ret_reg->s32_max_value = meta->msize_max_value; + ret_reg->smin_value = -MAX_ERRNO; + ret_reg->s32_min_value = -MAX_ERRNO; + reg_bounds_sync(ret_reg); + break; + case BPF_FUNC_get_smp_processor_id: + ret_reg->umax_value = nr_cpu_ids - 1; + ret_reg->u32_max_value = nr_cpu_ids - 1; + ret_reg->smax_value = nr_cpu_ids - 1; + ret_reg->s32_max_value = nr_cpu_ids - 1; + ret_reg->umin_value = 0; + ret_reg->u32_min_value = 0; + ret_reg->smin_value = 0; + ret_reg->s32_min_value = 0; + reg_bounds_sync(ret_reg); + break; + } } static int @@ -10050,15 +10083,6 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, return true; } - -static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { -#ifdef CONFIG_NET - [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], - [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], - [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], -#endif -}; - enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 98c1544cf572..f95cfb5bf2ee 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -493,7 +493,6 @@ static inline void rcu_expedite_gp(void) { } static inline void rcu_unexpedite_gp(void) { } static inline void rcu_async_hurry(void) { } static inline void rcu_async_relax(void) { } -static inline void rcu_request_urgent_qs_task(struct task_struct *t) { } #else /* #ifdef CONFIG_TINY_RCU */ bool rcu_gp_is_normal(void); /* Internal RCU use. */ bool rcu_gp_is_expedited(void); /* Internal RCU use. */ @@ -508,7 +507,6 @@ void show_rcu_tasks_gp_kthreads(void); #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ static inline void show_rcu_tasks_gp_kthreads(void) {} #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ -void rcu_request_urgent_qs_task(struct task_struct *t); #endif /* #else #ifdef CONFIG_TINY_RCU */ #define RCU_SCHEDULER_INACTIVE 0 diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index bd1a42b23f3f..79719979a635 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2376,9 +2376,13 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, if (is_tracepoint || is_syscall_tp) { *buf = is_tracepoint ? event->tp_event->tp->name : event->tp_event->name; - *fd_type = BPF_FD_TYPE_TRACEPOINT; - *probe_offset = 0x0; - *probe_addr = 0x0; + /* We allow NULL pointer for tracepoint */ + if (fd_type) + *fd_type = BPF_FD_TYPE_TRACEPOINT; + if (probe_offset) + *probe_offset = 0x0; + if (probe_addr) + *probe_addr = 0x0; } else { /* kprobe/uprobe */ err = -EOPNOTSUPP; @@ -2391,7 +2395,7 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, #ifdef CONFIG_UPROBE_EVENTS if (flags & TRACE_EVENT_FL_UPROBE) err = bpf_get_uprobe_info(event, fd_type, buf, - probe_offset, + probe_offset, probe_addr, event->attr.type == PERF_TYPE_TRACEPOINT); #endif } @@ -2476,6 +2480,7 @@ struct bpf_kprobe_multi_link { u32 cnt; u32 mods_cnt; struct module **mods; + u32 flags; }; struct bpf_kprobe_multi_run_ctx { @@ -2565,9 +2570,44 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) kfree(kmulti_link); } +static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs); + struct bpf_kprobe_multi_link *kmulti_link; + u32 ucount = info->kprobe_multi.count; + int err = 0, i; + + if (!uaddrs ^ !ucount) + return -EINVAL; + + kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); + info->kprobe_multi.count = kmulti_link->cnt; + info->kprobe_multi.flags = kmulti_link->flags; + + if (!uaddrs) + return 0; + if (ucount < kmulti_link->cnt) + err = -ENOSPC; + else + ucount = kmulti_link->cnt; + + if (kallsyms_show_value(current_cred())) { + if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64))) + return -EFAULT; + } else { + for (i = 0; i < ucount; i++) { + if (put_user(0, uaddrs + i)) + return -EFAULT; + } + } + return err; +} + static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { .release = bpf_kprobe_multi_link_release, .dealloc = bpf_kprobe_multi_link_dealloc, + .fill_link_info = bpf_kprobe_multi_link_fill_link_info, }; static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv) @@ -2881,6 +2921,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr link->addrs = addrs; link->cookies = cookies; link->cnt = cnt; + link->flags = flags; if (cookies) { /* diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 23dba01831f7..17c21c0b2dd1 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1561,15 +1561,10 @@ int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE : BPF_FD_TYPE_KPROBE; - if (tk->symbol) { - *symbol = tk->symbol; - *probe_offset = tk->rp.kp.offset; - *probe_addr = 0; - } else { - *symbol = NULL; - *probe_offset = 0; - *probe_addr = (unsigned long)tk->rp.kp.addr; - } + *probe_offset = tk->rp.kp.offset; + *probe_addr = kallsyms_show_value(current_cred()) ? + (unsigned long)tk->rp.kp.addr : 0; + *symbol = tk->symbol; return 0; } #endif /* CONFIG_PERF_EVENTS */ diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 688bf579f2f1..555c223c3232 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -1418,7 +1418,7 @@ static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, const char **filename, u64 *probe_offset, - bool perf_type_tracepoint) + u64 *probe_addr, bool perf_type_tracepoint) { const char *pevent = trace_event_name(event->tp_event); const char *group = event->tp_event->class->system; @@ -1435,6 +1435,7 @@ int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, : BPF_FD_TYPE_UPROBE; *filename = tu->filename; *probe_offset = tu->offset; + *probe_addr = 0; return 0; } #endif /* CONFIG_PERF_EVENTS */ diff --git a/lib/nlattr.c b/lib/nlattr.c index 489e15bde5c1..7a2b6c38fd59 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -355,6 +355,12 @@ static int nla_validate_mask(const struct nla_policy *pt, case NLA_U64: value = nla_get_u64(nla); break; + case NLA_BE16: + value = ntohs(nla_get_be16(nla)); + break; + case NLA_BE32: + value = ntohl(nla_get_be32(nla)); + break; default: return -EINVAL; } diff --git a/lib/test_bpf.c b/lib/test_bpf.c index fa0833410ac1..913a7a079239 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -14381,25 +14381,15 @@ static void *generate_test_data(struct bpf_test *test, int sub) * single fragment to the skb, filled with * test->frag_data. */ - void *ptr; - page = alloc_page(GFP_KERNEL); - if (!page) goto err_kfree_skb; - ptr = kmap(page); - if (!ptr) - goto err_free_page; - memcpy(ptr, test->frag_data, MAX_DATA); - kunmap(page); + memcpy(page_address(page), test->frag_data, MAX_DATA); skb_add_rx_frag(skb, 0, page, 0, MAX_DATA, MAX_DATA); } return skb; - -err_free_page: - __free_page(page); err_kfree_skb: kfree_skb(skb); return NULL; diff --git a/lib/ts_bm.c b/lib/ts_bm.c index c8ecbf74ef29..e5f30f9177df 100644 --- a/lib/ts_bm.c +++ b/lib/ts_bm.c @@ -55,6 +55,24 @@ struct ts_bm unsigned int good_shift[]; }; +static unsigned int matchpat(const u8 *pattern, unsigned int patlen, + const u8 *text, bool icase) +{ + unsigned int i; + + for (i = 0; i < patlen; i++) { + u8 t = *(text-i); + + if (icase) + t = toupper(t); + + if (t != *(pattern-i)) + break; + } + + return i; +} + static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) { struct ts_bm *bm = ts_config_priv(conf); @@ -72,19 +90,18 @@ static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) break; while (shift < text_len) { - DEBUGP("Searching in position %d (%c)\n", - shift, text[shift]); - for (i = 0; i < bm->patlen; i++) - if ((icase ? toupper(text[shift-i]) - : text[shift-i]) - != bm->pattern[bm->patlen-1-i]) - goto next; - - /* London calling... */ - DEBUGP("found!\n"); - return consumed + (shift-(bm->patlen-1)); - -next: bs = bm->bad_shift[text[shift-i]]; + DEBUGP("Searching in position %d (%c)\n", + shift, text[shift]); + + i = matchpat(&bm->pattern[bm->patlen-1], bm->patlen, + &text[shift], icase); + if (i == bm->patlen) { + /* London calling... */ + DEBUGP("found!\n"); + return consumed + (shift-(bm->patlen-1)); + } + + bs = bm->bad_shift[text[shift-i]]; /* Now jumping to... */ shift = max_t(int, shift-i+bs, shift+bm->good_shift[i]); diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index b90781b9ece6..2a7f1b15714a 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -354,6 +354,26 @@ out: return 0; } +static int vlan_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) +{ + struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; + + return generic_hwtstamp_get_lower(real_dev, cfg); +} + +static int vlan_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; + + if (!net_eq(dev_net(dev), dev_net(real_dev))) + return -EOPNOTSUPP; + + return generic_hwtstamp_set_lower(real_dev, cfg, extack); +} + static int vlan_dev_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) { struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; @@ -365,14 +385,9 @@ static int vlan_dev_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) ifrr.ifr_ifru = ifr->ifr_ifru; switch (cmd) { - case SIOCSHWTSTAMP: - if (!net_eq(dev_net(dev), dev_net(real_dev))) - break; - fallthrough; case SIOCGMIIPHY: case SIOCGMIIREG: case SIOCSMIIREG: - case SIOCGHWTSTAMP: if (netif_device_present(real_dev) && ops->ndo_eth_ioctl) err = ops->ndo_eth_ioctl(real_dev, &ifrr, cmd); break; @@ -1081,6 +1096,8 @@ static const struct net_device_ops vlan_netdev_ops = { .ndo_fix_features = vlan_dev_fix_features, .ndo_get_iflink = vlan_dev_get_iflink, .ndo_fill_forward_path = vlan_dev_fill_forward_path, + .ndo_hwtstamp_get = vlan_hwtstamp_get, + .ndo_hwtstamp_set = vlan_hwtstamp_set, }; static void vlan_dev_free(struct net_device *dev) diff --git a/net/Kconfig b/net/Kconfig index 2fb25b534df5..d532ec33f1fe 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -52,6 +52,11 @@ config NET_INGRESS config NET_EGRESS bool +config NET_XGRESS + select NET_INGRESS + select NET_EGRESS + bool + config NET_REDIRECT bool diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 2321bd2f9964..7d47f53f20c1 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -555,12 +555,23 @@ __bpf_kfunc u32 bpf_fentry_test9(u32 *a) return *a; } +void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo) +{ +} + __bpf_kfunc int bpf_modify_return_test(int a, int *b) { *b += 1; return a + *b; } +__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d, + void *e, char f, int g) +{ + *b += 1; + return a + *b + c + d + (long)e + f + g; +} + int noinline bpf_fentry_shadow_test(int a) { return a + 1; @@ -596,6 +607,7 @@ __diag_pop(); BTF_SET8_START(bpf_test_modify_return_ids) BTF_ID_FLAGS(func, bpf_modify_return_test) +BTF_ID_FLAGS(func, bpf_modify_return_test2) BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE) BTF_SET8_END(bpf_test_modify_return_ids) @@ -663,7 +675,11 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, case BPF_MODIFY_RETURN: ret = bpf_modify_return_test(1, &b); if (b != 2) - side_effect = 1; + side_effect++; + b = 2; + ret += bpf_modify_return_test2(1, &b, 3, 4, (void *)5, 6, 7); + if (b != 2) + side_effect++; break; default: goto out; diff --git a/net/bridge/br.c b/net/bridge/br.c index 4f5098d33a46..a6e94ceb7c9a 100644 --- a/net/bridge/br.c +++ b/net/bridge/br.c @@ -234,6 +234,14 @@ static int br_switchdev_blocking_event(struct notifier_block *nb, br_switchdev_port_unoffload(p, b->ctx, b->atomic_nb, b->blocking_nb); break; + case SWITCHDEV_BRPORT_REPLAY: + brport_info = ptr; + b = &brport_info->brport; + + err = br_switchdev_port_replay(p, b->dev, b->ctx, b->atomic_nb, + b->blocking_nb, extack); + err = notifier_from_errno(err); + break; } out: diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c index 6116eba1bd89..9d7bc8b96b53 100644 --- a/net/bridge/br_forward.c +++ b/net/bridge/br_forward.c @@ -154,6 +154,7 @@ void br_forward(const struct net_bridge_port *to, backup_port = rcu_dereference(to->backup_port); if (unlikely(!backup_port)) goto out; + BR_INPUT_SKB_CB(skb)->backup_nhid = READ_ONCE(to->backup_nhid); to = backup_port; } diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c index 05c5863d2e20..10f0d33d8ccf 100644 --- a/net/bridge/br_netlink.c +++ b/net/bridge/br_netlink.c @@ -211,6 +211,7 @@ static inline size_t br_port_info_size(void) + nla_total_size(sizeof(u8)) /* IFLA_BRPORT_MRP_IN_OPEN */ + nla_total_size(sizeof(u32)) /* IFLA_BRPORT_MCAST_EHT_HOSTS_LIMIT */ + nla_total_size(sizeof(u32)) /* IFLA_BRPORT_MCAST_EHT_HOSTS_CNT */ + + nla_total_size(sizeof(u32)) /* IFLA_BRPORT_BACKUP_NHID */ + 0; } @@ -319,6 +320,10 @@ static int br_port_fill_attrs(struct sk_buff *skb, backup_p->dev->ifindex); rcu_read_unlock(); + if (p->backup_nhid && + nla_put_u32(skb, IFLA_BRPORT_BACKUP_NHID, p->backup_nhid)) + return -EMSGSIZE; + return 0; } @@ -895,6 +900,7 @@ static const struct nla_policy br_port_policy[IFLA_BRPORT_MAX + 1] = { [IFLA_BRPORT_MCAST_N_GROUPS] = { .type = NLA_REJECT }, [IFLA_BRPORT_MCAST_MAX_GROUPS] = { .type = NLA_U32 }, [IFLA_BRPORT_NEIGH_VLAN_SUPPRESS] = NLA_POLICY_MAX(NLA_U8, 1), + [IFLA_BRPORT_BACKUP_NHID] = { .type = NLA_U32 }, }; /* Change the state of the port and notify spanning tree */ @@ -1065,6 +1071,12 @@ static int br_setport(struct net_bridge_port *p, struct nlattr *tb[], return err; } + if (tb[IFLA_BRPORT_BACKUP_NHID]) { + u32 backup_nhid = nla_get_u32(tb[IFLA_BRPORT_BACKUP_NHID]); + + WRITE_ONCE(p->backup_nhid, backup_nhid); + } + return 0; } diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index a63b32c1638e..a1f4acfa6994 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -387,6 +387,7 @@ struct net_bridge_port { struct net_bridge_vlan_group __rcu *vlgrp; #endif struct net_bridge_port __rcu *backup_port; + u32 backup_nhid; /* STP */ u8 priority; @@ -605,6 +606,8 @@ struct br_input_skb_cb { */ unsigned long fwd_hwdoms; #endif + + u32 backup_nhid; }; #define BR_INPUT_SKB_CB(__skb) ((struct br_input_skb_cb *)(__skb)->cb) @@ -971,7 +974,6 @@ int br_multicast_set_vlan_router(struct net_bridge_vlan *v, u8 mcast_router); int br_multicast_toggle(struct net_bridge *br, unsigned long val, struct netlink_ext_ack *extack); int br_multicast_set_querier(struct net_bridge_mcast *brmctx, unsigned long val); -int br_multicast_set_hash_max(struct net_bridge *br, unsigned long val); int br_multicast_set_igmp_version(struct net_bridge_mcast *brmctx, unsigned long val); #if IS_ENABLED(CONFIG_IPV6) @@ -2115,6 +2117,12 @@ void br_switchdev_port_unoffload(struct net_bridge_port *p, const void *ctx, struct notifier_block *atomic_nb, struct notifier_block *blocking_nb); +int br_switchdev_port_replay(struct net_bridge_port *p, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack); + bool br_switchdev_frame_uses_tx_fwd_offload(struct sk_buff *skb); void br_switchdev_frame_set_offload_fwd_mark(struct sk_buff *skb); @@ -2165,6 +2173,16 @@ br_switchdev_port_unoffload(struct net_bridge_port *p, const void *ctx, { } +static inline int +br_switchdev_port_replay(struct net_bridge_port *p, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + static inline bool br_switchdev_frame_uses_tx_fwd_offload(struct sk_buff *skb) { return false; diff --git a/net/bridge/br_switchdev.c b/net/bridge/br_switchdev.c index ba95c4d74a60..ee84e783e1df 100644 --- a/net/bridge/br_switchdev.c +++ b/net/bridge/br_switchdev.c @@ -727,6 +727,8 @@ br_switchdev_mdb_replay(struct net_device *br_dev, struct net_device *dev, err = br_switchdev_mdb_replay_one(nb, dev, SWITCHDEV_OBJ_PORT_MDB(obj), action, ctx, extack); + if (err == -EOPNOTSUPP) + err = 0; if (err) goto out_free_mdb; } @@ -759,8 +761,10 @@ static int nbp_switchdev_sync_objs(struct net_bridge_port *p, const void *ctx, err = br_switchdev_mdb_replay(br_dev, dev, ctx, true, blocking_nb, extack); - if (err && err != -EOPNOTSUPP) + if (err) { + /* -EOPNOTSUPP not propagated from MDB replay. */ return err; + } err = br_switchdev_fdb_replay(br_dev, ctx, true, atomic_nb); if (err && err != -EOPNOTSUPP) @@ -825,3 +829,12 @@ void br_switchdev_port_unoffload(struct net_bridge_port *p, const void *ctx, nbp_switchdev_del(p); } + +int br_switchdev_port_replay(struct net_bridge_port *p, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack) +{ + return nbp_switchdev_sync_objs(p, ctx, atomic_nb, blocking_nb, extack); +} diff --git a/net/bridge/br_vlan_tunnel.c b/net/bridge/br_vlan_tunnel.c index 6399a8a69d07..81833ca7a2c7 100644 --- a/net/bridge/br_vlan_tunnel.c +++ b/net/bridge/br_vlan_tunnel.c @@ -201,6 +201,21 @@ int br_handle_egress_vlan_tunnel(struct sk_buff *skb, if (err) return err; + if (BR_INPUT_SKB_CB(skb)->backup_nhid) { + tunnel_dst = __ip_tun_set_dst(0, 0, 0, 0, 0, TUNNEL_KEY, + tunnel_id, 0); + if (!tunnel_dst) + return -ENOMEM; + + tunnel_dst->u.tun_info.mode |= IP_TUNNEL_INFO_TX | + IP_TUNNEL_INFO_BRIDGE; + tunnel_dst->u.tun_info.key.nhid = + BR_INPUT_SKB_CB(skb)->backup_nhid; + skb_dst_set(skb, &tunnel_dst->dst); + + return 0; + } + tunnel_dst = rcu_dereference(vlan->tinfo.tunnel_dst); if (tunnel_dst && dst_hold_safe(&tunnel_dst->dst)) skb_dst_set(skb, &tunnel_dst->dst); diff --git a/net/core/dev.c b/net/core/dev.c index 69a3e544676c..10e5a036c706 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -107,6 +107,7 @@ #include <net/pkt_cls.h> #include <net/checksum.h> #include <net/xfrm.h> +#include <net/tcx.h> #include <linux/highmem.h> #include <linux/init.h> #include <linux/module.h> @@ -154,7 +155,6 @@ #include "dev.h" #include "net-sysfs.h" - static DEFINE_SPINLOCK(ptype_lock); struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; struct list_head ptype_all __read_mostly; /* Taps */ @@ -388,6 +388,8 @@ static void list_netdevice(struct net_device *dev) hlist_add_head_rcu(&dev->index_hlist, dev_index_hash(net, dev->ifindex)); write_unlock(&dev_base_lock); + /* We reserved the ifindex, this can't fail */ + WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); dev_base_seq_inc(net); } @@ -397,8 +399,12 @@ static void list_netdevice(struct net_device *dev) */ static void unlist_netdevice(struct net_device *dev, bool lock) { + struct net *net = dev_net(dev); + ASSERT_RTNL(); + xa_erase(&net->dev_by_index, dev->ifindex); + /* Unlink dev from the device chain */ if (lock) write_lock(&dev_base_lock); @@ -2384,8 +2390,7 @@ static bool remove_xps_queue(struct xps_dev_maps *dev_maps, struct xps_map *map = NULL; int pos; - if (dev_maps) - map = xmap_dereference(dev_maps->attr_map[tci]); + map = xmap_dereference(dev_maps->attr_map[tci]); if (!map) return false; @@ -3882,69 +3887,198 @@ int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) EXPORT_SYMBOL(dev_loopback_xmit); #ifdef CONFIG_NET_EGRESS -static struct sk_buff * -sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) +static struct netdev_queue * +netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) { + int qm = skb_get_queue_mapping(skb); + + return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm)); +} + +static bool netdev_xmit_txqueue_skipped(void) +{ + return __this_cpu_read(softnet_data.xmit.skip_txqueue); +} + +void netdev_xmit_skip_txqueue(bool skip) +{ + __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); +} +EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); +#endif /* CONFIG_NET_EGRESS */ + +#ifdef CONFIG_NET_XGRESS +static int tc_run(struct tcx_entry *entry, struct sk_buff *skb) +{ + int ret = TC_ACT_UNSPEC; #ifdef CONFIG_NET_CLS_ACT - struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress); - struct tcf_result cl_res; + struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); + struct tcf_result res; if (!miniq) - return skb; + return ret; - /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ tc_skb_cb(skb)->mru = 0; tc_skb_cb(skb)->post_ct = false; - mini_qdisc_bstats_cpu_update(miniq, skb); - switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) { + mini_qdisc_bstats_cpu_update(miniq, skb); + ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); + /* Only tcf related quirks below. */ + switch (ret) { + case TC_ACT_SHOT: + mini_qdisc_qstats_cpu_drop(miniq); + break; case TC_ACT_OK: case TC_ACT_RECLASSIFY: - skb->tc_index = TC_H_MIN(cl_res.classid); + skb->tc_index = TC_H_MIN(res.classid); break; + } +#endif /* CONFIG_NET_CLS_ACT */ + return ret; +} + +static DEFINE_STATIC_KEY_FALSE(tcx_needed_key); + +void tcx_inc(void) +{ + static_branch_inc(&tcx_needed_key); +} + +void tcx_dec(void) +{ + static_branch_dec(&tcx_needed_key); +} + +static __always_inline enum tcx_action_base +tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, + const bool needs_mac) +{ + const struct bpf_mprog_fp *fp; + const struct bpf_prog *prog; + int ret = TCX_NEXT; + + if (needs_mac) + __skb_push(skb, skb->mac_len); + bpf_mprog_foreach_prog(entry, fp, prog) { + bpf_compute_data_pointers(skb); + ret = bpf_prog_run(prog, skb); + if (ret != TCX_NEXT) + break; + } + if (needs_mac) + __skb_pull(skb, skb->mac_len); + return tcx_action_code(skb, ret); +} + +static __always_inline struct sk_buff * +sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, + struct net_device *orig_dev, bool *another) +{ + struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); + int sch_ret; + + if (!entry) + return skb; + if (*pt_prev) { + *ret = deliver_skb(skb, *pt_prev, orig_dev); + *pt_prev = NULL; + } + + qdisc_skb_cb(skb)->pkt_len = skb->len; + tcx_set_ingress(skb, true); + + if (static_branch_unlikely(&tcx_needed_key)) { + sch_ret = tcx_run(entry, skb, true); + if (sch_ret != TC_ACT_UNSPEC) + goto ingress_verdict; + } + sch_ret = tc_run(tcx_entry(entry), skb); +ingress_verdict: + switch (sch_ret) { + case TC_ACT_REDIRECT: + /* skb_mac_header check was done by BPF, so we can safely + * push the L2 header back before redirecting to another + * netdev. + */ + __skb_push(skb, skb->mac_len); + if (skb_do_redirect(skb) == -EAGAIN) { + __skb_pull(skb, skb->mac_len); + *another = true; + break; + } + *ret = NET_RX_SUCCESS; + return NULL; case TC_ACT_SHOT: - mini_qdisc_qstats_cpu_drop(miniq); - *ret = NET_XMIT_DROP; - kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS); + kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS); + *ret = NET_RX_DROP; return NULL; + /* used by tc_run */ case TC_ACT_STOLEN: case TC_ACT_QUEUED: case TC_ACT_TRAP: - *ret = NET_XMIT_SUCCESS; consume_skb(skb); + fallthrough; + case TC_ACT_CONSUMED: + *ret = NET_RX_SUCCESS; return NULL; + } + + return skb; +} + +static __always_inline struct sk_buff * +sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) +{ + struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); + int sch_ret; + + if (!entry) + return skb; + + /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was + * already set by the caller. + */ + if (static_branch_unlikely(&tcx_needed_key)) { + sch_ret = tcx_run(entry, skb, false); + if (sch_ret != TC_ACT_UNSPEC) + goto egress_verdict; + } + sch_ret = tc_run(tcx_entry(entry), skb); +egress_verdict: + switch (sch_ret) { case TC_ACT_REDIRECT: /* No need to push/pop skb's mac_header here on egress! */ skb_do_redirect(skb); *ret = NET_XMIT_SUCCESS; return NULL; - default: - break; + case TC_ACT_SHOT: + kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS); + *ret = NET_XMIT_DROP; + return NULL; + /* used by tc_run */ + case TC_ACT_STOLEN: + case TC_ACT_QUEUED: + case TC_ACT_TRAP: + *ret = NET_XMIT_SUCCESS; + return NULL; } -#endif /* CONFIG_NET_CLS_ACT */ return skb; } - -static struct netdev_queue * -netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) -{ - int qm = skb_get_queue_mapping(skb); - - return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm)); -} - -static bool netdev_xmit_txqueue_skipped(void) +#else +static __always_inline struct sk_buff * +sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, + struct net_device *orig_dev, bool *another) { - return __this_cpu_read(softnet_data.xmit.skip_txqueue); + return skb; } -void netdev_xmit_skip_txqueue(bool skip) +static __always_inline struct sk_buff * +sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) { - __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); + return skb; } -EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); -#endif /* CONFIG_NET_EGRESS */ +#endif /* CONFIG_NET_XGRESS */ #ifdef CONFIG_XPS static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, @@ -4128,9 +4262,7 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) skb_update_prio(skb); qdisc_pkt_len_init(skb); -#ifdef CONFIG_NET_CLS_ACT - skb->tc_at_ingress = 0; -#endif + tcx_set_ingress(skb, false); #ifdef CONFIG_NET_EGRESS if (static_branch_unlikely(&egress_needed_key)) { if (nf_hook_egress_active()) { @@ -5064,72 +5196,6 @@ int (*br_fdb_test_addr_hook)(struct net_device *dev, EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); #endif -static inline struct sk_buff * -sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, - struct net_device *orig_dev, bool *another) -{ -#ifdef CONFIG_NET_CLS_ACT - struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress); - struct tcf_result cl_res; - - /* If there's at least one ingress present somewhere (so - * we get here via enabled static key), remaining devices - * that are not configured with an ingress qdisc will bail - * out here. - */ - if (!miniq) - return skb; - - if (*pt_prev) { - *ret = deliver_skb(skb, *pt_prev, orig_dev); - *pt_prev = NULL; - } - - qdisc_skb_cb(skb)->pkt_len = skb->len; - tc_skb_cb(skb)->mru = 0; - tc_skb_cb(skb)->post_ct = false; - skb->tc_at_ingress = 1; - mini_qdisc_bstats_cpu_update(miniq, skb); - - switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) { - case TC_ACT_OK: - case TC_ACT_RECLASSIFY: - skb->tc_index = TC_H_MIN(cl_res.classid); - break; - case TC_ACT_SHOT: - mini_qdisc_qstats_cpu_drop(miniq); - kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS); - *ret = NET_RX_DROP; - return NULL; - case TC_ACT_STOLEN: - case TC_ACT_QUEUED: - case TC_ACT_TRAP: - consume_skb(skb); - *ret = NET_RX_SUCCESS; - return NULL; - case TC_ACT_REDIRECT: - /* skb_mac_header check was done by cls/act_bpf, so - * we can safely push the L2 header back before - * redirecting to another netdev - */ - __skb_push(skb, skb->mac_len); - if (skb_do_redirect(skb) == -EAGAIN) { - __skb_pull(skb, skb->mac_len); - *another = true; - break; - } - *ret = NET_RX_SUCCESS; - return NULL; - case TC_ACT_CONSUMED: - *ret = NET_RX_SUCCESS; - return NULL; - default: - break; - } -#endif /* CONFIG_NET_CLS_ACT */ - return skb; -} - /** * netdev_is_rx_handler_busy - check if receive handler is registered * @dev: device to check @@ -6316,12 +6382,8 @@ int dev_set_threaded(struct net_device *dev, bool threaded) * softirq mode will happen in the next round of napi_schedule(). * This should not cause hiccups/stalls to the live traffic. */ - list_for_each_entry(napi, &dev->napi_list, dev_list) { - if (threaded) - set_bit(NAPI_STATE_THREADED, &napi->state); - else - clear_bit(NAPI_STATE_THREADED, &napi->state); - } + list_for_each_entry(napi, &dev->napi_list, dev_list) + assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); return err; } @@ -9509,23 +9571,35 @@ err_out: } /** - * dev_new_index - allocate an ifindex - * @net: the applicable net namespace + * dev_index_reserve() - allocate an ifindex in a namespace + * @net: the applicable net namespace + * @ifindex: requested ifindex, pass %0 to get one allocated * - * Returns a suitable unique value for a new device interface - * number. The caller must hold the rtnl semaphore or the - * dev_base_lock to be sure it remains unique. + * Allocate a ifindex for a new device. Caller must either use the ifindex + * to store the device (via list_netdevice()) or call dev_index_release() + * to give the index up. + * + * Return: a suitable unique value for a new device interface number or -errno. */ -static int dev_new_index(struct net *net) +static int dev_index_reserve(struct net *net, u32 ifindex) { - int ifindex = net->ifindex; + int err; - for (;;) { - if (++ifindex <= 0) - ifindex = 1; - if (!__dev_get_by_index(net, ifindex)) - return net->ifindex = ifindex; - } + if (!ifindex) + err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, + xa_limit_31b, &net->ifindex, GFP_KERNEL); + else + err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); + if (err < 0) + return err; + + return ifindex; +} + +static void dev_index_release(struct net *net, int ifindex) +{ + /* Expect only unused indexes, unlist_netdevice() removes the used */ + WARN_ON(xa_erase(&net->dev_by_index, ifindex)); } /* Delayed registration/unregisteration */ @@ -9995,11 +10069,10 @@ int register_netdevice(struct net_device *dev) goto err_uninit; } - ret = -EBUSY; - if (!dev->ifindex) - dev->ifindex = dev_new_index(net); - else if (__dev_get_by_index(net, dev->ifindex)) + ret = dev_index_reserve(net, dev->ifindex); + if (ret < 0) goto err_uninit; + dev->ifindex = ret; /* Transfer changeable features to wanted_features and enable * software offloads (GSO and GRO). @@ -10046,7 +10119,7 @@ int register_netdevice(struct net_device *dev) ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); ret = notifier_to_errno(ret); if (ret) - goto err_uninit; + goto err_ifindex_release; ret = netdev_register_kobject(dev); write_lock(&dev_base_lock); @@ -10102,6 +10175,8 @@ out: err_uninit_notify: call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev); +err_ifindex_release: + dev_index_release(net, dev->ifindex); err_uninit: if (dev->netdev_ops->ndo_uninit) dev->netdev_ops->ndo_uninit(dev); @@ -10617,6 +10692,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, dev_net_set(dev, &init_net); dev->gso_max_size = GSO_LEGACY_MAX_SIZE; + dev->xdp_zc_max_segs = 1; dev->gso_max_segs = GSO_MAX_SEGS; dev->gro_max_size = GRO_LEGACY_MAX_SIZE; dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; @@ -10838,7 +10914,7 @@ void unregister_netdevice_many_notify(struct list_head *head, /* Shutdown queueing discipline. */ dev_shutdown(dev); - + dev_tcx_uninstall(dev); dev_xdp_uninstall(dev); bpf_dev_bound_netdev_unregister(dev); @@ -10978,9 +11054,19 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, } /* Check that new_ifindex isn't used yet. */ - err = -EBUSY; - if (new_ifindex && __dev_get_by_index(net, new_ifindex)) - goto out; + if (new_ifindex) { + err = dev_index_reserve(net, new_ifindex); + if (err < 0) + goto out; + } else { + /* If there is an ifindex conflict assign a new one */ + err = dev_index_reserve(net, dev->ifindex); + if (err == -EBUSY) + err = dev_index_reserve(net, 0); + if (err < 0) + goto out; + new_ifindex = err; + } /* * And now a mini version of register_netdevice unregister_netdevice. @@ -11008,13 +11094,6 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, rcu_barrier(); new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); - /* If there is an ifindex conflict assign a new one */ - if (!new_ifindex) { - if (__dev_get_by_index(net, dev->ifindex)) - new_ifindex = dev_new_index(net); - else - new_ifindex = dev->ifindex; - } rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, new_ifindex); @@ -11192,6 +11271,8 @@ static int __net_init netdev_init(struct net *net) if (net->dev_index_head == NULL) goto err_idx; + xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); + RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); return 0; @@ -11289,6 +11370,7 @@ static void __net_exit netdev_exit(struct net *net) { kfree(net->dev_name_head); kfree(net->dev_index_head); + xa_destroy(&net->dev_by_index); if (net != &init_net) WARN_ON_ONCE(!list_empty(&net->dev_base_head)); } diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 3730945ee294..72e077022348 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -5,6 +5,7 @@ #include <linux/etherdevice.h> #include <linux/rtnetlink.h> #include <linux/net_tstamp.h> +#include <linux/phylib_stubs.h> #include <linux/wireless.h> #include <linux/if_bridge.h> #include <net/dsa_stubs.h> @@ -252,14 +253,118 @@ static int dev_eth_ioctl(struct net_device *dev, return ops->ndo_eth_ioctl(dev, ifr, cmd); } +/** + * dev_get_hwtstamp_phylib() - Get hardware timestamping settings of NIC + * or of attached phylib PHY + * @dev: Network device + * @cfg: Timestamping configuration structure + * + * Helper for enforcing a common policy that phylib timestamping, if available, + * should take precedence in front of hardware timestamping provided by the + * netdev. + * + * Note: phy_mii_ioctl() only handles SIOCSHWTSTAMP (not SIOCGHWTSTAMP), and + * there only exists a phydev->mii_ts->hwtstamp() method. So this will return + * -EOPNOTSUPP for phylib for now, which is still more accurate than letting + * the netdev handle the GET request. + */ +static int dev_get_hwtstamp_phylib(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) +{ + if (phy_has_hwtstamp(dev->phydev)) + return phy_hwtstamp_get(dev->phydev, cfg); + + return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); +} + static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - return dev_eth_ioctl(dev, ifr, SIOCGHWTSTAMP); + const struct net_device_ops *ops = dev->netdev_ops; + struct kernel_hwtstamp_config kernel_cfg = {}; + struct hwtstamp_config cfg; + int err; + + if (!ops->ndo_hwtstamp_get) + return dev_eth_ioctl(dev, ifr, SIOCGHWTSTAMP); /* legacy */ + + if (!netif_device_present(dev)) + return -ENODEV; + + kernel_cfg.ifr = ifr; + err = dev_get_hwtstamp_phylib(dev, &kernel_cfg); + if (err) + return err; + + /* If the request was resolved through an unconverted driver, omit + * the copy_to_user(), since the implementation has already done that + */ + if (!kernel_cfg.copied_to_user) { + hwtstamp_config_from_kernel(&cfg, &kernel_cfg); + + if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg))) + return -EFAULT; + } + + return 0; +} + +/** + * dev_set_hwtstamp_phylib() - Change hardware timestamping of NIC + * or of attached phylib PHY + * @dev: Network device + * @cfg: Timestamping configuration structure + * @extack: Netlink extended ack message structure, for error reporting + * + * Helper for enforcing a common policy that phylib timestamping, if available, + * should take precedence in front of hardware timestamping provided by the + * netdev. If the netdev driver needs to perform specific actions even for PHY + * timestamping to work properly (a switch port must trap the timestamped + * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in + * dev->priv_flags. + */ +static int dev_set_hwtstamp_phylib(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + const struct net_device_ops *ops = dev->netdev_ops; + bool phy_ts = phy_has_hwtstamp(dev->phydev); + struct kernel_hwtstamp_config old_cfg = {}; + bool changed = false; + int err; + + cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; + + if (!phy_ts || (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { + err = ops->ndo_hwtstamp_get(dev, &old_cfg); + if (err) + return err; + + err = ops->ndo_hwtstamp_set(dev, cfg, extack); + if (err) { + if (extack->_msg) + netdev_err(dev, "%s\n", extack->_msg); + return err; + } + + changed = kernel_hwtstamp_config_changed(&old_cfg, cfg); + } + + if (phy_ts) { + err = phy_hwtstamp_set(dev->phydev, cfg, extack); + if (err) { + if (changed) + ops->ndo_hwtstamp_set(dev, &old_cfg, NULL); + return err; + } + } + + return 0; } static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - struct kernel_hwtstamp_config kernel_cfg; + const struct net_device_ops *ops = dev->netdev_ops; + struct kernel_hwtstamp_config kernel_cfg = {}; struct netlink_ext_ack extack = {}; struct hwtstamp_config cfg; int err; @@ -268,6 +373,7 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) return -EFAULT; hwtstamp_config_to_kernel(&kernel_cfg, &cfg); + kernel_cfg.ifr = ifr; err = net_hwtstamp_validate(&kernel_cfg); if (err) @@ -280,8 +386,80 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) return err; } - return dev_eth_ioctl(dev, ifr, SIOCSHWTSTAMP); + if (!ops->ndo_hwtstamp_set) + return dev_eth_ioctl(dev, ifr, SIOCSHWTSTAMP); /* legacy */ + + if (!netif_device_present(dev)) + return -ENODEV; + + err = dev_set_hwtstamp_phylib(dev, &kernel_cfg, &extack); + if (err) + return err; + + /* The driver may have modified the configuration, so copy the + * updated version of it back to user space + */ + if (!kernel_cfg.copied_to_user) { + hwtstamp_config_from_kernel(&cfg, &kernel_cfg); + + if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg))) + return -EFAULT; + } + + return 0; +} + +static int generic_hwtstamp_ioctl_lower(struct net_device *dev, int cmd, + struct kernel_hwtstamp_config *kernel_cfg) +{ + struct ifreq ifrr; + int err; + + strscpy_pad(ifrr.ifr_name, dev->name, IFNAMSIZ); + ifrr.ifr_ifru = kernel_cfg->ifr->ifr_ifru; + + err = dev_eth_ioctl(dev, &ifrr, cmd); + if (err) + return err; + + kernel_cfg->ifr->ifr_ifru = ifrr.ifr_ifru; + kernel_cfg->copied_to_user = true; + + return 0; +} + +int generic_hwtstamp_get_lower(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_cfg) +{ + const struct net_device_ops *ops = dev->netdev_ops; + + if (!netif_device_present(dev)) + return -ENODEV; + + if (ops->ndo_hwtstamp_get) + return dev_get_hwtstamp_phylib(dev, kernel_cfg); + + /* Legacy path: unconverted lower driver */ + return generic_hwtstamp_ioctl_lower(dev, SIOCGHWTSTAMP, kernel_cfg); +} +EXPORT_SYMBOL(generic_hwtstamp_get_lower); + +int generic_hwtstamp_set_lower(struct net_device *dev, + struct kernel_hwtstamp_config *kernel_cfg, + struct netlink_ext_ack *extack) +{ + const struct net_device_ops *ops = dev->netdev_ops; + + if (!netif_device_present(dev)) + return -ENODEV; + + if (ops->ndo_hwtstamp_set) + return dev_set_hwtstamp_phylib(dev, kernel_cfg, extack); + + /* Legacy path: unconverted lower driver */ + return generic_hwtstamp_ioctl_lower(dev, SIOCSHWTSTAMP, kernel_cfg); } +EXPORT_SYMBOL(generic_hwtstamp_set_lower); static int dev_siocbond(struct net_device *dev, struct ifreq *ifr, unsigned int cmd) diff --git a/net/core/filter.c b/net/core/filter.c index 06ba0e56e369..797e8f039696 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4345,13 +4345,8 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); enum bpf_map_type map_type = ri->map_type; - if (map_type == BPF_MAP_TYPE_XSKMAP) { - /* XDP_REDIRECT is not supported AF_XDP yet. */ - if (unlikely(xdp_buff_has_frags(xdp))) - return -EOPNOTSUPP; - + if (map_type == BPF_MAP_TYPE_XSKMAP) return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog); - } return __xdp_do_redirect_frame(ri, dev, xdp_convert_buff_to_frame(xdp), xdp_prog); @@ -9312,7 +9307,7 @@ static struct bpf_insn *bpf_convert_tstamp_read(const struct bpf_prog *prog, __u8 value_reg = si->dst_reg; __u8 skb_reg = si->src_reg; -#ifdef CONFIG_NET_CLS_ACT +#ifdef CONFIG_NET_XGRESS /* If the tstamp_type is read, * the bpf prog is aware the tstamp could have delivery time. * Thus, read skb->tstamp as is if tstamp_type_access is true. @@ -9346,7 +9341,7 @@ static struct bpf_insn *bpf_convert_tstamp_write(const struct bpf_prog *prog, __u8 value_reg = si->src_reg; __u8 skb_reg = si->dst_reg; -#ifdef CONFIG_NET_CLS_ACT +#ifdef CONFIG_NET_XGRESS /* If the tstamp_type is read, * the bpf prog is aware the tstamp could have delivery time. * Thus, write skb->tstamp as is if tstamp_type_access is true. diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 85a2d0d9bd39..89d15ceaf9af 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -40,7 +40,7 @@ static void dissector_set_key(struct flow_dissector *flow_dissector, enum flow_dissector_key_id key_id) { - flow_dissector->used_keys |= (1 << key_id); + flow_dissector->used_keys |= (1ULL << key_id); } void skb_flow_dissector_init(struct flow_dissector *flow_dissector, @@ -205,6 +205,50 @@ static void __skb_flow_dissect_icmp(const struct sk_buff *skb, skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen); } +static void __skb_flow_dissect_ah(const struct sk_buff *skb, + struct flow_dissector *flow_dissector, + void *target_container, const void *data, + int nhoff, int hlen) +{ + struct flow_dissector_key_ipsec *key_ah; + struct ip_auth_hdr _hdr, *hdr; + + if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPSEC)) + return; + + hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); + if (!hdr) + return; + + key_ah = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_IPSEC, + target_container); + + key_ah->spi = hdr->spi; +} + +static void __skb_flow_dissect_esp(const struct sk_buff *skb, + struct flow_dissector *flow_dissector, + void *target_container, const void *data, + int nhoff, int hlen) +{ + struct flow_dissector_key_ipsec *key_esp; + struct ip_esp_hdr _hdr, *hdr; + + if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPSEC)) + return; + + hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); + if (!hdr) + return; + + key_esp = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_IPSEC, + target_container); + + key_esp->spi = hdr->spi; +} + static void __skb_flow_dissect_l2tpv3(const struct sk_buff *skb, struct flow_dissector *flow_dissector, void *target_container, const void *data, @@ -1571,7 +1615,14 @@ ip_proto_again: __skb_flow_dissect_l2tpv3(skb, flow_dissector, target_container, data, nhoff, hlen); break; - + case IPPROTO_ESP: + __skb_flow_dissect_esp(skb, flow_dissector, target_container, + data, nhoff, hlen); + break; + case IPPROTO_AH: + __skb_flow_dissect_ah(skb, flow_dissector, target_container, + data, nhoff, hlen); + break; default: break; } diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c index acfc1f88ea79..bc5169482710 100644 --- a/net/core/flow_offload.c +++ b/net/core/flow_offload.c @@ -146,6 +146,13 @@ void flow_rule_match_tcp(const struct flow_rule *rule, } EXPORT_SYMBOL(flow_rule_match_tcp); +void flow_rule_match_ipsec(const struct flow_rule *rule, + struct flow_match_ipsec *out) +{ + FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPSEC, out); +} +EXPORT_SYMBOL(flow_rule_match_ipsec); + void flow_rule_match_icmp(const struct flow_rule *rule, struct flow_match_icmp *out) { diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index a4270fafdf11..797c813c7c77 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -25,6 +25,14 @@ netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, return -EINVAL; } + if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { + if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, + netdev->xdp_zc_max_segs)) { + genlmsg_cancel(rsp, hdr); + return -EINVAL; + } + } + genlmsg_end(rsp, hdr); return 0; @@ -93,43 +101,22 @@ int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); struct net_device *netdev; - int idx = 0, s_idx; - int h, s_h; - int err; - - s_h = cb->args[0]; - s_idx = cb->args[1]; + int err = 0; rtnl_lock(); - - for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) { - struct hlist_head *head; - - idx = 0; - head = &net->dev_index_head[h]; - hlist_for_each_entry(netdev, head, index_hlist) { - if (idx < s_idx) - goto cont; - err = netdev_nl_dev_fill(netdev, skb, - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, 0, - NETDEV_CMD_DEV_GET); - if (err < 0) - break; -cont: - idx++; - } + for_each_netdev_dump(net, netdev, cb->args[0]) { + err = netdev_nl_dev_fill(netdev, skb, + NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, 0, + NETDEV_CMD_DEV_GET); + if (err < 0) + break; } - rtnl_unlock(); if (err != -EMSGSIZE) return err; - cb->args[1] = idx; - cb->args[0] = h; - cb->seq = net->dev_base_seq; - return skb->len; } diff --git a/net/core/of_net.c b/net/core/of_net.c index 55d3fe229269..93ea425b9248 100644 --- a/net/core/of_net.c +++ b/net/core/of_net.c @@ -8,6 +8,7 @@ #include <linux/kernel.h> #include <linux/of_net.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/phy.h> #include <linux/export.h> #include <linux/device.h> diff --git a/net/core/page_pool.c b/net/core/page_pool.c index a3e12a61d456..5d615a169718 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -58,6 +58,17 @@ static const char pp_stats[][ETH_GSTRING_LEN] = { "rx_pp_recycle_released_ref", }; +/** + * page_pool_get_stats() - fetch page pool stats + * @pool: pool from which page was allocated + * @stats: struct page_pool_stats to fill in + * + * Retrieve statistics about the page_pool. This API is only available + * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``. + * A pointer to a caller allocated struct page_pool_stats structure + * is passed to this API which is filled in. The caller can then report + * those stats to the user (perhaps via ethtool, debugfs, etc.). + */ bool page_pool_get_stats(struct page_pool *pool, struct page_pool_stats *stats) { @@ -224,6 +235,10 @@ static int page_pool_init(struct page_pool *pool, return 0; } +/** + * page_pool_create() - create a page pool. + * @params: parameters, see struct page_pool_params + */ struct page_pool *page_pool_create(const struct page_pool_params *params) { struct page_pool *pool; @@ -492,7 +507,7 @@ static s32 page_pool_inflight(struct page_pool *pool) * a regular page (that will eventually be returned to the normal * page-allocator via put_page). */ -void page_pool_release_page(struct page_pool *pool, struct page *page) +static void page_pool_return_page(struct page_pool *pool, struct page *page) { dma_addr_t dma; int count; @@ -518,13 +533,6 @@ skip_dma_unmap: */ count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); trace_page_pool_state_release(pool, page, count); -} -EXPORT_SYMBOL(page_pool_release_page); - -/* Return a page to the page allocator, cleaning up our state */ -static void page_pool_return_page(struct page_pool *pool, struct page *page) -{ - page_pool_release_page(pool, page); put_page(page); /* An optimization would be to call __free_pages(page, pool->p.order) @@ -616,9 +624,7 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, * will be invoking put_page. */ recycle_stat_inc(pool, released_refcnt); - /* Do not replace this with page_pool_return_page() */ - page_pool_release_page(pool, page); - put_page(page); + page_pool_return_page(pool, page); return NULL; } @@ -635,7 +641,21 @@ void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, } EXPORT_SYMBOL(page_pool_put_defragged_page); -/* Caller must not use data area after call, as this function overwrites it */ +/** + * page_pool_put_page_bulk() - release references on multiple pages + * @pool: pool from which pages were allocated + * @data: array holding page pointers + * @count: number of pages in @data + * + * Tries to refill a number of pages into the ptr_ring cache holding ptr_ring + * producer lock. If the ptr_ring is full, page_pool_put_page_bulk() + * will release leftover pages to the page allocator. + * page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx + * completion loop for the XDP_REDIRECT use case. + * + * Please note the caller must not use data area after running + * page_pool_put_page_bulk(), as this function overwrites it. + */ void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count) { diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index aef25aa5cf1d..6d2180b8edb3 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -61,7 +61,7 @@ #include "dev.h" #define RTNL_MAX_TYPE 50 -#define RTNL_SLAVE_MAX_TYPE 43 +#define RTNL_SLAVE_MAX_TYPE 44 struct rtnl_link { rtnl_doit_func doit; @@ -1273,7 +1273,6 @@ static noinline_for_stack int rtnl_fill_stats(struct sk_buff *skb, static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb, struct net_device *dev, int vfs_num, - struct nlattr *vfinfo, u32 ext_filter_mask) { struct ifla_vf_rss_query_en vf_rss_query_en; @@ -1343,7 +1342,7 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb, vf_trust.setting = ivi.trusted; vf = nla_nest_start_noflag(skb, IFLA_VF_INFO); if (!vf) - goto nla_put_vfinfo_failure; + return -EMSGSIZE; if (nla_put(skb, IFLA_VF_MAC, sizeof(vf_mac), &vf_mac) || nla_put(skb, IFLA_VF_BROADCAST, sizeof(vf_broadcast), &vf_broadcast) || nla_put(skb, IFLA_VF_VLAN, sizeof(vf_vlan), &vf_vlan) || @@ -1414,8 +1413,6 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb, nla_put_vf_failure: nla_nest_cancel(skb, vf); -nla_put_vfinfo_failure: - nla_nest_cancel(skb, vfinfo); return -EMSGSIZE; } @@ -1441,8 +1438,10 @@ static noinline_for_stack int rtnl_fill_vf(struct sk_buff *skb, return -EMSGSIZE; for (i = 0; i < num_vfs; i++) { - if (rtnl_fill_vfinfo(skb, dev, i, vfinfo, ext_filter_mask)) + if (rtnl_fill_vfinfo(skb, dev, i, ext_filter_mask)) { + nla_nest_cancel(skb, vfinfo); return -EMSGSIZE; + } } nla_nest_end(skb, vfinfo); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a298992060e6..c6f98245582c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -6204,7 +6204,7 @@ EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); * * @header_len: size of linear part * @data_len: needed length in frags - * @max_page_order: max page order desired. + * @order: max page order desired. * @errcode: pointer to error code if any * @gfp_mask: allocation mask * @@ -6212,21 +6212,17 @@ EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); */ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, unsigned long data_len, - int max_page_order, + int order, int *errcode, gfp_t gfp_mask) { - int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; unsigned long chunk; struct sk_buff *skb; struct page *page; - int i; + int nr_frags = 0; *errcode = -EMSGSIZE; - /* Note this test could be relaxed, if we succeed to allocate - * high order pages... - */ - if (npages > MAX_SKB_FRAGS) + if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) return NULL; *errcode = -ENOBUFS; @@ -6234,34 +6230,32 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, if (!skb) return NULL; - skb->truesize += npages << PAGE_SHIFT; - - for (i = 0; npages > 0; i++) { - int order = max_page_order; - - while (order) { - if (npages >= 1 << order) { - page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | - __GFP_COMP | - __GFP_NOWARN, - order); - if (page) - goto fill_page; - /* Do not retry other high order allocations */ - order = 1; - max_page_order = 0; - } + while (data_len) { + if (nr_frags == MAX_SKB_FRAGS - 1) + goto failure; + while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) order--; + + if (order) { + page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | + __GFP_COMP | + __GFP_NOWARN, + order); + if (!page) { + order--; + continue; + } + } else { + page = alloc_page(gfp_mask); + if (!page) + goto failure; } - page = alloc_page(gfp_mask); - if (!page) - goto failure; -fill_page: chunk = min_t(unsigned long, data_len, PAGE_SIZE << order); - skb_fill_page_desc(skb, i, page, 0, chunk); + skb_fill_page_desc(skb, nr_frags, page, 0, chunk); + nr_frags++; + skb->truesize += (PAGE_SIZE << order); data_len -= chunk; - npages -= 1 << order; } return skb; diff --git a/net/core/sock.c b/net/core/sock.c index 6d4f28efe29a..49915801d53a 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1247,17 +1247,11 @@ set_sndbuf: break; case SO_PASSCRED: - if (valbool) - set_bit(SOCK_PASSCRED, &sock->flags); - else - clear_bit(SOCK_PASSCRED, &sock->flags); + assign_bit(SOCK_PASSCRED, &sock->flags, valbool); break; case SO_PASSPIDFD: - if (valbool) - set_bit(SOCK_PASSPIDFD, &sock->flags); - else - clear_bit(SOCK_PASSPIDFD, &sock->flags); + assign_bit(SOCK_PASSPIDFD, &sock->flags, valbool); break; case SO_TIMESTAMP_OLD: @@ -1361,10 +1355,7 @@ set_sndbuf: break; case SO_PASSSEC: - if (valbool) - set_bit(SOCK_PASSSEC, &sock->flags); - else - clear_bit(SOCK_PASSSEC, &sock->flags); + assign_bit(SOCK_PASSSEC, &sock->flags, valbool); break; case SO_MARK: if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && @@ -1823,14 +1814,14 @@ int sk_getsockopt(struct sock *sk, int level, int optname, case SO_PEERNAME: { - char address[128]; + struct sockaddr_storage address; - lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); + lv = sock->ops->getname(sock, (struct sockaddr *)&address, 2); if (lv < 0) return -ENOTCONN; if (lv < len) return -EINVAL; - if (copy_to_sockptr(optval, address, len)) + if (copy_to_sockptr(optval, &address, len)) return -EFAULT; goto lenout; } diff --git a/net/dccp/feat.h b/net/dccp/feat.h index d76c9be5bfca..57d9c026aa3f 100644 --- a/net/dccp/feat.h +++ b/net/dccp/feat.h @@ -105,7 +105,6 @@ extern int sysctl_dccp_rx_ccid; extern int sysctl_dccp_tx_ccid; int dccp_feat_init(struct sock *sk); -void dccp_feat_initialise_sysctls(void); int dccp_feat_register_sp(struct sock *sk, u8 feat, u8 is_local, u8 const *list, u8 len); int dccp_feat_parse_options(struct sock *, struct dccp_request_sock *, diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c index fa8079303cb0..8e919cfe6e23 100644 --- a/net/dccp/ipv4.c +++ b/net/dccp/ipv4.c @@ -474,7 +474,8 @@ static struct dst_entry* dccp_v4_route_skb(struct net *net, struct sock *sk, .flowi4_oif = inet_iif(skb), .daddr = iph->saddr, .saddr = iph->daddr, - .flowi4_tos = RT_CONN_FLAGS(sk), + .flowi4_tos = ip_sock_rt_tos(sk), + .flowi4_scope = ip_sock_rt_scope(sk), .flowi4_proto = sk->sk_protocol, .fl4_sport = dccp_hdr(skb)->dccph_dport, .fl4_dport = dccp_hdr(skb)->dccph_sport, diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c index d29d1163203d..686090bc5945 100644 --- a/net/dccp/ipv6.c +++ b/net/dccp/ipv6.c @@ -1056,6 +1056,7 @@ static struct proto dccp_v6_prot = { .orphan_count = &dccp_orphan_count, .max_header = MAX_DCCP_HEADER, .obj_size = sizeof(struct dccp6_sock), + .ipv6_pinfo_offset = offsetof(struct dccp6_sock, inet6), .slab_flags = SLAB_TYPESAFE_BY_RCU, .rsk_prot = &dccp6_request_sock_ops, .twsk_prot = &dccp6_timewait_sock_ops, diff --git a/net/dccp/ipv6.h b/net/dccp/ipv6.h index 7e4c2a3b322b..c5d14c48def1 100644 --- a/net/dccp/ipv6.h +++ b/net/dccp/ipv6.h @@ -13,10 +13,6 @@ struct dccp6_sock { struct dccp_sock dccp; - /* - * ipv6_pinfo has to be the last member of dccp6_sock, - * see inet6_sk_generic. - */ struct ipv6_pinfo inet6; }; diff --git a/net/devlink/leftover.c b/net/devlink/leftover.c index 1f00f874471f..5128b9c7eea8 100644 --- a/net/devlink/leftover.c +++ b/net/devlink/leftover.c @@ -3946,7 +3946,7 @@ static int devlink_param_get(struct devlink *devlink, const struct devlink_param *param, struct devlink_param_gset_ctx *ctx) { - if (!param->get || devlink->reload_failed) + if (!param->get) return -EOPNOTSUPP; return param->get(devlink, param->id, ctx); } @@ -3955,7 +3955,7 @@ static int devlink_param_set(struct devlink *devlink, const struct devlink_param *param, struct devlink_param_gset_ctx *ctx) { - if (!param->set || devlink->reload_failed) + if (!param->set) return -EOPNOTSUPP; return param->set(devlink, param->id, ctx); } diff --git a/net/dsa/port.c b/net/dsa/port.c index 2f6195d7b741..24015e11255f 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -1568,27 +1568,6 @@ static void dsa_port_phylink_validate(struct phylink_config *config, phylink_generic_validate(config, supported, state); } -static void dsa_port_phylink_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - struct dsa_port *dp = container_of(config, struct dsa_port, pl_config); - struct dsa_switch *ds = dp->ds; - int err; - - /* Only called for inband modes */ - if (!ds->ops->phylink_mac_link_state) { - state->link = 0; - return; - } - - err = ds->ops->phylink_mac_link_state(ds, dp->index, state); - if (err < 0) { - dev_err(ds->dev, "p%d: phylink_mac_link_state() failed: %d\n", - dp->index, err); - state->link = 0; - } -} - static struct phylink_pcs * dsa_port_phylink_mac_select_pcs(struct phylink_config *config, phy_interface_t interface) @@ -1646,17 +1625,6 @@ static int dsa_port_phylink_mac_finish(struct phylink_config *config, return err; } -static void dsa_port_phylink_mac_an_restart(struct phylink_config *config) -{ - struct dsa_port *dp = container_of(config, struct dsa_port, pl_config); - struct dsa_switch *ds = dp->ds; - - if (!ds->ops->phylink_mac_an_restart) - return; - - ds->ops->phylink_mac_an_restart(ds, dp->index); -} - static void dsa_port_phylink_mac_link_down(struct phylink_config *config, unsigned int mode, phy_interface_t interface) @@ -1700,11 +1668,9 @@ static void dsa_port_phylink_mac_link_up(struct phylink_config *config, static const struct phylink_mac_ops dsa_port_phylink_mac_ops = { .validate = dsa_port_phylink_validate, .mac_select_pcs = dsa_port_phylink_mac_select_pcs, - .mac_pcs_get_state = dsa_port_phylink_mac_pcs_get_state, .mac_prepare = dsa_port_phylink_mac_prepare, .mac_config = dsa_port_phylink_mac_config, .mac_finish = dsa_port_phylink_mac_finish, - .mac_an_restart = dsa_port_phylink_mac_an_restart, .mac_link_down = dsa_port_phylink_mac_link_down, .mac_link_up = dsa_port_phylink_mac_link_up, }; @@ -1720,13 +1686,6 @@ int dsa_port_phylink_create(struct dsa_port *dp) if (err) mode = PHY_INTERFACE_MODE_NA; - /* Presence of phylink_mac_link_state or phylink_mac_an_restart is - * an indicator of a legacy phylink driver. - */ - if (ds->ops->phylink_mac_link_state || - ds->ops->phylink_mac_an_restart) - dp->pl_config.legacy_pre_march2020 = true; - if (ds->ops->phylink_get_caps) { ds->ops->phylink_get_caps(ds, dp->index, &dp->pl_config); } else { diff --git a/net/dsa/slave.c b/net/dsa/slave.c index 527b1d576460..48db91b33390 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -21,6 +21,7 @@ #include <linux/if_hsr.h> #include <net/dcbnl.h> #include <linux/netpoll.h> +#include <linux/string.h> #include "dsa.h" #include "port.h" @@ -1056,10 +1057,10 @@ static void dsa_slave_get_strings(struct net_device *dev, if (stringset == ETH_SS_STATS) { int len = ETH_GSTRING_LEN; - strncpy(data, "tx_packets", len); - strncpy(data + len, "tx_bytes", len); - strncpy(data + 2 * len, "rx_packets", len); - strncpy(data + 3 * len, "rx_bytes", len); + strscpy_pad(data, "tx_packets", len); + strscpy_pad(data + len, "tx_bytes", len); + strscpy_pad(data + 2 * len, "rx_packets", len); + strscpy_pad(data + 3 * len, "rx_bytes", len); if (ds->ops->get_strings) ds->ops->get_strings(ds, dp->index, stringset, data + 4 * len); diff --git a/net/dsa/tag_qca.c b/net/dsa/tag_qca.c index e757c8de06f1..e5ff7c34e577 100644 --- a/net/dsa/tag_qca.c +++ b/net/dsa/tag_qca.c @@ -75,10 +75,6 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) return NULL; } - /* Remove QCA tag and recalculate checksum */ - skb_pull_rcsum(skb, QCA_HDR_LEN); - dsa_strip_etype_header(skb, QCA_HDR_LEN); - /* Get source port information */ port = FIELD_GET(QCA_HDR_RECV_SOURCE_PORT, hdr); @@ -86,6 +82,10 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) if (!skb->dev) return NULL; + /* Remove QCA tag and recalculate checksum */ + skb_pull_rcsum(skb, QCA_HDR_LEN); + dsa_strip_etype_header(skb, QCA_HDR_LEN); + return skb; } diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 5fb19050991e..f5598c5f50de 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -665,9 +665,8 @@ const struct ethtool_phy_ops *ethtool_phy_ops; void ethtool_set_ethtool_phy_ops(const struct ethtool_phy_ops *ops) { - rtnl_lock(); + ASSERT_RTNL(); ethtool_phy_ops = ops; - rtnl_unlock(); } EXPORT_SYMBOL_GPL(ethtool_set_ethtool_phy_ops); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 4a51e0ec295c..0b0ce4f81c01 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -907,6 +907,38 @@ static int ethtool_rxnfc_copy_to_compat(void __user *useraddr, return 0; } +static int ethtool_rxnfc_copy_struct(u32 cmd, struct ethtool_rxnfc *info, + size_t *info_size, void __user *useraddr) +{ + /* struct ethtool_rxnfc was originally defined for + * ETHTOOL_{G,S}RXFH with only the cmd, flow_type and data + * members. User-space might still be using that + * definition. + */ + if (cmd == ETHTOOL_GRXFH || cmd == ETHTOOL_SRXFH) + *info_size = (offsetof(struct ethtool_rxnfc, data) + + sizeof(info->data)); + + if (ethtool_rxnfc_copy_from_user(info, useraddr, *info_size)) + return -EFAULT; + + if ((cmd == ETHTOOL_GRXFH || cmd == ETHTOOL_SRXFH) && info->flow_type & FLOW_RSS) { + *info_size = sizeof(*info); + if (ethtool_rxnfc_copy_from_user(info, useraddr, *info_size)) + return -EFAULT; + /* Since malicious users may modify the original data, + * we need to check whether FLOW_RSS is still requested. + */ + if (!(info->flow_type & FLOW_RSS)) + return -EINVAL; + } + + if (info->cmd != cmd) + return -EINVAL; + + return 0; +} + static int ethtool_rxnfc_copy_to_user(void __user *useraddr, const struct ethtool_rxnfc *rxnfc, size_t size, const u32 *rule_buf) @@ -944,16 +976,9 @@ static noinline_for_stack int ethtool_set_rxnfc(struct net_device *dev, if (!dev->ethtool_ops->set_rxnfc) return -EOPNOTSUPP; - /* struct ethtool_rxnfc was originally defined for - * ETHTOOL_{G,S}RXFH with only the cmd, flow_type and data - * members. User-space might still be using that - * definition. */ - if (cmd == ETHTOOL_SRXFH) - info_size = (offsetof(struct ethtool_rxnfc, data) + - sizeof(info.data)); - - if (ethtool_rxnfc_copy_from_user(&info, useraddr, info_size)) - return -EFAULT; + rc = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr); + if (rc) + return rc; rc = dev->ethtool_ops->set_rxnfc(dev, &info); if (rc) @@ -978,33 +1003,9 @@ static noinline_for_stack int ethtool_get_rxnfc(struct net_device *dev, if (!ops->get_rxnfc) return -EOPNOTSUPP; - /* struct ethtool_rxnfc was originally defined for - * ETHTOOL_{G,S}RXFH with only the cmd, flow_type and data - * members. User-space might still be using that - * definition. */ - if (cmd == ETHTOOL_GRXFH) - info_size = (offsetof(struct ethtool_rxnfc, data) + - sizeof(info.data)); - - if (ethtool_rxnfc_copy_from_user(&info, useraddr, info_size)) - return -EFAULT; - - /* If FLOW_RSS was requested then user-space must be using the - * new definition, as FLOW_RSS is newer. - */ - if (cmd == ETHTOOL_GRXFH && info.flow_type & FLOW_RSS) { - info_size = sizeof(info); - if (ethtool_rxnfc_copy_from_user(&info, useraddr, info_size)) - return -EFAULT; - /* Since malicious users may modify the original data, - * we need to check whether FLOW_RSS is still requested. - */ - if (!(info.flow_type & FLOW_RSS)) - return -EINVAL; - } - - if (info.cmd != cmd) - return -EINVAL; + ret = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr); + if (ret) + return ret; if (info.cmd == ETHTOOL_GRXCLSRLALL) { if (info.rule_cnt > 0) { @@ -3207,7 +3208,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) if (v4_m_spec->ip4src || v4_m_spec->ip4dst) { match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS); + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS); match->dissector.offset[FLOW_DISSECTOR_KEY_IPV4_ADDRS] = offsetof(struct ethtool_rx_flow_key, ipv4); } @@ -3222,7 +3223,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) if (v4_m_spec->psrc || v4_m_spec->pdst) { match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_PORTS); + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); match->dissector.offset[FLOW_DISSECTOR_KEY_PORTS] = offsetof(struct ethtool_rx_flow_key, tp); } @@ -3259,7 +3260,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) if (!ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6src) || !ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6dst)) { match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS); + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS); match->dissector.offset[FLOW_DISSECTOR_KEY_IPV6_ADDRS] = offsetof(struct ethtool_rx_flow_key, ipv6); } @@ -3274,7 +3275,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) if (v6_m_spec->psrc || v6_m_spec->pdst) { match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_PORTS); + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); match->dissector.offset[FLOW_DISSECTOR_KEY_PORTS] = offsetof(struct ethtool_rx_flow_key, tp); } @@ -3282,7 +3283,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) match->key.ip.tos = v6_spec->tclass; match->mask.ip.tos = v6_m_spec->tclass; match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_IP); + BIT_ULL(FLOW_DISSECTOR_KEY_IP); match->dissector.offset[FLOW_DISSECTOR_KEY_IP] = offsetof(struct ethtool_rx_flow_key, ip); } @@ -3306,7 +3307,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) break; } - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_BASIC); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); match->dissector.offset[FLOW_DISSECTOR_KEY_BASIC] = offsetof(struct ethtool_rx_flow_key, basic); @@ -3339,7 +3340,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) if (ext_m_spec->vlan_etype || ext_m_spec->vlan_tci) { match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_VLAN); + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN); match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] = offsetof(struct ethtool_rx_flow_key, vlan); } @@ -3354,7 +3355,7 @@ ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input) ETH_ALEN); match->dissector.used_keys |= - BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS); + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS); match->dissector.offset[FLOW_DISSECTOR_KEY_ETH_ADDRS] = offsetof(struct ethtool_rx_flow_key, eth_addrs); } diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 39a459b0111b..ae344f1b0bbd 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -252,8 +252,7 @@ int ethnl_multicast(struct sk_buff *skb, struct net_device *dev) * @ops: request ops of currently processed message type * @req_info: parsed request header of processed request * @reply_data: data needed to compose the reply - * @pos_hash: saved iteration position - hashbucket - * @pos_idx: saved iteration position - index + * @pos_ifindex: saved iteration position - ifindex * * These parameters are kept in struct netlink_callback as context preserved * between iterations. They are initialized by ethnl_default_start() and used @@ -263,8 +262,7 @@ struct ethnl_dump_ctx { const struct ethnl_request_ops *ops; struct ethnl_req_info *req_info; struct ethnl_reply_data *reply_data; - int pos_hash; - int pos_idx; + unsigned long pos_ifindex; }; static const struct ethnl_request_ops * @@ -490,55 +488,27 @@ static int ethnl_default_dumpit(struct sk_buff *skb, { struct ethnl_dump_ctx *ctx = ethnl_dump_context(cb); struct net *net = sock_net(skb->sk); - int s_idx = ctx->pos_idx; - int h, idx = 0; + struct net_device *dev; int ret = 0; rtnl_lock(); - for (h = ctx->pos_hash; h < NETDEV_HASHENTRIES; h++, s_idx = 0) { - struct hlist_head *head; - struct net_device *dev; - unsigned int seq; - - head = &net->dev_index_head[h]; - -restart_chain: - seq = net->dev_base_seq; - cb->seq = seq; - idx = 0; - hlist_for_each_entry(dev, head, index_hlist) { - if (idx < s_idx) - goto cont; - dev_hold(dev); - rtnl_unlock(); - - ret = ethnl_default_dump_one(skb, dev, ctx, cb); - dev_put(dev); - if (ret < 0) { - if (ret == -EOPNOTSUPP) - goto lock_and_cont; - if (likely(skb->len)) - ret = skb->len; - goto out; - } -lock_and_cont: - rtnl_lock(); - if (net->dev_base_seq != seq) { - s_idx = idx + 1; - goto restart_chain; - } -cont: - idx++; - } + for_each_netdev_dump(net, dev, ctx->pos_ifindex) { + dev_hold(dev); + rtnl_unlock(); + + ret = ethnl_default_dump_one(skb, dev, ctx, cb); + + rtnl_lock(); + dev_put(dev); + if (ret < 0 && ret != -EOPNOTSUPP) { + if (likely(skb->len)) + ret = skb->len; + break; + } } rtnl_unlock(); -out: - ctx->pos_hash = h; - ctx->pos_idx = idx; - nl_dump_check_consistent(cb, nlmsg_hdr(skb)); - return ret; } @@ -584,8 +554,7 @@ static int ethnl_default_start(struct netlink_callback *cb) ctx->ops = ops; ctx->req_info = req_info; ctx->reply_data = reply_data; - ctx->pos_hash = 0; - ctx->pos_idx = 0; + ctx->pos_ifindex = 0; return 0; diff --git a/net/ethtool/tunnels.c b/net/ethtool/tunnels.c index 67fb414ca859..05f752557b5e 100644 --- a/net/ethtool/tunnels.c +++ b/net/ethtool/tunnels.c @@ -212,8 +212,7 @@ err_unlock_rtnl: struct ethnl_tunnel_info_dump_ctx { struct ethnl_req_info req_info; - int pos_hash; - int pos_idx; + unsigned long ifindex; }; int ethnl_tunnel_info_start(struct netlink_callback *cb) @@ -243,57 +242,39 @@ int ethnl_tunnel_info_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct ethnl_tunnel_info_dump_ctx *ctx = (void *)cb->ctx; struct net *net = sock_net(skb->sk); - int s_idx = ctx->pos_idx; - int h, idx = 0; + struct net_device *dev; int ret = 0; void *ehdr; rtnl_lock(); - cb->seq = net->dev_base_seq; - for (h = ctx->pos_hash; h < NETDEV_HASHENTRIES; h++, s_idx = 0) { - struct hlist_head *head; - struct net_device *dev; - - head = &net->dev_index_head[h]; - idx = 0; - hlist_for_each_entry(dev, head, index_hlist) { - if (idx < s_idx) - goto cont; - - ehdr = ethnl_dump_put(skb, cb, - ETHTOOL_MSG_TUNNEL_INFO_GET_REPLY); - if (!ehdr) { - ret = -EMSGSIZE; - goto out; - } - - ret = ethnl_fill_reply_header(skb, dev, ETHTOOL_A_TUNNEL_INFO_HEADER); - if (ret < 0) { - genlmsg_cancel(skb, ehdr); - goto out; - } - - ctx->req_info.dev = dev; - ret = ethnl_tunnel_info_fill_reply(&ctx->req_info, skb); - ctx->req_info.dev = NULL; - if (ret < 0) { - genlmsg_cancel(skb, ehdr); - if (ret == -EOPNOTSUPP) - goto cont; - goto out; - } - genlmsg_end(skb, ehdr); -cont: - idx++; + for_each_netdev_dump(net, dev, ctx->ifindex) { + ehdr = ethnl_dump_put(skb, cb, + ETHTOOL_MSG_TUNNEL_INFO_GET_REPLY); + if (!ehdr) { + ret = -EMSGSIZE; + break; } + + ret = ethnl_fill_reply_header(skb, dev, + ETHTOOL_A_TUNNEL_INFO_HEADER); + if (ret < 0) { + genlmsg_cancel(skb, ehdr); + break; + } + + ctx->req_info.dev = dev; + ret = ethnl_tunnel_info_fill_reply(&ctx->req_info, skb); + ctx->req_info.dev = NULL; + if (ret < 0) { + genlmsg_cancel(skb, ehdr); + if (ret == -EOPNOTSUPP) + continue; + break; + } + genlmsg_end(skb, ehdr); } -out: rtnl_unlock(); - ctx->pos_hash = h; - ctx->pos_idx = idx; - nl_dump_check_consistent(cb, nlmsg_hdr(skb)); - if (ret == -EMSGSIZE && skb->len) return skb->len; return ret; diff --git a/net/handshake/Makefile b/net/handshake/Makefile index 247d73c6ff6e..ef4d9a2112bd 100644 --- a/net/handshake/Makefile +++ b/net/handshake/Makefile @@ -8,6 +8,6 @@ # obj-y += handshake.o -handshake-y := genl.o netlink.o request.o tlshd.o trace.o +handshake-y := alert.o genl.o netlink.o request.o tlshd.o trace.o obj-$(CONFIG_NET_HANDSHAKE_KUNIT_TEST) += handshake-test.o diff --git a/net/handshake/alert.c b/net/handshake/alert.c new file mode 100644 index 000000000000..329d91984683 --- /dev/null +++ b/net/handshake/alert.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Handle the TLS Alert protocol + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2023, Oracle and/or its affiliates. + */ + +#include <linux/types.h> +#include <linux/socket.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/skbuff.h> +#include <linux/inet.h> + +#include <net/sock.h> +#include <net/handshake.h> +#include <net/tls.h> +#include <net/tls_prot.h> + +#include "handshake.h" + +#include <trace/events/handshake.h> + +/** + * tls_alert_send - send a TLS Alert on a kTLS socket + * @sock: open kTLS socket to send on + * @level: TLS Alert level + * @description: TLS Alert description + * + * Returns zero on success or a negative errno. + */ +int tls_alert_send(struct socket *sock, u8 level, u8 description) +{ + u8 record_type = TLS_RECORD_TYPE_ALERT; + u8 buf[CMSG_SPACE(sizeof(record_type))]; + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg; + struct kvec iov; + u8 alert[2]; + int ret; + + trace_tls_alert_send(sock->sk, level, description); + + alert[0] = level; + alert[1] = description; + iov.iov_base = alert; + iov.iov_len = sizeof(alert); + + memset(buf, 0, sizeof(buf)); + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + msg.msg_flags = MSG_DONTWAIT; + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_TLS; + cmsg->cmsg_type = TLS_SET_RECORD_TYPE; + cmsg->cmsg_len = CMSG_LEN(sizeof(record_type)); + memcpy(CMSG_DATA(cmsg), &record_type, sizeof(record_type)); + + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &iov, 1, iov.iov_len); + ret = sock_sendmsg(sock, &msg); + return ret < 0 ? ret : 0; +} + +/** + * tls_get_record_type - Look for TLS RECORD_TYPE information + * @sk: socket (for IP address information) + * @cmsg: incoming message to be parsed + * + * Returns zero or a TLS_RECORD_TYPE value. + */ +u8 tls_get_record_type(const struct sock *sk, const struct cmsghdr *cmsg) +{ + u8 record_type; + + if (cmsg->cmsg_level != SOL_TLS) + return 0; + if (cmsg->cmsg_type != TLS_GET_RECORD_TYPE) + return 0; + + record_type = *((u8 *)CMSG_DATA(cmsg)); + trace_tls_contenttype(sk, record_type); + return record_type; +} +EXPORT_SYMBOL(tls_get_record_type); + +/** + * tls_alert_recv - Parse TLS Alert messages + * @sk: socket (for IP address information) + * @msg: incoming message to be parsed + * @level: OUT - TLS AlertLevel value + * @description: OUT - TLS AlertDescription value + * + */ +void tls_alert_recv(const struct sock *sk, const struct msghdr *msg, + u8 *level, u8 *description) +{ + const struct kvec *iov; + u8 *data; + + iov = msg->msg_iter.kvec; + data = iov->iov_base; + *level = data[0]; + *description = data[1]; + + trace_tls_alert_recv(sk, *level, *description); +} +EXPORT_SYMBOL(tls_alert_recv); diff --git a/net/handshake/handshake.h b/net/handshake/handshake.h index 4dac965c99df..a48163765a7a 100644 --- a/net/handshake/handshake.h +++ b/net/handshake/handshake.h @@ -41,8 +41,11 @@ struct handshake_req { enum hr_flags_bits { HANDSHAKE_F_REQ_COMPLETED, + HANDSHAKE_F_REQ_SESSION, }; +struct genl_info; + /* Invariants for all handshake requests for one transport layer * security protocol */ @@ -63,6 +66,9 @@ enum hp_flags_bits { HANDSHAKE_F_PROTO_NOTIFY, }; +/* alert.c */ +int tls_alert_send(struct socket *sock, u8 level, u8 description); + /* netlink.c */ int handshake_genl_notify(struct net *net, const struct handshake_proto *proto, gfp_t flags); diff --git a/net/handshake/tlshd.c b/net/handshake/tlshd.c index b735f5cced2f..bbfb4095ddd6 100644 --- a/net/handshake/tlshd.c +++ b/net/handshake/tlshd.c @@ -18,6 +18,7 @@ #include <net/sock.h> #include <net/handshake.h> #include <net/genetlink.h> +#include <net/tls_prot.h> #include <uapi/linux/keyctl.h> #include <uapi/linux/handshake.h> @@ -100,6 +101,9 @@ static void tls_handshake_done(struct handshake_req *req, if (info) tls_handshake_remote_peerids(treq, info); + if (!status) + set_bit(HANDSHAKE_F_REQ_SESSION, &req->hr_flags); + treq->th_consumer_done(treq->th_consumer_data, -status, treq->th_peerid[0]); } @@ -424,3 +428,22 @@ bool tls_handshake_cancel(struct sock *sk) return handshake_req_cancel(sk); } EXPORT_SYMBOL(tls_handshake_cancel); + +/** + * tls_handshake_close - send a Closure alert + * @sock: an open socket + * + */ +void tls_handshake_close(struct socket *sock) +{ + struct handshake_req *req; + + req = handshake_req_hash_lookup(sock->sk); + if (!req) + return; + if (!test_and_clear_bit(HANDSHAKE_F_REQ_SESSION, &req->hr_flags)) + return; + tls_alert_send(sock, TLS_ALERT_LEVEL_WARNING, + TLS_ALERT_DESC_CLOSE_NOTIFY); +} +EXPORT_SYMBOL(tls_handshake_close); diff --git a/net/handshake/trace.c b/net/handshake/trace.c index 1c4d8e27e17a..44432d0857b9 100644 --- a/net/handshake/trace.c +++ b/net/handshake/trace.c @@ -8,8 +8,10 @@ */ #include <linux/types.h> +#include <linux/ipv6.h> #include <net/sock.h> +#include <net/inet_sock.h> #include <net/netlink.h> #include <net/genetlink.h> diff --git a/net/hsr/hsr_netlink.h b/net/hsr/hsr_netlink.h index 501552d9753b..8c99e64e1cea 100644 --- a/net/hsr/hsr_netlink.h +++ b/net/hsr/hsr_netlink.h @@ -23,7 +23,5 @@ void __exit hsr_netlink_exit(void); void hsr_nl_ringerror(struct hsr_priv *hsr, unsigned char addr[ETH_ALEN], struct hsr_port *port); void hsr_nl_nodedown(struct hsr_priv *hsr, unsigned char addr[ETH_ALEN]); -void hsr_nl_framedrop(int dropcount, int dev_idx); -void hsr_nl_linkdown(int dev_idx); #endif /* __HSR_NETLINK_H */ diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 4406d796cc2f..39dcccf0f174 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -51,8 +51,6 @@ static bool is_unsupported(u32 member_offset) return false; } -extern struct btf *btf_vmlinux; - static bool bpf_tcp_ca_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index f95142e56da0..93f14d39fef6 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -1152,41 +1152,64 @@ static bool ipv4_good_nh(const struct fib_nh *nh) return !!(state & NUD_VALID); } -static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash) +static bool nexthop_is_good_nh(const struct nexthop *nh) +{ + struct nh_info *nhi = rcu_dereference(nh->nh_info); + + switch (nhi->family) { + case AF_INET: + return ipv4_good_nh(&nhi->fib_nh); + case AF_INET6: + return ipv6_good_nh(&nhi->fib6_nh); + } + + return false; +} + +static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash) { - struct nexthop *rc = NULL; int i; - for (i = 0; i < nhg->num_nh; ++i) { + for (i = 0; i < nhg->num_nh; i++) { struct nh_grp_entry *nhge = &nhg->nh_entries[i]; - struct nh_info *nhi; if (hash > atomic_read(&nhge->hthr.upper_bound)) continue; - nhi = rcu_dereference(nhge->nh->nh_info); - if (nhi->fdb_nh) - return nhge->nh; + return nhge->nh; + } + + WARN_ON_ONCE(1); + return NULL; +} + +static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash) +{ + struct nexthop *rc = NULL; + int i; + + if (nhg->fdb_nh) + return nexthop_select_path_fdb(nhg, hash); + + for (i = 0; i < nhg->num_nh; ++i) { + struct nh_grp_entry *nhge = &nhg->nh_entries[i]; /* nexthops always check if it is good and does * not rely on a sysctl for this behavior */ - switch (nhi->family) { - case AF_INET: - if (ipv4_good_nh(&nhi->fib_nh)) - return nhge->nh; - break; - case AF_INET6: - if (ipv6_good_nh(&nhi->fib6_nh)) - return nhge->nh; - break; - } + if (!nexthop_is_good_nh(nhge->nh)) + continue; if (!rc) rc = nhge->nh; + + if (hash > atomic_read(&nhge->hthr.upper_bound)) + continue; + + return nhge->nh; } - return rc; + return rc ? : nhg->nh_entries[0].nh; } static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 8ed52e1e3c99..aca5620cf3ba 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -457,6 +457,7 @@ void tcp_init_sock(struct sock *sk) WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1])); WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1])); + tcp_scaling_ratio_init(sk); set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); sk_sockets_allocated_inc(sk); @@ -1700,7 +1701,7 @@ EXPORT_SYMBOL(tcp_peek_len); /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ int tcp_set_rcvlowat(struct sock *sk, int val) { - int cap; + int space, cap; if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) cap = sk->sk_rcvbuf >> 1; @@ -1715,10 +1716,10 @@ int tcp_set_rcvlowat(struct sock *sk, int val) if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) return 0; - val <<= 1; - if (val > sk->sk_rcvbuf) { - WRITE_ONCE(sk->sk_rcvbuf, val); - tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val); + space = tcp_space_from_win(sk, val); + if (space > sk->sk_rcvbuf) { + WRITE_ONCE(sk->sk_rcvbuf, space); + tcp_sk(sk)->window_clamp = val; } return 0; } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 57c8af1859c1..670c3dab24f2 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -237,6 +237,16 @@ static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) */ len = skb_shinfo(skb)->gso_size ? : skb->len; if (len >= icsk->icsk_ack.rcv_mss) { + /* Note: divides are still a bit expensive. + * For the moment, only adjust scaling_ratio + * when we update icsk_ack.rcv_mss. + */ + if (unlikely(len != icsk->icsk_ack.rcv_mss)) { + u64 val = (u64)skb->len << TCP_RMEM_TO_WIN_SCALE; + + do_div(val, skb->truesize); + tcp_sk(sk)->scaling_ratio = val ? val : 1; + } icsk->icsk_ack.rcv_mss = min_t(unsigned int, len, tcp_sk(sk)->advmss); /* Account for possibly-removed options */ @@ -287,7 +297,7 @@ static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks) icsk->icsk_ack.quick = quickacks; } -void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks) +static void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks) { struct inet_connection_sock *icsk = inet_csk(sk); @@ -295,7 +305,6 @@ void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks) inet_csk_exit_pingpong_mode(sk); icsk->icsk_ack.ato = TCP_ATO_MIN; } -EXPORT_SYMBOL(tcp_enter_quickack_mode); /* Send ACKs quickly, if "quick" count is not exhausted * and the session is not interactive. @@ -727,8 +736,8 @@ void tcp_rcv_space_adjust(struct sock *sk) if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) && !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { - int rcvmem, rcvbuf; u64 rcvwin, grow; + int rcvbuf; /* minimal window to cope with packet losses, assuming * steady state. Add some cushion because of small variations. @@ -740,12 +749,7 @@ void tcp_rcv_space_adjust(struct sock *sk) do_div(grow, tp->rcvq_space.space); rcvwin += (grow << 1); - rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER); - while (tcp_win_from_space(sk, rcvmem) < tp->advmss) - rcvmem += 128; - - do_div(rcvwin, tp->advmss); - rcvbuf = min_t(u64, rcvwin * rcvmem, + rcvbuf = min_t(u64, tcp_space_from_win(sk, rcvwin), READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); if (rcvbuf > sk->sk_rcvbuf) { WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); @@ -4308,10 +4312,16 @@ static inline bool tcp_paws_discard(const struct sock *sk, * (borrowed from freebsd) */ -static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq) +static enum skb_drop_reason tcp_sequence(const struct tcp_sock *tp, + u32 seq, u32 end_seq) { - return !before(end_seq, tp->rcv_wup) && - !after(seq, tp->rcv_nxt + tcp_receive_window(tp)); + if (before(end_seq, tp->rcv_wup)) + return SKB_DROP_REASON_TCP_OLD_SEQUENCE; + + if (after(seq, tp->rcv_nxt + tcp_receive_window(tp))) + return SKB_DROP_REASON_TCP_INVALID_SEQUENCE; + + return SKB_NOT_DROPPED_YET; } /* When we get a reset we do this. */ @@ -5734,7 +5744,8 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, } /* Step 1: check sequence number */ - if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) { + reason = tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); + if (reason) { /* RFC793, page 37: "In all states except SYN-SENT, all reset * (RST) segments are validated by checking their SEQ-fields." * And page 69: "If an incoming segment is not acceptable, @@ -5751,7 +5762,6 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, } else if (tcp_reset_check(sk, skb)) { goto reset; } - SKB_DR_SET(reason, TCP_INVALID_SEQUENCE); goto discard; } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index a59cc4b83861..5b18a048f613 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -57,6 +57,7 @@ #include <linux/init.h> #include <linux/times.h> #include <linux/slab.h> +#include <linux/sched.h> #include <net/net_namespace.h> #include <net/icmp.h> @@ -2448,6 +2449,8 @@ static void *established_get_first(struct seq_file *seq) struct hlist_nulls_node *node; spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket); + cond_resched(); + /* Lockless fast path for the common case of empty buckets */ if (empty_bucket(hinfo, st)) continue; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index abfa860367aa..1ee9e56dc79a 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1557,7 +1557,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) spin_unlock(&list->lock); if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk); + INDIRECT_CALL_1(sk->sk_data_ready, sock_def_readable, sk); busylock_release(busy); return 0; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 94cec2075eee..5184bd0ceb12 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -202,6 +202,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .ra_defrtr_metric = IP6_RT_PRIO_USER, .accept_ra_from_local = 0, .accept_ra_min_hop_limit= 1, + .accept_ra_min_lft = 0, .accept_ra_pinfo = 1, #ifdef CONFIG_IPV6_ROUTER_PREF .accept_ra_rtr_pref = 1, @@ -262,6 +263,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .ra_defrtr_metric = IP6_RT_PRIO_USER, .accept_ra_from_local = 0, .accept_ra_min_hop_limit= 1, + .accept_ra_min_lft = 0, .accept_ra_pinfo = 1, #ifdef CONFIG_IPV6_ROUTER_PREF .accept_ra_rtr_pref = 1, @@ -1061,20 +1063,28 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg, struct fib6_info *f6i = NULL; int err = 0; - if (addr_type == IPV6_ADDR_ANY || - (addr_type & IPV6_ADDR_MULTICAST && - !(cfg->ifa_flags & IFA_F_MCAUTOJOIN)) || - (!(idev->dev->flags & IFF_LOOPBACK) && - !netif_is_l3_master(idev->dev) && - addr_type & IPV6_ADDR_LOOPBACK)) + if (addr_type == IPV6_ADDR_ANY) { + NL_SET_ERR_MSG_MOD(extack, "Invalid address"); return ERR_PTR(-EADDRNOTAVAIL); + } else if (addr_type & IPV6_ADDR_MULTICAST && + !(cfg->ifa_flags & IFA_F_MCAUTOJOIN)) { + NL_SET_ERR_MSG_MOD(extack, "Cannot assign multicast address without \"IFA_F_MCAUTOJOIN\" flag"); + return ERR_PTR(-EADDRNOTAVAIL); + } else if (!(idev->dev->flags & IFF_LOOPBACK) && + !netif_is_l3_master(idev->dev) && + addr_type & IPV6_ADDR_LOOPBACK) { + NL_SET_ERR_MSG_MOD(extack, "Cannot assign loopback address on this device"); + return ERR_PTR(-EADDRNOTAVAIL); + } if (idev->dead) { - err = -ENODEV; /*XXX*/ + NL_SET_ERR_MSG_MOD(extack, "device is going away"); + err = -ENODEV; goto out; } if (idev->cnf.disable_ipv6) { + NL_SET_ERR_MSG_MOD(extack, "IPv6 is disabled on this device"); err = -EACCES; goto out; } @@ -1101,7 +1111,7 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg, goto out; } - f6i = addrconf_f6i_alloc(net, idev, cfg->pfx, false, gfp_flags); + f6i = addrconf_f6i_alloc(net, idev, cfg->pfx, false, gfp_flags, extack); if (IS_ERR(f6i)) { err = PTR_ERR(f6i); f6i = NULL; @@ -2731,6 +2741,9 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) return; } + if (valid_lft != 0 && valid_lft < in6_dev->cnf.accept_ra_min_lft) + return; + /* * Two things going on here: * 1) Add routes for on-link prefixes @@ -2925,30 +2938,40 @@ static int inet6_addr_add(struct net *net, int ifindex, ASSERT_RTNL(); - if (cfg->plen > 128) + if (cfg->plen > 128) { + NL_SET_ERR_MSG_MOD(extack, "Invalid prefix length"); return -EINVAL; + } /* check the lifetime */ - if (!cfg->valid_lft || cfg->preferred_lft > cfg->valid_lft) + if (!cfg->valid_lft || cfg->preferred_lft > cfg->valid_lft) { + NL_SET_ERR_MSG_MOD(extack, "address lifetime invalid"); return -EINVAL; + } - if (cfg->ifa_flags & IFA_F_MANAGETEMPADDR && cfg->plen != 64) + if (cfg->ifa_flags & IFA_F_MANAGETEMPADDR && cfg->plen != 64) { + NL_SET_ERR_MSG_MOD(extack, "address with \"mngtmpaddr\" flag must have a prefix length of 64"); return -EINVAL; + } dev = __dev_get_by_index(net, ifindex); if (!dev) return -ENODEV; idev = addrconf_add_dev(dev); - if (IS_ERR(idev)) + if (IS_ERR(idev)) { + NL_SET_ERR_MSG_MOD(extack, "IPv6 is disabled on this device"); return PTR_ERR(idev); + } if (cfg->ifa_flags & IFA_F_MCAUTOJOIN) { int ret = ipv6_mc_config(net->ipv6.mc_autojoin_sk, true, cfg->pfx, ifindex); - if (ret < 0) + if (ret < 0) { + NL_SET_ERR_MSG_MOD(extack, "Multicast auto join failed"); return ret; + } } cfg->scope = ipv6_addr_scope(cfg->pfx); @@ -3005,22 +3028,29 @@ static int inet6_addr_add(struct net *net, int ifindex, } static int inet6_addr_del(struct net *net, int ifindex, u32 ifa_flags, - const struct in6_addr *pfx, unsigned int plen) + const struct in6_addr *pfx, unsigned int plen, + struct netlink_ext_ack *extack) { struct inet6_ifaddr *ifp; struct inet6_dev *idev; struct net_device *dev; - if (plen > 128) + if (plen > 128) { + NL_SET_ERR_MSG_MOD(extack, "Invalid prefix length"); return -EINVAL; + } dev = __dev_get_by_index(net, ifindex); - if (!dev) + if (!dev) { + NL_SET_ERR_MSG_MOD(extack, "Unable to find the interface"); return -ENODEV; + } idev = __in6_dev_get(dev); - if (!idev) + if (!idev) { + NL_SET_ERR_MSG_MOD(extack, "IPv6 is disabled on this device"); return -ENXIO; + } read_lock_bh(&idev->lock); list_for_each_entry(ifp, &idev->addr_list, if_list) { @@ -3043,6 +3073,8 @@ static int inet6_addr_del(struct net *net, int ifindex, u32 ifa_flags, } } read_unlock_bh(&idev->lock); + + NL_SET_ERR_MSG_MOD(extack, "address not found"); return -EADDRNOTAVAIL; } @@ -3085,7 +3117,7 @@ int addrconf_del_ifaddr(struct net *net, void __user *arg) rtnl_lock(); err = inet6_addr_del(net, ireq.ifr6_ifindex, 0, &ireq.ifr6_addr, - ireq.ifr6_prefixlen); + ireq.ifr6_prefixlen, NULL); rtnl_unlock(); return err; } @@ -3488,7 +3520,7 @@ static int fixup_permanent_addr(struct net *net, struct fib6_info *f6i, *prev; f6i = addrconf_f6i_alloc(net, idev, &ifp->addr, false, - GFP_ATOMIC); + GFP_ATOMIC, NULL); if (IS_ERR(f6i)) return PTR_ERR(f6i); @@ -4698,7 +4730,7 @@ inet6_rtm_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh, ifa_flags &= IFA_F_MANAGETEMPADDR; return inet6_addr_del(net, ifm->ifa_index, ifa_flags, pfx, - ifm->ifa_prefixlen); + ifm->ifa_prefixlen, extack); } static int modify_prefix_route(struct inet6_ifaddr *ifp, @@ -4903,8 +4935,10 @@ inet6_rtm_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh, } dev = __dev_get_by_index(net, ifm->ifa_index); - if (!dev) + if (!dev) { + NL_SET_ERR_MSG_MOD(extack, "Unable to find the interface"); return -ENODEV; + } if (tb[IFA_FLAGS]) cfg.ifa_flags = nla_get_u32(tb[IFA_FLAGS]); @@ -4939,10 +4973,12 @@ inet6_rtm_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh, } if (nlh->nlmsg_flags & NLM_F_EXCL || - !(nlh->nlmsg_flags & NLM_F_REPLACE)) + !(nlh->nlmsg_flags & NLM_F_REPLACE)) { + NL_SET_ERR_MSG_MOD(extack, "address already assigned"); err = -EEXIST; - else + } else { err = inet6_addr_modify(net, ifa, &cfg); + } in6_ifa_put(ifa); @@ -5602,6 +5638,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_IOAM6_ID_WIDE] = cnf->ioam6_id_wide; array[DEVCONF_NDISC_EVICT_NOCARRIER] = cnf->ndisc_evict_nocarrier; array[DEVCONF_ACCEPT_UNTRACKED_NA] = cnf->accept_untracked_na; + array[DEVCONF_ACCEPT_RA_MIN_LFT] = cnf->accept_ra_min_lft; } static inline size_t inet6_ifla6_size(void) @@ -6796,6 +6833,13 @@ static const struct ctl_table addrconf_sysctl[] = { .proc_handler = proc_dointvec, }, { + .procname = "accept_ra_min_lft", + .data = &ipv6_devconf.accept_ra_min_lft, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { .procname = "accept_ra_pinfo", .data = &ipv6_devconf.accept_ra_pinfo, .maxlen = sizeof(int), diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 5d593ddc0347..9f9c4b838664 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -102,9 +102,9 @@ bool ipv6_mod_enabled(void) } EXPORT_SYMBOL_GPL(ipv6_mod_enabled); -static __inline__ struct ipv6_pinfo *inet6_sk_generic(struct sock *sk) +static struct ipv6_pinfo *inet6_sk_generic(struct sock *sk) { - const int offset = sk->sk_prot->obj_size - sizeof(struct ipv6_pinfo); + const int offset = sk->sk_prot->ipv6_pinfo_offset; return (struct ipv6_pinfo *)(((u8 *)sk) + offset); } diff --git a/net/ipv6/anycast.c b/net/ipv6/anycast.c index dacdea7fcb62..bb17f484ee2c 100644 --- a/net/ipv6/anycast.c +++ b/net/ipv6/anycast.c @@ -305,7 +305,7 @@ int __ipv6_dev_ac_inc(struct inet6_dev *idev, const struct in6_addr *addr) } net = dev_net(idev->dev); - f6i = addrconf_f6i_alloc(net, idev, addr, true, GFP_ATOMIC); + f6i = addrconf_f6i_alloc(net, idev, addr, true, GFP_ATOMIC, NULL); if (IS_ERR(f6i)) { err = PTR_ERR(f6i); goto out; diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index 9b6818453afe..d80d6024cafa 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -38,10 +38,11 @@ static bool ipv6_mapped_addr_any(const struct in6_addr *a) return ipv6_addr_v4mapped(a) && (a->s6_addr32[3] == 0); } -static void ip6_datagram_flow_key_init(struct flowi6 *fl6, struct sock *sk) +static void ip6_datagram_flow_key_init(struct flowi6 *fl6, + const struct sock *sk) { - struct inet_sock *inet = inet_sk(sk); - struct ipv6_pinfo *np = inet6_sk(sk); + const struct inet_sock *inet = inet_sk(sk); + const struct ipv6_pinfo *np = inet6_sk(sk); int oif = sk->sk_bound_dev_if; memset(fl6, 0, sizeof(*fl6)); diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 202fc3aaa83c..f4bfccae003c 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -612,8 +612,6 @@ looped_back: kfree(buf); - skb_dst_drop(skb); - ip6_route_input(skb); if (skb_dst(skb)->error) { diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 65fa5014bc85..6d88f5248c1f 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -1034,11 +1034,9 @@ drop_no_count: return 0; } -void icmpv6_flow_init(struct sock *sk, struct flowi6 *fl6, - u8 type, +void icmpv6_flow_init(const struct sock *sk, struct flowi6 *fl6, u8 type, const struct in6_addr *saddr, - const struct in6_addr *daddr, - int oif) + const struct in6_addr *daddr, int oif) { memset(fl6, 0, sizeof(*fl6)); fl6->saddr = *saddr; diff --git a/net/ipv6/ila/ila_main.c b/net/ipv6/ila/ila_main.c index 3faf62530d6a..69caed07315f 100644 --- a/net/ipv6/ila/ila_main.c +++ b/net/ipv6/ila/ila_main.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 #include <net/genetlink.h> -#include <net/ila.h> #include <net/netns/generic.h> #include <uapi/linux/genetlink.h> #include "ila.h" diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c index bee45dfeb187..67e8c9440977 100644 --- a/net/ipv6/ila/ila_xlat.c +++ b/net/ipv6/ila/ila_xlat.c @@ -5,7 +5,6 @@ #include <linux/rhashtable.h> #include <linux/vmalloc.h> #include <net/genetlink.h> -#include <net/ila.h> #include <net/netns/generic.h> #include <uapi/linux/genetlink.h> #include "ila.h" diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 1e8c90e97608..bc96559bbf0f 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1693,7 +1693,10 @@ alloc_new_skb: fraglen = datalen + fragheaderlen; copy = datalen - transhdrlen - fraggap - pagedlen; - if (copy < 0) { + /* [!] NOTE: copy may be negative if pagedlen>0 + * because then the equation may reduces to -fraggap. + */ + if (copy < 0 && !(flags & MSG_SPLICE_PAGES)) { err = -EINVAL; goto error; } @@ -1744,6 +1747,8 @@ alloc_new_skb: err = -EFAULT; kfree_skb(skb); goto error; + } else if (flags & MSG_SPLICE_PAGES) { + copy = 0; } offset += copy; @@ -1791,6 +1796,10 @@ alloc_new_skb: } else if (flags & MSG_SPLICE_PAGES) { struct msghdr *msg = from; + err = -EIO; + if (WARN_ON_ONCE(copy > msg->msg_iter.count)) + goto error; + err = skb_splice_from_iter(skb, &msg->msg_iter, copy, sk->sk_allocation); if (err < 0) diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 714cdc9e2b8e..5ce25bcb9974 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -1699,11 +1699,9 @@ mld_scount(struct ifmcaddr6 *pmc, int type, int gdeleted, int sdeleted) return scount; } -static void ip6_mc_hdr(struct sock *sk, struct sk_buff *skb, - struct net_device *dev, - const struct in6_addr *saddr, - const struct in6_addr *daddr, - int proto, int len) +static void ip6_mc_hdr(const struct sock *sk, struct sk_buff *skb, + struct net_device *dev, const struct in6_addr *saddr, + const struct in6_addr *daddr, int proto, int len) { struct ipv6hdr *hdr; diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index 18634ebd20a4..c394799be38f 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1266,10 +1266,6 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb) } #endif - /* - * set the RA_RECV flag in the interface - */ - in6_dev = __in6_dev_get(skb->dev); if (!in6_dev) { ND_PRINTK(0, err, "RA: can't find inet6 device for %s\n", @@ -1327,6 +1323,14 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb) goto skip_defrtr; } + lifetime = ntohs(ra_msg->icmph.icmp6_rt_lifetime); + if (lifetime != 0 && lifetime < in6_dev->cnf.accept_ra_min_lft) { + ND_PRINTK(2, info, + "RA: router lifetime (%ds) is too short: %s\n", + lifetime, skb->dev->name); + goto skip_defrtr; + } + /* Do not accept RA with source-addr found on local machine unless * accept_ra_from_local is set to true. */ @@ -1339,8 +1343,6 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb) goto skip_defrtr; } - lifetime = ntohs(ra_msg->icmph.icmp6_rt_lifetime); - #ifdef CONFIG_IPV6_ROUTER_PREF pref = ra_msg->icmph.icmp6_router_pref; /* 10b is handled as if it were 00b (medium) */ @@ -1516,6 +1518,9 @@ skip_linkparms: if (ri->prefix_len == 0 && !in6_dev->cnf.accept_ra_defrtr) continue; + if (ri->lifetime != 0 && + ntohl(ri->lifetime) < in6_dev->cnf.accept_ra_min_lft) + continue; if (ri->prefix_len < in6_dev->cnf.accept_ra_rt_info_min_plen) continue; if (ri->prefix_len > in6_dev->cnf.accept_ra_rt_info_max_plen) diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index c2c291827a2c..1b2772834972 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -215,6 +215,7 @@ struct proto pingv6_prot = { .get_port = ping_get_port, .put_port = ping_unhash, .obj_size = sizeof(struct raw6_sock), + .ipv6_pinfo_offset = offsetof(struct raw6_sock, inet6), }; EXPORT_SYMBOL_GPL(pingv6_prot); diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 49381f35b623..ea16734f5e1f 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -1216,6 +1216,7 @@ struct proto rawv6_prot = { .hash = raw_hash_sk, .unhash = raw_unhash_sk, .obj_size = sizeof(struct raw6_sock), + .ipv6_pinfo_offset = offsetof(struct raw6_sock, inet6), .useroffset = offsetof(struct raw6_sock, filter), .usersize = sizeof_field(struct raw6_sock, filter), .h.raw_hash = &raw_v6_hashinfo, diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 56a55585eb79..10751df16dab 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4544,7 +4544,8 @@ static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff struct fib6_info *addrconf_f6i_alloc(struct net *net, struct inet6_dev *idev, const struct in6_addr *addr, - bool anycast, gfp_t gfp_flags) + bool anycast, gfp_t gfp_flags, + struct netlink_ext_ack *extack) { struct fib6_config cfg = { .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, @@ -4566,7 +4567,7 @@ struct fib6_info *addrconf_f6i_alloc(struct net *net, cfg.fc_flags |= RTF_LOCAL; } - f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); + f6i = ip6_route_info_create(&cfg, gfp_flags, extack); if (!IS_ERR(f6i)) { f6i->dst_nocount = true; diff --git a/net/ipv6/rpl_iptunnel.c b/net/ipv6/rpl_iptunnel.c index b1c028df686e..a013b92cbb86 100644 --- a/net/ipv6/rpl_iptunnel.c +++ b/net/ipv6/rpl_iptunnel.c @@ -272,8 +272,6 @@ static int rpl_input(struct sk_buff *skb) dst = dst_cache_get(&rlwt->cache); preempt_enable(); - skb_dst_drop(skb); - if (!dst) { ip6_route_input(skb); dst = skb_dst(skb); @@ -284,6 +282,7 @@ static int rpl_input(struct sk_buff *skb) preempt_enable(); } } else { + skb_dst_drop(skb); skb_dst_set(skb, dst); } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 6e86721e1cdb..3a88545a265d 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -2176,6 +2176,7 @@ struct proto tcpv6_prot = { .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), .max_header = MAX_TCP_HEADER, .obj_size = sizeof(struct tcp6_sock), + .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6), .slab_flags = SLAB_TYPESAFE_BY_RCU, .twsk_prot = &tcp6_timewait_sock_ops, .rsk_prot = &tcp6_request_sock_ops, diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index f787e6b8424c..486d893b8e3c 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -1802,6 +1802,7 @@ struct proto udpv6_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min), .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min), .obj_size = sizeof(struct udp6_sock), + .ipv6_pinfo_offset = offsetof(struct udp6_sock, inet6), .h.udp_table = NULL, .diag_destroy = udp_abort, }; diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c index 8e010d07917a..267d491e9707 100644 --- a/net/ipv6/udplite.c +++ b/net/ipv6/udplite.c @@ -67,6 +67,7 @@ struct proto udplitev6_prot = { .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min), .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min), .obj_size = sizeof(struct udp6_sock), + .ipv6_pinfo_offset = offsetof(struct udp6_sock, inet6), .h.udp_table = &udplite_table, }; diff --git a/net/key/af_key.c b/net/key/af_key.c index ede3c6a60353..542439b6a59c 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -1281,7 +1281,6 @@ static struct xfrm_state * pfkey_msg2xfrm_state(struct net *net, ext_hdrs[SADB_X_EXT_NAT_T_DPORT-1]; natt->encap_dport = n_port->sadb_x_nat_t_port_port; } - memset(&natt->encap_oa, 0, sizeof(natt->encap_oa)); } err = xfrm_init_state(x); diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index ff78217f0cb1..ed8ebb6f5909 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -36,9 +36,6 @@ struct l2tp_ip6_sock { u32 conn_id; u32 peer_conn_id; - /* ipv6_pinfo has to be the last member of l2tp_ip6_sock, see - * inet6_sk_generic - */ struct ipv6_pinfo inet6; }; @@ -730,6 +727,7 @@ static struct proto l2tp_ip6_prot = { .hash = l2tp_ip6_hash, .unhash = l2tp_ip6_unhash, .obj_size = sizeof(struct l2tp_ip6_sock), + .ipv6_pinfo_offset = offsetof(struct l2tp_ip6_sock, inet6), }; static const struct proto_ops l2tp_ip6_ops = { diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 3317d1cca156..65ee949a8a44 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -90,6 +90,7 @@ static int __mptcp_socket_create(struct mptcp_sock *msk) if (err) return err; + msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; WRITE_ONCE(msk->first, ssock->sk); WRITE_ONCE(msk->subflow, ssock); subflow = mptcp_subflow_ctx(ssock->sk); @@ -1881,6 +1882,7 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; + u8 scaling_ratio = U8_MAX; u32 time, advmss = 1; u64 rtt_us, mstamp; @@ -1911,9 +1913,11 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) rtt_us = max(sf_rtt_us, rtt_us); advmss = max(sf_advmss, advmss); + scaling_ratio = min(tp->scaling_ratio, scaling_ratio); } msk->rcvq_space.rtt_us = rtt_us; + msk->scaling_ratio = scaling_ratio; if (time < (rtt_us >> 3) || rtt_us == 0) return; @@ -1922,8 +1926,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) && !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { - int rcvmem, rcvbuf; u64 rcvwin, grow; + int rcvbuf; rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; @@ -1932,18 +1936,13 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) do_div(grow, msk->rcvq_space.space); rcvwin += (grow << 1); - rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); - while (tcp_win_from_space(sk, rcvmem) < advmss) - rcvmem += 128; - - do_div(rcvwin, advmss); - rcvbuf = min_t(u64, rcvwin * rcvmem, + rcvbuf = min_t(u64, __tcp_space_from_win(scaling_ratio, rcvwin), READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); if (rcvbuf > sk->sk_rcvbuf) { u32 window_clamp; - window_clamp = tcp_win_from_space(sk, rcvbuf); + window_clamp = __tcp_win_from_space(scaling_ratio, rcvbuf); WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); /* Make subflows follow along. If we do not do this, we @@ -3987,6 +3986,7 @@ int __init mptcp_proto_v6_init(void) strcpy(mptcp_v6_prot.name, "MPTCPv6"); mptcp_v6_prot.slab = NULL; mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); + mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np); err = proto_register(&mptcp_v6_prot, 1); if (err) diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 37fbe22e2433..795f422e8597 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -321,6 +321,7 @@ struct mptcp_sock { u64 time; /* start time of measurement window */ u64 rtt_us; /* last maximum rtt of subflows */ } rcvq_space; + u8 scaling_ratio; u32 subflow_id; u32 setsockopt_seq; @@ -351,9 +352,14 @@ static inline int __mptcp_rmem(const struct sock *sk) return atomic_read(&sk->sk_rmem_alloc) - READ_ONCE(mptcp_sk(sk)->rmem_released); } +static inline int mptcp_win_from_space(const struct sock *sk, int space) +{ + return __tcp_win_from_space(mptcp_sk(sk)->scaling_ratio, space); +} + static inline int __mptcp_space(const struct sock *sk) { - return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk)); + return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk)); } static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 9ee3b7abbaf6..ad7080fabb2f 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -1359,7 +1359,7 @@ void mptcp_space(const struct sock *ssk, int *space, int *full_space) const struct sock *sk = subflow->conn; *space = __mptcp_space(sk); - *full_space = tcp_full_space(sk); + *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); } void __mptcp_error_report(struct sock *sk) diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 992393102d5f..9f6f2e643575 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -1756,7 +1756,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, cnet = nf_ct_pernet(net); if (cnet->expect_count) { spin_lock_bh(&nf_conntrack_expect_lock); - exp = nf_ct_find_expectation(net, zone, tuple); + exp = nf_ct_find_expectation(net, zone, tuple, !tmpl || nf_ct_is_confirmed(tmpl)); if (exp) { /* Welcome, Mr. Bond. We've been expecting you... */ __set_bit(IPS_EXPECTED_BIT, &ct->status); diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 96948e98ec53..81ca348915c9 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -171,7 +171,7 @@ EXPORT_SYMBOL_GPL(nf_ct_expect_find_get); struct nf_conntrack_expect * nf_ct_find_expectation(struct net *net, const struct nf_conntrack_zone *zone, - const struct nf_conntrack_tuple *tuple) + const struct nf_conntrack_tuple *tuple, bool unlink) { struct nf_conntrack_net *cnet = nf_ct_pernet(net); struct nf_conntrack_expect *i, *exp = NULL; @@ -211,7 +211,7 @@ nf_ct_find_expectation(struct net *net, !refcount_inc_not_zero(&exp->master->ct_general.use))) return NULL; - if (exp->flags & NF_CT_EXPECT_PERMANENT) { + if (exp->flags & NF_CT_EXPECT_PERMANENT || !unlink) { refcount_inc(&exp->use); return exp; } else if (del_timer(&exp->timeout)) { diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 69c8c8c7e9b8..334db22199c1 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1321,15 +1321,11 @@ static int ctnetlink_parse_tuple_ip(struct nlattr *attr, struct nlattr *tb[CTA_IP_MAX+1]; int ret = 0; - ret = nla_parse_nested_deprecated(tb, CTA_IP_MAX, attr, NULL, NULL); + ret = nla_parse_nested_deprecated(tb, CTA_IP_MAX, attr, + cta_ip_nla_policy, NULL); if (ret < 0) return ret; - ret = nla_validate_nested_deprecated(attr, CTA_IP_MAX, - cta_ip_nla_policy, NULL); - if (ret) - return ret; - switch (tuple->src.l3num) { case NFPROTO_IPV4: ret = ipv4_nlattr_to_tuple(tb, tuple, flags); diff --git a/net/netfilter/nf_conntrack_proto_dccp.c b/net/netfilter/nf_conntrack_proto_dccp.c index d4fd626d2b8c..e2db1f4ec2df 100644 --- a/net/netfilter/nf_conntrack_proto_dccp.c +++ b/net/netfilter/nf_conntrack_proto_dccp.c @@ -69,6 +69,7 @@ #define DCCP_MSL (2 * 60 * HZ) +#ifdef CONFIG_NF_CONNTRACK_PROCFS static const char * const dccp_state_names[] = { [CT_DCCP_NONE] = "NONE", [CT_DCCP_REQUEST] = "REQUEST", @@ -81,6 +82,7 @@ static const char * const dccp_state_names[] = { [CT_DCCP_IGNORE] = "IGNORE", [CT_DCCP_INVALID] = "INVALID", }; +#endif #define sNO CT_DCCP_NONE #define sRQ CT_DCCP_REQUEST diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 1c26f03fc661..a010b25076ca 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -34,7 +34,7 @@ static void nf_flow_rule_lwt_match(struct nf_flow_match *match, { struct nf_flow_key *mask = &match->mask; struct nf_flow_key *key = &match->key; - unsigned int enc_keys; + unsigned long long enc_keys; if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX)) return; @@ -43,8 +43,8 @@ static void nf_flow_rule_lwt_match(struct nf_flow_match *match, NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id); mask->enc_key_id.keyid = 0xffffffff; - enc_keys = BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | - BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL); + enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL); if (ip_tunnel_info_af(tun_info) == AF_INET) { NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, @@ -55,7 +55,7 @@ static void nf_flow_rule_lwt_match(struct nf_flow_match *match, mask->enc_ipv4.src = 0xffffffff; if (key->enc_ipv4.dst) mask->enc_ipv4.dst = 0xffffffff; - enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); + enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; } else { memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst, @@ -70,7 +70,7 @@ static void nf_flow_rule_lwt_match(struct nf_flow_match *match, sizeof(struct in6_addr))) memset(&mask->enc_ipv6.dst, 0xff, sizeof(struct in6_addr)); - enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS); + enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS); key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; } @@ -163,14 +163,14 @@ static int nf_flow_rule_match(struct nf_flow_match *match, return -EOPNOTSUPP; } mask->control.addr_type = 0xffff; - match->dissector.used_keys |= BIT(key->control.addr_type); + match->dissector.used_keys |= BIT_ULL(key->control.addr_type); mask->basic.n_proto = 0xffff; switch (tuple->l4proto) { case IPPROTO_TCP: key->tcp.flags = 0; mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16); - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP); break; case IPPROTO_UDP: case IPPROTO_GRE: @@ -182,9 +182,9 @@ static int nf_flow_rule_match(struct nf_flow_match *match, key->basic.ip_proto = tuple->l4proto; mask->basic.ip_proto = 0xff; - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_META) | - BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) | + BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); switch (tuple->l4proto) { case IPPROTO_TCP: @@ -194,7 +194,7 @@ static int nf_flow_rule_match(struct nf_flow_match *match, key->tp.dst = tuple->dst_port; mask->tp.dst = 0xffff; - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_PORTS); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); break; } diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c index 910ef881c3b8..12ab78fa5d84 100644 --- a/net/netfilter/nf_tables_offload.c +++ b/net/netfilter/nf_tables_offload.c @@ -35,12 +35,12 @@ void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow, struct nft_flow_key *mask = &match->mask; struct nft_flow_key *key = &match->key; - if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) + if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) return; key->control.addr_type = addr_type; mask->control.addr_type = 0xffff; - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL); match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] = offsetof(struct nft_flow_key, control); } @@ -59,7 +59,7 @@ static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx, .mask = match->mask.basic.n_proto, }; - if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) && + if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) && (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) || match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) { match->key.basic.n_proto = match->key.cvlan.vlan_tpid; @@ -70,8 +70,9 @@ static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx, match->mask.vlan.vlan_tpid = ethertype.mask; match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] = offsetof(struct nft_flow_key, cvlan); - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN); - } else if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC) && + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN); + } else if (match->dissector.used_keys & + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) && (match->key.basic.n_proto == htons(ETH_P_8021Q) || match->key.basic.n_proto == htons(ETH_P_8021AD))) { match->key.basic.n_proto = match->key.vlan.vlan_tpid; @@ -80,7 +81,7 @@ static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx, match->mask.vlan.vlan_tpid = ethertype.mask; match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] = offsetof(struct nft_flow_key, vlan); - match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN); + match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_VLAN); } } diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c index 6eb21a4f5698..cd4652259095 100644 --- a/net/netfilter/nft_cmp.c +++ b/net/netfilter/nft_cmp.c @@ -162,7 +162,7 @@ static int __nft_cmp_offload(struct nft_offload_ctx *ctx, memcpy(key + reg->offset, data, reg->len); memcpy(mask + reg->offset, datamask, reg->len); - flow->match.dissector.used_keys |= BIT(reg->key); + flow->match.dissector.used_keys |= BIT_ULL(reg->key); flow->match.dissector.offset[reg->key] = reg->base_offset; if (reg->key == FLOW_DISSECTOR_KEY_META && diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 38958e067aa8..e87fd4314c68 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -262,6 +262,7 @@ static void nft_ct_set_zone_eval(const struct nft_expr *expr, regs->verdict.code = NF_DROP; return; } + __set_bit(IPS_CONFIRMED_BIT, &ct->status); } nf_ct_set(skb, ct, IP_CT_NEW); @@ -368,6 +369,7 @@ static bool nft_ct_tmpl_alloc_pcpu(void) return false; } + __set_bit(IPS_CONFIRMED_BIT, &tmp->status); per_cpu(nft_ct_pcpu_template, cpu) = tmp; } diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index 6e049fd48760..601c9e09d07a 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -14,17 +14,18 @@ #include <net/netfilter/nf_tables.h> #include <net/netfilter/nft_fib.h> +#define NFTA_FIB_F_ALL (NFTA_FIB_F_SADDR | NFTA_FIB_F_DADDR | \ + NFTA_FIB_F_MARK | NFTA_FIB_F_IIF | NFTA_FIB_F_OIF | \ + NFTA_FIB_F_PRESENT) + const struct nla_policy nft_fib_policy[NFTA_FIB_MAX + 1] = { [NFTA_FIB_DREG] = { .type = NLA_U32 }, [NFTA_FIB_RESULT] = { .type = NLA_U32 }, - [NFTA_FIB_FLAGS] = { .type = NLA_U32 }, + [NFTA_FIB_FLAGS] = + NLA_POLICY_MASK(NLA_BE32, NFTA_FIB_F_ALL), }; EXPORT_SYMBOL(nft_fib_policy); -#define NFTA_FIB_F_ALL (NFTA_FIB_F_SADDR | NFTA_FIB_F_DADDR | \ - NFTA_FIB_F_MARK | NFTA_FIB_F_IIF | NFTA_FIB_F_OIF | \ - NFTA_FIB_F_PRESENT) - int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nft_data **data) { @@ -77,7 +78,7 @@ int nft_fib_init(const struct nft_ctx *ctx, const struct nft_expr *expr, priv->flags = ntohl(nla_get_be32(tb[NFTA_FIB_FLAGS])); - if (priv->flags == 0 || (priv->flags & ~NFTA_FIB_F_ALL)) + if (priv->flags == 0) return -EINVAL; if ((priv->flags & (NFTA_FIB_F_SADDR | NFTA_FIB_F_DADDR)) == diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index 29ac48cdd6db..870e5b113d13 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -90,7 +90,8 @@ static const struct nla_policy nft_lookup_policy[NFTA_LOOKUP_MAX + 1] = { [NFTA_LOOKUP_SET_ID] = { .type = NLA_U32 }, [NFTA_LOOKUP_SREG] = { .type = NLA_U32 }, [NFTA_LOOKUP_DREG] = { .type = NLA_U32 }, - [NFTA_LOOKUP_FLAGS] = { .type = NLA_U32 }, + [NFTA_LOOKUP_FLAGS] = + NLA_POLICY_MASK(NLA_BE32, NFT_LOOKUP_F_INV), }; static int nft_lookup_init(const struct nft_ctx *ctx, @@ -120,9 +121,6 @@ static int nft_lookup_init(const struct nft_ctx *ctx, if (tb[NFTA_LOOKUP_FLAGS]) { flags = ntohl(nla_get_be32(tb[NFTA_LOOKUP_FLAGS])); - if (flags & ~NFT_LOOKUP_F_INV) - return -EINVAL; - if (flags & NFT_LOOKUP_F_INV) priv->invert = true; } diff --git a/net/netfilter/nft_masq.c b/net/netfilter/nft_masq.c index b115d77fbbc7..8a14aaca93bb 100644 --- a/net/netfilter/nft_masq.c +++ b/net/netfilter/nft_masq.c @@ -20,7 +20,8 @@ struct nft_masq { }; static const struct nla_policy nft_masq_policy[NFTA_MASQ_MAX + 1] = { - [NFTA_MASQ_FLAGS] = { .type = NLA_U32 }, + [NFTA_MASQ_FLAGS] = + NLA_POLICY_MASK(NLA_BE32, NF_NAT_RANGE_MASK), [NFTA_MASQ_REG_PROTO_MIN] = { .type = NLA_U32 }, [NFTA_MASQ_REG_PROTO_MAX] = { .type = NLA_U32 }, }; @@ -47,11 +48,8 @@ static int nft_masq_init(const struct nft_ctx *ctx, struct nft_masq *priv = nft_expr_priv(expr); int err; - if (tb[NFTA_MASQ_FLAGS]) { + if (tb[NFTA_MASQ_FLAGS]) priv->flags = ntohl(nla_get_be32(tb[NFTA_MASQ_FLAGS])); - if (priv->flags & ~NF_NAT_RANGE_MASK) - return -EINVAL; - } if (tb[NFTA_MASQ_REG_PROTO_MIN]) { err = nft_parse_register_load(tb[NFTA_MASQ_REG_PROTO_MIN], diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index 5c29915ab028..583885ce7232 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -132,7 +132,8 @@ static const struct nla_policy nft_nat_policy[NFTA_NAT_MAX + 1] = { [NFTA_NAT_REG_ADDR_MAX] = { .type = NLA_U32 }, [NFTA_NAT_REG_PROTO_MIN] = { .type = NLA_U32 }, [NFTA_NAT_REG_PROTO_MAX] = { .type = NLA_U32 }, - [NFTA_NAT_FLAGS] = { .type = NLA_U32 }, + [NFTA_NAT_FLAGS] = + NLA_POLICY_MASK(NLA_BE32, NF_NAT_RANGE_MASK), }; static int nft_nat_validate(const struct nft_ctx *ctx, @@ -246,11 +247,8 @@ static int nft_nat_init(const struct nft_ctx *ctx, const struct nft_expr *expr, priv->flags |= NF_NAT_RANGE_PROTO_SPECIFIED; } - if (tb[NFTA_NAT_FLAGS]) { + if (tb[NFTA_NAT_FLAGS]) priv->flags |= ntohl(nla_get_be32(tb[NFTA_NAT_FLAGS])); - if (priv->flags & ~NF_NAT_RANGE_MASK) - return -EOPNOTSUPP; - } return nf_ct_netns_get(ctx->net, family); } diff --git a/net/netfilter/nft_redir.c b/net/netfilter/nft_redir.c index a70196ffcb1e..a58bd8d291ff 100644 --- a/net/netfilter/nft_redir.c +++ b/net/netfilter/nft_redir.c @@ -22,7 +22,8 @@ struct nft_redir { static const struct nla_policy nft_redir_policy[NFTA_REDIR_MAX + 1] = { [NFTA_REDIR_REG_PROTO_MIN] = { .type = NLA_U32 }, [NFTA_REDIR_REG_PROTO_MAX] = { .type = NLA_U32 }, - [NFTA_REDIR_FLAGS] = { .type = NLA_U32 }, + [NFTA_REDIR_FLAGS] = + NLA_POLICY_MASK(NLA_BE32, NF_NAT_RANGE_MASK), }; static int nft_redir_validate(const struct nft_ctx *ctx, @@ -68,11 +69,8 @@ static int nft_redir_init(const struct nft_ctx *ctx, priv->flags |= NF_NAT_RANGE_PROTO_SPECIFIED; } - if (tb[NFTA_REDIR_FLAGS]) { + if (tb[NFTA_REDIR_FLAGS]) priv->flags = ntohl(nla_get_be32(tb[NFTA_REDIR_FLAGS])); - if (priv->flags & ~NF_NAT_RANGE_MASK) - return -EINVAL; - } return nf_ct_netns_get(ctx->net, ctx->family); } diff --git a/net/netlabel/netlabel_cipso_v4.h b/net/netlabel/netlabel_cipso_v4.h index 85d7ecb05728..9518ab56ec98 100644 --- a/net/netlabel/netlabel_cipso_v4.h +++ b/net/netlabel/netlabel_cipso_v4.h @@ -149,7 +149,4 @@ enum { /* NetLabel protocol functions */ int netlbl_cipsov4_genl_init(void); -/* Free the memory associated with a CIPSOv4 DOI definition */ -void netlbl_cipsov4_doi_free(struct rcu_head *entry); - #endif diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 383631873748..96c605e45235 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -677,6 +677,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, struct netlink_sock *nlk; int (*bind)(struct net *net, int group); void (*unbind)(struct net *net, int group); + void (*release)(struct sock *sock, unsigned long *groups); int err = 0; sock->state = SS_UNCONNECTED; @@ -704,6 +705,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, cb_mutex = nl_table[protocol].cb_mutex; bind = nl_table[protocol].bind; unbind = nl_table[protocol].unbind; + release = nl_table[protocol].release; netlink_unlock_table(); if (err < 0) @@ -719,6 +721,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, nlk->module = module; nlk->netlink_bind = bind; nlk->netlink_unbind = unbind; + nlk->netlink_release = release; out: return err; @@ -763,6 +766,8 @@ static int netlink_release(struct socket *sock) * OK. Socket is unlinked, any packets that arrive now * will be purged. */ + if (nlk->netlink_release) + nlk->netlink_release(sk, nlk->groups); /* must not acquire netlink_table_lock in any way again before unbind * and notifying genetlink is done as otherwise it might deadlock @@ -1432,6 +1437,8 @@ struct netlink_broadcast_data { int delivered; gfp_t allocation; struct sk_buff *skb, *skb2; + int (*tx_filter)(struct sock *dsk, struct sk_buff *skb, void *data); + void *tx_data; }; static void do_one_broadcast(struct sock *sk, @@ -1485,6 +1492,13 @@ static void do_one_broadcast(struct sock *sk, p->delivery_failure = 1; goto out; } + + if (p->tx_filter && p->tx_filter(sk, p->skb2, p->tx_data)) { + kfree_skb(p->skb2); + p->skb2 = NULL; + goto out; + } + if (sk_filter(sk, p->skb2)) { kfree_skb(p->skb2); p->skb2 = NULL; @@ -1507,8 +1521,12 @@ out: sock_put(sk); } -int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid, - u32 group, gfp_t allocation) +int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb, + u32 portid, + u32 group, gfp_t allocation, + int (*filter)(struct sock *dsk, + struct sk_buff *skb, void *data), + void *filter_data) { struct net *net = sock_net(ssk); struct netlink_broadcast_data info; @@ -1527,6 +1545,8 @@ int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid, info.allocation = allocation; info.skb = skb; info.skb2 = NULL; + info.tx_filter = filter; + info.tx_data = filter_data; /* While we sleep in clone, do not allow to change socket list */ @@ -1552,6 +1572,14 @@ int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid, } return -ESRCH; } +EXPORT_SYMBOL(netlink_broadcast_filtered); + +int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid, + u32 group, gfp_t allocation) +{ + return netlink_broadcast_filtered(ssk, skb, portid, group, allocation, + NULL, NULL); +} EXPORT_SYMBOL(netlink_broadcast); struct netlink_set_err_data { @@ -1629,10 +1657,7 @@ static void netlink_update_socket_mc(struct netlink_sock *nlk, old = test_bit(group - 1, nlk->groups); subscriptions = nlk->subscriptions - old + new; - if (new) - __set_bit(group - 1, nlk->groups); - else - __clear_bit(group - 1, nlk->groups); + __assign_bit(group - 1, nlk->groups, new); netlink_update_subscriptions(&nlk->sk, subscriptions); netlink_update_listeners(&nlk->sk); } @@ -2069,6 +2094,7 @@ __netlink_kernel_create(struct net *net, int unit, struct module *module, if (cfg) { nl_table[unit].bind = cfg->bind; nl_table[unit].unbind = cfg->unbind; + nl_table[unit].release = cfg->release; nl_table[unit].flags = cfg->flags; } nl_table[unit].registered = 1; diff --git a/net/netlink/af_netlink.h b/net/netlink/af_netlink.h index 90a3198a9b7f..fd424cd63f31 100644 --- a/net/netlink/af_netlink.h +++ b/net/netlink/af_netlink.h @@ -42,6 +42,8 @@ struct netlink_sock { void (*netlink_rcv)(struct sk_buff *skb); int (*netlink_bind)(struct net *net, int group); void (*netlink_unbind)(struct net *net, int group); + void (*netlink_release)(struct sock *sk, + unsigned long *groups); struct module *module; struct rhash_head node; @@ -64,6 +66,8 @@ struct netlink_table { struct module *module; int (*bind)(struct net *net, int group); void (*unbind)(struct net *net, int group); + void (*release)(struct sock *sk, + unsigned long *groups); int registered; }; diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index a157247a1e45..6bd2ce51271f 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -593,8 +593,12 @@ static int genl_validate_ops(const struct genl_family *family) return -EINVAL; /* Check sort order */ - if (a->cmd < b->cmd) + if (a->cmd < b->cmd) { continue; + } else if (a->cmd > b->cmd) { + WARN_ON(1); + return -EINVAL; + } if (a->internal_flags != b->internal_flags || ((a->flags ^ b->flags) & ~(GENL_CMD_CAP_DO | diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 331730fd3580..fa955e892210 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -455,45 +455,6 @@ static int ovs_ct_handle_fragments(struct net *net, struct sw_flow_key *key, return 0; } -static struct nf_conntrack_expect * -ovs_ct_expect_find(struct net *net, const struct nf_conntrack_zone *zone, - u16 proto, const struct sk_buff *skb) -{ - struct nf_conntrack_tuple tuple; - struct nf_conntrack_expect *exp; - - if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), proto, net, &tuple)) - return NULL; - - exp = __nf_ct_expect_find(net, zone, &tuple); - if (exp) { - struct nf_conntrack_tuple_hash *h; - - /* Delete existing conntrack entry, if it clashes with the - * expectation. This can happen since conntrack ALGs do not - * check for clashes between (new) expectations and existing - * conntrack entries. nf_conntrack_in() will check the - * expectations only if a conntrack entry can not be found, - * which can lead to OVS finding the expectation (here) in the - * init direction, but which will not be removed by the - * nf_conntrack_in() call, if a matching conntrack entry is - * found instead. In this case all init direction packets - * would be reported as new related packets, while reply - * direction packets would be reported as un-related - * established packets. - */ - h = nf_conntrack_find_get(net, zone, &tuple); - if (h) { - struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); - - nf_ct_delete(ct, 0, 0); - nf_ct_put(ct); - } - } - - return exp; -} - /* This replicates logic from nf_conntrack_core.c that is not exported. */ static enum ip_conntrack_info ovs_ct_get_info(const struct nf_conntrack_tuple_hash *h) @@ -852,36 +813,16 @@ static int ovs_ct_lookup(struct net *net, struct sw_flow_key *key, const struct ovs_conntrack_info *info, struct sk_buff *skb) { - struct nf_conntrack_expect *exp; - - /* If we pass an expected packet through nf_conntrack_in() the - * expectation is typically removed, but the packet could still be - * lost in upcall processing. To prevent this from happening we - * perform an explicit expectation lookup. Expected connections are - * always new, and will be passed through conntrack only when they are - * committed, as it is OK to remove the expectation at that time. - */ - exp = ovs_ct_expect_find(net, &info->zone, info->family, skb); - if (exp) { - u8 state; - - /* NOTE: New connections are NATted and Helped only when - * committed, so we are not calling into NAT here. - */ - state = OVS_CS_F_TRACKED | OVS_CS_F_NEW | OVS_CS_F_RELATED; - __ovs_ct_update_key(key, state, &info->zone, exp->master); - } else { - struct nf_conn *ct; - int err; + struct nf_conn *ct; + int err; - err = __ovs_ct_lookup(net, key, info, skb); - if (err) - return err; + err = __ovs_ct_lookup(net, key, info, skb); + if (err) + return err; - ct = (struct nf_conn *)skb_nfct(skb); - if (ct) - nf_ct_deliver_cached_events(ct); - } + ct = (struct nf_conn *)skb_nfct(skb); + if (ct) + nf_ct_deliver_cached_events(ct); return 0; } @@ -1460,7 +1401,8 @@ int ovs_ct_copy_action(struct net *net, const struct nlattr *attr, if (err) goto err_free_ct; - __set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status); + if (ct_info.commit) + __set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status); return 0; err_free_ct: __ovs_ct_free_action(&ct_info); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index a4631cb457a9..0a65e696a4c6 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2927,8 +2927,10 @@ static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - err, 0); + err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return NULL; diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index 78beb74146e7..41ece61eb57a 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -23,6 +23,8 @@ #define QRTR_EPH_PORT_RANGE \ XA_LIMIT(QRTR_MIN_EPH_SOCKET, QRTR_MAX_EPH_SOCKET) +#define QRTR_PORT_CTRL_LEGACY 0xffff + /** * struct qrtr_hdr_v1 - (I|R)PCrouter packet header version 1 * @version: protocol version @@ -495,6 +497,9 @@ int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len) goto err; } + if (cb->dst_port == QRTR_PORT_CTRL_LEGACY) + cb->dst_port = QRTR_PORT_CTRL; + if (!size || len != ALIGN(size, 4) + hdrlen) goto err; diff --git a/net/qrtr/ns.c b/net/qrtr/ns.c index 0f7a729f1a1f..b1db0b519179 100644 --- a/net/qrtr/ns.c +++ b/net/qrtr/ns.c @@ -16,7 +16,7 @@ #define CREATE_TRACE_POINTS #include <trace/events/qrtr.h> -static RADIX_TREE(nodes, GFP_KERNEL); +static DEFINE_XARRAY(nodes); static struct { struct socket *sock; @@ -66,14 +66,14 @@ struct qrtr_server { struct qrtr_node { unsigned int id; - struct radix_tree_root servers; + struct xarray servers; }; static struct qrtr_node *node_get(unsigned int node_id) { struct qrtr_node *node; - node = radix_tree_lookup(&nodes, node_id); + node = xa_load(&nodes, node_id); if (node) return node; @@ -83,8 +83,9 @@ static struct qrtr_node *node_get(unsigned int node_id) return NULL; node->id = node_id; + xa_init(&node->servers); - if (radix_tree_insert(&nodes, node_id, node)) { + if (xa_store(&nodes, node_id, node, GFP_KERNEL)) { kfree(node); return NULL; } @@ -193,40 +194,23 @@ static void lookup_notify(struct sockaddr_qrtr *to, struct qrtr_server *srv, static int announce_servers(struct sockaddr_qrtr *sq) { - struct radix_tree_iter iter; struct qrtr_server *srv; struct qrtr_node *node; - void __rcu **slot; + unsigned long index; int ret; node = node_get(qrtr_ns.local_node); if (!node) return 0; - rcu_read_lock(); /* Announce the list of servers registered in this node */ - radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { - srv = radix_tree_deref_slot(slot); - if (!srv) - continue; - if (radix_tree_deref_retry(srv)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - slot = radix_tree_iter_resume(slot, &iter); - rcu_read_unlock(); - + xa_for_each(&node->servers, index, srv) { ret = service_announce_new(sq, srv); if (ret < 0) { pr_err("failed to announce new service\n"); return ret; } - - rcu_read_lock(); } - - rcu_read_unlock(); - return 0; } @@ -256,14 +240,17 @@ static struct qrtr_server *server_add(unsigned int service, goto err; /* Delete the old server on the same port */ - old = radix_tree_lookup(&node->servers, port); + old = xa_store(&node->servers, port, srv, GFP_KERNEL); if (old) { - radix_tree_delete(&node->servers, port); - kfree(old); + if (xa_is_err(old)) { + pr_err("failed to add server [0x%x:0x%x] ret:%d\n", + srv->service, srv->instance, xa_err(old)); + goto err; + } else { + kfree(old); + } } - radix_tree_insert(&node->servers, port, srv); - trace_qrtr_ns_server_add(srv->service, srv->instance, srv->node, srv->port); @@ -280,11 +267,11 @@ static int server_del(struct qrtr_node *node, unsigned int port, bool bcast) struct qrtr_server *srv; struct list_head *li; - srv = radix_tree_lookup(&node->servers, port); + srv = xa_load(&node->servers, port); if (!srv) return -ENOENT; - radix_tree_delete(&node->servers, port); + xa_erase(&node->servers, port); /* Broadcast the removal of local servers */ if (srv->node == qrtr_ns.local_node && bcast) @@ -344,13 +331,12 @@ static int ctrl_cmd_hello(struct sockaddr_qrtr *sq) static int ctrl_cmd_bye(struct sockaddr_qrtr *from) { struct qrtr_node *local_node; - struct radix_tree_iter iter; struct qrtr_ctrl_pkt pkt; struct qrtr_server *srv; struct sockaddr_qrtr sq; struct msghdr msg = { }; struct qrtr_node *node; - void __rcu **slot; + unsigned long index; struct kvec iv; int ret; @@ -361,22 +347,9 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) if (!node) return 0; - rcu_read_lock(); /* Advertise removal of this client to all servers of remote node */ - radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { - srv = radix_tree_deref_slot(slot); - if (!srv) - continue; - if (radix_tree_deref_retry(srv)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - slot = radix_tree_iter_resume(slot, &iter); - rcu_read_unlock(); + xa_for_each(&node->servers, index, srv) server_del(node, srv->port, true); - rcu_read_lock(); - } - rcu_read_unlock(); /* Advertise the removal of this client to all local servers */ local_node = node_get(qrtr_ns.local_node); @@ -387,18 +360,7 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) pkt.cmd = cpu_to_le32(QRTR_TYPE_BYE); pkt.client.node = cpu_to_le32(from->sq_node); - rcu_read_lock(); - radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { - srv = radix_tree_deref_slot(slot); - if (!srv) - continue; - if (radix_tree_deref_retry(srv)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - slot = radix_tree_iter_resume(slot, &iter); - rcu_read_unlock(); - + xa_for_each(&local_node->servers, index, srv) { sq.sq_family = AF_QIPCRTR; sq.sq_node = srv->node; sq.sq_port = srv->port; @@ -411,11 +373,7 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) pr_err("failed to send bye cmd\n"); return ret; } - rcu_read_lock(); } - - rcu_read_unlock(); - return 0; } @@ -423,7 +381,6 @@ static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, unsigned int node_id, unsigned int port) { struct qrtr_node *local_node; - struct radix_tree_iter iter; struct qrtr_lookup *lookup; struct qrtr_ctrl_pkt pkt; struct msghdr msg = { }; @@ -432,7 +389,7 @@ static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, struct qrtr_node *node; struct list_head *tmp; struct list_head *li; - void __rcu **slot; + unsigned long index; struct kvec iv; int ret; @@ -477,18 +434,7 @@ static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, pkt.client.node = cpu_to_le32(node_id); pkt.client.port = cpu_to_le32(port); - rcu_read_lock(); - radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { - srv = radix_tree_deref_slot(slot); - if (!srv) - continue; - if (radix_tree_deref_retry(srv)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - slot = radix_tree_iter_resume(slot, &iter); - rcu_read_unlock(); - + xa_for_each(&local_node->servers, index, srv) { sq.sq_family = AF_QIPCRTR; sq.sq_node = srv->node; sq.sq_port = srv->port; @@ -501,11 +447,7 @@ static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, pr_err("failed to send del client cmd\n"); return ret; } - rcu_read_lock(); } - - rcu_read_unlock(); - return 0; } @@ -576,13 +518,12 @@ static int ctrl_cmd_del_server(struct sockaddr_qrtr *from, static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, unsigned int service, unsigned int instance) { - struct radix_tree_iter node_iter; struct qrtr_server_filter filter; - struct radix_tree_iter srv_iter; struct qrtr_lookup *lookup; + struct qrtr_server *srv; struct qrtr_node *node; - void __rcu **node_slot; - void __rcu **srv_slot; + unsigned long node_idx; + unsigned long srv_idx; /* Accept only local observers */ if (from->sq_node != qrtr_ns.local_node) @@ -601,40 +542,14 @@ static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, filter.service = service; filter.instance = instance; - rcu_read_lock(); - radix_tree_for_each_slot(node_slot, &nodes, &node_iter, 0) { - node = radix_tree_deref_slot(node_slot); - if (!node) - continue; - if (radix_tree_deref_retry(node)) { - node_slot = radix_tree_iter_retry(&node_iter); - continue; - } - node_slot = radix_tree_iter_resume(node_slot, &node_iter); - - radix_tree_for_each_slot(srv_slot, &node->servers, - &srv_iter, 0) { - struct qrtr_server *srv; - - srv = radix_tree_deref_slot(srv_slot); - if (!srv) - continue; - if (radix_tree_deref_retry(srv)) { - srv_slot = radix_tree_iter_retry(&srv_iter); - continue; - } - + xa_for_each(&nodes, node_idx, node) { + xa_for_each(&node->servers, srv_idx, srv) { if (!server_match(srv, &filter)) continue; - srv_slot = radix_tree_iter_resume(srv_slot, &srv_iter); - - rcu_read_unlock(); lookup_notify(from, srv, true); - rcu_read_lock(); } } - rcu_read_unlock(); /* Empty notification, to indicate end of listing */ lookup_notify(from, NULL, true); diff --git a/net/sched/Kconfig b/net/sched/Kconfig index 4b95cb1ac435..470c70deffe2 100644 --- a/net/sched/Kconfig +++ b/net/sched/Kconfig @@ -347,8 +347,7 @@ config NET_SCH_FQ_PIE config NET_SCH_INGRESS tristate "Ingress/classifier-action Qdisc" depends on NET_CLS_ACT - select NET_INGRESS - select NET_EGRESS + select NET_XGRESS help Say Y here if you want to use classifiers for incoming and/or outgoing packets. This qdisc doesn't do anything else besides running classifiers, @@ -679,6 +678,7 @@ config NET_EMATCH_IPT config NET_CLS_ACT bool "Actions" select NET_CLS + select NET_XGRESS help Say Y here if you want to use traffic control actions. Actions get attached to classifiers and are invoked after a successful diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index abc71a06d634..7c652d14528b 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1238,7 +1238,8 @@ static int tcf_ct_fill_params(struct net *net, } } - __set_bit(IPS_CONFIRMED_BIT, &tmpl->status); + if (p->ct_action & TCA_CT_ACT_COMMIT) + __set_bit(IPS_CONFIRMED_BIT, &tmpl->status); return 0; err: nf_ct_put(p->tmpl); diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 9f0711da9c95..e5314a31f75a 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -72,6 +72,7 @@ struct fl_flow_key { struct flow_dissector_key_num_of_vlans num_of_vlans; struct flow_dissector_key_pppoe pppoe; struct flow_dissector_key_l2tpv3 l2tpv3; + struct flow_dissector_key_ipsec ipsec; struct flow_dissector_key_cfm cfm; } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ @@ -726,6 +727,8 @@ static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { [TCA_FLOWER_KEY_PPPOE_SID] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_PPP_PROTO] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_L2TPV3_SID] = { .type = NLA_U32 }, + [TCA_FLOWER_KEY_SPI] = { .type = NLA_U32 }, + [TCA_FLOWER_KEY_SPI_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_L2_MISS] = NLA_POLICY_MAX(NLA_U8, 1), [TCA_FLOWER_KEY_CFM] = { .type = NLA_NESTED }, }; @@ -796,6 +799,24 @@ static void fl_set_key_val(struct nlattr **tb, nla_memcpy(mask, tb[mask_type], len); } +static int fl_set_key_spi(struct nlattr **tb, struct fl_flow_key *key, + struct fl_flow_key *mask, + struct netlink_ext_ack *extack) +{ + if (key->basic.ip_proto != IPPROTO_ESP && + key->basic.ip_proto != IPPROTO_AH) { + NL_SET_ERR_MSG(extack, + "Protocol must be either ESP or AH"); + return -EINVAL; + } + + fl_set_key_val(tb, &key->ipsec.spi, + TCA_FLOWER_KEY_SPI, + &mask->ipsec.spi, TCA_FLOWER_KEY_SPI_MASK, + sizeof(key->ipsec.spi)); + return 0; +} + static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, struct fl_flow_key *mask, struct netlink_ext_ack *extack) @@ -1895,6 +1916,12 @@ static int fl_set_key(struct net *net, struct nlattr **tb, return ret; } + if (tb[TCA_FLOWER_KEY_SPI]) { + ret = fl_set_key_spi(tb, key, mask, extack); + if (ret) + return ret; + } + if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; @@ -2068,6 +2095,8 @@ static void fl_init_dissector(struct flow_dissector *dissector, FL_KEY_SET_IF_MASKED(mask, keys, cnt, FLOW_DISSECTOR_KEY_L2TPV3, l2tpv3); FL_KEY_SET_IF_MASKED(mask, keys, cnt, + FLOW_DISSECTOR_KEY_IPSEC, ipsec); + FL_KEY_SET_IF_MASKED(mask, keys, cnt, FLOW_DISSECTOR_KEY_CFM, cfm); skb_flow_dissector_init(dissector, keys, cnt); @@ -3365,6 +3394,12 @@ static int fl_dump_key(struct sk_buff *skb, struct net *net, sizeof(key->l2tpv3.session_id))) goto nla_put_failure; + if (key->ipsec.spi && + fl_dump_key_val(skb, &key->ipsec.spi, TCA_FLOWER_KEY_SPI, + &mask->ipsec.spi, TCA_FLOWER_KEY_SPI_MASK, + sizeof(key->ipsec.spi))) + goto nla_put_failure; + if ((key->basic.ip_proto == IPPROTO_TCP || key->basic.ip_proto == IPPROTO_UDP || key->basic.ip_proto == IPPROTO_SCTP) && diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index e35a4e90f4e6..19901e77cd3b 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -17,7 +17,6 @@ struct drr_class { struct Qdisc_class_common common; - unsigned int filter_cnt; struct gnet_stats_basic_sync bstats; struct gnet_stats_queue qstats; @@ -150,8 +149,10 @@ static int drr_delete_class(struct Qdisc *sch, unsigned long arg, struct drr_sched *q = qdisc_priv(sch); struct drr_class *cl = (struct drr_class *)arg; - if (cl->filter_cnt > 0) + if (qdisc_class_in_use(&cl->common)) { + NL_SET_ERR_MSG(extack, "DRR class is in use"); return -EBUSY; + } sch_tree_lock(sch); @@ -187,8 +188,8 @@ static unsigned long drr_bind_tcf(struct Qdisc *sch, unsigned long parent, { struct drr_class *cl = drr_find_class(sch, classid); - if (cl != NULL) - cl->filter_cnt++; + if (cl) + qdisc_class_get(&cl->common); return (unsigned long)cl; } @@ -197,7 +198,7 @@ static void drr_unbind_tcf(struct Qdisc *sch, unsigned long arg) { struct drr_class *cl = (struct drr_class *)arg; - cl->filter_cnt--; + qdisc_class_put(&cl->common); } static int drr_graft_class(struct Qdisc *sch, unsigned long arg, diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 70b0c5873d32..98805303218d 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -116,7 +116,6 @@ struct hfsc_class { struct net_rate_estimator __rcu *rate_est; struct tcf_proto __rcu *filter_list; /* filter list */ struct tcf_block *block; - unsigned int filter_cnt; /* filter count */ unsigned int level; /* class level in hierarchy */ struct hfsc_sched *sched; /* scheduler data */ @@ -1094,8 +1093,11 @@ hfsc_delete_class(struct Qdisc *sch, unsigned long arg, struct hfsc_sched *q = qdisc_priv(sch); struct hfsc_class *cl = (struct hfsc_class *)arg; - if (cl->level > 0 || cl->filter_cnt > 0 || cl == &q->root) + if (cl->level > 0 || qdisc_class_in_use(&cl->cl_common) || + cl == &q->root) { + NL_SET_ERR_MSG(extack, "HFSC class in use"); return -EBUSY; + } sch_tree_lock(sch); @@ -1223,7 +1225,7 @@ hfsc_bind_tcf(struct Qdisc *sch, unsigned long parent, u32 classid) if (cl != NULL) { if (p != NULL && p->level <= cl->level) return 0; - cl->filter_cnt++; + qdisc_class_get(&cl->cl_common); } return (unsigned long)cl; @@ -1234,7 +1236,7 @@ hfsc_unbind_tcf(struct Qdisc *sch, unsigned long arg) { struct hfsc_class *cl = (struct hfsc_class *)arg; - cl->filter_cnt--; + qdisc_class_put(&cl->cl_common); } static struct tcf_block *hfsc_tcf_block(struct Qdisc *sch, unsigned long arg, diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index 325c29041c7d..0d947414e616 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -102,7 +102,6 @@ struct htb_class { struct tcf_proto __rcu *filter_list; /* class attached filters */ struct tcf_block *block; - int filter_cnt; int level; /* our level (see above) */ unsigned int children; @@ -1710,8 +1709,10 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg, * tc subsys guarantee us that in htb_destroy it holds no class * refs so that we can remove children safely there ? */ - if (cl->children || cl->filter_cnt) + if (cl->children || qdisc_class_in_use(&cl->common)) { + NL_SET_ERR_MSG(extack, "HTB class in use"); return -EBUSY; + } if (!cl->level && htb_parent_last_child(cl)) last_child = 1; @@ -1810,10 +1811,6 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, NL_SET_ERR_MSG(extack, "HTB offload doesn't support the mpu parameter"); goto failure; } - if (hopt->quantum) { - NL_SET_ERR_MSG(extack, "HTB offload doesn't support the quantum parameter"); - goto failure; - } } /* Keeping backward compatible with rate_table based iproute2 tc */ @@ -1910,6 +1907,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, .rate = max_t(u64, hopt->rate.rate, rate64), .ceil = max_t(u64, hopt->ceil.rate, ceil64), .prio = hopt->prio, + .quantum = hopt->quantum, .extack = extack, }; err = htb_offload(dev, &offload_opt); @@ -1931,6 +1929,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, .rate = max_t(u64, hopt->rate.rate, rate64), .ceil = max_t(u64, hopt->ceil.rate, ceil64), .prio = hopt->prio, + .quantum = hopt->quantum, .extack = extack, }; err = htb_offload(dev, &offload_opt); @@ -2017,6 +2016,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, .rate = max_t(u64, hopt->rate.rate, rate64), .ceil = max_t(u64, hopt->ceil.rate, ceil64), .prio = hopt->prio, + .quantum = hopt->quantum, .extack = extack, }; err = htb_offload(dev, &offload_opt); @@ -2108,7 +2108,7 @@ static unsigned long htb_bind_filter(struct Qdisc *sch, unsigned long parent, * be broken by class during destroy IIUC. */ if (cl) - cl->filter_cnt++; + qdisc_class_get(&cl->common); return (unsigned long)cl; } @@ -2116,8 +2116,7 @@ static void htb_unbind_filter(struct Qdisc *sch, unsigned long arg) { struct htb_class *cl = (struct htb_class *)arg; - if (cl) - cl->filter_cnt--; + qdisc_class_put(&cl->common); } static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg) diff --git a/net/sched/sch_ingress.c b/net/sched/sch_ingress.c index e43a45499372..a463a63192c3 100644 --- a/net/sched/sch_ingress.c +++ b/net/sched/sch_ingress.c @@ -13,6 +13,7 @@ #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> +#include <net/tcx.h> struct ingress_sched_data { struct tcf_block *block; @@ -78,6 +79,8 @@ static int ingress_init(struct Qdisc *sch, struct nlattr *opt, { struct ingress_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); + struct bpf_mprog_entry *entry; + bool created; int err; if (sch->parent != TC_H_INGRESS) @@ -85,7 +88,13 @@ static int ingress_init(struct Qdisc *sch, struct nlattr *opt, net_inc_ingress_queue(); - mini_qdisc_pair_init(&q->miniqp, sch, &dev->miniq_ingress); + entry = tcx_entry_fetch_or_create(dev, true, &created); + if (!entry) + return -ENOMEM; + tcx_miniq_set_active(entry, true); + mini_qdisc_pair_init(&q->miniqp, sch, &tcx_entry(entry)->miniq); + if (created) + tcx_entry_update(dev, entry, true); q->block_info.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; q->block_info.chain_head_change = clsact_chain_head_change; @@ -103,11 +112,22 @@ static int ingress_init(struct Qdisc *sch, struct nlattr *opt, static void ingress_destroy(struct Qdisc *sch) { struct ingress_sched_data *q = qdisc_priv(sch); + struct net_device *dev = qdisc_dev(sch); + struct bpf_mprog_entry *entry = rtnl_dereference(dev->tcx_ingress); if (sch->parent != TC_H_INGRESS) return; tcf_block_put_ext(q->block, sch, &q->block_info); + + if (entry) { + tcx_miniq_set_active(entry, false); + if (!tcx_entry_is_active(entry)) { + tcx_entry_update(dev, NULL, true); + tcx_entry_free(entry); + } + } + net_dec_ingress_queue(); } @@ -223,6 +243,8 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, { struct clsact_sched_data *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); + struct bpf_mprog_entry *entry; + bool created; int err; if (sch->parent != TC_H_CLSACT) @@ -231,7 +253,13 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, net_inc_ingress_queue(); net_inc_egress_queue(); - mini_qdisc_pair_init(&q->miniqp_ingress, sch, &dev->miniq_ingress); + entry = tcx_entry_fetch_or_create(dev, true, &created); + if (!entry) + return -ENOMEM; + tcx_miniq_set_active(entry, true); + mini_qdisc_pair_init(&q->miniqp_ingress, sch, &tcx_entry(entry)->miniq); + if (created) + tcx_entry_update(dev, entry, true); q->ingress_block_info.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; q->ingress_block_info.chain_head_change = clsact_chain_head_change; @@ -244,7 +272,13 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, mini_qdisc_pair_block_init(&q->miniqp_ingress, q->ingress_block); - mini_qdisc_pair_init(&q->miniqp_egress, sch, &dev->miniq_egress); + entry = tcx_entry_fetch_or_create(dev, false, &created); + if (!entry) + return -ENOMEM; + tcx_miniq_set_active(entry, true); + mini_qdisc_pair_init(&q->miniqp_egress, sch, &tcx_entry(entry)->miniq); + if (created) + tcx_entry_update(dev, entry, false); q->egress_block_info.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS; q->egress_block_info.chain_head_change = clsact_chain_head_change; @@ -256,12 +290,31 @@ static int clsact_init(struct Qdisc *sch, struct nlattr *opt, static void clsact_destroy(struct Qdisc *sch) { struct clsact_sched_data *q = qdisc_priv(sch); + struct net_device *dev = qdisc_dev(sch); + struct bpf_mprog_entry *ingress_entry = rtnl_dereference(dev->tcx_ingress); + struct bpf_mprog_entry *egress_entry = rtnl_dereference(dev->tcx_egress); if (sch->parent != TC_H_CLSACT) return; - tcf_block_put_ext(q->egress_block, sch, &q->egress_block_info); tcf_block_put_ext(q->ingress_block, sch, &q->ingress_block_info); + tcf_block_put_ext(q->egress_block, sch, &q->egress_block_info); + + if (ingress_entry) { + tcx_miniq_set_active(ingress_entry, false); + if (!tcx_entry_is_active(ingress_entry)) { + tcx_entry_update(dev, NULL, true); + tcx_entry_free(ingress_entry); + } + } + + if (egress_entry) { + tcx_miniq_set_active(egress_entry, false); + if (!tcx_entry_is_active(egress_entry)) { + tcx_entry_update(dev, NULL, false); + tcx_entry_free(egress_entry); + } + } net_dec_ingress_queue(); net_dec_egress_queue(); diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index befaf74b33ca..1a25752f1a9a 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -130,8 +130,6 @@ struct qfq_aggregate; struct qfq_class { struct Qdisc_class_common common; - unsigned int filter_cnt; - struct gnet_stats_basic_sync bstats; struct gnet_stats_queue qstats; struct net_rate_estimator __rcu *rate_est; @@ -545,8 +543,10 @@ static int qfq_delete_class(struct Qdisc *sch, unsigned long arg, struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl = (struct qfq_class *)arg; - if (cl->filter_cnt > 0) + if (qdisc_class_in_use(&cl->common)) { + NL_SET_ERR_MSG_MOD(extack, "QFQ class in use"); return -EBUSY; + } sch_tree_lock(sch); @@ -580,8 +580,8 @@ static unsigned long qfq_bind_tcf(struct Qdisc *sch, unsigned long parent, { struct qfq_class *cl = qfq_find_class(sch, classid); - if (cl != NULL) - cl->filter_cnt++; + if (cl) + qdisc_class_get(&cl->common); return (unsigned long)cl; } @@ -590,7 +590,7 @@ static void qfq_unbind_tcf(struct Qdisc *sch, unsigned long arg) { struct qfq_class *cl = (struct qfq_class *)arg; - cl->filter_cnt--; + qdisc_class_put(&cl->common); } static int qfq_graft_class(struct Qdisc *sch, unsigned long arg, diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index 274d07bd774f..33c0895e101c 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -435,7 +435,8 @@ static void sctp_v4_get_dst(struct sctp_transport *t, union sctp_addr *saddr, fl4->fl4_dport = daddr->v4.sin_port; fl4->flowi4_proto = IPPROTO_SCTP; if (asoc) { - fl4->flowi4_tos = RT_CONN_FLAGS_TOS(asoc->base.sk, tos); + fl4->flowi4_tos = RT_TOS(tos); + fl4->flowi4_scope = ip_sock_rt_scope(asoc->base.sk); fl4->flowi4_oif = asoc->base.sk->sk_bound_dev_if; fl4->fl4_sport = htons(asoc->base.bind_addr.port); } diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 9388d98aebc0..6e3d28aa587c 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -9732,6 +9732,7 @@ struct proto sctpv6_prot = { .unhash = sctp_unhash, .no_autobind = true, .obj_size = sizeof(struct sctp6_sock), + .ipv6_pinfo_offset = offsetof(struct sctp6_sock, inet6), .useroffset = offsetof(struct sctp6_sock, sctp.subscribe), .usersize = offsetof(struct sctp6_sock, sctp.initmsg) - offsetof(struct sctp6_sock, sctp.subscribe) + diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h index 1645fba0d2d3..3c1b31bfa1cf 100644 --- a/net/smc/smc_core.h +++ b/net/smc/smc_core.h @@ -539,7 +539,6 @@ int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini); void smc_conn_free(struct smc_connection *conn); int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini); -void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr); int smc_core_init(void); void smc_core_exit(void); diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h index 034295676e88..4df5f8c8a0a1 100644 --- a/net/smc/smc_ib.h +++ b/net/smc/smc_ib.h @@ -96,7 +96,6 @@ void smc_ib_destroy_queue_pair(struct smc_link *lnk); int smc_ib_create_queue_pair(struct smc_link *lnk); int smc_ib_ready_link(struct smc_link *lnk); int smc_ib_modify_qp_rts(struct smc_link *lnk); -int smc_ib_modify_qp_reset(struct smc_link *lnk); int smc_ib_modify_qp_error(struct smc_link *lnk); long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev); int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index e43f26382411..2ed29e40c6a9 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -43,7 +43,7 @@ #include <net/udp.h> #include <net/tcp.h> #include <net/tcp_states.h> -#include <net/tls.h> +#include <net/tls_prot.h> #include <net/handshake.h> #include <linux/uaccess.h> #include <linux/highmem.h> @@ -226,27 +226,30 @@ static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining) } static int -svc_tcp_sock_process_cmsg(struct svc_sock *svsk, struct msghdr *msg, +svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg, struct cmsghdr *cmsg, int ret) { - if (cmsg->cmsg_level == SOL_TLS && - cmsg->cmsg_type == TLS_GET_RECORD_TYPE) { - u8 content_type = *((u8 *)CMSG_DATA(cmsg)); - - switch (content_type) { - case TLS_RECORD_TYPE_DATA: - /* TLS sets EOR at the end of each application data - * record, even though there might be more frames - * waiting to be decrypted. - */ - msg->msg_flags &= ~MSG_EOR; - break; - case TLS_RECORD_TYPE_ALERT: - ret = -ENOTCONN; - break; - default: - ret = -EAGAIN; - } + u8 content_type = tls_get_record_type(sock->sk, cmsg); + u8 level, description; + + switch (content_type) { + case 0: + break; + case TLS_RECORD_TYPE_DATA: + /* TLS sets EOR at the end of each application data + * record, even though there might be more frames + * waiting to be decrypted. + */ + msg->msg_flags &= ~MSG_EOR; + break; + case TLS_RECORD_TYPE_ALERT: + tls_alert_recv(sock->sk, msg, &level, &description); + ret = (level == TLS_ALERT_LEVEL_FATAL) ? + -ENOTCONN : -EAGAIN; + break; + default: + /* discard this record type */ + ret = -EAGAIN; } return ret; } @@ -258,13 +261,14 @@ svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg) struct cmsghdr cmsg; u8 buf[CMSG_SPACE(sizeof(u8))]; } u; + struct socket *sock = svsk->sk_sock; int ret; msg->msg_control = &u; msg->msg_controllen = sizeof(u); - ret = sock_recvmsg(svsk->sk_sock, msg, MSG_DONTWAIT); + ret = sock_recvmsg(sock, msg, MSG_DONTWAIT); if (unlikely(msg->msg_controllen != sizeof(u))) - ret = svc_tcp_sock_process_cmsg(svsk, msg, &u.cmsg, ret); + ret = svc_tcp_sock_process_cmsg(sock, msg, &u.cmsg, ret); return ret; } @@ -1621,6 +1625,8 @@ static void svc_tcp_sock_detach(struct svc_xprt *xprt) { struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); + tls_handshake_close(svsk->sk_sock); + svc_sock_detach(xprt); if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) { diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 9f010369100a..268a2cc61acd 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -47,7 +47,7 @@ #include <net/checksum.h> #include <net/udp.h> #include <net/tcp.h> -#include <net/tls.h> +#include <net/tls_prot.h> #include <net/handshake.h> #include <linux/bvec.h> @@ -360,24 +360,27 @@ static int xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, struct cmsghdr *cmsg, int ret) { - if (cmsg->cmsg_level == SOL_TLS && - cmsg->cmsg_type == TLS_GET_RECORD_TYPE) { - u8 content_type = *((u8 *)CMSG_DATA(cmsg)); - - switch (content_type) { - case TLS_RECORD_TYPE_DATA: - /* TLS sets EOR at the end of each application data - * record, even though there might be more frames - * waiting to be decrypted. - */ - msg->msg_flags &= ~MSG_EOR; - break; - case TLS_RECORD_TYPE_ALERT: - ret = -ENOTCONN; - break; - default: - ret = -EAGAIN; - } + u8 content_type = tls_get_record_type(sock->sk, cmsg); + u8 level, description; + + switch (content_type) { + case 0: + break; + case TLS_RECORD_TYPE_DATA: + /* TLS sets EOR at the end of each application data + * record, even though there might be more frames + * waiting to be decrypted. + */ + msg->msg_flags &= ~MSG_EOR; + break; + case TLS_RECORD_TYPE_ALERT: + tls_alert_recv(sock->sk, msg, &level, &description); + ret = (level == TLS_ALERT_LEVEL_FATAL) ? + -EACCES : -EAGAIN; + break; + default: + /* discard this record type */ + ret = -EAGAIN; } return ret; } @@ -777,6 +780,8 @@ static void xs_stream_data_receive(struct sock_xprt *transport) } if (ret == -ESHUTDOWN) kernel_sock_shutdown(transport->sock, SHUT_RDWR); + else if (ret == -EACCES) + xprt_wake_pending_tasks(&transport->xprt, -EACCES); else xs_poll_check_readable(transport); out: @@ -1292,6 +1297,8 @@ static void xs_close(struct rpc_xprt *xprt) dprintk("RPC: xs_close xprt %p\n", xprt); + if (transport->sock) + tls_handshake_close(transport->sock); xs_reset_transport(transport); xprt->reestablish_timeout = 0; } diff --git a/net/switchdev/switchdev.c b/net/switchdev/switchdev.c index 8cc42aea19c7..5b045284849e 100644 --- a/net/switchdev/switchdev.c +++ b/net/switchdev/switchdev.c @@ -862,3 +862,28 @@ void switchdev_bridge_port_unoffload(struct net_device *brport_dev, NULL); } EXPORT_SYMBOL_GPL(switchdev_bridge_port_unoffload); + +int switchdev_bridge_port_replay(struct net_device *brport_dev, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack) +{ + struct switchdev_notifier_brport_info brport_info = { + .brport = { + .dev = dev, + .ctx = ctx, + .atomic_nb = atomic_nb, + .blocking_nb = blocking_nb, + }, + }; + int err; + + ASSERT_RTNL(); + + err = call_switchdev_blocking_notifiers(SWITCHDEV_BRPORT_REPLAY, + brport_dev, &brport_info.info, + extack); + return notifier_to_errno(err); +} +EXPORT_SYMBOL_GPL(switchdev_bridge_port_replay); diff --git a/net/tipc/addr.h b/net/tipc/addr.h index 0772cfadaa0d..93f82398283d 100644 --- a/net/tipc/addr.h +++ b/net/tipc/addr.h @@ -131,6 +131,5 @@ bool tipc_in_scope(bool legacy_format, u32 domain, u32 addr); void tipc_set_node_id(struct net *net, u8 *id); void tipc_set_node_addr(struct net *net, u32 addr); char *tipc_nodeid2string(char *str, u8 *id); -u32 tipc_node_id2hash(u8 *id128); #endif diff --git a/net/tipc/bearer.h b/net/tipc/bearer.h index 1ee60649bd17..41eac1ee0c09 100644 --- a/net/tipc/bearer.h +++ b/net/tipc/bearer.h @@ -214,8 +214,6 @@ int tipc_nl_media_get(struct sk_buff *skb, struct genl_info *info); int tipc_nl_media_set(struct sk_buff *skb, struct genl_info *info); int __tipc_nl_media_set(struct sk_buff *skb, struct genl_info *info); -int tipc_media_set_priority(const char *name, u32 new_value); -int tipc_media_set_window(const char *name, u32 new_value); int tipc_media_addr_printf(char *buf, int len, struct tipc_media_addr *a); int tipc_enable_l2_media(struct net *net, struct tipc_bearer *b, struct nlattr *attrs[]); diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h index e231e6964d61..c677f6f082df 100644 --- a/net/tipc/name_distr.h +++ b/net/tipc/name_distr.h @@ -67,7 +67,6 @@ struct distr_item { __be32 key; }; -void tipc_named_bcast(struct net *net, struct sk_buff *skb); struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ); struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ); void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities); diff --git a/net/tipc/net.h b/net/tipc/net.h index d0c91d2df20a..1cb1e43cf34a 100644 --- a/net/tipc/net.h +++ b/net/tipc/net.h @@ -43,7 +43,6 @@ extern const struct nla_policy tipc_nl_net_policy[]; int tipc_net_init(struct net *net, u8 *node_id, u32 addr); void tipc_net_finalize_work(struct work_struct *work); -void tipc_sched_net_finalize(struct net *net, u32 addr); void tipc_net_stop(struct net *net); int tipc_nl_net_dump(struct sk_buff *skb, struct netlink_callback *cb); int tipc_nl_net_set(struct sk_buff *skb, struct genl_info *info); diff --git a/net/tls/tls.h b/net/tls/tls.h index 86cef1c68e03..37539ac3ac2a 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -39,6 +39,7 @@ #include <linux/types.h> #include <linux/skmsg.h> #include <net/tls.h> +#include <net/tls_prot.h> #define TLS_PAGE_ORDER (min_t(unsigned int, PAGE_ALLOC_COSTLY_ORDER, \ TLS_MAX_PAYLOAD_SIZE >> PAGE_SHIFT)) @@ -110,6 +111,8 @@ bool tls_sw_sock_is_readable(struct sock *sk); ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); +int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, + sk_read_actor_t read_actor); int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); void tls_device_splice_eof(struct socket *sock); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 2021fe557e50..5df18f696d7f 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -441,9 +441,13 @@ static int tls_push_data(struct sock *sk, long timeo; if (flags & - ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SPLICE_PAGES)) + ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | + MSG_SPLICE_PAGES | MSG_EOR)) return -EOPNOTSUPP; + if ((flags & (MSG_MORE | MSG_EOR)) == (MSG_MORE | MSG_EOR)) + return -EINVAL; + if (unlikely(sk->sk_err)) return -sk->sk_err; diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index b6896126bb92..7dbb8cd8f809 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -962,10 +962,12 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG] ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE]; ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll; + ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock; ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE]; ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; + ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock; #ifdef CONFIG_TLS_DEVICE ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index f37f4a0fcd3c..ca1e0e198ceb 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -369,7 +369,6 @@ static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, static int tls_strp_read_copyin(struct tls_strparser *strp) { - struct socket *sock = strp->sk->sk_socket; read_descriptor_t desc; desc.arg.data = strp; @@ -377,7 +376,7 @@ static int tls_strp_read_copyin(struct tls_strparser *strp) desc.count = 1; /* give more than one skb per call */ /* sk should be locked here, so okay to do read_sock */ - sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin); + tcp_read_sock(strp->sk, &desc, tls_strp_copyin); return desc.error; } diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 53f944e6d8ef..9c1f13541708 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -984,6 +984,9 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, int ret = 0; int pending; + if (!eor && (msg->msg_flags & MSG_EOR)) + return -EINVAL; + if (unlikely(msg->msg_controllen)) { ret = tls_process_cmsg(sk, msg, &record_type); if (ret) { @@ -1193,7 +1196,7 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) int ret; if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | - MSG_CMSG_COMPAT | MSG_SPLICE_PAGES | + MSG_CMSG_COMPAT | MSG_SPLICE_PAGES | MSG_EOR | MSG_SENDPAGE_NOPOLICY)) return -EOPNOTSUPP; @@ -1845,13 +1848,10 @@ tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot, return sk_flush_backlog(sk); } -static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, - bool nonblock) +static int tls_rx_reader_acquire(struct sock *sk, struct tls_sw_context_rx *ctx, + bool nonblock) { long timeo; - int err; - - lock_sock(sk); timeo = sock_rcvtimeo(sk, nonblock); @@ -1865,26 +1865,30 @@ static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, !READ_ONCE(ctx->reader_present), &wait); remove_wait_queue(&ctx->wq, &wait); - if (timeo <= 0) { - err = -EAGAIN; - goto err_unlock; - } - if (signal_pending(current)) { - err = sock_intr_errno(timeo); - goto err_unlock; - } + if (timeo <= 0) + return -EAGAIN; + if (signal_pending(current)) + return sock_intr_errno(timeo); } WRITE_ONCE(ctx->reader_present, 1); return 0; +} -err_unlock: - release_sock(sk); +static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, + bool nonblock) +{ + int err; + + lock_sock(sk); + err = tls_rx_reader_acquire(sk, ctx, nonblock); + if (err) + release_sock(sk); return err; } -static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) +static void tls_rx_reader_release(struct sock *sk, struct tls_sw_context_rx *ctx) { if (unlikely(ctx->reader_contended)) { if (wq_has_sleeper(&ctx->wq)) @@ -1896,6 +1900,11 @@ static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) } WRITE_ONCE(ctx->reader_present, 0); +} + +static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) +{ + tls_rx_reader_release(sk, ctx); release_sock(sk); } @@ -2193,6 +2202,105 @@ splice_requeue: goto splice_read_end; } +int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, + sk_read_actor_t read_actor) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); + struct tls_prot_info *prot = &tls_ctx->prot_info; + struct strp_msg *rxm = NULL; + struct sk_buff *skb = NULL; + struct sk_psock *psock; + size_t flushed_at = 0; + bool released = true; + struct tls_msg *tlm; + ssize_t copied = 0; + ssize_t decrypted; + int err, used; + + psock = sk_psock_get(sk); + if (psock) { + sk_psock_put(sk, psock); + return -EINVAL; + } + err = tls_rx_reader_acquire(sk, ctx, true); + if (err < 0) + return err; + + /* If crypto failed the connection is broken */ + err = ctx->async_wait.err; + if (err) + goto read_sock_end; + + decrypted = 0; + do { + if (!skb_queue_empty(&ctx->rx_list)) { + skb = __skb_dequeue(&ctx->rx_list); + rxm = strp_msg(skb); + tlm = tls_msg(skb); + } else { + struct tls_decrypt_arg darg; + int to_decrypt; + + err = tls_rx_rec_wait(sk, NULL, true, released); + if (err <= 0) + goto read_sock_end; + + memset(&darg.inargs, 0, sizeof(darg.inargs)); + + rxm = strp_msg(tls_strp_msg(ctx)); + tlm = tls_msg(tls_strp_msg(ctx)); + + to_decrypt = rxm->full_len - prot->overhead_size; + + err = tls_rx_one_record(sk, NULL, &darg); + if (err < 0) { + tls_err_abort(sk, -EBADMSG); + goto read_sock_end; + } + + released = tls_read_flush_backlog(sk, prot, rxm->full_len, to_decrypt, + decrypted, &flushed_at); + skb = darg.skb; + decrypted += rxm->full_len; + + tls_rx_rec_done(ctx); + } + + /* read_sock does not support reading control messages */ + if (tlm->control != TLS_RECORD_TYPE_DATA) { + err = -EINVAL; + goto read_sock_requeue; + } + + used = read_actor(desc, skb, rxm->offset, rxm->full_len); + if (used <= 0) { + if (!copied) + err = used; + goto read_sock_requeue; + } + copied += used; + if (used < rxm->full_len) { + rxm->offset += used; + rxm->full_len -= used; + if (!desc->count) + goto read_sock_requeue; + } else { + consume_skb(skb); + if (!desc->count) + skb = NULL; + } + } while (skb); + +read_sock_end: + tls_rx_reader_release(sk, ctx); + return copied ? : err; + +read_sock_requeue: + __skb_queue_head(&ctx->rx_list, skb); + goto read_sock_end; +} + bool tls_sw_sock_is_readable(struct sock *sk) { struct tls_context *tls_ctx = tls_get_ctx(sk); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index b769fc258931..352d042b130b 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -348,37 +348,34 @@ virtio_transport_stream_do_peek(struct vsock_sock *vsk, size_t len) { struct virtio_vsock_sock *vvs = vsk->trans; - size_t bytes, total = 0, off; - struct sk_buff *skb, *tmp; - int err = -EFAULT; + struct sk_buff *skb; + size_t total = 0; + int err; spin_lock_bh(&vvs->rx_lock); - skb_queue_walk_safe(&vvs->rx_queue, skb, tmp) { - off = 0; + skb_queue_walk(&vvs->rx_queue, skb) { + size_t bytes; - if (total == len) - break; + bytes = len - total; + if (bytes > skb->len) + bytes = skb->len; - while (total < len && off < skb->len) { - bytes = len - total; - if (bytes > skb->len - off) - bytes = skb->len - off; + spin_unlock_bh(&vvs->rx_lock); - /* sk_lock is held by caller so no one else can dequeue. - * Unlock rx_lock since memcpy_to_msg() may sleep. - */ - spin_unlock_bh(&vvs->rx_lock); + /* sk_lock is held by caller so no one else can dequeue. + * Unlock rx_lock since memcpy_to_msg() may sleep. + */ + err = memcpy_to_msg(msg, skb->data, bytes); + if (err) + goto out; - err = memcpy_to_msg(msg, skb->data + off, bytes); - if (err) - goto out; + total += bytes; - spin_lock_bh(&vvs->rx_lock); + spin_lock_bh(&vvs->rx_lock); - total += bytes; - off += bytes; - } + if (total == len) + break; } spin_unlock_bh(&vvs->rx_lock); @@ -463,6 +460,63 @@ out: return err; } +static ssize_t +virtio_transport_seqpacket_do_peek(struct vsock_sock *vsk, + struct msghdr *msg) +{ + struct virtio_vsock_sock *vvs = vsk->trans; + struct sk_buff *skb; + size_t total, len; + + spin_lock_bh(&vvs->rx_lock); + + if (!vvs->msg_count) { + spin_unlock_bh(&vvs->rx_lock); + return 0; + } + + total = 0; + len = msg_data_left(msg); + + skb_queue_walk(&vvs->rx_queue, skb) { + struct virtio_vsock_hdr *hdr; + + if (total < len) { + size_t bytes; + int err; + + bytes = len - total; + if (bytes > skb->len) + bytes = skb->len; + + spin_unlock_bh(&vvs->rx_lock); + + /* sk_lock is held by caller so no one else can dequeue. + * Unlock rx_lock since memcpy_to_msg() may sleep. + */ + err = memcpy_to_msg(msg, skb->data, bytes); + if (err) + return err; + + spin_lock_bh(&vvs->rx_lock); + } + + total += skb->len; + hdr = virtio_vsock_hdr(skb); + + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { + if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) + msg->msg_flags |= MSG_EOR; + + break; + } + } + + spin_unlock_bh(&vvs->rx_lock); + + return total; +} + static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk, struct msghdr *msg, int flags) @@ -557,9 +611,9 @@ virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk, int flags) { if (flags & MSG_PEEK) - return -EOPNOTSUPP; - - return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags); + return virtio_transport_seqpacket_do_peek(vsk, msg); + else + return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags); } EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue); diff --git a/net/vmw_vsock/vmci_transport.h b/net/vmw_vsock/vmci_transport.h index b7b072194282..dbda3ababa14 100644 --- a/net/vmw_vsock/vmci_transport.h +++ b/net/vmw_vsock/vmci_transport.h @@ -116,9 +116,6 @@ struct vmci_transport { spinlock_t lock; /* protects sk. */ }; -int vmci_transport_register(void); -void vmci_transport_unregister(void); - int vmci_transport_send_wrote_bh(struct sockaddr_vm *dst, struct sockaddr_vm *src); int vmci_transport_send_read_bh(struct sockaddr_vm *dst, diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index b89adb52a977..d4ccffcab982 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -135,14 +135,14 @@ int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool, return 0; } -static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) +static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len, + u32 flags) { - struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); u64 addr; int err; addr = xp_get_handle(xskb); - err = xskq_prod_reserve_desc(xs->rx, addr, len); + err = xskq_prod_reserve_desc(xs->rx, addr, len, flags); if (err) { xs->rx_queue_full++; return err; @@ -152,48 +152,138 @@ static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) return 0; } -static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len) +static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) { - void *from_buf, *to_buf; - u32 metalen; + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); + u32 frags = xdp_buff_has_frags(xdp); + struct xdp_buff_xsk *pos, *tmp; + struct list_head *xskb_list; + u32 contd = 0; + int err; - if (unlikely(xdp_data_meta_unsupported(from))) { - from_buf = from->data; - to_buf = to->data; - metalen = 0; - } else { - from_buf = from->data_meta; - metalen = from->data - from->data_meta; - to_buf = to->data - metalen; + if (frags) + contd = XDP_PKT_CONTD; + + err = __xsk_rcv_zc(xs, xskb, len, contd); + if (err || likely(!frags)) + goto out; + + xskb_list = &xskb->pool->xskb_list; + list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) { + if (list_is_singular(xskb_list)) + contd = 0; + len = pos->xdp.data_end - pos->xdp.data; + err = __xsk_rcv_zc(xs, pos, len, contd); + if (err) + return err; + list_del(&pos->xskb_list_node); } - memcpy(to_buf, from_buf, len + metalen); +out: + return err; } -static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) +static void *xsk_copy_xdp_start(struct xdp_buff *from) { + if (unlikely(xdp_data_meta_unsupported(from))) + return from->data; + else + return from->data_meta; +} + +static u32 xsk_copy_xdp(void *to, void **from, u32 to_len, + u32 *from_len, skb_frag_t **frag, u32 rem) +{ + u32 copied = 0; + + while (1) { + u32 copy_len = min_t(u32, *from_len, to_len); + + memcpy(to, *from, copy_len); + copied += copy_len; + if (rem == copied) + return copied; + + if (*from_len == copy_len) { + *from = skb_frag_address(*frag); + *from_len = skb_frag_size((*frag)++); + } else { + *from += copy_len; + *from_len -= copy_len; + } + if (to_len == copy_len) + return copied; + + to_len -= copy_len; + to += copy_len; + } +} + +static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) +{ + u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool); + void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; + u32 from_len, meta_len, rem, num_desc; + struct xdp_buff_xsk *xskb; struct xdp_buff *xsk_xdp; - int err; - u32 len; + skb_frag_t *frag; - len = xdp->data_end - xdp->data; - if (len > xsk_pool_get_rx_frame_size(xs->pool)) { - xs->rx_dropped++; - return -ENOSPC; + from_len = xdp->data_end - copy_from; + meta_len = xdp->data - copy_from; + rem = len + meta_len; + + if (len <= frame_size && !xdp_buff_has_frags(xdp)) { + int err; + + xsk_xdp = xsk_buff_alloc(xs->pool); + if (!xsk_xdp) { + xs->rx_dropped++; + return -ENOMEM; + } + memcpy(xsk_xdp->data - meta_len, copy_from, rem); + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + err = __xsk_rcv_zc(xs, xskb, len, 0); + if (err) { + xsk_buff_free(xsk_xdp); + return err; + } + + return 0; } - xsk_xdp = xsk_buff_alloc(xs->pool); - if (!xsk_xdp) { + num_desc = (len - 1) / frame_size + 1; + + if (!xsk_buff_can_alloc(xs->pool, num_desc)) { xs->rx_dropped++; return -ENOMEM; } + if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) { + xs->rx_queue_full++; + return -ENOBUFS; + } - xsk_copy_xdp(xsk_xdp, xdp, len); - err = __xsk_rcv_zc(xs, xsk_xdp, len); - if (err) { - xsk_buff_free(xsk_xdp); - return err; + if (xdp_buff_has_frags(xdp)) { + struct skb_shared_info *sinfo; + + sinfo = xdp_get_shared_info_from_buff(xdp); + frag = &sinfo->frags[0]; } + + do { + u32 to_len = frame_size + meta_len; + u32 copied; + + xsk_xdp = xsk_buff_alloc(xs->pool); + copy_to = xsk_xdp->data - meta_len; + + copied = xsk_copy_xdp(copy_to, ©_from, to_len, &from_len, &frag, rem); + rem -= copied; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + __xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0); + meta_len = 0; + } while (rem); + return 0; } @@ -215,7 +305,7 @@ static bool xsk_is_bound(struct xdp_sock *xs) return false; } -static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp) +static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) { if (!xsk_is_bound(xs)) return -ENXIO; @@ -223,6 +313,11 @@ static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp) if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) return -EINVAL; + if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) { + xs->rx_dropped++; + return -ENOSPC; + } + sk_mark_napi_id_once_xdp(&xs->sk, xdp); return 0; } @@ -236,12 +331,13 @@ static void xsk_flush(struct xdp_sock *xs) int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { + u32 len = xdp_get_buff_len(xdp); int err; spin_lock_bh(&xs->rx_lock); - err = xsk_rcv_check(xs, xdp); + err = xsk_rcv_check(xs, xdp, len); if (!err) { - err = __xsk_rcv(xs, xdp); + err = __xsk_rcv(xs, xdp, len); xsk_flush(xs); } spin_unlock_bh(&xs->rx_lock); @@ -250,19 +346,19 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { + u32 len = xdp_get_buff_len(xdp); int err; - u32 len; - err = xsk_rcv_check(xs, xdp); + err = xsk_rcv_check(xs, xdp, len); if (err) return err; if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) { len = xdp->data_end - xdp->data; - return __xsk_rcv_zc(xs, xdp, len); + return xsk_rcv_zc(xs, xdp, len); } - err = __xsk_rcv(xs, xdp); + err = __xsk_rcv(xs, xdp, len); if (!err) xdp_return_buff(xdp); return err; @@ -321,7 +417,8 @@ bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) rcu_read_lock(); list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) { if (!xskq_cons_peek_desc(xs->tx, desc, pool)) { - xs->tx->queue_empty_descs++; + if (xskq_has_descs(xs->tx)) + xskq_cons_release(xs->tx); continue; } @@ -408,37 +505,91 @@ static int xsk_wakeup(struct xdp_sock *xs, u8 flags) return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags); } -static void xsk_destruct_skb(struct sk_buff *skb) +static int xsk_cq_reserve_addr_locked(struct xdp_sock *xs, u64 addr) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&xs->pool->cq_lock, flags); + ret = xskq_prod_reserve_addr(xs->pool->cq, addr); + spin_unlock_irqrestore(&xs->pool->cq_lock, flags); + + return ret; +} + +static void xsk_cq_submit_locked(struct xdp_sock *xs, u32 n) { - u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg; - struct xdp_sock *xs = xdp_sk(skb->sk); unsigned long flags; spin_lock_irqsave(&xs->pool->cq_lock, flags); - xskq_prod_submit_addr(xs->pool->cq, addr); + xskq_prod_submit_n(xs->pool->cq, n); spin_unlock_irqrestore(&xs->pool->cq_lock, flags); +} + +static void xsk_cq_cancel_locked(struct xdp_sock *xs, u32 n) +{ + unsigned long flags; + spin_lock_irqsave(&xs->pool->cq_lock, flags); + xskq_prod_cancel_n(xs->pool->cq, n); + spin_unlock_irqrestore(&xs->pool->cq_lock, flags); +} + +static u32 xsk_get_num_desc(struct sk_buff *skb) +{ + return skb ? (long)skb_shinfo(skb)->destructor_arg : 0; +} + +static void xsk_destruct_skb(struct sk_buff *skb) +{ + xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb)); sock_wfree(skb); } +static void xsk_set_destructor_arg(struct sk_buff *skb) +{ + long num = xsk_get_num_desc(xdp_sk(skb->sk)->skb) + 1; + + skb_shinfo(skb)->destructor_arg = (void *)num; +} + +static void xsk_consume_skb(struct sk_buff *skb) +{ + struct xdp_sock *xs = xdp_sk(skb->sk); + + skb->destructor = sock_wfree; + xsk_cq_cancel_locked(xs, xsk_get_num_desc(skb)); + /* Free skb without triggering the perf drop trace */ + consume_skb(skb); + xs->skb = NULL; +} + +static void xsk_drop_skb(struct sk_buff *skb) +{ + xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb); + xsk_consume_skb(skb); +} + static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, struct xdp_desc *desc) { struct xsk_buff_pool *pool = xs->pool; u32 hr, len, ts, offset, copy, copied; - struct sk_buff *skb; + struct sk_buff *skb = xs->skb; struct page *page; void *buffer; int err, i; u64 addr; - hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom)); + if (!skb) { + hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom)); - skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err); - if (unlikely(!skb)) - return ERR_PTR(err); + skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err); + if (unlikely(!skb)) + return ERR_PTR(err); - skb_reserve(skb, hr); + skb_reserve(skb, hr); + } addr = desc->addr; len = desc->len; @@ -448,7 +599,10 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, offset = offset_in_page(buffer); addr = buffer - pool->addrs; - for (copied = 0, i = 0; copied < len; i++) { + for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) { + if (unlikely(i >= MAX_SKB_FRAGS)) + return ERR_PTR(-EFAULT); + page = pool->umem->pgs[addr >> PAGE_SHIFT]; get_page(page); @@ -473,43 +627,77 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct xdp_desc *desc) { struct net_device *dev = xs->dev; - struct sk_buff *skb; + struct sk_buff *skb = xs->skb; + int err; if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) { skb = xsk_build_skb_zerocopy(xs, desc); - if (IS_ERR(skb)) - return skb; + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + goto free_err; + } } else { u32 hr, tr, len; void *buffer; - int err; - hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); - tr = dev->needed_tailroom; + buffer = xsk_buff_raw_get_data(xs->pool, desc->addr); len = desc->len; - skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err); - if (unlikely(!skb)) - return ERR_PTR(err); + if (!skb) { + hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); + tr = dev->needed_tailroom; + skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err); + if (unlikely(!skb)) + goto free_err; - skb_reserve(skb, hr); - skb_put(skb, len); + skb_reserve(skb, hr); + skb_put(skb, len); - buffer = xsk_buff_raw_get_data(xs->pool, desc->addr); - err = skb_store_bits(skb, 0, buffer, len); - if (unlikely(err)) { - kfree_skb(skb); - return ERR_PTR(err); + err = skb_store_bits(skb, 0, buffer, len); + if (unlikely(err)) + goto free_err; + } else { + int nr_frags = skb_shinfo(skb)->nr_frags; + struct page *page; + u8 *vaddr; + + if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) { + err = -EFAULT; + goto free_err; + } + + page = alloc_page(xs->sk.sk_allocation); + if (unlikely(!page)) { + err = -EAGAIN; + goto free_err; + } + + vaddr = kmap_local_page(page); + memcpy(vaddr, buffer, len); + kunmap_local(vaddr); + + skb_add_rx_frag(skb, nr_frags, page, 0, len, 0); } } skb->dev = dev; skb->priority = xs->sk.sk_priority; skb->mark = READ_ONCE(xs->sk.sk_mark); - skb_shinfo(skb)->destructor_arg = (void *)(long)desc->addr; skb->destructor = xsk_destruct_skb; + xsk_set_destructor_arg(skb); return skb; + +free_err: + if (err == -EAGAIN) { + xsk_cq_cancel_locked(xs, 1); + } else { + xsk_set_destructor_arg(skb); + xsk_drop_skb(skb); + xskq_cons_release(xs->tx); + } + + return ERR_PTR(err); } static int __xsk_generic_xmit(struct sock *sk) @@ -519,7 +707,6 @@ static int __xsk_generic_xmit(struct sock *sk) bool sent_frame = false; struct xdp_desc desc; struct sk_buff *skb; - unsigned long flags; int err = 0; mutex_lock(&xs->mutex); @@ -544,47 +731,51 @@ static int __xsk_generic_xmit(struct sock *sk) * if there is space in it. This avoids having to implement * any buffering in the Tx path. */ - spin_lock_irqsave(&xs->pool->cq_lock, flags); - if (xskq_prod_reserve(xs->pool->cq)) { - spin_unlock_irqrestore(&xs->pool->cq_lock, flags); + if (xsk_cq_reserve_addr_locked(xs, desc.addr)) goto out; - } - spin_unlock_irqrestore(&xs->pool->cq_lock, flags); skb = xsk_build_skb(xs, &desc); if (IS_ERR(skb)) { err = PTR_ERR(skb); - spin_lock_irqsave(&xs->pool->cq_lock, flags); - xskq_prod_cancel(xs->pool->cq); - spin_unlock_irqrestore(&xs->pool->cq_lock, flags); - goto out; + if (err == -EAGAIN) + goto out; + err = 0; + continue; + } + + xskq_cons_release(xs->tx); + + if (xp_mb_desc(&desc)) { + xs->skb = skb; + continue; } err = __dev_direct_xmit(skb, xs->queue_id); if (err == NETDEV_TX_BUSY) { /* Tell user-space to retry the send */ - skb->destructor = sock_wfree; - spin_lock_irqsave(&xs->pool->cq_lock, flags); - xskq_prod_cancel(xs->pool->cq); - spin_unlock_irqrestore(&xs->pool->cq_lock, flags); - /* Free skb without triggering the perf drop trace */ - consume_skb(skb); + xskq_cons_cancel_n(xs->tx, xsk_get_num_desc(skb)); + xsk_consume_skb(skb); err = -EAGAIN; goto out; } - xskq_cons_release(xs->tx); /* Ignore NET_XMIT_CN as packet might have been sent */ if (err == NET_XMIT_DROP) { /* SKB completed but not sent */ err = -EBUSY; + xs->skb = NULL; goto out; } sent_frame = true; + xs->skb = NULL; } - xs->tx->queue_empty_descs++; + if (xskq_has_descs(xs->tx)) { + if (xs->skb) + xsk_drop_skb(xs->skb); + xskq_cons_release(xs->tx); + } out: if (sent_frame) @@ -834,6 +1025,9 @@ static int xsk_release(struct socket *sock) net = sock_net(sk); + if (xs->skb) + xsk_drop_skb(xs->skb); + mutex_lock(&net->xdp.lock); sk_del_node_init_rcu(sk); mutex_unlock(&net->xdp.lock); @@ -897,7 +1091,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) flags = sxdp->sxdp_flags; if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY | - XDP_USE_NEED_WAKEUP)) + XDP_USE_NEED_WAKEUP | XDP_USE_SG)) return -EINVAL; bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); @@ -929,7 +1123,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) struct socket *sock; if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) || - (flags & XDP_USE_NEED_WAKEUP)) { + (flags & XDP_USE_NEED_WAKEUP) || (flags & XDP_USE_SG)) { /* Cannot specify flags for shared sockets. */ err = -EINVAL; goto out_unlock; @@ -1028,6 +1222,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) xs->dev = dev; xs->zc = xs->umem->zc; + xs->sg = !!(flags & XDP_USE_SG); xs->queue_id = qid; xp_add_xsk(xs->pool, xs); diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 26f6d304451e..b3f7b310811e 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -86,6 +86,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, pool->umem = umem; pool->addrs = umem->addrs; INIT_LIST_HEAD(&pool->free_list); + INIT_LIST_HEAD(&pool->xskb_list); INIT_LIST_HEAD(&pool->xsk_tx_list); spin_lock_init(&pool->xsk_tx_list_lock); spin_lock_init(&pool->cq_lock); @@ -99,6 +100,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, xskb->pool = pool; xskb->xdp.frame_sz = umem->chunk_size - umem->headroom; INIT_LIST_HEAD(&xskb->free_list_node); + INIT_LIST_HEAD(&xskb->xskb_list_node); if (pool->unaligned) pool->free_heads[i] = xskb; else @@ -187,6 +189,11 @@ int xp_assign_dev(struct xsk_buff_pool *pool, goto err_unreg_pool; } + if (netdev->xdp_zc_max_segs == 1 && (flags & XDP_USE_SG)) { + err = -EOPNOTSUPP; + goto err_unreg_pool; + } + bpf.command = XDP_SETUP_XSK_POOL; bpf.xsk.pool = pool; bpf.xsk.queue_id = queue_id; diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 6d40a77fccbe..13354a1e4280 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -48,6 +48,11 @@ struct xsk_queue { size_t ring_vmalloc_size; }; +struct parsed_desc { + u32 mb; + u32 valid; +}; + /* The structure of the shared state of the rings are a simple * circular buffer, as outlined in * Documentation/core-api/circular-buffers.rst. For the Rx and @@ -130,18 +135,26 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr) return false; } +static inline bool xp_unused_options_set(u32 options) +{ + return options & ~XDP_PKT_CONTD; +} + static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) { u64 offset = desc->addr & (pool->chunk_size - 1); + if (!desc->len) + return false; + if (offset + desc->len > pool->chunk_size) return false; if (desc->addr >= pool->addrs_cnt) return false; - if (desc->options) + if (xp_unused_options_set(desc->options)) return false; return true; } @@ -151,6 +164,9 @@ static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool, { u64 addr = xp_unaligned_add_offset_to_addr(desc->addr); + if (!desc->len) + return false; + if (desc->len > pool->chunk_size) return false; @@ -158,7 +174,7 @@ static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool, xp_desc_crosses_non_contig_pg(pool, addr, desc->len)) return false; - if (desc->options) + if (xp_unused_options_set(desc->options)) return false; return true; } @@ -170,6 +186,11 @@ static inline bool xp_validate_desc(struct xsk_buff_pool *pool, xp_aligned_validate_desc(pool, desc); } +static inline bool xskq_has_descs(struct xsk_queue *q) +{ + return q->cached_cons != q->cached_prod; +} + static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d, struct xsk_buff_pool *pool) @@ -185,17 +206,15 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q, struct xdp_desc *desc, struct xsk_buff_pool *pool) { - while (q->cached_cons != q->cached_prod) { + if (q->cached_cons != q->cached_prod) { struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; u32 idx = q->cached_cons & q->ring_mask; *desc = ring->desc[idx]; - if (xskq_cons_is_valid_desc(q, desc, pool)) - return true; - - q->cached_cons++; + return xskq_cons_is_valid_desc(q, desc, pool); } + q->queue_empty_descs++; return false; } @@ -204,30 +223,52 @@ static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt) q->cached_cons += cnt; } -static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, - u32 max) +static inline void parse_desc(struct xsk_queue *q, struct xsk_buff_pool *pool, + struct xdp_desc *desc, struct parsed_desc *parsed) +{ + parsed->valid = xskq_cons_is_valid_desc(q, desc, pool); + parsed->mb = xp_mb_desc(desc); +} + +static inline +u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, + u32 max) { u32 cached_cons = q->cached_cons, nb_entries = 0; struct xdp_desc *descs = pool->tx_descs; + u32 total_descs = 0, nr_frags = 0; + /* track first entry, if stumble upon *any* invalid descriptor, rewind + * current packet that consists of frags and stop the processing + */ while (cached_cons != q->cached_prod && nb_entries < max) { struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; u32 idx = cached_cons & q->ring_mask; + struct parsed_desc parsed; descs[nb_entries] = ring->desc[idx]; - if (unlikely(!xskq_cons_is_valid_desc(q, &descs[nb_entries], pool))) { - /* Skip the entry */ - cached_cons++; - continue; + cached_cons++; + parse_desc(q, pool, &descs[nb_entries], &parsed); + if (unlikely(!parsed.valid)) + break; + + if (likely(!parsed.mb)) { + total_descs += (nr_frags + 1); + nr_frags = 0; + } else { + nr_frags++; + if (nr_frags == pool->netdev->xdp_zc_max_segs) { + nr_frags = 0; + break; + } } - nb_entries++; - cached_cons++; } + cached_cons -= nr_frags; /* Release valid plus any invalid entries */ xskq_cons_release_n(q, cached_cons - q->cached_cons); - return nb_entries; + return total_descs; } /* Functions for consumers */ @@ -292,6 +333,11 @@ static inline void xskq_cons_release(struct xsk_queue *q) q->cached_cons++; } +static inline void xskq_cons_cancel_n(struct xsk_queue *q, u32 cnt) +{ + q->cached_cons -= cnt; +} + static inline u32 xskq_cons_present_entries(struct xsk_queue *q) { /* No barriers needed since data is not accessed */ @@ -319,9 +365,9 @@ static inline bool xskq_prod_is_full(struct xsk_queue *q) return xskq_prod_nb_free(q, 1) ? false : true; } -static inline void xskq_prod_cancel(struct xsk_queue *q) +static inline void xskq_prod_cancel_n(struct xsk_queue *q, u32 cnt) { - q->cached_prod--; + q->cached_prod -= cnt; } static inline int xskq_prod_reserve(struct xsk_queue *q) @@ -360,7 +406,7 @@ static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_de } static inline int xskq_prod_reserve_desc(struct xsk_queue *q, - u64 addr, u32 len) + u64 addr, u32 len, u32 flags) { struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; u32 idx; @@ -372,6 +418,7 @@ static inline int xskq_prod_reserve_desc(struct xsk_queue *q, idx = q->cached_prod++ & q->ring_mask; ring->desc[idx].addr = addr; ring->desc[idx].len = len; + ring->desc[idx].options = flags; return 0; } @@ -386,16 +433,6 @@ static inline void xskq_prod_submit(struct xsk_queue *q) __xskq_prod_submit(q, q->cached_prod); } -static inline void xskq_prod_submit_addr(struct xsk_queue *q, u64 addr) -{ - struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring; - u32 idx = q->ring->producer; - - ring->desc[idx++ & q->ring_mask] = addr; - - __xskq_prod_submit(q, idx); -} - static inline void xskq_prod_submit_n(struct xsk_queue *q, u32 nb_entries) { __xskq_prod_submit(q, q->ring->producer + nb_entries); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 533697e2488f..3784534c9185 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -247,12 +247,6 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, return -EINVAL; } - /* We don't yet support UDP encapsulation and TFC padding. */ - if (x->encap || x->tfcpad) { - NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded"); - return -EINVAL; - } - if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) { NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); @@ -260,6 +254,13 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET; + + /* We don't yet support UDP encapsulation and TFC padding. */ + if ((!is_packet_offload && x->encap) || x->tfcpad) { + NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded"); + return -EINVAL; + } + dev = dev_get_by_index(net, xuo->ifindex); if (!dev) { if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 615f24ebc49c..595b98d825ce 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -248,7 +248,7 @@ BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help 2>&1 | grep dwarfris) BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help 2>&1 | grep BTF) BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --help 2>&1 | grep -i 'usage.*llvm') BTF_LLVM_PROBE := $(shell echo "int main() { return 0; }" | \ - $(CLANG) -target bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \ + $(CLANG) --target=bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \ $(LLVM_READELF) -S ./llvm_btf_verify.o | grep BTF; \ /bin/rm -f ./llvm_btf_verify.o) @@ -370,7 +370,7 @@ endif clean-files += vmlinux.h # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -392,7 +392,7 @@ $(obj)/xdp_router_ipv4.bpf.o: $(obj)/xdp_sample.bpf.o $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/xdp_sample_shared.h @echo " CLANG-BPF " $@ - $(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(SRCARCH) \ + $(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \ -Wno-compare-distinct-pointer-types -I$(srctree)/include \ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \ diff --git a/samples/bpf/README.rst b/samples/bpf/README.rst index 57f93edd1957..f16fc48e55a5 100644 --- a/samples/bpf/README.rst +++ b/samples/bpf/README.rst @@ -8,11 +8,14 @@ Build dependencies ================== Compiling requires having installed: - * clang >= version 3.4.0 - * llvm >= version 3.7.1 + * clang + * llvm + * pahole -Note that LLVM's tool 'llc' must support target 'bpf', list version -and supported targets with command: ``llc --version`` +Consult :ref:`Documentation/process/changes.rst <changes>` for the minimum +version numbers required and how to update them. Note that LLVM's tool +'llc' must support target 'bpf', list version and supported targets with +command: ``llc --version`` Clean and configuration ----------------------- @@ -24,7 +27,8 @@ after some changes (on demand):: make -C samples/bpf clean make clean -Configure kernel, defconfig for instance:: +Configure kernel, defconfig for instance +(see "tools/testing/selftests/bpf/config" for a reference config):: make defconfig diff --git a/samples/bpf/gnu/stubs.h b/samples/bpf/gnu/stubs.h index 719225b16626..1c638d9dce1a 100644 --- a/samples/bpf/gnu/stubs.h +++ b/samples/bpf/gnu/stubs.h @@ -1 +1 @@ -/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ +/* dummy .h to trick /usr/include/features.h to work with 'clang --target=bpf' */ diff --git a/samples/bpf/syscall_tp_kern.c b/samples/bpf/syscall_tp_kern.c index e7121dd1ee37..090fecfe641a 100644 --- a/samples/bpf/syscall_tp_kern.c +++ b/samples/bpf/syscall_tp_kern.c @@ -44,12 +44,14 @@ static __always_inline void count(void *map) bpf_map_update_elem(map, &key, &init_val, BPF_NOEXIST); } +#if !defined(__aarch64__) SEC("tracepoint/syscalls/sys_enter_open") int trace_enter_open(struct syscalls_enter_open_args *ctx) { count(&enter_open_map); return 0; } +#endif SEC("tracepoint/syscalls/sys_enter_openat") int trace_enter_open_at(struct syscalls_enter_open_args *ctx) @@ -65,12 +67,14 @@ int trace_enter_open_at2(struct syscalls_enter_open_args *ctx) return 0; } +#if !defined(__aarch64__) SEC("tracepoint/syscalls/sys_exit_open") int trace_enter_exit(struct syscalls_exit_open_args *ctx) { count(&exit_open_map); return 0; } +#endif SEC("tracepoint/syscalls/sys_exit_openat") int trace_enter_exit_at(struct syscalls_exit_open_args *ctx) diff --git a/samples/bpf/test_lwt_bpf.sh b/samples/bpf/test_lwt_bpf.sh index 0bf2d0f6bf4b..148e2df6cdce 100755 --- a/samples/bpf/test_lwt_bpf.sh +++ b/samples/bpf/test_lwt_bpf.sh @@ -376,7 +376,7 @@ DST_MAC=$(lookup_mac $VETH1 $NS1) SRC_MAC=$(lookup_mac $VETH0) DST_IFINDEX=$(cat /sys/class/net/$VETH0/ifindex) -CLANG_OPTS="-O2 -target bpf -I ../include/" +CLANG_OPTS="-O2 --target=bpf -I ../include/" CLANG_OPTS+=" -DSRC_MAC=$SRC_MAC -DDST_MAC=$DST_MAC -DDST_IFINDEX=$DST_IFINDEX" clang $CLANG_OPTS -c $PROG_SRC -o $BPF_PROG diff --git a/samples/hid/Makefile b/samples/hid/Makefile index 026288280a03..9f7fe29dd749 100644 --- a/samples/hid/Makefile +++ b/samples/hid/Makefile @@ -86,7 +86,7 @@ BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help 2>&1 | grep dwarfris) BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help 2>&1 | grep BTF) BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --help 2>&1 | grep -i 'usage.*llvm') BTF_LLVM_PROBE := $(shell echo "int main() { return 0; }" | \ - $(CLANG) -target bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \ + $(CLANG) --target=bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \ $(LLVM_READELF) -S ./llvm_btf_verify.o | grep BTF; \ /bin/rm -f ./llvm_btf_verify.o) @@ -181,7 +181,7 @@ endif clean-files += vmlinux.h # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -198,7 +198,7 @@ EXTRA_BPF_HEADERS_SRC := $(addprefix $(src)/,$(EXTRA_BPF_HEADERS)) $(obj)/%.bpf.o: $(src)/%.bpf.c $(EXTRA_BPF_HEADERS_SRC) $(obj)/vmlinux.h @echo " CLANG-BPF " $@ - $(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(SRCARCH) \ + $(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \ -Wno-compare-distinct-pointer-types -I$(srctree)/include \ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \ diff --git a/security/security.c b/security/security.c index b720424ca37d..2dfc7b9f6ed9 100644 --- a/security/security.c +++ b/security/security.c @@ -4396,7 +4396,7 @@ void security_sk_clone(const struct sock *sk, struct sock *newsk) } EXPORT_SYMBOL(security_sk_clone); -void security_sk_classify_flow(struct sock *sk, struct flowi_common *flic) +void security_sk_classify_flow(const struct sock *sk, struct flowi_common *flic) { call_void_hook(sk_getsecid, sk, &flic->flowic_secid); } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index d06e350fedee..2bdc48dd8670 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -5167,12 +5167,12 @@ static void selinux_sk_clone_security(const struct sock *sk, struct sock *newsk) selinux_netlbl_sk_security_reset(newsksec); } -static void selinux_sk_getsecid(struct sock *sk, u32 *secid) +static void selinux_sk_getsecid(const struct sock *sk, u32 *secid) { if (!sk) *secid = SECINITSID_ANY_SOCKET; else { - struct sk_security_struct *sksec = sk->sk_security; + const struct sk_security_struct *sksec = sk->sk_security; *secid = sksec->sid; } diff --git a/tools/bpf/bpftool/Documentation/bpftool-gen.rst b/tools/bpf/bpftool/Documentation/bpftool-gen.rst index 68454ef28f58..5006e724d1bc 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-gen.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-gen.rst @@ -260,9 +260,9 @@ EXAMPLES This is example BPF application with two BPF programs and a mix of BPF maps and global variables. Source code is split across two source code files. -**$ clang -target bpf -g example1.bpf.c -o example1.bpf.o** +**$ clang --target=bpf -g example1.bpf.c -o example1.bpf.o** -**$ clang -target bpf -g example2.bpf.c -o example2.bpf.o** +**$ clang --target=bpf -g example2.bpf.c -o example2.bpf.o** **$ bpftool gen object example.bpf.o example1.bpf.o example2.bpf.o** diff --git a/tools/bpf/bpftool/Documentation/bpftool-net.rst b/tools/bpf/bpftool/Documentation/bpftool-net.rst index f4e0a516335a..5e2abd3de5ab 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-net.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-net.rst @@ -4,7 +4,7 @@ bpftool-net ================ ------------------------------------------------------------------------------- -tool for inspection of netdev/tc related bpf prog attachments +tool for inspection of networking related bpf prog attachments ------------------------------------------------------------------------------- :Manual section: 8 @@ -37,10 +37,13 @@ DESCRIPTION **bpftool net { show | list }** [ **dev** *NAME* ] List bpf program attachments in the kernel networking subsystem. - Currently, only device driver xdp attachments and tc filter - classification/action attachments are implemented, i.e., for - program types **BPF_PROG_TYPE_SCHED_CLS**, - **BPF_PROG_TYPE_SCHED_ACT** and **BPF_PROG_TYPE_XDP**. + Currently, device driver xdp attachments, tcx and old-style tc + classifier/action attachments, flow_dissector as well as netfilter + attachments are implemented, i.e., for + program types **BPF_PROG_TYPE_XDP**, **BPF_PROG_TYPE_SCHED_CLS**, + **BPF_PROG_TYPE_SCHED_ACT**, **BPF_PROG_TYPE_FLOW_DISSECTOR**, + **BPF_PROG_TYPE_NETFILTER**. + For programs attached to a particular cgroup, e.g., **BPF_PROG_TYPE_CGROUP_SKB**, **BPF_PROG_TYPE_CGROUP_SOCK**, **BPF_PROG_TYPE_SOCK_OPS** and **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, @@ -49,12 +52,13 @@ DESCRIPTION bpf programs, users should consult other tools, e.g., iproute2. The current output will start with all xdp program attachments, followed by - all tc class/qdisc bpf program attachments. Both xdp programs and - tc programs are ordered based on ifindex number. If multiple bpf - programs attached to the same networking device through **tc filter**, - the order will be first all bpf programs attached to tc classes, then - all bpf programs attached to non clsact qdiscs, and finally all - bpf programs attached to root and clsact qdisc. + all tcx, then tc class/qdisc bpf program attachments, then flow_dissector + and finally netfilter programs. Both xdp programs and tcx/tc programs are + ordered based on ifindex number. If multiple bpf programs attached + to the same networking device through **tc**, the order will be first + all bpf programs attached to tcx, then tc classes, then all bpf programs + attached to non clsact qdiscs, and finally all bpf programs attached + to root and clsact qdisc. **bpftool** **net attach** *ATTACH_TYPE* *PROG* **dev** *NAME* [ **overwrite** ] Attach bpf program *PROG* to network interface *NAME* with diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 681fbcc5ed50..e9154ace80ff 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -216,7 +216,7 @@ $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF_BOOTSTRAP) -I$(srctree)/tools/include/uapi/ \ -I$(LIBBPF_BOOTSTRAP_INCLUDE) \ -g -O2 -Wall -fno-stack-protector \ - -target bpf -c $< -o $@ + --target=bpf -c $< -o $@ $(Q)$(LLVM_STRIP) -g $@ $(OUTPUT)%.skel.h: $(OUTPUT)%.bpf.o $(BPFTOOL_BOOTSTRAP) diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 294de231db99..1b7f69714604 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -835,7 +835,7 @@ static void dotlabel_puts(const char *s) case '|': case ' ': putchar('\\'); - /* fallthrough */ + fallthrough; default: putchar(*s); } diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 0675d6a46413..edda4fc2c4d0 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -757,7 +757,7 @@ probe_helpers_for_progtype(enum bpf_prog_type prog_type, case BPF_FUNC_probe_write_user: if (!full_mode) continue; - /* fallthrough */ + fallthrough; default: probe_res |= probe_helper_for_progtype(prog_type, supported_type, define_prefix, id, prog_type_str, diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 2d786072ed0d..65a168df63bc 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -5,6 +5,7 @@ #include <linux/err.h> #include <linux/netfilter.h> #include <linux/netfilter_arp.h> +#include <linux/perf_event.h> #include <net/if.h> #include <stdio.h> #include <unistd.h> @@ -14,8 +15,78 @@ #include "json_writer.h" #include "main.h" +#include "xlated_dumper.h" + +#define PERF_HW_CACHE_LEN 128 static struct hashmap *link_table; +static struct dump_data dd; + +static const char *perf_type_name[PERF_TYPE_MAX] = { + [PERF_TYPE_HARDWARE] = "hardware", + [PERF_TYPE_SOFTWARE] = "software", + [PERF_TYPE_TRACEPOINT] = "tracepoint", + [PERF_TYPE_HW_CACHE] = "hw-cache", + [PERF_TYPE_RAW] = "raw", + [PERF_TYPE_BREAKPOINT] = "breakpoint", +}; + +const char *event_symbols_hw[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = "cpu-cycles", + [PERF_COUNT_HW_INSTRUCTIONS] = "instructions", + [PERF_COUNT_HW_CACHE_REFERENCES] = "cache-references", + [PERF_COUNT_HW_CACHE_MISSES] = "cache-misses", + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "branch-instructions", + [PERF_COUNT_HW_BRANCH_MISSES] = "branch-misses", + [PERF_COUNT_HW_BUS_CYCLES] = "bus-cycles", + [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "stalled-cycles-frontend", + [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "stalled-cycles-backend", + [PERF_COUNT_HW_REF_CPU_CYCLES] = "ref-cycles", +}; + +const char *event_symbols_sw[PERF_COUNT_SW_MAX] = { + [PERF_COUNT_SW_CPU_CLOCK] = "cpu-clock", + [PERF_COUNT_SW_TASK_CLOCK] = "task-clock", + [PERF_COUNT_SW_PAGE_FAULTS] = "page-faults", + [PERF_COUNT_SW_CONTEXT_SWITCHES] = "context-switches", + [PERF_COUNT_SW_CPU_MIGRATIONS] = "cpu-migrations", + [PERF_COUNT_SW_PAGE_FAULTS_MIN] = "minor-faults", + [PERF_COUNT_SW_PAGE_FAULTS_MAJ] = "major-faults", + [PERF_COUNT_SW_ALIGNMENT_FAULTS] = "alignment-faults", + [PERF_COUNT_SW_EMULATION_FAULTS] = "emulation-faults", + [PERF_COUNT_SW_DUMMY] = "dummy", + [PERF_COUNT_SW_BPF_OUTPUT] = "bpf-output", + [PERF_COUNT_SW_CGROUP_SWITCHES] = "cgroup-switches", +}; + +const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX] = { + [PERF_COUNT_HW_CACHE_L1D] = "L1-dcache", + [PERF_COUNT_HW_CACHE_L1I] = "L1-icache", + [PERF_COUNT_HW_CACHE_LL] = "LLC", + [PERF_COUNT_HW_CACHE_DTLB] = "dTLB", + [PERF_COUNT_HW_CACHE_ITLB] = "iTLB", + [PERF_COUNT_HW_CACHE_BPU] = "branch", + [PERF_COUNT_HW_CACHE_NODE] = "node", +}; + +const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX] = { + [PERF_COUNT_HW_CACHE_OP_READ] = "load", + [PERF_COUNT_HW_CACHE_OP_WRITE] = "store", + [PERF_COUNT_HW_CACHE_OP_PREFETCH] = "prefetch", +}; + +const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [PERF_COUNT_HW_CACHE_RESULT_ACCESS] = "refs", + [PERF_COUNT_HW_CACHE_RESULT_MISS] = "misses", +}; + +#define perf_event_name(array, id) ({ \ + const char *event_str = NULL; \ + \ + if ((id) >= 0 && (id) < ARRAY_SIZE(array)) \ + event_str = array[id]; \ + event_str; \ +}) static int link_parse_fd(int *argc, char ***argv) { @@ -166,6 +237,154 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info) return err; } +static int cmp_u64(const void *A, const void *B) +{ + const __u64 *a = A, *b = B; + + return *a - *b; +} + +static void +show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + __u32 i, j = 0; + __u64 *addrs; + + jsonw_bool_field(json_wtr, "retprobe", + info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN); + jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count); + jsonw_name(json_wtr, "funcs"); + jsonw_start_array(json_wtr); + addrs = u64_to_ptr(info->kprobe_multi.addrs); + qsort(addrs, info->kprobe_multi.count, sizeof(addrs[0]), cmp_u64); + + /* Load it once for all. */ + if (!dd.sym_count) + kernel_syms_load(&dd); + for (i = 0; i < dd.sym_count; i++) { + if (dd.sym_mapping[i].address != addrs[j]) + continue; + jsonw_start_object(json_wtr); + jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address); + jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name); + /* Print null if it is vmlinux */ + if (dd.sym_mapping[i].module[0] == '\0') { + jsonw_name(json_wtr, "module"); + jsonw_null(json_wtr); + } else { + jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module); + } + jsonw_end_object(json_wtr); + if (j++ == info->kprobe_multi.count) + break; + } + jsonw_end_array(json_wtr); +} + +static void +show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_KRETPROBE); + jsonw_uint_field(wtr, "addr", info->perf_event.kprobe.addr); + jsonw_string_field(wtr, "func", + u64_to_ptr(info->perf_event.kprobe.func_name)); + jsonw_uint_field(wtr, "offset", info->perf_event.kprobe.offset); +} + +static void +show_perf_event_uprobe_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_URETPROBE); + jsonw_string_field(wtr, "file", + u64_to_ptr(info->perf_event.uprobe.file_name)); + jsonw_uint_field(wtr, "offset", info->perf_event.uprobe.offset); +} + +static void +show_perf_event_tracepoint_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_string_field(wtr, "tracepoint", + u64_to_ptr(info->perf_event.tracepoint.tp_name)); +} + +static char *perf_config_hw_cache_str(__u64 config) +{ + const char *hw_cache, *result, *op; + char *str = malloc(PERF_HW_CACHE_LEN); + + if (!str) { + p_err("mem alloc failed"); + return NULL; + } + + hw_cache = perf_event_name(evsel__hw_cache, config & 0xff); + if (hw_cache) + snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache); + else + snprintf(str, PERF_HW_CACHE_LEN, "%lld-", config & 0xff); + + op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff); + if (op) + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%s-", op); + else + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%lld-", (config >> 8) & 0xff); + + result = perf_event_name(evsel__hw_cache_result, config >> 16); + if (result) + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%s", result); + else + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%lld", config >> 16); + return str; +} + +static const char *perf_config_str(__u32 type, __u64 config) +{ + const char *perf_config; + + switch (type) { + case PERF_TYPE_HARDWARE: + perf_config = perf_event_name(event_symbols_hw, config); + break; + case PERF_TYPE_SOFTWARE: + perf_config = perf_event_name(event_symbols_sw, config); + break; + case PERF_TYPE_HW_CACHE: + perf_config = perf_config_hw_cache_str(config); + break; + default: + perf_config = NULL; + break; + } + return perf_config; +} + +static void +show_perf_event_event_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + __u64 config = info->perf_event.event.config; + __u32 type = info->perf_event.event.type; + const char *perf_type, *perf_config; + + perf_type = perf_event_name(perf_type_name, type); + if (perf_type) + jsonw_string_field(wtr, "event_type", perf_type); + else + jsonw_uint_field(wtr, "event_type", type); + + perf_config = perf_config_str(type, config); + if (perf_config) + jsonw_string_field(wtr, "event_config", perf_config); + else + jsonw_uint_field(wtr, "event_config", config); + + if (type == PERF_TYPE_HW_CACHE && perf_config) + free((void *)perf_config); +} + static int show_link_close_json(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; @@ -218,6 +437,29 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) jsonw_uint_field(json_wtr, "map_id", info->struct_ops.map_id); break; + case BPF_LINK_TYPE_KPROBE_MULTI: + show_kprobe_multi_json(info, json_wtr); + break; + case BPF_LINK_TYPE_PERF_EVENT: + switch (info->perf_event.type) { + case BPF_PERF_EVENT_EVENT: + show_perf_event_event_json(info, json_wtr); + break; + case BPF_PERF_EVENT_TRACEPOINT: + show_perf_event_tracepoint_json(info, json_wtr); + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + show_perf_event_kprobe_json(info, json_wtr); + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + show_perf_event_uprobe_json(info, json_wtr); + break; + default: + break; + } + break; default: break; } @@ -351,6 +593,113 @@ void netfilter_dump_plain(const struct bpf_link_info *info) printf(" flags 0x%x", info->netfilter.flags); } +static void show_kprobe_multi_plain(struct bpf_link_info *info) +{ + __u32 i, j = 0; + __u64 *addrs; + + if (!info->kprobe_multi.count) + return; + + if (info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN) + printf("\n\tkretprobe.multi "); + else + printf("\n\tkprobe.multi "); + printf("func_cnt %u ", info->kprobe_multi.count); + addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs); + qsort(addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64); + + /* Load it once for all. */ + if (!dd.sym_count) + kernel_syms_load(&dd); + if (!dd.sym_count) + return; + + printf("\n\t%-16s %s", "addr", "func [module]"); + for (i = 0; i < dd.sym_count; i++) { + if (dd.sym_mapping[i].address != addrs[j]) + continue; + printf("\n\t%016lx %s", + dd.sym_mapping[i].address, dd.sym_mapping[i].name); + if (dd.sym_mapping[i].module[0] != '\0') + printf(" [%s] ", dd.sym_mapping[i].module); + else + printf(" "); + + if (j++ == info->kprobe_multi.count) + break; + } +} + +static void show_perf_event_kprobe_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.kprobe.func_name); + if (buf[0] == '\0' && !info->perf_event.kprobe.addr) + return; + + if (info->perf_event.type == BPF_PERF_EVENT_KRETPROBE) + printf("\n\tkretprobe "); + else + printf("\n\tkprobe "); + if (info->perf_event.kprobe.addr) + printf("%llx ", info->perf_event.kprobe.addr); + printf("%s", buf); + if (info->perf_event.kprobe.offset) + printf("+%#x", info->perf_event.kprobe.offset); + printf(" "); +} + +static void show_perf_event_uprobe_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.uprobe.file_name); + if (buf[0] == '\0') + return; + + if (info->perf_event.type == BPF_PERF_EVENT_URETPROBE) + printf("\n\turetprobe "); + else + printf("\n\tuprobe "); + printf("%s+%#x ", buf, info->perf_event.uprobe.offset); +} + +static void show_perf_event_tracepoint_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.tracepoint.tp_name); + if (buf[0] == '\0') + return; + + printf("\n\ttracepoint %s ", buf); +} + +static void show_perf_event_event_plain(struct bpf_link_info *info) +{ + __u64 config = info->perf_event.event.config; + __u32 type = info->perf_event.event.type; + const char *perf_type, *perf_config; + + printf("\n\tevent "); + perf_type = perf_event_name(perf_type_name, type); + if (perf_type) + printf("%s:", perf_type); + else + printf("%u :", type); + + perf_config = perf_config_str(type, config); + if (perf_config) + printf("%s ", perf_config); + else + printf("%llu ", config); + + if (type == PERF_TYPE_HW_CACHE && perf_config) + free((void *)perf_config); +} + static int show_link_close_plain(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; @@ -396,6 +745,29 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_NETFILTER: netfilter_dump_plain(info); break; + case BPF_LINK_TYPE_KPROBE_MULTI: + show_kprobe_multi_plain(info); + break; + case BPF_LINK_TYPE_PERF_EVENT: + switch (info->perf_event.type) { + case BPF_PERF_EVENT_EVENT: + show_perf_event_event_plain(info); + break; + case BPF_PERF_EVENT_TRACEPOINT: + show_perf_event_tracepoint_plain(info); + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + show_perf_event_kprobe_plain(info); + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + show_perf_event_uprobe_plain(info); + break; + default: + break; + } + break; default: break; } @@ -417,10 +789,13 @@ static int do_show_link(int fd) { struct bpf_link_info info; __u32 len = sizeof(info); - char buf[256]; + __u64 *addrs = NULL; + char buf[PATH_MAX]; + int count; int err; memset(&info, 0, sizeof(info)); + buf[0] = '\0'; again: err = bpf_link_get_info_by_fd(fd, &info, &len); if (err) { @@ -431,22 +806,67 @@ again: } if (info.type == BPF_LINK_TYPE_RAW_TRACEPOINT && !info.raw_tracepoint.tp_name) { - info.raw_tracepoint.tp_name = (unsigned long)&buf; + info.raw_tracepoint.tp_name = ptr_to_u64(&buf); info.raw_tracepoint.tp_name_len = sizeof(buf); goto again; } if (info.type == BPF_LINK_TYPE_ITER && !info.iter.target_name) { - info.iter.target_name = (unsigned long)&buf; + info.iter.target_name = ptr_to_u64(&buf); info.iter.target_name_len = sizeof(buf); goto again; } + if (info.type == BPF_LINK_TYPE_KPROBE_MULTI && + !info.kprobe_multi.addrs) { + count = info.kprobe_multi.count; + if (count) { + addrs = calloc(count, sizeof(__u64)); + if (!addrs) { + p_err("mem alloc failed"); + close(fd); + return -ENOMEM; + } + info.kprobe_multi.addrs = ptr_to_u64(addrs); + goto again; + } + } + if (info.type == BPF_LINK_TYPE_PERF_EVENT) { + switch (info.perf_event.type) { + case BPF_PERF_EVENT_TRACEPOINT: + if (!info.perf_event.tracepoint.tp_name) { + info.perf_event.tracepoint.tp_name = ptr_to_u64(&buf); + info.perf_event.tracepoint.name_len = sizeof(buf); + goto again; + } + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + if (!info.perf_event.kprobe.func_name) { + info.perf_event.kprobe.func_name = ptr_to_u64(&buf); + info.perf_event.kprobe.name_len = sizeof(buf); + goto again; + } + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + if (!info.perf_event.uprobe.file_name) { + info.perf_event.uprobe.file_name = ptr_to_u64(&buf); + info.perf_event.uprobe.name_len = sizeof(buf); + goto again; + } + break; + default: + break; + } + } if (json_output) show_link_close_json(fd, &info); else show_link_close_plain(fd, &info); + if (addrs) + free(addrs); close(fd); return 0; } @@ -471,7 +891,8 @@ static int do_show(int argc, char **argv) fd = link_parse_fd(&argc, &argv); if (fd < 0) return fd; - return do_show_link(fd); + do_show_link(fd); + goto out; } if (argc) @@ -510,6 +931,9 @@ static int do_show(int argc, char **argv) if (show_pinned) delete_pinned_obj_table(link_table); +out: + if (dd.sym_count) + kernel_syms_destroy(&dd); return errno == ENOENT ? 0 : -1; } diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index 26a49965bf71..66a8ce8ae012 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -76,6 +76,11 @@ static const char * const attach_type_strings[] = { [NET_ATTACH_TYPE_XDP_OFFLOAD] = "xdpoffload", }; +static const char * const attach_loc_strings[] = { + [BPF_TCX_INGRESS] = "tcx/ingress", + [BPF_TCX_EGRESS] = "tcx/egress", +}; + const size_t net_attach_type_size = ARRAY_SIZE(attach_type_strings); static enum net_attach_type parse_attach_type(const char *str) @@ -422,8 +427,89 @@ static int dump_filter_nlmsg(void *cookie, void *msg, struct nlattr **tb) filter_info->devname, filter_info->ifindex); } -static int show_dev_tc_bpf(int sock, unsigned int nl_pid, - struct ip_devname_ifindex *dev) +static int __show_dev_tc_bpf_name(__u32 id, char *name, size_t len) +{ + struct bpf_prog_info info = {}; + __u32 ilen = sizeof(info); + int fd, ret; + + fd = bpf_prog_get_fd_by_id(id); + if (fd < 0) + return fd; + ret = bpf_obj_get_info_by_fd(fd, &info, &ilen); + if (ret < 0) + goto out; + ret = -ENOENT; + if (info.name[0]) { + get_prog_full_name(&info, fd, name, len); + ret = 0; + } +out: + close(fd); + return ret; +} + +static void __show_dev_tc_bpf(const struct ip_devname_ifindex *dev, + const enum bpf_attach_type loc) +{ + __u32 prog_flags[64] = {}, link_flags[64] = {}, i, j; + __u32 prog_ids[64] = {}, link_ids[64] = {}; + LIBBPF_OPTS(bpf_prog_query_opts, optq); + char prog_name[MAX_PROG_FULL_NAME]; + int ret; + + optq.prog_ids = prog_ids; + optq.prog_attach_flags = prog_flags; + optq.link_ids = link_ids; + optq.link_attach_flags = link_flags; + optq.count = ARRAY_SIZE(prog_ids); + + ret = bpf_prog_query_opts(dev->ifindex, loc, &optq); + if (ret) + return; + for (i = 0; i < optq.count; i++) { + NET_START_OBJECT; + NET_DUMP_STR("devname", "%s", dev->devname); + NET_DUMP_UINT("ifindex", "(%u)", dev->ifindex); + NET_DUMP_STR("kind", " %s", attach_loc_strings[loc]); + ret = __show_dev_tc_bpf_name(prog_ids[i], prog_name, + sizeof(prog_name)); + if (!ret) + NET_DUMP_STR("name", " %s", prog_name); + NET_DUMP_UINT("prog_id", " prog_id %u ", prog_ids[i]); + if (prog_flags[i] || json_output) { + NET_START_ARRAY("prog_flags", "%s "); + for (j = 0; prog_flags[i] && j < 32; j++) { + if (!(prog_flags[i] & (1 << j))) + continue; + NET_DUMP_UINT_ONLY(1 << j); + } + NET_END_ARRAY(""); + } + if (link_ids[i] || json_output) { + NET_DUMP_UINT("link_id", "link_id %u ", link_ids[i]); + if (link_flags[i] || json_output) { + NET_START_ARRAY("link_flags", "%s "); + for (j = 0; link_flags[i] && j < 32; j++) { + if (!(link_flags[i] & (1 << j))) + continue; + NET_DUMP_UINT_ONLY(1 << j); + } + NET_END_ARRAY(""); + } + } + NET_END_OBJECT_FINAL; + } +} + +static void show_dev_tc_bpf(struct ip_devname_ifindex *dev) +{ + __show_dev_tc_bpf(dev, BPF_TCX_INGRESS); + __show_dev_tc_bpf(dev, BPF_TCX_EGRESS); +} + +static int show_dev_tc_bpf_classic(int sock, unsigned int nl_pid, + struct ip_devname_ifindex *dev) { struct bpf_filter_t filter_info; struct bpf_tcinfo_t tcinfo; @@ -790,8 +876,9 @@ static int do_show(int argc, char **argv) if (!ret) { NET_START_ARRAY("tc", "%s:\n"); for (i = 0; i < dev_array.used_len; i++) { - ret = show_dev_tc_bpf(sock, nl_pid, - &dev_array.devices[i]); + show_dev_tc_bpf(&dev_array.devices[i]); + ret = show_dev_tc_bpf_classic(sock, nl_pid, + &dev_array.devices[i]); if (ret) break; } @@ -839,7 +926,8 @@ static int do_help(int argc, char **argv) " ATTACH_TYPE := { xdp | xdpgeneric | xdpdrv | xdpoffload }\n" " " HELP_SPEC_OPTIONS " }\n" "\n" - "Note: Only xdp and tc attachments are supported now.\n" + "Note: Only xdp, tcx, tc, flow_dissector and netfilter attachments\n" + " are currently supported.\n" " For progs attached to cgroups, use \"bpftool cgroup\"\n" " to dump program attachments. For program types\n" " sk_{filter,skb,msg,reuseport} and lwt/seg6, please\n" diff --git a/tools/bpf/bpftool/netlink_dumper.h b/tools/bpf/bpftool/netlink_dumper.h index 774af6c62ef5..96318106fb49 100644 --- a/tools/bpf/bpftool/netlink_dumper.h +++ b/tools/bpf/bpftool/netlink_dumper.h @@ -76,6 +76,14 @@ fprintf(stdout, fmt_str, val); \ } +#define NET_DUMP_UINT_ONLY(str) \ +{ \ + if (json_output) \ + jsonw_uint(json_wtr, str); \ + else \ + fprintf(stdout, "%u ", str); \ +} + #define NET_DUMP_STR(name, fmt_str, str) \ { \ if (json_output) \ diff --git a/tools/bpf/bpftool/skeleton/pid_iter.bpf.c b/tools/bpf/bpftool/skeleton/pid_iter.bpf.c index eb05ea53afb1..26004f0c5a6a 100644 --- a/tools/bpf/bpftool/skeleton/pid_iter.bpf.c +++ b/tools/bpf/bpftool/skeleton/pid_iter.bpf.c @@ -15,6 +15,19 @@ enum bpf_obj_type { BPF_OBJ_BTF, }; +struct bpf_perf_link___local { + struct bpf_link link; + struct file *perf_file; +} __attribute__((preserve_access_index)); + +struct perf_event___local { + u64 bpf_cookie; +} __attribute__((preserve_access_index)); + +enum bpf_link_type___local { + BPF_LINK_TYPE_PERF_EVENT___local = 7, +}; + extern const void bpf_link_fops __ksym; extern const void bpf_map_fops __ksym; extern const void bpf_prog_fops __ksym; @@ -41,10 +54,10 @@ static __always_inline __u32 get_obj_id(void *ent, enum bpf_obj_type type) /* could be used only with BPF_LINK_TYPE_PERF_EVENT links */ static __u64 get_bpf_cookie(struct bpf_link *link) { - struct bpf_perf_link *perf_link; - struct perf_event *event; + struct bpf_perf_link___local *perf_link; + struct perf_event___local *event; - perf_link = container_of(link, struct bpf_perf_link, link); + perf_link = container_of(link, struct bpf_perf_link___local, link); event = BPF_CORE_READ(perf_link, perf_file, private_data); return BPF_CORE_READ(event, bpf_cookie); } @@ -84,10 +97,13 @@ int iter(struct bpf_iter__task_file *ctx) e.pid = task->tgid; e.id = get_obj_id(file->private_data, obj_type); - if (obj_type == BPF_OBJ_LINK) { + if (obj_type == BPF_OBJ_LINK && + bpf_core_enum_value_exists(enum bpf_link_type___local, + BPF_LINK_TYPE_PERF_EVENT___local)) { struct bpf_link *link = (struct bpf_link *) file->private_data; - if (BPF_CORE_READ(link, type) == BPF_LINK_TYPE_PERF_EVENT) { + if (link->type == bpf_core_enum_value(enum bpf_link_type___local, + BPF_LINK_TYPE_PERF_EVENT___local)) { e.has_bpf_cookie = true; e.bpf_cookie = get_bpf_cookie(link); } diff --git a/tools/bpf/bpftool/skeleton/profiler.bpf.c b/tools/bpf/bpftool/skeleton/profiler.bpf.c index ce5b65e07ab1..2f80edc682f1 100644 --- a/tools/bpf/bpftool/skeleton/profiler.bpf.c +++ b/tools/bpf/bpftool/skeleton/profiler.bpf.c @@ -4,6 +4,12 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +struct bpf_perf_event_value___local { + __u64 counter; + __u64 enabled; + __u64 running; +} __attribute__((preserve_access_index)); + /* map of perf event fds, num_cpu * num_metric entries */ struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); @@ -15,14 +21,14 @@ struct { struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(value_size, sizeof(struct bpf_perf_event_value___local)); } fentry_readings SEC(".maps"); /* accumulated readings */ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(value_size, sizeof(struct bpf_perf_event_value___local)); } accum_readings SEC(".maps"); /* sample counts, one per cpu */ @@ -39,7 +45,7 @@ const volatile __u32 num_metric = 1; SEC("fentry/XXX") int BPF_PROG(fentry_XXX) { - struct bpf_perf_event_value *ptrs[MAX_NUM_MATRICS]; + struct bpf_perf_event_value___local *ptrs[MAX_NUM_MATRICS]; u32 key = bpf_get_smp_processor_id(); u32 i; @@ -53,10 +59,10 @@ int BPF_PROG(fentry_XXX) } for (i = 0; i < num_metric && i < MAX_NUM_MATRICS; i++) { - struct bpf_perf_event_value reading; + struct bpf_perf_event_value___local reading; int err; - err = bpf_perf_event_read_value(&events, key, &reading, + err = bpf_perf_event_read_value(&events, key, (void *)&reading, sizeof(reading)); if (err) return 0; @@ -68,14 +74,14 @@ int BPF_PROG(fentry_XXX) } static inline void -fexit_update_maps(u32 id, struct bpf_perf_event_value *after) +fexit_update_maps(u32 id, struct bpf_perf_event_value___local *after) { - struct bpf_perf_event_value *before, diff; + struct bpf_perf_event_value___local *before, diff; before = bpf_map_lookup_elem(&fentry_readings, &id); /* only account samples with a valid fentry_reading */ if (before && before->counter) { - struct bpf_perf_event_value *accum; + struct bpf_perf_event_value___local *accum; diff.counter = after->counter - before->counter; diff.enabled = after->enabled - before->enabled; @@ -93,7 +99,7 @@ fexit_update_maps(u32 id, struct bpf_perf_event_value *after) SEC("fexit/XXX") int BPF_PROG(fexit_XXX) { - struct bpf_perf_event_value readings[MAX_NUM_MATRICS]; + struct bpf_perf_event_value___local readings[MAX_NUM_MATRICS]; u32 cpu = bpf_get_smp_processor_id(); u32 i, zero = 0; int err; @@ -102,7 +108,8 @@ int BPF_PROG(fexit_XXX) /* read all events before updating the maps, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_MATRICS; i++) { err = bpf_perf_event_read_value(&events, cpu + i * num_cpu, - readings + i, sizeof(*readings)); + (void *)(readings + i), + sizeof(*readings)); if (err) return 0; } diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index da608e10c843..567f56dfd9f1 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -46,7 +46,11 @@ out: } dd->sym_mapping = tmp; sym = &dd->sym_mapping[dd->sym_count]; - if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2) + + /* module is optional */ + sym->module[0] = '\0'; + /* trim the square brackets around the module name */ + if (sscanf(buff, "%p %*c %s [%[^]]s", &address, sym->name, sym->module) < 2) continue; sym->address = (unsigned long)address; if (!strcmp(sym->name, "__bpf_call_base")) { diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h index 9a946377b0e6..db3ba0671501 100644 --- a/tools/bpf/bpftool/xlated_dumper.h +++ b/tools/bpf/bpftool/xlated_dumper.h @@ -5,12 +5,14 @@ #define __BPF_TOOL_XLATED_DUMPER_H #define SYM_MAX_NAME 256 +#define MODULE_MAX_NAME 64 struct bpf_prog_linfo; struct kernel_sym { unsigned long address; char name[SYM_MAX_NAME]; + char module[MODULE_MAX_NAME]; }; struct dump_data { diff --git a/tools/bpf/runqslower/Makefile b/tools/bpf/runqslower/Makefile index 47acf6936516..d8288936c912 100644 --- a/tools/bpf/runqslower/Makefile +++ b/tools/bpf/runqslower/Makefile @@ -62,7 +62,7 @@ $(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(BPFTOOL) $(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@ $(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) | $(OUTPUT) - $(QUIET_GEN)$(CLANG) -g -O2 -target bpf $(INCLUDES) \ + $(QUIET_GEN)$(CLANG) -g -O2 --target=bpf $(INCLUDES) \ -c $(filter %.c,$^) -o $@ && \ $(LLVM_STRIP) -g $@ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 2cd6dbbee088..f0c5de018a95 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -372,7 +372,7 @@ $(OUTPUT)test-libzstd.bin: $(BUILD) -lzstd $(OUTPUT)test-clang-bpf-co-re.bin: - $(CLANG) -S -g -target bpf -o - $(patsubst %.bin,%.c,$(@F)) | \ + $(CLANG) -S -g --target=bpf -o - $(patsubst %.bin,%.c,$(@F)) | \ grep BTF_KIND_VAR $(OUTPUT)test-file-handle.bin: diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 60a9d59beeab..739c15906a65 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1036,6 +1036,8 @@ enum bpf_attach_type { BPF_LSM_CGROUP, BPF_STRUCT_OPS, BPF_NETFILTER, + BPF_TCX_INGRESS, + BPF_TCX_EGRESS, __MAX_BPF_ATTACH_TYPE }; @@ -1053,10 +1055,20 @@ enum bpf_link_type { BPF_LINK_TYPE_KPROBE_MULTI = 8, BPF_LINK_TYPE_STRUCT_OPS = 9, BPF_LINK_TYPE_NETFILTER = 10, - + BPF_LINK_TYPE_TCX = 11, MAX_BPF_LINK_TYPE, }; +enum bpf_perf_event_type { + BPF_PERF_EVENT_UNSPEC = 0, + BPF_PERF_EVENT_UPROBE = 1, + BPF_PERF_EVENT_URETPROBE = 2, + BPF_PERF_EVENT_KPROBE = 3, + BPF_PERF_EVENT_KRETPROBE = 4, + BPF_PERF_EVENT_TRACEPOINT = 5, + BPF_PERF_EVENT_EVENT = 6, +}; + /* cgroup-bpf attach flags used in BPF_PROG_ATTACH command * * NONE(default): No further bpf programs allowed in the subtree. @@ -1103,7 +1115,12 @@ enum bpf_link_type { */ #define BPF_F_ALLOW_OVERRIDE (1U << 0) #define BPF_F_ALLOW_MULTI (1U << 1) +/* Generic attachment flags. */ #define BPF_F_REPLACE (1U << 2) +#define BPF_F_BEFORE (1U << 3) +#define BPF_F_AFTER (1U << 4) +#define BPF_F_ID (1U << 5) +#define BPF_F_LINK BPF_F_LINK /* 1 << 13 */ /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the * verifier will perform strict alignment checking as if the kernel @@ -1434,14 +1451,19 @@ union bpf_attr { }; struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ - __u32 target_fd; /* container object to attach to */ - __u32 attach_bpf_fd; /* eBPF program to attach */ + union { + __u32 target_fd; /* target object to attach to or ... */ + __u32 target_ifindex; /* target ifindex */ + }; + __u32 attach_bpf_fd; __u32 attach_type; __u32 attach_flags; - __u32 replace_bpf_fd; /* previously attached eBPF - * program to replace if - * BPF_F_REPLACE is used - */ + __u32 replace_bpf_fd; + union { + __u32 relative_fd; + __u32 relative_id; + }; + __u64 expected_revision; }; struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ @@ -1487,16 +1509,26 @@ union bpf_attr { } info; struct { /* anonymous struct used by BPF_PROG_QUERY command */ - __u32 target_fd; /* container object to query */ + union { + __u32 target_fd; /* target object to query or ... */ + __u32 target_ifindex; /* target ifindex */ + }; __u32 attach_type; __u32 query_flags; __u32 attach_flags; __aligned_u64 prog_ids; - __u32 prog_cnt; + union { + __u32 prog_cnt; + __u32 count; + }; + __u32 :32; /* output: per-program attach_flags. * not allowed to be set during effective query. */ __aligned_u64 prog_attach_flags; + __aligned_u64 link_ids; + __aligned_u64 link_attach_flags; + __u64 revision; } query; struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ @@ -1539,13 +1571,13 @@ union bpf_attr { __u32 map_fd; /* struct_ops to attach */ }; union { - __u32 target_fd; /* object to attach to */ - __u32 target_ifindex; /* target ifindex */ + __u32 target_fd; /* target object to attach to or ... */ + __u32 target_ifindex; /* target ifindex */ }; __u32 attach_type; /* attach type */ __u32 flags; /* extra flags */ union { - __u32 target_btf_id; /* btf_id of target to attach to */ + __u32 target_btf_id; /* btf_id of target to attach to */ struct { __aligned_u64 iter_info; /* extra bpf_iter_link_info */ __u32 iter_info_len; /* iter_info length */ @@ -1579,6 +1611,13 @@ union bpf_attr { __s32 priority; __u32 flags; } netfilter; + struct { + union { + __u32 relative_fd; + __u32 relative_id; + }; + __u64 expected_revision; + } tcx; }; } link_create; @@ -6187,6 +6226,19 @@ struct bpf_sock_tuple { }; }; +/* (Simplified) user return codes for tcx prog type. + * A valid tcx program must return one of these defined values. All other + * return codes are reserved for future use. Must remain compatible with + * their TC_ACT_* counter-parts. For compatibility in behavior, unknown + * return codes are mapped to TCX_NEXT. + */ +enum tcx_action_base { + TCX_NEXT = -1, + TCX_PASS = 0, + TCX_DROP = 2, + TCX_REDIRECT = 7, +}; + struct bpf_xdp_sock { __u32 queue_id; }; @@ -6439,6 +6491,40 @@ struct bpf_link_info { __s32 priority; __u32 flags; } netfilter; + struct { + __aligned_u64 addrs; + __u32 count; /* in/out: kprobe_multi function count */ + __u32 flags; + } kprobe_multi; + struct { + __u32 type; /* enum bpf_perf_event_type */ + __u32 :32; + union { + struct { + __aligned_u64 file_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from file_name */ + } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */ + struct { + __aligned_u64 func_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from func_name */ + __u64 addr; + } kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */ + struct { + __aligned_u64 tp_name; /* in/out */ + __u32 name_len; + } tracepoint; /* BPF_PERF_EVENT_TRACEPOINT */ + struct { + __u64 config; + __u32 type; + } event; /* BPF_PERF_EVENT_EVENT */ + }; + } perf_event; + struct { + __u32 ifindex; + __u32 attach_type; + } tcx; }; } __attribute__((aligned(8))); @@ -7012,6 +7098,7 @@ struct bpf_list_head { struct bpf_list_node { __u64 :64; __u64 :64; + __u64 :64; } __attribute__((aligned(8))); struct bpf_rb_root { @@ -7023,6 +7110,7 @@ struct bpf_rb_node { __u64 :64; __u64 :64; __u64 :64; + __u64 :64; } __attribute__((aligned(8))); struct bpf_refcount { diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h index a78a8096f4ce..73a47da885dc 100644 --- a/tools/include/uapi/linux/if_xdp.h +++ b/tools/include/uapi/linux/if_xdp.h @@ -25,6 +25,12 @@ * application. */ #define XDP_USE_NEED_WAKEUP (1 << 3) +/* By setting this option, userspace application indicates that it can + * handle multiple descriptors per packet thus enabling xsk core to split + * multi-buffer XDP frames into multiple Rx descriptors. Without this set + * such frames will be dropped by xsk. + */ +#define XDP_USE_SG (1 << 4) /* Flags for xsk_umem_config flags */ #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) @@ -106,6 +112,9 @@ struct xdp_desc { __u32 options; }; +/* Flag indicating packet constitutes of multiple buffers*/ +#define XDP_PKT_CONTD (1 << 0) + /* UMEM descriptor is __u64 */ #endif /* _LINUX_IF_XDP_H */ diff --git a/tools/include/uapi/linux/netdev.h b/tools/include/uapi/linux/netdev.h index 639524b59930..c1634b95c223 100644 --- a/tools/include/uapi/linux/netdev.h +++ b/tools/include/uapi/linux/netdev.h @@ -11,7 +11,7 @@ /** * enum netdev_xdp_act - * @NETDEV_XDP_ACT_BASIC: XDP feautues set supported by all drivers + * @NETDEV_XDP_ACT_BASIC: XDP features set supported by all drivers * (XDP_ABORTED, XDP_DROP, XDP_PASS, XDP_TX) * @NETDEV_XDP_ACT_REDIRECT: The netdev supports XDP_REDIRECT * @NETDEV_XDP_ACT_NDO_XMIT: This feature informs if netdev implements @@ -34,6 +34,7 @@ enum netdev_xdp_act { NETDEV_XDP_ACT_RX_SG = 32, NETDEV_XDP_ACT_NDO_XMIT_SG = 64, + /* private: */ NETDEV_XDP_ACT_MASK = 127, }; @@ -41,6 +42,7 @@ enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, NETDEV_A_DEV_XDP_FEATURES, + NETDEV_A_DEV_XDP_ZC_MAX_SEGS, __NETDEV_A_DEV_MAX, NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index ed86b37d8024..c9b6b311a441 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -629,55 +629,89 @@ int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type, return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts); } -int bpf_prog_attach_opts(int prog_fd, int target_fd, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts) +int bpf_prog_attach_opts(int prog_fd, int target, enum bpf_attach_type type, + const struct bpf_prog_attach_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); + const size_t attr_sz = offsetofend(union bpf_attr, expected_revision); + __u32 relative_id, flags; + int ret, relative_fd; union bpf_attr attr; - int ret; if (!OPTS_VALID(opts, bpf_prog_attach_opts)) return libbpf_err(-EINVAL); + relative_id = OPTS_GET(opts, relative_id, 0); + relative_fd = OPTS_GET(opts, relative_fd, 0); + flags = OPTS_GET(opts, flags, 0); + + /* validate we don't have unexpected combinations of non-zero fields */ + if (relative_fd && relative_id) + return libbpf_err(-EINVAL); + memset(&attr, 0, attr_sz); - attr.target_fd = target_fd; - attr.attach_bpf_fd = prog_fd; - attr.attach_type = type; - attr.attach_flags = OPTS_GET(opts, flags, 0); - attr.replace_bpf_fd = OPTS_GET(opts, replace_prog_fd, 0); + attr.target_fd = target; + attr.attach_bpf_fd = prog_fd; + attr.attach_type = type; + attr.replace_bpf_fd = OPTS_GET(opts, replace_fd, 0); + attr.expected_revision = OPTS_GET(opts, expected_revision, 0); + + if (relative_id) { + attr.attach_flags = flags | BPF_F_ID; + attr.relative_id = relative_id; + } else { + attr.attach_flags = flags; + attr.relative_fd = relative_fd; + } ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz); return libbpf_err_errno(ret); } -int bpf_prog_detach(int target_fd, enum bpf_attach_type type) +int bpf_prog_detach_opts(int prog_fd, int target, enum bpf_attach_type type, + const struct bpf_prog_detach_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); + const size_t attr_sz = offsetofend(union bpf_attr, expected_revision); + __u32 relative_id, flags; + int ret, relative_fd; union bpf_attr attr; - int ret; + + if (!OPTS_VALID(opts, bpf_prog_detach_opts)) + return libbpf_err(-EINVAL); + + relative_id = OPTS_GET(opts, relative_id, 0); + relative_fd = OPTS_GET(opts, relative_fd, 0); + flags = OPTS_GET(opts, flags, 0); + + /* validate we don't have unexpected combinations of non-zero fields */ + if (relative_fd && relative_id) + return libbpf_err(-EINVAL); memset(&attr, 0, attr_sz); - attr.target_fd = target_fd; - attr.attach_type = type; + attr.target_fd = target; + attr.attach_bpf_fd = prog_fd; + attr.attach_type = type; + attr.expected_revision = OPTS_GET(opts, expected_revision, 0); + + if (relative_id) { + attr.attach_flags = flags | BPF_F_ID; + attr.relative_id = relative_id; + } else { + attr.attach_flags = flags; + attr.relative_fd = relative_fd; + } ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz); return libbpf_err_errno(ret); } -int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type) +int bpf_prog_detach(int target_fd, enum bpf_attach_type type) { - const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.target_fd = target_fd; - attr.attach_bpf_fd = prog_fd; - attr.attach_type = type; + return bpf_prog_detach_opts(0, target_fd, type, NULL); +} - ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz); - return libbpf_err_errno(ret); +int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type) +{ + return bpf_prog_detach_opts(prog_fd, target_fd, type, NULL); } int bpf_link_create(int prog_fd, int target_fd, @@ -685,9 +719,9 @@ int bpf_link_create(int prog_fd, int target_fd, const struct bpf_link_create_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, link_create); - __u32 target_btf_id, iter_info_len; + __u32 target_btf_id, iter_info_len, relative_id; + int fd, err, relative_fd; union bpf_attr attr; - int fd, err; if (!OPTS_VALID(opts, bpf_link_create_opts)) return libbpf_err(-EINVAL); @@ -741,6 +775,30 @@ int bpf_link_create(int prog_fd, int target_fd, if (!OPTS_ZEROED(opts, tracing)) return libbpf_err(-EINVAL); break; + case BPF_NETFILTER: + attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0); + attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0); + attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0); + attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0); + if (!OPTS_ZEROED(opts, netfilter)) + return libbpf_err(-EINVAL); + break; + case BPF_TCX_INGRESS: + case BPF_TCX_EGRESS: + relative_fd = OPTS_GET(opts, tcx.relative_fd, 0); + relative_id = OPTS_GET(opts, tcx.relative_id, 0); + if (relative_fd && relative_id) + return libbpf_err(-EINVAL); + if (relative_id) { + attr.link_create.tcx.relative_id = relative_id; + attr.link_create.flags |= BPF_F_ID; + } else { + attr.link_create.tcx.relative_fd = relative_fd; + } + attr.link_create.tcx.expected_revision = OPTS_GET(opts, tcx.expected_revision, 0); + if (!OPTS_ZEROED(opts, tcx)) + return libbpf_err(-EINVAL); + break; default: if (!OPTS_ZEROED(opts, flags)) return libbpf_err(-EINVAL); @@ -833,8 +891,7 @@ int bpf_iter_create(int link_fd) return libbpf_err_errno(fd); } -int bpf_prog_query_opts(int target_fd, - enum bpf_attach_type type, +int bpf_prog_query_opts(int target, enum bpf_attach_type type, struct bpf_prog_query_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, query); @@ -845,18 +902,20 @@ int bpf_prog_query_opts(int target_fd, return libbpf_err(-EINVAL); memset(&attr, 0, attr_sz); - - attr.query.target_fd = target_fd; - attr.query.attach_type = type; - attr.query.query_flags = OPTS_GET(opts, query_flags, 0); - attr.query.prog_cnt = OPTS_GET(opts, prog_cnt, 0); - attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); - attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); + attr.query.target_fd = target; + attr.query.attach_type = type; + attr.query.query_flags = OPTS_GET(opts, query_flags, 0); + attr.query.count = OPTS_GET(opts, count, 0); + attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); + attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL)); + attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); + attr.query.link_attach_flags = ptr_to_u64(OPTS_GET(opts, link_attach_flags, NULL)); ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz); OPTS_SET(opts, attach_flags, attr.query.attach_flags); - OPTS_SET(opts, prog_cnt, attr.query.prog_cnt); + OPTS_SET(opts, revision, attr.query.revision); + OPTS_SET(opts, count, attr.query.count); return libbpf_err_errno(ret); } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 9aa0ee473754..044a74ffc38a 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -312,22 +312,68 @@ LIBBPF_API int bpf_obj_get(const char *pathname); LIBBPF_API int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts); -struct bpf_prog_attach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - unsigned int flags; - int replace_prog_fd; -}; -#define bpf_prog_attach_opts__last_field replace_prog_fd - LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, enum bpf_attach_type type, unsigned int flags); -LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int attachable_fd, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts); LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type); LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd, enum bpf_attach_type type); +struct bpf_prog_attach_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + __u32 flags; + union { + int replace_prog_fd; + int replace_fd; + }; + int relative_fd; + __u32 relative_id; + __u64 expected_revision; + size_t :0; +}; +#define bpf_prog_attach_opts__last_field expected_revision + +struct bpf_prog_detach_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + __u32 flags; + int relative_fd; + __u32 relative_id; + __u64 expected_revision; + size_t :0; +}; +#define bpf_prog_detach_opts__last_field expected_revision + +/** + * @brief **bpf_prog_attach_opts()** attaches the BPF program corresponding to + * *prog_fd* to a *target* which can represent a file descriptor or netdevice + * ifindex. + * + * @param prog_fd BPF program file descriptor + * @param target attach location file descriptor or ifindex + * @param type attach type for the BPF program + * @param opts options for configuring the attachment + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ +LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int target, + enum bpf_attach_type type, + const struct bpf_prog_attach_opts *opts); + +/** + * @brief **bpf_prog_detach_opts()** detaches the BPF program corresponding to + * *prog_fd* from a *target* which can represent a file descriptor or netdevice + * ifindex. + * + * @param prog_fd BPF program file descriptor + * @param target detach location file descriptor or ifindex + * @param type detach type for the BPF program + * @param opts options for configuring the detachment + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ +LIBBPF_API int bpf_prog_detach_opts(int prog_fd, int target, + enum bpf_attach_type type, + const struct bpf_prog_detach_opts *opts); + union bpf_iter_link_info; /* defined in up-to-date linux/bpf.h */ struct bpf_link_create_opts { size_t sz; /* size of this struct for forward/backward compatibility */ @@ -349,6 +395,17 @@ struct bpf_link_create_opts { struct { __u64 cookie; } tracing; + struct { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + } netfilter; + struct { + __u32 relative_fd; + __u32 relative_id; + __u64 expected_revision; + } tcx; }; size_t :0; }; @@ -489,13 +546,31 @@ struct bpf_prog_query_opts { __u32 query_flags; __u32 attach_flags; /* output argument */ __u32 *prog_ids; - __u32 prog_cnt; /* input+output argument */ + union { + /* input+output argument */ + __u32 prog_cnt; + __u32 count; + }; __u32 *prog_attach_flags; + __u32 *link_ids; + __u32 *link_attach_flags; + __u64 revision; + size_t :0; }; -#define bpf_prog_query_opts__last_field prog_attach_flags +#define bpf_prog_query_opts__last_field revision -LIBBPF_API int bpf_prog_query_opts(int target_fd, - enum bpf_attach_type type, +/** + * @brief **bpf_prog_query_opts()** queries the BPF programs and BPF links + * which are attached to *target* which can represent a file descriptor or + * netdevice ifindex. + * + * @param target query location file descriptor or ifindex + * @param type attach type for the BPF program + * @param opts options for configuring the query + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ +LIBBPF_API int bpf_prog_query_opts(int target, enum bpf_attach_type type, struct bpf_prog_query_opts *opts); LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, __u32 *attach_flags, diff --git a/tools/lib/bpf/hashmap.h b/tools/lib/bpf/hashmap.h index 0a5bf1937a7c..c12f8320e668 100644 --- a/tools/lib/bpf/hashmap.h +++ b/tools/lib/bpf/hashmap.h @@ -80,16 +80,6 @@ struct hashmap { size_t sz; }; -#define HASHMAP_INIT(hash_fn, equal_fn, ctx) { \ - .hash_fn = (hash_fn), \ - .equal_fn = (equal_fn), \ - .ctx = (ctx), \ - .buckets = NULL, \ - .cap = 0, \ - .cap_bits = 0, \ - .sz = 0, \ -} - void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, hashmap_equal_fn equal_fn, void *ctx); struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 214f828ece6b..17883f5a44b9 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -118,6 +118,8 @@ static const char * const attach_type_name[] = { [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", [BPF_STRUCT_OPS] = "struct_ops", [BPF_NETFILTER] = "netfilter", + [BPF_TCX_INGRESS] = "tcx_ingress", + [BPF_TCX_EGRESS] = "tcx_egress", }; static const char * const link_type_name[] = { @@ -132,6 +134,7 @@ static const char * const link_type_name[] = { [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", [BPF_LINK_TYPE_NETFILTER] = "netfilter", + [BPF_LINK_TYPE_TCX] = "tcx", }; static const char * const map_type_name[] = { @@ -5471,6 +5474,10 @@ static int load_module_btfs(struct bpf_object *obj) err = bpf_btf_get_next_id(id, &id); if (err && errno == ENOENT) return 0; + if (err && errno == EPERM) { + pr_debug("skipping module BTFs loading, missing privileges\n"); + return 0; + } if (err) { err = -errno; pr_warn("failed to iterate BTF objects: %d\n", err); @@ -6157,7 +6164,11 @@ static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_progra if (main_prog == subprog) return 0; relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); - if (!relos) + /* if new count is zero, reallocarray can return a valid NULL result; + * in this case the previous pointer will be freed, so we *have to* + * reassign old pointer to the new value (even if it's NULL) + */ + if (!relos && new_cnt) return -ENOMEM; if (subprog->nr_reloc) memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, @@ -8528,7 +8539,8 @@ int bpf_program__set_insns(struct bpf_program *prog, return -EBUSY; insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); - if (!insns) { + /* NULL is a valid return from reallocarray if the new count is zero */ + if (!insns && new_insn_cnt) { pr_warn("prog '%s': failed to realloc prog code\n", prog->name); return -ENOMEM; } @@ -8558,13 +8570,31 @@ enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) return prog->type; } +static size_t custom_sec_def_cnt; +static struct bpf_sec_def *custom_sec_defs; +static struct bpf_sec_def custom_fallback_def; +static bool has_custom_fallback_def; +static int last_custom_sec_def_handler_id; + int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) { if (prog->obj->loaded) return libbpf_err(-EBUSY); + /* if type is not changed, do nothing */ + if (prog->type == type) + return 0; + prog->type = type; - prog->sec_def = NULL; + + /* If a program type was changed, we need to reset associated SEC() + * handler, as it will be invalid now. The only exception is a generic + * fallback handler, which by definition is program type-agnostic and + * is a catch-all custom handler, optionally set by the application, + * so should be able to handle any type of BPF program. + */ + if (prog->sec_def != &custom_fallback_def) + prog->sec_def = NULL; return 0; } @@ -8669,9 +8699,13 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), - SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), - SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), - SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), + SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ + SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ + SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), + SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), + SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ + SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ + SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), @@ -8740,13 +8774,6 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), }; -static size_t custom_sec_def_cnt; -static struct bpf_sec_def *custom_sec_defs; -static struct bpf_sec_def custom_fallback_def; -static bool has_custom_fallback_def; - -static int last_custom_sec_def_handler_id; - int libbpf_register_prog_handler(const char *sec, enum bpf_prog_type prog_type, enum bpf_attach_type exp_attach_type, @@ -8826,7 +8853,11 @@ int libbpf_unregister_prog_handler(int handler_id) /* try to shrink the array, but it's ok if we couldn't */ sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); - if (sec_defs) + /* if new count is zero, reallocarray can return a valid NULL result; + * in this case the previous pointer will be freed, so we *have to* + * reassign old pointer to the new value (even if it's NULL) + */ + if (sec_defs || custom_sec_def_cnt == 0) custom_sec_defs = sec_defs; return 0; @@ -10224,6 +10255,18 @@ static const char *tracefs_uprobe_events(void) return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; } +static const char *tracefs_available_filter_functions(void) +{ + return use_debugfs() ? DEBUGFS"/available_filter_functions" + : TRACEFS"/available_filter_functions"; +} + +static const char *tracefs_available_filter_functions_addrs(void) +{ + return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" + : TRACEFS"/available_filter_functions_addrs"; +} + static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, const char *kfunc_name, size_t offset) { @@ -10539,25 +10582,158 @@ struct kprobe_multi_resolve { size_t cnt; }; -static int -resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type, - const char *sym_name, void *ctx) +struct avail_kallsyms_data { + char **syms; + size_t cnt; + struct kprobe_multi_resolve *res; +}; + +static int avail_func_cmp(const void *a, const void *b) { - struct kprobe_multi_resolve *res = ctx; + return strcmp(*(const char **)a, *(const char **)b); +} + +static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, + const char *sym_name, void *ctx) +{ + struct avail_kallsyms_data *data = ctx; + struct kprobe_multi_resolve *res = data->res; int err; - if (!glob_match(sym_name, res->pattern)) + if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) return 0; - err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long), - res->cnt + 1); + err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); if (err) return err; - res->addrs[res->cnt++] = (unsigned long) sym_addr; + res->addrs[res->cnt++] = (unsigned long)sym_addr; return 0; } +static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) +{ + const char *available_functions_file = tracefs_available_filter_functions(); + struct avail_kallsyms_data data; + char sym_name[500]; + FILE *f; + int err = 0, ret, i; + char **syms = NULL; + size_t cap = 0, cnt = 0; + + f = fopen(available_functions_file, "re"); + if (!f) { + err = -errno; + pr_warn("failed to open %s: %d\n", available_functions_file, err); + return err; + } + + while (true) { + char *name; + + ret = fscanf(f, "%499s%*[^\n]\n", sym_name); + if (ret == EOF && feof(f)) + break; + + if (ret != 1) { + pr_warn("failed to parse available_filter_functions entry: %d\n", ret); + err = -EINVAL; + goto cleanup; + } + + if (!glob_match(sym_name, res->pattern)) + continue; + + err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); + if (err) + goto cleanup; + + name = strdup(sym_name); + if (!name) { + err = -errno; + goto cleanup; + } + + syms[cnt++] = name; + } + + /* no entries found, bail out */ + if (cnt == 0) { + err = -ENOENT; + goto cleanup; + } + + /* sort available functions */ + qsort(syms, cnt, sizeof(*syms), avail_func_cmp); + + data.syms = syms; + data.res = res; + data.cnt = cnt; + libbpf_kallsyms_parse(avail_kallsyms_cb, &data); + + if (res->cnt == 0) + err = -ENOENT; + +cleanup: + for (i = 0; i < cnt; i++) + free((char *)syms[i]); + free(syms); + + fclose(f); + return err; +} + +static bool has_available_filter_functions_addrs(void) +{ + return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; +} + +static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) +{ + const char *available_path = tracefs_available_filter_functions_addrs(); + char sym_name[500]; + FILE *f; + int ret, err = 0; + unsigned long long sym_addr; + + f = fopen(available_path, "re"); + if (!f) { + err = -errno; + pr_warn("failed to open %s: %d\n", available_path, err); + return err; + } + + while (true) { + ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); + if (ret == EOF && feof(f)) + break; + + if (ret != 2) { + pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", + ret); + err = -EINVAL; + goto cleanup; + } + + if (!glob_match(sym_name, res->pattern)) + continue; + + err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, + sizeof(*res->addrs), res->cnt + 1); + if (err) + goto cleanup; + + res->addrs[res->cnt++] = (unsigned long)sym_addr; + } + + if (res->cnt == 0) + err = -ENOENT; + +cleanup: + fclose(f); + return err; +} + struct bpf_link * bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, const char *pattern, @@ -10594,13 +10770,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, return libbpf_err_ptr(-EINVAL); if (pattern) { - err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res); + if (has_available_filter_functions_addrs()) + err = libbpf_available_kprobes_parse(&res); + else + err = libbpf_available_kallsyms_parse(&res); if (err) goto error; - if (!res.cnt) { - err = -ENOENT; - goto error; - } addrs = res.addrs; cnt = res.cnt; } @@ -11680,11 +11855,10 @@ static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_li } static struct bpf_link * -bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id, - const char *target_name) +bpf_program_attach_fd(const struct bpf_program *prog, + int target_fd, const char *target_name, + const struct bpf_link_create_opts *opts) { - DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, - .target_btf_id = btf_id); enum bpf_attach_type attach_type; char errmsg[STRERR_BUFSIZE]; struct bpf_link *link; @@ -11702,7 +11876,7 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id link->detach = &bpf_link__detach_fd; attach_type = bpf_program__expected_attach_type(prog); - link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); + link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); if (link_fd < 0) { link_fd = -errno; free(link); @@ -11718,19 +11892,54 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id struct bpf_link * bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) { - return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); + return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); } struct bpf_link * bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) { - return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); + return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); } struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) { /* target_fd/target_ifindex use the same field in LINK_CREATE */ - return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); + return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); +} + +struct bpf_link * +bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, + const struct bpf_tcx_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); + __u32 relative_id; + int relative_fd; + + if (!OPTS_VALID(opts, bpf_tcx_opts)) + return libbpf_err_ptr(-EINVAL); + + relative_id = OPTS_GET(opts, relative_id, 0); + relative_fd = OPTS_GET(opts, relative_fd, 0); + + /* validate we don't have unexpected combinations of non-zero fields */ + if (!ifindex) { + pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", + prog->name); + return libbpf_err_ptr(-EINVAL); + } + if (relative_fd && relative_id) { + pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", + prog->name); + return libbpf_err_ptr(-EINVAL); + } + + link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); + link_create_opts.tcx.relative_fd = relative_fd; + link_create_opts.tcx.relative_id = relative_id; + link_create_opts.flags = OPTS_GET(opts, flags, 0); + + /* target_fd/target_ifindex use the same field in LINK_CREATE */ + return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); } struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, @@ -11752,11 +11961,16 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, } if (target_fd) { + LIBBPF_OPTS(bpf_link_create_opts, target_opts); + btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); if (btf_id < 0) return libbpf_err_ptr(btf_id); - return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); + target_opts.target_btf_id = btf_id; + + return bpf_program_attach_fd(prog, target_fd, "freplace", + &target_opts); } else { /* no target, so use raw_tracepoint_open for compatibility * with old kernels @@ -11811,6 +12025,48 @@ static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_l return libbpf_get_error(*link); } +struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, + const struct bpf_netfilter_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, lopts); + struct bpf_link *link; + int prog_fd, link_fd; + + if (!OPTS_VALID(opts, bpf_netfilter_opts)) + return libbpf_err_ptr(-EINVAL); + + prog_fd = bpf_program__fd(prog); + if (prog_fd < 0) { + pr_warn("prog '%s': can't attach before loaded\n", prog->name); + return libbpf_err_ptr(-EINVAL); + } + + link = calloc(1, sizeof(*link)); + if (!link) + return libbpf_err_ptr(-ENOMEM); + + link->detach = &bpf_link__detach_fd; + + lopts.netfilter.pf = OPTS_GET(opts, pf, 0); + lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); + lopts.netfilter.priority = OPTS_GET(opts, priority, 0); + lopts.netfilter.flags = OPTS_GET(opts, flags, 0); + + link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); + if (link_fd < 0) { + char errmsg[STRERR_BUFSIZE]; + + link_fd = -errno; + free(link); + pr_warn("prog '%s': failed to attach to netfilter: %s\n", + prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); + return libbpf_err_ptr(link_fd); + } + link->fd = link_fd; + + return link; +} + struct bpf_link *bpf_program__attach(const struct bpf_program *prog) { struct bpf_link *link = NULL; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 754da73c643b..55b97b208754 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -718,6 +718,36 @@ LIBBPF_API struct bpf_link * bpf_program__attach_freplace(const struct bpf_program *prog, int target_fd, const char *attach_func_name); +struct bpf_netfilter_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; +}; +#define bpf_netfilter_opts__last_field flags + +LIBBPF_API struct bpf_link * +bpf_program__attach_netfilter(const struct bpf_program *prog, + const struct bpf_netfilter_opts *opts); + +struct bpf_tcx_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + __u32 flags; + __u32 relative_fd; + __u32 relative_id; + __u64 expected_revision; + size_t :0; +}; +#define bpf_tcx_opts__last_field expected_revision + +LIBBPF_API struct bpf_link * +bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, + const struct bpf_tcx_opts *opts); + struct bpf_map; LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map); @@ -1090,9 +1120,10 @@ struct bpf_xdp_query_opts { __u32 skb_prog_id; /* output */ __u8 attach_mode; /* output */ __u64 feature_flags; /* output */ + __u32 xdp_zc_max_segs; /* output */ size_t :0; }; -#define bpf_xdp_query_opts__last_field feature_flags +#define bpf_xdp_query_opts__last_field xdp_zc_max_segs LIBBPF_API int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts); diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 7521a2fb7626..9c7538dd5835 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -395,4 +395,7 @@ LIBBPF_1.2.0 { LIBBPF_1.3.0 { global: bpf_obj_pin_opts; + bpf_prog_detach_opts; + bpf_program__attach_netfilter; + bpf_program__attach_tcx; } LIBBPF_1.2.0; diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h index 9a7937f339df..b7060f254486 100644 --- a/tools/lib/bpf/libbpf_common.h +++ b/tools/lib/bpf/libbpf_common.h @@ -70,4 +70,20 @@ }; \ }) +/* Helper macro to clear and optionally reinitialize libbpf options struct + * + * Small helper macro to reset all fields and to reinitialize the common + * structure size member. Values provided by users in struct initializer- + * syntax as varargs can be provided as well to reinitialize options struct + * specific members. + */ +#define LIBBPF_OPTS_RESET(NAME, ...) \ + do { \ + memset(&NAME, 0, sizeof(NAME)); \ + NAME = (typeof(NAME)) { \ + .sz = sizeof(NAME), \ + __VA_ARGS__ \ + }; \ + } while (0) + #endif /* __LIBBPF_LIBBPF_COMMON_H */ diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index 84dd5fa14905..090bcf6e3b3d 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -45,6 +45,7 @@ struct xdp_id_md { struct xdp_features_md { int ifindex; + __u32 xdp_zc_max_segs; __u64 flags; }; @@ -421,6 +422,9 @@ static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, return NL_CONT; md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]); + if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]) + md->xdp_zc_max_segs = + libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]); return NL_DONE; } @@ -493,6 +497,7 @@ int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts) return libbpf_err(err); opts->feature_flags = md.flags; + opts->xdp_zc_max_segs = md.xdp_zc_max_segs; skip_feature_flags: return 0; diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index f1a141555f08..37455d00b239 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -852,8 +852,11 @@ static int bpf_link_usdt_detach(struct bpf_link *link) * system is so exhausted on memory, it's the least of user's * concerns, probably. * So just do our best here to return those IDs to usdt_manager. + * Another edge case when we can legitimately get NULL is when + * new_cnt is zero, which can happen in some edge cases, so we + * need to be careful about that. */ - if (new_free_ids) { + if (new_free_ids || new_cnt == 0) { memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); man->free_spec_ids = new_free_ids; diff --git a/tools/net/ynl/Makefile b/tools/net/ynl/Makefile index d664b36deb5b..8156f03e23ac 100644 --- a/tools/net/ynl/Makefile +++ b/tools/net/ynl/Makefile @@ -3,6 +3,7 @@ SUBDIRS = lib generated samples all: $(SUBDIRS) + ./ynl-regen.sh -f -p $(PWD)/../../../ $(SUBDIRS): @if [ -f "$@/Makefile" ] ; then \ diff --git a/tools/net/ynl/generated/netdev-user.c b/tools/net/ynl/generated/netdev-user.c index 4eb8aefef0cd..68b408ca0f7f 100644 --- a/tools/net/ynl/generated/netdev-user.c +++ b/tools/net/ynl/generated/netdev-user.c @@ -50,6 +50,7 @@ struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { [NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, }, [NETDEV_A_DEV_PAD] = { .name = "pad", .type = YNL_PT_IGNORE, }, [NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, }, + [NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, }, }; struct ynl_policy_nest netdev_dev_nest = { @@ -91,6 +92,11 @@ int netdev_dev_get_rsp_parse(const struct nlmsghdr *nlh, void *data) return MNL_CB_ERROR; dst->_present.xdp_features = 1; dst->xdp_features = mnl_attr_get_u64(attr); + } else if (type == NETDEV_A_DEV_XDP_ZC_MAX_SEGS) { + if (ynl_attr_validate(yarg, attr)) + return MNL_CB_ERROR; + dst->_present.xdp_zc_max_segs = 1; + dst->xdp_zc_max_segs = mnl_attr_get_u32(attr); } } diff --git a/tools/net/ynl/generated/netdev-user.h b/tools/net/ynl/generated/netdev-user.h index 5554dc69bb9c..0952d3261f4d 100644 --- a/tools/net/ynl/generated/netdev-user.h +++ b/tools/net/ynl/generated/netdev-user.h @@ -47,10 +47,12 @@ struct netdev_dev_get_rsp { struct { __u32 ifindex:1; __u32 xdp_features:1; + __u32 xdp_zc_max_segs:1; } _present; __u32 ifindex; __u64 xdp_features; + __u32 xdp_zc_max_segs; }; void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp); diff --git a/tools/net/ynl/samples/netdev.c b/tools/net/ynl/samples/netdev.c index d31268aa47c5..06433400dddd 100644 --- a/tools/net/ynl/samples/netdev.c +++ b/tools/net/ynl/samples/netdev.c @@ -38,6 +38,8 @@ static void netdev_print_device(struct netdev_dev_get_rsp *d, unsigned int op) printf(" %s", netdev_xdp_act_str(1 << i)); } + printf(" xdp-zc-max-segs=%u", d->xdp_zc_max_segs); + name = netdev_op_str(op); if (name) printf(" (ntf: %s)", name); diff --git a/tools/net/ynl/ynl-gen-c.py b/tools/net/ynl/ynl-gen-c.py index 71c5e79e877f..650be9b8b693 100755 --- a/tools/net/ynl/ynl-gen-c.py +++ b/tools/net/ynl/ynl-gen-c.py @@ -2125,6 +2125,7 @@ def render_uapi(family, cw): if const.get('render-max', False): cw.nl() + cw.p('/* private: */') if const['type'] == 'flags': max_name = c_upper(name_pfx + 'mask') max_val = f' = {enum.get_mask()},' diff --git a/tools/net/ynl/ynl-regen.sh b/tools/net/ynl/ynl-regen.sh index 8d4ca6a50582..bdba24066cf1 100755 --- a/tools/net/ynl/ynl-regen.sh +++ b/tools/net/ynl/ynl-regen.sh @@ -4,15 +4,18 @@ TOOL=$(dirname $(realpath $0))/ynl-gen-c.py force= +search= while [ ! -z "$1" ]; do case "$1" in -f ) force=yes; shift ;; + -p ) search=$2; shift 2 ;; * ) echo "Unrecognized option '$1'"; exit 1 ;; esac done KDIR=$(dirname $(dirname $(dirname $(dirname $(realpath $0))))) +pushd ${search:-$KDIR} >>/dev/null files=$(git grep --files-with-matches '^/\* YNL-GEN \(kernel\|uapi\|user\)') for f in $files; do @@ -30,3 +33,5 @@ for f in $files; do $TOOL --mode ${params[2]} --${params[3]} --spec $KDIR/${params[0]} \ $args -o $f done + +popd >>/dev/null diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 666b56f22a41..5c60a7cea732 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -8,6 +8,7 @@ TARGETS += cachestat TARGETS += capabilities TARGETS += cgroup TARGETS += clone3 +TARGETS += connector TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug diff --git a/tools/testing/selftests/bpf/DENYLIST.aarch64 b/tools/testing/selftests/bpf/DENYLIST.aarch64 index 08adc805878b..3b61e8b35d62 100644 --- a/tools/testing/selftests/bpf/DENYLIST.aarch64 +++ b/tools/testing/selftests/bpf/DENYLIST.aarch64 @@ -10,3 +10,5 @@ kprobe_multi_test/link_api_addrs # link_fd unexpected link_fd: a kprobe_multi_test/link_api_syms # link_fd unexpected link_fd: actual -95 < expected 0 kprobe_multi_test/skel_api # libbpf: failed to load BPF skeleton 'kprobe_multi': -3 module_attach # prog 'kprobe_multi': failed to auto-attach: -95 +fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524 +fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524 diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 538df8fb8c42..882be03b179f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -12,7 +12,11 @@ BPFDIR := $(LIBDIR)/bpf TOOLSINCDIR := $(TOOLSDIR)/include BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool APIDIR := $(TOOLSINCDIR)/uapi +ifneq ($(O),) +GENDIR := $(O)/include/generated +else GENDIR := $(abspath ../../../../include/generated) +endif GENHDR := $(GENDIR)/autoconf.h HOSTPKG_CONFIG := pkg-config @@ -331,7 +335,7 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ OUTPUT=$(HOST_BUILD_DIR)/resolve_btfids/ BPFOBJ=$(HOST_BPFOBJ) # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -372,12 +376,12 @@ $(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h # $3 - CFLAGS define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v3 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2 endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v2 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2 endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE @@ -644,11 +648,13 @@ $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o: $(OUTPUT)/local_storage_rcu_tasks_trace_bench.skel.h $(OUTPUT)/bench_local_storage_create.o: $(OUTPUT)/bench_local_storage_create.skel.h $(OUTPUT)/bench_bpf_hashmap_lookup.o: $(OUTPUT)/bpf_hashmap_lookup.skel.h +$(OUTPUT)/bench_htab_mem.o: $(OUTPUT)/htab_mem_bench.skel.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(TESTING_HELPERS) \ $(TRACE_HELPERS) \ + $(CGROUP_HELPERS) \ $(OUTPUT)/bench_count.o \ $(OUTPUT)/bench_rename.o \ $(OUTPUT)/bench_trigger.o \ @@ -661,6 +667,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o \ $(OUTPUT)/bench_bpf_hashmap_lookup.o \ $(OUTPUT)/bench_local_storage_create.o \ + $(OUTPUT)/bench_htab_mem.o \ # $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 41fe5a82b88b..73ce11b0547d 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -279,6 +279,7 @@ extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; extern struct argp bench_hashmap_lookup_argp; extern struct argp bench_local_storage_create_argp; +extern struct argp bench_htab_mem_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -290,6 +291,7 @@ static const struct argp_child bench_parsers[] = { "local_storage RCU Tasks Trace slowdown benchmark", 0 }, { &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 }, { &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 }, + { &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 }, {}, }; @@ -520,6 +522,7 @@ extern const struct bench bench_local_storage_cache_hashmap_control; extern const struct bench bench_local_storage_tasks_trace; extern const struct bench bench_bpf_hashmap_lookup; extern const struct bench bench_local_storage_create; +extern const struct bench bench_htab_mem; static const struct bench *benchs[] = { &bench_count_global, @@ -561,6 +564,7 @@ static const struct bench *benchs[] = { &bench_local_storage_tasks_trace, &bench_bpf_hashmap_lookup, &bench_local_storage_create, + &bench_htab_mem, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_htab_mem.c b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c new file mode 100644 index 000000000000..9146d3f414d2 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include <argp.h> +#include <stdbool.h> +#include <pthread.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/param.h> +#include <fcntl.h> + +#include "bench.h" +#include "bpf_util.h" +#include "cgroup_helpers.h" +#include "htab_mem_bench.skel.h" + +struct htab_mem_use_case { + const char *name; + const char **progs; + /* Do synchronization between addition thread and deletion thread */ + bool need_sync; +}; + +static struct htab_mem_ctx { + const struct htab_mem_use_case *uc; + struct htab_mem_bench *skel; + pthread_barrier_t *notify; + int fd; +} ctx; + +const char *ow_progs[] = {"overwrite", NULL}; +const char *batch_progs[] = {"batch_add_batch_del", NULL}; +const char *add_del_progs[] = {"add_only", "del_only", NULL}; +const static struct htab_mem_use_case use_cases[] = { + { .name = "overwrite", .progs = ow_progs }, + { .name = "batch_add_batch_del", .progs = batch_progs }, + { .name = "add_del_on_diff_cpu", .progs = add_del_progs, .need_sync = true }, +}; + +static struct htab_mem_args { + u32 value_size; + const char *use_case; + bool preallocated; +} args = { + .value_size = 8, + .use_case = "overwrite", + .preallocated = false, +}; + +enum { + ARG_VALUE_SIZE = 10000, + ARG_USE_CASE = 10001, + ARG_PREALLOCATED = 10002, +}; + +static const struct argp_option opts[] = { + { "value-size", ARG_VALUE_SIZE, "VALUE_SIZE", 0, + "Set the value size of hash map (default 8)" }, + { "use-case", ARG_USE_CASE, "USE_CASE", 0, + "Set the use case of hash map: overwrite|batch_add_batch_del|add_del_on_diff_cpu" }, + { "preallocated", ARG_PREALLOCATED, NULL, 0, "use preallocated hash map" }, + {}, +}; + +static error_t htab_mem_parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_VALUE_SIZE: + args.value_size = strtoul(arg, NULL, 10); + if (args.value_size > 4096) { + fprintf(stderr, "too big value size %u\n", args.value_size); + argp_usage(state); + } + break; + case ARG_USE_CASE: + args.use_case = strdup(arg); + if (!args.use_case) { + fprintf(stderr, "no mem for use-case\n"); + argp_usage(state); + } + break; + case ARG_PREALLOCATED: + args.preallocated = true; + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_htab_mem_argp = { + .options = opts, + .parser = htab_mem_parse_arg, +}; + +static void htab_mem_validate(void) +{ + if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) { + fprintf(stderr, "%s needs an even number of producers\n", args.use_case); + exit(1); + } +} + +static int htab_mem_bench_init_barriers(void) +{ + pthread_barrier_t *barriers; + unsigned int i, nr; + + if (!ctx.uc->need_sync) + return 0; + + nr = (env.producer_cnt + 1) / 2; + barriers = calloc(nr, sizeof(*barriers)); + if (!barriers) + return -1; + + /* Used for synchronization between two threads */ + for (i = 0; i < nr; i++) + pthread_barrier_init(&barriers[i], NULL, 2); + + ctx.notify = barriers; + return 0; +} + +static void htab_mem_bench_exit_barriers(void) +{ + unsigned int i, nr; + + if (!ctx.notify) + return; + + nr = (env.producer_cnt + 1) / 2; + for (i = 0; i < nr; i++) + pthread_barrier_destroy(&ctx.notify[i]); + free(ctx.notify); +} + +static const struct htab_mem_use_case *htab_mem_find_use_case_or_exit(const char *name) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(use_cases); i++) { + if (!strcmp(name, use_cases[i].name)) + return &use_cases[i]; + } + + fprintf(stderr, "no such use-case: %s\n", name); + fprintf(stderr, "available use case:"); + for (i = 0; i < ARRAY_SIZE(use_cases); i++) + fprintf(stderr, " %s", use_cases[i].name); + fprintf(stderr, "\n"); + exit(1); +} + +static void htab_mem_setup(void) +{ + struct bpf_map *map; + const char **names; + int err; + + setup_libbpf(); + + ctx.uc = htab_mem_find_use_case_or_exit(args.use_case); + err = htab_mem_bench_init_barriers(); + if (err) { + fprintf(stderr, "failed to init barrier\n"); + exit(1); + } + + ctx.fd = cgroup_setup_and_join("/htab_mem"); + if (ctx.fd < 0) + goto cleanup; + + ctx.skel = htab_mem_bench__open(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + goto cleanup; + } + + map = ctx.skel->maps.htab; + bpf_map__set_value_size(map, args.value_size); + /* Ensure that different CPUs can operate on different subset */ + bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus)); + if (args.preallocated) + bpf_map__set_map_flags(map, bpf_map__map_flags(map) & ~BPF_F_NO_PREALLOC); + + names = ctx.uc->progs; + while (*names) { + struct bpf_program *prog; + + prog = bpf_object__find_program_by_name(ctx.skel->obj, *names); + if (!prog) { + fprintf(stderr, "no such program %s\n", *names); + goto cleanup; + } + bpf_program__set_autoload(prog, true); + names++; + } + ctx.skel->bss->nr_thread = env.producer_cnt; + + err = htab_mem_bench__load(ctx.skel); + if (err) { + fprintf(stderr, "failed to load skeleton\n"); + goto cleanup; + } + err = htab_mem_bench__attach(ctx.skel); + if (err) { + fprintf(stderr, "failed to attach skeleton\n"); + goto cleanup; + } + return; + +cleanup: + htab_mem_bench__destroy(ctx.skel); + htab_mem_bench_exit_barriers(); + if (ctx.fd >= 0) { + close(ctx.fd); + cleanup_cgroup_environment(); + } + exit(1); +} + +static void htab_mem_add_fn(pthread_barrier_t *notify) +{ + while (true) { + /* Do addition */ + (void)syscall(__NR_getpgid, 0); + /* Notify deletion thread to do deletion */ + pthread_barrier_wait(notify); + /* Wait for deletion to complete */ + pthread_barrier_wait(notify); + } +} + +static void htab_mem_delete_fn(pthread_barrier_t *notify) +{ + while (true) { + /* Wait for addition to complete */ + pthread_barrier_wait(notify); + /* Do deletion */ + (void)syscall(__NR_getppid); + /* Notify addition thread to do addition */ + pthread_barrier_wait(notify); + } +} + +static void *htab_mem_producer(void *arg) +{ + pthread_barrier_t *notify; + int seq; + + if (!ctx.uc->need_sync) { + while (true) + (void)syscall(__NR_getpgid, 0); + return NULL; + } + + seq = (long)arg; + notify = &ctx.notify[seq / 2]; + if (seq & 1) + htab_mem_delete_fn(notify); + else + htab_mem_add_fn(notify); + return NULL; +} + +static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value) +{ + char buf[32]; + ssize_t got; + int fd; + + fd = openat(ctx.fd, name, O_RDONLY); + if (fd < 0) { + /* cgroup v1 ? */ + fprintf(stderr, "no %s\n", name); + *value = 0; + return; + } + + got = read(fd, buf, sizeof(buf) - 1); + if (got <= 0) { + *value = 0; + return; + } + buf[got] = 0; + + *value = strtoull(buf, NULL, 0); + + close(fd); +} + +static void htab_mem_measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt; + htab_mem_read_mem_cgrp_file("memory.current", &res->gp_ct); +} + +static void htab_mem_report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double loop, mem; + + loop = res->hits / 1000.0 / (delta_ns / 1000000000.0); + mem = res->gp_ct / 1048576.0; + printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0); + printf("per-prod-op %7.2lfk/s, memory usage %7.2lfMiB\n", loop, mem); +} + +static void htab_mem_report_final(struct bench_res res[], int res_cnt) +{ + double mem_mean = 0.0, mem_stddev = 0.0; + double loop_mean = 0.0, loop_stddev = 0.0; + unsigned long peak_mem; + int i; + + for (i = 0; i < res_cnt; i++) { + loop_mean += res[i].hits / 1000.0 / (0.0 + res_cnt); + mem_mean += res[i].gp_ct / 1048576.0 / (0.0 + res_cnt); + } + if (res_cnt > 1) { + for (i = 0; i < res_cnt; i++) { + loop_stddev += (loop_mean - res[i].hits / 1000.0) * + (loop_mean - res[i].hits / 1000.0) / + (res_cnt - 1.0); + mem_stddev += (mem_mean - res[i].gp_ct / 1048576.0) * + (mem_mean - res[i].gp_ct / 1048576.0) / + (res_cnt - 1.0); + } + loop_stddev = sqrt(loop_stddev); + mem_stddev = sqrt(mem_stddev); + } + + htab_mem_read_mem_cgrp_file("memory.peak", &peak_mem); + printf("Summary: per-prod-op %7.2lf \u00B1 %7.2lfk/s, memory usage %7.2lf \u00B1 %7.2lfMiB," + " peak memory usage %7.2lfMiB\n", + loop_mean, loop_stddev, mem_mean, mem_stddev, peak_mem / 1048576.0); + + cleanup_cgroup_environment(); +} + +const struct bench bench_htab_mem = { + .name = "htab-mem", + .argp = &bench_htab_mem_argp, + .validate = htab_mem_validate, + .setup = htab_mem_setup, + .producer_thread = htab_mem_producer, + .measure = htab_mem_measure, + .report_progress = htab_mem_report_progress, + .report_final = htab_mem_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c index 3ca14ad36607..e1ee979e6acc 100644 --- a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c +++ b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c @@ -399,7 +399,7 @@ static void perfbuf_libbpf_setup(void) ctx->skel = perfbuf_setup_skeleton(); memset(&attr, 0, sizeof(attr)); - attr.config = PERF_COUNT_SW_BPF_OUTPUT, + attr.config = PERF_COUNT_SW_BPF_OUTPUT; attr.type = PERF_TYPE_SOFTWARE; attr.sample_type = PERF_SAMPLE_RAW; /* notify only every Nth sample */ diff --git a/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh b/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh new file mode 100755 index 000000000000..9ff5832463a2 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +htab_mem() +{ + echo -n "per-prod-op: " + echo -n "$*" | sed -E "s/.* per-prod-op\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+k\/s).*/\1/" + echo -n -e ", avg mem: " + echo -n "$*" | sed -E "s/.* memory usage\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+MiB).*/\1/" + echo -n ", peak mem: " + echo "$*" | sed -E "s/.* peak memory usage\s+([0-9]+\.[0-9]+MiB).*/\1/" +} + +summarize_htab_mem() +{ + local bench="$1" + local summary=$(echo $2 | tail -n1) + + printf "%-20s %s\n" "$bench" "$(htab_mem $summary)" +} + +htab_mem_bench() +{ + local name + + for name in overwrite batch_add_batch_del add_del_on_diff_cpu + do + summarize_htab_mem "$name" "$($RUN_BENCH htab-mem --use-case $name -p8 "$@")" + done +} + +header "preallocated" +htab_mem_bench "--preallocated" + +header "normal bpf ma" +htab_mem_bench diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c index aaf6ef1201c7..a6f991b56345 100644 --- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c @@ -34,6 +34,11 @@ struct bpf_testmod_struct_arg_3 { int b[]; }; +struct bpf_testmod_struct_arg_4 { + u64 a; + int b; +}; + __diag_push(); __diag_ignore_all("-Wmissing-prototypes", "Global functions as their definitions will be in bpf_testmod.ko BTF"); @@ -75,6 +80,24 @@ bpf_testmod_test_struct_arg_6(struct bpf_testmod_struct_arg_3 *a) { return bpf_testmod_test_struct_arg_result; } +noinline int +bpf_testmod_test_struct_arg_7(u64 a, void *b, short c, int d, void *e, + struct bpf_testmod_struct_arg_4 f) +{ + bpf_testmod_test_struct_arg_result = a + (long)b + c + d + + (long)e + f.a + f.b; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_8(u64 a, void *b, short c, int d, void *e, + struct bpf_testmod_struct_arg_4 f, int g) +{ + bpf_testmod_test_struct_arg_result = a + (long)b + c + d + + (long)e + f.a + f.b + g; + return bpf_testmod_test_struct_arg_result; +} + __bpf_kfunc void bpf_testmod_test_mod_kfunc(int i) { @@ -191,6 +214,20 @@ noinline int bpf_testmod_fentry_test3(char a, int b, u64 c) return a + b + c; } +noinline int bpf_testmod_fentry_test7(u64 a, void *b, short c, int d, + void *e, char f, int g) +{ + return a + (long)b + c + d + (long)e + f + g; +} + +noinline int bpf_testmod_fentry_test11(u64 a, void *b, short c, int d, + void *e, char f, int g, + unsigned int h, long i, __u64 j, + unsigned long k) +{ + return a + (long)b + c + d + (long)e + f + g + h + i + j + k; +} + int bpf_testmod_fentry_ok; noinline ssize_t @@ -206,6 +243,7 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, struct bpf_testmod_struct_arg_1 struct_arg1 = {10}; struct bpf_testmod_struct_arg_2 struct_arg2 = {2, 3}; struct bpf_testmod_struct_arg_3 *struct_arg3; + struct bpf_testmod_struct_arg_4 struct_arg4 = {21, 22}; int i = 1; while (bpf_testmod_return_ptr(i)) @@ -216,6 +254,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, (void)bpf_testmod_test_struct_arg_3(1, 4, struct_arg2); (void)bpf_testmod_test_struct_arg_4(struct_arg1, 1, 2, 3, struct_arg2); (void)bpf_testmod_test_struct_arg_5(); + (void)bpf_testmod_test_struct_arg_7(16, (void *)17, 18, 19, + (void *)20, struct_arg4); + (void)bpf_testmod_test_struct_arg_8(16, (void *)17, 18, 19, + (void *)20, struct_arg4, 23); + struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) + sizeof(int)), GFP_KERNEL); @@ -243,7 +286,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, if (bpf_testmod_fentry_test1(1) != 2 || bpf_testmod_fentry_test2(2, 3) != 5 || - bpf_testmod_fentry_test3(4, 5, 6) != 15) + bpf_testmod_fentry_test3(4, 5, 6) != 15 || + bpf_testmod_fentry_test7(16, (void *)17, 18, 19, (void *)20, + 21, 22) != 133 || + bpf_testmod_fentry_test11(16, (void *)17, 18, 19, (void *)20, + 21, 22, 23, 24, 25, 26) != 231) goto out; bpf_testmod_fentry_ok = 1; diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 9e95b37a7dff..2caee8423ee0 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -278,6 +278,18 @@ int join_cgroup(const char *relative_path) } /** + * join_root_cgroup() - Join the root cgroup + * + * This function joins the root cgroup. + * + * On success, it returns 0, otherwise on failure it returns 1. + */ +int join_root_cgroup(void) +{ + return join_cgroup_from_top(CGROUP_MOUNT_PATH); +} + +/** * join_parent_cgroup() - Join a cgroup in the parent process workdir * @relative_path: The cgroup path, relative to parent process workdir, to join * diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index f099a166c94d..5c2cb9c8b546 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -22,6 +22,7 @@ void remove_cgroup(const char *relative_path); unsigned long long get_cgroup_id(const char *relative_path); int join_cgroup(const char *relative_path); +int join_root_cgroup(void); int join_parent_cgroup(const char *relative_path); int setup_cgroup_environment(void); diff --git a/tools/testing/selftests/bpf/cgroup_tcp_skb.h b/tools/testing/selftests/bpf/cgroup_tcp_skb.h new file mode 100644 index 000000000000..7f6b24f102fb --- /dev/null +++ b/tools/testing/selftests/bpf/cgroup_tcp_skb.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +/* Define states of a socket to tracking messages sending to and from the + * socket. + * + * These states are based on rfc9293 with some modifications to support + * tracking of messages sent out from a socket. For example, when a SYN is + * received, a new socket is transiting to the SYN_RECV state defined in + * rfc9293. But, we put it in SYN_RECV_SENDING_SYN_ACK state and when + * SYN-ACK is sent out, it moves to SYN_RECV state. With this modification, + * we can track the message sent out from a socket. + */ + +#ifndef __CGROUP_TCP_SKB_H__ +#define __CGROUP_TCP_SKB_H__ + +enum { + INIT, + CLOSED, + SYN_SENT, + SYN_RECV_SENDING_SYN_ACK, + SYN_RECV, + ESTABLISHED, + FIN_WAIT1, + FIN_WAIT2, + CLOSE_WAIT_SENDING_ACK, + CLOSE_WAIT, + CLOSING, + LAST_ACK, + TIME_WAIT_SENDING_ACK, + TIME_WAIT, +}; + +#endif /* __CGROUP_TCP_SKB_H__ */ diff --git a/tools/testing/selftests/bpf/gnu/stubs.h b/tools/testing/selftests/bpf/gnu/stubs.h index 719225b16626..1c638d9dce1a 100644 --- a/tools/testing/selftests/bpf/gnu/stubs.h +++ b/tools/testing/selftests/bpf/gnu/stubs.h @@ -1 +1 @@ -/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ +/* dummy .h to trick /usr/include/features.h to work with 'clang --target=bpf' */ diff --git a/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c new file mode 100644 index 000000000000..1a9eeefda9a8 --- /dev/null +++ b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include <errno.h> +#include <unistd.h> +#include <pthread.h> + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> + +#include <bpf_util.h> +#include <test_maps.h> + +#include "map_percpu_stats.skel.h" + +#define MAX_ENTRIES 16384 +#define MAX_ENTRIES_HASH_OF_MAPS 64 +#define N_THREADS 8 +#define MAX_MAP_KEY_SIZE 4 + +static void map_info(int map_fd, struct bpf_map_info *info) +{ + __u32 len = sizeof(*info); + int ret; + + memset(info, 0, sizeof(*info)); + + ret = bpf_obj_get_info_by_fd(map_fd, info, &len); + CHECK(ret < 0, "bpf_obj_get_info_by_fd", "error: %s\n", strerror(errno)); +} + +static const char *map_type_to_s(__u32 type) +{ + switch (type) { + case BPF_MAP_TYPE_HASH: + return "HASH"; + case BPF_MAP_TYPE_PERCPU_HASH: + return "PERCPU_HASH"; + case BPF_MAP_TYPE_LRU_HASH: + return "LRU_HASH"; + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + return "LRU_PERCPU_HASH"; + case BPF_MAP_TYPE_HASH_OF_MAPS: + return "BPF_MAP_TYPE_HASH_OF_MAPS"; + default: + return "<define-me>"; + } +} + +static __u32 map_count_elements(__u32 type, int map_fd) +{ + __u32 key = -1; + int n = 0; + + while (!bpf_map_get_next_key(map_fd, &key, &key)) + n++; + return n; +} + +#define BATCH true + +static void delete_and_lookup_batch(int map_fd, void *keys, __u32 count) +{ + static __u8 values[(8 << 10) * MAX_ENTRIES]; + void *in_batch = NULL, *out_batch; + __u32 save_count = count; + int ret; + + ret = bpf_map_lookup_and_delete_batch(map_fd, + &in_batch, &out_batch, + keys, values, &count, + NULL); + + /* + * Despite what uapi header says, lookup_and_delete_batch will return + * -ENOENT in case we successfully have deleted all elements, so check + * this separately + */ + CHECK(ret < 0 && (errno != ENOENT || !count), "bpf_map_lookup_and_delete_batch", + "error: %s\n", strerror(errno)); + + CHECK(count != save_count, + "bpf_map_lookup_and_delete_batch", + "deleted not all elements: removed=%u expected=%u\n", + count, save_count); +} + +static void delete_all_elements(__u32 type, int map_fd, bool batch) +{ + static __u8 val[8 << 10]; /* enough for 1024 CPUs */ + __u32 key = -1; + void *keys; + __u32 i, n; + int ret; + + keys = calloc(MAX_MAP_KEY_SIZE, MAX_ENTRIES); + CHECK(!keys, "calloc", "error: %s\n", strerror(errno)); + + for (n = 0; !bpf_map_get_next_key(map_fd, &key, &key); n++) + memcpy(keys + n*MAX_MAP_KEY_SIZE, &key, MAX_MAP_KEY_SIZE); + + if (batch) { + /* Can't mix delete_batch and delete_and_lookup_batch because + * they have different semantics in relation to the keys + * argument. However, delete_batch utilize map_delete_elem, + * so we actually test it in non-batch scenario */ + delete_and_lookup_batch(map_fd, keys, n); + } else { + /* Intentionally mix delete and lookup_and_delete so we can test both */ + for (i = 0; i < n; i++) { + void *keyp = keys + i*MAX_MAP_KEY_SIZE; + + if (i % 2 || type == BPF_MAP_TYPE_HASH_OF_MAPS) { + ret = bpf_map_delete_elem(map_fd, keyp); + CHECK(ret < 0, "bpf_map_delete_elem", + "error: key %u: %s\n", i, strerror(errno)); + } else { + ret = bpf_map_lookup_and_delete_elem(map_fd, keyp, val); + CHECK(ret < 0, "bpf_map_lookup_and_delete_elem", + "error: key %u: %s\n", i, strerror(errno)); + } + } + } + + free(keys); +} + +static bool is_lru(__u32 map_type) +{ + return map_type == BPF_MAP_TYPE_LRU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; +} + +struct upsert_opts { + __u32 map_type; + int map_fd; + __u32 n; +}; + +static int create_small_hash(void) +{ + int map_fd; + + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "small", 4, 4, 4, NULL); + CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n", + strerror(errno), "small"); + + return map_fd; +} + +static void *patch_map_thread(void *arg) +{ + struct upsert_opts *opts = arg; + int val; + int ret; + int i; + + for (i = 0; i < opts->n; i++) { + if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) + val = create_small_hash(); + else + val = rand(); + ret = bpf_map_update_elem(opts->map_fd, &i, &val, 0); + CHECK(ret < 0, "bpf_map_update_elem", "key=%d error: %s\n", i, strerror(errno)); + + if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) + close(val); + } + return NULL; +} + +static void upsert_elements(struct upsert_opts *opts) +{ + pthread_t threads[N_THREADS]; + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + ret = pthread_create(&i[threads], NULL, patch_map_thread, opts); + CHECK(ret != 0, "pthread_create", "error: %s\n", strerror(ret)); + } + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + ret = pthread_join(i[threads], NULL); + CHECK(ret != 0, "pthread_join", "error: %s\n", strerror(ret)); + } +} + +static __u32 read_cur_elements(int iter_fd) +{ + char buf[64]; + ssize_t n; + __u32 ret; + + n = read(iter_fd, buf, sizeof(buf)-1); + CHECK(n <= 0, "read", "error: %s\n", strerror(errno)); + buf[n] = '\0'; + + errno = 0; + ret = (__u32)strtol(buf, NULL, 10); + CHECK(errno != 0, "strtol", "error: %s\n", strerror(errno)); + + return ret; +} + +static __u32 get_cur_elements(int map_id) +{ + struct map_percpu_stats *skel; + struct bpf_link *link; + __u32 n_elements; + int iter_fd; + int ret; + + skel = map_percpu_stats__open(); + CHECK(skel == NULL, "map_percpu_stats__open", "error: %s", strerror(errno)); + + skel->bss->target_id = map_id; + + ret = map_percpu_stats__load(skel); + CHECK(ret != 0, "map_percpu_stats__load", "error: %s", strerror(errno)); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL); + CHECK(!link, "bpf_program__attach_iter", "error: %s\n", strerror(errno)); + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + CHECK(iter_fd < 0, "bpf_iter_create", "error: %s\n", strerror(errno)); + + n_elements = read_cur_elements(iter_fd); + + close(iter_fd); + bpf_link__destroy(link); + map_percpu_stats__destroy(skel); + + return n_elements; +} + +static void check_expected_number_elements(__u32 n_inserted, int map_fd, + struct bpf_map_info *info) +{ + __u32 n_real; + __u32 n_iter; + + /* Count the current number of elements in the map by iterating through + * all the map keys via bpf_get_next_key + */ + n_real = map_count_elements(info->type, map_fd); + + /* The "real" number of elements should be the same as the inserted + * number of elements in all cases except LRU maps, where some elements + * may have been evicted + */ + if (n_inserted == 0 || !is_lru(info->type)) + CHECK(n_inserted != n_real, "map_count_elements", + "n_real(%u) != n_inserted(%u)\n", n_real, n_inserted); + + /* Count the current number of elements in the map using an iterator */ + n_iter = get_cur_elements(info->id); + + /* Both counts should be the same, as all updates are over */ + CHECK(n_iter != n_real, "get_cur_elements", + "n_iter=%u, expected %u (map_type=%s,map_flags=%08x)\n", + n_iter, n_real, map_type_to_s(info->type), info->map_flags); +} + +static void __test(int map_fd) +{ + struct upsert_opts opts = { + .map_fd = map_fd, + }; + struct bpf_map_info info; + + map_info(map_fd, &info); + opts.map_type = info.type; + opts.n = info.max_entries; + + /* Reduce the number of elements we are updating such that we don't + * bump into -E2BIG from non-preallocated hash maps, but still will + * have some evictions for LRU maps */ + if (opts.map_type != BPF_MAP_TYPE_HASH_OF_MAPS) + opts.n -= 512; + else + opts.n /= 2; + + /* + * Upsert keys [0, n) under some competition: with random values from + * N_THREADS threads. Check values, then delete all elements and check + * values again. + */ + upsert_elements(&opts); + check_expected_number_elements(opts.n, map_fd, &info); + delete_all_elements(info.type, map_fd, !BATCH); + check_expected_number_elements(0, map_fd, &info); + + /* Now do the same, but using batch delete operations */ + upsert_elements(&opts); + check_expected_number_elements(opts.n, map_fd, &info); + delete_all_elements(info.type, map_fd, BATCH); + check_expected_number_elements(0, map_fd, &info); + + close(map_fd); +} + +static int map_create_opts(__u32 type, const char *name, + struct bpf_map_create_opts *map_opts, + __u32 key_size, __u32 val_size) +{ + int max_entries; + int map_fd; + + if (type == BPF_MAP_TYPE_HASH_OF_MAPS) + max_entries = MAX_ENTRIES_HASH_OF_MAPS; + else + max_entries = MAX_ENTRIES; + + map_fd = bpf_map_create(type, name, key_size, val_size, max_entries, map_opts); + CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n", + strerror(errno), name); + + return map_fd; +} + +static int map_create(__u32 type, const char *name, struct bpf_map_create_opts *map_opts) +{ + return map_create_opts(type, name, map_opts, sizeof(int), sizeof(int)); +} + +static int create_hash(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + }; + + return map_create(BPF_MAP_TYPE_HASH, "hash", &map_opts); +} + +static int create_percpu_hash(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + }; + + return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash", &map_opts); +} + +static int create_hash_prealloc(void) +{ + return map_create(BPF_MAP_TYPE_HASH, "hash", NULL); +} + +static int create_percpu_hash_prealloc(void) +{ + return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash_prealloc", NULL); +} + +static int create_lru_hash(__u32 type, __u32 map_flags) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = map_flags, + }; + + return map_create(type, "lru_hash", &map_opts); +} + +static int create_hash_of_maps(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + .inner_map_fd = create_small_hash(), + }; + int ret; + + ret = map_create_opts(BPF_MAP_TYPE_HASH_OF_MAPS, "hash_of_maps", + &map_opts, sizeof(int), sizeof(int)); + close(map_opts.inner_map_fd); + return ret; +} + +static void map_percpu_stats_hash(void) +{ + __test(create_hash()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_hash(void) +{ + __test(create_percpu_hash()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_hash_prealloc(void) +{ + __test(create_hash_prealloc()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_hash_prealloc(void) +{ + __test(create_percpu_hash_prealloc()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_lru_hash(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, 0)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_lru_hash_no_common(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_lru_hash(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_lru_hash_no_common(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, BPF_F_NO_COMMON_LRU)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_hash_of_maps(void) +{ + __test(create_hash_of_maps()); + printf("test_%s:PASS\n", __func__); +} + +void test_map_percpu_stats(void) +{ + map_percpu_stats_hash(); + map_percpu_stats_percpu_hash(); + map_percpu_stats_hash_prealloc(); + map_percpu_stats_percpu_hash_prealloc(); + map_percpu_stats_lru_hash(); + map_percpu_stats_lru_hash_no_common(); + map_percpu_stats_percpu_lru_hash(); + map_percpu_stats_percpu_lru_hash_no_common(); + map_percpu_stats_hash_of_maps(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index c8ba4009e4ab..b30ff6b3b81a 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -123,12 +123,13 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->data->test_snat_addr, 0, "Test for source natting"); ASSERT_EQ(skel->data->test_dnat_addr, 0, "Test for destination natting"); end: - if (srv_client_fd != -1) - close(srv_client_fd); if (client_fd != -1) close(client_fd); + if (srv_client_fd != -1) + close(srv_client_fd); if (srv_fd != -1) close(srv_fd); + snprintf(cmd, sizeof(cmd), iptables, "-D"); system(cmd); test_bpf_nf__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c b/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c new file mode 100644 index 000000000000..95bab61a1e57 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Facebook */ +#include <test_progs.h> +#include <linux/in6.h> +#include <sys/socket.h> +#include <sched.h> +#include <unistd.h> +#include "cgroup_helpers.h" +#include "testing_helpers.h" +#include "cgroup_tcp_skb.skel.h" +#include "cgroup_tcp_skb.h" + +#define CGROUP_TCP_SKB_PATH "/test_cgroup_tcp_skb" + +static int install_filters(int cgroup_fd, + struct bpf_link **egress_link, + struct bpf_link **ingress_link, + struct bpf_program *egress_prog, + struct bpf_program *ingress_prog, + struct cgroup_tcp_skb *skel) +{ + /* Prepare filters */ + skel->bss->g_sock_state = 0; + skel->bss->g_unexpected = 0; + *egress_link = + bpf_program__attach_cgroup(egress_prog, + cgroup_fd); + if (!ASSERT_OK_PTR(egress_link, "egress_link")) + return -1; + *ingress_link = + bpf_program__attach_cgroup(ingress_prog, + cgroup_fd); + if (!ASSERT_OK_PTR(ingress_link, "ingress_link")) + return -1; + + return 0; +} + +static void uninstall_filters(struct bpf_link **egress_link, + struct bpf_link **ingress_link) +{ + bpf_link__destroy(*egress_link); + *egress_link = NULL; + bpf_link__destroy(*ingress_link); + *ingress_link = NULL; +} + +static int create_client_sock_v6(void) +{ + int fd; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (fd < 0) { + perror("socket"); + return -1; + } + + return fd; +} + +static int create_server_sock_v6(void) +{ + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_port = htons(0), + .sin6_addr = IN6ADDR_LOOPBACK_INIT, + }; + int fd, err; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (fd < 0) { + perror("socket"); + return -1; + } + + err = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); + if (err < 0) { + perror("bind"); + return -1; + } + + err = listen(fd, 1); + if (err < 0) { + perror("listen"); + return -1; + } + + return fd; +} + +static int get_sock_port_v6(int fd) +{ + struct sockaddr_in6 addr; + socklen_t len; + int err; + + len = sizeof(addr); + err = getsockname(fd, (struct sockaddr *)&addr, &len); + if (err < 0) { + perror("getsockname"); + return -1; + } + + return ntohs(addr.sin6_port); +} + +static int connect_client_server_v6(int client_fd, int listen_fd) +{ + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT, + }; + int err; + + addr.sin6_port = htons(get_sock_port_v6(listen_fd)); + if (addr.sin6_port < 0) + return -1; + + err = connect(client_fd, (struct sockaddr *)&addr, sizeof(addr)); + if (err < 0) { + perror("connect"); + return -1; + } + + return 0; +} + +/* Connect to the server in a cgroup from the outside of the cgroup. */ +static int talk_to_cgroup(int *client_fd, int *listen_fd, int *service_fd, + struct cgroup_tcp_skb *skel) +{ + int err, cp; + char buf[5]; + + /* Create client & server socket */ + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + *client_fd = create_client_sock_v6(); + if (!ASSERT_GE(*client_fd, 0, "client_fd")) + return -1; + err = join_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_OK(err, "join_cgroup")) + return -1; + *listen_fd = create_server_sock_v6(); + if (!ASSERT_GE(*listen_fd, 0, "listen_fd")) + return -1; + skel->bss->g_sock_port = get_sock_port_v6(*listen_fd); + + /* Connect client to server */ + err = connect_client_server_v6(*client_fd, *listen_fd); + if (!ASSERT_OK(err, "connect_client_server_v6")) + return -1; + *service_fd = accept(*listen_fd, NULL, NULL); + if (!ASSERT_GE(*service_fd, 0, "service_fd")) + return -1; + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + cp = write(*client_fd, "hello", 5); + if (!ASSERT_EQ(cp, 5, "write")) + return -1; + cp = read(*service_fd, buf, 5); + if (!ASSERT_EQ(cp, 5, "read")) + return -1; + + return 0; +} + +/* Connect to the server out of a cgroup from inside the cgroup. */ +static int talk_to_outside(int *client_fd, int *listen_fd, int *service_fd, + struct cgroup_tcp_skb *skel) + +{ + int err, cp; + char buf[5]; + + /* Create client & server socket */ + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + *listen_fd = create_server_sock_v6(); + if (!ASSERT_GE(*listen_fd, 0, "listen_fd")) + return -1; + err = join_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_OK(err, "join_cgroup")) + return -1; + *client_fd = create_client_sock_v6(); + if (!ASSERT_GE(*client_fd, 0, "client_fd")) + return -1; + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + skel->bss->g_sock_port = get_sock_port_v6(*listen_fd); + + /* Connect client to server */ + err = connect_client_server_v6(*client_fd, *listen_fd); + if (!ASSERT_OK(err, "connect_client_server_v6")) + return -1; + *service_fd = accept(*listen_fd, NULL, NULL); + if (!ASSERT_GE(*service_fd, 0, "service_fd")) + return -1; + cp = write(*client_fd, "hello", 5); + if (!ASSERT_EQ(cp, 5, "write")) + return -1; + cp = read(*service_fd, buf, 5); + if (!ASSERT_EQ(cp, 5, "read")) + return -1; + + return 0; +} + +static int close_connection(int *closing_fd, int *peer_fd, int *listen_fd, + struct cgroup_tcp_skb *skel) +{ + __u32 saved_packet_count = 0; + int err; + int i; + + /* Wait for ACKs to be sent */ + saved_packet_count = skel->bss->g_packet_count; + usleep(100000); /* 0.1s */ + for (i = 0; + skel->bss->g_packet_count != saved_packet_count && i < 10; + i++) { + saved_packet_count = skel->bss->g_packet_count; + usleep(100000); /* 0.1s */ + } + if (!ASSERT_EQ(skel->bss->g_packet_count, saved_packet_count, + "packet_count")) + return -1; + + skel->bss->g_packet_count = 0; + saved_packet_count = 0; + + /* Half shutdown to make sure the closing socket having a chance to + * receive a FIN from the peer. + */ + err = shutdown(*closing_fd, SHUT_WR); + if (!ASSERT_OK(err, "shutdown closing_fd")) + return -1; + + /* Wait for FIN and the ACK of the FIN to be observed */ + for (i = 0; + skel->bss->g_packet_count < saved_packet_count + 2 && i < 10; + i++) + usleep(100000); /* 0.1s */ + if (!ASSERT_GE(skel->bss->g_packet_count, saved_packet_count + 2, + "packet_count")) + return -1; + + saved_packet_count = skel->bss->g_packet_count; + + /* Fully shutdown the connection */ + err = close(*peer_fd); + if (!ASSERT_OK(err, "close peer_fd")) + return -1; + *peer_fd = -1; + + /* Wait for FIN and the ACK of the FIN to be observed */ + for (i = 0; + skel->bss->g_packet_count < saved_packet_count + 2 && i < 10; + i++) + usleep(100000); /* 0.1s */ + if (!ASSERT_GE(skel->bss->g_packet_count, saved_packet_count + 2, + "packet_count")) + return -1; + + err = close(*closing_fd); + if (!ASSERT_OK(err, "close closing_fd")) + return -1; + *closing_fd = -1; + + close(*listen_fd); + *listen_fd = -1; + + return 0; +} + +/* This test case includes four scenarios: + * 1. Connect to the server from outside the cgroup and close the connection + * from outside the cgroup. + * 2. Connect to the server from outside the cgroup and close the connection + * from inside the cgroup. + * 3. Connect to the server from inside the cgroup and close the connection + * from outside the cgroup. + * 4. Connect to the server from inside the cgroup and close the connection + * from inside the cgroup. + * + * The test case is to verify that cgroup_skb/{egress,ingress} filters + * receive expected packets including SYN, SYN/ACK, ACK, FIN, and FIN/ACK. + */ +void test_cgroup_tcp_skb(void) +{ + struct bpf_link *ingress_link = NULL; + struct bpf_link *egress_link = NULL; + int client_fd = -1, listen_fd = -1; + struct cgroup_tcp_skb *skel; + int service_fd = -1; + int cgroup_fd = -1; + int err; + + skel = cgroup_tcp_skb__open_and_load(); + if (!ASSERT_OK(!skel, "skel_open_load")) + return; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + goto cleanup; + + cgroup_fd = create_and_get_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "cgroup_fd")) + goto cleanup; + + /* Scenario 1 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.server_egress, + skel->progs.server_ingress, + skel); + if (!ASSERT_OK(err, "install_filters")) + goto cleanup; + + err = talk_to_cgroup(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_cgroup")) + goto cleanup; + + err = close_connection(&client_fd, &service_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, CLOSED, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 2 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.server_egress_srv, + skel->progs.server_ingress_srv, + skel); + + err = talk_to_cgroup(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_cgroup")) + goto cleanup; + + err = close_connection(&service_fd, &client_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, TIME_WAIT, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 3 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.client_egress_srv, + skel->progs.client_ingress_srv, + skel); + + err = talk_to_outside(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_outside")) + goto cleanup; + + err = close_connection(&service_fd, &client_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, CLOSED, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 4 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.client_egress, + skel->progs.client_ingress, + skel); + + err = talk_to_outside(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_outside")) + goto cleanup; + + err = close_connection(&client_fd, &service_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, TIME_WAIT, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + +cleanup: + close(client_fd); + close(listen_fd); + close(service_fd); + close(cgroup_fd); + bpf_link__destroy(egress_link); + bpf_link__destroy(ingress_link); + cleanup_cgroup_environment(); + cgroup_tcp_skb__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fentry_test.c b/tools/testing/selftests/bpf/prog_tests/fentry_test.c index c0d1d61d5f66..aee1bc77a17f 100644 --- a/tools/testing/selftests/bpf/prog_tests/fentry_test.c +++ b/tools/testing/selftests/bpf/prog_tests/fentry_test.c @@ -2,8 +2,9 @@ /* Copyright (c) 2019 Facebook */ #include <test_progs.h> #include "fentry_test.lskel.h" +#include "fentry_many_args.skel.h" -static int fentry_test(struct fentry_test_lskel *fentry_skel) +static int fentry_test_common(struct fentry_test_lskel *fentry_skel) { int err, prog_fd, i; int link_fd; @@ -37,7 +38,7 @@ static int fentry_test(struct fentry_test_lskel *fentry_skel) return 0; } -void test_fentry_test(void) +static void fentry_test(void) { struct fentry_test_lskel *fentry_skel = NULL; int err; @@ -46,13 +47,47 @@ void test_fentry_test(void) if (!ASSERT_OK_PTR(fentry_skel, "fentry_skel_load")) goto cleanup; - err = fentry_test(fentry_skel); + err = fentry_test_common(fentry_skel); if (!ASSERT_OK(err, "fentry_first_attach")) goto cleanup; - err = fentry_test(fentry_skel); + err = fentry_test_common(fentry_skel); ASSERT_OK(err, "fentry_second_attach"); cleanup: fentry_test_lskel__destroy(fentry_skel); } + +static void fentry_many_args(void) +{ + struct fentry_many_args *fentry_skel = NULL; + int err; + + fentry_skel = fentry_many_args__open_and_load(); + if (!ASSERT_OK_PTR(fentry_skel, "fentry_many_args_skel_load")) + goto cleanup; + + err = fentry_many_args__attach(fentry_skel); + if (!ASSERT_OK(err, "fentry_many_args_attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + ASSERT_EQ(fentry_skel->bss->test1_result, 1, + "fentry_many_args_result1"); + ASSERT_EQ(fentry_skel->bss->test2_result, 1, + "fentry_many_args_result2"); + ASSERT_EQ(fentry_skel->bss->test3_result, 1, + "fentry_many_args_result3"); + +cleanup: + fentry_many_args__destroy(fentry_skel); +} + +void test_fentry_test(void) +{ + if (test__start_subtest("fentry")) + fentry_test(); + if (test__start_subtest("fentry_many_args")) + fentry_many_args(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_test.c b/tools/testing/selftests/bpf/prog_tests/fexit_test.c index 101b7343036b..1c13007e37dd 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_test.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_test.c @@ -2,8 +2,9 @@ /* Copyright (c) 2019 Facebook */ #include <test_progs.h> #include "fexit_test.lskel.h" +#include "fexit_many_args.skel.h" -static int fexit_test(struct fexit_test_lskel *fexit_skel) +static int fexit_test_common(struct fexit_test_lskel *fexit_skel) { int err, prog_fd, i; int link_fd; @@ -37,7 +38,7 @@ static int fexit_test(struct fexit_test_lskel *fexit_skel) return 0; } -void test_fexit_test(void) +static void fexit_test(void) { struct fexit_test_lskel *fexit_skel = NULL; int err; @@ -46,13 +47,47 @@ void test_fexit_test(void) if (!ASSERT_OK_PTR(fexit_skel, "fexit_skel_load")) goto cleanup; - err = fexit_test(fexit_skel); + err = fexit_test_common(fexit_skel); if (!ASSERT_OK(err, "fexit_first_attach")) goto cleanup; - err = fexit_test(fexit_skel); + err = fexit_test_common(fexit_skel); ASSERT_OK(err, "fexit_second_attach"); cleanup: fexit_test_lskel__destroy(fexit_skel); } + +static void fexit_many_args(void) +{ + struct fexit_many_args *fexit_skel = NULL; + int err; + + fexit_skel = fexit_many_args__open_and_load(); + if (!ASSERT_OK_PTR(fexit_skel, "fexit_many_args_skel_load")) + goto cleanup; + + err = fexit_many_args__attach(fexit_skel); + if (!ASSERT_OK(err, "fexit_many_args_attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + ASSERT_EQ(fexit_skel->bss->test1_result, 1, + "fexit_many_args_result1"); + ASSERT_EQ(fexit_skel->bss->test2_result, 1, + "fexit_many_args_result2"); + ASSERT_EQ(fexit_skel->bss->test3_result, 1, + "fexit_many_args_result3"); + +cleanup: + fexit_many_args__destroy(fexit_skel); +} + +void test_fexit_test(void) +{ + if (test__start_subtest("fexit")) + fexit_test(); + if (test__start_subtest("fexit_many_args")) + fexit_many_args(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c index 28cf63963cb7..64a9c95d4acf 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c @@ -30,7 +30,9 @@ void test_get_func_args_test(void) prog_fd = bpf_program__fd(skel->progs.fmod_ret_test); err = bpf_prog_test_run_opts(prog_fd, &topts); ASSERT_OK(err, "test_run"); - ASSERT_EQ(topts.retval, 1234, "test_run"); + + ASSERT_EQ(topts.retval >> 16, 1, "test_run"); + ASSERT_EQ(topts.retval & 0xffff, 1234 + 29, "test_run"); ASSERT_EQ(skel->bss->test1_result, 1, "test1_result"); ASSERT_EQ(skel->bss->test2_result, 1, "test2_result"); diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index fd41425d2e5c..56b5baef35c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -22,7 +22,7 @@ static void global_map_resize_bss_subtest(void) struct test_global_map_resize *skel; struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; - size_t array_len, actual_sz; + size_t array_len, actual_sz, new_sz; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -42,6 +42,10 @@ static void global_map_resize_bss_subtest(void) if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "resize")) goto teardown; + new_sz = sizeof(skel->data_percpu_arr->percpu_arr[0]) * libbpf_num_possible_cpus(); + err = bpf_map__set_value_size(skel->maps.data_percpu_arr, new_sz); + ASSERT_OK(err, "percpu_arr_resize"); + /* set the expected number of elements based on the resized array */ array_len = (desired_sz - sizeof(skel->bss->sum)) / sizeof(skel->bss->array[0]); if (!ASSERT_GT(array_len, 1, "array_len")) @@ -84,11 +88,11 @@ teardown: static void global_map_resize_data_subtest(void) { - int err; struct test_global_map_resize *skel; struct bpf_map *map; const __u32 desired_sz = sysconf(_SC_PAGE_SIZE) * 2; - size_t array_len, actual_sz; + size_t array_len, actual_sz, new_sz; + int err; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -108,6 +112,10 @@ static void global_map_resize_data_subtest(void) if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "resize")) goto teardown; + new_sz = sizeof(skel->data_percpu_arr->percpu_arr[0]) * libbpf_num_possible_cpus(); + err = bpf_map__set_value_size(skel->maps.data_percpu_arr, new_sz); + ASSERT_OK(err, "percpu_arr_resize"); + /* set the expected number of elements based on the resized array */ array_len = (desired_sz - sizeof(skel->bss->sum)) / sizeof(skel->data_custom->my_array[0]); if (!ASSERT_GT(array_len, 1, "array_len")) diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index f63309fd0e28..18cf7b17463d 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -23,7 +23,7 @@ static struct { "bpf_spin_lock at off=" #off " must be held for bpf_list_head" }, \ { #test "_missing_lock_pop_back", \ "bpf_spin_lock at off=" #off " must be held for bpf_list_head" }, - TEST(kptr, 32) + TEST(kptr, 40) TEST(global, 16) TEST(map, 0) TEST(inner_map, 0) @@ -31,7 +31,7 @@ static struct { #define TEST(test, op) \ { #test "_kptr_incorrect_lock_" #op, \ "held lock and object are not in the same allocation\n" \ - "bpf_spin_lock at off=32 must be held for bpf_list_head" }, \ + "bpf_spin_lock at off=40 must be held for bpf_list_head" }, \ { #test "_global_incorrect_lock_" #op, \ "held lock and object are not in the same allocation\n" \ "bpf_spin_lock at off=16 must be held for bpf_list_head" }, \ @@ -84,23 +84,23 @@ static struct { { "double_push_back", "arg#1 expected pointer to allocated object" }, { "no_node_value_type", "bpf_list_node not found at offset=0" }, { "incorrect_value_type", - "operation on bpf_list_head expects arg#1 bpf_list_node at offset=40 in struct foo, " + "operation on bpf_list_head expects arg#1 bpf_list_node at offset=48 in struct foo, " "but arg is at offset=0 in struct bar" }, { "incorrect_node_var_off", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, - { "incorrect_node_off1", "bpf_list_node not found at offset=41" }, - { "incorrect_node_off2", "arg#1 offset=0, but expected bpf_list_node at offset=40 in struct foo" }, + { "incorrect_node_off1", "bpf_list_node not found at offset=49" }, + { "incorrect_node_off2", "arg#1 offset=0, but expected bpf_list_node at offset=48 in struct foo" }, { "no_head_type", "bpf_list_head not found at offset=0" }, { "incorrect_head_var_off1", "R1 doesn't have constant offset" }, { "incorrect_head_var_off2", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, - { "incorrect_head_off1", "bpf_list_head not found at offset=17" }, + { "incorrect_head_off1", "bpf_list_head not found at offset=25" }, { "incorrect_head_off2", "bpf_list_head not found at offset=1" }, { "pop_front_off", - "15: (bf) r1 = r6 ; R1_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=40,imm=0) " - "R6_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=40,imm=0) refs=2,4\n" + "15: (bf) r1 = r6 ; R1_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=48,imm=0) " + "R6_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=48,imm=0) refs=2,4\n" "16: (85) call bpf_this_cpu_ptr#154\nR1 type=ptr_or_null_ expected=percpu_ptr_" }, { "pop_back_off", - "15: (bf) r1 = r6 ; R1_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=40,imm=0) " - "R6_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=40,imm=0) refs=2,4\n" + "15: (bf) r1 = r6 ; R1_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=48,imm=0) " + "R6_w=ptr_or_null_foo(id=4,ref_obj_id=4,off=48,imm=0) refs=2,4\n" "16: (85) call bpf_this_cpu_ptr#154\nR1 type=ptr_or_null_ expected=percpu_ptr_" }, }; @@ -257,7 +257,7 @@ static struct btf *init_btf(void) hid = btf__add_struct(btf, "bpf_list_head", 16); if (!ASSERT_EQ(hid, LIST_HEAD, "btf__add_struct bpf_list_head")) goto end; - nid = btf__add_struct(btf, "bpf_list_node", 16); + nid = btf__add_struct(btf, "bpf_list_node", 24); if (!ASSERT_EQ(nid, LIST_NODE, "btf__add_struct bpf_list_node")) goto end; return btf; @@ -276,7 +276,7 @@ static void list_and_rb_node_same_struct(bool refcount_field) if (!ASSERT_OK_PTR(btf, "init_btf")) return; - bpf_rb_node_btf_id = btf__add_struct(btf, "bpf_rb_node", 24); + bpf_rb_node_btf_id = btf__add_struct(btf, "bpf_rb_node", 32); if (!ASSERT_GT(bpf_rb_node_btf_id, 0, "btf__add_struct bpf_rb_node")) return; @@ -286,17 +286,17 @@ static void list_and_rb_node_same_struct(bool refcount_field) return; } - id = btf__add_struct(btf, "bar", refcount_field ? 44 : 40); + id = btf__add_struct(btf, "bar", refcount_field ? 60 : 56); if (!ASSERT_GT(id, 0, "btf__add_struct bar")) return; err = btf__add_field(btf, "a", LIST_NODE, 0, 0); if (!ASSERT_OK(err, "btf__add_field bar::a")) return; - err = btf__add_field(btf, "c", bpf_rb_node_btf_id, 128, 0); + err = btf__add_field(btf, "c", bpf_rb_node_btf_id, 192, 0); if (!ASSERT_OK(err, "btf__add_field bar::c")) return; if (refcount_field) { - err = btf__add_field(btf, "ref", bpf_refcount_btf_id, 320, 0); + err = btf__add_field(btf, "ref", bpf_refcount_btf_id, 448, 0); if (!ASSERT_OK(err, "btf__add_field bar::ref")) return; } @@ -527,7 +527,7 @@ static void test_btf(void) btf = init_btf(); if (!ASSERT_OK_PTR(btf, "init_btf")) break; - id = btf__add_struct(btf, "foo", 36); + id = btf__add_struct(btf, "foo", 44); if (!ASSERT_EQ(id, 5, "btf__add_struct foo")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -536,7 +536,7 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field foo::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field foo::c")) break; id = btf__add_decl_tag(btf, "contains:foo:b", 5, 0); @@ -553,7 +553,7 @@ static void test_btf(void) btf = init_btf(); if (!ASSERT_OK_PTR(btf, "init_btf")) break; - id = btf__add_struct(btf, "foo", 36); + id = btf__add_struct(btf, "foo", 44); if (!ASSERT_EQ(id, 5, "btf__add_struct foo")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -562,13 +562,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field foo::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field foo::c")) break; id = btf__add_decl_tag(btf, "contains:bar:b", 5, 0); if (!ASSERT_EQ(id, 6, "btf__add_decl_tag contains:bar:b")) break; - id = btf__add_struct(btf, "bar", 36); + id = btf__add_struct(btf, "bar", 44); if (!ASSERT_EQ(id, 7, "btf__add_struct bar")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -577,7 +577,7 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field bar::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field bar::c")) break; id = btf__add_decl_tag(btf, "contains:foo:b", 7, 0); @@ -594,19 +594,19 @@ static void test_btf(void) btf = init_btf(); if (!ASSERT_OK_PTR(btf, "init_btf")) break; - id = btf__add_struct(btf, "foo", 20); + id = btf__add_struct(btf, "foo", 28); if (!ASSERT_EQ(id, 5, "btf__add_struct foo")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); if (!ASSERT_OK(err, "btf__add_field foo::a")) break; - err = btf__add_field(btf, "b", SPIN_LOCK, 128, 0); + err = btf__add_field(btf, "b", SPIN_LOCK, 192, 0); if (!ASSERT_OK(err, "btf__add_field foo::b")) break; id = btf__add_decl_tag(btf, "contains:bar:a", 5, 0); if (!ASSERT_EQ(id, 6, "btf__add_decl_tag contains:bar:a")) break; - id = btf__add_struct(btf, "bar", 16); + id = btf__add_struct(btf, "bar", 24); if (!ASSERT_EQ(id, 7, "btf__add_struct bar")) break; err = btf__add_field(btf, "a", LIST_NODE, 0, 0); @@ -623,19 +623,19 @@ static void test_btf(void) btf = init_btf(); if (!ASSERT_OK_PTR(btf, "init_btf")) break; - id = btf__add_struct(btf, "foo", 20); + id = btf__add_struct(btf, "foo", 28); if (!ASSERT_EQ(id, 5, "btf__add_struct foo")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); if (!ASSERT_OK(err, "btf__add_field foo::a")) break; - err = btf__add_field(btf, "b", SPIN_LOCK, 128, 0); + err = btf__add_field(btf, "b", SPIN_LOCK, 192, 0); if (!ASSERT_OK(err, "btf__add_field foo::b")) break; id = btf__add_decl_tag(btf, "contains:bar:b", 5, 0); if (!ASSERT_EQ(id, 6, "btf__add_decl_tag contains:bar:b")) break; - id = btf__add_struct(btf, "bar", 36); + id = btf__add_struct(btf, "bar", 44); if (!ASSERT_EQ(id, 7, "btf__add_struct bar")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -644,13 +644,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field bar::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field bar::c")) break; id = btf__add_decl_tag(btf, "contains:baz:a", 7, 0); if (!ASSERT_EQ(id, 8, "btf__add_decl_tag contains:baz:a")) break; - id = btf__add_struct(btf, "baz", 16); + id = btf__add_struct(btf, "baz", 24); if (!ASSERT_EQ(id, 9, "btf__add_struct baz")) break; err = btf__add_field(btf, "a", LIST_NODE, 0, 0); @@ -667,7 +667,7 @@ static void test_btf(void) btf = init_btf(); if (!ASSERT_OK_PTR(btf, "init_btf")) break; - id = btf__add_struct(btf, "foo", 36); + id = btf__add_struct(btf, "foo", 44); if (!ASSERT_EQ(id, 5, "btf__add_struct foo")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -676,13 +676,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field foo::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field foo::c")) break; id = btf__add_decl_tag(btf, "contains:bar:b", 5, 0); if (!ASSERT_EQ(id, 6, "btf__add_decl_tag contains:bar:b")) break; - id = btf__add_struct(btf, "bar", 36); + id = btf__add_struct(btf, "bar", 44); if (!ASSERT_EQ(id, 7, "btf__add_struct bar")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -691,13 +691,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field bar:b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field bar:c")) break; id = btf__add_decl_tag(btf, "contains:baz:a", 7, 0); if (!ASSERT_EQ(id, 8, "btf__add_decl_tag contains:baz:a")) break; - id = btf__add_struct(btf, "baz", 16); + id = btf__add_struct(btf, "baz", 24); if (!ASSERT_EQ(id, 9, "btf__add_struct baz")) break; err = btf__add_field(btf, "a", LIST_NODE, 0, 0); @@ -726,7 +726,7 @@ static void test_btf(void) id = btf__add_decl_tag(btf, "contains:bar:b", 5, 0); if (!ASSERT_EQ(id, 6, "btf__add_decl_tag contains:bar:b")) break; - id = btf__add_struct(btf, "bar", 36); + id = btf__add_struct(btf, "bar", 44); if (!ASSERT_EQ(id, 7, "btf__add_struct bar")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -735,13 +735,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field bar::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field bar::c")) break; id = btf__add_decl_tag(btf, "contains:baz:b", 7, 0); if (!ASSERT_EQ(id, 8, "btf__add_decl_tag")) break; - id = btf__add_struct(btf, "baz", 36); + id = btf__add_struct(btf, "baz", 44); if (!ASSERT_EQ(id, 9, "btf__add_struct baz")) break; err = btf__add_field(btf, "a", LIST_HEAD, 0, 0); @@ -750,13 +750,13 @@ static void test_btf(void) err = btf__add_field(btf, "b", LIST_NODE, 128, 0); if (!ASSERT_OK(err, "btf__add_field bar::b")) break; - err = btf__add_field(btf, "c", SPIN_LOCK, 256, 0); + err = btf__add_field(btf, "c", SPIN_LOCK, 320, 0); if (!ASSERT_OK(err, "btf__add_field bar::c")) break; id = btf__add_decl_tag(btf, "contains:bam:a", 9, 0); if (!ASSERT_EQ(id, 10, "btf__add_decl_tag contains:bam:a")) break; - id = btf__add_struct(btf, "bam", 16); + id = btf__add_struct(btf, "bam", 24); if (!ASSERT_EQ(id, 11, "btf__add_struct bam")) break; err = btf__add_field(btf, "a", LIST_NODE, 0, 0); diff --git a/tools/testing/selftests/bpf/prog_tests/modify_return.c b/tools/testing/selftests/bpf/prog_tests/modify_return.c index 5d9955af6247..a70c99c2f8c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/modify_return.c +++ b/tools/testing/selftests/bpf/prog_tests/modify_return.c @@ -41,6 +41,10 @@ static void run_test(__u32 input_retval, __u16 want_side_effect, __s16 want_ret) ASSERT_EQ(skel->bss->fexit_result, 1, "modify_return fexit_result"); ASSERT_EQ(skel->bss->fmod_ret_result, 1, "modify_return fmod_ret_result"); + ASSERT_EQ(skel->bss->fentry_result2, 1, "modify_return fentry_result2"); + ASSERT_EQ(skel->bss->fexit_result2, 1, "modify_return fexit_result2"); + ASSERT_EQ(skel->bss->fmod_ret_result2, 1, "modify_return fmod_ret_result2"); + cleanup: modify_return__destroy(skel); } @@ -49,9 +53,9 @@ cleanup: void serial_test_modify_return(void) { run_test(0 /* input_retval */, - 1 /* want_side_effect */, - 4 /* want_ret */); + 2 /* want_side_effect */, + 33 /* want_ret */); run_test(-EINVAL /* input_retval */, 0 /* want_side_effect */, - -EINVAL /* want_ret */); + -EINVAL * 2 /* want_ret */); } diff --git a/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c b/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c new file mode 100644 index 000000000000..4297a2a4cb11 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <netinet/in.h> +#include <linux/netfilter.h> + +#include "test_progs.h" +#include "test_netfilter_link_attach.skel.h" + +struct nf_link_test { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + + bool expect_success; + const char * const name; +}; + +static const struct nf_link_test nf_hook_link_tests[] = { + { .name = "allzero", }, + { .pf = NFPROTO_NUMPROTO, .name = "invalid-pf", }, + { .pf = NFPROTO_IPV4, .hooknum = 42, .name = "invalid-hooknum", }, + { .pf = NFPROTO_IPV4, .priority = INT_MIN, .name = "invalid-priority-min", }, + { .pf = NFPROTO_IPV4, .priority = INT_MAX, .name = "invalid-priority-max", }, + { .pf = NFPROTO_IPV4, .flags = UINT_MAX, .name = "invalid-flags", }, + + { .pf = NFPROTO_INET, .priority = 1, .name = "invalid-inet-not-supported", }, + + { .pf = NFPROTO_IPV4, .priority = -10000, .expect_success = true, .name = "attach ipv4", }, + { .pf = NFPROTO_IPV6, .priority = 10001, .expect_success = true, .name = "attach ipv6", }, +}; + +void test_netfilter_link_attach(void) +{ + struct test_netfilter_link_attach *skel; + struct bpf_program *prog; + LIBBPF_OPTS(bpf_netfilter_opts, opts); + int i; + + skel = test_netfilter_link_attach__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_netfilter_link_attach__open_and_load")) + goto out; + + prog = skel->progs.nf_link_attach_test; + if (!ASSERT_OK_PTR(prog, "attach program")) + goto out; + + for (i = 0; i < ARRAY_SIZE(nf_hook_link_tests); i++) { + struct bpf_link *link; + + if (!test__start_subtest(nf_hook_link_tests[i].name)) + continue; + +#define X(opts, m, i) opts.m = nf_hook_link_tests[(i)].m + X(opts, pf, i); + X(opts, hooknum, i); + X(opts, priority, i); + X(opts, flags, i); +#undef X + link = bpf_program__attach_netfilter(prog, &opts); + if (nf_hook_link_tests[i].expect_success) { + struct bpf_link *link2; + + if (!ASSERT_OK_PTR(link, "program attach successful")) + continue; + + link2 = bpf_program__attach_netfilter(prog, &opts); + ASSERT_ERR_PTR(link2, "attach program with same pf/hook/priority"); + + if (!ASSERT_OK(bpf_link__destroy(link), "link destroy")) + break; + + link2 = bpf_program__attach_netfilter(prog, &opts); + if (!ASSERT_OK_PTR(link2, "program reattach successful")) + continue; + if (!ASSERT_OK(bpf_link__destroy(link2), "link destroy")) + break; + } else { + ASSERT_ERR_PTR(link, "program load failure"); + } + } + +out: + test_netfilter_link_attach__destroy(skel); +} + diff --git a/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c b/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c new file mode 100644 index 000000000000..8d077d150c56 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */ + +#include <string.h> +#include <linux/bpf.h> +#include <test_progs.h> +#include "test_ptr_untrusted.skel.h" + +#define TP_NAME "sched_switch" + +void serial_test_ptr_untrusted(void) +{ + struct test_ptr_untrusted *skel; + int err; + + skel = test_ptr_untrusted__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + /* First, attach lsm prog */ + skel->links.lsm_run = bpf_program__attach_lsm(skel->progs.lsm_run); + if (!ASSERT_OK_PTR(skel->links.lsm_run, "lsm_attach")) + goto cleanup; + + /* Second, attach raw_tp prog. The lsm prog will be triggered. */ + skel->links.raw_tp_run = bpf_program__attach_raw_tracepoint(skel->progs.raw_tp_run, + TP_NAME); + if (!ASSERT_OK_PTR(skel->links.raw_tp_run, "raw_tp_attach")) + goto cleanup; + + err = strncmp(skel->bss->tp_name, TP_NAME, strlen(TP_NAME)); + ASSERT_EQ(err, 0, "cmp_tp_name"); + +cleanup: + test_ptr_untrusted__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c index 595cbf92bff5..7423983472c7 100644 --- a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c @@ -14,3 +14,7 @@ void test_refcounted_kptr(void) void test_refcounted_kptr_fail(void) { } + +void test_refcounted_kptr_wrong_owner(void) +{ +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_helpers.h b/tools/testing/selftests/bpf/prog_tests/tc_helpers.h new file mode 100644 index 000000000000..6c93215be8a3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_helpers.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Isovalent */ +#ifndef TC_HELPERS +#define TC_HELPERS +#include <test_progs.h> + +static inline __u32 id_from_prog_fd(int fd) +{ + struct bpf_prog_info prog_info = {}; + __u32 prog_info_len = sizeof(prog_info); + int err; + + err = bpf_obj_get_info_by_fd(fd, &prog_info, &prog_info_len); + if (!ASSERT_OK(err, "id_from_prog_fd")) + return 0; + + ASSERT_NEQ(prog_info.id, 0, "prog_info.id"); + return prog_info.id; +} + +static inline __u32 id_from_link_fd(int fd) +{ + struct bpf_link_info link_info = {}; + __u32 link_info_len = sizeof(link_info); + int err; + + err = bpf_link_get_info_by_fd(fd, &link_info, &link_info_len); + if (!ASSERT_OK(err, "id_from_link_fd")) + return 0; + + ASSERT_NEQ(link_info.id, 0, "link_info.id"); + return link_info.id; +} + +static inline __u32 ifindex_from_link_fd(int fd) +{ + struct bpf_link_info link_info = {}; + __u32 link_info_len = sizeof(link_info); + int err; + + err = bpf_link_get_info_by_fd(fd, &link_info, &link_info_len); + if (!ASSERT_OK(err, "id_from_link_fd")) + return 0; + + return link_info.tcx.ifindex; +} + +static inline void __assert_mprog_count(int target, int expected, bool miniq, int ifindex) +{ + __u32 count = 0, attach_flags = 0; + int err; + + err = bpf_prog_query(ifindex, target, 0, &attach_flags, + NULL, &count); + ASSERT_EQ(count, expected, "count"); + if (!expected && !miniq) + ASSERT_EQ(err, -ENOENT, "prog_query"); + else + ASSERT_EQ(err, 0, "prog_query"); +} + +static inline void assert_mprog_count(int target, int expected) +{ + __assert_mprog_count(target, expected, false, loopback); +} + +static inline void assert_mprog_count_ifindex(int ifindex, int target, int expected) +{ + __assert_mprog_count(target, expected, false, ifindex); +} + +#endif /* TC_HELPERS */ diff --git a/tools/testing/selftests/bpf/prog_tests/tc_links.c b/tools/testing/selftests/bpf/prog_tests/tc_links.c new file mode 100644 index 000000000000..81eea5f10742 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_links.c @@ -0,0 +1,1583 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ +#include <uapi/linux/if_link.h> +#include <net/if.h> +#include <test_progs.h> + +#define loopback 1 +#define ping_cmd "ping -q -c1 -w1 127.0.0.1 > /dev/null" + +#include "test_tc_link.skel.h" +#include "tc_helpers.h" + +void serial_test_tc_links_basic(void) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[2], link_ids[2]; + __u32 pid1, pid2, lid1, lid2; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + + assert_mprog_count(BPF_TCX_INGRESS, 0); + assert_mprog_count(BPF_TCX_EGRESS, 0); + + ASSERT_EQ(skel->bss->seen_tc1, false, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(BPF_TCX_INGRESS, 1); + assert_mprog_count(BPF_TCX_EGRESS, 0); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, BPF_TCX_INGRESS, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 2, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], 0, "link_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + ASSERT_NEQ(lid1, lid2, "link_ids_1_2"); + + assert_mprog_count(BPF_TCX_INGRESS, 1); + assert_mprog_count(BPF_TCX_EGRESS, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, BPF_TCX_EGRESS, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 2, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid2, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid2, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], 0, "link_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); +cleanup: + test_tc_link__destroy(skel); + + assert_mprog_count(BPF_TCX_INGRESS, 0); + assert_mprog_count(BPF_TCX_EGRESS, 0); +} + +static void test_tc_links_before_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[5], link_ids[5]; + __u32 pid1, pid2, pid3, pid4; + __u32 lid1, lid2, lid3, lid4; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + .relative_fd = bpf_program__fd(skel->progs.tc2), + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc3 = link; + + lid3 = id_from_link_fd(bpf_link__fd(skel->links.tc3)); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_LINK, + .relative_id = lid1, + ); + + link = bpf_program__attach_tcx(skel->progs.tc4, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc4 = link; + + lid4 = id_from_link_fd(bpf_link__fd(skel->links.tc4)); + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid4, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid4, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid1, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid1, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], pid3, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], lid3, "link_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], pid2, "prog_ids[3]"); + ASSERT_EQ(optq.link_ids[3], lid2, "link_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + ASSERT_EQ(optq.link_ids[4], 0, "link_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_before(void) +{ + test_tc_links_before_target(BPF_TCX_INGRESS); + test_tc_links_before_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_after_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[5], link_ids[5]; + __u32 pid1, pid2, pid3, pid4; + __u32 lid1, lid2, lid3, lid4; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc3 = link; + + lid3 = id_from_link_fd(bpf_link__fd(skel->links.tc3)); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER | BPF_F_LINK, + .relative_fd = bpf_link__fd(skel->links.tc2), + ); + + link = bpf_program__attach_tcx(skel->progs.tc4, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc4 = link; + + lid4 = id_from_link_fd(bpf_link__fd(skel->links.tc4)); + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid3, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid3, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], pid2, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], lid2, "link_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], pid4, "prog_ids[3]"); + ASSERT_EQ(optq.link_ids[3], lid4, "link_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + ASSERT_EQ(optq.link_ids[4], 0, "link_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_after(void) +{ + test_tc_links_after_target(BPF_TCX_INGRESS); + test_tc_links_after_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_revision_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[3], link_ids[3]; + __u32 pid1, pid2, lid1, lid2; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + + assert_mprog_count(target, 0); + + optl.expected_revision = 1; + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + optl.expected_revision = 1; + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 1); + + optl.expected_revision = 2; + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_revision(void) +{ + test_tc_links_revision_target(BPF_TCX_INGRESS); + test_tc_links_revision_target(BPF_TCX_EGRESS); +} + +static void test_tc_chain_classic(int target, bool chain_tc_old) +{ + LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1); + LIBBPF_OPTS(bpf_tc_hook, tc_hook, .ifindex = loopback); + bool hook_created = false, tc_attached = false; + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 pid1, pid2, pid3; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + if (chain_tc_old) { + tc_hook.attach_point = target == BPF_TCX_INGRESS ? + BPF_TC_INGRESS : BPF_TC_EGRESS; + err = bpf_tc_hook_create(&tc_hook); + if (err == 0) + hook_created = true; + err = err == -EEXIST ? 0 : err; + if (!ASSERT_OK(err, "bpf_tc_hook_create")) + goto cleanup; + + tc_opts.prog_fd = bpf_program__fd(skel->progs.tc3); + err = bpf_tc_attach(&tc_hook, &tc_opts); + if (!ASSERT_OK(err, "bpf_tc_attach")) + goto cleanup; + tc_attached = true; + } + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + assert_mprog_count(target, 2); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, chain_tc_old, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + err = bpf_link__detach(skel->links.tc2); + if (!ASSERT_OK(err, "prog_detach")) + goto cleanup; + + assert_mprog_count(target, 1); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, chain_tc_old, "seen_tc3"); +cleanup: + if (tc_attached) { + tc_opts.flags = tc_opts.prog_fd = tc_opts.prog_id = 0; + err = bpf_tc_detach(&tc_hook, &tc_opts); + ASSERT_OK(err, "bpf_tc_detach"); + } + if (hook_created) { + tc_hook.attach_point = BPF_TC_INGRESS | BPF_TC_EGRESS; + bpf_tc_hook_destroy(&tc_hook); + } + assert_mprog_count(target, 1); + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_chain_classic(void) +{ + test_tc_chain_classic(BPF_TCX_INGRESS, false); + test_tc_chain_classic(BPF_TCX_EGRESS, false); + test_tc_chain_classic(BPF_TCX_INGRESS, true); + test_tc_chain_classic(BPF_TCX_EGRESS, true); +} + +static void test_tc_links_replace_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 pid1, pid2, pid3, lid1, lid2; + __u32 prog_ids[4], link_ids[4]; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + optl.expected_revision = 1; + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + .relative_id = pid1, + .expected_revision = 2, + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid2, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid2, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid1, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid1, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_REPLACE, + .relative_fd = bpf_program__fd(skel->progs.tc2), + .expected_revision = 3, + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_REPLACE | BPF_F_LINK, + .relative_fd = bpf_link__fd(skel->links.tc2), + .expected_revision = 3, + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_REPLACE | BPF_F_LINK | BPF_F_AFTER, + .relative_id = lid2, + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 2); + + err = bpf_link__update_program(skel->links.tc2, skel->progs.tc3); + if (!ASSERT_OK(err, "link_update")) + goto cleanup; + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 4, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid3, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid2, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid1, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid1, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + err = bpf_link__detach(skel->links.tc2); + if (!ASSERT_OK(err, "link_detach")) + goto cleanup; + + assert_mprog_count(target, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], 0, "link_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + err = bpf_link__update_program(skel->links.tc1, skel->progs.tc1); + if (!ASSERT_OK(err, "link_update_self")) + goto cleanup; + + assert_mprog_count(target, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], 0, "link_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_replace(void) +{ + test_tc_links_replace_target(BPF_TCX_INGRESS); + test_tc_links_replace_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_invalid_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 pid1, pid2, lid1; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + + assert_mprog_count(target, 0); + + optl.flags = BPF_F_BEFORE | BPF_F_AFTER; + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_ID, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER | BPF_F_ID, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_ID, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_LINK, + .relative_fd = bpf_program__fd(skel->progs.tc2), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_LINK, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .relative_fd = bpf_program__fd(skel->progs.tc2), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_AFTER, + .relative_fd = bpf_program__fd(skel->progs.tc2), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_ID, + .relative_id = pid2, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_ID, + .relative_id = 42, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_LINK, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl); + + link = bpf_program__attach_tcx(skel->progs.tc1, 0, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER | BPF_F_LINK, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optl); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER | BPF_F_LINK, + .relative_fd = bpf_program__fd(skel->progs.tc1), + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_LINK | BPF_F_ID, + .relative_id = ~0, + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_LINK | BPF_F_ID, + .relative_id = lid1, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_ID, + .relative_id = pid1, + ); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_ERR_PTR(link, "link_attach_should_fail")) { + bpf_link__destroy(link); + goto cleanup; + } + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE | BPF_F_LINK | BPF_F_ID, + .relative_id = lid1, + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + assert_mprog_count(target, 2); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_invalid(void) +{ + test_tc_links_invalid_target(BPF_TCX_INGRESS); + test_tc_links_invalid_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_prepend_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[5], link_ids[5]; + __u32 pid1, pid2, pid3, pid4; + __u32 lid1, lid2, lid3, lid4; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid2, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid2, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid1, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid1, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc3 = link; + + lid3 = id_from_link_fd(bpf_link__fd(skel->links.tc3)); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_BEFORE, + ); + + link = bpf_program__attach_tcx(skel->progs.tc4, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc4 = link; + + lid4 = id_from_link_fd(bpf_link__fd(skel->links.tc4)); + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid4, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid4, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid3, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid3, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], pid2, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], lid2, "link_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], pid1, "prog_ids[3]"); + ASSERT_EQ(optq.link_ids[3], lid1, "link_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + ASSERT_EQ(optq.link_ids[4], 0, "link_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_prepend(void) +{ + test_tc_links_prepend_target(BPF_TCX_INGRESS); + test_tc_links_prepend_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_append_target(int target) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 prog_ids[5], link_ids[5]; + __u32 pid1, pid2, pid3, pid4; + __u32 lid1, lid2, lid3, lid4; + struct test_tc_link *skel; + struct bpf_link *link; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + link = bpf_program__attach_tcx(skel->progs.tc1, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + lid1 = id_from_link_fd(bpf_link__fd(skel->links.tc1)); + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER, + ); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + optq.link_ids = link_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], 0, "link_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER, + ); + + link = bpf_program__attach_tcx(skel->progs.tc3, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc3 = link; + + lid3 = id_from_link_fd(bpf_link__fd(skel->links.tc3)); + + LIBBPF_OPTS_RESET(optl, + .flags = BPF_F_AFTER, + ); + + link = bpf_program__attach_tcx(skel->progs.tc4, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc4 = link; + + lid4 = id_from_link_fd(bpf_link__fd(skel->links.tc4)); + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(link_ids, 0, sizeof(link_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid1, "prog_ids[0]"); + ASSERT_EQ(optq.link_ids[0], lid1, "link_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], pid3, "prog_ids[2]"); + ASSERT_EQ(optq.link_ids[2], lid3, "link_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], pid4, "prog_ids[3]"); + ASSERT_EQ(optq.link_ids[3], lid4, "link_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + ASSERT_EQ(optq.link_ids[4], 0, "link_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_links_append(void) +{ + test_tc_links_append_target(BPF_TCX_INGRESS); + test_tc_links_append_target(BPF_TCX_EGRESS); +} + +static void test_tc_links_dev_cleanup_target(int target) +{ + LIBBPF_OPTS(bpf_tcx_opts, optl); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 pid1, pid2, pid3, pid4; + struct test_tc_link *skel; + struct bpf_link *link; + int err, ifindex; + + ASSERT_OK(system("ip link add dev tcx_opts1 type veth peer name tcx_opts2"), "add veth"); + ifindex = if_nametoindex("tcx_opts1"); + ASSERT_NEQ(ifindex, 0, "non_zero_ifindex"); + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + link = bpf_program__attach_tcx(skel->progs.tc1, ifindex, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc1 = link; + + assert_mprog_count_ifindex(ifindex, target, 1); + + link = bpf_program__attach_tcx(skel->progs.tc2, ifindex, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc2 = link; + + assert_mprog_count_ifindex(ifindex, target, 2); + + link = bpf_program__attach_tcx(skel->progs.tc3, ifindex, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc3 = link; + + assert_mprog_count_ifindex(ifindex, target, 3); + + link = bpf_program__attach_tcx(skel->progs.tc4, ifindex, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup; + + skel->links.tc4 = link; + + assert_mprog_count_ifindex(ifindex, target, 4); + + ASSERT_OK(system("ip link del dev tcx_opts1"), "del veth"); + ASSERT_EQ(if_nametoindex("tcx_opts1"), 0, "dev1_removed"); + ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed"); + + ASSERT_EQ(ifindex_from_link_fd(bpf_link__fd(skel->links.tc1)), 0, "tc1_ifindex"); + ASSERT_EQ(ifindex_from_link_fd(bpf_link__fd(skel->links.tc2)), 0, "tc2_ifindex"); + ASSERT_EQ(ifindex_from_link_fd(bpf_link__fd(skel->links.tc3)), 0, "tc3_ifindex"); + ASSERT_EQ(ifindex_from_link_fd(bpf_link__fd(skel->links.tc4)), 0, "tc4_ifindex"); + + test_tc_link__destroy(skel); + return; +cleanup: + test_tc_link__destroy(skel); + + ASSERT_OK(system("ip link del dev tcx_opts1"), "del veth"); + ASSERT_EQ(if_nametoindex("tcx_opts1"), 0, "dev1_removed"); + ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed"); +} + +void serial_test_tc_links_dev_cleanup(void) +{ + test_tc_links_dev_cleanup_target(BPF_TCX_INGRESS); + test_tc_links_dev_cleanup_target(BPF_TCX_EGRESS); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_opts.c b/tools/testing/selftests/bpf/prog_tests/tc_opts.c new file mode 100644 index 000000000000..7914100f9b46 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_opts.c @@ -0,0 +1,2239 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ +#include <uapi/linux/if_link.h> +#include <net/if.h> +#include <test_progs.h> + +#define loopback 1 +#define ping_cmd "ping -q -c1 -w1 127.0.0.1 > /dev/null" + +#include "test_tc_link.skel.h" +#include "tc_helpers.h" + +void serial_test_tc_opts_basic(void) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, id1, id2; + struct test_tc_link *skel; + __u32 prog_ids[2]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + + assert_mprog_count(BPF_TCX_INGRESS, 0); + assert_mprog_count(BPF_TCX_EGRESS, 0); + + ASSERT_EQ(skel->bss->seen_tc1, false, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + + err = bpf_prog_attach_opts(fd1, loopback, BPF_TCX_INGRESS, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(BPF_TCX_INGRESS, 1); + assert_mprog_count(BPF_TCX_EGRESS, 0); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, BPF_TCX_INGRESS, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_in; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 2, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + + err = bpf_prog_attach_opts(fd2, loopback, BPF_TCX_EGRESS, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_in; + + assert_mprog_count(BPF_TCX_INGRESS, 1); + assert_mprog_count(BPF_TCX_EGRESS, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, BPF_TCX_EGRESS, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_eg; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 2, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + +cleanup_eg: + err = bpf_prog_detach_opts(fd2, loopback, BPF_TCX_EGRESS, &optd); + ASSERT_OK(err, "prog_detach_eg"); + + assert_mprog_count(BPF_TCX_INGRESS, 1); + assert_mprog_count(BPF_TCX_EGRESS, 0); + +cleanup_in: + err = bpf_prog_detach_opts(fd1, loopback, BPF_TCX_INGRESS, &optd); + ASSERT_OK(err, "prog_detach_in"); + + assert_mprog_count(BPF_TCX_INGRESS, 0); + assert_mprog_count(BPF_TCX_EGRESS, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +static void test_tc_opts_before_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + .relative_fd = fd2, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target3; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 4, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id2, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + .relative_id = id1, + ); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target3; + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id4, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id1, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id3, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id2, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); + +cleanup_target4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup_target3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup_target2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup_target: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_before(void) +{ + test_tc_opts_before_target(BPF_TCX_INGRESS); + test_tc_opts_before_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_after_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target3; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 4, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id2, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + .relative_id = id2, + ); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target3; + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id2, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); + +cleanup_target4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target3; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 6, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id2, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + +cleanup_target3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 7, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + +cleanup_target2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 8, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + +cleanup_target: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_after(void) +{ + test_tc_opts_after_target(BPF_TCX_INGRESS); + test_tc_opts_after_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_revision_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, id1, id2; + struct test_tc_link *skel; + __u32 prog_ids[3]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .expected_revision = 1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .expected_revision = 1, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, -ESTALE, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .expected_revision = 2, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + + LIBBPF_OPTS_RESET(optd, + .expected_revision = 2, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_EQ(err, -ESTALE, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup_target2: + LIBBPF_OPTS_RESET(optd, + .expected_revision = 3, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup_target: + LIBBPF_OPTS_RESET(optd); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_revision(void) +{ + test_tc_opts_revision_target(BPF_TCX_INGRESS); + test_tc_opts_revision_target(BPF_TCX_EGRESS); +} + +static void test_tc_chain_classic(int target, bool chain_tc_old) +{ + LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1); + LIBBPF_OPTS(bpf_tc_hook, tc_hook, .ifindex = loopback); + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + bool hook_created = false, tc_attached = false; + __u32 fd1, fd2, fd3, id1, id2, id3; + struct test_tc_link *skel; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + if (chain_tc_old) { + tc_hook.attach_point = target == BPF_TCX_INGRESS ? + BPF_TC_INGRESS : BPF_TC_EGRESS; + err = bpf_tc_hook_create(&tc_hook); + if (err == 0) + hook_created = true; + err = err == -EEXIST ? 0 : err; + if (!ASSERT_OK(err, "bpf_tc_hook_create")) + goto cleanup; + + tc_opts.prog_fd = fd3; + err = bpf_tc_attach(&tc_hook, &tc_opts); + if (!ASSERT_OK(err, "bpf_tc_attach")) + goto cleanup; + tc_attached = true; + } + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_detach; + + assert_mprog_count(target, 2); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, chain_tc_old, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + if (!ASSERT_OK(err, "prog_detach")) + goto cleanup_detach; + + assert_mprog_count(target, 1); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, chain_tc_old, "seen_tc3"); + +cleanup_detach: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + if (!ASSERT_OK(err, "prog_detach")) + goto cleanup; + + __assert_mprog_count(target, 0, chain_tc_old, loopback); +cleanup: + if (tc_attached) { + tc_opts.flags = tc_opts.prog_fd = tc_opts.prog_id = 0; + err = bpf_tc_detach(&tc_hook, &tc_opts); + ASSERT_OK(err, "bpf_tc_detach"); + } + if (hook_created) { + tc_hook.attach_point = BPF_TC_INGRESS | BPF_TC_EGRESS; + bpf_tc_hook_destroy(&tc_hook); + } + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_opts_chain_classic(void) +{ + test_tc_chain_classic(BPF_TCX_INGRESS, false); + test_tc_chain_classic(BPF_TCX_EGRESS, false); + test_tc_chain_classic(BPF_TCX_INGRESS, true); + test_tc_chain_classic(BPF_TCX_EGRESS, true); +} + +static void test_tc_opts_replace_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, id1, id2, id3, detach_fd; + __u32 prog_ids[4], prog_flags[4]; + struct test_tc_link *skel; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .expected_revision = 1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + .relative_id = id1, + .expected_revision = 2, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + detach_fd = fd2; + + assert_mprog_count(target, 2); + + optq.prog_attach_flags = prog_flags; + optq.prog_ids = prog_ids; + + memset(prog_flags, 0, sizeof(prog_flags)); + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id1, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_EQ(optq.prog_attach_flags[0], 0, "prog_flags[0]"); + ASSERT_EQ(optq.prog_attach_flags[1], 0, "prog_flags[1]"); + ASSERT_EQ(optq.prog_attach_flags[2], 0, "prog_flags[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = fd2, + .expected_revision = 3, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + detach_fd = fd3; + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 4, "revision"); + ASSERT_EQ(optq.prog_ids[0], id3, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id1, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, false, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + + skel->bss->seen_tc1 = false; + skel->bss->seen_tc2 = false; + skel->bss->seen_tc3 = false; + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE | BPF_F_BEFORE, + .replace_prog_fd = fd3, + .relative_fd = fd1, + .expected_revision = 4, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + detach_fd = fd2; + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id1, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = fd2, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE | BPF_F_AFTER, + .replace_prog_fd = fd2, + .relative_fd = fd1, + .expected_revision = 5, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + ASSERT_EQ(err, -ERANGE, "prog_attach"); + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE | BPF_F_AFTER | BPF_F_REPLACE, + .replace_prog_fd = fd2, + .relative_fd = fd1, + .expected_revision = 5, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + ASSERT_EQ(err, -ERANGE, "prog_attach"); + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_id = id1, + .expected_revision = 5, + ); + +cleanup_target2: + err = bpf_prog_detach_opts(detach_fd, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup_target: + LIBBPF_OPTS_RESET(optd); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_replace(void) +{ + test_tc_opts_replace_target(BPF_TCX_INGRESS); + test_tc_opts_replace_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_invalid_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + __u32 fd1, fd2, id1, id2; + struct test_tc_link *skel; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE | BPF_F_AFTER, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ERANGE, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE | BPF_F_ID, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ENOENT, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER | BPF_F_ID, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ENOENT, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .relative_fd = fd2, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EINVAL, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE | BPF_F_AFTER, + .relative_fd = fd2, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ENOENT, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_ID, + .relative_id = id2, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EINVAL, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ENOENT, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -ENOENT, "prog_attach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(opta); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .relative_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EINVAL, "prog_attach_x1"); + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = fd1, + ); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + assert_mprog_count(target, 1); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_invalid(void) +{ + test_tc_opts_invalid_target(BPF_TCX_INGRESS); + test_tc_opts_invalid_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_prepend_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id1, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target3; + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id4, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id2, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id1, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); + +cleanup_target4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup_target3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup_target2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup_target: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_prepend(void) +{ + test_tc_opts_prepend_target(BPF_TCX_INGRESS); + test_tc_opts_prepend_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_append_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target; + + assert_mprog_count(target, 2); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target2; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 3, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, false, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, false, "seen_tc4"); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target2; + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup_target3; + + assert_mprog_count(target, 4); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup_target4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id3, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + + ASSERT_EQ(skel->bss->seen_tc1, true, "seen_tc1"); + ASSERT_EQ(skel->bss->seen_tc2, true, "seen_tc2"); + ASSERT_EQ(skel->bss->seen_tc3, true, "seen_tc3"); + ASSERT_EQ(skel->bss->seen_tc4, true, "seen_tc4"); + +cleanup_target4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup_target3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup_target2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup_target: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_append(void) +{ + test_tc_opts_append_target(BPF_TCX_INGRESS); + test_tc_opts_append_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_dev_cleanup_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + int err, ifindex; + + ASSERT_OK(system("ip link add dev tcx_opts1 type veth peer name tcx_opts2"), "add veth"); + ifindex = if_nametoindex("tcx_opts1"); + ASSERT_NEQ(ifindex, 0, "non_zero_ifindex"); + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count_ifindex(ifindex, target, 0); + + err = bpf_prog_attach_opts(fd1, ifindex, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count_ifindex(ifindex, target, 1); + + err = bpf_prog_attach_opts(fd2, ifindex, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup1; + + assert_mprog_count_ifindex(ifindex, target, 2); + + err = bpf_prog_attach_opts(fd3, ifindex, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup2; + + assert_mprog_count_ifindex(ifindex, target, 3); + + err = bpf_prog_attach_opts(fd4, ifindex, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup3; + + assert_mprog_count_ifindex(ifindex, target, 4); + + ASSERT_OK(system("ip link del dev tcx_opts1"), "del veth"); + ASSERT_EQ(if_nametoindex("tcx_opts1"), 0, "dev1_removed"); + ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed"); + return; +cleanup3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count_ifindex(ifindex, target, 2); +cleanup2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count_ifindex(ifindex, target, 1); +cleanup1: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count_ifindex(ifindex, target, 0); +cleanup: + test_tc_link__destroy(skel); + + ASSERT_OK(system("ip link del dev tcx_opts1"), "del veth"); + ASSERT_EQ(if_nametoindex("tcx_opts1"), 0, "dev1_removed"); + ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed"); +} + +void serial_test_tc_opts_dev_cleanup(void) +{ + test_tc_opts_dev_cleanup_target(BPF_TCX_INGRESS); + test_tc_opts_dev_cleanup_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_mixed_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + LIBBPF_OPTS(bpf_tcx_opts, optl); + __u32 pid1, pid2, pid3, pid4, lid2, lid4; + __u32 prog_flags[4], link_flags[4]; + __u32 prog_ids[4], link_ids[4]; + struct test_tc_link *skel; + struct bpf_link *link; + int err, detach_fd; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc3, target), + 0, "tc3_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc4, target), + 0, "tc4_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + pid3 = id_from_prog_fd(bpf_program__fd(skel->progs.tc3)); + pid4 = id_from_prog_fd(bpf_program__fd(skel->progs.tc4)); + + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + ASSERT_NEQ(pid3, pid4, "prog_ids_3_4"); + ASSERT_NEQ(pid2, pid3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc1), + loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + detach_fd = bpf_program__fd(skel->progs.tc1); + + assert_mprog_count(target, 1); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup1; + skel->links.tc2 = link; + + lid2 = id_from_link_fd(bpf_link__fd(skel->links.tc2)); + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = bpf_program__fd(skel->progs.tc1), + ); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc2), + loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = bpf_program__fd(skel->progs.tc2), + ); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc1), + loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = bpf_program__fd(skel->progs.tc2), + ); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc3), + loopback, target, &opta); + ASSERT_EQ(err, -EBUSY, "prog_attach"); + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = bpf_program__fd(skel->progs.tc1), + ); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc3), + loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup1; + + detach_fd = bpf_program__fd(skel->progs.tc3); + + assert_mprog_count(target, 2); + + link = bpf_program__attach_tcx(skel->progs.tc4, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup1; + skel->links.tc4 = link; + + lid4 = id_from_link_fd(bpf_link__fd(skel->links.tc4)); + + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(opta, + .flags = BPF_F_REPLACE, + .replace_prog_fd = bpf_program__fd(skel->progs.tc4), + ); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc2), + loopback, target, &opta); + ASSERT_EQ(err, -EEXIST, "prog_attach"); + + optq.prog_ids = prog_ids; + optq.prog_attach_flags = prog_flags; + optq.link_ids = link_ids; + optq.link_attach_flags = link_flags; + + memset(prog_ids, 0, sizeof(prog_ids)); + memset(prog_flags, 0, sizeof(prog_flags)); + memset(link_ids, 0, sizeof(link_ids)); + memset(link_flags, 0, sizeof(link_flags)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup1; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], pid3, "prog_ids[0]"); + ASSERT_EQ(optq.prog_attach_flags[0], 0, "prog_flags[0]"); + ASSERT_EQ(optq.link_ids[0], 0, "link_ids[0]"); + ASSERT_EQ(optq.link_attach_flags[0], 0, "link_flags[0]"); + ASSERT_EQ(optq.prog_ids[1], pid2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_attach_flags[1], 0, "prog_flags[1]"); + ASSERT_EQ(optq.link_ids[1], lid2, "link_ids[1]"); + ASSERT_EQ(optq.link_attach_flags[1], 0, "link_flags[1]"); + ASSERT_EQ(optq.prog_ids[2], pid4, "prog_ids[2]"); + ASSERT_EQ(optq.prog_attach_flags[2], 0, "prog_flags[2]"); + ASSERT_EQ(optq.link_ids[2], lid4, "link_ids[2]"); + ASSERT_EQ(optq.link_attach_flags[2], 0, "link_flags[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + ASSERT_EQ(optq.prog_attach_flags[3], 0, "prog_flags[3]"); + ASSERT_EQ(optq.link_ids[3], 0, "link_ids[3]"); + ASSERT_EQ(optq.link_attach_flags[3], 0, "link_flags[3]"); + + ASSERT_OK(system(ping_cmd), ping_cmd); + +cleanup1: + err = bpf_prog_detach_opts(detach_fd, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_opts_mixed(void) +{ + test_tc_opts_mixed_target(BPF_TCX_INGRESS); + test_tc_opts_mixed_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_demixed_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_tcx_opts, optl); + struct test_tc_link *skel; + struct bpf_link *link; + __u32 pid1, pid2; + int err; + + skel = test_tc_link__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc1, target), + 0, "tc1_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc2, target), + 0, "tc2_attach_type"); + + err = test_tc_link__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + pid1 = id_from_prog_fd(bpf_program__fd(skel->progs.tc1)); + pid2 = id_from_prog_fd(bpf_program__fd(skel->progs.tc2)); + ASSERT_NEQ(pid1, pid2, "prog_ids_1_2"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(bpf_program__fd(skel->progs.tc1), + loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + link = bpf_program__attach_tcx(skel->progs.tc2, loopback, &optl); + if (!ASSERT_OK_PTR(link, "link_attach")) + goto cleanup1; + skel->links.tc2 = link; + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_EQ(err, -EBUSY, "prog_detach"); + + assert_mprog_count(target, 2); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 1); + goto cleanup; + +cleanup1: + err = bpf_prog_detach_opts(bpf_program__fd(skel->progs.tc1), + loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup: + test_tc_link__destroy(skel); + assert_mprog_count(target, 0); +} + +void serial_test_tc_opts_demixed(void) +{ + test_tc_opts_demixed_target(BPF_TCX_INGRESS); + test_tc_opts_demixed_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_detach_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup1; + + assert_mprog_count(target, 2); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup2; + + assert_mprog_count(target, 3); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup3; + + assert_mprog_count(target, 4); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id3, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 3); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 6, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id4, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 7, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + LIBBPF_OPTS_RESET(optd); + + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_EQ(err, -ENOENT, "prog_detach"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_EQ(err, -ENOENT, "prog_detach"); + goto cleanup; + +cleanup4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup1: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_detach(void) +{ + test_tc_opts_detach_target(BPF_TCX_INGRESS); + test_tc_opts_detach_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_detach_before_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup1; + + assert_mprog_count(target, 2); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup2; + + assert_mprog_count(target, 3); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup3; + + assert_mprog_count(target, 4); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id3, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd2, + ); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 3); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 6, "revision"); + ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id4, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd2, + ); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_EQ(err, -ENOENT, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd4, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_EQ(err, -ERANGE, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_EQ(err, -ENOENT, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd3, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 7, "revision"); + ASSERT_EQ(optq.prog_ids[0], id3, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id4, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + .relative_fd = fd4, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 8, "revision"); + ASSERT_EQ(optq.prog_ids[0], id4, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_BEFORE, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 0); + goto cleanup; + +cleanup4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup1: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_detach_before(void) +{ + test_tc_opts_detach_before_target(BPF_TCX_INGRESS); + test_tc_opts_detach_before_target(BPF_TCX_EGRESS); +} + +static void test_tc_opts_detach_after_target(int target) +{ + LIBBPF_OPTS(bpf_prog_attach_opts, opta); + LIBBPF_OPTS(bpf_prog_detach_opts, optd); + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; + struct test_tc_link *skel; + __u32 prog_ids[5]; + int err; + + skel = test_tc_link__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + fd1 = bpf_program__fd(skel->progs.tc1); + fd2 = bpf_program__fd(skel->progs.tc2); + fd3 = bpf_program__fd(skel->progs.tc3); + fd4 = bpf_program__fd(skel->progs.tc4); + + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + + ASSERT_NEQ(id1, id2, "prog_ids_1_2"); + ASSERT_NEQ(id3, id4, "prog_ids_3_4"); + ASSERT_NEQ(id2, id3, "prog_ids_2_3"); + + assert_mprog_count(target, 0); + + err = bpf_prog_attach_opts(fd1, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup; + + assert_mprog_count(target, 1); + + err = bpf_prog_attach_opts(fd2, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup1; + + assert_mprog_count(target, 2); + + err = bpf_prog_attach_opts(fd3, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup2; + + assert_mprog_count(target, 3); + + err = bpf_prog_attach_opts(fd4, loopback, target, &opta); + if (!ASSERT_EQ(err, 0, "prog_attach")) + goto cleanup3; + + assert_mprog_count(target, 4); + + optq.prog_ids = prog_ids; + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 4, "count"); + ASSERT_EQ(optq.revision, 5, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id2, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id3, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]"); + ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 3); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 3, "count"); + ASSERT_EQ(optq.revision, 6, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], id4, "prog_ids[2]"); + ASSERT_EQ(optq.prog_ids[3], 0, "prog_ids[3]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_EQ(err, -ENOENT, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd4, + ); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_EQ(err, -ERANGE, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd3, + ); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_EQ(err, -ERANGE, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_EQ(err, -ERANGE, "prog_detach"); + assert_mprog_count(target, 3); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 2); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 2, "count"); + ASSERT_EQ(optq.revision, 7, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], id4, "prog_ids[1]"); + ASSERT_EQ(optq.prog_ids[2], 0, "prog_ids[2]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + .relative_fd = fd1, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 1); + + memset(prog_ids, 0, sizeof(prog_ids)); + optq.count = ARRAY_SIZE(prog_ids); + + err = bpf_prog_query_opts(loopback, target, &optq); + if (!ASSERT_OK(err, "prog_query")) + goto cleanup4; + + ASSERT_EQ(optq.count, 1, "count"); + ASSERT_EQ(optq.revision, 8, "revision"); + ASSERT_EQ(optq.prog_ids[0], id1, "prog_ids[0]"); + ASSERT_EQ(optq.prog_ids[1], 0, "prog_ids[1]"); + + LIBBPF_OPTS_RESET(optd, + .flags = BPF_F_AFTER, + ); + + err = bpf_prog_detach_opts(0, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + + assert_mprog_count(target, 0); + goto cleanup; + +cleanup4: + err = bpf_prog_detach_opts(fd4, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 3); + +cleanup3: + err = bpf_prog_detach_opts(fd3, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 2); + +cleanup2: + err = bpf_prog_detach_opts(fd2, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 1); + +cleanup1: + err = bpf_prog_detach_opts(fd1, loopback, target, &optd); + ASSERT_OK(err, "prog_detach"); + assert_mprog_count(target, 0); + +cleanup: + test_tc_link__destroy(skel); +} + +void serial_test_tc_opts_detach_after(void) +{ + test_tc_opts_detach_after_target(BPF_TCX_INGRESS); + test_tc_opts_detach_after_target(BPF_TCX_EGRESS); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c index 13bcaeb028b8..56685fc03c7e 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c @@ -347,7 +347,7 @@ static void syncookie_estab(void) exp_active_estab_in.max_delack_ms = 22; exp_passive_hdr_stg.syncookie = true; - exp_active_hdr_stg.resend_syn = true, + exp_active_hdr_stg.resend_syn = true; prepare_out(); diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 1c75a32186d6..fe0fb0c9849a 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -55,6 +55,25 @@ static void test_fentry(void) ASSERT_EQ(skel->bss->t6, 1, "t6 ret"); + ASSERT_EQ(skel->bss->t7_a, 16, "t7:a"); + ASSERT_EQ(skel->bss->t7_b, 17, "t7:b"); + ASSERT_EQ(skel->bss->t7_c, 18, "t7:c"); + ASSERT_EQ(skel->bss->t7_d, 19, "t7:d"); + ASSERT_EQ(skel->bss->t7_e, 20, "t7:e"); + ASSERT_EQ(skel->bss->t7_f_a, 21, "t7:f.a"); + ASSERT_EQ(skel->bss->t7_f_b, 22, "t7:f.b"); + ASSERT_EQ(skel->bss->t7_ret, 133, "t7 ret"); + + ASSERT_EQ(skel->bss->t8_a, 16, "t8:a"); + ASSERT_EQ(skel->bss->t8_b, 17, "t8:b"); + ASSERT_EQ(skel->bss->t8_c, 18, "t8:c"); + ASSERT_EQ(skel->bss->t8_d, 19, "t8:d"); + ASSERT_EQ(skel->bss->t8_e, 20, "t8:e"); + ASSERT_EQ(skel->bss->t8_f_a, 21, "t8:f.a"); + ASSERT_EQ(skel->bss->t8_f_b, 22, "t8:f.b"); + ASSERT_EQ(skel->bss->t8_g, 23, "t8:g"); + ASSERT_EQ(skel->bss->t8_ret, 156, "t8 ret"); + tracing_struct__detach(skel); destroy_skel: tracing_struct__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index e91d0d1769f1..6cd7349d4a2b 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -88,8 +88,8 @@ void serial_test_trampoline_count(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) goto cleanup; - ASSERT_EQ(opts.retval & 0xffff, 4, "bpf_modify_return_test.result"); - ASSERT_EQ(opts.retval >> 16, 1, "bpf_modify_return_test.side_effect"); + ASSERT_EQ(opts.retval & 0xffff, 33, "bpf_modify_return_test.result"); + ASSERT_EQ(opts.retval >> 16, 2, "bpf_modify_return_test.side_effect"); cleanup: for (; i >= 0; i--) { diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 070a13833c3f..c375e59ff28d 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -58,6 +58,7 @@ #include "verifier_stack_ptr.skel.h" #include "verifier_subprog_precision.skel.h" #include "verifier_subreg.skel.h" +#include "verifier_typedef.skel.h" #include "verifier_uninit.skel.h" #include "verifier_unpriv.skel.h" #include "verifier_unpriv_perf.skel.h" @@ -159,6 +160,7 @@ void test_verifier_spin_lock(void) { RUN(verifier_spin_lock); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } void test_verifier_subreg(void) { RUN(verifier_subreg); } +void test_verifier_typedef(void) { RUN(verifier_typedef); } void test_verifier_uninit(void) { RUN(verifier_uninit); } void test_verifier_unpriv(void) { RUN(verifier_unpriv); } void test_verifier_unpriv_perf(void) { RUN(verifier_unpriv_perf); } diff --git a/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c b/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c new file mode 100644 index 000000000000..1e2e73f3b749 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_endian.h> +#include <bpf/bpf_helpers.h> + +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/in6.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> + +#include <sys/types.h> +#include <sys/socket.h> + +#include "cgroup_tcp_skb.h" + +char _license[] SEC("license") = "GPL"; + +__u16 g_sock_port = 0; +__u32 g_sock_state = 0; +int g_unexpected = 0; +__u32 g_packet_count = 0; + +int needed_tcp_pkt(struct __sk_buff *skb, struct tcphdr *tcph) +{ + struct ipv6hdr ip6h; + + if (skb->protocol != bpf_htons(ETH_P_IPV6)) + return 0; + if (bpf_skb_load_bytes(skb, 0, &ip6h, sizeof(ip6h))) + return 0; + + if (ip6h.nexthdr != IPPROTO_TCP) + return 0; + + if (bpf_skb_load_bytes(skb, sizeof(ip6h), tcph, sizeof(*tcph))) + return 0; + + if (tcph->source != bpf_htons(g_sock_port) && + tcph->dest != bpf_htons(g_sock_port)) + return 0; + + return 1; +} + +/* Run accept() on a socket in the cgroup to receive a new connection. */ +static int egress_accept(struct tcphdr *tcph) +{ + if (g_sock_state == SYN_RECV_SENDING_SYN_ACK) { + if (tcph->fin || !tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_RECV; + return 1; + } + + return 0; +} + +static int ingress_accept(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case INIT: + if (!tcph->syn || tcph->fin || tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_RECV_SENDING_SYN_ACK; + break; + case SYN_RECV: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = ESTABLISHED; + break; + default: + return 0; + } + + return 1; +} + +/* Run connect() on a socket in the cgroup to start a new connection. */ +static int egress_connect(struct tcphdr *tcph) +{ + if (g_sock_state == INIT) { + if (!tcph->syn || tcph->fin || tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_SENT; + return 1; + } + + return 0; +} + +static int ingress_connect(struct tcphdr *tcph) +{ + if (g_sock_state == SYN_SENT) { + if (tcph->fin || !tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = ESTABLISHED; + return 1; + } + + return 0; +} + +/* The connection is closed by the peer outside the cgroup. */ +static int egress_close_remote(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + break; + case CLOSE_WAIT_SENDING_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = CLOSE_WAIT; + break; + case CLOSE_WAIT: + if (!tcph->fin) + g_unexpected++; + else + g_sock_state = LAST_ACK; + break; + default: + return 0; + } + + return 1; +} + +static int ingress_close_remote(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + if (tcph->fin) + g_sock_state = CLOSE_WAIT_SENDING_ACK; + break; + case LAST_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = CLOSED; + break; + default: + return 0; + } + + return 1; +} + +/* The connection is closed by the endpoint inside the cgroup. */ +static int egress_close_local(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + if (tcph->fin) + g_sock_state = FIN_WAIT1; + break; + case TIME_WAIT_SENDING_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = TIME_WAIT; + break; + default: + return 0; + } + + return 1; +} + +static int ingress_close_local(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + break; + case FIN_WAIT1: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = FIN_WAIT2; + break; + case FIN_WAIT2: + if (!tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = TIME_WAIT_SENDING_ACK; + break; + default: + return 0; + } + + return 1; +} + +/* Check the types of outgoing packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/egress") +int server_egress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_accept(&tcph) || egress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/ingress") +int server_ingress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_accept(&tcph) || ingress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/egress") +int server_egress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_accept(&tcph) || egress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/ingress") +int server_ingress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_accept(&tcph) || ingress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/egress") +int client_egress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_connect(&tcph) || egress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/ingress") +int client_ingress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_connect(&tcph) || ingress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/egress") +int client_egress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_connect(&tcph) || egress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/ingress") +int client_ingress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_connect(&tcph) || ingress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} diff --git a/tools/testing/selftests/bpf/progs/fentry_many_args.c b/tools/testing/selftests/bpf/progs/fentry_many_args.c new file mode 100644 index 000000000000..b61bb92fee2c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_many_args.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Tencent */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_result = 0; +SEC("fentry/bpf_testmod_fentry_test7") +int BPF_PROG(test1, __u64 a, void *b, short c, int d, void *e, char f, + int g) +{ + test1_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22; + return 0; +} + +__u64 test2_result = 0; +SEC("fentry/bpf_testmod_fentry_test11") +int BPF_PROG(test2, __u64 a, void *b, short c, int d, void *e, char f, + int g, unsigned int h, long i, __u64 j, unsigned long k) +{ + test2_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26; + return 0; +} + +__u64 test3_result = 0; +SEC("fentry/bpf_testmod_fentry_test11") +int BPF_PROG(test3, __u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, + __u64 g, __u64 h, __u64 i, __u64 j, __u64 k) +{ + test3_result = a == 16 && b == 17 && c == 18 && d == 19 && + e == 20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fexit_many_args.c b/tools/testing/selftests/bpf/progs/fexit_many_args.c new file mode 100644 index 000000000000..53b335c2dafb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fexit_many_args.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Tencent */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_result = 0; +SEC("fexit/bpf_testmod_fentry_test7") +int BPF_PROG(test1, __u64 a, void *b, short c, int d, void *e, char f, + int g, int ret) +{ + test1_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && ret == 133; + return 0; +} + +__u64 test2_result = 0; +SEC("fexit/bpf_testmod_fentry_test11") +int BPF_PROG(test2, __u64 a, void *b, short c, int d, void *e, char f, + int g, unsigned int h, long i, __u64 j, unsigned long k, + int ret) +{ + test2_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26 && ret == 231; + return 0; +} + +__u64 test3_result = 0; +SEC("fexit/bpf_testmod_fentry_test11") +int BPF_PROG(test3, __u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, + __u64 g, __u64 h, __u64 i, __u64 j, __u64 k, __u64 ret) +{ + test3_result = a == 16 && b == 17 && c == 18 && d == 19 && + e == 20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26 && ret == 231; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/htab_mem_bench.c b/tools/testing/selftests/bpf/progs/htab_mem_bench.c new file mode 100644 index 000000000000..b1b721b14d67 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/htab_mem_bench.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include <stdbool.h> +#include <errno.h> +#include <linux/types.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define OP_BATCH 64 + +struct update_ctx { + unsigned int from; + unsigned int step; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, 4); + __uint(map_flags, BPF_F_NO_PREALLOC); +} htab SEC(".maps"); + +char _license[] SEC("license") = "GPL"; + +unsigned char zeroed_value[4096]; +unsigned int nr_thread = 0; +long op_cnt = 0; + +static int write_htab(unsigned int i, struct update_ctx *ctx, unsigned int flags) +{ + bpf_map_update_elem(&htab, &ctx->from, zeroed_value, flags); + ctx->from += ctx->step; + + return 0; +} + +static int overwrite_htab(unsigned int i, struct update_ctx *ctx) +{ + return write_htab(i, ctx, 0); +} + +static int newwrite_htab(unsigned int i, struct update_ctx *ctx) +{ + return write_htab(i, ctx, BPF_NOEXIST); +} + +static int del_htab(unsigned int i, struct update_ctx *ctx) +{ + bpf_map_delete_elem(&htab, &ctx->from); + ctx->from += ctx->step; + + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int overwrite(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id(); + update.step = nr_thread; + bpf_loop(OP_BATCH, overwrite_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int batch_add_batch_del(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id(); + update.step = nr_thread; + bpf_loop(OP_BATCH, overwrite_htab, &update, 0); + + update.from = bpf_get_smp_processor_id(); + bpf_loop(OP_BATCH, del_htab, &update, 0); + + __sync_fetch_and_add(&op_cnt, 2); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int add_only(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id() / 2; + update.step = nr_thread / 2; + bpf_loop(OP_BATCH, newwrite_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getppid") +int del_only(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id() / 2; + update.step = nr_thread / 2; + bpf_loop(OP_BATCH, del_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/linked_list.c b/tools/testing/selftests/bpf/progs/linked_list.c index 57440a554304..84d1777a9e6c 100644 --- a/tools/testing/selftests/bpf/progs/linked_list.c +++ b/tools/testing/selftests/bpf/progs/linked_list.c @@ -96,7 +96,7 @@ static __always_inline int list_push_pop_multiple(struct bpf_spin_lock *lock, struct bpf_list_head *head, bool leave_in_map) { struct bpf_list_node *n; - struct foo *f[8], *pf; + struct foo *f[200], *pf; int i; /* Loop following this check adds nodes 2-at-a-time in order to diff --git a/tools/testing/selftests/bpf/progs/map_percpu_stats.c b/tools/testing/selftests/bpf/progs/map_percpu_stats.c new file mode 100644 index 000000000000..10b2325c1720 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/map_percpu_stats.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +__u32 target_id; + +__s64 bpf_map_sum_elem_count(struct bpf_map *map) __ksym; + +SEC("iter/bpf_map") +int dump_bpf_map(struct bpf_iter__bpf_map *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct bpf_map *map = ctx->map; + + if (map && map->id == target_id) + BPF_SEQ_PRINTF(seq, "%lld", bpf_map_sum_elem_count(map)); + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index db388f593d0a..3325da17ec81 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -103,6 +103,8 @@ struct { __type(value, __u32); } m_hash SEC(".maps"); +__s64 bpf_map_sum_elem_count(struct bpf_map *map) __ksym; + static inline int check_hash(void) { struct bpf_htab *hash = (struct bpf_htab *)&m_hash; @@ -115,6 +117,8 @@ static inline int check_hash(void) VERIFY(hash->elem_size == 64); VERIFY(hash->count.counter == 0); + VERIFY(bpf_map_sum_elem_count(map) == 0); + for (i = 0; i < HALF_ENTRIES; ++i) { const __u32 key = i; const __u32 val = 1; @@ -123,6 +127,7 @@ static inline int check_hash(void) return 0; } VERIFY(hash->count.counter == HALF_ENTRIES); + VERIFY(bpf_map_sum_elem_count(map) == HALF_ENTRIES); return 1; } diff --git a/tools/testing/selftests/bpf/progs/modify_return.c b/tools/testing/selftests/bpf/progs/modify_return.c index 8b7466a15c6b..3376d4849f58 100644 --- a/tools/testing/selftests/bpf/progs/modify_return.c +++ b/tools/testing/selftests/bpf/progs/modify_return.c @@ -47,3 +47,43 @@ int BPF_PROG(fexit_test, int a, __u64 b, int ret) return 0; } + +static int sequence2; + +__u64 fentry_result2 = 0; +SEC("fentry/bpf_modify_return_test2") +int BPF_PROG(fentry_test2, int a, int *b, short c, int d, void *e, char f, + int g) +{ + sequence2++; + fentry_result2 = (sequence2 == 1); + return 0; +} + +__u64 fmod_ret_result2 = 0; +SEC("fmod_ret/bpf_modify_return_test2") +int BPF_PROG(fmod_ret_test2, int a, int *b, short c, int d, void *e, char f, + int g, int ret) +{ + sequence2++; + /* This is the first fmod_ret program, the ret passed should be 0 */ + fmod_ret_result2 = (sequence2 == 2 && ret == 0); + return input_retval; +} + +__u64 fexit_result2 = 0; +SEC("fexit/bpf_modify_return_test2") +int BPF_PROG(fexit_test2, int a, int *b, short c, int d, void *e, char f, + int g, int ret) +{ + sequence2++; + /* If the input_reval is non-zero a successful modification should have + * occurred. + */ + if (input_retval) + fexit_result2 = (sequence2 == 3 && ret == input_retval); + else + fexit_result2 = (sequence2 == 3 && ret == 29); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/nested_trust_failure.c b/tools/testing/selftests/bpf/progs/nested_trust_failure.c index 0d1aa6bbace4..ea39497f11ed 100644 --- a/tools/testing/selftests/bpf/progs/nested_trust_failure.c +++ b/tools/testing/selftests/bpf/progs/nested_trust_failure.c @@ -10,6 +10,13 @@ char _license[] SEC("license") = "GPL"; +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, u64); +} sk_storage_map SEC(".maps"); + /* Prototype for all of the program trace events below: * * TRACE_EVENT(task_newtask, @@ -31,3 +38,12 @@ int BPF_PROG(test_invalid_nested_offset, struct task_struct *task, u64 clone_fla bpf_cpumask_first_zero(&task->cpus_mask); return 0; } + +/* Although R2 is of type sk_buff but sock_common is expected, we will hit untrusted ptr first. */ +SEC("tp_btf/tcp_probe") +__failure __msg("R2 type=untrusted_ptr_ expected=ptr_, trusted_ptr_, rcu_ptr_") +int BPF_PROG(test_invalid_skb_field, struct sock *sk, struct sk_buff *skb) +{ + bpf_sk_storage_get(&sk_storage_map, skb->next, 0, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/nested_trust_success.c b/tools/testing/selftests/bpf/progs/nested_trust_success.c index 886ade4aa99d..833840bffd3b 100644 --- a/tools/testing/selftests/bpf/progs/nested_trust_success.c +++ b/tools/testing/selftests/bpf/progs/nested_trust_success.c @@ -10,6 +10,13 @@ char _license[] SEC("license") = "GPL"; +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, u64); +} sk_storage_map SEC(".maps"); + SEC("tp_btf/task_newtask") __success int BPF_PROG(test_read_cpumask, struct task_struct *task, u64 clone_flags) @@ -17,3 +24,11 @@ int BPF_PROG(test_read_cpumask, struct task_struct *task, u64 clone_flags) bpf_cpumask_test_cpu(0, task->cpus_ptr); return 0; } + +SEC("tp_btf/tcp_probe") +__success +int BPF_PROG(test_skb_field, struct sock *sk, struct sk_buff *skb) +{ + bpf_sk_storage_get(&sk_storage_map, skb->sk, 0, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr.c b/tools/testing/selftests/bpf/progs/refcounted_kptr.c index a3da610b1e6b..c55652fdc63a 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr.c @@ -24,7 +24,7 @@ struct { __uint(type, BPF_MAP_TYPE_ARRAY); __type(key, int); __type(value, struct map_value); - __uint(max_entries, 1); + __uint(max_entries, 2); } stashed_nodes SEC(".maps"); struct node_acquire { @@ -42,6 +42,9 @@ private(A) struct bpf_list_head head __contains(node_data, l); private(B) struct bpf_spin_lock alock; private(B) struct bpf_rb_root aroot __contains(node_acquire, node); +private(C) struct bpf_spin_lock block; +private(C) struct bpf_rb_root broot __contains(node_data, r); + static bool less(struct bpf_rb_node *node_a, const struct bpf_rb_node *node_b) { struct node_data *a; @@ -405,4 +408,93 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } +static long __stash_map_empty_xchg(struct node_data *n, int idx) +{ + struct map_value *mapval = bpf_map_lookup_elem(&stashed_nodes, &idx); + + if (!mapval) { + bpf_obj_drop(n); + return 1; + } + n = bpf_kptr_xchg(&mapval->node, n); + if (n) { + bpf_obj_drop(n); + return 2; + } + return 0; +} + +SEC("tc") +long rbtree_wrong_owner_remove_fail_a1(void *ctx) +{ + struct node_data *n, *m; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return 1; + m = bpf_refcount_acquire(n); + + if (__stash_map_empty_xchg(n, 0)) { + bpf_obj_drop(m); + return 2; + } + + if (__stash_map_empty_xchg(m, 1)) + return 3; + + return 0; +} + +SEC("tc") +long rbtree_wrong_owner_remove_fail_b(void *ctx) +{ + struct map_value *mapval; + struct node_data *n; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_nodes, &idx); + if (!mapval) + return 1; + + n = bpf_kptr_xchg(&mapval->node, NULL); + if (!n) + return 2; + + bpf_spin_lock(&block); + + bpf_rbtree_add(&broot, &n->r, less); + + bpf_spin_unlock(&block); + return 0; +} + +SEC("tc") +long rbtree_wrong_owner_remove_fail_a2(void *ctx) +{ + struct map_value *mapval; + struct bpf_rb_node *res; + struct node_data *m; + int idx = 1; + + mapval = bpf_map_lookup_elem(&stashed_nodes, &idx); + if (!mapval) + return 1; + + m = bpf_kptr_xchg(&mapval->node, NULL); + if (!m) + return 2; + bpf_spin_lock(&lock); + + /* make m non-owning ref */ + bpf_list_push_back(&head, &m->l); + res = bpf_rbtree_remove(&root, &m->r); + + bpf_spin_unlock(&lock); + if (res) { + bpf_obj_drop(container_of(res, struct node_data, r)); + return 3; + } + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_global_map_resize.c b/tools/testing/selftests/bpf/progs/test_global_map_resize.c index 2588f2384246..1fbb73d3e5d5 100644 --- a/tools/testing/selftests/bpf/progs/test_global_map_resize.c +++ b/tools/testing/selftests/bpf/progs/test_global_map_resize.c @@ -29,13 +29,16 @@ int my_int SEC(".data.non_array"); int my_array_first[1] SEC(".data.array_not_last"); int my_int_last SEC(".data.array_not_last"); +int percpu_arr[1] SEC(".data.percpu_arr"); + SEC("tp/syscalls/sys_enter_getpid") int bss_array_sum(void *ctx) { if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; - sum = 0; + /* this will be zero, we just rely on verifier not rejecting this */ + sum = percpu_arr[bpf_get_smp_processor_id()]; for (size_t i = 0; i < bss_array_len; ++i) sum += array[i]; @@ -49,7 +52,8 @@ int data_array_sum(void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; - sum = 0; + /* this will be zero, we just rely on verifier not rejecting this */ + sum = percpu_arr[bpf_get_smp_processor_id()]; for (size_t i = 0; i < data_array_len; ++i) sum += my_array[i]; diff --git a/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c b/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c new file mode 100644 index 000000000000..03a475160abe --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +#define NF_ACCEPT 1 + +SEC("netfilter") +int nf_link_attach_test(struct bpf_nf_ctx *ctx) +{ + return NF_ACCEPT; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c b/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c new file mode 100644 index 000000000000..4bdd65b5aa2d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */ + +#include "vmlinux.h" +#include <bpf/bpf_tracing.h> + +char tp_name[128]; + +SEC("lsm/bpf") +int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size) +{ + switch (cmd) { + case BPF_RAW_TRACEPOINT_OPEN: + bpf_probe_read_user_str(tp_name, sizeof(tp_name) - 1, + (void *)attr->raw_tracepoint.name); + break; + default: + break; + } + return 0; +} + +SEC("raw_tracepoint") +int BPF_PROG(raw_tp_run) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_link.c b/tools/testing/selftests/bpf/progs/test_tc_link.c new file mode 100644 index 000000000000..ed1fd0e9cee9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tc_link.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +char LICENSE[] SEC("license") = "GPL"; + +bool seen_tc1; +bool seen_tc2; +bool seen_tc3; +bool seen_tc4; + +SEC("tc/ingress") +int tc1(struct __sk_buff *skb) +{ + seen_tc1 = true; + return TCX_NEXT; +} + +SEC("tc/egress") +int tc2(struct __sk_buff *skb) +{ + seen_tc2 = true; + return TCX_NEXT; +} + +SEC("tc/egress") +int tc3(struct __sk_buff *skb) +{ + seen_tc3 = true; + return TCX_NEXT; +} + +SEC("tc/egress") +int tc4(struct __sk_buff *skb) +{ + seen_tc4 = true; + return TCX_NEXT; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_struct.c b/tools/testing/selftests/bpf/progs/tracing_struct.c index c435a3a8328a..515daef3c84b 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct.c @@ -18,6 +18,11 @@ struct bpf_testmod_struct_arg_3 { int b[]; }; +struct bpf_testmod_struct_arg_4 { + u64 a; + int b; +}; + long t1_a_a, t1_a_b, t1_b, t1_c, t1_ret, t1_nregs; __u64 t1_reg0, t1_reg1, t1_reg2, t1_reg3; long t2_a, t2_b_a, t2_b_b, t2_c, t2_ret; @@ -25,6 +30,9 @@ long t3_a, t3_b, t3_c_a, t3_c_b, t3_ret; long t4_a_a, t4_b, t4_c, t4_d, t4_e_a, t4_e_b, t4_ret; long t5_ret; int t6; +long t7_a, t7_b, t7_c, t7_d, t7_e, t7_f_a, t7_f_b, t7_ret; +long t8_a, t8_b, t8_c, t8_d, t8_e, t8_f_a, t8_f_b, t8_g, t8_ret; + SEC("fentry/bpf_testmod_test_struct_arg_1") int BPF_PROG2(test_struct_arg_1, struct bpf_testmod_struct_arg_2, a, int, b, int, c) @@ -130,4 +138,50 @@ int BPF_PROG2(test_struct_arg_11, struct bpf_testmod_struct_arg_3 *, a) return 0; } +SEC("fentry/bpf_testmod_test_struct_arg_7") +int BPF_PROG2(test_struct_arg_12, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f) +{ + t7_a = a; + t7_b = (long)b; + t7_c = c; + t7_d = d; + t7_e = (long)e; + t7_f_a = f.a; + t7_f_b = f.b; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_7") +int BPF_PROG2(test_struct_arg_13, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, ret) +{ + t7_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_8") +int BPF_PROG2(test_struct_arg_14, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, g) +{ + t8_a = a; + t8_b = (long)b; + t8_c = c; + t8_d = d; + t8_e = (long)e; + t8_f_a = f.a; + t8_f_b = f.b; + t8_g = g; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_8") +int BPF_PROG2(test_struct_arg_15, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, g, + int, ret) +{ + t8_ret = ret; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_typedef.c b/tools/testing/selftests/bpf/progs/verifier_typedef.c new file mode 100644 index 000000000000..08481cfaac4b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_typedef.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +SEC("fentry/bpf_fentry_test_sinfo") +__description("typedef: resolve") +__success __retval(0) +__naked void resolve_typedef(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 +0); \ + r2 = *(u64 *)(r1 +%[frags_offs]); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(frags_offs, + offsetof(struct skb_shared_info, frags)) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xsk_xdp_progs.c b/tools/testing/selftests/bpf/progs/xsk_xdp_progs.c index a630c95c7471..24369f242853 100644 --- a/tools/testing/selftests/bpf/progs/xsk_xdp_progs.c +++ b/tools/testing/selftests/bpf/progs/xsk_xdp_progs.c @@ -15,12 +15,12 @@ struct { static unsigned int idx; int count = 0; -SEC("xdp") int xsk_def_prog(struct xdp_md *xdp) +SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) { return bpf_redirect_map(&xsk, 0, XDP_DROP); } -SEC("xdp") int xsk_xdp_drop(struct xdp_md *xdp) +SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) { /* Drop every other packet */ if (idx++ % 2) @@ -29,7 +29,7 @@ SEC("xdp") int xsk_xdp_drop(struct xdp_md *xdp) return bpf_redirect_map(&xsk, 0, XDP_DROP); } -SEC("xdp") int xsk_xdp_populate_metadata(struct xdp_md *xdp) +SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) { void *data, *data_meta; struct xdp_info *meta; diff --git a/tools/testing/selftests/bpf/test_xsk.sh b/tools/testing/selftests/bpf/test_xsk.sh index c2ad50f26b63..2aa5a3445056 100755 --- a/tools/testing/selftests/bpf/test_xsk.sh +++ b/tools/testing/selftests/bpf/test_xsk.sh @@ -171,7 +171,10 @@ exec_xskxceiver if [ -z $ETH ]; then cleanup_exit ${VETH0} ${VETH1} +else + cleanup_iface ${ETH} ${MTU} fi + TEST_NAME="XSK_SELFTESTS_${VETH0}_BUSY_POLL" busy_poll=1 @@ -184,6 +187,8 @@ exec_xskxceiver if [ -z $ETH ]; then cleanup_exit ${VETH0} ${VETH1} +else + cleanup_iface ${ETH} ${MTU} fi failures=0 diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 9b070cdf44ac..f83d9f65c65b 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -18,7 +18,7 @@ #define TRACEFS_PIPE "/sys/kernel/tracing/trace_pipe" #define DEBUGFS_PIPE "/sys/kernel/debug/tracing/trace_pipe" -#define MAX_SYMS 300000 +#define MAX_SYMS 400000 static struct ksym syms[MAX_SYMS]; static int sym_cnt; @@ -46,6 +46,9 @@ int load_kallsyms_refresh(void) break; if (!addr) continue; + if (i >= MAX_SYMS) + return -EFBIG; + syms[i].addr = (long) addr; syms[i].name = strdup(func); i++; diff --git a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c index b39665f33524..319337bdcfc8 100644 --- a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c +++ b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c @@ -242,4 +242,5 @@ .result = REJECT, .errstr = "R0 invalid mem access", .errstr_unpriv = "R10 partial copy of pointer", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, diff --git a/tools/testing/selftests/bpf/verifier/ctx_skb.c b/tools/testing/selftests/bpf/verifier/ctx_skb.c index 83cecfbd6739..0b394a7f7a2d 100644 --- a/tools/testing/selftests/bpf/verifier/ctx_skb.c +++ b/tools/testing/selftests/bpf/verifier/ctx_skb.c @@ -1169,6 +1169,7 @@ }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SK_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "pkt_end < pkt taken check", @@ -1190,4 +1191,5 @@ }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SK_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, diff --git a/tools/testing/selftests/bpf/verifier/jmp32.c b/tools/testing/selftests/bpf/verifier/jmp32.c index 1a27a6210554..43776f6f92f4 100644 --- a/tools/testing/selftests/bpf/verifier/jmp32.c +++ b/tools/testing/selftests/bpf/verifier/jmp32.c @@ -290,6 +290,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jgt32: BPF_K", @@ -360,6 +361,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jle32: BPF_K", @@ -430,6 +432,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jlt32: BPF_K", @@ -500,6 +503,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsge32: BPF_K", @@ -570,6 +574,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsgt32: BPF_K", @@ -640,6 +645,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsle32: BPF_K", @@ -710,6 +716,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jslt32: BPF_K", @@ -780,6 +787,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jgt32: range bound deduction, reg op imm", diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index a0cfc06d75bc..d25c3e9605f1 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c @@ -68,6 +68,7 @@ .fixup_map_kptr = { 1 }, .result = REJECT, .errstr = "kptr access cannot have variable offset", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map_kptr: bpf_kptr_xchg non-const var_off", @@ -121,6 +122,7 @@ .fixup_map_kptr = { 1 }, .result = REJECT, .errstr = "kptr access misaligned expected=0 off=7", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map_kptr: reject var_off != 0", diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 99272bb890da..0d84dd1f38b6 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -216,7 +216,7 @@ }, .fixup_map_ringbuf = { 1 }, .prog_type = BPF_PROG_TYPE_XDP, - .flags = BPF_F_TEST_STATE_FREQ, + .flags = BPF_F_TEST_STATE_FREQ | F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, .errstr = "invalid access to memory, mem_size=1 off=42 size=8", .result = REJECT, }, diff --git a/tools/testing/selftests/bpf/xsk.c b/tools/testing/selftests/bpf/xsk.c index 687d83e707f8..d9fb2b730a2c 100644 --- a/tools/testing/selftests/bpf/xsk.c +++ b/tools/testing/selftests/bpf/xsk.c @@ -18,17 +18,19 @@ #include <linux/ethtool.h> #include <linux/filter.h> #include <linux/if_ether.h> +#include <linux/if_link.h> #include <linux/if_packet.h> #include <linux/if_xdp.h> #include <linux/kernel.h> #include <linux/list.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> #include <linux/sockios.h> #include <net/if.h> #include <sys/ioctl.h> #include <sys/mman.h> #include <sys/socket.h> #include <sys/types.h> -#include <linux/if_link.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> @@ -81,6 +83,12 @@ struct xsk_socket { int fd; }; +struct nl_mtu_req { + struct nlmsghdr nh; + struct ifinfomsg msg; + char buf[512]; +}; + int xsk_umem__fd(const struct xsk_umem *umem) { return umem ? umem->fd : -EINVAL; @@ -286,6 +294,132 @@ bool xsk_is_in_mode(u32 ifindex, int mode) return false; } +/* Lifted from netlink.c in tools/lib/bpf */ +static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags) +{ + int len; + + do { + len = recvmsg(sock, mhdr, flags); + } while (len < 0 && (errno == EINTR || errno == EAGAIN)); + + if (len < 0) + return -errno; + return len; +} + +/* Lifted from netlink.c in tools/lib/bpf */ +static int alloc_iov(struct iovec *iov, int len) +{ + void *nbuf; + + nbuf = realloc(iov->iov_base, len); + if (!nbuf) + return -ENOMEM; + + iov->iov_base = nbuf; + iov->iov_len = len; + return 0; +} + +/* Original version lifted from netlink.c in tools/lib/bpf */ +static int netlink_recv(int sock) +{ + struct iovec iov = {}; + struct msghdr mhdr = { + .msg_iov = &iov, + .msg_iovlen = 1, + }; + bool multipart = true; + struct nlmsgerr *err; + struct nlmsghdr *nh; + int len, ret; + + ret = alloc_iov(&iov, 4096); + if (ret) + goto done; + + while (multipart) { + multipart = false; + len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC); + if (len < 0) { + ret = len; + goto done; + } + + if (len > iov.iov_len) { + ret = alloc_iov(&iov, len); + if (ret) + goto done; + } + + len = netlink_recvmsg(sock, &mhdr, 0); + if (len < 0) { + ret = len; + goto done; + } + + if (len == 0) + break; + + for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len); + nh = NLMSG_NEXT(nh, len)) { + if (nh->nlmsg_flags & NLM_F_MULTI) + multipart = true; + switch (nh->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(nh); + if (!err->error) + continue; + ret = err->error; + goto done; + case NLMSG_DONE: + ret = 0; + goto done; + default: + break; + } + } + } + ret = 0; +done: + free(iov.iov_base); + return ret; +} + +int xsk_set_mtu(int ifindex, int mtu) +{ + struct nl_mtu_req req; + struct rtattr *rta; + int fd, ret; + + fd = socket(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE); + if (fd < 0) + return fd; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_type = RTM_NEWLINK; + req.msg.ifi_family = AF_UNSPEC; + req.msg.ifi_index = ifindex; + rta = (struct rtattr *)(((char *)&req) + NLMSG_ALIGN(req.nh.nlmsg_len)); + rta->rta_type = IFLA_MTU; + rta->rta_len = RTA_LENGTH(sizeof(unsigned int)); + req.nh.nlmsg_len = NLMSG_ALIGN(req.nh.nlmsg_len) + RTA_LENGTH(sizeof(mtu)); + memcpy(RTA_DATA(rta), &mtu, sizeof(mtu)); + + ret = send(fd, &req, req.nh.nlmsg_len, 0); + if (ret < 0) { + close(fd); + return errno; + } + + ret = netlink_recv(fd); + close(fd); + return ret; +} + int xsk_attach_xdp_program(struct bpf_program *prog, int ifindex, u32 xdp_flags) { int prog_fd; diff --git a/tools/testing/selftests/bpf/xsk.h b/tools/testing/selftests/bpf/xsk.h index 8da8d557768b..d93200fdaa8d 100644 --- a/tools/testing/selftests/bpf/xsk.h +++ b/tools/testing/selftests/bpf/xsk.h @@ -239,6 +239,8 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr, int xsk_umem__delete(struct xsk_umem *umem); void xsk_socket__delete(struct xsk_socket *xsk); +int xsk_set_mtu(int ifindex, int mtu); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/testing/selftests/bpf/xsk_prereqs.sh b/tools/testing/selftests/bpf/xsk_prereqs.sh index ae697a10a056..29175682c44d 100755 --- a/tools/testing/selftests/bpf/xsk_prereqs.sh +++ b/tools/testing/selftests/bpf/xsk_prereqs.sh @@ -53,6 +53,13 @@ test_exit() exit 1 } +cleanup_iface() +{ + ip link set $1 mtu $2 + ip link set $1 xdp off + ip link set $1 xdpgeneric off +} + clear_configs() { [ $(ip link show $1 &>/dev/null; echo $?;) == 0 ] && diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index 218d7f694e5c..3ff436706640 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -49,8 +49,11 @@ * h. tests for invalid and corner case Tx descriptors so that the correct ones * are discarded and let through, respectively. * i. 2K frame size tests - * - * Total tests: 12 + * j. If multi-buffer is supported, send 9k packets divided into 3 frames + * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame + * using unaligned mode + * l. If multi-buffer is supported, try various nasty combinations of descriptors to + * check if they pass the validation or not * * Flow: * ----- @@ -73,10 +76,10 @@ #include <fcntl.h> #include <errno.h> #include <getopt.h> -#include <asm/barrier.h> #include <linux/if_link.h> #include <linux/if_ether.h> #include <linux/mman.h> +#include <linux/netdev.h> #include <arpa/inet.h> #include <net/if.h> #include <locale.h> @@ -91,7 +94,6 @@ #include <sys/socket.h> #include <sys/time.h> #include <sys/types.h> -#include <time.h> #include <unistd.h> #include "xsk_xdp_progs.skel.h" @@ -253,6 +255,8 @@ static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_i cfg.bind_flags = ifobject->bind_flags; if (shared) cfg.bind_flags |= XDP_SHARED_UMEM; + if (ifobject->pkt_stream && ifobject->mtu > MAX_ETH_PKT_SIZE) + cfg.bind_flags |= XDP_USE_SG; txr = ifobject->tx_on ? &xsk->tx : NULL; rxr = ifobject->rx_on ? &xsk->rx : NULL; @@ -415,6 +419,7 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, test->total_steps = 1; test->nb_sockets = 1; test->fail = false; + test->mtu = MAX_ETH_PKT_SIZE; test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; @@ -468,6 +473,26 @@ static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *x test->xskmap_tx = xskmap_tx; } +static int test_spec_set_mtu(struct test_spec *test, int mtu) +{ + int err; + + if (test->ifobj_rx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); + if (err) + return err; + test->ifobj_rx->mtu = mtu; + } + if (test->ifobj_tx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); + if (err) + return err; + test->ifobj_tx->mtu = mtu; + } + + return 0; +} + static void pkt_stream_reset(struct pkt_stream *pkt_stream) { if (pkt_stream) @@ -533,23 +558,49 @@ static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) return pkt_stream; } +static bool pkt_continues(u32 options) +{ + return options & XDP_PKT_CONTD; +} + static u32 ceil_u32(u32 a, u32 b) { return (a + b - 1) / b; } -static u32 pkt_nb_frags(u32 frame_size, struct pkt *pkt) +static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) { - if (!pkt || !pkt->valid) + u32 nb_frags = 1, next_frag; + + if (!pkt) return 1; - return ceil_u32(pkt->len, frame_size); + + if (!pkt_stream->verbatim) { + if (!pkt->valid || !pkt->len) + return 1; + return ceil_u32(pkt->len, frame_size); + } + + /* Search for the end of the packet in verbatim mode */ + if (!pkt_continues(pkt->options)) + return nb_frags; + + next_frag = pkt_stream->current_pkt_nb; + pkt++; + while (next_frag++ < pkt_stream->nb_pkts) { + nb_frags++; + if (!pkt_continues(pkt->options) || !pkt->valid) + break; + pkt++; + } + return nb_frags; } static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, int offset, u32 len) { pkt->offset = offset; pkt->len = len; - if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom) + if (len > MAX_ETH_JUMBO_SIZE) pkt->valid = false; else pkt->valid = true; @@ -637,6 +688,11 @@ static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) return pkt->offset + umem_alloc_buffer(umem); } +static void pkt_stream_cancel(struct pkt_stream *pkt_stream) +{ + pkt_stream->current_pkt_nb--; +} + static void pkt_generate(struct ifobject *ifobject, u64 addr, u32 len, u32 pkt_nb, u32 bytes_written) { @@ -657,34 +713,59 @@ static void pkt_generate(struct ifobject *ifobject, u64 addr, u32 len, u32 pkt_n write_payload(data, pkt_nb, bytes_written, len); } -static void __pkt_stream_generate_custom(struct ifobject *ifobj, - struct pkt *pkts, u32 nb_pkts) +static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, + u32 nb_frames, bool verbatim) { + u32 i, len = 0, pkt_nb = 0, payload = 0; struct pkt_stream *pkt_stream; - u32 i; - pkt_stream = __pkt_stream_alloc(nb_pkts); + pkt_stream = __pkt_stream_alloc(nb_frames); if (!pkt_stream) exit_with_error(ENOMEM); - for (i = 0; i < nb_pkts; i++) { - struct pkt *pkt = &pkt_stream->pkts[i]; + for (i = 0; i < nb_frames; i++) { + struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; + struct pkt *frame = &frames[i]; - pkt->offset = pkts[i].offset; - pkt->len = pkts[i].len; - pkt->pkt_nb = i; - pkt->valid = pkts[i].valid; - if (pkt->len > pkt_stream->max_pkt_len) + pkt->offset = frame->offset; + if (verbatim) { + *pkt = *frame; + pkt->pkt_nb = payload; + if (!frame->valid || !pkt_continues(frame->options)) + payload++; + } else { + if (frame->valid) + len += frame->len; + if (frame->valid && pkt_continues(frame->options)) + continue; + + pkt->pkt_nb = pkt_nb; + pkt->len = len; + pkt->valid = frame->valid; + pkt->options = 0; + + len = 0; + } + + if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) pkt_stream->max_pkt_len = pkt->len; + pkt_nb++; } - ifobj->pkt_stream = pkt_stream; + pkt_stream->nb_pkts = pkt_nb; + pkt_stream->verbatim = verbatim; + return pkt_stream; } static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) { - __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts); - __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts); + struct pkt_stream *pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); + test->ifobj_tx->pkt_stream = pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); + test->ifobj_rx->pkt_stream = pkt_stream; } static void pkt_print_data(u32 *data, u32 cnt) @@ -765,43 +846,76 @@ static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) return true; } -static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) +static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, + u32 bytes_processed) { - void *data = xsk_umem__get_data(buffer, addr); - u32 seqnum, pkt_data; + u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; + void *data = xsk_umem__get_data(umem->buffer, addr); - if (!pkt) { - ksft_print_msg("[%s] too many packets received\n", __func__); - goto error; + addr -= umem->base_addr; + + if (addr >= umem->num_frames * umem->frame_size || + addr + len > umem->num_frames * umem->frame_size) { + ksft_print_msg("Frag invalid addr: %llx len: %u\n", addr, len); + return false; + } + if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { + ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", addr, len); + return false; } - if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) { - /* Do not try to verify packets that are smaller than minimum size. */ - return true; + pkt_data = data; + if (!bytes_processed) { + pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); + len -= PKT_HDR_SIZE; + } else { + bytes_processed -= PKT_HDR_SIZE; } - if (pkt->len != len) { - ksft_print_msg("[%s] expected length [%d], got length [%d]\n", - __func__, pkt->len, len); + expected_seqnum = bytes_processed / sizeof(*pkt_data); + seqnum = ntohl(*pkt_data) & 0xffff; + pkt_nb = ntohl(*pkt_data) >> 16; + + if (expected_pkt_nb != pkt_nb) { + ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", + __func__, expected_pkt_nb, pkt_nb); + goto error; + } + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); goto error; } - pkt_data = ntohl(*((u32 *)(data + PKT_HDR_SIZE))); - seqnum = pkt_data >> 16; - - if (pkt->pkt_nb != seqnum) { - ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n", - __func__, pkt->pkt_nb, seqnum); + words_to_end = len / sizeof(*pkt_data) - 1; + pkt_data += words_to_end; + seqnum = ntohl(*pkt_data) & 0xffff; + expected_seqnum += words_to_end; + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); goto error; } return true; error: - pkt_dump(data, len, true); + pkt_dump(data, len, !bytes_processed); return false; } +static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) +{ + if (pkt->len != len) { + ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", + __func__, pkt->len, len); + pkt_dump(xsk_umem__get_data(buffer, addr), len, true); + return false; + } + + return true; +} + static void kick_tx(struct xsk_socket_info *xsk) { int ret; @@ -854,8 +968,8 @@ static int receive_pkts(struct test_spec *test, struct pollfd *fds) { struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream; - u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0; struct xsk_socket_info *xsk = test->ifobj_rx->xsk; + u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; struct ifobject *ifobj = test->ifobj_rx; struct xsk_umem_info *umem = xsk->umem; struct pkt *pkt; @@ -868,6 +982,9 @@ static int receive_pkts(struct test_spec *test, struct pollfd *fds) pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); while (pkt) { + u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; + u64 first_addr; + ret = gettimeofday(&tv_now, NULL); if (ret) exit_with_error(errno); @@ -888,7 +1005,6 @@ static int receive_pkts(struct test_spec *test, struct pollfd *fds) ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); return TEST_FAILURE; - } if (!(fds->revents & POLLIN)) @@ -913,27 +1029,59 @@ static int receive_pkts(struct test_spec *test, struct pollfd *fds) } } - for (i = 0; i < rcvd; i++) { + while (frags_processed < rcvd) { const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); u64 addr = desc->addr, orig; orig = xsk_umem__extract_addr(addr); addr = xsk_umem__add_offset_to_addr(addr); - if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) || + if (!pkt) { + ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", + __func__, addr, desc->len); + return TEST_FAILURE; + } + + if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && !is_metadata_correct(pkt, umem->buffer, addr))) return TEST_FAILURE; + if (!nb_frags++) + first_addr = addr; + frags_processed++; + pkt_len += desc->len; if (ifobj->use_fill_ring) *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; + + if (pkt_continues(desc->options)) + continue; + + /* The complete packet has been received */ + if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || + !is_offset_correct(umem, pkt, addr)) + return TEST_FAILURE; + pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); + nb_frags = 0; + pkt_len = 0; + } + + if (nb_frags) { + /* In the middle of a packet. Start over from beginning of packet. */ + idx_rx -= nb_frags; + xsk_ring_cons__cancel(&xsk->rx, nb_frags); + if (ifobj->use_fill_ring) { + idx_fq -= nb_frags; + xsk_ring_prod__cancel(&umem->fq, nb_frags); + } + frags_processed -= nb_frags; } if (ifobj->use_fill_ring) - xsk_ring_prod__submit(&umem->fq, rcvd); + xsk_ring_prod__submit(&umem->fq, frags_processed); if (ifobj->release_rx) - xsk_ring_cons__release(&xsk->rx, rcvd); + xsk_ring_cons__release(&xsk->rx, frags_processed); pthread_mutex_lock(&pacing_mutex); pkts_in_flight -= pkts_sent; @@ -946,13 +1094,14 @@ static int receive_pkts(struct test_spec *test, struct pollfd *fds) static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeout) { + u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct pkt_stream *pkt_stream = ifobject->pkt_stream; struct xsk_socket_info *xsk = ifobject->xsk; struct xsk_umem_info *umem = ifobject->umem; - u32 i, idx = 0, valid_pkts = 0, buffer_len; bool use_poll = ifobject->use_poll; int ret; - buffer_len = pkt_get_buffer_len(umem, ifobject->pkt_stream->max_pkt_len); + buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); /* pkts_in_flight might be negative if many invalid packets are sent */ if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) { kick_tx(xsk); @@ -983,17 +1132,49 @@ static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeo } for (i = 0; i < BATCH_SIZE; i++) { - struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); - struct pkt *pkt = pkt_stream_get_next_tx_pkt(ifobject->pkt_stream); + struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + u32 nb_frags_left, nb_frags, bytes_written = 0; if (!pkt) break; - tx_desc->addr = pkt_get_addr(pkt, umem); - tx_desc->len = pkt->len; - if (pkt->valid) { + nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); + if (nb_frags > BATCH_SIZE - i) { + pkt_stream_cancel(pkt_stream); + xsk_ring_prod__cancel(&xsk->tx, BATCH_SIZE - i); + break; + } + nb_frags_left = nb_frags; + + while (nb_frags_left--) { + struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); + + tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); + if (pkt_stream->verbatim) { + tx_desc->len = pkt->len; + tx_desc->options = pkt->options; + } else if (nb_frags_left) { + tx_desc->len = umem->frame_size; + tx_desc->options = XDP_PKT_CONTD; + } else { + tx_desc->len = pkt->len - bytes_written; + tx_desc->options = 0; + } + if (pkt->valid) + pkt_generate(ifobject, tx_desc->addr, tx_desc->len, pkt->pkt_nb, + bytes_written); + bytes_written += tx_desc->len; + + if (nb_frags_left) { + i++; + if (pkt_stream->verbatim) + pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + } + } + + if (pkt && pkt->valid) { valid_pkts++; - pkt_generate(ifobject, tx_desc->addr, tx_desc->len, pkt->pkt_nb, 0); + valid_frags += nb_frags; } } @@ -1002,7 +1183,7 @@ static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeo pthread_mutex_unlock(&pacing_mutex); xsk_ring_prod__submit(&xsk->tx, i); - xsk->outstanding_tx += valid_pkts; + xsk->outstanding_tx += valid_frags; if (use_poll) { ret = poll(fds, 1, POLL_TMOUT); @@ -1222,7 +1403,7 @@ static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream u64 addr; u32 i; - for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt); i++) { + for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { if (!pkt) { if (!fill_up) break; @@ -1415,6 +1596,25 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i struct ifobject *ifobj2) { pthread_t t0, t1; + int err; + + if (test->mtu > MAX_ETH_PKT_SIZE) { + if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || + (ifobj2 && !ifobj2->multi_buff_zc_supp))) { + ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); + return TEST_SKIP; + } + if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || + (ifobj2 && !ifobj2->multi_buff_supp))) { + ksft_test_result_skip("Multi buffer not supported.\n"); + return TEST_SKIP; + } + } + err = test_spec_set_mtu(test, test->mtu); + if (err) { + ksft_print_msg("Error, could not set mtu.\n"); + exit_with_error(err); + } if (ifobj2) { if (pthread_barrier_init(&barr, NULL, 2)) @@ -1616,6 +1816,16 @@ static int testapp_unaligned(struct test_spec *test) return testapp_validate_traffic(test); } +static int testapp_unaligned_mb(struct test_spec *test) +{ + test_spec_set_name(test, "UNALIGNED_MODE_9K"); + test->mtu = MAX_ETH_JUMBO_SIZE; + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + return testapp_validate_traffic(test); +} + static int testapp_single_pkt(struct test_spec *test) { struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; @@ -1624,6 +1834,55 @@ static int testapp_single_pkt(struct test_spec *test) return testapp_validate_traffic(test); } +static int testapp_multi_buffer(struct test_spec *test) +{ + test_spec_set_name(test, "RUN_TO_COMPLETION_9K_PACKETS"); + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + + return testapp_validate_traffic(test); +} + +static int testapp_invalid_desc_mb(struct test_spec *test) +{ + struct xsk_umem_info *umem = test->ifobj_tx->umem; + u64 umem_size = umem->num_frames * umem->frame_size; + struct pkt pkts[] = { + /* Valid packet for synch to start with */ + {0, MIN_PKT_SIZE, 0, true, 0}, + /* Zero frame len is not legal */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, 0, 0, false, 0}, + /* Invalid address in the second frame */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid len in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid options in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, + /* Transmit 2 frags, receive 3 */ + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, + /* Middle frame crosses chunk boundary with small length */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, + /* Valid packet for synch so that something is received */ + {0, MIN_PKT_SIZE, 0, true, 0}}; + + if (umem->unaligned_mode) { + /* Crossing a chunk boundary allowed */ + pkts[12].valid = true; + pkts[13].valid = true; + } + + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + static int testapp_invalid_desc(struct test_spec *test) { struct xsk_umem_info *umem = test->ifobj_tx->umem; @@ -1690,7 +1949,6 @@ static int testapp_xdp_metadata_count(struct test_spec *test) int count = 0; int key = 0; - test_spec_set_name(test, "XDP_METADATA_COUNT"); test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, skel_tx->progs.xsk_xdp_populate_metadata, skel_rx->maps.xsk, skel_tx->maps.xsk); @@ -1724,6 +1982,48 @@ static int testapp_poll_rxq_tmout(struct test_spec *test) return testapp_validate_traffic_single_thread(test, test->ifobj_rx); } +static int testapp_too_many_frags(struct test_spec *test) +{ + struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {}; + u32 max_frags, i; + + test_spec_set_name(test, "TOO_MANY_FRAGS"); + if (test->mode == TEST_MODE_ZC) + max_frags = test->ifobj_tx->xdp_zc_max_segs; + else + max_frags = XSK_DESC__MAX_SKB_FRAGS; + + test->mtu = MAX_ETH_JUMBO_SIZE; + + /* Valid packet for synch */ + pkts[0].len = MIN_PKT_SIZE; + pkts[0].valid = true; + + /* One valid packet with the max amount of frags */ + for (i = 1; i < max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = true; + } + pkts[max_frags].options = 0; + + /* An invalid packet with the max amount of frags but signals packet + * continues on the last frag + */ + for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = false; + } + + /* Valid packet for synch */ + pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; + pkts[2 * max_frags + 1].valid = true; + + pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); + return testapp_validate_traffic(test); +} + static int xsk_load_xdp_programs(struct ifobject *ifobj) { ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); @@ -1757,6 +2057,7 @@ static bool hugepages_present(void) static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac, thread_func_t func_ptr) { + LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); int err; memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN); @@ -1772,6 +2073,22 @@ static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char * if (hugepages_present()) ifobj->unaligned_supp = true; + + err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); + if (err) { + ksft_print_msg("Error querrying XDP capabilities\n"); + exit_with_error(-err); + } + if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) + ifobj->multi_buff_supp = true; + if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { + if (query_opts.xdp_zc_max_segs > 1) { + ifobj->multi_buff_zc_supp = true; + ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; + } else { + ifobj->xdp_zc_max_segs = 0; + } + } } static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type) @@ -1804,6 +2121,9 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ test_spec_set_name(test, "RUN_TO_COMPLETION"); ret = testapp_validate_traffic(test); break; + case TEST_TYPE_RUN_TO_COMPLETION_MB: + ret = testapp_multi_buffer(test); + break; case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT: test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT"); ret = testapp_single_pkt(test); @@ -1866,9 +2186,22 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ ret = testapp_invalid_desc(test); break; } + case TEST_TYPE_ALIGNED_INV_DESC_MB: + test_spec_set_name(test, "ALIGNED_INV_DESC_MULTI_BUFF"); + ret = testapp_invalid_desc_mb(test); + break; + case TEST_TYPE_UNALIGNED_INV_DESC_MB: + test_spec_set_name(test, "UNALIGNED_INV_DESC_MULTI_BUFF"); + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + ret = testapp_invalid_desc_mb(test); + break; case TEST_TYPE_UNALIGNED: ret = testapp_unaligned(test); break; + case TEST_TYPE_UNALIGNED_MB: + ret = testapp_unaligned_mb(test); + break; case TEST_TYPE_HEADROOM: ret = testapp_headroom(test); break; @@ -1876,8 +2209,17 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ ret = testapp_xdp_drop(test); break; case TEST_TYPE_XDP_METADATA_COUNT: + test_spec_set_name(test, "XDP_METADATA_COUNT"); + ret = testapp_xdp_metadata_count(test); + break; + case TEST_TYPE_XDP_METADATA_COUNT_MB: + test_spec_set_name(test, "XDP_METADATA_COUNT_MULTI_BUFF"); + test->mtu = MAX_ETH_JUMBO_SIZE; ret = testapp_xdp_metadata_count(test); break; + case TEST_TYPE_TOO_MANY_FRAGS: + ret = testapp_too_many_frags(test); + break; default: break; } diff --git a/tools/testing/selftests/bpf/xskxceiver.h b/tools/testing/selftests/bpf/xskxceiver.h index aaf27e067640..233b66cef64a 100644 --- a/tools/testing/selftests/bpf/xskxceiver.h +++ b/tools/testing/selftests/bpf/xskxceiver.h @@ -38,6 +38,8 @@ #define MAX_TEARDOWN_ITER 10 #define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) /* Just to align the data in the packet */ #define MIN_PKT_SIZE 64 +#define MAX_ETH_PKT_SIZE 1518 +#define MAX_ETH_JUMBO_SIZE 9000 #define USLEEP_MAX 10000 #define SOCK_RECONF_CTR 10 #define BATCH_SIZE 64 @@ -47,7 +49,11 @@ #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) #define RX_FULL_RXQSIZE 32 #define UMEM_HEADROOM_TEST_SIZE 128 -#define XSK_UMEM__INVALID_FRAME_SIZE (XSK_UMEM__DEFAULT_FRAME_SIZE + 1) +#define XSK_UMEM__INVALID_FRAME_SIZE (MAX_ETH_JUMBO_SIZE + 1) +#define XSK_UMEM__LARGE_FRAME_SIZE (3 * 1024) +#define XSK_UMEM__MAX_FRAME_SIZE (4 * 1024) +#define XSK_DESC__INVALID_OPTION (0xffff) +#define XSK_DESC__MAX_SKB_FRAGS 18 #define HUGEPAGE_SIZE (2 * 1024 * 1024) #define PKT_DUMP_NB_TO_PRINT 16 @@ -83,6 +89,12 @@ enum test_type { TEST_TYPE_BPF_RES, TEST_TYPE_XDP_DROP_HALF, TEST_TYPE_XDP_METADATA_COUNT, + TEST_TYPE_XDP_METADATA_COUNT_MB, + TEST_TYPE_RUN_TO_COMPLETION_MB, + TEST_TYPE_UNALIGNED_MB, + TEST_TYPE_ALIGNED_INV_DESC_MB, + TEST_TYPE_UNALIGNED_INV_DESC_MB, + TEST_TYPE_TOO_MANY_FRAGS, TEST_TYPE_MAX }; @@ -115,6 +127,7 @@ struct pkt { u32 len; u32 pkt_nb; bool valid; + u16 options; }; struct pkt_stream { @@ -122,6 +135,7 @@ struct pkt_stream { u32 current_pkt_nb; struct pkt *pkts; u32 max_pkt_len; + bool verbatim; }; struct ifobject; @@ -141,7 +155,9 @@ struct ifobject { struct bpf_program *xdp_prog; enum test_mode mode; int ifindex; + int mtu; u32 bind_flags; + u32 xdp_zc_max_segs; bool tx_on; bool rx_on; bool use_poll; @@ -151,6 +167,8 @@ struct ifobject { bool shared_umem; bool use_metadata; bool unaligned_supp; + bool multi_buff_supp; + bool multi_buff_zc_supp; u8 dst_mac[ETH_ALEN]; u8 src_mac[ETH_ALEN]; }; @@ -164,6 +182,7 @@ struct test_spec { struct bpf_program *xdp_prog_tx; struct bpf_map *xskmap_rx; struct bpf_map *xskmap_tx; + int mtu; u16 total_steps; u16 current_step; u16 nb_sockets; diff --git a/tools/testing/selftests/connector/.gitignore b/tools/testing/selftests/connector/.gitignore new file mode 100644 index 000000000000..c90098199a44 --- /dev/null +++ b/tools/testing/selftests/connector/.gitignore @@ -0,0 +1 @@ +proc_filter diff --git a/tools/testing/selftests/connector/Makefile b/tools/testing/selftests/connector/Makefile new file mode 100644 index 000000000000..92188b9bac5c --- /dev/null +++ b/tools/testing/selftests/connector/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +CFLAGS += -Wall $(KHDR_INCLUDES) + +TEST_GEN_PROGS = proc_filter + +include ../lib.mk diff --git a/tools/testing/selftests/connector/proc_filter.c b/tools/testing/selftests/connector/proc_filter.c new file mode 100644 index 000000000000..4a825b997666 --- /dev/null +++ b/tools/testing/selftests/connector/proc_filter.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <sys/types.h> +#include <sys/epoll.h> +#include <sys/socket.h> +#include <linux/netlink.h> +#include <linux/connector.h> +#include <linux/cn_proc.h> + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <strings.h> +#include <errno.h> +#include <signal.h> +#include <string.h> + +#include "../kselftest.h" + +#define NL_MESSAGE_SIZE (sizeof(struct nlmsghdr) + sizeof(struct cn_msg) + \ + sizeof(struct proc_input)) +#define NL_MESSAGE_SIZE_NF (sizeof(struct nlmsghdr) + sizeof(struct cn_msg) + \ + sizeof(int)) + +#define MAX_EVENTS 1 + +volatile static int interrupted; +static int nl_sock, ret_errno, tcount; +static struct epoll_event evn; + +static int filter; + +#ifdef ENABLE_PRINTS +#define Printf printf +#else +#define Printf ksft_print_msg +#endif + +int send_message(void *pinp) +{ + char buff[NL_MESSAGE_SIZE]; + struct nlmsghdr *hdr; + struct cn_msg *msg; + + hdr = (struct nlmsghdr *)buff; + if (filter) + hdr->nlmsg_len = NL_MESSAGE_SIZE; + else + hdr->nlmsg_len = NL_MESSAGE_SIZE_NF; + hdr->nlmsg_type = NLMSG_DONE; + hdr->nlmsg_flags = 0; + hdr->nlmsg_seq = 0; + hdr->nlmsg_pid = getpid(); + + msg = (struct cn_msg *)NLMSG_DATA(hdr); + msg->id.idx = CN_IDX_PROC; + msg->id.val = CN_VAL_PROC; + msg->seq = 0; + msg->ack = 0; + msg->flags = 0; + + if (filter) { + msg->len = sizeof(struct proc_input); + ((struct proc_input *)msg->data)->mcast_op = + ((struct proc_input *)pinp)->mcast_op; + ((struct proc_input *)msg->data)->event_type = + ((struct proc_input *)pinp)->event_type; + } else { + msg->len = sizeof(int); + *(int *)msg->data = *(enum proc_cn_mcast_op *)pinp; + } + + if (send(nl_sock, hdr, hdr->nlmsg_len, 0) == -1) { + ret_errno = errno; + perror("send failed"); + return -3; + } + return 0; +} + +int register_proc_netlink(int *efd, void *input) +{ + struct sockaddr_nl sa_nl; + int err = 0, epoll_fd; + + nl_sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR); + + if (nl_sock == -1) { + ret_errno = errno; + perror("socket failed"); + return -1; + } + + bzero(&sa_nl, sizeof(sa_nl)); + sa_nl.nl_family = AF_NETLINK; + sa_nl.nl_groups = CN_IDX_PROC; + sa_nl.nl_pid = getpid(); + + if (bind(nl_sock, (struct sockaddr *)&sa_nl, sizeof(sa_nl)) == -1) { + ret_errno = errno; + perror("bind failed"); + return -2; + } + + epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if (epoll_fd < 0) { + ret_errno = errno; + perror("epoll_create1 failed"); + return -2; + } + + err = send_message(input); + + if (err < 0) + return err; + + evn.events = EPOLLIN; + evn.data.fd = nl_sock; + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, nl_sock, &evn) < 0) { + ret_errno = errno; + perror("epoll_ctl failed"); + return -3; + } + *efd = epoll_fd; + return 0; +} + +static void sigint(int sig) +{ + interrupted = 1; +} + +int handle_packet(char *buff, int fd, struct proc_event *event) +{ + struct nlmsghdr *hdr; + + hdr = (struct nlmsghdr *)buff; + + if (hdr->nlmsg_type == NLMSG_ERROR) { + perror("NLMSG_ERROR error\n"); + return -3; + } else if (hdr->nlmsg_type == NLMSG_DONE) { + event = (struct proc_event *) + ((struct cn_msg *)NLMSG_DATA(hdr))->data; + tcount++; + switch (event->what) { + case PROC_EVENT_EXIT: + Printf("Exit process %d (tgid %d) with code %d, signal %d\n", + event->event_data.exit.process_pid, + event->event_data.exit.process_tgid, + event->event_data.exit.exit_code, + event->event_data.exit.exit_signal); + break; + case PROC_EVENT_FORK: + Printf("Fork process %d (tgid %d), parent %d (tgid %d)\n", + event->event_data.fork.child_pid, + event->event_data.fork.child_tgid, + event->event_data.fork.parent_pid, + event->event_data.fork.parent_tgid); + break; + case PROC_EVENT_EXEC: + Printf("Exec process %d (tgid %d)\n", + event->event_data.exec.process_pid, + event->event_data.exec.process_tgid); + break; + case PROC_EVENT_UID: + Printf("UID process %d (tgid %d) uid %d euid %d\n", + event->event_data.id.process_pid, + event->event_data.id.process_tgid, + event->event_data.id.r.ruid, + event->event_data.id.e.euid); + break; + case PROC_EVENT_GID: + Printf("GID process %d (tgid %d) gid %d egid %d\n", + event->event_data.id.process_pid, + event->event_data.id.process_tgid, + event->event_data.id.r.rgid, + event->event_data.id.e.egid); + break; + case PROC_EVENT_SID: + Printf("SID process %d (tgid %d)\n", + event->event_data.sid.process_pid, + event->event_data.sid.process_tgid); + break; + case PROC_EVENT_PTRACE: + Printf("Ptrace process %d (tgid %d), Tracer %d (tgid %d)\n", + event->event_data.ptrace.process_pid, + event->event_data.ptrace.process_tgid, + event->event_data.ptrace.tracer_pid, + event->event_data.ptrace.tracer_tgid); + break; + case PROC_EVENT_COMM: + Printf("Comm process %d (tgid %d) comm %s\n", + event->event_data.comm.process_pid, + event->event_data.comm.process_tgid, + event->event_data.comm.comm); + break; + case PROC_EVENT_COREDUMP: + Printf("Coredump process %d (tgid %d) parent %d, (tgid %d)\n", + event->event_data.coredump.process_pid, + event->event_data.coredump.process_tgid, + event->event_data.coredump.parent_pid, + event->event_data.coredump.parent_tgid); + break; + default: + break; + } + } + return 0; +} + +int handle_events(int epoll_fd, struct proc_event *pev) +{ + char buff[CONNECTOR_MAX_MSG_SIZE]; + struct epoll_event ev[MAX_EVENTS]; + int i, event_count = 0, err = 0; + + event_count = epoll_wait(epoll_fd, ev, MAX_EVENTS, -1); + if (event_count < 0) { + ret_errno = errno; + if (ret_errno != EINTR) + perror("epoll_wait failed"); + return -3; + } + for (i = 0; i < event_count; i++) { + if (!(ev[i].events & EPOLLIN)) + continue; + if (recv(ev[i].data.fd, buff, sizeof(buff), 0) == -1) { + ret_errno = errno; + perror("recv failed"); + return -3; + } + err = handle_packet(buff, ev[i].data.fd, pev); + if (err < 0) + return err; + } + return 0; +} + +int main(int argc, char *argv[]) +{ + int epoll_fd, err; + struct proc_event proc_ev; + struct proc_input input; + + signal(SIGINT, sigint); + + if (argc > 2) { + printf("Expected 0(assume no-filter) or 1 argument(-f)\n"); + exit(KSFT_SKIP); + } + + if (argc == 2) { + if (strcmp(argv[1], "-f") == 0) { + filter = 1; + } else { + printf("Valid option : -f (for filter feature)\n"); + exit(KSFT_SKIP); + } + } + + if (filter) { + input.event_type = PROC_EVENT_NONZERO_EXIT; + input.mcast_op = PROC_CN_MCAST_LISTEN; + err = register_proc_netlink(&epoll_fd, (void*)&input); + } else { + enum proc_cn_mcast_op op = PROC_CN_MCAST_LISTEN; + err = register_proc_netlink(&epoll_fd, (void*)&op); + } + + if (err < 0) { + if (err == -2) + close(nl_sock); + if (err == -3) { + close(nl_sock); + close(epoll_fd); + } + exit(1); + } + + while (!interrupted) { + err = handle_events(epoll_fd, &proc_ev); + if (err < 0) { + if (ret_errno == EINTR) + continue; + if (err == -2) + close(nl_sock); + if (err == -3) { + close(nl_sock); + close(epoll_fd); + } + exit(1); + } + } + + if (filter) { + input.mcast_op = PROC_CN_MCAST_IGNORE; + send_message((void*)&input); + } else { + enum proc_cn_mcast_op op = PROC_CN_MCAST_IGNORE; + send_message((void*)&op); + } + + close(epoll_fd); + close(nl_sock); + + printf("Done total count: %d\n", tcount); + exit(0); +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh b/tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh new file mode 100755 index 000000000000..b1f0781f6b25 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test that filters that match on the same port range, but with different +# combination of IPv4/IPv6 and TCP/UDP all use the same port range register by +# observing port range registers' occupancy via devlink-resource. + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + port_range_occ_test +" +NUM_NETIFS=2 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh + +h1_create() +{ + simple_if_init $h1 +} + +h1_destroy() +{ + simple_if_fini $h1 +} + +switch_create() +{ + simple_if_init $swp1 + tc qdisc add dev $swp1 clsact +} + +switch_destroy() +{ + tc qdisc del dev $swp1 clsact + simple_if_fini $swp1 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + vrf_prepare + + h1_create + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + h1_destroy + + vrf_cleanup +} + +port_range_occ_get() +{ + devlink_resource_occ_get port_range_registers +} + +port_range_occ_test() +{ + RET=0 + + local occ=$(port_range_occ_get) + + # Two port range registers are used, for source and destination port + # ranges. + tc filter add dev $swp1 ingress pref 1 handle 101 proto ip \ + flower skip_sw ip_proto udp src_port 1-100 dst_port 1-100 \ + action pass + (( occ + 2 == $(port_range_occ_get) )) + check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))" + + tc filter add dev $swp1 ingress pref 1 handle 102 proto ip \ + flower skip_sw ip_proto tcp src_port 1-100 dst_port 1-100 \ + action pass + tc filter add dev $swp1 ingress pref 2 handle 103 proto ipv6 \ + flower skip_sw ip_proto udp src_port 1-100 dst_port 1-100 \ + action pass + tc filter add dev $swp1 ingress pref 2 handle 104 proto ipv6 \ + flower skip_sw ip_proto tcp src_port 1-100 dst_port 1-100 \ + action pass + (( occ + 2 == $(port_range_occ_get) )) + check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))" + + tc filter del dev $swp1 ingress pref 2 handle 104 flower + tc filter del dev $swp1 ingress pref 2 handle 103 flower + tc filter del dev $swp1 ingress pref 1 handle 102 flower + (( occ + 2 == $(port_range_occ_get) )) + check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))" + + tc filter del dev $swp1 ingress pref 1 handle 101 flower + (( occ == $(port_range_occ_get) )) + check_err $? "Got occupancy $(port_range_occ_get), expected $occ" + + log_test "port range occupancy" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh new file mode 100644 index 000000000000..2a70840ff14b --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: GPL-2.0 + +PORT_RANGE_NUM_NETIFS=2 + +port_range_h1_create() +{ + simple_if_init $h1 +} + +port_range_h1_destroy() +{ + simple_if_fini $h1 +} + +port_range_switch_create() +{ + simple_if_init $swp1 + tc qdisc add dev $swp1 clsact +} + +port_range_switch_destroy() +{ + tc qdisc del dev $swp1 clsact + simple_if_fini $swp1 +} + +port_range_rules_create() +{ + local count=$1; shift + local should_fail=$1; shift + local batch_file="$(mktemp)" + + for ((i = 0; i < count; ++i)); do + cat >> $batch_file <<-EOF + filter add dev $swp1 ingress \ + prot ipv4 \ + pref 1000 \ + flower skip_sw \ + ip_proto udp dst_port 1-$((100 + i)) \ + action pass + EOF + done + + tc -b $batch_file + check_err_fail $should_fail $? "Rule insertion" + + rm -f $batch_file +} + +__port_range_test() +{ + local count=$1; shift + local should_fail=$1; shift + + port_range_rules_create $count $should_fail + + offload_count=$(tc -j filter show dev $swp1 ingress | + jq "[.[] | select(.options.in_hw == true)] | length") + ((offload_count == count)) + check_err_fail $should_fail $? "port range offload count" +} + +port_range_test() +{ + local count=$1; shift + local should_fail=$1; shift + + if ! tc_offload_check $PORT_RANGE_NUM_NETIFS; then + check_err 1 "Could not test offloaded functionality" + return + fi + + __port_range_test $count $should_fail +} + +port_range_setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + vrf_prepare + + port_range_h1_create + port_range_switch_create +} + +port_range_cleanup() +{ + pre_cleanup + + port_range_switch_destroy + port_range_h1_destroy + + vrf_cleanup +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/rif_bridge.sh b/tools/testing/selftests/drivers/net/mlxsw/rif_bridge.sh new file mode 100755 index 000000000000..b79542a4dcc7 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/rif_bridge.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + bridge_rif_add + bridge_rif_nomaster + bridge_rif_remaster + bridge_rif_nomaster_addr + bridge_rif_nomaster_port + bridge_rif_remaster_port +" + +NUM_NETIFS=2 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh + +setup_prepare() +{ + swp1=${NETIFS[p1]} + swp2=${NETIFS[p2]} + + team_create lag1 lacp + ip link set dev lag1 addrgenmode none + ip link set dev lag1 address $(mac_get $swp1) + + team_create lag2 lacp + ip link set dev lag2 addrgenmode none + ip link set dev lag2 address $(mac_get $swp2) + + ip link add name br1 type bridge vlan_filtering 1 + ip link set dev br1 addrgenmode none + ip link set dev br1 address $(mac_get lag1) + ip link set dev br1 up + + ip link set dev lag1 master br1 + + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + + ip link set dev $swp2 master lag2 + ip link set dev $swp2 up +} + +cleanup() +{ + pre_cleanup + + ip link set dev $swp2 nomaster + ip link set dev $swp2 down + + ip link set dev $swp1 nomaster + ip link set dev $swp1 down + + ip link del dev lag2 + ip link set dev lag1 nomaster + ip link del dev lag1 + + ip link del dev br1 +} + +bridge_rif_add() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + __addr_add_del br1 add 192.0.2.2/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIF for bridge on address addition" +} + +bridge_rif_nomaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev lag1 nomaster + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 - 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Drop RIF for bridge on LAG deslavement" +} + +bridge_rif_remaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev lag1 master br1 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIF for bridge on LAG reenslavement" +} + +bridge_rif_nomaster_addr() +{ + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + + # Adding an address while the LAG is enslaved shouldn't generate a RIF. + __addr_add_del lag1 add 192.0.2.65/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "After adding IP: Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + # Removing the LAG from the bridge should drop RIF for the bridge (as + # tested in bridge_rif_lag_nomaster), but since the LAG now has an + # address, it should gain a RIF. + ip link set dev lag1 nomaster + sleep 1 + local rifs_occ_t2=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0)) + + ((expected_rifs == rifs_occ_t2)) + check_err $? "After deslaving: Expected $expected_rifs RIFs, $rifs_occ_t2 are used" + + log_test "Add RIF for LAG on deslavement from bridge" + + __addr_add_del lag1 del 192.0.2.65/28 + ip link set dev lag1 master br1 + sleep 1 +} + +bridge_rif_nomaster_port() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 nomaster + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 - 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Drop RIF for bridge on deslavement of port from LAG" +} + +bridge_rif_remaster_port() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 down + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + setup_wait_dev $swp1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIF for bridge on reenslavement of port to LAG" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/rif_lag.sh b/tools/testing/selftests/drivers/net/mlxsw/rif_lag.sh new file mode 100755 index 000000000000..e28f978104f3 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/rif_lag.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + lag_rif_add + lag_rif_nomaster + lag_rif_remaster + lag_rif_nomaster_addr +" + +NUM_NETIFS=2 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh + +setup_prepare() +{ + swp1=${NETIFS[p1]} + swp2=${NETIFS[p2]} + + team_create lag1 lacp + ip link set dev lag1 addrgenmode none + ip link set dev lag1 address $(mac_get $swp1) + + team_create lag2 lacp + ip link set dev lag2 addrgenmode none + ip link set dev lag2 address $(mac_get $swp2) + + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + + ip link set dev $swp2 master lag2 + ip link set dev $swp2 up +} + +cleanup() +{ + pre_cleanup + + ip link set dev $swp2 nomaster + ip link set dev $swp2 down + + ip link set dev $swp1 nomaster + ip link set dev $swp1 down + + ip link del dev lag2 + ip link del dev lag1 +} + +lag_rif_add() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + __addr_add_del lag1 add 192.0.2.2/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIF for LAG on address addition" +} + +lag_rif_nomaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 nomaster + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 - 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Drop RIF for LAG on port deslavement" +} + +lag_rif_remaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 down + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + setup_wait_dev $swp1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 1)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIF for LAG on port reenslavement" +} + +lag_rif_nomaster_addr() +{ + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + + # Adding an address while the port is LAG'd shouldn't generate a RIF. + __addr_add_del $swp1 add 192.0.2.65/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "After adding IP: Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + # Removing the port from LAG should drop RIF for the LAG (as tested in + # lag_rif_nomaster), but since the port now has an address, it should + # gain a RIF. + ip link set dev $swp1 nomaster + sleep 1 + local rifs_occ_t2=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0)) + + ((expected_rifs == rifs_occ_t2)) + check_err $? "After deslaving: Expected $expected_rifs RIFs, $rifs_occ_t2 are used" + + __addr_add_del $swp1 del 192.0.2.65/28 + log_test "Add RIF for port on deslavement from LAG" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/rif_lag_vlan.sh b/tools/testing/selftests/drivers/net/mlxsw/rif_lag_vlan.sh new file mode 100755 index 000000000000..6318cfa6434c --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/rif_lag_vlan.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + lag_rif_add + lag_rif_nomaster + lag_rif_remaster + lag_rif_nomaster_addr +" + +NUM_NETIFS=2 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh + +setup_prepare() +{ + swp1=${NETIFS[p1]} + swp2=${NETIFS[p2]} + + team_create lag1 lacp + ip link set dev lag1 addrgenmode none + ip link set dev lag1 address $(mac_get $swp1) + + team_create lag2 lacp + ip link set dev lag2 addrgenmode none + ip link set dev lag2 address $(mac_get $swp2) + + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + + ip link set dev $swp2 master lag2 + ip link set dev $swp2 up + + vlan_create lag1 100 + ip link set dev lag1.100 addrgenmode none + + vlan_create lag1 200 + ip link set dev lag1.200 addrgenmode none +} + +cleanup() +{ + pre_cleanup + + ip link del dev lag1.200 + ip link del dev lag1.100 + + ip link set dev $swp2 nomaster + ip link set dev $swp2 down + + ip link set dev $swp1 nomaster + ip link set dev $swp1 down + + ip link del dev lag2 + ip link del dev lag1 +} + +lag_rif_add() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + __addr_add_del lag1.100 add 192.0.2.2/28 + __addr_add_del lag1.200 add 192.0.2.18/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 2)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIFs for LAG VLANs on address addition" +} + +lag_rif_nomaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 nomaster + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 - 2)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Drop RIFs for LAG VLANs on port deslavement" +} + +lag_rif_remaster() +{ + RET=0 + + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + ip link set dev $swp1 down + ip link set dev $swp1 master lag1 + ip link set dev $swp1 up + setup_wait_dev $swp1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 + 2)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + log_test "Add RIFs for LAG VLANs on port reenslavement" +} + +lag_rif_nomaster_addr() +{ + local rifs_occ_t0=$(devlink_resource_occ_get rifs) + + # Adding an address while the port is LAG'd shouldn't generate a RIF. + __addr_add_del $swp1 add 192.0.2.65/28 + sleep 1 + local rifs_occ_t1=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0)) + + ((expected_rifs == rifs_occ_t1)) + check_err $? "After adding IP: Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + # Removing the port from LAG should drop two RIFs for the LAG VLANs (as + # tested in lag_rif_nomaster), but since the port now has an address, it + # should gain a RIF. + ip link set dev $swp1 nomaster + sleep 1 + local rifs_occ_t2=$(devlink_resource_occ_get rifs) + local expected_rifs=$((rifs_occ_t0 - 1)) + + ((expected_rifs == rifs_occ_t2)) + check_err $? "After deslaving: Expected $expected_rifs RIFs, $rifs_occ_t2 are used" + + __addr_add_del $swp1 del 192.0.2.65/28 + log_test "Add RIF for port on deslavement from LAG" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/rtnetlink.sh b/tools/testing/selftests/drivers/net/mlxsw/rtnetlink.sh index 5e89657857c7..893a693ad805 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/rtnetlink.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/rtnetlink.sh @@ -16,7 +16,6 @@ ALL_TESTS=" bridge_deletion_test bridge_vlan_flags_test vlan_1_test - lag_bridge_upper_test duplicate_vlans_test vlan_rif_refcount_test subport_rif_refcount_test @@ -211,33 +210,6 @@ vlan_1_test() ip link del dev $swp1.1 } -lag_bridge_upper_test() -{ - # Test that ports cannot be enslaved to LAG devices that have uppers - # and that failure is handled gracefully. See commit b3529af6bb0d - # ("spectrum: Reference count VLAN entries") for more details - RET=0 - - ip link add name bond1 type bond mode 802.3ad - - ip link add name br0 type bridge vlan_filtering 1 - ip link set dev bond1 master br0 - - ip link set dev $swp1 down - ip link set dev $swp1 master bond1 &> /dev/null - check_fail $? "managed to enslave port to lag when should not" - - # This might generate a trace, if we did not handle the failure - # correctly - ip -6 address add 2001:db8:1::1/64 dev $swp1 - ip -6 address del 2001:db8:1::1/64 dev $swp1 - - log_test "lag with bridge upper" - - ip link del dev br0 - ip link del dev bond1 -} - duplicate_vlans_test() { # Test that on a given port a VLAN is only used once. Either as VLAN @@ -510,9 +482,6 @@ vlan_interface_uppers_test() ip link set dev $swp1 master br0 ip link add link br0 name br0.10 type vlan id 10 - ip link add link br0.10 name macvlan0 \ - type macvlan mode private &> /dev/null - check_fail $? "managed to create a macvlan when should not" ip -6 address add 2001:db8:1::1/64 dev br0.10 ip link add link br0.10 name macvlan0 type macvlan mode private diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh new file mode 120000 index 000000000000..bd670d9dc4e5 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh @@ -0,0 +1 @@ +../spectrum/port_range_scale.sh
\ No newline at end of file diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/resource_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/resource_scale.sh index 688338bbeb97..a88d8a8c85f2 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/resource_scale.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/resource_scale.sh @@ -33,6 +33,7 @@ ALL_TESTS=" port rif_mac_profile rif_counter + port_range " for current_test in ${TESTS:-$ALL_TESTS}; do diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh new file mode 100644 index 000000000000..d0847e8ea270 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +source ../port_range_scale.sh + +port_range_get_target() +{ + local should_fail=$1; shift + local target + + target=$(devlink_resource_size_get port_range_registers) + + if ((! should_fail)); then + echo $target + else + echo $((target + 1)) + fi +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh index 95d9f710a630..f981c957f097 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh @@ -30,6 +30,7 @@ ALL_TESTS=" port rif_mac_profile rif_counter + port_range " for current_test in ${TESTS:-$ALL_TESTS}; do diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 01c0491d64da..2e986cbf1a46 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile @@ -167,7 +167,7 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ OUTPUT=$(HOST_BUILD_DIR)/resolve_btfids/ BPFOBJ=$(HOST_BPFOBJ) # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -196,12 +196,12 @@ CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \ # $3 - CFLAGS define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v3 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2 endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v2 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2 endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 7f3ab2a93ed6..04341e1b38f0 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -3,6 +3,8 @@ CFLAGS = -Wall -Wl,--no-as-needed -O2 -g CFLAGS += -I../../../../usr/include/ $(KHDR_INCLUDES) +# Additional include paths needed by kselftest.h +CFLAGS += -I../ TEST_PROGS := run_netsocktests run_afpackettests test_bpf.sh netdevice.sh \ rtnetlink.sh xfrm_policy.sh test_blackhole_dev.sh @@ -85,6 +87,7 @@ TEST_GEN_FILES += bind_wildcard TEST_PROGS += test_vxlan_mdb.sh TEST_PROGS += test_bridge_neigh_suppress.sh TEST_PROGS += test_vxlan_nolocalbypass.sh +TEST_PROGS += test_bridge_backup_port.sh TEST_FILES := settings @@ -113,7 +116,7 @@ $(MAKE_DIRS): mkdir -p $@ # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -131,7 +134,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) $(OUTPUT)/nat6to4.o: nat6to4.c $(BPFOBJ) | $(MAKE_DIRS) - $(CLANG) -O2 -target bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ + $(CLANG) -O2 --target=bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ $(APIDIR)/linux/bpf.h \ diff --git a/tools/testing/selftests/net/csum.c b/tools/testing/selftests/net/csum.c index 82a1c1839da6..90eb06fefa59 100644 --- a/tools/testing/selftests/net/csum.c +++ b/tools/testing/selftests/net/csum.c @@ -91,6 +91,8 @@ #include <sys/types.h> #include <unistd.h> +#include "kselftest.h" + static bool cfg_bad_csum; static int cfg_family = PF_INET6; static int cfg_num_pkt = 4; @@ -450,7 +452,7 @@ static void send_packet(int fd, const char *buf, int len) iov[2].iov_len = len; msg.msg_iov = iov; - msg.msg_iovlen = sizeof(iov) / sizeof(iov[0]); + msg.msg_iovlen = ARRAY_SIZE(iov); msg.msg_name = &addr; msg.msg_namelen = sizeof(addr); @@ -505,7 +507,7 @@ static void __recv_prepare_packet_filter(int fd, int off_nexthdr, int off_dport) struct sock_fprog prog = {}; prog.filter = filter; - prog.len = sizeof(filter) / sizeof(struct sock_filter); + prog.len = ARRAY_SIZE(filter); if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) error(1, errno, "setsockopt filter"); } diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index 0f5e88c8f4ff..54ec2b7b7b8c 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -29,6 +29,7 @@ IPV4_TESTS=" ipv4_large_res_grp ipv4_compat_mode ipv4_fdb_grp_fcnal + ipv4_mpath_select ipv4_torture ipv4_res_torture " @@ -42,6 +43,7 @@ IPV6_TESTS=" ipv6_large_res_grp ipv6_compat_mode ipv6_fdb_grp_fcnal + ipv6_mpath_select ipv6_torture ipv6_res_torture " @@ -370,6 +372,27 @@ check_large_res_grp() log_test $? 0 "Dump large (x$buckets) nexthop buckets" } +get_route_dev() +{ + local pfx="$1" + local out + + if out=$($IP -j route get "$pfx" | jq -re ".[0].dev"); then + echo "$out" + fi +} + +check_route_dev() +{ + local pfx="$1" + local expected="$2" + local out + + out=$(get_route_dev "$pfx") + + check_output "$out" "$expected" +} + start_ip_monitor() { local mtype=$1 @@ -575,6 +598,112 @@ ipv4_fdb_grp_fcnal() $IP link del dev vx10 } +ipv4_mpath_select() +{ + local rc dev match h addr + + echo + echo "IPv4 multipath selection" + echo "------------------------" + if [ ! -x "$(command -v jq)" ]; then + echo "SKIP: Could not run test; need jq tool" + return $ksft_skip + fi + + # Use status of existing neighbor entry when determining nexthop for + # multipath routes. + local -A gws + gws=([veth1]=172.16.1.2 [veth3]=172.16.2.2) + local -A other_dev + other_dev=([veth1]=veth3 [veth3]=veth1) + + run_cmd "$IP nexthop add id 1 via ${gws["veth1"]} dev veth1" + run_cmd "$IP nexthop add id 2 via ${gws["veth3"]} dev veth3" + run_cmd "$IP nexthop add id 1001 group 1/2" + run_cmd "$IP ro add 172.16.101.0/24 nhid 1001" + rc=0 + for dev in veth1 veth3; do + match=0 + for h in {1..254}; do + addr="172.16.101.$h" + if [ "$(get_route_dev "$addr")" = "$dev" ]; then + match=1 + break + fi + done + if (( match == 0 )); then + echo "SKIP: Did not find a route using device $dev" + return $ksft_skip + fi + run_cmd "$IP neigh add ${gws[$dev]} dev $dev nud failed" + if ! check_route_dev "$addr" "${other_dev[$dev]}"; then + rc=1 + break + fi + run_cmd "$IP neigh del ${gws[$dev]} dev $dev" + done + log_test $rc 0 "Use valid neighbor during multipath selection" + + run_cmd "$IP neigh add 172.16.1.2 dev veth1 nud incomplete" + run_cmd "$IP neigh add 172.16.2.2 dev veth3 nud incomplete" + run_cmd "$IP route get 172.16.101.1" + # if we did not crash, success + log_test $rc 0 "Multipath selection with no valid neighbor" +} + +ipv6_mpath_select() +{ + local rc dev match h addr + + echo + echo "IPv6 multipath selection" + echo "------------------------" + if [ ! -x "$(command -v jq)" ]; then + echo "SKIP: Could not run test; need jq tool" + return $ksft_skip + fi + + # Use status of existing neighbor entry when determining nexthop for + # multipath routes. + local -A gws + gws=([veth1]=2001:db8:91::2 [veth3]=2001:db8:92::2) + local -A other_dev + other_dev=([veth1]=veth3 [veth3]=veth1) + + run_cmd "$IP nexthop add id 1 via ${gws["veth1"]} dev veth1" + run_cmd "$IP nexthop add id 2 via ${gws["veth3"]} dev veth3" + run_cmd "$IP nexthop add id 1001 group 1/2" + run_cmd "$IP ro add 2001:db8:101::/64 nhid 1001" + rc=0 + for dev in veth1 veth3; do + match=0 + for h in {1..65535}; do + addr=$(printf "2001:db8:101::%x" $h) + if [ "$(get_route_dev "$addr")" = "$dev" ]; then + match=1 + break + fi + done + if (( match == 0 )); then + echo "SKIP: Did not find a route using device $dev" + return $ksft_skip + fi + run_cmd "$IP neigh add ${gws[$dev]} dev $dev nud failed" + if ! check_route_dev "$addr" "${other_dev[$dev]}"; then + rc=1 + break + fi + run_cmd "$IP neigh del ${gws[$dev]} dev $dev" + done + log_test $rc 0 "Use valid neighbor during multipath selection" + + run_cmd "$IP neigh add 2001:db8:91::2 dev veth1 nud incomplete" + run_cmd "$IP neigh add 2001:db8:92::2 dev veth3 nud incomplete" + run_cmd "$IP route get 2001:db8:101::1" + # if we did not crash, success + log_test $rc 0 "Multipath selection with no valid neighbor" +} + ################################################################################ # basic operations (add, delete, replace) on nexthops and nexthop groups # diff --git a/tools/testing/selftests/net/forwarding/Makefile b/tools/testing/selftests/net/forwarding/Makefile index 770efbe24f0d..74e754e266c3 100644 --- a/tools/testing/selftests/net/forwarding/Makefile +++ b/tools/testing/selftests/net/forwarding/Makefile @@ -64,7 +64,13 @@ TEST_PROGS = bridge_igmp.sh \ q_in_vni_ipv6.sh \ q_in_vni.sh \ router_bridge.sh \ + router_bridge_1d.sh \ + router_bridge_1d_lag.sh \ + router_bridge_lag.sh \ router_bridge_vlan.sh \ + router_bridge_vlan_upper.sh \ + router_bridge_pvid_vlan_upper.sh \ + router_bridge_vlan_upper_pvid.sh \ router_broadcast.sh \ router_mpath_nh_res.sh \ router_mpath_nh.sh \ @@ -85,6 +91,7 @@ TEST_PROGS = bridge_igmp.sh \ tc_flower.sh \ tc_flower_l2_miss.sh \ tc_flower_cfm.sh \ + tc_flower_port_range.sh \ tc_mpls_l2vpn.sh \ tc_police.sh \ tc_shblocks.sh \ diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index 9ddb68dd6a08..71f7c0c49677 100755 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -1215,6 +1215,15 @@ ping_test() log_test "ping$3" } +ping_test_fails() +{ + RET=0 + + ping_do $1 $2 + check_fail $? + log_test "ping fails$3" +} + ping6_do() { local if_name=$1 @@ -1237,6 +1246,15 @@ ping6_test() log_test "ping6$3" } +ping6_test_fails() +{ + RET=0 + + ping6_do $1 $2 + check_fail $? + log_test "ping6 fails$3" +} + learning_test() { local bridge=$1 diff --git a/tools/testing/selftests/net/forwarding/router_bridge.sh b/tools/testing/selftests/net/forwarding/router_bridge.sh index 8ce0aed54ece..0182eb2abfa6 100755 --- a/tools/testing/selftests/net/forwarding/router_bridge.sh +++ b/tools/testing/selftests/net/forwarding/router_bridge.sh @@ -1,9 +1,39 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +# +------------------------+ +----------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | + $h1 | | + $h2 | +# | | 192.0.2.1/28 | | | 192.0.2.130/28 | +# | | 2001:db8:1::1/64 | | | 2001:db8:2::2/64 | +# | | | | | | +# +----|-------------------+ +--|-------------------+ +# | | +# +----|--------------------------------------------------|-------------------+ +# | SW | | | +# | +--|-----------------------------+ + $swp2 | +# | | + $swp1 BR1 (802.1q) | 192.0.2.129/28 | +# | | 192.0.2.2/28 | 2001:db8:2::1/64 | +# | | 2001:db8:1::1/64 | | +# | | | | +# | +--------------------------------+ | +# +---------------------------------------------------------------------------+ + ALL_TESTS=" ping_ipv4 ping_ipv6 + config_remaster + ping_ipv4 + ping_ipv6 + config_remove_pvid + ping_ipv4_fails + ping_ipv6_fails + config_add_pvid + ping_ipv4 + ping_ipv6 + config_late_pvid + ping_ipv4 + ping_ipv6 " NUM_NETIFS=4 source lib.sh @@ -62,6 +92,42 @@ router_destroy() ip link del dev br1 } +config_remaster() +{ + log_info "Remaster bridge slave" + + ip link set dev $swp1 nomaster + sleep 2 + ip link set dev $swp1 master br1 +} + +config_remove_pvid() +{ + log_info "Remove PVID from the bridge" + + bridge vlan add dev br1 vid 1 self + sleep 2 +} + +config_add_pvid() +{ + log_info "Add PVID to the bridge" + + bridge vlan add dev br1 vid 1 self pvid untagged + sleep 2 +} + +config_late_pvid() +{ + log_info "Add bridge PVID after enslaving port" + + ip link set dev $swp1 nomaster + ip link set dev br1 type bridge vlan_default_pvid 0 + sleep 2 + ip link set dev $swp1 master br1 + ip link set dev br1 type bridge vlan_default_pvid 1 +} + setup_prepare() { h1=${NETIFS[p1]} @@ -104,6 +170,16 @@ ping_ipv6() ping6_test $h1 2001:db8:2::2 } +ping_ipv4_fails() +{ + ping_test_fails $h1 192.0.2.130 +} + +ping_ipv6_fails() +{ + ping6_test_fails $h1 2001:db8:2::2 +} + trap cleanup EXIT setup_prepare diff --git a/tools/testing/selftests/net/forwarding/router_bridge_1d.sh b/tools/testing/selftests/net/forwarding/router_bridge_1d.sh new file mode 100755 index 000000000000..6d51f2ca72a2 --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_1d.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +---------------------------------------------+ +----------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | + $h1.100 + $h1.200 | | + $h2 | +# | | 192.0.2.1/28 | 192.0.2.17/28 | | | 192.0.2.130/28 | +# | | 2001:db8:1::1/64 | 2001:db8:3::1/64 | | | 192.0.2.146/28 | +# | \_________ __________/ | | | 2001:db8:2::2/64 | +# | V | | | 2001:db8:4::2/64 | +# | + $h1 | | | | +# +--------------|------------------------------+ +--|-------------------+ +# | | +# +--------------|----------------------------------------|-------------------+ +# | SW + $swp1 + $swp2 | +# | | 192.0.2.129/28 | +# | | 192.0.2.145/28 | +# | | 2001:db8:2::1/64 | +# | ________^___________________________ 2001:db8:4::1/64 | +# | / \ | +# | +---|------------------------------+ +---|------------------------------+ | +# | | + $swp1.100 BR1 (802.1d) | | + $swp1.200 BR2 (802.1d) | | +# | | 192.0.2.2/28 | | 192.0.2.18/28 | | +# | | 2001:db8:1::2/64 | | 2001:db8:3::2/64 | | +# | | | | | | +# | +----------------------------------+ +----------------------------------+ | +# +---------------------------------------------------------------------------+ + +ALL_TESTS=" + ping_ipv4 + ping_ipv6 + config_remaster + ping_ipv4 + ping_ipv6 +" +NUM_NETIFS=4 +source lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 100 v$h1 192.0.2.1/28 2001:db8:1::1/64 + vlan_create $h1 200 v$h1 192.0.2.17/28 2001:db8:3::1/64 + ip -4 route add 192.0.2.128/28 vrf v$h1 nexthop via 192.0.2.2 + ip -4 route add 192.0.2.144/28 vrf v$h1 nexthop via 192.0.2.18 + ip -6 route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::2 + ip -6 route add 2001:db8:4::/64 vrf v$h1 nexthop via 2001:db8:3::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:4::/64 vrf v$h1 + ip -6 route del 2001:db8:2::/64 vrf v$h1 + ip -4 route del 192.0.2.144/28 vrf v$h1 + ip -4 route del 192.0.2.128/28 vrf v$h1 + vlan_destroy $h1 200 + vlan_destroy $h1 100 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 192.0.2.130/28 2001:db8:2::2/64 \ + 192.0.2.146/28 2001:db8:4::2/64 + ip -4 route add 192.0.2.0/28 vrf v$h2 nexthop via 192.0.2.129 + ip -4 route add 192.0.2.16/28 vrf v$h2 nexthop via 192.0.2.145 + ip -6 route add 2001:db8:1::/64 vrf v$h2 nexthop via 2001:db8:2::1 + ip -6 route add 2001:db8:3::/64 vrf v$h2 nexthop via 2001:db8:4::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:3::/64 vrf v$h2 + ip -6 route del 2001:db8:1::/64 vrf v$h2 + ip -4 route del 192.0.2.16/28 vrf v$h2 + ip -4 route del 192.0.2.0/28 vrf v$h2 + simple_if_fini $h2 192.0.2.130/28 2001:db8:2::2/64 \ + 192.0.2.146/28 2001:db8:4::2/64 +} + +router_create() +{ + ip link set dev $swp1 up + + vlan_create $swp1 100 + ip link add name br1 type bridge vlan_filtering 0 + ip link set dev br1 address $(mac_get $swp1.100) + ip link set dev $swp1.100 master br1 + __addr_add_del br1 add 192.0.2.2/28 2001:db8:1::2/64 + ip link set dev br1 up + + vlan_create $swp1 200 + ip link add name br2 type bridge vlan_filtering 0 + ip link set dev br2 address $(mac_get $swp1.200) + ip link set dev $swp1.200 master br2 + __addr_add_del br2 add 192.0.2.18/28 2001:db8:3::2/64 + ip link set dev br2 up + + ip link set dev $swp2 up + __addr_add_del $swp2 add 192.0.2.129/28 2001:db8:2::1/64 \ + 192.0.2.145/28 2001:db8:4::1/64 +} + +router_destroy() +{ + __addr_add_del $swp2 del 192.0.2.129/28 2001:db8:2::1/64 \ + 192.0.2.145/28 2001:db8:4::1/64 + ip link set dev $swp2 down + + __addr_add_del br2 del 192.0.2.18/28 2001:db8:3::2/64 + ip link set dev $swp1.200 nomaster + ip link del dev br2 + vlan_destroy $swp1 200 + + __addr_add_del br1 del 192.0.2.2/28 2001:db8:1::2/64 + ip link set dev $swp1.100 nomaster + ip link del dev br1 + vlan_destroy $swp1 100 + + ip link set dev $swp1 down +} + +config_remaster() +{ + log_info "Remaster bridge slaves" + + ip link set dev $swp1.100 nomaster + ip link set dev $swp1.200 nomaster + sleep 2 + ip link set dev $swp1.200 master br2 + ip link set dev $swp1.100 master br1 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.130 ": via 100" + ping_test $h1 192.0.2.146 ": via 200" +} + +ping_ipv6() +{ + ping6_test $h1 2001:db8:2::2 ": via 100" + ping6_test $h1 2001:db8:4::2 ": via 200" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/router_bridge_1d_lag.sh b/tools/testing/selftests/net/forwarding/router_bridge_1d_lag.sh new file mode 100755 index 000000000000..e064b946e821 --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_1d_lag.sh @@ -0,0 +1,408 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +--------------------------------------------+ +# | H1 (vrf) | +# | | +# | + LAG1.100 + LAG1.200 | +# | | 192.0.2.1/28 | 192.0.2.17/28 | +# | | 2001:db8:1::1/64 | 2001:db8:3:1/64 | +# | \___________ _______/ | +# | v | +# | + LAG1 (team) | +# | | | +# | ____^____ | +# | / \ | +# | + $h1 + $h4 | +# | | | | +# +----------|-----------|---------------------+ +# | | +# +----------|-----------|---------------------+ +# | SW | | | +# | + $swp1 + $swp4 | +# | \____ ____/ | +# | v | +# | LAG2 (team) + | +# | | | +# | _______^______________ | +# | / \ | +# | +------|------------+ +-------|----------+ | +# | | + LAG2.100 | | + LAG2.200 | | +# | | | | | | +# | | BR1 (802.1d) | | BR2 (802.1d) | | +# | | 192.0.2.2/28 | | 192.0.2.18/28 | | +# | | 2001:db8:1::2/64 | | 2001:db8:3:2/64 | | +# | | | | | | +# | +-------------------+ +------------------+ | +# | | +# | + LAG3.100 + LAG3.200 | +# | | 192.0.2.129/28 | 192.0.2.145/28 | +# | | 2001:db8:2::1/64 | 2001:db8:4::1/64 | +# | | | | +# | \_________ ___________/ | +# | v | +# | + LAG3 (team) | +# | ____|____ | +# | / \ | +# | + $swp2 + $swp3 | +# | | | | +# +-------|---------|--------------------------+ +# | | +# +-------|---------|--------------------------+ +# | | | | +# | + $h2 + $h3 | +# | \____ ___/ | +# | | | +# | + LAG4 (team) | +# | | | +# | __________^__________ | +# | / \ | +# | | | | +# | + LAG4.100 + LAG4.200 | +# | 192.0.2.130/28 192.0.2.146/28 | +# | 2001:db8:2::2/64 2001:db8:4::2/64 | +# | | +# | H2 (vrf) | +# +--------------------------------------------+ + +ALL_TESTS=" + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG2 slaves ) + config_deslave_swp4 + config_wait + ping_ipv4 + ping_ipv6 + config_enslave_swp4 + config_deslave_swp1 + config_wait + ping_ipv4 + ping_ipv6 + config_deslave_swp4 + config_enslave_swp1 + config_enslave_swp4 + config_wait + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG2 itself ) + config_remaster_lag2 + config_wait + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG3 slaves ) + config_deslave_swp2 + config_wait + ping_ipv4 + ping_ipv6 + config_enslave_swp2 + config_deslave_swp3 + config_wait + ping_ipv4 + ping_ipv6 + config_deslave_swp2 + config_enslave_swp3 + config_enslave_swp2 + config_wait + ping_ipv4 + ping_ipv6 +" +NUM_NETIFS=8 +source lib.sh + +h1_create() +{ + team_create lag1 lacp + ip link set dev lag1 addrgenmode none + ip link set dev lag1 address $(mac_get $h1) + ip link set dev $h1 master lag1 + ip link set dev $h4 master lag1 + simple_if_init lag1 + ip link set dev $h1 up + ip link set dev $h4 up + + vlan_create lag1 100 vlag1 192.0.2.1/28 2001:db8:1::1/64 + vlan_create lag1 200 vlag1 192.0.2.17/28 2001:db8:3::1/64 + + ip -4 route add 192.0.2.128/28 vrf vlag1 nexthop via 192.0.2.2 + ip -6 route add 2001:db8:2::/64 vrf vlag1 nexthop via 2001:db8:1::2 + + ip -4 route add 192.0.2.144/28 vrf vlag1 nexthop via 192.0.2.18 + ip -6 route add 2001:db8:4::/64 vrf vlag1 nexthop via 2001:db8:3::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:4::/64 vrf vlag1 + ip -4 route del 192.0.2.144/28 vrf vlag1 + + ip -6 route del 2001:db8:2::/64 vrf vlag1 + ip -4 route del 192.0.2.128/28 vrf vlag1 + + vlan_destroy lag1 200 + vlan_destroy lag1 100 + + ip link set dev $h4 down + ip link set dev $h1 down + simple_if_fini lag1 + ip link set dev $h4 nomaster + ip link set dev $h1 nomaster + team_destroy lag1 +} + +h2_create() +{ + team_create lag4 lacp + ip link set dev lag4 addrgenmode none + ip link set dev lag4 address $(mac_get $h2) + ip link set dev $h2 master lag4 + ip link set dev $h3 master lag4 + simple_if_init lag4 + ip link set dev $h2 up + ip link set dev $h3 up + + vlan_create lag4 100 vlag4 192.0.2.130/28 2001:db8:2::2/64 + vlan_create lag4 200 vlag4 192.0.2.146/28 2001:db8:4::2/64 + + ip -4 route add 192.0.2.0/28 vrf vlag4 nexthop via 192.0.2.129 + ip -6 route add 2001:db8:1::/64 vrf vlag4 nexthop via 2001:db8:2::1 + + ip -4 route add 192.0.2.16/28 vrf vlag4 nexthop via 192.0.2.145 + ip -6 route add 2001:db8:3::/64 vrf vlag4 nexthop via 2001:db8:4::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:3::/64 vrf vlag4 + ip -4 route del 192.0.2.16/28 vrf vlag4 + + ip -6 route del 2001:db8:1::/64 vrf vlag4 + ip -4 route del 192.0.2.0/28 vrf vlag4 + + vlan_destroy lag4 200 + vlan_destroy lag4 100 + + ip link set dev $h3 down + ip link set dev $h2 down + simple_if_fini lag4 + ip link set dev $h3 nomaster + ip link set dev $h2 nomaster + team_destroy lag4 +} + +router_create() +{ + team_create lag2 lacp + ip link set dev lag2 addrgenmode none + ip link set dev lag2 address $(mac_get $swp1) + ip link set dev $swp1 master lag2 + ip link set dev $swp4 master lag2 + + vlan_create lag2 100 + vlan_create lag2 200 + + ip link add name br1 type bridge vlan_filtering 0 + ip link set dev br1 address $(mac_get lag2.100) + ip link set dev lag2.100 master br1 + + ip link add name br2 type bridge vlan_filtering 0 + ip link set dev br2 address $(mac_get lag2.200) + ip link set dev lag2.200 master br2 + + ip link set dev $swp1 up + ip link set dev $swp4 up + ip link set dev br1 up + ip link set dev br2 up + + __addr_add_del br1 add 192.0.2.2/28 2001:db8:1::2/64 + __addr_add_del br2 add 192.0.2.18/28 2001:db8:3::2/64 + + team_create lag3 lacp + ip link set dev lag3 addrgenmode none + ip link set dev lag3 address $(mac_get $swp2) + ip link set dev $swp2 master lag3 + ip link set dev $swp3 master lag3 + ip link set dev $swp2 up + ip link set dev $swp3 up + + vlan_create lag3 100 + vlan_create lag3 200 + + __addr_add_del lag3.100 add 192.0.2.129/28 2001:db8:2::1/64 + __addr_add_del lag3.200 add 192.0.2.145/28 2001:db8:4::1/64 +} + +router_destroy() +{ + __addr_add_del lag3.200 del 192.0.2.145/28 2001:db8:4::1/64 + __addr_add_del lag3.100 del 192.0.2.129/28 2001:db8:2::1/64 + + vlan_destroy lag3 200 + vlan_destroy lag3 100 + + ip link set dev $swp3 down + ip link set dev $swp2 down + ip link set dev $swp3 nomaster + ip link set dev $swp2 nomaster + team_destroy lag3 + + __addr_add_del br2 del 192.0.2.18/28 2001:db8:3::2/64 + __addr_add_del br1 del 192.0.2.2/28 2001:db8:1::2/64 + + ip link set dev br2 down + ip link set dev br1 down + ip link set dev $swp4 down + ip link set dev $swp1 down + + ip link set dev lag2.200 nomaster + ip link del dev br2 + + ip link set dev lag2.100 nomaster + ip link del dev br1 + + vlan_destroy lag2 200 + vlan_destroy lag2 100 + + ip link set dev $swp4 nomaster + ip link set dev $swp1 nomaster + team_destroy lag2 +} + +config_remaster_lag2() +{ + log_info "Remaster bridge slaves" + + ip link set dev lag2.200 nomaster + ip link set dev lag2.100 nomaster + sleep 2 + ip link set dev lag2.100 master br1 + ip link set dev lag2.200 master br2 +} + +config_deslave() +{ + local netdev=$1; shift + + log_info "Deslave $netdev" + ip link set dev $netdev down + ip link set dev $netdev nomaster + ip link set dev $netdev up +} + +config_deslave_swp1() +{ + config_deslave $swp1 +} + +config_deslave_swp2() +{ + config_deslave $swp2 +} + +config_deslave_swp3() +{ + config_deslave $swp3 +} + +config_deslave_swp4() +{ + config_deslave $swp4 +} + +config_enslave() +{ + local netdev=$1; shift + local master=$1; shift + + log_info "Enslave $netdev to $master" + ip link set dev $netdev down + ip link set dev $netdev master $master + ip link set dev $netdev up +} + +config_enslave_swp1() +{ + config_enslave $swp1 lag2 +} + +config_enslave_swp2() +{ + config_enslave $swp2 lag3 +} + +config_enslave_swp3() +{ + config_enslave $swp3 lag3 +} + +config_enslave_swp4() +{ + config_enslave $swp4 lag2 +} + +config_wait() +{ + setup_wait_dev lag2 + setup_wait_dev lag3 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + h3=${NETIFS[p6]} + + h4=${NETIFS[p7]} + swp4=${NETIFS[p8]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test lag1.100 192.0.2.130 ": via 100" + ping_test lag1.200 192.0.2.146 ": via 200" +} + +ping_ipv6() +{ + ping6_test lag1.100 2001:db8:2::2 ": via 100" + ping6_test lag1.200 2001:db8:4::2 ": via 200" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/router_bridge_lag.sh b/tools/testing/selftests/net/forwarding/router_bridge_lag.sh new file mode 100755 index 000000000000..f05ffe213c46 --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_lag.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +----------------------------+ +--------------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | | | | +# | + LAG1 (team) | | + LAG4 (team) | +# | | 192.0.2.1/28 | | | 192.0.2.130/28 | +# | | 2001:db8:1::1/64 | | | 2001:db8:2::2/64 | +# | __^___ | | __^_____ | +# | / \ | | / \ | +# | + $h1 + $h4 | | + $h2 + $h3 | +# | | | | | | | | +# +----|--------|--------------+ +-|----------|-------------+ +# | | | | +# +----|--------|------------------------------------|----------|-------------+ +# | SW | | | | | +# | + $swp1 + $swp4 + $swp2 + $swp3 | +# | \__ ___/ \__ _____/ | +# | v v | +# | +------|-------------------------------+ | | +# | | + LAG2 BR1 (802.1q) | + LAG3 (team) | +# | | (team) 192.0.2.2/28 | 192.0.2.129/28 | +# | | 2001:db8:1::2/64 | 2001:db8:2::1/64 | +# | | | | +# | +--------------------------------------+ | +# +---------------------------------------------------------------------------+ + +: ${ALL_TESTS:=" + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG2 slaves ) + config_deslave_swp4 + config_wait + ping_ipv4 + ping_ipv6 + config_enslave_swp4 + config_deslave_swp1 + config_wait + ping_ipv4 + ping_ipv6 + config_deslave_swp4 + config_enslave_swp1 + config_enslave_swp4 + config_wait + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG2 itself ) + config_remaster_lag2 + config_wait + ping_ipv4 + ping_ipv6 + + $(: exercise remastering of LAG3 slaves ) + config_deslave_swp2 + config_wait + ping_ipv4 + ping_ipv6 + config_enslave_swp2 + config_deslave_swp3 + config_wait + ping_ipv4 + ping_ipv6 + config_deslave_swp2 + config_enslave_swp3 + config_enslave_swp2 + config_wait + ping_ipv4 + ping_ipv6 + + $(: move LAG3 to a bridge and then out ) + config_remaster_lag3 + config_wait + ping_ipv4 + ping_ipv6 + "} +NUM_NETIFS=8 +: ${lib_dir:=.} +source $lib_dir/lib.sh +$EXTRA_SOURCE + +h1_create() +{ + team_create lag1 lacp + ip link set dev lag1 address $(mac_get $h1) + ip link set dev $h1 master lag1 + ip link set dev $h4 master lag1 + simple_if_init lag1 192.0.2.1/28 2001:db8:1::1/64 + ip link set dev $h1 up + ip link set dev $h4 up + ip -4 route add 192.0.2.128/28 vrf vlag1 nexthop via 192.0.2.2 + ip -6 route add 2001:db8:2::/64 vrf vlag1 nexthop via 2001:db8:1::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:2::/64 vrf vlag1 + ip -4 route del 192.0.2.128/28 vrf vlag1 + ip link set dev $h4 down + ip link set dev $h1 down + simple_if_fini lag1 192.0.2.1/28 2001:db8:1::1/64 + ip link set dev $h4 nomaster + ip link set dev $h1 nomaster + team_destroy lag1 +} + +h2_create() +{ + team_create lag4 lacp + ip link set dev lag4 address $(mac_get $h2) + ip link set dev $h2 master lag4 + ip link set dev $h3 master lag4 + simple_if_init lag4 192.0.2.130/28 2001:db8:2::2/64 + ip link set dev $h2 up + ip link set dev $h3 up + ip -4 route add 192.0.2.0/28 vrf vlag4 nexthop via 192.0.2.129 + ip -6 route add 2001:db8:1::/64 vrf vlag4 nexthop via 2001:db8:2::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:1::/64 vrf vlag4 + ip -4 route del 192.0.2.0/28 vrf vlag4 + ip link set dev $h3 down + ip link set dev $h2 down + simple_if_fini lag4 192.0.2.130/28 2001:db8:2::2/64 + ip link set dev $h3 nomaster + ip link set dev $h2 nomaster + team_destroy lag4 +} + +router_create() +{ + team_create lag2 lacp + ip link set dev lag2 address $(mac_get $swp1) + ip link set dev $swp1 master lag2 + ip link set dev $swp4 master lag2 + + ip link add name br1 address $(mac_get lag2) \ + type bridge vlan_filtering 1 + ip link set dev lag2 master br1 + + ip link set dev $swp1 up + ip link set dev $swp4 up + ip link set dev br1 up + + __addr_add_del br1 add 192.0.2.2/28 2001:db8:1::2/64 + + team_create lag3 lacp + ip link set dev lag3 address $(mac_get $swp2) + ip link set dev $swp2 master lag3 + ip link set dev $swp3 master lag3 + ip link set dev $swp2 up + ip link set dev $swp3 up + __addr_add_del lag3 add 192.0.2.129/28 2001:db8:2::1/64 +} + +router_destroy() +{ + __addr_add_del lag3 del 192.0.2.129/28 2001:db8:2::1/64 + ip link set dev $swp3 down + ip link set dev $swp2 down + ip link set dev $swp3 nomaster + ip link set dev $swp2 nomaster + team_destroy lag3 + + __addr_add_del br1 del 192.0.2.2/28 2001:db8:1::2/64 + + ip link set dev $swp4 down + ip link set dev $swp1 down + ip link set dev br1 down + + ip link set dev lag2 nomaster + ip link del dev br1 + + ip link set dev $swp4 nomaster + ip link set dev $swp1 nomaster + team_destroy lag2 +} + +config_remaster_lag2() +{ + log_info "Remaster bridge slave" + + ip link set dev lag2 nomaster + sleep 2 + ip link set dev lag2 master br1 +} + +config_remaster_lag3() +{ + log_info "Move lag3 to the bridge, then out again" + + ip link set dev lag3 master br1 + sleep 2 + ip link set dev lag3 nomaster +} + +config_deslave() +{ + local netdev=$1; shift + + log_info "Deslave $netdev" + ip link set dev $netdev down + ip link set dev $netdev nomaster + ip link set dev $netdev up +} + +config_deslave_swp1() +{ + config_deslave $swp1 +} + +config_deslave_swp2() +{ + config_deslave $swp2 +} + +config_deslave_swp3() +{ + config_deslave $swp3 +} + +config_deslave_swp4() +{ + config_deslave $swp4 +} + +config_enslave() +{ + local netdev=$1; shift + local master=$1; shift + + log_info "Enslave $netdev to $master" + ip link set dev $netdev down + ip link set dev $netdev master $master + ip link set dev $netdev up +} + +config_enslave_swp1() +{ + config_enslave $swp1 lag2 +} + +config_enslave_swp2() +{ + config_enslave $swp2 lag3 +} + +config_enslave_swp3() +{ + config_enslave $swp3 lag3 +} + +config_enslave_swp4() +{ + config_enslave $swp4 lag2 +} + +config_wait() +{ + setup_wait_dev lag2 + setup_wait_dev lag3 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + h3=${NETIFS[p6]} + + h4=${NETIFS[p7]} + swp4=${NETIFS[p8]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test lag1 192.0.2.130 +} + +ping_ipv6() +{ + ping6_test lag1 2001:db8:2::2 +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/router_bridge_pvid_vlan_upper.sh b/tools/testing/selftests/net/forwarding/router_bridge_pvid_vlan_upper.sh new file mode 100755 index 000000000000..76e4941fef73 --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_pvid_vlan_upper.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +----------------------------+ +# | H1 (vrf) | +# | + $h1.10 | +----------------------+ +# | | 192.0.2.1/28 | | H2 (vrf) | +# | | 2001:db8:1::1/64 | | + $h2 | +# | | | | | 192.0.2.130/28 | +# | + $h1 | | | 2001:db8:2::2/64 | +# +---|------------------------+ +--|-------------------+ +# | | +# +---|--------------------------------------------------|-------------------+ +# | | router (main VRF) | | +# | +-|----------------------------------+ + $swp2 | +# | | + $swp1 BR1 (802.1q, pvid=10) | 192.0.2.129/28 | +# | | 192.0.2.2/28 | 2001:db8:2::1/64 | +# | | 2001:db8:1::2/64 | | +# | +------------------------------------+ | +# +--------------------------------------------------------------------------+ + +ALL_TESTS=" + ping_ipv4 + ping_ipv6 + shuffle_pvid + ping_ipv4 + ping_ipv6 +" +NUM_NETIFS=4 +source lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 10 v$h1 192.0.2.1/28 2001:db8:1::1/64 + ip -4 route add 192.0.2.128/28 vrf v$h1 nexthop via 192.0.2.2 + ip -6 route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:2::/64 vrf v$h1 + ip -4 route del 192.0.2.128/28 vrf v$h1 + vlan_destroy $h1 10 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 192.0.2.130/28 2001:db8:2::2/64 + ip -4 route add 192.0.2.0/28 vrf v$h2 nexthop via 192.0.2.129 + ip -6 route add 2001:db8:1::/64 vrf v$h2 nexthop via 2001:db8:2::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:1::/64 vrf v$h2 + ip -4 route del 192.0.2.0/28 vrf v$h2 + simple_if_fini $h2 192.0.2.130/28 2001:db8:2::2/64 +} + +router_create() +{ + ip link add name br1 address $(mac_get $swp1) \ + type bridge vlan_filtering 1 vlan_default_pvid 0 + ip link set dev br1 up + __addr_add_del br1 add 192.0.2.2/28 2001:db8:1::2/64 + + ip link set dev $swp1 master br1 + ip link set dev $swp1 up + + ip link set dev $swp2 up + __addr_add_del $swp2 add 192.0.2.129/28 2001:db8:2::1/64 + + bridge vlan add dev br1 vid 10 pvid untagged self + bridge vlan add dev $swp1 vid 10 +} + +router_destroy() +{ + bridge vlan del dev $swp1 vid 10 + bridge vlan del dev br1 vid 10 self + + __addr_add_del $swp2 del 192.0.2.129/28 2001:db8:2::1/64 + ip link set dev $swp2 down + + ip link set dev $swp1 down + ip link set dev $swp1 nomaster + + __addr_add_del br1 del 192.0.2.2/28 2001:db8:1::2/64 + ip link del dev br1 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +shuffle_pvid() +{ + log_info "Add and remove VLAN upper for PVID VLAN" + + # Adding and removing a VLAN upper for the PVID VLAN shouldn't change + # anything. The address is arbitrary, just to make sure it will be an L3 + # netdevice. + vlan_create br1 10 "" 192.0.2.33/28 + sleep 1 + vlan_destroy br1 10 +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.130 +} + +ping_ipv6() +{ + ping6_test $h1 2001:db8:2::2 +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/router_bridge_vlan.sh b/tools/testing/selftests/net/forwarding/router_bridge_vlan.sh index de2b2d5480dd..b76a4a707a5b 100755 --- a/tools/testing/selftests/net/forwarding/router_bridge_vlan.sh +++ b/tools/testing/selftests/net/forwarding/router_bridge_vlan.sh @@ -1,25 +1,28 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 -# +------------------------+ +----------------------+ -# | H1 (vrf) | | H2 (vrf) | -# | + $h1.555 | | + $h2 | -# | | 192.0.2.1/28 | | | 192.0.2.130/28 | -# | | 2001:db8:1::1/64 | | | 2001:db8:2::2/64 | -# | | | | | | -# | + $h1 | | | | -# +----|-------------------+ +--|-------------------+ +# +------------------------------------------------+ +----------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | + $h1.555 + $h1.777 | | + $h2 | +# | | 192.0.2.1/28 | 192.0.2.17/28 | | | 192.0.2.130/28 | +# | | 2001:db8:1::1/64 | 2001:db8:3::1/64 | | | 192.0.2.146/28 | +# | | .-----------------' | | | 2001:db8:2::2/64 | +# | |/ | | | 2001:db8:4::2/64 | +# | + $h1 | | | | +# +----|-------------------------------------------+ +--|-------------------+ # | | # +----|--------------------------------------------------|-------------------+ # | SW | | | # | +--|-------------------------------+ + $swp2 | # | | + $swp1 | 192.0.2.129/28 | -# | | vid 555 | 2001:db8:2::1/64 | -# | | | | -# | | + BR1 (802.1q) | | +# | | vid 555 777 | 192.0.2.145/28 | +# | | | 2001:db8:2::1/64 | +# | | + BR1 (802.1q) | 2001:db8:4::1/64 | # | | vid 555 pvid untagged | | # | | 192.0.2.2/28 | | +# | | 192.0.2.18/28 | | # | | 2001:db8:1::2/64 | | +# | | 2001:db8:3::2/64 | | # | +----------------------------------+ | # +---------------------------------------------------------------------------+ @@ -27,6 +30,14 @@ ALL_TESTS=" ping_ipv4 ping_ipv6 vlan + config_777 + ping_ipv4_fails + ping_ipv6_fails + ping_ipv4_777 + ping_ipv6_777 + config_555 + ping_ipv4 + ping_ipv6 " NUM_NETIFS=4 source lib.sh @@ -34,31 +45,47 @@ source lib.sh h1_create() { simple_if_init $h1 + vlan_create $h1 555 v$h1 192.0.2.1/28 2001:db8:1::1/64 ip -4 route add 192.0.2.128/28 vrf v$h1 nexthop via 192.0.2.2 ip -6 route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::2 + + vlan_create $h1 777 v$h1 192.0.2.17/28 2001:db8:3::1/64 + ip -4 route add 192.0.2.144/28 vrf v$h1 nexthop via 192.0.2.18 + ip -6 route add 2001:db8:4::/64 vrf v$h1 nexthop via 2001:db8:3::2 } h1_destroy() { + ip -6 route del 2001:db8:4::/64 vrf v$h1 + ip -4 route del 192.0.2.144/28 vrf v$h1 + vlan_destroy $h1 777 + ip -6 route del 2001:db8:2::/64 vrf v$h1 ip -4 route del 192.0.2.128/28 vrf v$h1 vlan_destroy $h1 555 + simple_if_fini $h1 } h2_create() { - simple_if_init $h2 192.0.2.130/28 2001:db8:2::2/64 + simple_if_init $h2 192.0.2.130/28 2001:db8:2::2/64 \ + 192.0.2.146/28 2001:db8:4::2/64 ip -4 route add 192.0.2.0/28 vrf v$h2 nexthop via 192.0.2.129 + ip -4 route add 192.0.2.16/28 vrf v$h2 nexthop via 192.0.2.145 ip -6 route add 2001:db8:1::/64 vrf v$h2 nexthop via 2001:db8:2::1 + ip -6 route add 2001:db8:3::/64 vrf v$h2 nexthop via 2001:db8:4::1 } h2_destroy() { + ip -6 route del 2001:db8:3::/64 vrf v$h2 ip -6 route del 2001:db8:1::/64 vrf v$h2 + ip -4 route del 192.0.2.16/28 vrf v$h2 ip -4 route del 192.0.2.0/28 vrf v$h2 - simple_if_fini $h2 192.0.2.130/28 2001:db8:2::2/64 + simple_if_fini $h2 192.0.2.146/28 2001:db8:4::2/64 \ + 192.0.2.130/28 2001:db8:2::2/64 } router_create() @@ -71,18 +98,23 @@ router_create() bridge vlan add dev br1 vid 555 self pvid untagged bridge vlan add dev $swp1 vid 555 + bridge vlan add dev $swp1 vid 777 __addr_add_del br1 add 192.0.2.2/28 2001:db8:1::2/64 + __addr_add_del br1 add 192.0.2.18/28 2001:db8:3::2/64 ip link set dev $swp2 up __addr_add_del $swp2 add 192.0.2.129/28 2001:db8:2::1/64 + __addr_add_del $swp2 add 192.0.2.145/28 2001:db8:4::1/64 } router_destroy() { + __addr_add_del $swp2 del 192.0.2.145/28 2001:db8:4::1/64 __addr_add_del $swp2 del 192.0.2.129/28 2001:db8:2::1/64 ip link set dev $swp2 down + __addr_add_del br1 del 192.0.2.18/28 2001:db8:3::2/64 __addr_add_del br1 del 192.0.2.2/28 2001:db8:1::2/64 ip link set dev $swp1 down ip link set dev $swp1 nomaster @@ -108,6 +140,24 @@ setup_prepare() forwarding_enable } +config_555() +{ + log_info "Configure VLAN 555 as PVID" + + bridge vlan add dev br1 vid 555 self pvid untagged + bridge vlan del dev br1 vid 777 self + sleep 2 +} + +config_777() +{ + log_info "Configure VLAN 777 as PVID" + + bridge vlan add dev br1 vid 777 self pvid untagged + bridge vlan del dev br1 vid 555 self + sleep 2 +} + cleanup() { pre_cleanup @@ -136,12 +186,32 @@ vlan() ping_ipv4() { - ping_test $h1 192.0.2.130 + ping_test $h1.555 192.0.2.130 } ping_ipv6() { - ping6_test $h1 2001:db8:2::2 + ping6_test $h1.555 2001:db8:2::2 +} + +ping_ipv4_fails() +{ + ping_test_fails $h1.555 192.0.2.130 ": via 555" +} + +ping_ipv6_fails() +{ + ping6_test_fails $h1.555 2001:db8:2::2 ": via 555" +} + +ping_ipv4_777() +{ + ping_test $h1.777 192.0.2.146 ": via 777" +} + +ping_ipv6_777() +{ + ping6_test $h1.777 2001:db8:4::2 ": via 777" } trap cleanup EXIT diff --git a/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper.sh b/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper.sh new file mode 100755 index 000000000000..215309ea1c8c --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +------------------------+ +----------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | + $h1.555 | | + $h2.777 | +# | | 192.0.2.1/28 | | | 192.0.2.18/28 | +# | | 2001:db8:1::1/64 | | | 2001:db8:2::2/64 | +# | | | | | | +# | + $h1 | | + $h2 | +# +----|-------------------+ +--|-------------------+ +# | | +# +----|--------------------------------------------------|-------------------+ +# | SW | | | +# | +--|--------------------------------------------------|-----------------+ | +# | | + $swp1 BR1 (802.1q) + $swp2 | | +# | | | | +# | +------+------------------------------------------+---------------------+ | +# | | | | +# | + br1.555 + br1.777 | +# | 192.0.2.2/28 192.0.2.17/28 | +# | 2001:db8:1::2/64 2001:db8:2::1/64 | +# +---------------------------------------------------------------------------+ + +ALL_TESTS=" + ping_ipv4 + ping_ipv6 + respin_config + ping_ipv4 + ping_ipv6 +" +NUM_NETIFS=4 +source lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 555 v$h1 192.0.2.1/28 2001:db8:1::1/64 + ip -4 route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 + ip -6 route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:2::/64 vrf v$h1 + ip -4 route del 192.0.2.16/28 vrf v$h1 + vlan_destroy $h1 555 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + vlan_create $h2 777 v$h2 192.0.2.18/28 2001:db8:2::2/64 + ip -4 route add 192.0.2.0/28 vrf v$h2 nexthop via 192.0.2.17 + ip -6 route add 2001:db8:1::/64 vrf v$h2 nexthop via 2001:db8:2::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:1::/64 vrf v$h2 + ip -4 route del 192.0.2.0/28 vrf v$h2 + vlan_destroy $h2 777 + simple_if_fini $h2 +} + +router_create() +{ + ip link add name br1 address $(mac_get $swp1) \ + type bridge vlan_filtering 1 + ip link set dev br1 up + + ip link set dev $swp1 master br1 + ip link set dev $swp2 master br1 + ip link set dev $swp1 up + ip link set dev $swp2 up + + bridge vlan add dev br1 vid 555 self + bridge vlan add dev br1 vid 777 self + bridge vlan add dev $swp1 vid 555 + bridge vlan add dev $swp2 vid 777 + + vlan_create br1 555 "" 192.0.2.2/28 2001:db8:1::2/64 + vlan_create br1 777 "" 192.0.2.17/28 2001:db8:2::1/64 +} + +router_destroy() +{ + vlan_destroy br1 777 + vlan_destroy br1 555 + + bridge vlan del dev $swp2 vid 777 + bridge vlan del dev $swp1 vid 555 + bridge vlan del dev br1 vid 777 self + bridge vlan del dev br1 vid 555 self + + ip link set dev $swp2 down nomaster + ip link set dev $swp1 down nomaster + + ip link set dev br1 down + ip link del dev br1 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.18 +} + +ping_ipv6() +{ + ping6_test $h1 2001:db8:2::2 +} + +respin_config() +{ + log_info "Remaster bridge slave" + + ip link set dev $swp2 nomaster + ip link set dev $swp1 nomaster + + sleep 2 + + ip link set dev $swp1 master br1 + ip link set dev $swp2 master br1 + + bridge vlan add dev $swp1 vid 555 + bridge vlan add dev $swp2 vid 777 +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper_pvid.sh b/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper_pvid.sh new file mode 100755 index 000000000000..138558452402 --- /dev/null +++ b/tools/testing/selftests/net/forwarding/router_bridge_vlan_upper_pvid.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +----------------------------+ +# | H1 (vrf) | +# | + $h1.10 | +----------------------+ +# | | 192.0.2.1/28 | | H2 (vrf) | +# | | 2001:db8:1::1/64 | | + $h2 | +# | | | | | 192.0.2.130/28 | +# | + $h1 | | | 2001:db8:2::2/64 | +# +---|------------------------+ +--|-------------------+ +# | | +# +---|--------------------------------------------------|-------------------+ +# | | router (main VRF) | | +# | +-|--------------------------+ + $swp2 | +# | | + $swp1 BR1 (802.1q) | 192.0.2.129/28 | +# | +-----+----------------------+ 2001:db8:2::1/64 | +# | | | +# | + br1.10 | +# | 192.0.2.2/28 | +# | 2001:db8:1::2/64 | +# +--------------------------------------------------------------------------+ + +ALL_TESTS=" + ping_ipv4 + ping_ipv6 + pvid_set_unset + ping_ipv4 + ping_ipv6 + pvid_set_move + ping_ipv4 + ping_ipv6 +" +NUM_NETIFS=4 +source lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 10 v$h1 192.0.2.1/28 2001:db8:1::1/64 + ip -4 route add 192.0.2.128/28 vrf v$h1 nexthop via 192.0.2.2 + ip -6 route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::2 +} + +h1_destroy() +{ + ip -6 route del 2001:db8:2::/64 vrf v$h1 + ip -4 route del 192.0.2.128/28 vrf v$h1 + vlan_destroy $h1 10 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 192.0.2.130/28 2001:db8:2::2/64 + ip -4 route add 192.0.2.0/28 vrf v$h2 nexthop via 192.0.2.129 + ip -6 route add 2001:db8:1::/64 vrf v$h2 nexthop via 2001:db8:2::1 +} + +h2_destroy() +{ + ip -6 route del 2001:db8:1::/64 vrf v$h2 + ip -4 route del 192.0.2.0/28 vrf v$h2 + simple_if_fini $h2 192.0.2.130/28 2001:db8:2::2/64 +} + +router_create() +{ + ip link add name br1 address $(mac_get $swp1) \ + type bridge vlan_filtering 1 vlan_default_pvid 0 + ip link set dev br1 up + + ip link set dev $swp1 master br1 + ip link set dev $swp1 up + + ip link set dev $swp2 up + __addr_add_del $swp2 add 192.0.2.129/28 2001:db8:2::1/64 + + bridge vlan add dev br1 vid 10 self + bridge vlan add dev $swp1 vid 10 + vlan_create br1 10 "" 192.0.2.2/28 2001:db8:1::2/64 +} + +router_destroy() +{ + vlan_destroy br1 10 + bridge vlan del dev $swp1 vid 10 + bridge vlan del dev br1 vid 10 self + + __addr_add_del $swp2 del 192.0.2.129/28 2001:db8:2::1/64 + ip link set dev $swp2 down + + ip link set dev $swp1 down + ip link set dev $swp1 nomaster + + ip link del dev br1 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + + h1_create + h2_create + + router_create + + forwarding_enable +} + +pvid_set_unset() +{ + log_info "Set and unset PVID on VLAN 10" + + bridge vlan add dev br1 vid 10 pvid self + sleep 1 + bridge vlan add dev br1 vid 10 self +} + +pvid_set_move() +{ + log_info "Set PVID on VLAN 10, then move it to VLAN 20" + + bridge vlan add dev br1 vid 10 pvid self + sleep 1 + bridge vlan add dev br1 vid 20 pvid self +} + +shuffle_vlan() +{ + log_info "" +} + +cleanup() +{ + pre_cleanup + + forwarding_restore + + router_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.130 +} + +ping_ipv6() +{ + ping6_test $h1 2001:db8:2::2 +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/tc_flower_port_range.sh b/tools/testing/selftests/net/forwarding/tc_flower_port_range.sh new file mode 100755 index 000000000000..3885a2a91f7d --- /dev/null +++ b/tools/testing/selftests/net/forwarding/tc_flower_port_range.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# +-----------------------+ +----------------------+ +# | H1 (vrf) | | H2 (vrf) | +# | + $h1 | | $h2 + | +# | | 192.0.2.1/28 | | 192.0.2.2/28 | | +# | | 2001:db8:1::1/64 | | 2001:db8:1::2/64 | | +# +----|------------------+ +------------------|---+ +# | | +# +----|-------------------------------------------------------------------|---+ +# | SW | | | +# | +-|-------------------------------------------------------------------|-+ | +# | | + $swp1 BR $swp2 + | | +# | +-----------------------------------------------------------------------+ | +# +----------------------------------------------------------------------------+ + +ALL_TESTS=" + test_port_range_ipv4_udp + test_port_range_ipv4_tcp + test_port_range_ipv6_udp + test_port_range_ipv6_tcp +" + +NUM_NETIFS=4 +source lib.sh +source tc_common.sh + +h1_create() +{ + simple_if_init $h1 192.0.2.1/28 2001:db8:1::1/64 +} + +h1_destroy() +{ + simple_if_fini $h1 192.0.2.1/28 2001:db8:1::1/64 +} + +h2_create() +{ + simple_if_init $h2 192.0.2.2/28 2001:db8:1::2/64 +} + +h2_destroy() +{ + simple_if_fini $h2 192.0.2.2/28 2001:db8:1::2/64 +} + +switch_create() +{ + ip link add name br1 type bridge + ip link set dev $swp1 master br1 + ip link set dev $swp1 up + ip link set dev $swp2 master br1 + ip link set dev $swp2 up + ip link set dev br1 up + + tc qdisc add dev $swp1 clsact + tc qdisc add dev $swp2 clsact +} + +switch_destroy() +{ + tc qdisc del dev $swp2 clsact + tc qdisc del dev $swp1 clsact + + ip link set dev br1 down + ip link set dev $swp2 down + ip link set dev $swp2 nomaster + ip link set dev $swp1 down + ip link set dev $swp1 nomaster + ip link del dev br1 +} + +__test_port_range() +{ + local proto=$1; shift + local ip_proto=$1; shift + local sip=$1; shift + local dip=$1; shift + local mode=$1; shift + local name=$1; shift + local dmac=$(mac_get $h2) + local smac=$(mac_get $h1) + local sport_min=100 + local sport_max=200 + local sport_mid=$((sport_min + (sport_max - sport_min) / 2)) + local dport_min=300 + local dport_max=400 + local dport_mid=$((dport_min + (dport_max - dport_min) / 2)) + + RET=0 + + tc filter add dev $swp1 ingress protocol $proto handle 101 pref 1 \ + flower src_ip $sip dst_ip $dip ip_proto $ip_proto \ + src_port $sport_min-$sport_max \ + dst_port $dport_min-$dport_max \ + action pass + tc filter add dev $swp2 egress protocol $proto handle 101 pref 1 \ + flower src_ip $sip dst_ip $dip ip_proto $ip_proto \ + src_port $sport_min-$sport_max \ + dst_port $dport_min-$dport_max \ + action drop + + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$sport_min,dp=$dport_min" + tc_check_packets "dev $swp1 ingress" 101 1 + check_err $? "Ingress filter not hit with minimum ports" + tc_check_packets "dev $swp2 egress" 101 1 + check_err $? "Egress filter not hit with minimum ports" + + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$sport_mid,dp=$dport_mid" + tc_check_packets "dev $swp1 ingress" 101 2 + check_err $? "Ingress filter not hit with middle ports" + tc_check_packets "dev $swp2 egress" 101 2 + check_err $? "Egress filter not hit with middle ports" + + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$sport_max,dp=$dport_max" + tc_check_packets "dev $swp1 ingress" 101 3 + check_err $? "Ingress filter not hit with maximum ports" + tc_check_packets "dev $swp2 egress" 101 3 + check_err $? "Egress filter not hit with maximum ports" + + # Send traffic when both ports are out of range and when only one port + # is out of range. + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$((sport_min - 1)),dp=$dport_min" + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$((sport_max + 1)),dp=$dport_min" + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$sport_min,dp=$((dport_min - 1))" + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$sport_min,dp=$((dport_max + 1))" + $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \ + -t $ip_proto "sp=$((sport_max + 1)),dp=$((dport_max + 1))" + tc_check_packets "dev $swp1 ingress" 101 3 + check_err $? "Ingress filter was hit when should not" + tc_check_packets "dev $swp2 egress" 101 3 + check_err $? "Egress filter was hit when should not" + + tc filter del dev $swp2 egress protocol $proto pref 1 handle 101 flower + tc filter del dev $swp1 ingress protocol $proto pref 1 handle 101 flower + + log_test "Port range matching - $name" +} + +test_port_range_ipv4_udp() +{ + local proto=ipv4 + local ip_proto=udp + local sip=192.0.2.1 + local dip=192.0.2.2 + local mode="-4" + local name="IPv4 UDP" + + __test_port_range $proto $ip_proto $sip $dip $mode "$name" +} + +test_port_range_ipv4_tcp() +{ + local proto=ipv4 + local ip_proto=tcp + local sip=192.0.2.1 + local dip=192.0.2.2 + local mode="-4" + local name="IPv4 TCP" + + __test_port_range $proto $ip_proto $sip $dip $mode "$name" +} + +test_port_range_ipv6_udp() +{ + local proto=ipv6 + local ip_proto=udp + local sip=2001:db8:1::1 + local dip=2001:db8:1::2 + local mode="-6" + local name="IPv6 UDP" + + __test_port_range $proto $ip_proto $sip $dip $mode "$name" +} + +test_port_range_ipv6_tcp() +{ + local proto=ipv6 + local ip_proto=tcp + local sip=2001:db8:1::1 + local dip=2001:db8:1::2 + local mode="-6" + local name="IPv6 TCP" + + __test_port_range $proto $ip_proto $sip $dip $mode "$name" +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + h1_create + h2_create + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + h2_destroy + h1_destroy + vrf_cleanup +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/hwtstamp_config.c b/tools/testing/selftests/net/hwtstamp_config.c index e1fdee841021..170728c96c46 100644 --- a/tools/testing/selftests/net/hwtstamp_config.c +++ b/tools/testing/selftests/net/hwtstamp_config.c @@ -16,6 +16,8 @@ #include <linux/net_tstamp.h> #include <linux/sockios.h> +#include "kselftest.h" + static int lookup_value(const char **names, int size, const char *name) { @@ -50,7 +52,7 @@ static const char *tx_types[] = { TX_TYPE(ONESTEP_SYNC) #undef TX_TYPE }; -#define N_TX_TYPES ((int)(sizeof(tx_types) / sizeof(tx_types[0]))) +#define N_TX_TYPES ((int)(ARRAY_SIZE(tx_types))) static const char *rx_filters[] = { #define RX_FILTER(name) [HWTSTAMP_FILTER_ ## name] = #name @@ -71,7 +73,7 @@ static const char *rx_filters[] = { RX_FILTER(PTP_V2_DELAY_REQ), #undef RX_FILTER }; -#define N_RX_FILTERS ((int)(sizeof(rx_filters) / sizeof(rx_filters[0]))) +#define N_RX_FILTERS ((int)(ARRAY_SIZE(rx_filters))) static void usage(void) { diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh index fa9e09ad97d9..85a8ee9395b3 100755 --- a/tools/testing/selftests/net/mptcp/diag.sh +++ b/tools/testing/selftests/net/mptcp/diag.sh @@ -65,12 +65,15 @@ __chk_nr() if [ $nr != $expected ]; then if [ $nr = "$skip" ] && ! mptcp_lib_expect_all_features; then echo "[ skip ] Feature probably not supported" + mptcp_lib_result_skip "${msg}" else echo "[ fail ] expected $expected found $nr" + mptcp_lib_result_fail "${msg}" ret=$test_cnt fi else echo "[ ok ]" + mptcp_lib_result_pass "${msg}" fi test_cnt=$((test_cnt+1)) } @@ -111,12 +114,15 @@ wait_msk_nr() printf "%-50s" "$msg" if [ $i -ge $timeout ]; then echo "[ fail ] timeout while expecting $expected max $max last $nr" + mptcp_lib_result_fail "${msg} # timeout" ret=$test_cnt elif [ $nr != $expected ]; then echo "[ fail ] expected $expected found $nr" + mptcp_lib_result_fail "${msg} # unexpected result" ret=$test_cnt else echo "[ ok ]" + mptcp_lib_result_pass "${msg}" fi test_cnt=$((test_cnt+1)) } @@ -276,4 +282,5 @@ flush_pids chk_msk_inuse 0 "....chk 0 msk in use after flush" +mptcp_lib_result_print_all_tap exit $ret diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh index bbae40882bfa..b1fc8afd072d 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh @@ -7,6 +7,7 @@ time_start=$(date +%s) optstring="S:R:d:e:l:r:h4cm:f:tC" ret=0 +final_ret=0 sin="" sout="" cin_disconnect="" @@ -128,6 +129,7 @@ ns3="ns3-$rndh" ns4="ns4-$rndh" TEST_COUNT=0 +TEST_GROUP="" cleanup() { @@ -285,6 +287,7 @@ check_mptcp_disabled() # net.mptcp.enabled should be enabled by default if [ "$(ip netns exec ${disabled_ns} sysctl net.mptcp.enabled | awk '{ print $3 }')" -ne 1 ]; then echo -e "net.mptcp.enabled sysctl is not 1 by default\t\t[ FAIL ]" + mptcp_lib_result_fail "net.mptcp.enabled sysctl is not 1 by default" ret=1 return 1 fi @@ -297,11 +300,13 @@ check_mptcp_disabled() if [ ${err} -eq 0 ]; then echo -e "New MPTCP socket cannot be blocked via sysctl\t\t[ FAIL ]" + mptcp_lib_result_fail "New MPTCP socket cannot be blocked via sysctl" ret=1 return 1 fi echo -e "New MPTCP socket can be blocked via sysctl\t\t[ OK ]" + mptcp_lib_result_pass "New MPTCP socket can be blocked via sysctl" return 0 } @@ -317,14 +322,16 @@ do_ping() local connector_ns="$2" local connect_addr="$3" local ping_args="-q -c 1" + local rc=0 if is_v6 "${connect_addr}"; then $ipv6 || return 0 ping_args="${ping_args} -6" fi - ip netns exec ${connector_ns} ping ${ping_args} $connect_addr >/dev/null - if [ $? -ne 0 ] ; then + ip netns exec ${connector_ns} ping ${ping_args} $connect_addr >/dev/null || rc=1 + + if [ $rc -ne 0 ] ; then echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2 ret=1 @@ -403,7 +410,9 @@ do_transfer() local addr_port addr_port=$(printf "%s:%d" ${connect_addr} ${port}) - printf "%.3s %-5s -> %.3s (%-20s) %-5s\t" ${connector_ns} ${cl_proto} ${listener_ns} ${addr_port} ${srv_proto} + local result_msg + result_msg="$(printf "%.3s %-5s -> %.3s (%-20s) %-5s" ${connector_ns} ${cl_proto} ${listener_ns} ${addr_port} ${srv_proto})" + printf "%s\t" "${result_msg}" if $capture; then local capuser @@ -478,6 +487,7 @@ do_transfer() local duration duration=$((stop-start)) + result_msg+=" # time=${duration}ms" printf "(duration %05sms) " "${duration}" if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then echo "[ FAIL ] client exit code $retc, server $rets" 1>&2 @@ -490,6 +500,7 @@ do_transfer() echo cat "$capout" + mptcp_lib_result_fail "${TEST_GROUP}: ${result_msg}" return 1 fi @@ -549,6 +560,9 @@ do_transfer() if [ $retc -eq 0 ] && [ $rets -eq 0 ]; then printf "[ OK ]" + mptcp_lib_result_pass "${TEST_GROUP}: ${result_msg}" + else + mptcp_lib_result_fail "${TEST_GROUP}: ${result_msg}" fi if [ $cookies -eq 2 ];then @@ -691,6 +705,8 @@ run_test_transparent() local lret=0 local r6flag="" + TEST_GROUP="${msg}" + # skip if we don't want v6 if ! $ipv6 && is_v6 "${connect_addr}"; then return 0 @@ -702,6 +718,7 @@ run_test_transparent() # checking for a specific kernel version. if ! mptcp_lib_kallsyms_has "T __ip_sock_set_tos$"; then echo "INFO: ${msg} not supported by the kernel: SKIP" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -719,6 +736,7 @@ EOF if [ $? -ne 0 ]; then echo "SKIP: $msg, could not load nft ruleset" mptcp_lib_fail_if_expected_feature "nft rules" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -735,6 +753,7 @@ EOF ip netns exec "$listener_ns" nft flush ruleset echo "SKIP: $msg, ip $r6flag rule failed" mptcp_lib_fail_if_expected_feature "ip rule" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -744,6 +763,7 @@ EOF ip -net "$listener_ns" $r6flag rule del fwmark 1 lookup 100 echo "SKIP: $msg, ip route add local $local_addr failed" mptcp_lib_fail_if_expected_feature "ip route" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -773,6 +793,7 @@ run_tests_peekmode() { local peekmode="$1" + TEST_GROUP="peek mode: ${peekmode}" echo "INFO: with peek mode: ${peekmode}" run_tests_lo "$ns1" "$ns1" 10.0.1.1 1 "-P ${peekmode}" run_tests_lo "$ns1" "$ns1" dead:beef:1::1 1 "-P ${peekmode}" @@ -780,8 +801,11 @@ run_tests_peekmode() run_tests_mptfo() { + TEST_GROUP="MPTFO" + if ! mptcp_lib_kallsyms_has "mptcp_fastopen_"; then echo "INFO: TFO not supported by the kernel: SKIP" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -805,8 +829,11 @@ run_tests_disconnect() local old_cin=$cin local old_sin=$sin + TEST_GROUP="full disconnect" + if ! mptcp_lib_kallsyms_has "mptcp_pm_data_reset$"; then echo "INFO: Full disconnect not supported: SKIP" + mptcp_lib_result_skip "${TEST_GROUP}" return fi @@ -837,14 +864,26 @@ display_time() echo "Time: ${time_run} seconds" } -stop_if_error() +log_if_error() { local msg="$1" if [ ${ret} -ne 0 ]; then echo "FAIL: ${msg}" 1>&2 + + final_ret=${ret} + ret=0 + + return ${final_ret} + fi +} + +stop_if_error() +{ + if ! log_if_error "${@}"; then display_time - exit ${ret} + mptcp_lib_result_print_all_tap + exit ${final_ret} fi } @@ -874,6 +913,8 @@ for sender in "$ns1" "$ns2" "$ns3" "$ns4";do do_ping "$ns4" $sender dead:beef:3::1 done +mptcp_lib_result_code "${ret}" "ping tests" + stop_if_error "Could not even run ping tests" [ -n "$tc_loss" ] && tc -net "$ns2" qdisc add dev ns2eth3 root netem loss random $tc_loss delay ${tc_delay}ms @@ -903,12 +944,15 @@ echo "on ns3eth4" tc -net "$ns3" qdisc add dev ns3eth4 root netem delay ${reorder_delay}ms $tc_reorder +TEST_GROUP="loopback v4" run_tests_lo "$ns1" "$ns1" 10.0.1.1 1 stop_if_error "Could not even run loopback test" +TEST_GROUP="loopback v6" run_tests_lo "$ns1" "$ns1" dead:beef:1::1 1 stop_if_error "Could not even run loopback v6 test" +TEST_GROUP="multihosts" for sender in $ns1 $ns2 $ns3 $ns4;do # ns1<->ns2 is not subject to reordering/tc delays. Use it to test # mptcp syncookie support. @@ -934,23 +978,25 @@ for sender in $ns1 $ns2 $ns3 $ns4;do run_tests "$ns4" $sender 10.0.3.1 run_tests "$ns4" $sender dead:beef:3::1 - stop_if_error "Tests with $sender as a sender have failed" + log_if_error "Tests with $sender as a sender have failed" done run_tests_peekmode "saveWithPeek" run_tests_peekmode "saveAfterPeek" -stop_if_error "Tests with peek mode have failed" +log_if_error "Tests with peek mode have failed" # MPTFO (MultiPath TCP Fatopen tests) run_tests_mptfo -stop_if_error "Tests with MPTFO have failed" +log_if_error "Tests with MPTFO have failed" # connect to ns4 ip address, ns2 should intercept/proxy run_test_transparent 10.0.3.1 "tproxy ipv4" run_test_transparent dead:beef:3::1 "tproxy ipv6" -stop_if_error "Tests with tproxy have failed" +log_if_error "Tests with tproxy have failed" run_tests_disconnect +log_if_error "Tests of the full disconnection have failed" display_time -exit $ret +mptcp_lib_result_print_all_tap +exit ${final_ret} diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 3c2096ac97ef..63658b0416fb 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -39,7 +39,9 @@ evts_ns1="" evts_ns2="" evts_ns1_pid=0 evts_ns2_pid=0 -stats_dumped=0 +last_test_failed=0 +last_test_skipped=0 +last_test_ignored=1 declare -A all_tests declare -a only_tests_ids @@ -47,13 +49,17 @@ declare -a only_tests_names declare -A failed_tests TEST_COUNT=0 TEST_NAME="" -nr_blank=40 - -export FAILING_LINKS="" -export test_linkfail=0 -export addr_nr_ns1=0 -export addr_nr_ns2=0 -export sflags="" +nr_blank=6 + +# These var are used only in some tests, make sure they are not already set +unset FAILING_LINKS +unset test_linkfail +unset addr_nr_ns1 +unset addr_nr_ns2 +unset sflags +unset fastclose +unset fullmesh +unset speed # generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) || # (ip6 && (ip6[74] & 0xf0) == 0x30)'" @@ -97,10 +103,8 @@ init_partial() fi done - stats_dumped=0 check_invert=0 validate_checksum=$checksum - FAILING_LINKS="" # ns1 ns2 # ns1eth1 ns2eth1 @@ -183,8 +187,8 @@ init() { trap cleanup EXIT - make_file "$cin" "client" 1 - make_file "$sin" "server" 1 + make_file "$cin" "client" 1 >/dev/null + make_file "$sin" "server" 1 >/dev/null } cleanup() @@ -196,10 +200,37 @@ cleanup() cleanup_partial } -# $1: msg print_title() { - printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${1}" + printf "%03u %s\n" "${TEST_COUNT}" "${TEST_NAME}" +} + +print_check() +{ + printf "%-${nr_blank}s%-36s" " " "${*}" +} + +print_info() +{ + # It can be empty, no need to print anything then + [ -z "${1}" ] && return + + mptcp_lib_print_info " Info: ${*}" +} + +print_ok() +{ + mptcp_lib_print_ok "[ ok ]${1:+ ${*}}" +} + +print_fail() +{ + mptcp_lib_print_err "[fail]${1:+ ${*}}" +} + +print_skip() +{ + mptcp_lib_print_warn "[skip]${1:+ ${*}}" } # [ $1: fail msg ] @@ -209,8 +240,10 @@ mark_as_skipped() mptcp_lib_fail_if_expected_feature "${msg}" - print_title "[ skip ] ${msg}" - printf "\n" + print_check "${msg}" + print_skip + + last_test_skipped=1 } # $@: condition @@ -243,17 +276,37 @@ skip_test() return 0 } +append_prev_results() +{ + if [ ${last_test_failed} -eq 1 ]; then + mptcp_lib_result_fail "${TEST_NAME}" + elif [ ${last_test_skipped} -eq 1 ]; then + mptcp_lib_result_skip "${TEST_NAME}" + elif [ ${last_test_ignored} -ne 1 ]; then + mptcp_lib_result_pass "${TEST_NAME}" + fi + + last_test_failed=0 + last_test_skipped=0 + last_test_ignored=0 +} + # $1: test name reset() { + append_prev_results + TEST_NAME="${1}" TEST_COUNT=$((TEST_COUNT+1)) if skip_test; then + last_test_ignored=1 return 1 fi + print_title + if [ "${init}" != "1" ]; then init else @@ -434,13 +487,19 @@ reset_with_tcp_filter() fi } +# $1: err msg fail_test() { ret=1 - failed_tests[${TEST_COUNT}]="${TEST_NAME}" - [ "${stats_dumped}" = 0 ] && dump_stats - stats_dumped=1 + print_fail "${@}" + + # just in case a test is marked twice as failed + if [ ${last_test_failed} -eq 0 ]; then + failed_tests[${TEST_COUNT}]="${TEST_NAME}" + dump_stats + last_test_failed=1 + fi } get_failed_tests_ids() @@ -455,7 +514,7 @@ get_failed_tests_ids() print_file_err() { ls -l "$1" 1>&2 - echo "Trailing bytes are: " + echo -n "Trailing bytes are: " tail -c 27 "$1" } @@ -473,8 +532,7 @@ check_transfer() # when truncating we must check the size explicitly out_size=$(wc -c $out | awk '{print $1}') if [ $out_size -ne $bytes ]; then - echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)" - fail_test + fail_test "$what output file has wrong size ($out_size, $bytes)" return 1 fi @@ -489,14 +547,13 @@ check_transfer() cmp -l "$in" "$out" | while read -r i a b; do local sum=$((0${a} + 0${b})) if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then - echo "[ FAIL ] $what does not match (in, out):" + fail_test "$what does not match (in, out):" print_file_err "$in" print_file_err "$out" - fail_test return 1 else - echo "$what has inverted byte at ${i}" + print_info "$what has inverted byte at ${i}" fi done @@ -510,8 +567,7 @@ do_ping() local connect_addr="$3" if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then - echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2 - fail_test + fail_test "$listener_ns -> $connect_addr connectivity" fi } @@ -749,10 +805,9 @@ pm_nl_change_endpoint() pm_nl_check_endpoint() { local line expected_line - local need_title=$1 - local msg="$2" - local ns=$3 - local addr=$4 + local msg="$1" + local ns=$2 + local addr=$3 local _flags="" local flags local _port @@ -761,13 +816,9 @@ pm_nl_check_endpoint() local _id local id - if [ "${need_title}" = 1 ]; then - printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}" - else - printf "%-${nr_blank}s %s" " " "${msg}" - fi + print_check "${msg}" - shift 4 + shift 3 while [ -n "$1" ]; do if [ $1 = "flags" ]; then _flags=$2 @@ -790,7 +841,7 @@ pm_nl_check_endpoint() done if [ -z "$id" ]; then - echo "[skip] bad test - missing endpoint id" + test_fail "bad test - missing endpoint id" return fi @@ -813,10 +864,9 @@ pm_nl_check_endpoint() [ -n "$_port" ] && expected_line="$expected_line $_port" fi if [ "$line" = "$expected_line" ]; then - echo "[ ok ]" + print_ok else - echo "[fail] expected '$expected_line' found '$line'" - fail_test + fail_test "expected '$expected_line' found '$line'" fi } @@ -826,6 +876,17 @@ pm_nl_set_endpoint() local connector_ns="$2" local connect_addr="$3" + local addr_nr_ns1=${addr_nr_ns1:-0} + local addr_nr_ns2=${addr_nr_ns2:-0} + local sflags=${sflags:-""} + local fullmesh=${fullmesh:-""} + + local flags="subflow" + if [ -n "${fullmesh}" ]; then + flags="${flags},fullmesh" + addr_nr_ns2=${fullmesh} + fi + # let the mptcp subflow be established in background before # do endpoint manipulation if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then @@ -973,10 +1034,12 @@ do_transfer() local cl_proto="$3" local srv_proto="$4" local connect_addr="$5" - local speed="$6" local port=$((10000 + TEST_COUNT - 1)) local cappid + local FAILING_LINKS=${FAILING_LINKS:-""} + local fastclose=${fastclose:-""} + local speed=${speed:-"fast"} :> "$cout" :> "$sout" @@ -1009,24 +1072,22 @@ do_transfer() extra_args="-j" elif [ $speed = "slow" ]; then extra_args="-r 50" - elif [[ $speed = "speed_"* ]]; then - extra_args="-r ${speed:6}" + elif [ $speed -gt 0 ]; then + extra_args="-r ${speed}" fi - local flags="subflow" local extra_cl_args="" local extra_srv_args="" local trunc_size="" - if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then + if [ -n "${fastclose}" ]; then if [ ${test_linkfail} -le 1 ]; then - echo "fastclose tests need test_linkfail argument" - fail_test + fail_test "fastclose tests need test_linkfail argument" return 1 fi # disconnect trunc_size=${test_linkfail} - local side=${addr_nr_ns2:10} + local side=${fastclose} if [ ${side} = "client" ]; then extra_cl_args="-f ${test_linkfail}" @@ -1035,14 +1096,9 @@ do_transfer() extra_srv_args="-f ${test_linkfail}" extra_cl_args="-f -1" else - echo "wrong/unknown fastclose spec ${side}" - fail_test + fail_test "wrong/unknown fastclose spec ${side}" return 1 fi - addr_nr_ns2=0 - elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then - flags="${flags},fullmesh" - addr_nr_ns2=${addr_nr_ns2:9} fi extra_srv_args="$extra_args $extra_srv_args" @@ -1101,7 +1157,7 @@ do_transfer() nstat | grep Tcp > /tmp/${connector_ns}.out if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then - echo " client exit code $retc, server $rets" 1>&2 + fail_test "client exit code $retc, server $rets" echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2 ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port" cat /tmp/${listener_ns}.out @@ -1110,7 +1166,6 @@ do_transfer() cat /tmp/${connector_ns}.out cat "$capout" - fail_test return 1 fi @@ -1145,7 +1200,7 @@ make_file() dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name" - echo "Created $name (size $size KB) containing data sent by $who" + print_info "Test file (size $size KB) for $who" } run_tests() @@ -1153,9 +1208,9 @@ run_tests() local listener_ns="$1" local connector_ns="$2" local connect_addr="$3" - local speed="${4:-fast}" local size + local test_linkfail=${test_linkfail:-0} # The values above 2 are reused to make test files # with the given sizes (KB) @@ -1197,7 +1252,7 @@ run_tests() make_file "$sinfail" "server" $size fi - do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} ${speed} + do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} } dump_stats() @@ -1226,36 +1281,34 @@ chk_csum_nr() csum_ns2=${csum_ns2:1} fi - printf "%-${nr_blank}s %s" " " "sum" + print_check "sum" count=$(get_counter ${ns1} "MPTcpExtDataCsumErr") if [ "$count" != "$csum_ns1" ]; then extra_msg="$extra_msg ns1=$count" fi if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } || { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then - echo "[fail] got $count data checksum error[s] expected $csum_ns1" - fail_test + fail_test "got $count data checksum error[s] expected $csum_ns1" else - echo -n "[ ok ]" + print_ok fi - echo -n " - csum " + print_check "csum" count=$(get_counter ${ns2} "MPTcpExtDataCsumErr") if [ "$count" != "$csum_ns2" ]; then extra_msg="$extra_msg ns2=$count" fi if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } || { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then - echo "[fail] got $count data checksum error[s] expected $csum_ns2" - fail_test + fail_test "got $count data checksum error[s] expected $csum_ns2" else - echo -n "[ ok ]" + print_ok fi - echo "$extra_msg" + print_info "$extra_msg" } chk_fail_nr() @@ -1273,7 +1326,7 @@ chk_fail_nr() if [[ $ns_invert = "invert" ]]; then ns_tx=$ns2 ns_rx=$ns1 - extra_msg=" invert" + extra_msg="invert" fi if [[ "${fail_tx}" = "-"* ]]; then @@ -1285,37 +1338,35 @@ chk_fail_nr() fail_rx=${fail_rx:1} fi - printf "%-${nr_blank}s %s" " " "ftx" + print_check "ftx" count=$(get_counter ${ns_tx} "MPTcpExtMPFailTx") if [ "$count" != "$fail_tx" ]; then extra_msg="$extra_msg,tx=$count" fi if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } || { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then - echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx" - fail_test + fail_test "got $count MP_FAIL[s] TX expected $fail_tx" else - echo -n "[ ok ]" + print_ok fi - echo -n " - failrx" + print_check "failrx" count=$(get_counter ${ns_rx} "MPTcpExtMPFailRx") if [ "$count" != "$fail_rx" ]; then extra_msg="$extra_msg,rx=$count" fi if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } || { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then - echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx" - fail_test + fail_test "got $count MP_FAIL[s] RX expected $fail_rx" else - echo -n "[ ok ]" + print_ok fi - echo "$extra_msg" + print_info "$extra_msg" } chk_fclose_nr() @@ -1326,39 +1377,37 @@ chk_fclose_nr() local count local ns_tx=$ns2 local ns_rx=$ns1 - local extra_msg=" " + local extra_msg="" if [[ $ns_invert = "invert" ]]; then ns_tx=$ns1 ns_rx=$ns2 - extra_msg=${extra_msg}"invert" + extra_msg="invert" fi - printf "%-${nr_blank}s %s" " " "ctx" + print_check "ctx" count=$(get_counter ${ns_tx} "MPTcpExtMPFastcloseTx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$fclose_tx" ]; then extra_msg="$extra_msg,tx=$count" - echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx" - fail_test + fail_test "got $count MP_FASTCLOSE[s] TX expected $fclose_tx" else - echo -n "[ ok ]" + print_ok fi - echo -n " - fclzrx" + print_check "fclzrx" count=$(get_counter ${ns_rx} "MPTcpExtMPFastcloseRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$fclose_rx" ]; then extra_msg="$extra_msg,rx=$count" - echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx" - fail_test + fail_test "got $count MP_FASTCLOSE[s] RX expected $fclose_rx" else - echo -n "[ ok ]" + print_ok fi - echo "$extra_msg" + print_info "$extra_msg" } chk_rst_nr() @@ -1374,32 +1423,30 @@ chk_rst_nr() if [[ $ns_invert = "invert" ]]; then ns_tx=$ns2 ns_rx=$ns1 - extra_msg=" invert" + extra_msg="invert" fi - printf "%-${nr_blank}s %s" " " "rtx" + print_check "rtx" count=$(get_counter ${ns_tx} "MPTcpExtMPRstTx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ $count -lt $rst_tx ]; then - echo "[fail] got $count MP_RST[s] TX expected $rst_tx" - fail_test + fail_test "got $count MP_RST[s] TX expected $rst_tx" else - echo -n "[ ok ]" + print_ok fi - echo -n " - rstrx " + print_check "rstrx" count=$(get_counter ${ns_rx} "MPTcpExtMPRstRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" -lt "$rst_rx" ]; then - echo "[fail] got $count MP_RST[s] RX expected $rst_rx" - fail_test + fail_test "got $count MP_RST[s] RX expected $rst_rx" else - echo -n "[ ok ]" + print_ok fi - echo "$extra_msg" + print_info "$extra_msg" } chk_infi_nr() @@ -1408,26 +1455,24 @@ chk_infi_nr() local infi_rx=$2 local count - printf "%-${nr_blank}s %s" " " "itx" + print_check "itx" count=$(get_counter ${ns2} "MPTcpExtInfiniteMapTx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$infi_tx" ]; then - echo "[fail] got $count infinite map[s] TX expected $infi_tx" - fail_test + fail_test "got $count infinite map[s] TX expected $infi_tx" else - echo -n "[ ok ]" + print_ok fi - echo -n " - infirx" + print_check "infirx" count=$(get_counter ${ns1} "MPTcpExtInfiniteMapRx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$infi_rx" ]; then - echo "[fail] got $count infinite map[s] RX expected $infi_rx" - fail_test + fail_test "got $count infinite map[s] RX expected $infi_rx" else - echo "[ ok ]" + print_ok fi } @@ -1444,51 +1489,47 @@ chk_join_nr() local corrupted_pkts=${9:-0} local count local with_cookie - local title="${TEST_NAME}" if [ "${corrupted_pkts}" -gt 0 ]; then - title+=": ${corrupted_pkts} corrupted pkts" + print_info "${corrupted_pkts} corrupted pkts" fi - printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn" + print_check "syn" count=$(get_counter ${ns1} "MPTcpExtMPJoinSynRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$syn_nr" ]; then - echo "[fail] got $count JOIN[s] syn expected $syn_nr" - fail_test + fail_test "got $count JOIN[s] syn expected $syn_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - synack" + print_check "synack" with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies) count=$(get_counter ${ns2} "MPTcpExtMPJoinSynAckRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$syn_ack_nr" ]; then # simult connections exceeding the limit with cookie enabled could go up to # synack validation as the conn limit can be enforced reliably only after # the subflow creation if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then - echo -n "[ ok ]" + print_ok else - echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr" - fail_test + fail_test "got $count JOIN[s] synack expected $syn_ack_nr" fi else - echo -n "[ ok ]" + print_ok fi - echo -n " - ack" + print_check "ack" count=$(get_counter ${ns1} "MPTcpExtMPJoinAckRx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$ack_nr" ]; then - echo "[fail] got $count JOIN[s] ack expected $ack_nr" - fail_test + fail_test "got $count JOIN[s] ack expected $ack_nr" else - echo "[ ok ]" + print_ok fi if [ $validate_checksum -eq 1 ]; then chk_csum_nr $csum_ns1 $csum_ns2 @@ -1513,22 +1554,21 @@ chk_stale_nr() local stale_nr local recover_nr - printf "%-${nr_blank}s %-18s" " " "stale" + print_check "stale" stale_nr=$(get_counter ${ns} "MPTcpExtSubflowStale") recover_nr=$(get_counter ${ns} "MPTcpExtSubflowRecover") if [ -z "$stale_nr" ] || [ -z "$recover_nr" ]; then - echo "[skip]" + print_skip elif [ $stale_nr -lt $stale_min ] || { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } || [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then - echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \ + fail_test "got $stale_nr stale[s] $recover_nr recover[s], " \ " expected stale in range [$stale_min..$stale_max]," \ - " stale-recover delta $stale_delta " - fail_test + " stale-recover delta $stale_delta" dump_stats=1 else - echo "[ ok ]" + print_ok fi if [ "${dump_stats}" = 1 ]; then @@ -1553,103 +1593,93 @@ chk_add_nr() timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout) - printf "%-${nr_blank}s %s" " " "add" + print_check "add" count=$(get_counter ${ns2} "MPTcpExtAddAddr") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip # if the test configured a short timeout tolerate greater then expected # add addrs options, due to retransmissions elif [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then - echo "[fail] got $count ADD_ADDR[s] expected $add_nr" - fail_test + fail_test "got $count ADD_ADDR[s] expected $add_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - echo " + print_check "echo" count=$(get_counter ${ns1} "MPTcpExtEchoAdd") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$echo_nr" ]; then - echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr" - fail_test + fail_test "got $count ADD_ADDR echo[s] expected $echo_nr" else - echo -n "[ ok ]" + print_ok fi if [ $port_nr -gt 0 ]; then - echo -n " - pt " + print_check "pt" count=$(get_counter ${ns2} "MPTcpExtPortAdd") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$port_nr" ]; then - echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr" - fail_test + fail_test "got $count ADD_ADDR[s] with a port-number expected $port_nr" else - echo "[ ok ]" + print_ok fi - printf "%-${nr_blank}s %s" " " "syn" + print_check "syn" count=$(get_counter ${ns1} "MPTcpExtMPJoinPortSynRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$syn_nr" ]; then - echo "[fail] got $count JOIN[s] syn with a different \ - port-number expected $syn_nr" - fail_test + fail_test "got $count JOIN[s] syn with a different \ + port-number expected $syn_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - synack" + print_check "synack" count=$(get_counter ${ns2} "MPTcpExtMPJoinPortSynAckRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$syn_ack_nr" ]; then - echo "[fail] got $count JOIN[s] synack with a different \ - port-number expected $syn_ack_nr" - fail_test + fail_test "got $count JOIN[s] synack with a different \ + port-number expected $syn_ack_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - ack" + print_check "ack" count=$(get_counter ${ns1} "MPTcpExtMPJoinPortAckRx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$ack_nr" ]; then - echo "[fail] got $count JOIN[s] ack with a different \ - port-number expected $ack_nr" - fail_test + fail_test "got $count JOIN[s] ack with a different \ + port-number expected $ack_nr" else - echo "[ ok ]" + print_ok fi - printf "%-${nr_blank}s %s" " " "syn" + print_check "syn" count=$(get_counter ${ns1} "MPTcpExtMismatchPortSynRx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$mis_syn_nr" ]; then - echo "[fail] got $count JOIN[s] syn with a mismatched \ - port-number expected $mis_syn_nr" - fail_test + fail_test "got $count JOIN[s] syn with a mismatched \ + port-number expected $mis_syn_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - ack " + print_check "ack" count=$(get_counter ${ns1} "MPTcpExtMismatchPortAckRx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$mis_ack_nr" ]; then - echo "[fail] got $count JOIN[s] ack with a mismatched \ - port-number expected $mis_ack_nr" - fail_test + fail_test "got $count JOIN[s] ack with a mismatched \ + port-number expected $mis_ack_nr" else - echo "[ ok ]" + print_ok fi - else - echo "" fi } @@ -1662,28 +1692,26 @@ chk_add_tx_nr() timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout) - printf "%-${nr_blank}s %s" " " "add TX" + print_check "add TX" count=$(get_counter ${ns1} "MPTcpExtAddAddrTx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip # if the test configured a short timeout tolerate greater then expected # add addrs options, due to retransmissions elif [ "$count" != "$add_tx_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_tx_nr" ]; }; then - echo "[fail] got $count ADD_ADDR[s] TX, expected $add_tx_nr" - fail_test + fail_test "got $count ADD_ADDR[s] TX, expected $add_tx_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - echo TX " + print_check "echo TX" count=$(get_counter ${ns2} "MPTcpExtEchoAddTx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$echo_tx_nr" ]; then - echo "[fail] got $count ADD_ADDR echo[s] TX, expected $echo_tx_nr" - fail_test + fail_test "got $count ADD_ADDR echo[s] TX, expected $echo_tx_nr" else - echo "[ ok ]" + print_ok fi } @@ -1711,24 +1739,23 @@ chk_rm_nr() elif [ $invert = "true" ]; then addr_ns=$ns2 subflow_ns=$ns1 - extra_msg=" invert" + extra_msg="invert" fi - printf "%-${nr_blank}s %s" " " "rm " + print_check "rm" count=$(get_counter ${addr_ns} "MPTcpExtRmAddr") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$rm_addr_nr" ]; then - echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr" - fail_test + fail_test "got $count RM_ADDR[s] expected $rm_addr_nr" else - echo -n "[ ok ]" + print_ok fi - echo -n " - rmsf " + print_check "rmsf" count=$(get_counter ${subflow_ns} "MPTcpExtRmSubflow") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ -n "$simult" ]; then local cnt suffix @@ -1740,34 +1767,31 @@ chk_rm_nr() [ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]" if [ $count -ge "$rm_subflow_nr" ] && \ [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then - echo -n "[ ok ] $suffix" + print_ok "$suffix" else - echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]" - fail_test + fail_test "got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]" fi elif [ "$count" != "$rm_subflow_nr" ]; then - echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr" - fail_test + fail_test "got $count RM_SUBFLOW[s] expected $rm_subflow_nr" else - echo -n "[ ok ]" + print_ok fi - echo "$extra_msg" + print_info "$extra_msg" } chk_rm_tx_nr() { local rm_addr_tx_nr=$1 - printf "%-${nr_blank}s %s" " " "rm TX " + print_check "rm TX" count=$(get_counter ${ns2} "MPTcpExtRmAddrTx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$rm_addr_tx_nr" ]; then - echo "[fail] got $count RM_ADDR[s] expected $rm_addr_tx_nr" - fail_test + fail_test "got $count RM_ADDR[s] expected $rm_addr_tx_nr" else - echo "[ ok ]" + print_ok fi } @@ -1777,52 +1801,44 @@ chk_prio_nr() local mp_prio_nr_rx=$2 local count - printf "%-${nr_blank}s %s" " " "ptx" + print_check "ptx" count=$(get_counter ${ns1} "MPTcpExtMPPrioTx") if [ -z "$count" ]; then - echo -n "[skip]" + print_skip elif [ "$count" != "$mp_prio_nr_tx" ]; then - echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx" - fail_test + fail_test "got $count MP_PRIO[s] TX expected $mp_prio_nr_tx" else - echo -n "[ ok ]" + print_ok fi - echo -n " - prx " + print_check "prx" count=$(get_counter ${ns1} "MPTcpExtMPPrioRx") if [ -z "$count" ]; then - echo "[skip]" + print_skip elif [ "$count" != "$mp_prio_nr_rx" ]; then - echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx" - fail_test + fail_test "got $count MP_PRIO[s] RX expected $mp_prio_nr_rx" else - echo "[ ok ]" + print_ok fi } chk_subflow_nr() { - local need_title="$1" - local msg="$2" - local subflow_nr=$3 + local msg="$1" + local subflow_nr=$2 local cnt1 local cnt2 local dump_stats - if [ -n "${need_title}" ]; then - printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}" - else - printf "%-${nr_blank}s %s" " " "${msg}" - fi + print_check "${msg}" cnt1=$(ss -N $ns1 -tOni | grep -c token) cnt2=$(ss -N $ns2 -tOni | grep -c token) if [ "$cnt1" != "$subflow_nr" ] || [ "$cnt2" != "$subflow_nr" ]; then - echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr" - fail_test + fail_test "got $cnt1:$cnt2 subflows expected $subflow_nr" dump_stats=1 else - echo "[ ok ]" + print_ok fi if [ "${dump_stats}" = 1 ]; then @@ -1842,7 +1858,7 @@ chk_mptcp_info() local cnt2 local dump_stats - printf "%-${nr_blank}s %-30s" " " "mptcp_info $info1:$info2=$exp1:$exp2" + print_check "mptcp_info ${info1:0:8}=$exp1:$exp2" cnt1=$(ss -N $ns1 -inmHM | grep "$info1:" | sed -n 's/.*\('"$info1"':\)\([[:digit:]]*\).*$/\2/p;q') @@ -1853,11 +1869,10 @@ chk_mptcp_info() [ -z "$cnt2" ] && cnt2=0 if [ "$cnt1" != "$exp1" ] || [ "$cnt2" != "$exp2" ]; then - echo "[fail] got $cnt1:$cnt2 $info1:$info2 expected $exp1:$exp2" - fail_test + fail_test "got $cnt1:$cnt2 $info1:$info2 expected $exp1:$exp2" dump_stats=1 else - echo "[ ok ]" + print_ok fi if [ "$dump_stats" = 1 ]; then @@ -1879,13 +1894,12 @@ chk_link_usage() local tx_rate=$((tx_link * 100 / tx_total)) local tolerance=5 - printf "%-${nr_blank}s %-18s" " " "link usage" + print_check "link usage" if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \ [ $tx_rate -gt $((expected_rate + tolerance)) ]; then - echo "[fail] got $tx_rate% usage, expected $expected_rate%" - fail_test + fail_test "got $tx_rate% usage, expected $expected_rate%" else - echo "[ ok ]" + print_ok fi } @@ -1986,7 +2000,8 @@ subflows_error_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 fi @@ -1997,7 +2012,8 @@ subflows_error_tests() pm_nl_set_limits $ns2 0 2 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 fi @@ -2008,7 +2024,8 @@ subflows_error_tests() pm_nl_set_limits $ns2 0 2 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 fi @@ -2020,7 +2037,8 @@ subflows_error_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - run_tests $ns1 $ns2 10.0.1.1 slow & + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 & # mpj subflow will be in TW after the reset wait_attempt_fail $ns2 @@ -2119,7 +2137,8 @@ signal_address_tests() # the peer could possibly miss some addr notification, allow retransmission ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1 - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 # It is not directly linked to the commit introducing this # symbol but for the parent one which is linked anyway. @@ -2229,7 +2248,8 @@ add_addr_timeout_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_tx_nr 4 4 chk_add_nr 4 0 @@ -2240,7 +2260,8 @@ add_addr_timeout_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal - run_tests $ns1 $ns2 dead:beef:1::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 1 1 1 chk_add_nr 4 0 fi @@ -2251,7 +2272,8 @@ add_addr_timeout_tests() pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_set_limits $ns2 2 2 - run_tests $ns1 $ns2 10.0.1.1 speed_10 + speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_add_nr 8 0 fi @@ -2262,7 +2284,8 @@ add_addr_timeout_tests() pm_nl_add_endpoint $ns1 10.0.12.1 flags signal pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_set_limits $ns2 2 2 - run_tests $ns1 $ns2 10.0.1.1 speed_10 + speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 8 0 fi @@ -2275,8 +2298,8 @@ remove_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns2=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_rm_tx_nr 1 chk_rm_nr 1 1 @@ -2288,8 +2311,8 @@ remove_tests() pm_nl_set_limits $ns2 0 2 pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns2=-2 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=-2 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_rm_nr 2 2 fi @@ -2299,8 +2322,8 @@ remove_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_set_limits $ns2 1 1 - addr_nr_ns1=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 chk_rm_nr 1 1 invert @@ -2312,8 +2335,8 @@ remove_tests() pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_set_limits $ns2 1 2 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns1=-1 addr_nr_ns2=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-1 addr_nr_ns2=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_add_nr 1 1 chk_rm_nr 1 1 @@ -2326,8 +2349,8 @@ remove_tests() pm_nl_set_limits $ns2 1 3 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow - addr_nr_ns1=-1 addr_nr_ns2=-2 \ - run_tests $ns1 $ns2 10.0.1.1 speed_10 + addr_nr_ns1=-1 addr_nr_ns2=-2 speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 1 1 chk_rm_nr 2 2 @@ -2340,8 +2363,8 @@ remove_tests() pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_add_endpoint $ns1 10.0.4.1 flags signal pm_nl_set_limits $ns2 3 3 - addr_nr_ns1=-3 \ - run_tests $ns1 $ns2 10.0.1.1 speed_10 + addr_nr_ns1=-3 speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 3 3 chk_rm_nr 3 3 invert @@ -2354,8 +2377,8 @@ remove_tests() pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_add_endpoint $ns1 10.0.14.1 flags signal pm_nl_set_limits $ns2 3 3 - addr_nr_ns1=-3 \ - run_tests $ns1 $ns2 10.0.1.1 speed_10 + addr_nr_ns1=-3 speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 3 3 chk_rm_nr 3 1 invert @@ -2368,8 +2391,8 @@ remove_tests() pm_nl_set_limits $ns2 1 3 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow - addr_nr_ns1=-8 addr_nr_ns2=-8 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 1 1 chk_rm_nr 1 3 invert simult @@ -2382,8 +2405,8 @@ remove_tests() pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow - addr_nr_ns1=-8 addr_nr_ns2=-8 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 if mptcp_lib_kversion_ge 5.18; then @@ -2401,8 +2424,8 @@ remove_tests() pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_add_endpoint $ns1 10.0.4.1 flags signal pm_nl_set_limits $ns2 3 3 - addr_nr_ns1=-8 addr_nr_ns2=-8 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 3 3 chk_rm_nr 3 3 invert simult @@ -2415,8 +2438,8 @@ remove_tests() pm_nl_add_endpoint $ns1 10.0.3.1 flags signal pm_nl_add_endpoint $ns1 10.0.14.1 flags signal pm_nl_set_limits $ns2 3 3 - addr_nr_ns1=-8 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-8 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 3 3 chk_rm_nr 3 1 invert @@ -2427,8 +2450,8 @@ remove_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns2=-9 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=-9 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_rm_nr 1 1 fi @@ -2438,8 +2461,8 @@ remove_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_set_limits $ns2 1 1 - addr_nr_ns1=-9 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-9 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 chk_rm_nr 1 1 invert @@ -2452,8 +2475,8 @@ add_tests() if reset "add single subflow"; then pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 - addr_nr_ns2=1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 fi @@ -2461,8 +2484,8 @@ add_tests() if reset "add signal address"; then pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 1 1 - addr_nr_ns1=1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 fi @@ -2471,8 +2494,8 @@ add_tests() if reset "add multiple subflows"; then pm_nl_set_limits $ns1 0 2 pm_nl_set_limits $ns2 0 2 - addr_nr_ns2=2 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=2 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 fi @@ -2480,8 +2503,8 @@ add_tests() if reset "add multiple subflows IPv6"; then pm_nl_set_limits $ns1 0 2 pm_nl_set_limits $ns2 0 2 - addr_nr_ns2=2 \ - run_tests $ns1 $ns2 dead:beef:1::1 slow + addr_nr_ns2=2 speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 2 2 2 fi @@ -2489,8 +2512,8 @@ add_tests() if reset "add multiple addresses IPv6"; then pm_nl_set_limits $ns1 0 2 pm_nl_set_limits $ns2 2 2 - addr_nr_ns1=2 \ - run_tests $ns1 $ns2 dead:beef:1::1 slow + addr_nr_ns1=2 speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 2 2 2 chk_add_nr 2 2 fi @@ -2503,14 +2526,16 @@ ipv6_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow - run_tests $ns1 $ns2 dead:beef:1::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 1 1 1 fi # add_address, unused IPv6 if reset "unused signal address IPv6"; then pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal - run_tests $ns1 $ns2 dead:beef:1::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 0 0 0 chk_add_nr 1 1 fi @@ -2520,7 +2545,8 @@ ipv6_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal pm_nl_set_limits $ns2 1 1 - run_tests $ns1 $ns2 dead:beef:1::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 1 1 1 chk_add_nr 1 1 fi @@ -2530,8 +2556,8 @@ ipv6_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal pm_nl_set_limits $ns2 1 1 - addr_nr_ns1=-1 \ - run_tests $ns1 $ns2 dead:beef:1::1 slow + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 1 1 1 chk_add_nr 1 1 chk_rm_nr 1 1 invert @@ -2543,8 +2569,8 @@ ipv6_tests() pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal pm_nl_set_limits $ns2 1 2 pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow - addr_nr_ns1=-1 addr_nr_ns2=-1 \ - run_tests $ns1 $ns2 dead:beef:1::1 slow + addr_nr_ns1=-1 addr_nr_ns2=-1 speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 2 2 2 chk_add_nr 1 1 chk_rm_nr 1 1 @@ -2645,7 +2671,8 @@ mixed_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 fi @@ -2655,7 +2682,8 @@ mixed_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns1 10.0.1.1 flags signal - run_tests $ns1 $ns2 dead:beef:2::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:2::1 chk_join_nr 1 1 1 fi @@ -2666,7 +2694,8 @@ mixed_tests() pm_nl_set_limits $ns2 1 4 pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow,fullmesh pm_nl_add_endpoint $ns1 10.0.1.1 flags signal - run_tests $ns1 $ns2 dead:beef:2::1 slow + speed=slow \ + run_tests $ns1 $ns2 dead:beef:2::1 chk_join_nr 1 1 1 fi @@ -2678,8 +2707,8 @@ mixed_tests() pm_nl_set_limits $ns2 2 4 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal - addr_nr_ns2=fullmesh_1 \ - run_tests $ns1 $ns2 dead:beef:1::1 slow + fullmesh=1 speed=slow \ + run_tests $ns1 $ns2 dead:beef:1::1 chk_join_nr 4 4 4 fi } @@ -2692,8 +2721,8 @@ backup_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup - sflags=nobackup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=nobackup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_prio_nr 0 1 fi @@ -2704,8 +2733,8 @@ backup_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal pm_nl_set_limits $ns2 1 1 - sflags=backup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=backup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 chk_prio_nr 1 1 @@ -2717,8 +2746,8 @@ backup_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100 pm_nl_set_limits $ns2 1 1 - sflags=backup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=backup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 chk_prio_nr 1 1 @@ -2727,7 +2756,8 @@ backup_tests() if reset "mpc backup" && continue_if mptcp_lib_kallsyms_doesnt_have "mptcp_subflow_send_ack$"; then pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_prio_nr 0 1 fi @@ -2736,7 +2766,8 @@ backup_tests() continue_if mptcp_lib_kallsyms_doesnt_have "mptcp_subflow_send_ack$"; then pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup - run_tests $ns1 $ns2 10.0.1.1 slow + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_prio_nr 1 1 fi @@ -2744,8 +2775,8 @@ backup_tests() if reset "mpc switch to backup" && continue_if mptcp_lib_kallsyms_doesnt_have "mptcp_subflow_send_ack$"; then pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow - sflags=backup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=backup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_prio_nr 0 1 fi @@ -2754,8 +2785,8 @@ backup_tests() continue_if mptcp_lib_kallsyms_doesnt_have "mptcp_subflow_send_ack$"; then pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow - sflags=backup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=backup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_prio_nr 1 1 fi @@ -2783,15 +2814,15 @@ verify_listener_events() if [ $e_type = $LISTENER_CREATED ]; then name="LISTENER_CREATED" elif [ $e_type = $LISTENER_CLOSED ]; then - name="LISTENER_CLOSED" + name="LISTENER_CLOSED " else name="$e_type" fi - printf "%-${nr_blank}s %s %s:%s " " " "$name" "$e_saddr" "$e_sport" + print_check "$name $e_saddr:$e_sport" if ! mptcp_lib_kallsyms_has "mptcp_event_pm_listener$"; then - printf "[skip]: event not supported\n" + print_skip "event not supported" return fi @@ -2808,11 +2839,10 @@ verify_listener_events() [ $family ] && [ $family = $e_family ] && [ $saddr ] && [ $saddr = $e_saddr ] && [ $sport ] && [ $sport = $e_sport ]; then - echo "[ ok ]" + print_ok return 0 fi - fail_test - echo "[fail]" + fail_test "$e_type:$type $e_family:$family $e_saddr:$saddr $e_sport:$sport" } add_addr_ports_tests() @@ -2844,8 +2874,8 @@ add_addr_ports_tests() pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100 pm_nl_set_limits $ns2 1 1 - addr_nr_ns1=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 chk_add_nr 1 1 1 chk_rm_nr 1 1 invert @@ -2861,8 +2891,8 @@ add_addr_ports_tests() pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100 pm_nl_set_limits $ns2 1 2 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns1=-1 addr_nr_ns2=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-1 addr_nr_ns2=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_add_nr 1 1 1 chk_rm_nr 1 1 @@ -2875,8 +2905,8 @@ add_addr_ports_tests() pm_nl_set_limits $ns2 1 3 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow - addr_nr_ns1=-8 addr_nr_ns2=-2 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=-8 addr_nr_ns2=-2 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 1 1 chk_rm_nr 1 3 invert simult @@ -3078,8 +3108,8 @@ fullmesh_tests() pm_nl_set_limits $ns2 1 4 pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh - addr_nr_ns1=1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns1=1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 4 4 4 chk_add_nr 1 1 fi @@ -3091,8 +3121,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 1 3 pm_nl_set_limits $ns2 1 3 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - addr_nr_ns2=fullmesh_1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + fullmesh=1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 chk_add_nr 1 1 fi @@ -3104,8 +3134,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 2 5 pm_nl_set_limits $ns2 1 5 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - addr_nr_ns2=fullmesh_2 \ - run_tests $ns1 $ns2 10.0.1.1 slow + fullmesh=2 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 5 5 5 chk_add_nr 1 1 fi @@ -3118,8 +3148,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 2 4 pm_nl_set_limits $ns2 1 4 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - addr_nr_ns2=fullmesh_2 \ - run_tests $ns1 $ns2 10.0.1.1 slow + fullmesh=2 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 4 4 4 chk_add_nr 1 1 fi @@ -3130,8 +3160,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 4 4 pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow pm_nl_set_limits $ns2 4 4 - addr_nr_ns2=1 sflags=fullmesh \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=1 sflags=fullmesh speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_rm_nr 0 1 fi @@ -3142,8 +3172,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 4 4 pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh pm_nl_set_limits $ns2 4 4 - addr_nr_ns2=fullmesh_1 sflags=nofullmesh \ - run_tests $ns1 $ns2 10.0.1.1 slow + fullmesh=1 sflags=nofullmesh speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_rm_nr 0 1 fi @@ -3154,8 +3184,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 4 4 pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow pm_nl_set_limits $ns2 4 4 - addr_nr_ns2=1 sflags=backup,fullmesh \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=1 sflags=backup,fullmesh speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_prio_nr 0 1 chk_rm_nr 0 1 @@ -3167,8 +3197,8 @@ fullmesh_tests() pm_nl_set_limits $ns1 4 4 pm_nl_set_limits $ns2 4 4 pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh - sflags=nobackup,nofullmesh \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=nobackup,nofullmesh speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 chk_prio_nr 0 1 chk_rm_nr 0 1 @@ -3178,7 +3208,7 @@ fullmesh_tests() fastclose_tests() { if reset_check_counter "fastclose test" "MPTcpExtMPFastcloseTx"; then - test_linkfail=1024 addr_nr_ns2=fastclose_client \ + test_linkfail=1024 fastclose=client \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_fclose_nr 1 1 @@ -3186,7 +3216,7 @@ fastclose_tests() fi if reset_check_counter "fastclose server test" "MPTcpExtMPFastcloseRx"; then - test_linkfail=1024 addr_nr_ns2=fastclose_server \ + test_linkfail=1024 fastclose=server \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_fclose_nr 1 1 invert @@ -3343,8 +3373,8 @@ userspace_tests() pm_nl_set_limits $ns1 1 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - sflags=backup \ - run_tests $ns1 $ns2 10.0.1.1 slow + sflags=backup speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 0 chk_prio_nr 0 0 fi @@ -3357,8 +3387,8 @@ userspace_tests() pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - addr_nr_ns2=-1 \ - run_tests $ns1 $ns2 10.0.1.1 slow + addr_nr_ns2=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 0 0 0 chk_rm_nr 0 0 fi @@ -3368,7 +3398,8 @@ userspace_tests() continue_if mptcp_lib_has_file '/proc/sys/net/mptcp/pm_type'; then set_userspace_pm $ns1 pm_nl_set_limits $ns2 1 1 - run_tests $ns1 $ns2 10.0.1.1 speed_10 & + speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! wait_mpj $ns1 userspace_pm_add_addr 10.0.2.1 10 @@ -3388,7 +3419,8 @@ userspace_tests() continue_if mptcp_lib_has_file '/proc/sys/net/mptcp/pm_type'; then set_userspace_pm $ns2 pm_nl_set_limits $ns1 0 1 - run_tests $ns1 $ns2 10.0.1.1 speed_10 & + speed=10 \ + run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! wait_mpj $ns2 userspace_pm_add_sf 10.0.3.2 20 @@ -3411,20 +3443,21 @@ endpoint_tests() pm_nl_set_limits $ns1 2 2 pm_nl_set_limits $ns2 2 2 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - run_tests $ns1 $ns2 10.0.1.1 slow 2>/dev/null & + speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 2>/dev/null & wait_mpj $ns1 - pm_nl_check_endpoint 1 "creation" \ + pm_nl_check_endpoint "creation" \ $ns2 10.0.2.2 id 1 flags implicit chk_mptcp_info subflows 1 subflows 1 chk_mptcp_info add_addr_signal 1 add_addr_accepted 1 - pm_nl_add_endpoint $ns2 10.0.2.2 id 33 - pm_nl_check_endpoint 0 "ID change is prevented" \ + pm_nl_add_endpoint $ns2 10.0.2.2 id 33 2>/dev/null + pm_nl_check_endpoint "ID change is prevented" \ $ns2 10.0.2.2 id 1 flags implicit pm_nl_add_endpoint $ns2 10.0.2.2 flags signal - pm_nl_check_endpoint 0 "modif is allowed" \ + pm_nl_check_endpoint "modif is allowed" \ $ns2 10.0.2.2 id 1 flags signal kill_tests_wait fi @@ -3434,21 +3467,21 @@ endpoint_tests() pm_nl_set_limits $ns1 1 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow - test_linkfail=4 \ - run_tests $ns1 $ns2 10.0.1.1 speed_20 2>/dev/null & + test_linkfail=4 speed=20 \ + run_tests $ns1 $ns2 10.0.1.1 2>/dev/null & wait_mpj $ns2 - chk_subflow_nr needtitle "before delete" 2 + chk_subflow_nr "before delete" 2 chk_mptcp_info subflows 1 subflows 1 pm_nl_del_endpoint $ns2 2 10.0.2.2 sleep 0.5 - chk_subflow_nr "" "after delete" 1 + chk_subflow_nr "after delete" 1 chk_mptcp_info subflows 0 subflows 0 pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow wait_mpj $ns2 - chk_subflow_nr "" "after re-add" 2 + chk_subflow_nr "after re-add" 2 chk_mptcp_info subflows 1 subflows 1 kill_tests_wait fi @@ -3566,4 +3599,7 @@ if [ ${ret} -ne 0 ]; then echo fi +append_prev_results +mptcp_lib_result_print_all_tap + exit $ret diff --git a/tools/testing/selftests/net/mptcp/mptcp_lib.sh b/tools/testing/selftests/net/mptcp/mptcp_lib.sh index f32045b23b89..92a5befe8039 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_lib.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_lib.sh @@ -1,9 +1,52 @@ #! /bin/bash # SPDX-License-Identifier: GPL-2.0 +readonly KSFT_PASS=0 readonly KSFT_FAIL=1 readonly KSFT_SKIP=4 +# shellcheck disable=SC2155 # declare and assign separately +readonly KSFT_TEST=$(basename "${0}" | sed 's/\.sh$//g') + +MPTCP_LIB_SUBTESTS=() + +# only if supported (or forced) and not disabled, see no-color.org +if { [ -t 1 ] || [ "${SELFTESTS_MPTCP_LIB_COLOR_FORCE:-}" = "1" ]; } && + [ "${NO_COLOR:-}" != "1" ]; then + readonly MPTCP_LIB_COLOR_RED="\E[1;31m" + readonly MPTCP_LIB_COLOR_GREEN="\E[1;32m" + readonly MPTCP_LIB_COLOR_YELLOW="\E[1;33m" + readonly MPTCP_LIB_COLOR_BLUE="\E[1;34m" + readonly MPTCP_LIB_COLOR_RESET="\E[0m" +else + readonly MPTCP_LIB_COLOR_RED= + readonly MPTCP_LIB_COLOR_GREEN= + readonly MPTCP_LIB_COLOR_YELLOW= + readonly MPTCP_LIB_COLOR_BLUE= + readonly MPTCP_LIB_COLOR_RESET= +fi + +# $1: color, $2: text +mptcp_lib_print_color() { + echo -e "${MPTCP_LIB_START_PRINT:-}${*}${MPTCP_LIB_COLOR_RESET}" +} + +mptcp_lib_print_ok() { + mptcp_lib_print_color "${MPTCP_LIB_COLOR_GREEN}${*}" +} + +mptcp_lib_print_warn() { + mptcp_lib_print_color "${MPTCP_LIB_COLOR_YELLOW}${*}" +} + +mptcp_lib_print_info() { + mptcp_lib_print_color "${MPTCP_LIB_COLOR_BLUE}${*}" +} + +mptcp_lib_print_err() { + mptcp_lib_print_color "${MPTCP_LIB_COLOR_RED}${*}" +} + # SELFTESTS_MPTCP_LIB_EXPECT_ALL_FEATURES env var can be set when validating all # features using the last version of the kernel and the selftests to make sure # a test is not being skipped by mistake. @@ -102,3 +145,65 @@ mptcp_lib_kversion_ge() { mptcp_lib_fail_if_expected_feature "kernel version ${1} lower than ${v}" } + +__mptcp_lib_result_add() { + local result="${1}" + shift + + local id=$((${#MPTCP_LIB_SUBTESTS[@]} + 1)) + + MPTCP_LIB_SUBTESTS+=("${result} ${id} - ${KSFT_TEST}: ${*}") +} + +# $1: test name +mptcp_lib_result_pass() { + __mptcp_lib_result_add "ok" "${1}" +} + +# $1: test name +mptcp_lib_result_fail() { + __mptcp_lib_result_add "not ok" "${1}" +} + +# $1: test name +mptcp_lib_result_skip() { + __mptcp_lib_result_add "ok" "${1} # SKIP" +} + +# $1: result code ; $2: test name +mptcp_lib_result_code() { + local ret="${1}" + local name="${2}" + + case "${ret}" in + "${KSFT_PASS}") + mptcp_lib_result_pass "${name}" + ;; + "${KSFT_FAIL}") + mptcp_lib_result_fail "${name}" + ;; + "${KSFT_SKIP}") + mptcp_lib_result_skip "${name}" + ;; + *) + echo "ERROR: wrong result code: ${ret}" + exit ${KSFT_FAIL} + ;; + esac +} + +mptcp_lib_result_print_all_tap() { + local subtest + + if [ ${#MPTCP_LIB_SUBTESTS[@]} -eq 0 ] || + [ "${SELFTESTS_MPTCP_LIB_NO_TAP:-}" = "1" ]; then + return + fi + + printf "\nTAP version 13\n" + printf "1..%d\n" "${#MPTCP_LIB_SUBTESTS[@]}" + + for subtest in "${MPTCP_LIB_SUBTESTS[@]}"; do + printf "%s\n" "${subtest}" + done +} diff --git a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh index dc8d473fc82c..8c8694f21e7d 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh @@ -183,11 +183,13 @@ do_transfer() local mptcp_connect="./mptcp_connect -r 20" - local local_addr + local local_addr ip if is_v6 "${connect_addr}"; then local_addr="::" + ip=ipv6 else local_addr="0.0.0.0" + ip=ipv4 fi cmsg="TIMESTAMPNS" @@ -223,6 +225,8 @@ do_transfer() echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2 ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port" + mptcp_lib_result_fail "transfer ${ip}" + ret=1 return 1 fi @@ -236,9 +240,11 @@ do_transfer() fi check_transfer $cin $sout "file received by server" - rets=$? + mptcp_lib_result_code "${retc}" "mark ${ip}" + mptcp_lib_result_code "${rets}" "transfer ${ip}" + if [ $retc -eq 0 ] && [ $rets -eq 0 ];then return 0 fi @@ -264,6 +270,7 @@ do_mptcp_sockopt_tests() if ! mptcp_lib_kallsyms_has "mptcp_diag_fill_info$"; then echo "INFO: MPTCP sockopt not supported: SKIP" + mptcp_lib_result_skip "sockopt" return fi @@ -272,18 +279,22 @@ do_mptcp_sockopt_tests() if [ $lret -ne 0 ]; then echo "FAIL: SOL_MPTCP getsockopt" 1>&2 + mptcp_lib_result_fail "sockopt v4" ret=$lret return fi + mptcp_lib_result_pass "sockopt v4" ip netns exec "$ns_sbox" ./mptcp_sockopt -6 lret=$? if [ $lret -ne 0 ]; then echo "FAIL: SOL_MPTCP getsockopt (ipv6)" 1>&2 + mptcp_lib_result_fail "sockopt v6" ret=$lret return fi + mptcp_lib_result_pass "sockopt v6" } run_tests() @@ -310,10 +321,12 @@ do_tcpinq_test() if [ $lret -ne 0 ];then ret=$lret echo "FAIL: mptcp_inq $@" 1>&2 + mptcp_lib_result_fail "TCP_INQ: $*" return $lret fi echo "PASS: TCP_INQ cmsg/ioctl $@" + mptcp_lib_result_pass "TCP_INQ: $*" return $lret } @@ -323,6 +336,7 @@ do_tcpinq_tests() if ! mptcp_lib_kallsyms_has "mptcp_ioctl$"; then echo "INFO: TCP_INQ not supported: SKIP" + mptcp_lib_result_skip "TCP_INQ" return fi @@ -367,4 +381,6 @@ if [ $ret -eq 0 ];then fi do_tcpinq_tests + +mptcp_lib_result_print_all_tap exit $ret diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh index d02e0d63a8f9..8f4ff123a7eb 100755 --- a/tools/testing/selftests/net/mptcp/pm_netlink.sh +++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh @@ -58,16 +58,19 @@ check() local out=`$cmd 2>$err` local cmd_ret=$? - printf "%-50s %s" "$msg" + printf "%-50s" "$msg" if [ $cmd_ret -ne 0 ]; then echo "[FAIL] command execution '$cmd' stderr " cat $err + mptcp_lib_result_fail "${msg} # error ${cmd_ret}" ret=1 elif [ "$out" = "$expected" ]; then echo "[ OK ]" + mptcp_lib_result_pass "${msg}" else echo -n "[FAIL] " echo "expected '$expected' got '$out'" + mptcp_lib_result_fail "${msg} # different output" ret=1 fi } @@ -96,7 +99,7 @@ check "ip netns exec $ns1 ./pm_nl_ctl dump" \ "id 1 flags 10.0.1.1 id 3 flags signal,backup 10.0.1.3" "dump addrs after del" -ip netns exec $ns1 ./pm_nl_ctl add 10.0.1.3 +ip netns exec $ns1 ./pm_nl_ctl add 10.0.1.3 2>/dev/null check "ip netns exec $ns1 ./pm_nl_ctl get 4" "" "duplicate addr" ip netns exec $ns1 ./pm_nl_ctl add 10.0.1.4 flags signal @@ -124,10 +127,10 @@ id 8 flags signal 10.0.1.8" "id limit" ip netns exec $ns1 ./pm_nl_ctl flush check "ip netns exec $ns1 ./pm_nl_ctl dump" "" "flush addrs" -ip netns exec $ns1 ./pm_nl_ctl limits 9 1 +ip netns exec $ns1 ./pm_nl_ctl limits 9 1 2>/dev/null check "ip netns exec $ns1 ./pm_nl_ctl limits" "$default_limits" "rcv addrs above hard limit" -ip netns exec $ns1 ./pm_nl_ctl limits 1 9 +ip netns exec $ns1 ./pm_nl_ctl limits 1 9 2>/dev/null check "ip netns exec $ns1 ./pm_nl_ctl limits" "$default_limits" "subflows above hard limit" ip netns exec $ns1 ./pm_nl_ctl limits 8 8 @@ -193,4 +196,5 @@ subflow 10.0.1.1" " (nofullmesh)" subflow,backup,fullmesh 10.0.1.1" " (backup,fullmesh)" fi +mptcp_lib_result_print_all_tap exit $ret diff --git a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c index 1887bd61bd9a..49369c4a5f26 100644 --- a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c +++ b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c @@ -66,20 +66,25 @@ static int init_genl_req(char *data, int family, int cmd, int version) return off; } -static void nl_error(struct nlmsghdr *nh) +static int nl_error(struct nlmsghdr *nh) { struct nlmsgerr *err = (struct nlmsgerr *)NLMSG_DATA(nh); int len = nh->nlmsg_len - sizeof(*nh); uint32_t off; - if (len < sizeof(struct nlmsgerr)) + if (len < sizeof(struct nlmsgerr)) { error(1, 0, "netlink error message truncated %d min %ld", len, sizeof(struct nlmsgerr)); + return -1; + } - if (!err->error) { + if (err->error) { /* check messages from kernel */ struct rtattr *attrs = (struct rtattr *)NLMSG_DATA(nh); + fprintf(stderr, "netlink error %d (%s)\n", + err->error, strerror(-err->error)); + while (RTA_OK(attrs, len)) { if (attrs->rta_type == NLMSGERR_ATTR_MSG) fprintf(stderr, "netlink ext ack msg: %s\n", @@ -91,9 +96,10 @@ static void nl_error(struct nlmsghdr *nh) } attrs = RTA_NEXT(attrs, len); } - } else { - fprintf(stderr, "netlink error %d", err->error); + return -1; } + + return 0; } static int capture_events(int fd, int event_group) @@ -198,7 +204,7 @@ static int capture_events(int fd, int event_group) return 0; } -/* do a netlink command and, if max > 0, fetch the reply */ +/* do a netlink command and, if max > 0, fetch the reply ; nh's size >1024B */ static int do_nl_req(int fd, struct nlmsghdr *nh, int len, int max) { struct sockaddr_nl nladdr = { .nl_family = AF_NETLINK }; @@ -207,12 +213,16 @@ static int do_nl_req(int fd, struct nlmsghdr *nh, int len, int max) int rem, ret; int err = 0; + /* If no expected answer, ask for an ACK to look for errors if any */ + if (max == 0) { + nh->nlmsg_flags |= NLM_F_ACK; + max = 1024; + } + nh->nlmsg_len = len; ret = sendto(fd, data, len, 0, (void *)&nladdr, sizeof(nladdr)); if (ret != len) error(1, errno, "send netlink: %uB != %uB\n", ret, len); - if (max == 0) - return 0; addr_len = sizeof(nladdr); rem = ret = recvfrom(fd, data, max, 0, (void *)&nladdr, &addr_len); @@ -221,10 +231,11 @@ static int do_nl_req(int fd, struct nlmsghdr *nh, int len, int max) /* Beware: the NLMSG_NEXT macro updates the 'rem' argument */ for (; NLMSG_OK(nh, rem); nh = NLMSG_NEXT(nh, rem)) { - if (nh->nlmsg_type == NLMSG_ERROR) { - nl_error(nh); + if (nh->nlmsg_type == NLMSG_DONE) + break; + + if (nh->nlmsg_type == NLMSG_ERROR && nl_error(nh)) err = 1; - } } if (err) error(1, 0, "bailing out due to netlink error[s]"); diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index 36a3c9d92e20..ce9203b817f8 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -261,6 +261,7 @@ run_test() printf "%-60s" "$msg" do_transfer $small $large $time lret=$? + mptcp_lib_result_code "${lret}" "${msg}" if [ $lret -ne 0 ]; then ret=$lret [ $bail -eq 0 ] || exit $ret @@ -269,6 +270,7 @@ run_test() printf "%-60s" "$msg - reverse direction" do_transfer $large $small $time lret=$? + mptcp_lib_result_code "${lret}" "${msg}" if [ $lret -ne 0 ]; then ret=$lret [ $bail -eq 0 ] || exit $ret @@ -305,4 +307,6 @@ run_test 10 10 1 50 "balanced bwidth with unbalanced delay" run_test 30 10 0 0 "unbalanced bwidth" run_test 30 10 1 50 "unbalanced bwidth with unbalanced delay" run_test 30 10 50 1 "unbalanced bwidth with opposed, unbalanced delay" + +mptcp_lib_result_print_all_tap exit $ret diff --git a/tools/testing/selftests/net/mptcp/userspace_pm.sh b/tools/testing/selftests/net/mptcp/userspace_pm.sh index b180133a30af..b25a3e33eb25 100755 --- a/tools/testing/selftests/net/mptcp/userspace_pm.sh +++ b/tools/testing/selftests/net/mptcp/userspace_pm.sh @@ -1,6 +1,13 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +# Double quotes to prevent globbing and word splitting is recommended in new +# code but we accept it. +#shellcheck disable=SC2086 + +# Some variables are used below but indirectly, see check_expected_one() +#shellcheck disable=SC2034 + . "$(dirname "${0}")/mptcp_lib.sh" mptcp_lib_check_mptcp @@ -11,8 +18,7 @@ if ! mptcp_lib_has_file '/proc/sys/net/mptcp/pm_type'; then exit ${KSFT_SKIP} fi -ip -Version > /dev/null 2>&1 -if [ $? -ne 0 ];then +if ! ip -Version &> /dev/null; then echo "SKIP: Cannot not run test without ip tool" exit ${KSFT_SKIP} fi @@ -52,10 +58,54 @@ sec=$(date +%s) rndh=$(printf %x "$sec")-$(mktemp -u XXXXXX) ns1="ns1-$rndh" ns2="ns2-$rndh" +ret=0 +test_name="" + +_printf() { + stdbuf -o0 -e0 printf "${@}" +} print_title() { - stdbuf -o0 -e0 printf "INFO: %s\n" "${1}" + _printf "INFO: %s\n" "${1}" +} + +# $1: test name +print_test() +{ + test_name="${1}" + + _printf "%-63s" "${test_name}" +} + +print_results() +{ + _printf "[%s]\n" "${1}" +} + +test_pass() +{ + print_results " OK " + mptcp_lib_result_pass "${test_name}" +} + +test_skip() +{ + print_results "SKIP" + mptcp_lib_result_skip "${test_name}" +} + +# $1: msg +test_fail() +{ + print_results "FAIL" + ret=1 + + if [ -n "${1}" ]; then + _printf "\t%s\n" "${1}" + fi + + mptcp_lib_result_fail "${test_name}" } kill_wait() @@ -67,6 +117,8 @@ kill_wait() wait $1 2>/dev/null } +# This function is used in the cleanup trap +#shellcheck disable=SC2317 cleanup() { print_title "Cleanup" @@ -86,7 +138,7 @@ cleanup() rm -rf $file $client_evts $server_evts - stdbuf -o0 -e0 printf "Done\n" + _printf "Done\n" } trap cleanup EXIT @@ -118,7 +170,8 @@ ip -net "$ns2" addr add dead:beef:2::2/64 dev ns2eth1 nodad ip -net "$ns2" link set ns2eth1 up print_title "Init" -stdbuf -o0 -e0 printf "Created network namespaces ns1, ns2 \t\t\t[OK]\n" +print_test "Created network namespaces ns1, ns2" +test_pass make_file() { @@ -203,16 +256,14 @@ make_connection() server_serverside=$(grep "type:1," "$server_evts" | sed --unbuffered -n 's/.*\(server_side:\)\([[:digit:]]*\).*$/\2/p;q') - stdbuf -o0 -e0 printf "Established IP%s MPTCP Connection ns2 => ns1 \t\t" $is_v6 + print_test "Established IP${is_v6} MPTCP Connection ns2 => ns1" if [ "$client_token" != "" ] && [ "$server_token" != "" ] && [ "$client_serverside" = 0 ] && [ "$server_serverside" = 1 ] then - stdbuf -o0 -e0 printf "[OK]\n" + test_pass else - stdbuf -o0 -e0 printf "[FAIL]\n" - stdbuf -o0 -e0 printf "\tExpected tokens (c:%s - s:%s) and server (c:%d - s:%d)\n" \ - "${client_token}" "${server_token}" \ - "${client_serverside}" "${server_serverside}" + test_fail "Expected tokens (c:${client_token} - s:${server_token}) and server (c:${client_serverside} - s:${server_serverside})" + mptcp_lib_result_print_all_tap exit 1 fi @@ -246,10 +297,10 @@ check_expected_one() if [ "${prev_ret}" = "0" ] then - stdbuf -o0 -e0 printf "[FAIL]\n" + test_fail fi - stdbuf -o0 -e0 printf "\tExpected value for '%s': '%s', got '%s'.\n" \ + _printf "\tExpected value for '%s': '%s', got '%s'.\n" \ "${var}" "${!exp}" "${!var}" return 1 } @@ -257,21 +308,21 @@ check_expected_one() # $@: all var names to check check_expected() { - local ret=0 + local rc=0 local var for var in "${@}" do - check_expected_one "${var}" "${ret}" || ret=1 + check_expected_one "${var}" "${rc}" || rc=1 done - if [ ${ret} -eq 0 ] + if [ ${rc} -eq 0 ] then - stdbuf -o0 -e0 printf "[OK]\n" + test_pass return 0 fi - exit 1 + return 1 } verify_announce_event() @@ -317,21 +368,20 @@ test_announce() local type type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") - stdbuf -o0 -e0 printf "ADD_ADDR 10.0.2.2 (ns2) => ns1, invalid token \t\t" + print_test "ADD_ADDR 10.0.2.2 (ns2) => ns1, invalid token" if [ "$type" = "" ] then - stdbuf -o0 -e0 printf "[OK]\n" + test_pass else - stdbuf -o0 -e0 printf "[FAIL]\n\ttype defined: %s\n" "${type}" - exit 1 + test_fail "type defined: ${type}" fi # ADD_ADDR from the client to server machine reusing the subflow port :>"$server_evts" ip netns exec "$ns2"\ ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id $client_addr_id dev\ - ns2eth1 > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.2 (ns2) => ns1, reuse port \t\t" $client_addr_id + ns2eth1 + print_test "ADD_ADDR id:${client_addr_id} 10.0.2.2 (ns2) => ns1, reuse port" sleep 0.5 verify_announce_event $server_evts $ANNOUNCED $server4_token "10.0.2.2" $client_addr_id \ "$client4_port" @@ -339,8 +389,8 @@ test_announce() # ADD_ADDR6 from the client to server machine reusing the subflow port :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann\ - dead:beef:2::2 token "$client6_token" id $client_addr_id dev ns2eth1 > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR6 id:%d dead:beef:2::2 (ns2) => ns1, reuse port\t\t" $client_addr_id + dead:beef:2::2 token "$client6_token" id $client_addr_id dev ns2eth1 + print_test "ADD_ADDR6 id:${client_addr_id} dead:beef:2::2 (ns2) => ns1, reuse port" sleep 0.5 verify_announce_event "$server_evts" "$ANNOUNCED" "$server6_token" "dead:beef:2::2"\ "$client_addr_id" "$client6_port" "v6" @@ -349,8 +399,8 @@ test_announce() :>"$server_evts" client_addr_id=$((client_addr_id+1)) ip netns exec "$ns2" ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id\ - $client_addr_id dev ns2eth1 port $new4_port > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.2 (ns2) => ns1, new port \t\t\t" $client_addr_id + $client_addr_id dev ns2eth1 port $new4_port + print_test "ADD_ADDR id:${client_addr_id} 10.0.2.2 (ns2) => ns1, new port" sleep 0.5 verify_announce_event "$server_evts" "$ANNOUNCED" "$server4_token" "10.0.2.2"\ "$client_addr_id" "$new4_port" @@ -360,8 +410,8 @@ test_announce() # ADD_ADDR from the server to client machine reusing the subflow port ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ - $server_addr_id dev ns1eth2 > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.1 (ns1) => ns2, reuse port \t\t" $server_addr_id + $server_addr_id dev ns1eth2 + print_test "ADD_ADDR id:${server_addr_id} 10.0.2.1 (ns1) => ns2, reuse port" sleep 0.5 verify_announce_event "$client_evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ "$server_addr_id" "$app4_port" @@ -369,8 +419,8 @@ test_announce() # ADD_ADDR6 from the server to client machine reusing the subflow port :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann dead:beef:2::1 token "$server6_token" id\ - $server_addr_id dev ns1eth2 > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR6 id:%d dead:beef:2::1 (ns1) => ns2, reuse port\t\t" $server_addr_id + $server_addr_id dev ns1eth2 + print_test "ADD_ADDR6 id:${server_addr_id} dead:beef:2::1 (ns1) => ns2, reuse port" sleep 0.5 verify_announce_event "$client_evts" "$ANNOUNCED" "$client6_token" "dead:beef:2::1"\ "$server_addr_id" "$app6_port" "v6" @@ -379,8 +429,8 @@ test_announce() :>"$client_evts" server_addr_id=$((server_addr_id+1)) ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ - $server_addr_id dev ns1eth2 port $new4_port > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.1 (ns1) => ns2, new port \t\t\t" $server_addr_id + $server_addr_id dev ns1eth2 port $new4_port + print_test "ADD_ADDR id:${server_addr_id} 10.0.2.1 (ns1) => ns2, new port" sleep 0.5 verify_announce_event "$client_evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ "$server_addr_id" "$new4_port" @@ -414,39 +464,34 @@ test_remove() local invalid_token=$(( client4_token - 1 )) ip netns exec "$ns2" ./pm_nl_ctl rem token $invalid_token id\ $client_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1, invalid token \t"\ - $client_addr_id + print_test "RM_ADDR id:${client_addr_id} ns2 => ns1, invalid token" local type type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") if [ "$type" = "" ] then - stdbuf -o0 -e0 printf "[OK]\n" + test_pass else - stdbuf -o0 -e0 printf "[FAIL]\n" - exit 1 + test_fail fi # RM_ADDR using an invalid addr id should result in no action local invalid_id=$(( client_addr_id + 1 )) ip netns exec "$ns2" ./pm_nl_ctl rem token "$client4_token" id\ $invalid_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1, invalid id \t"\ - $invalid_id + print_test "RM_ADDR id:${invalid_id} ns2 => ns1, invalid id" type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") if [ "$type" = "" ] then - stdbuf -o0 -e0 printf "[OK]\n" + test_pass else - stdbuf -o0 -e0 printf "[FAIL]\n" - exit 1 + test_fail fi # RM_ADDR from the client to server machine :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl rem token "$client4_token" id\ - $client_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1 \t"\ - $client_addr_id + $client_addr_id + print_test "RM_ADDR id:${client_addr_id} ns2 => ns1" sleep 0.5 verify_remove_event "$server_evts" "$REMOVED" "$server4_token" "$client_addr_id" @@ -454,18 +499,16 @@ test_remove() :>"$server_evts" client_addr_id=$(( client_addr_id - 1 )) ip netns exec "$ns2" ./pm_nl_ctl rem token "$client4_token" id\ - $client_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1 \t"\ - $client_addr_id + $client_addr_id + print_test "RM_ADDR id:${client_addr_id} ns2 => ns1" sleep 0.5 verify_remove_event "$server_evts" "$REMOVED" "$server4_token" "$client_addr_id" # RM_ADDR6 from the client to server machine :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl rem token "$client6_token" id\ - $client_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR6 id:%d ns2 => ns1 \t"\ - $client_addr_id + $client_addr_id + print_test "RM_ADDR6 id:${client_addr_id} ns2 => ns1" sleep 0.5 verify_remove_event "$server_evts" "$REMOVED" "$server6_token" "$client_addr_id" @@ -474,9 +517,8 @@ test_remove() # RM_ADDR from the server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem token "$server4_token" id\ - $server_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns1 => ns2 \t"\ - $server_addr_id + $server_addr_id + print_test "RM_ADDR id:${server_addr_id} ns1 => ns2" sleep 0.5 verify_remove_event "$client_evts" "$REMOVED" "$client4_token" "$server_addr_id" @@ -484,16 +526,16 @@ test_remove() :>"$client_evts" server_addr_id=$(( server_addr_id - 1 )) ip netns exec "$ns1" ./pm_nl_ctl rem token "$server4_token" id\ - $server_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR id:%d ns1 => ns2 \t" $server_addr_id + $server_addr_id + print_test "RM_ADDR id:${server_addr_id} ns1 => ns2" sleep 0.5 verify_remove_event "$client_evts" "$REMOVED" "$client4_token" "$server_addr_id" # RM_ADDR6 from the server to client machine :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl rem token "$server6_token" id\ - $server_addr_id > /dev/null 2>&1 - stdbuf -o0 -e0 printf "RM_ADDR6 id:%d ns1 => ns2 \t" $server_addr_id + $server_addr_id + print_test "RM_ADDR6 id:${server_addr_id} ns1 => ns2" sleep 0.5 verify_remove_event "$client_evts" "$REMOVED" "$client6_token" "$server_addr_id" } @@ -520,25 +562,24 @@ verify_subflow_events() local dport local locid local remid + local info + + info="${e_saddr} (${e_from}) => ${e_daddr} (${e_to})" if [ "$e_type" = "$SUB_ESTABLISHED" ] then if [ "$e_family" = "$AF_INET6" ] then - stdbuf -o0 -e0 printf "CREATE_SUBFLOW6 %s (%s) => %s (%s) "\ - "$e_saddr" "$e_from" "$e_daddr" "$e_to" + print_test "CREATE_SUBFLOW6 ${info}" else - stdbuf -o0 -e0 printf "CREATE_SUBFLOW %s (%s) => %s (%s) \t"\ - "$e_saddr" "$e_from" "$e_daddr" "$e_to" + print_test "CREATE_SUBFLOW ${info}" fi else if [ "$e_family" = "$AF_INET6" ] then - stdbuf -o0 -e0 printf "DESTROY_SUBFLOW6 %s (%s) => %s (%s) "\ - "$e_saddr" "$e_from" "$e_daddr" "$e_to" + print_test "DESTROY_SUBFLOW6 ${info}" else - stdbuf -o0 -e0 printf "DESTROY_SUBFLOW %s (%s) => %s (%s) \t"\ - "$e_saddr" "$e_from" "$e_daddr" "$e_to" + print_test "DESTROY_SUBFLOW ${info}" fi fi @@ -569,18 +610,18 @@ test_subflows() # Attempt to add a listener at 10.0.2.2:<subflow-port> ip netns exec "$ns2" ./pm_nl_ctl listen 10.0.2.2\ - "$client4_port" > /dev/null 2>&1 & + "$client4_port" & local listener_pid=$! # ADD_ADDR from client to server machine reusing the subflow port ip netns exec "$ns2" ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id\ - $client_addr_id > /dev/null 2>&1 + $client_addr_id sleep 0.5 # CREATE_SUBFLOW from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2\ - rport "$client4_port" token "$server4_token" > /dev/null 2>&1 + rport "$client4_port" token "$server4_token" sleep 0.5 verify_subflow_events $server_evts $SUB_ESTABLISHED $server4_token $AF_INET "10.0.2.1" \ "10.0.2.2" "$client4_port" "23" "$client_addr_id" "ns1" "ns2" @@ -594,31 +635,31 @@ test_subflows() # DESTROY_SUBFLOW from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip 10.0.2.1 lport "$sport" rip 10.0.2.2 rport\ - "$client4_port" token "$server4_token" > /dev/null 2>&1 + "$client4_port" token "$server4_token" sleep 0.5 verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ "10.0.2.2" "$client4_port" "23" "$client_addr_id" "ns1" "ns2" # RM_ADDR from client to server machine ip netns exec "$ns2" ./pm_nl_ctl rem id $client_addr_id token\ - "$client4_token" > /dev/null 2>&1 + "$client4_token" sleep 0.5 # Attempt to add a listener at dead:beef:2::2:<subflow-port> ip netns exec "$ns2" ./pm_nl_ctl listen dead:beef:2::2\ - "$client6_port" > /dev/null 2>&1 & + "$client6_port" & listener_pid=$! # ADD_ADDR6 from client to server machine reusing the subflow port :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann dead:beef:2::2 token "$client6_token" id\ - $client_addr_id > /dev/null 2>&1 + $client_addr_id sleep 0.5 # CREATE_SUBFLOW6 from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip dead:beef:2::1 lid 23 rip\ - dead:beef:2::2 rport "$client6_port" token "$server6_token" > /dev/null 2>&1 + dead:beef:2::2 rport "$client6_port" token "$server6_token" sleep 0.5 verify_subflow_events "$server_evts" "$SUB_ESTABLISHED" "$server6_token" "$AF_INET6"\ "dead:beef:2::1" "dead:beef:2::2" "$client6_port" "23"\ @@ -632,7 +673,7 @@ test_subflows() # DESTROY_SUBFLOW6 from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip dead:beef:2::1 lport "$sport" rip\ - dead:beef:2::2 rport "$client6_port" token "$server6_token" > /dev/null 2>&1 + dead:beef:2::2 rport "$client6_port" token "$server6_token" sleep 0.5 verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server6_token" "$AF_INET6"\ "dead:beef:2::1" "dead:beef:2::2" "$client6_port" "23"\ @@ -640,24 +681,24 @@ test_subflows() # RM_ADDR from client to server machine ip netns exec "$ns2" ./pm_nl_ctl rem id $client_addr_id token\ - "$client6_token" > /dev/null 2>&1 + "$client6_token" sleep 0.5 # Attempt to add a listener at 10.0.2.2:<new-port> ip netns exec "$ns2" ./pm_nl_ctl listen 10.0.2.2\ - $new4_port > /dev/null 2>&1 & + $new4_port & listener_pid=$! # ADD_ADDR from client to server machine using a new port :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id\ - $client_addr_id port $new4_port > /dev/null 2>&1 + $client_addr_id port $new4_port sleep 0.5 # CREATE_SUBFLOW from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2 rport\ - $new4_port token "$server4_token" > /dev/null 2>&1 + $new4_port token "$server4_token" sleep 0.5 verify_subflow_events "$server_evts" "$SUB_ESTABLISHED" "$server4_token" "$AF_INET"\ "10.0.2.1" "10.0.2.2" "$new4_port" "23"\ @@ -671,32 +712,32 @@ test_subflows() # DESTROY_SUBFLOW from server to client machine :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip 10.0.2.1 lport "$sport" rip 10.0.2.2 rport\ - $new4_port token "$server4_token" > /dev/null 2>&1 + $new4_port token "$server4_token" sleep 0.5 verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ "10.0.2.2" "$new4_port" "23" "$client_addr_id" "ns1" "ns2" # RM_ADDR from client to server machine ip netns exec "$ns2" ./pm_nl_ctl rem id $client_addr_id token\ - "$client4_token" > /dev/null 2>&1 + "$client4_token" # Capture events on the network namespace running the client :>"$client_evts" # Attempt to add a listener at 10.0.2.1:<subflow-port> ip netns exec "$ns1" ./pm_nl_ctl listen 10.0.2.1\ - $app4_port > /dev/null 2>&1 & + $app4_port & listener_pid=$! # ADD_ADDR from server to client machine reusing the subflow port ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ - $server_addr_id > /dev/null 2>&1 + $server_addr_id sleep 0.5 # CREATE_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ - $app4_port token "$client4_token" > /dev/null 2>&1 + $app4_port token "$client4_token" sleep 0.5 verify_subflow_events $client_evts $SUB_ESTABLISHED $client4_token $AF_INET "10.0.2.2"\ "10.0.2.1" "$app4_port" "23" "$server_addr_id" "ns2" "ns1" @@ -709,31 +750,31 @@ test_subflows() # DESTROY_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ - $app4_port token "$client4_token" > /dev/null 2>&1 + $app4_port token "$client4_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ "10.0.2.1" "$app4_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR from server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ - "$server4_token" > /dev/null 2>&1 + "$server4_token" sleep 0.5 # Attempt to add a listener at dead:beef:2::1:<subflow-port> ip netns exec "$ns1" ./pm_nl_ctl listen dead:beef:2::1\ - $app6_port > /dev/null 2>&1 & + $app6_port & listener_pid=$! # ADD_ADDR6 from server to client machine reusing the subflow port :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann dead:beef:2::1 token "$server6_token" id\ - $server_addr_id > /dev/null 2>&1 + $server_addr_id sleep 0.5 # CREATE_SUBFLOW6 from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip dead:beef:2::2 lid 23 rip\ - dead:beef:2::1 rport $app6_port token "$client6_token" > /dev/null 2>&1 + dead:beef:2::1 rport $app6_port token "$client6_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client6_token"\ "$AF_INET6" "dead:beef:2::2"\ @@ -748,31 +789,31 @@ test_subflows() # DESTROY_SUBFLOW6 from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip dead:beef:2::2 lport "$sport" rip\ - dead:beef:2::1 rport $app6_port token "$client6_token" > /dev/null 2>&1 + dead:beef:2::1 rport $app6_port token "$client6_token" sleep 0.5 verify_subflow_events $client_evts $SUB_CLOSED $client6_token $AF_INET6 "dead:beef:2::2"\ "dead:beef:2::1" "$app6_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR6 from server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ - "$server6_token" > /dev/null 2>&1 + "$server6_token" sleep 0.5 # Attempt to add a listener at 10.0.2.1:<new-port> ip netns exec "$ns1" ./pm_nl_ctl listen 10.0.2.1\ - $new4_port > /dev/null 2>&1 & + $new4_port & listener_pid=$! # ADD_ADDR from server to client machine using a new port :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ - $server_addr_id port $new4_port > /dev/null 2>&1 + $server_addr_id port $new4_port sleep 0.5 # CREATE_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ - $new4_port token "$client4_token" > /dev/null 2>&1 + $new4_port token "$client4_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client4_token" "$AF_INET"\ "10.0.2.2" "10.0.2.1" "$new4_port" "23" "$server_addr_id" "ns2" "ns1" @@ -785,14 +826,14 @@ test_subflows() # DESTROY_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ - $new4_port token "$client4_token" > /dev/null 2>&1 + $new4_port token "$client4_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ "10.0.2.1" "$new4_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR from server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ - "$server4_token" > /dev/null 2>&1 + "$server4_token" } test_subflows_v4_v6_mix() @@ -801,15 +842,15 @@ test_subflows_v4_v6_mix() # Attempt to add a listener at 10.0.2.1:<subflow-port> ip netns exec "$ns1" ./pm_nl_ctl listen 10.0.2.1\ - $app6_port > /dev/null 2>&1 & + $app6_port & local listener_pid=$! # ADD_ADDR4 from server to client machine reusing the subflow port on # the established v6 connection :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server6_token" id\ - $server_addr_id dev ns1eth2 > /dev/null 2>&1 - stdbuf -o0 -e0 printf "ADD_ADDR4 id:%d 10.0.2.1 (ns1) => ns2, reuse port\t\t" $server_addr_id + $server_addr_id dev ns1eth2 + print_test "ADD_ADDR4 id:${server_addr_id} 10.0.2.1 (ns1) => ns2, reuse port" sleep 0.5 verify_announce_event "$client_evts" "$ANNOUNCED" "$client6_token" "10.0.2.1"\ "$server_addr_id" "$app6_port" @@ -817,7 +858,7 @@ test_subflows_v4_v6_mix() # CREATE_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ - $app6_port token "$client6_token" > /dev/null 2>&1 + $app6_port token "$client6_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client6_token"\ "$AF_INET" "10.0.2.2" "10.0.2.1" "$app6_port" "23"\ @@ -831,7 +872,7 @@ test_subflows_v4_v6_mix() # DESTROY_SUBFLOW from client to server machine :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ - $app6_port token "$client6_token" > /dev/null 2>&1 + $app6_port token "$client6_token" sleep 0.5 verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client6_token" \ "$AF_INET" "10.0.2.2" "10.0.2.1" "$app6_port" "23"\ @@ -839,7 +880,7 @@ test_subflows_v4_v6_mix() # RM_ADDR from server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ - "$server6_token" > /dev/null 2>&1 + "$server6_token" sleep 0.5 } @@ -854,25 +895,23 @@ test_prio() sleep 0.5 # Check TX - stdbuf -o0 -e0 printf "MP_PRIO TX \t" + print_test "MP_PRIO TX" count=$(ip netns exec "$ns2" nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}') [ -z "$count" ] && count=0 if [ $count != 1 ]; then - stdbuf -o0 -e0 printf "[FAIL]\n\tCount != 1: %d\n" "${count}" - exit 1 + test_fail "Count != 1: ${count}" else - stdbuf -o0 -e0 printf "[OK]\n" + test_pass fi # Check RX - stdbuf -o0 -e0 printf "MP_PRIO RX \t" + print_test "MP_PRIO RX" count=$(ip netns exec "$ns1" nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}') [ -z "$count" ] && count=0 if [ $count != 1 ]; then - stdbuf -o0 -e0 printf "[FAIL]\n\tCount != 1: %d\n" "${count}" - exit 1 + test_fail "Count != 1: ${count}" else - stdbuf -o0 -e0 printf "[OK]\n" + test_pass fi } @@ -889,11 +928,9 @@ verify_listener_events() local sport if [ $e_type = $LISTENER_CREATED ]; then - stdbuf -o0 -e0 printf "CREATE_LISTENER %s:%s\t\t\t\t\t"\ - $e_saddr $e_sport + print_test "CREATE_LISTENER $e_saddr:$e_sport" elif [ $e_type = $LISTENER_CLOSED ]; then - stdbuf -o0 -e0 printf "CLOSE_LISTENER %s:%s\t\t\t\t\t"\ - $e_saddr $e_sport + print_test "CLOSE_LISTENER $e_saddr:$e_sport" fi type=$(grep "type:$e_type," $evt | @@ -918,7 +955,8 @@ test_listener() print_title "Listener tests" if ! mptcp_lib_kallsyms_has "mptcp_event_pm_listener$"; then - stdbuf -o0 -e0 printf "LISTENER events \t[SKIP] Not supported\n" + print_test "LISTENER events" + test_skip return fi @@ -927,7 +965,7 @@ test_listener() # Attempt to add a listener at 10.0.2.2:<subflow-port> ip netns exec $ns2 ./pm_nl_ctl listen 10.0.2.2\ - $client4_port > /dev/null 2>&1 & + $client4_port & local listener_pid=$! sleep 0.5 @@ -935,12 +973,12 @@ test_listener() # ADD_ADDR from client to server machine reusing the subflow port ip netns exec $ns2 ./pm_nl_ctl ann 10.0.2.2 token $client4_token id\ - $client_addr_id > /dev/null 2>&1 + $client_addr_id sleep 0.5 # CREATE_SUBFLOW from server to client machine ip netns exec $ns1 ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2\ - rport $client4_port token $server4_token > /dev/null 2>&1 + rport $client4_port token $server4_token sleep 0.5 # Delete the listener from the client ns, if one was created @@ -961,4 +999,5 @@ test_subflows_v4_v6_mix test_prio test_listener -exit 0 +mptcp_lib_result_print_all_tap +exit ${ret} diff --git a/tools/testing/selftests/net/openvswitch/openvswitch.sh b/tools/testing/selftests/net/openvswitch/openvswitch.sh index 3117a4be0cd0..dced4f612a78 100755 --- a/tools/testing/selftests/net/openvswitch/openvswitch.sh +++ b/tools/testing/selftests/net/openvswitch/openvswitch.sh @@ -11,6 +11,10 @@ VERBOSE=0 TRACING=0 tests=" + arp_ping eth-arp: Basic arp ping between two NS + ct_connect_v4 ip4-ct-xon: Basic ipv4 tcp connection using ct + connect_v4 ip4-xon: Basic ipv4 ping between two NS + nat_connect_v4 ip4-nat-xon: Basic ipv4 tcp connection via NAT netlink_checks ovsnl: validate netlink attrs and settings upcall_interfaces ovs: test the upcall interfaces" @@ -127,6 +131,16 @@ ovs_add_netns_and_veths () { return 0 } +ovs_add_flow () { + info "Adding flow to DP: sbx:$1 br:$2 flow:$3 act:$4" + ovs_sbx "$1" python3 $ovs_base/ovs-dpctl.py add-flow "$2" "$3" "$4" + if [ $? -ne 0 ]; then + echo "Flow [ $3 : $4 ] failed" >> ${ovs_dir}/debug.log + return 1 + fi + return 0 +} + usage() { echo echo "$0 [OPTIONS] [TEST]..." @@ -141,6 +155,215 @@ usage() { exit 1 } +# arp_ping test +# - client has 1500 byte MTU +# - server has 1500 byte MTU +# - send ARP ping between two ns +test_arp_ping () { + + which arping >/dev/null 2>&1 || return $ksft_skip + + sbx_add "test_arp_ping" || return $? + + ovs_add_dp "test_arp_ping" arpping || return 1 + + info "create namespaces" + for ns in client server; do + ovs_add_netns_and_veths "test_arp_ping" "arpping" "$ns" \ + "${ns:0:1}0" "${ns:0:1}1" || return 1 + done + + # Setup client namespace + ip netns exec client ip addr add 172.31.110.10/24 dev c1 + ip netns exec client ip link set c1 up + HW_CLIENT=`ip netns exec client ip link show dev c1 | grep -E 'link/ether [0-9a-f:]+' | awk '{print $2;}'` + info "Client hwaddr: $HW_CLIENT" + + # Setup server namespace + ip netns exec server ip addr add 172.31.110.20/24 dev s1 + ip netns exec server ip link set s1 up + HW_SERVER=`ip netns exec server ip link show dev s1 | grep -E 'link/ether [0-9a-f:]+' | awk '{print $2;}'` + info "Server hwaddr: $HW_SERVER" + + ovs_add_flow "test_arp_ping" arpping \ + "in_port(1),eth(),eth_type(0x0806),arp(sip=172.31.110.10,tip=172.31.110.20,sha=$HW_CLIENT,tha=ff:ff:ff:ff:ff:ff)" '2' || return 1 + ovs_add_flow "test_arp_ping" arpping \ + "in_port(2),eth(),eth_type(0x0806),arp()" '1' || return 1 + + ovs_sbx "test_arp_ping" ip netns exec client arping -I c1 172.31.110.20 -c 1 || return 1 + + return 0 +} + +# ct_connect_v4 test +# - client has 1500 byte MTU +# - server has 1500 byte MTU +# - use ICMP to ping in each direction +# - only allow CT state stuff to pass through new in c -> s +test_ct_connect_v4 () { + + which nc >/dev/null 2>/dev/null || return $ksft_skip + + sbx_add "test_ct_connect_v4" || return $? + + ovs_add_dp "test_ct_connect_v4" ct4 || return 1 + info "create namespaces" + for ns in client server; do + ovs_add_netns_and_veths "test_ct_connect_v4" "ct4" "$ns" \ + "${ns:0:1}0" "${ns:0:1}1" || return 1 + done + + ip netns exec client ip addr add 172.31.110.10/24 dev c1 + ip netns exec client ip link set c1 up + ip netns exec server ip addr add 172.31.110.20/24 dev s1 + ip netns exec server ip link set s1 up + + # Add forwarding for ARP and ip packets - completely wildcarded + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'ct_state(-trk),eth(),eth_type(0x0800),ipv4()' \ + 'ct(commit),recirc(0x1)' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'recirc_id(0x1),ct_state(+trk+new),in_port(1),eth(),eth_type(0x0800),ipv4(src=172.31.110.10)' \ + '2' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'recirc_id(0x1),ct_state(+trk+est),in_port(1),eth(),eth_type(0x0800),ipv4(src=172.31.110.10)' \ + '2' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'recirc_id(0x1),ct_state(+trk+est),in_port(2),eth(),eth_type(0x0800),ipv4(dst=172.31.110.10)' \ + '1' || return 1 + ovs_add_flow "test_ct_connect_v4" ct4 \ + 'recirc_id(0x1),ct_state(+trk+inv),eth(),eth_type(0x0800),ipv4()' 'drop' || \ + return 1 + + # do a ping + ovs_sbx "test_ct_connect_v4" ip netns exec client ping 172.31.110.20 -c 3 || return 1 + + # create an echo server in 'server' + echo "server" | \ + ovs_netns_spawn_daemon "test_ct_connect_v4" "server" \ + nc -lvnp 4443 + ovs_sbx "test_ct_connect_v4" ip netns exec client nc -i 1 -zv 172.31.110.20 4443 || return 1 + + # Now test in the other direction (should fail) + echo "client" | \ + ovs_netns_spawn_daemon "test_ct_connect_v4" "client" \ + nc -lvnp 4443 + ovs_sbx "test_ct_connect_v4" ip netns exec client nc -i 1 -zv 172.31.110.10 4443 + if [ $? == 0 ]; then + info "ct connect to client was successful" + return 1 + fi + + info "done..." + return 0 +} + +# connect_v4 test +# - client has 1500 byte MTU +# - server has 1500 byte MTU +# - use ICMP to ping in each direction +test_connect_v4 () { + + sbx_add "test_connect_v4" || return $? + + ovs_add_dp "test_connect_v4" cv4 || return 1 + + info "create namespaces" + for ns in client server; do + ovs_add_netns_and_veths "test_connect_v4" "cv4" "$ns" \ + "${ns:0:1}0" "${ns:0:1}1" || return 1 + done + + + ip netns exec client ip addr add 172.31.110.10/24 dev c1 + ip netns exec client ip link set c1 up + ip netns exec server ip addr add 172.31.110.20/24 dev s1 + ip netns exec server ip link set s1 up + + # Add forwarding for ARP and ip packets - completely wildcarded + ovs_add_flow "test_connect_v4" cv4 \ + 'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1 + ovs_add_flow "test_connect_v4" cv4 \ + 'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1 + ovs_add_flow "test_connect_v4" cv4 \ + 'in_port(1),eth(),eth_type(0x0800),ipv4(src=172.31.110.10)' '2' || return 1 + ovs_add_flow "test_connect_v4" cv4 \ + 'in_port(2),eth(),eth_type(0x0800),ipv4(src=172.31.110.20)' '1' || return 1 + + # do a ping + ovs_sbx "test_connect_v4" ip netns exec client ping 172.31.110.20 -c 3 || return 1 + + info "done..." + return 0 +} + +# nat_connect_v4 test +# - client has 1500 byte MTU +# - server has 1500 byte MTU +# - use ICMP to ping in each direction +# - only allow CT state stuff to pass through new in c -> s +test_nat_connect_v4 () { + which nc >/dev/null 2>/dev/null || return $ksft_skip + + sbx_add "test_nat_connect_v4" || return $? + + ovs_add_dp "test_nat_connect_v4" nat4 || return 1 + info "create namespaces" + for ns in client server; do + ovs_add_netns_and_veths "test_nat_connect_v4" "nat4" "$ns" \ + "${ns:0:1}0" "${ns:0:1}1" || return 1 + done + + ip netns exec client ip addr add 172.31.110.10/24 dev c1 + ip netns exec client ip link set c1 up + ip netns exec server ip addr add 172.31.110.20/24 dev s1 + ip netns exec server ip link set s1 up + + ip netns exec client ip route add default via 172.31.110.20 + + ovs_add_flow "test_nat_connect_v4" nat4 \ + 'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1 + ovs_add_flow "test_nat_connect_v4" nat4 \ + 'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1 + ovs_add_flow "test_nat_connect_v4" nat4 \ + "ct_state(-trk),in_port(1),eth(),eth_type(0x0800),ipv4(dst=192.168.0.20)" \ + "ct(commit,nat(dst=172.31.110.20)),recirc(0x1)" + ovs_add_flow "test_nat_connect_v4" nat4 \ + "ct_state(-trk),in_port(2),eth(),eth_type(0x0800),ipv4()" \ + "ct(commit,nat),recirc(0x2)" + + ovs_add_flow "test_nat_connect_v4" nat4 \ + "recirc_id(0x1),ct_state(+trk-inv),in_port(1),eth(),eth_type(0x0800),ipv4()" "2" + ovs_add_flow "test_nat_connect_v4" nat4 \ + "recirc_id(0x2),ct_state(+trk-inv),in_port(2),eth(),eth_type(0x0800),ipv4()" "1" + + # do a ping + ovs_sbx "test_nat_connect_v4" ip netns exec client ping 192.168.0.20 -c 3 || return 1 + + # create an echo server in 'server' + echo "server" | \ + ovs_netns_spawn_daemon "test_nat_connect_v4" "server" \ + nc -lvnp 4443 + ovs_sbx "test_nat_connect_v4" ip netns exec client nc -i 1 -zv 192.168.0.20 4443 || return 1 + + # Now test in the other direction (should fail) + echo "client" | \ + ovs_netns_spawn_daemon "test_nat_connect_v4" "client" \ + nc -lvnp 4443 + ovs_sbx "test_nat_connect_v4" ip netns exec client nc -i 1 -zv 172.31.110.10 4443 + if [ $? == 0 ]; then + info "connect to client was successful" + return 1 + fi + + info "done..." + return 0 +} + # netlink_validation # - Create a dp # - check no warning with "old version" simulation diff --git a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py index 1c8b36bc15d4..fbdac15e3134 100644 --- a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py +++ b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py @@ -9,9 +9,12 @@ import errno import ipaddress import logging import multiprocessing +import re import struct import sys import time +import types +import uuid try: from pyroute2 import NDB @@ -59,24 +62,207 @@ def macstr(mac): return outstr -def convert_mac(mac_str, mask=False): - if mac_str is None or mac_str == "": - mac_str = "00:00:00:00:00:00" - if mask is True and mac_str != "00:00:00:00:00:00": - mac_str = "FF:FF:FF:FF:FF:FF" - mac_split = mac_str.split(":") - ret = bytearray([int(i, 16) for i in mac_split]) - return bytes(ret) +def strcspn(str1, str2): + tot = 0 + for char in str1: + if str2.find(char) != -1: + return tot + tot += 1 + return tot -def convert_ipv4(ip, mask=False): - if ip is None: - ip = 0 - if mask is True: - if ip != 0: - ip = int(ipaddress.IPv4Address(ip)) & 0xFFFFFFFF +def strspn(str1, str2): + tot = 0 + for char in str1: + if str2.find(char) == -1: + return tot + tot += 1 + return tot - return int(ipaddress.IPv4Address(ip)) + +def intparse(statestr, defmask="0xffffffff"): + totalparse = strspn(statestr, "0123456789abcdefABCDEFx/") + # scan until "/" + count = strspn(statestr, "x0123456789abcdefABCDEF") + + firstnum = statestr[:count] + if firstnum[-1] == "/": + firstnum = firstnum[:-1] + k = int(firstnum, 0) + + m = None + if defmask is not None: + secondnum = defmask + if statestr[count] == "/": + secondnum = statestr[count + 1 :] # this is wrong... + m = int(secondnum, 0) + + return statestr[totalparse + 1 :], k, m + + +def parse_flags(flag_str, flag_vals): + bitResult = 0 + maskResult = 0 + + if len(flag_str) == 0: + return flag_str, bitResult, maskResult + + if flag_str[0].isdigit(): + idx = 0 + while flag_str[idx].isdigit() or flag_str[idx] == "x": + idx += 1 + digits = flag_str[:idx] + flag_str = flag_str[idx:] + + bitResult = int(digits, 0) + maskResult = int(digits, 0) + + while len(flag_str) > 0 and (flag_str[0] == "+" or flag_str[0] == "-"): + if flag_str[0] == "+": + setFlag = True + elif flag_str[0] == "-": + setFlag = False + + flag_str = flag_str[1:] + + flag_len = 0 + while ( + flag_str[flag_len] != "+" + and flag_str[flag_len] != "-" + and flag_str[flag_len] != "," + and flag_str[flag_len] != ")" + ): + flag_len += 1 + + flag = flag_str[0:flag_len] + + if flag in flag_vals: + if maskResult & flag_vals[flag]: + raise KeyError( + "Flag %s set once, cannot be set in multiples" % flag + ) + + if setFlag: + bitResult |= flag_vals[flag] + + maskResult |= flag_vals[flag] + else: + raise KeyError("Missing flag value: %s" % flag) + + flag_str = flag_str[flag_len:] + + return flag_str, bitResult, maskResult + + +def parse_ct_state(statestr): + ct_flags = { + "new": 1 << 0, + "est": 1 << 1, + "rel": 1 << 2, + "rpl": 1 << 3, + "inv": 1 << 4, + "trk": 1 << 5, + "snat": 1 << 6, + "dnat": 1 << 7, + } + + return parse_flags(statestr, ct_flags) + + +def convert_mac(data): + def to_bytes(mac): + mac_split = mac.split(":") + ret = bytearray([int(i, 16) for i in mac_split]) + return bytes(ret) + + mac_str, _, mask_str = data.partition('/') + + if not mac_str: + mac_str = mask_str = "00:00:00:00:00:00" + elif not mask_str: + mask_str = "FF:FF:FF:FF:FF:FF" + + return to_bytes(mac_str), to_bytes(mask_str) + +def convert_ipv4(data): + ip, _, mask = data.partition('/') + + if not ip: + ip = mask = 0 + elif not mask: + mask = 0xFFFFFFFF + elif mask.isdigit(): + mask = (0xFFFFFFFF << (32 - int(mask))) & 0xFFFFFFFF + + return int(ipaddress.IPv4Address(ip)), int(ipaddress.IPv4Address(mask)) + +def convert_int(size): + def convert_int_sized(data): + value, _, mask = data.partition('/') + + if not value: + return 0, 0 + elif not mask: + return int(value, 0), pow(2, size) - 1 + else: + return int(value, 0), int(mask, 0) + + return convert_int_sized + +def parse_starts_block(block_str, scanstr, returnskipped, scanregex=False): + if scanregex: + m = re.search(scanstr, block_str) + if m is None: + if returnskipped: + return block_str + return False + if returnskipped: + block_str = block_str[len(m.group(0)) :] + return block_str + return True + + if block_str.startswith(scanstr): + if returnskipped: + block_str = block_str[len(scanstr) :] + else: + return True + + if returnskipped: + return block_str + + return False + + +def parse_extract_field( + block_str, fieldstr, scanfmt, convert, masked=False, defval=None +): + if fieldstr and not block_str.startswith(fieldstr): + return block_str, defval + + if fieldstr: + str_skiplen = len(fieldstr) + str_skipped = block_str[str_skiplen:] + if str_skiplen == 0: + return str_skipped, defval + else: + str_skiplen = 0 + str_skipped = block_str + + m = re.search(scanfmt, str_skipped) + if m is None: + raise ValueError("Bad fmt string") + + data = m.group(0) + if convert: + data = convert(m.group(0)) + + str_skipped = str_skipped[len(m.group(0)) :] + if masked: + if str_skipped[0] == "/": + raise ValueError("Masking support TBD...") + + str_skipped = str_skipped[strspn(str_skipped, ", ") :] + return str_skipped, data class ovs_dp_msg(genlmsg): @@ -278,6 +464,142 @@ class ovsactions(nla): return print_str + def parse(self, actstr): + while len(actstr) != 0: + parsed = False + if actstr.startswith("drop"): + # for now, drops have no explicit action, so we + # don't need to set any attributes. The final + # act of the processing chain will just drop the packet + return + + elif parse_starts_block(actstr, "^(\d+)", False, True): + actstr, output = parse_extract_field( + actstr, None, "(\d+)", lambda x: int(x), False, "0" + ) + self["attrs"].append(["OVS_ACTION_ATTR_OUTPUT", output]) + parsed = True + elif parse_starts_block(actstr, "recirc(", False): + actstr, recircid = parse_extract_field( + actstr, + "recirc(", + "([0-9a-fA-Fx]+)", + lambda x: int(x, 0), + False, + 0, + ) + self["attrs"].append(["OVS_ACTION_ATTR_RECIRC", recircid]) + parsed = True + + parse_flat_map = ( + ("ct_clear", "OVS_ACTION_ATTR_CT_CLEAR"), + ("pop_vlan", "OVS_ACTION_ATTR_POP_VLAN"), + ("pop_eth", "OVS_ACTION_ATTR_POP_ETH"), + ("pop_nsh", "OVS_ACTION_ATTR_POP_NSH"), + ) + + for flat_act in parse_flat_map: + if parse_starts_block(actstr, flat_act[0], False): + actstr += len(flat_act[0]) + self["attrs"].append([flat_act[1]]) + actstr = actstr[strspn(actstr, ", ") :] + parsed = True + + if parse_starts_block(actstr, "ct(", False): + actstr = actstr[len("ct(") :] + ctact = ovsactions.ctact() + + for scan in ( + ("commit", "OVS_CT_ATTR_COMMIT", None), + ("force_commit", "OVS_CT_ATTR_FORCE_COMMIT", None), + ("zone", "OVS_CT_ATTR_ZONE", int), + ("mark", "OVS_CT_ATTR_MARK", int), + ("helper", "OVS_CT_ATTR_HELPER", lambda x, y: str(x)), + ("timeout", "OVS_CT_ATTR_TIMEOUT", lambda x, y: str(x)), + ): + if actstr.startswith(scan[0]): + actstr = actstr[len(scan[0]) :] + if scan[2] is not None: + if actstr[0] != "=": + raise ValueError("Invalid ct attr") + actstr = actstr[1:] + pos = strcspn(actstr, ",)") + datum = scan[2](actstr[:pos], 0) + ctact["attrs"].append([scan[1], datum]) + actstr = actstr[pos:] + else: + ctact["attrs"].append([scan[1], None]) + actstr = actstr[strspn(actstr, ", ") :] + # it seems strange to put this here, but nat() is a complex + # sub-action and this lets it sit anywhere in the ct() action + if actstr.startswith("nat"): + actstr = actstr[3:] + natact = ovsactions.ctact.natattr() + + if actstr.startswith("("): + t = None + actstr = actstr[1:] + if actstr.startswith("src"): + t = "OVS_NAT_ATTR_SRC" + actstr = actstr[3:] + elif actstr.startswith("dst"): + t = "OVS_NAT_ATTR_DST" + actstr = actstr[3:] + + actstr, ip_block_min = parse_extract_field( + actstr, "=", "([0-9a-fA-F\.]+)", str, False + ) + actstr, ip_block_max = parse_extract_field( + actstr, "-", "([0-9a-fA-F\.]+)", str, False + ) + + actstr, proto_min = parse_extract_field( + actstr, ":", "(\d+)", int, False + ) + actstr, proto_max = parse_extract_field( + actstr, "-", "(\d+)", int, False + ) + + if t is not None: + natact["attrs"].append([t, None]) + + if ip_block_min is not None: + natact["attrs"].append( + ["OVS_NAT_ATTR_IP_MIN", ip_block_min] + ) + if ip_block_max is not None: + natact["attrs"].append( + ["OVS_NAT_ATTR_IP_MAX", ip_block_max] + ) + if proto_min is not None: + natact["attrs"].append( + ["OVS_NAT_ATTR_PROTO_MIN", proto_min] + ) + if proto_max is not None: + natact["attrs"].append( + ["OVS_NAT_ATTR_PROTO_MAX", proto_max] + ) + + for natscan in ( + ("persistent", "OVS_NAT_ATTR_PERSISTENT"), + ("hash", "OVS_NAT_ATTR_PROTO_HASH"), + ("random", "OVS_NAT_ATTR_PROTO_RANDOM"), + ): + if actstr.startswith(natscan[0]): + actstr = actstr[len(natscan[0]) :] + natact["attrs"].append([natscan[1], None]) + actstr = actstr[strspn(actstr, ", ") :] + + ctact["attrs"].append(["OVS_CT_ATTR_NAT", natact]) + actstr = actstr[strspn(actstr, ",) ") :] + + self["attrs"].append(["OVS_ACTION_ATTR_CT", ctact]) + parsed = True + + actstr = actstr[strspn(actstr, "), ") :] + if not parsed: + raise ValueError("Action str: '%s' not supported" % actstr) + class ovskey(nla): nla_flags = NLA_F_NESTED @@ -324,8 +646,10 @@ class ovskey(nla): ) fields_map = ( - ("src", "src", "%d", lambda x: int(x) if x is not None else 0), - ("dst", "dst", "%d", lambda x: int(x) if x is not None else 0), + ("src", "src", "%d", lambda x: int(x) if x else 0, + convert_int(16)), + ("dst", "dst", "%d", lambda x: int(x) if x else 0, + convert_int(16)), ) def __init__( @@ -347,6 +671,49 @@ class ovskey(nla): init=init, ) + def parse(self, flowstr, typeInst): + if not flowstr.startswith(self.proto_str): + return None, None + + k = typeInst() + m = typeInst() + + flowstr = flowstr[len(self.proto_str) :] + if flowstr.startswith("("): + flowstr = flowstr[1:] + + keybits = b"" + maskbits = b"" + for f in self.fields_map: + if flowstr.startswith(f[1]): + # the following assumes that the field looks + # something like 'field.' where '.' is a + # character that we don't exactly care about. + flowstr = flowstr[len(f[1]) + 1 :] + splitchar = 0 + for c in flowstr: + if c == "," or c == ")": + break + splitchar += 1 + data = flowstr[:splitchar] + flowstr = flowstr[splitchar:] + else: + data = "" + + if len(f) > 4: + k[f[0]], m[f[0]] = f[4](data) + else: + k[f[0]] = f[3](data) + m[f[0]] = f[3](data) + + flowstr = flowstr[strspn(flowstr, ", ") :] + if len(flowstr) == 0: + return flowstr, k, m + + flowstr = flowstr[strspn(flowstr, "), ") :] + + return flowstr, k, m + def dpstr(self, masked=None, more=False): outstr = self.proto_str + "(" first = False @@ -441,10 +808,14 @@ class ovskey(nla): int, convert_ipv4, ), - ("proto", "proto", "%d", lambda x: int(x) if x is not None else 0), - ("tos", "tos", "%d", lambda x: int(x) if x is not None else 0), - ("ttl", "ttl", "%d", lambda x: int(x) if x is not None else 0), - ("frag", "frag", "%d", lambda x: int(x) if x is not None else 0), + ("proto", "proto", "%d", lambda x: int(x) if x else 0, + convert_int(8)), + ("tos", "tos", "%d", lambda x: int(x) if x else 0, + convert_int(8)), + ("ttl", "ttl", "%d", lambda x: int(x) if x else 0, + convert_int(8)), + ("frag", "frag", "%d", lambda x: int(x) if x else 0, + convert_int(8)), ) def __init__( @@ -580,8 +951,8 @@ class ovskey(nla): ) fields_map = ( - ("type", "type", "%d", int), - ("code", "code", "%d", int), + ("type", "type", "%d", lambda x: int(x) if x else 0), + ("code", "code", "%d", lambda x: int(x) if x else 0), ) def __init__( @@ -646,7 +1017,7 @@ class ovskey(nla): int, convert_ipv4, ), - ("op", "op", "%d", lambda x: int(x) if x is not None else 0), + ("op", "op", "%d", lambda x: int(x) if x else 0), ( "sha", "sha", @@ -810,6 +1181,81 @@ class ovskey(nla): class ovs_key_mpls(nla): fields = (("lse", ">I"),) + def parse(self, flowstr, mask=None): + for field in ( + ("OVS_KEY_ATTR_PRIORITY", "skb_priority", intparse), + ("OVS_KEY_ATTR_SKB_MARK", "skb_mark", intparse), + ("OVS_KEY_ATTR_RECIRC_ID", "recirc_id", intparse), + ("OVS_KEY_ATTR_DP_HASH", "dp_hash", intparse), + ("OVS_KEY_ATTR_CT_STATE", "ct_state", parse_ct_state), + ("OVS_KEY_ATTR_CT_ZONE", "ct_zone", intparse), + ("OVS_KEY_ATTR_CT_MARK", "ct_mark", intparse), + ("OVS_KEY_ATTR_IN_PORT", "in_port", intparse), + ( + "OVS_KEY_ATTR_ETHERNET", + "eth", + ovskey.ethaddr, + ), + ( + "OVS_KEY_ATTR_ETHERTYPE", + "eth_type", + lambda x: intparse(x, "0xffff"), + ), + ( + "OVS_KEY_ATTR_IPV4", + "ipv4", + ovskey.ovs_key_ipv4, + ), + ( + "OVS_KEY_ATTR_IPV6", + "ipv6", + ovskey.ovs_key_ipv6, + ), + ( + "OVS_KEY_ATTR_ARP", + "arp", + ovskey.ovs_key_arp, + ), + ( + "OVS_KEY_ATTR_TCP", + "tcp", + ovskey.ovs_key_tcp, + ), + ( + "OVS_KEY_ATTR_UDP", + "udp", + ovskey.ovs_key_udp, + ), + ( + "OVS_KEY_ATTR_ICMP", + "icmp", + ovskey.ovs_key_icmp, + ), + ( + "OVS_KEY_ATTR_TCP_FLAGS", + "tcp_flags", + lambda x: parse_flags(x, None), + ), + ): + fld = field[1] + "(" + if not flowstr.startswith(fld): + continue + + if not isinstance(field[2], types.FunctionType): + nk = field[2]() + flowstr, k, m = nk.parse(flowstr, field[2]) + else: + flowstr = flowstr[len(fld) :] + flowstr, k, m = field[2](flowstr) + + if m and mask is not None: + mask["attrs"].append([field[0], m]) + self["attrs"].append([field[0], k]) + + flowstr = flowstr[strspn(flowstr, "),") :] + + return flowstr + def dpstr(self, mask=None, more=False): print_str = "" @@ -1358,11 +1804,92 @@ class OvsFlow(GenericNetlinkSocket): return print_str + def parse(self, flowstr, actstr, dpidx=0): + OVS_UFID_F_OMIT_KEY = 1 << 0 + OVS_UFID_F_OMIT_MASK = 1 << 1 + OVS_UFID_F_OMIT_ACTIONS = 1 << 2 + + self["cmd"] = 0 + self["version"] = 0 + self["reserved"] = 0 + self["dpifindex"] = 0 + + if flowstr.startswith("ufid:"): + count = 5 + while flowstr[count] != ",": + count += 1 + ufidstr = flowstr[5:count] + flowstr = flowstr[count + 1 :] + else: + ufidstr = str(uuid.uuid4()) + uuidRawObj = uuid.UUID(ufidstr).fields + + self["attrs"].append( + [ + "OVS_FLOW_ATTR_UFID", + [ + uuidRawObj[0], + uuidRawObj[1] << 16 | uuidRawObj[2], + uuidRawObj[3] << 24 + | uuidRawObj[4] << 16 + | uuidRawObj[5] & (0xFF << 32) >> 32, + uuidRawObj[5] & (0xFFFFFFFF), + ], + ] + ) + self["attrs"].append( + [ + "OVS_FLOW_ATTR_UFID_FLAGS", + int( + OVS_UFID_F_OMIT_KEY + | OVS_UFID_F_OMIT_MASK + | OVS_UFID_F_OMIT_ACTIONS + ), + ] + ) + + k = ovskey() + m = ovskey() + k.parse(flowstr, m) + self["attrs"].append(["OVS_FLOW_ATTR_KEY", k]) + self["attrs"].append(["OVS_FLOW_ATTR_MASK", m]) + + a = ovsactions() + a.parse(actstr) + self["attrs"].append(["OVS_FLOW_ATTR_ACTIONS", a]) + def __init__(self): GenericNetlinkSocket.__init__(self) self.bind(OVS_FLOW_FAMILY, OvsFlow.ovs_flow_msg) + def add_flow(self, dpifindex, flowmsg): + """ + Send a new flow message to the kernel. + + dpifindex should be a valid datapath obtained by calling + into the OvsDatapath lookup + + flowmsg is a flow object obtained by calling a dpparse + """ + + flowmsg["cmd"] = OVS_FLOW_CMD_NEW + flowmsg["version"] = OVS_DATAPATH_VERSION + flowmsg["reserved"] = 0 + flowmsg["dpifindex"] = dpifindex + + try: + reply = self.nlm_request( + flowmsg, + msg_type=self.prid, + msg_flags=NLM_F_REQUEST | NLM_F_ACK, + ) + reply = reply[0] + except NetlinkError as ne: + print(flowmsg) + raise ne + return reply + def dump(self, dpifindex, flowspec=None): """ Returns a list of messages containing flows. @@ -1514,6 +2041,11 @@ def main(argv): dumpflcmd = subparsers.add_parser("dump-flows") dumpflcmd.add_argument("dumpdp", help="Datapath Name") + addflcmd = subparsers.add_parser("add-flow") + addflcmd.add_argument("flbr", help="Datapath name") + addflcmd.add_argument("flow", help="Flow specification") + addflcmd.add_argument("acts", help="Flow actions") + args = parser.parse_args() if args.verbose > 0: @@ -1589,6 +2121,14 @@ def main(argv): rep = ovsflow.dump(rep["dpifindex"]) for flow in rep: print(flow.dpstr(True if args.verbose > 0 else False)) + elif hasattr(args, "flbr"): + rep = ovsdp.info(args.flbr, 0) + if rep is None: + print("DP '%s' not found." % args.flbr) + return 1 + flow = OvsFlow.ovs_flow_msg() + flow.parse(args.flow, args.acts, rep["dpifindex"]) + ovsflow.add_flow(rep["dpifindex"], flow) return 0 diff --git a/tools/testing/selftests/net/psock_lib.h b/tools/testing/selftests/net/psock_lib.h index faa884385c45..6e4fef560873 100644 --- a/tools/testing/selftests/net/psock_lib.h +++ b/tools/testing/selftests/net/psock_lib.h @@ -14,6 +14,8 @@ #include <arpa/inet.h> #include <unistd.h> +#include "kselftest.h" + #define DATA_LEN 100 #define DATA_CHAR 'a' #define DATA_CHAR_1 'b' @@ -63,7 +65,7 @@ static __maybe_unused void pair_udp_setfilter(int fd) struct sock_fprog bpf_prog; bpf_prog.filter = bpf_filter; - bpf_prog.len = sizeof(bpf_filter) / sizeof(struct sock_filter); + bpf_prog.len = ARRAY_SIZE(bpf_filter); if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &bpf_prog, sizeof(bpf_prog))) { diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh index ba286d680fd9..488f4964365e 100755 --- a/tools/testing/selftests/net/rtnetlink.sh +++ b/tools/testing/selftests/net/rtnetlink.sh @@ -21,6 +21,7 @@ ALL_TESTS=" kci_test_vrf kci_test_encap kci_test_macsec + kci_test_macsec_offload kci_test_ipsec kci_test_ipsec_offload kci_test_fdb_get @@ -643,6 +644,88 @@ kci_test_macsec() echo "PASS: macsec" } +kci_test_macsec_offload() +{ + sysfsd=/sys/kernel/debug/netdevsim/netdevsim0/ports/0/ + sysfsnet=/sys/bus/netdevsim/devices/netdevsim0/net/ + probed=false + local ret=0 + + ip macsec help 2>&1 | grep -q "^Usage: ip macsec" + if [ $? -ne 0 ]; then + echo "SKIP: macsec: iproute2 too old" + return $ksft_skip + fi + + # setup netdevsim since dummydev doesn't have offload support + if [ ! -w /sys/bus/netdevsim/new_device ] ; then + modprobe -q netdevsim + check_err $? + if [ $ret -ne 0 ]; then + echo "SKIP: macsec_offload can't load netdevsim" + return $ksft_skip + fi + probed=true + fi + + echo "0" > /sys/bus/netdevsim/new_device + while [ ! -d $sysfsnet ] ; do :; done + udevadm settle + dev=`ls $sysfsnet` + + ip link set $dev up + if [ ! -d $sysfsd ] ; then + echo "FAIL: macsec_offload can't create device $dev" + return 1 + fi + + ethtool -k $dev | grep -q 'macsec-hw-offload: on' + if [ $? -eq 1 ] ; then + echo "FAIL: macsec_offload netdevsim doesn't support MACsec offload" + return 1 + fi + + ip link add link $dev kci_macsec1 type macsec port 4 offload mac + check_err $? + + ip link add link $dev kci_macsec2 type macsec address "aa:bb:cc:dd:ee:ff" port 5 offload mac + check_err $? + + ip link add link $dev kci_macsec3 type macsec sci abbacdde01020304 offload mac + check_err $? + + ip link add link $dev kci_macsec4 type macsec port 8 offload mac 2> /dev/null + check_fail $? + + msname=kci_macsec1 + + ip macsec add "$msname" tx sa 0 pn 1024 on key 01 12345678901234567890123456789012 + check_err $? + + ip macsec add "$msname" rx port 1234 address "1c:ed:de:ad:be:ef" + check_err $? + + ip macsec add "$msname" rx port 1234 address "1c:ed:de:ad:be:ef" sa 0 pn 1 on \ + key 00 0123456789abcdef0123456789abcdef + check_err $? + + ip macsec add "$msname" rx port 1235 address "1c:ed:de:ad:be:ef" 2> /dev/null + check_fail $? + + # clean up any leftovers + for msdev in kci_macsec{1,2,3,4} ; do + ip link del $msdev 2> /dev/null + done + echo 0 > /sys/bus/netdevsim/del_device + $probed && rmmod netdevsim + + if [ $ret -ne 0 ]; then + echo "FAIL: macsec_offload" + return 1 + fi + echo "PASS: macsec_offload" +} + #------------------------------------------------------------------- # Example commands # ip x s add proto esp src 14.0.0.52 dst 14.0.0.70 \ diff --git a/tools/testing/selftests/net/tcp_mmap.c b/tools/testing/selftests/net/tcp_mmap.c index 6e59b1461dcc..4fcce5150850 100644 --- a/tools/testing/selftests/net/tcp_mmap.c +++ b/tools/testing/selftests/net/tcp_mmap.c @@ -153,6 +153,19 @@ static void *mmap_large_buffer(size_t need, size_t *allocated) return buffer; } +static uint32_t tcp_info_get_rcv_mss(int fd) +{ + socklen_t sz = sizeof(struct tcp_info); + struct tcp_info info; + + if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) { + fprintf(stderr, "Error fetching TCP_INFO\n"); + return 0; + } + + return info.tcpi_rcv_mss; +} + void *child_thread(void *arg) { unsigned char digest[SHA256_DIGEST_LENGTH]; @@ -288,7 +301,7 @@ end: total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec + 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec; printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n" - " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n", + " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n", total / (1024.0 * 1024.0), 100.0*total_mmap/total, (double)delta_usec / 1000000.0, @@ -296,7 +309,8 @@ end: (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0, (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0, (double)total_usec/mb, - ru.ru_nvcsw); + ru.ru_nvcsw, + tcp_info_get_rcv_mss(fd)); } error: munmap(buffer, buffer_sz); diff --git a/tools/testing/selftests/net/test_bridge_backup_port.sh b/tools/testing/selftests/net/test_bridge_backup_port.sh new file mode 100755 index 000000000000..112cfd8a10ad --- /dev/null +++ b/tools/testing/selftests/net/test_bridge_backup_port.sh @@ -0,0 +1,759 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test is for checking bridge backup port and backup nexthop ID +# functionality. The topology consists of two bridge (VTEPs) connected using +# VXLAN. The test checks that when the switch port (swp1) is down, traffic is +# redirected to the VXLAN port (vx0). When a backup nexthop ID is configured, +# the test checks that traffic is redirected with the correct nexthop +# information. +# +# +------------------------------------+ +------------------------------------+ +# | + swp1 + vx0 | | + swp1 + vx0 | +# | | | | | | | | +# | | br0 | | | | | | +# | +------------+-----------+ | | +------------+-----------+ | +# | | | | | | +# | | | | | | +# | + | | + | +# | br0 | | br0 | +# | + | | + | +# | | | | | | +# | | | | | | +# | + | | + | +# | br0.10 | | br0.10 | +# | 192.0.2.65/28 | | 192.0.2.66/28 | +# | | | | +# | | | | +# | 192.0.2.33 | | 192.0.2.34 | +# | + lo | | + lo | +# | | | | +# | | | | +# | 192.0.2.49/28 | | 192.0.2.50/28 | +# | veth0 +-------+ veth0 | +# | | | | +# | sw1 | | sw2 | +# +------------------------------------+ +------------------------------------+ + +ret=0 +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +# All tests in this script. Can be overridden with -t option. +TESTS=" + backup_port + backup_nhid + backup_nhid_invalid + backup_nhid_ping + backup_nhid_torture +" +VERBOSE=0 +PAUSE_ON_FAIL=no +PAUSE=no +PING_TIMEOUT=5 + +################################################################################ +# Utilities + +log_test() +{ + local rc=$1 + local expected=$2 + local msg="$3" + + if [ ${rc} -eq ${expected} ]; then + printf "TEST: %-60s [ OK ]\n" "${msg}" + nsuccess=$((nsuccess+1)) + else + ret=1 + nfail=$((nfail+1)) + printf "TEST: %-60s [FAIL]\n" "${msg}" + if [ "$VERBOSE" = "1" ]; then + echo " rc=$rc, expected $expected" + fi + + if [ "${PAUSE_ON_FAIL}" = "yes" ]; then + echo + echo "hit enter to continue, 'q' to quit" + read a + [ "$a" = "q" ] && exit 1 + fi + fi + + if [ "${PAUSE}" = "yes" ]; then + echo + echo "hit enter to continue, 'q' to quit" + read a + [ "$a" = "q" ] && exit 1 + fi + + [ "$VERBOSE" = "1" ] && echo +} + +run_cmd() +{ + local cmd="$1" + local out + local stderr="2>/dev/null" + + if [ "$VERBOSE" = "1" ]; then + printf "COMMAND: $cmd\n" + stderr= + fi + + out=$(eval $cmd $stderr) + rc=$? + if [ "$VERBOSE" = "1" -a -n "$out" ]; then + echo " $out" + fi + + return $rc +} + +tc_check_packets() +{ + local ns=$1; shift + local id=$1; shift + local handle=$1; shift + local count=$1; shift + local pkts + + sleep 0.1 + pkts=$(tc -n $ns -j -s filter show $id \ + | jq ".[] | select(.options.handle == $handle) | \ + .options.actions[0].stats.packets") + [[ $pkts == $count ]] +} + +################################################################################ +# Setup + +setup_topo_ns() +{ + local ns=$1; shift + + ip netns add $ns + ip -n $ns link set dev lo up + + ip netns exec $ns sysctl -qw net.ipv6.conf.all.keep_addr_on_down=1 + ip netns exec $ns sysctl -qw net.ipv6.conf.default.ignore_routes_with_linkdown=1 + ip netns exec $ns sysctl -qw net.ipv6.conf.all.accept_dad=0 + ip netns exec $ns sysctl -qw net.ipv6.conf.default.accept_dad=0 +} + +setup_topo() +{ + local ns + + for ns in sw1 sw2; do + setup_topo_ns $ns + done + + ip link add name veth0 type veth peer name veth1 + ip link set dev veth0 netns sw1 name veth0 + ip link set dev veth1 netns sw2 name veth0 +} + +setup_sw_common() +{ + local ns=$1; shift + local local_addr=$1; shift + local remote_addr=$1; shift + local veth_addr=$1; shift + local gw_addr=$1; shift + local br_addr=$1; shift + + ip -n $ns address add $local_addr/32 dev lo + + ip -n $ns link set dev veth0 up + ip -n $ns address add $veth_addr/28 dev veth0 + ip -n $ns route add default via $gw_addr + + ip -n $ns link add name br0 up type bridge vlan_filtering 1 \ + vlan_default_pvid 0 mcast_snooping 0 + + ip -n $ns link add link br0 name br0.10 up type vlan id 10 + bridge -n $ns vlan add vid 10 dev br0 self + ip -n $ns address add $br_addr/28 dev br0.10 + + ip -n $ns link add name swp1 up type dummy + ip -n $ns link set dev swp1 master br0 + bridge -n $ns vlan add vid 10 dev swp1 untagged + + ip -n $ns link add name vx0 up master br0 type vxlan \ + local $local_addr dstport 4789 nolearning external + bridge -n $ns link set dev vx0 vlan_tunnel on learning off + + bridge -n $ns vlan add vid 10 dev vx0 + bridge -n $ns vlan add vid 10 dev vx0 tunnel_info id 10010 +} + +setup_sw1() +{ + local ns=sw1 + local local_addr=192.0.2.33 + local remote_addr=192.0.2.34 + local veth_addr=192.0.2.49 + local gw_addr=192.0.2.50 + local br_addr=192.0.2.65 + + setup_sw_common $ns $local_addr $remote_addr $veth_addr $gw_addr \ + $br_addr +} + +setup_sw2() +{ + local ns=sw2 + local local_addr=192.0.2.34 + local remote_addr=192.0.2.33 + local veth_addr=192.0.2.50 + local gw_addr=192.0.2.49 + local br_addr=192.0.2.66 + + setup_sw_common $ns $local_addr $remote_addr $veth_addr $gw_addr \ + $br_addr +} + +setup() +{ + set -e + + setup_topo + setup_sw1 + setup_sw2 + + sleep 5 + + set +e +} + +cleanup() +{ + local ns + + for ns in h1 h2 sw1 sw2; do + ip netns del $ns &> /dev/null + done +} + +################################################################################ +# Tests + +backup_port() +{ + local dmac=00:11:22:33:44:55 + local smac=00:aa:bb:cc:dd:ee + + echo + echo "Backup port" + echo "-----------" + + run_cmd "tc -n sw1 qdisc replace dev swp1 clsact" + run_cmd "tc -n sw1 filter replace dev swp1 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + + run_cmd "tc -n sw1 qdisc replace dev vx0 clsact" + run_cmd "tc -n sw1 filter replace dev vx0 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + + run_cmd "bridge -n sw1 fdb replace $dmac dev swp1 master static vlan 10" + + # Initial state - check that packets are forwarded out of swp1 when it + # has a carrier and not forwarded out of any port when it does not have + # a carrier. + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 1 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 0 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 1 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 0 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier on" + log_test $? 0 "swp1 carrier on" + + # Configure vx0 as the backup port of swp1 and check that packets are + # forwarded out of swp1 when it has a carrier and out of vx0 when swp1 + # does not have a carrier. + run_cmd "bridge -n sw1 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_port vx0\"" + log_test $? 0 "vx0 configured as backup port of swp1" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 2 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 0 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 2 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "Forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier on" + log_test $? 0 "swp1 carrier on" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 3 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "No forwarding out of vx0" + + # Remove vx0 as the backup port of swp1 and check that packets are no + # longer forwarded out of vx0 when swp1 does not have a carrier. + run_cmd "bridge -n sw1 link set dev swp1 nobackup_port" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_port vx0\"" + log_test $? 1 "vx0 not configured as backup port of swp1" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 4 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 4 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "No forwarding out of vx0" +} + +backup_nhid() +{ + local dmac=00:11:22:33:44:55 + local smac=00:aa:bb:cc:dd:ee + + echo + echo "Backup nexthop ID" + echo "-----------------" + + run_cmd "tc -n sw1 qdisc replace dev swp1 clsact" + run_cmd "tc -n sw1 filter replace dev swp1 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + + run_cmd "tc -n sw1 qdisc replace dev vx0 clsact" + run_cmd "tc -n sw1 filter replace dev vx0 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + + run_cmd "ip -n sw1 nexthop replace id 1 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 2 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 10 group 1/2 fdb" + + run_cmd "bridge -n sw1 fdb replace $dmac dev swp1 master static vlan 10" + run_cmd "bridge -n sw1 fdb replace $dmac dev vx0 self static dst 192.0.2.36 src_vni 10010" + + run_cmd "ip -n sw2 address replace 192.0.2.36/32 dev lo" + + # The first filter matches on packets forwarded using the backup + # nexthop ID and the second filter matches on packets forwarded using a + # regular VXLAN FDB entry. + run_cmd "tc -n sw2 qdisc replace dev vx0 clsact" + run_cmd "tc -n sw2 filter replace dev vx0 ingress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac enc_key_id 10010 enc_dst_ip 192.0.2.34 action pass" + run_cmd "tc -n sw2 filter replace dev vx0 ingress pref 1 handle 102 proto ip flower src_mac $smac dst_mac $dmac enc_key_id 10010 enc_dst_ip 192.0.2.36 action pass" + + # Configure vx0 as the backup port of swp1 and check that packets are + # forwarded out of swp1 when it has a carrier and out of vx0 when swp1 + # does not have a carrier. When packets are forwarded out of vx0, check + # that they are forwarded by the VXLAN FDB entry. + run_cmd "bridge -n sw1 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_port vx0\"" + log_test $? 0 "vx0 configured as backup port of swp1" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 1 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 0 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 1 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 0 + log_test $? 0 "No forwarding using backup nexthop ID" + tc_check_packets sw2 "dev vx0 ingress" 102 1 + log_test $? 0 "Forwarding using VXLAN FDB entry" + + run_cmd "ip -n sw1 link set dev swp1 carrier on" + log_test $? 0 "swp1 carrier on" + + # Configure nexthop ID 10 as the backup nexthop ID of swp1 and check + # that when packets are forwarded out of vx0, they are forwarded using + # the backup nexthop ID. + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 10" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 10\"" + log_test $? 0 "nexthop ID 10 configured as backup nexthop ID of swp1" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 2 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "No forwarding out of vx0" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 2 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 2 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "Forwarding using backup nexthop ID" + tc_check_packets sw2 "dev vx0 ingress" 102 1 + log_test $? 0 "No forwarding using VXLAN FDB entry" + + run_cmd "ip -n sw1 link set dev swp1 carrier on" + log_test $? 0 "swp1 carrier on" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 3 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 2 + log_test $? 0 "No forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + tc_check_packets sw2 "dev vx0 ingress" 102 1 + log_test $? 0 "No forwarding using VXLAN FDB entry" + + # Reset the backup nexthop ID to 0 and check that packets are no longer + # forwarded using the backup nexthop ID when swp1 does not have a + # carrier and are instead forwarded by the VXLAN FDB. + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 0" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid\"" + log_test $? 1 "No backup nexthop ID configured for swp1" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 4 + log_test $? 0 "Forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 2 + log_test $? 0 "No forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + tc_check_packets sw2 "dev vx0 ingress" 102 1 + log_test $? 0 "No forwarding using VXLAN FDB entry" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 4 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 3 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + tc_check_packets sw2 "dev vx0 ingress" 102 2 + log_test $? 0 "Forwarding using VXLAN FDB entry" +} + +backup_nhid_invalid() +{ + local dmac=00:11:22:33:44:55 + local smac=00:aa:bb:cc:dd:ee + local tx_drop + + echo + echo "Backup nexthop ID - invalid IDs" + echo "-------------------------------" + + # Check that when traffic is redirected with an invalid nexthop ID, it + # is forwarded out of the VXLAN port, but dropped by the VXLAN driver + # and does not crash the host. + + run_cmd "tc -n sw1 qdisc replace dev swp1 clsact" + run_cmd "tc -n sw1 filter replace dev swp1 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + + run_cmd "tc -n sw1 qdisc replace dev vx0 clsact" + run_cmd "tc -n sw1 filter replace dev vx0 egress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac action pass" + # Drop all other Tx traffic to avoid changes to Tx drop counter. + run_cmd "tc -n sw1 filter replace dev vx0 egress pref 2 handle 102 proto all matchall action drop" + + tx_drop=$(ip -n sw1 -s -j link show dev vx0 | jq '.[]["stats64"]["tx"]["dropped"]') + + run_cmd "ip -n sw1 nexthop replace id 1 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 2 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 10 group 1/2 fdb" + + run_cmd "bridge -n sw1 fdb replace $dmac dev swp1 master static vlan 10" + + run_cmd "tc -n sw2 qdisc replace dev vx0 clsact" + run_cmd "tc -n sw2 filter replace dev vx0 ingress pref 1 handle 101 proto ip flower src_mac $smac dst_mac $dmac enc_key_id 10010 enc_dst_ip 192.0.2.34 action pass" + + # First, check that redirection works. + run_cmd "bridge -n sw1 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_port vx0\"" + log_test $? 0 "vx0 configured as backup port of swp1" + + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 10" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 10\"" + log_test $? 0 "Valid nexthop as backup nexthop" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + log_test $? 0 "swp1 carrier off" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 0 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 1 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "Forwarding using backup nexthop ID" + run_cmd "ip -n sw1 -s -j link show dev vx0 | jq -e '.[][\"stats64\"][\"tx\"][\"dropped\"] == $tx_drop'" + log_test $? 0 "No Tx drop increase" + + # Use a non-existent nexthop ID. + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 20" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 20\"" + log_test $? 0 "Non-existent nexthop as backup nexthop" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 0 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 2 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + run_cmd "ip -n sw1 -s -j link show dev vx0 | jq -e '.[][\"stats64\"][\"tx\"][\"dropped\"] == $((tx_drop + 1))'" + log_test $? 0 "Tx drop increased" + + # Use a blckhole nexthop. + run_cmd "ip -n sw1 nexthop replace id 30 blackhole" + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 30" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 30\"" + log_test $? 0 "Blackhole nexthop as backup nexthop" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 0 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 3 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + run_cmd "ip -n sw1 -s -j link show dev vx0 | jq -e '.[][\"stats64\"][\"tx\"][\"dropped\"] == $((tx_drop + 2))'" + log_test $? 0 "Tx drop increased" + + # Non-group FDB nexthop. + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 1" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 1\"" + log_test $? 0 "Non-group FDB nexthop as backup nexthop" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 0 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 4 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + run_cmd "ip -n sw1 -s -j link show dev vx0 | jq -e '.[][\"stats64\"][\"tx\"][\"dropped\"] == $((tx_drop + 3))'" + log_test $? 0 "Tx drop increased" + + # IPv6 address family nexthop. + run_cmd "ip -n sw1 nexthop replace id 100 via 2001:db8:100::1 fdb" + run_cmd "ip -n sw1 nexthop replace id 200 via 2001:db8:100::1 fdb" + run_cmd "ip -n sw1 nexthop replace id 300 group 100/200 fdb" + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 300" + run_cmd "bridge -n sw1 -d link show dev swp1 | grep \"backup_nhid 300\"" + log_test $? 0 "IPv6 address family nexthop as backup nexthop" + + run_cmd "ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 1" + tc_check_packets sw1 "dev swp1 egress" 101 0 + log_test $? 0 "No forwarding out of swp1" + tc_check_packets sw1 "dev vx0 egress" 101 5 + log_test $? 0 "Forwarding out of vx0" + tc_check_packets sw2 "dev vx0 ingress" 101 1 + log_test $? 0 "No forwarding using backup nexthop ID" + run_cmd "ip -n sw1 -s -j link show dev vx0 | jq -e '.[][\"stats64\"][\"tx\"][\"dropped\"] == $((tx_drop + 4))'" + log_test $? 0 "Tx drop increased" +} + +backup_nhid_ping() +{ + local sw1_mac + local sw2_mac + + echo + echo "Backup nexthop ID - ping" + echo "------------------------" + + # Test bidirectional traffic when traffic is redirected in both VTEPs. + sw1_mac=$(ip -n sw1 -j -p link show br0.10 | jq -r '.[]["address"]') + sw2_mac=$(ip -n sw2 -j -p link show br0.10 | jq -r '.[]["address"]') + + run_cmd "bridge -n sw1 fdb replace $sw2_mac dev swp1 master static vlan 10" + run_cmd "bridge -n sw2 fdb replace $sw1_mac dev swp1 master static vlan 10" + + run_cmd "ip -n sw1 neigh replace 192.0.2.66 lladdr $sw2_mac nud perm dev br0.10" + run_cmd "ip -n sw2 neigh replace 192.0.2.65 lladdr $sw1_mac nud perm dev br0.10" + + run_cmd "ip -n sw1 nexthop replace id 1 via 192.0.2.34 fdb" + run_cmd "ip -n sw2 nexthop replace id 1 via 192.0.2.33 fdb" + run_cmd "ip -n sw1 nexthop replace id 10 group 1 fdb" + run_cmd "ip -n sw2 nexthop replace id 10 group 1 fdb" + + run_cmd "bridge -n sw1 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw2 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 10" + run_cmd "bridge -n sw2 link set dev swp1 backup_nhid 10" + + run_cmd "ip -n sw1 link set dev swp1 carrier off" + run_cmd "ip -n sw2 link set dev swp1 carrier off" + + run_cmd "ip netns exec sw1 ping -i 0.1 -c 10 -w $PING_TIMEOUT 192.0.2.66" + log_test $? 0 "Ping with backup nexthop ID" + + # Reset the backup nexthop ID to 0 and check that ping fails. + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 0" + run_cmd "bridge -n sw2 link set dev swp1 backup_nhid 0" + + run_cmd "ip netns exec sw1 ping -i 0.1 -c 10 -w $PING_TIMEOUT 192.0.2.66" + log_test $? 1 "Ping after disabling backup nexthop ID" +} + +backup_nhid_add_del_loop() +{ + while true; do + ip -n sw1 nexthop del id 10 + ip -n sw1 nexthop replace id 10 group 1/2 fdb + done >/dev/null 2>&1 +} + +backup_nhid_torture() +{ + local dmac=00:11:22:33:44:55 + local smac=00:aa:bb:cc:dd:ee + local pid1 + local pid2 + local pid3 + + echo + echo "Backup nexthop ID - torture test" + echo "--------------------------------" + + # Continuously send traffic through the backup nexthop while adding and + # deleting the group. The test is considered successful if nothing + # crashed. + + run_cmd "ip -n sw1 nexthop replace id 1 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 2 via 192.0.2.34 fdb" + run_cmd "ip -n sw1 nexthop replace id 10 group 1/2 fdb" + + run_cmd "bridge -n sw1 fdb replace $dmac dev swp1 master static vlan 10" + + run_cmd "bridge -n sw1 link set dev swp1 backup_port vx0" + run_cmd "bridge -n sw1 link set dev swp1 backup_nhid 10" + run_cmd "ip -n sw1 link set dev swp1 carrier off" + + backup_nhid_add_del_loop & + pid1=$! + ip netns exec sw1 mausezahn br0.10 -a $smac -b $dmac -A 198.51.100.1 -B 198.51.100.2 -t ip -p 100 -q -c 0 & + pid2=$! + + sleep 30 + kill -9 $pid1 $pid2 + wait $pid1 $pid2 2>/dev/null + + log_test 0 0 "Torture test" +} + +################################################################################ +# Usage + +usage() +{ + cat <<EOF +usage: ${0##*/} OPTS + + -t <test> Test(s) to run (default: all) + (options: $TESTS) + -p Pause on fail + -P Pause after each test before cleanup + -v Verbose mode (show commands and output) + -w Timeout for ping +EOF +} + +################################################################################ +# Main + +trap cleanup EXIT + +while getopts ":t:pPvhw:" opt; do + case $opt in + t) TESTS=$OPTARG;; + p) PAUSE_ON_FAIL=yes;; + P) PAUSE=yes;; + v) VERBOSE=$(($VERBOSE + 1));; + w) PING_TIMEOUT=$OPTARG;; + h) usage; exit 0;; + *) usage; exit 1;; + esac +done + +# Make sure we don't pause twice. +[ "${PAUSE}" = "yes" ] && PAUSE_ON_FAIL=no + +if [ "$(id -u)" -ne 0 ];then + echo "SKIP: Need root privileges" + exit $ksft_skip; +fi + +if [ ! -x "$(command -v ip)" ]; then + echo "SKIP: Could not run test without ip tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v bridge)" ]; then + echo "SKIP: Could not run test without bridge tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v tc)" ]; then + echo "SKIP: Could not run test without tc tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v mausezahn)" ]; then + echo "SKIP: Could not run test without mausezahn tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v jq)" ]; then + echo "SKIP: Could not run test without jq tool" + exit $ksft_skip +fi + +bridge link help 2>&1 | grep -q "backup_nhid" +if [ $? -ne 0 ]; then + echo "SKIP: iproute2 bridge too old, missing backup nexthop ID support" + exit $ksft_skip +fi + +# Start clean. +cleanup + +for t in $TESTS +do + setup; $t; cleanup; +done + +if [ "$TESTS" != "none" ]; then + printf "\nTests passed: %3d\n" ${nsuccess} + printf "Tests failed: %3d\n" ${nfail} +fi + +exit $ret diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index a3c57004344c..4b63708c6a81 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -486,6 +486,17 @@ TEST_F(tls, msg_more_unsent) EXPECT_EQ(recv(self->cfd, buf, send_len, MSG_DONTWAIT), -1); } +TEST_F(tls, msg_eor) +{ + char const *test_str = "test_read"; + int send_len = 10; + char buf[10]; + + EXPECT_EQ(send(self->fd, test_str, send_len, MSG_EOR), send_len); + EXPECT_EQ(recv(self->cfd, buf, send_len, MSG_WAITALL), send_len); + EXPECT_EQ(memcmp(buf, test_str, send_len), 0); +} + TEST_F(tls, sendmsg_single) { struct msghdr msg; diff --git a/tools/testing/selftests/ptp/testptp.c b/tools/testing/selftests/ptp/testptp.c index e9438a1862ad..c9f6cca4feb4 100644 --- a/tools/testing/selftests/ptp/testptp.c +++ b/tools/testing/selftests/ptp/testptp.c @@ -143,8 +143,10 @@ static void usage(char *progname) " -S set the system time from the ptp clock time\n" " -t val shift the ptp clock time by 'val' seconds\n" " -T val set the ptp clock time to 'val' seconds\n" + " -x val get an extended ptp clock time with the desired number of samples (up to %d)\n" + " -X get a ptp clock cross timestamp\n" " -z test combinations of rising/falling external time stamp flags\n", - progname); + progname, PTP_MAX_SAMPLES); } int main(int argc, char *argv[]) @@ -158,6 +160,8 @@ int main(int argc, char *argv[]) struct timex tx; struct ptp_clock_time *pct; struct ptp_sys_offset *sysoff; + struct ptp_sys_offset_extended *soe; + struct ptp_sys_offset_precise *xts; char *progname; unsigned int i; @@ -176,6 +180,8 @@ int main(int argc, char *argv[]) int index = 0; int list_pins = 0; int pct_offset = 0; + int getextended = 0; + int getcross = 0; int n_samples = 0; int pin_index = -1, pin_func; int pps = -1; @@ -190,7 +196,7 @@ int main(int argc, char *argv[]) progname = strrchr(argv[0], '/'); progname = progname ? 1+progname : argv[0]; - while (EOF != (c = getopt(argc, argv, "cd:e:f:ghH:i:k:lL:n:o:p:P:sSt:T:w:z"))) { + while (EOF != (c = getopt(argc, argv, "cd:e:f:ghH:i:k:lL:n:o:p:P:sSt:T:w:x:Xz"))) { switch (c) { case 'c': capabilities = 1; @@ -255,6 +261,18 @@ int main(int argc, char *argv[]) case 'w': pulsewidth = atoi(optarg); break; + case 'x': + getextended = atoi(optarg); + if (getextended < 1 || getextended > PTP_MAX_SAMPLES) { + fprintf(stderr, + "number of extended timestamp samples must be between 1 and %d; was asked for %d\n", + PTP_MAX_SAMPLES, getextended); + return -1; + } + break; + case 'X': + getcross = 1; + break; case 'z': flagtest = 1; break; @@ -535,6 +553,57 @@ int main(int argc, char *argv[]) free(sysoff); } + if (getextended) { + soe = calloc(1, sizeof(*soe)); + if (!soe) { + perror("calloc"); + return -1; + } + + soe->n_samples = getextended; + + if (ioctl(fd, PTP_SYS_OFFSET_EXTENDED, soe)) { + perror("PTP_SYS_OFFSET_EXTENDED"); + } else { + printf("extended timestamp request returned %d samples\n", + getextended); + + for (i = 0; i < getextended; i++) { + printf("sample #%2d: system time before: %lld.%09u\n", + i, soe->ts[i][0].sec, soe->ts[i][0].nsec); + printf(" phc time: %lld.%09u\n", + soe->ts[i][1].sec, soe->ts[i][1].nsec); + printf(" system time after: %lld.%09u\n", + soe->ts[i][2].sec, soe->ts[i][2].nsec); + } + } + + free(soe); + } + + if (getcross) { + xts = calloc(1, sizeof(*xts)); + if (!xts) { + perror("calloc"); + return -1; + } + + if (ioctl(fd, PTP_SYS_OFFSET_PRECISE, xts)) { + perror("PTP_SYS_OFFSET_PRECISE"); + } else { + puts("system and phc crosstimestamping request okay"); + + printf("device time: %lld.%09u\n", + xts->device.sec, xts->device.nsec); + printf("system time: %lld.%09u\n", + xts->sys_realtime.sec, xts->sys_realtime.nsec); + printf("monoraw time: %lld.%09u\n", + xts->sys_monoraw.sec, xts->sys_monoraw.nsec); + } + + free(xts); + } + close(fd); return 0; } diff --git a/tools/testing/selftests/tc-testing/Makefile b/tools/testing/selftests/tc-testing/Makefile index cb553eac9f41..3c4b7fa05075 100644 --- a/tools/testing/selftests/tc-testing/Makefile +++ b/tools/testing/selftests/tc-testing/Makefile @@ -24,7 +24,7 @@ CLANG_FLAGS = -I. -I$(APIDIR) \ $(OUTPUT)/%.o: %.c $(CLANG) $(CLANG_FLAGS) \ - -O2 -target bpf -emit-llvm -c $< -o - | \ + -O2 --target=bpf -emit-llvm -c $< -o - | \ $(LLC) -march=bpf -mcpu=$(CPU) $(LLC_FLAGS) -filetype=obj -o $@ TEST_PROGS += ./tdc.sh diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c index ac1bd3ac1533..90718c2fd4ea 100644 --- a/tools/testing/vsock/vsock_test.c +++ b/tools/testing/vsock/vsock_test.c @@ -255,35 +255,142 @@ static void test_stream_multiconn_server(const struct test_opts *opts) close(fds[i]); } -static void test_stream_msg_peek_client(const struct test_opts *opts) +#define MSG_PEEK_BUF_LEN 64 + +static void test_msg_peek_client(const struct test_opts *opts, + bool seqpacket) { + unsigned char buf[MSG_PEEK_BUF_LEN]; + ssize_t send_size; int fd; + int i; + + if (seqpacket) + fd = vsock_seqpacket_connect(opts->peer_cid, 1234); + else + fd = vsock_stream_connect(opts->peer_cid, 1234); - fd = vsock_stream_connect(opts->peer_cid, 1234); if (fd < 0) { perror("connect"); exit(EXIT_FAILURE); } - send_byte(fd, 1, 0); + for (i = 0; i < sizeof(buf); i++) + buf[i] = rand() & 0xFF; + + control_expectln("SRVREADY"); + + send_size = send(fd, buf, sizeof(buf), 0); + + if (send_size < 0) { + perror("send"); + exit(EXIT_FAILURE); + } + + if (send_size != sizeof(buf)) { + fprintf(stderr, "Invalid send size %zi\n", send_size); + exit(EXIT_FAILURE); + } + close(fd); } -static void test_stream_msg_peek_server(const struct test_opts *opts) +static void test_msg_peek_server(const struct test_opts *opts, + bool seqpacket) { + unsigned char buf_half[MSG_PEEK_BUF_LEN / 2]; + unsigned char buf_normal[MSG_PEEK_BUF_LEN]; + unsigned char buf_peek[MSG_PEEK_BUF_LEN]; + ssize_t res; int fd; - fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); + if (seqpacket) + fd = vsock_seqpacket_accept(VMADDR_CID_ANY, 1234, NULL); + else + fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); + if (fd < 0) { perror("accept"); exit(EXIT_FAILURE); } - recv_byte(fd, 1, MSG_PEEK); - recv_byte(fd, 1, 0); + /* Peek from empty socket. */ + res = recv(fd, buf_peek, sizeof(buf_peek), MSG_PEEK | MSG_DONTWAIT); + if (res != -1) { + fprintf(stderr, "expected recv(2) failure, got %zi\n", res); + exit(EXIT_FAILURE); + } + + if (errno != EAGAIN) { + perror("EAGAIN expected"); + exit(EXIT_FAILURE); + } + + control_writeln("SRVREADY"); + + /* Peek part of data. */ + res = recv(fd, buf_half, sizeof(buf_half), MSG_PEEK); + if (res != sizeof(buf_half)) { + fprintf(stderr, "recv(2) + MSG_PEEK, expected %zu, got %zi\n", + sizeof(buf_half), res); + exit(EXIT_FAILURE); + } + + /* Peek whole data. */ + res = recv(fd, buf_peek, sizeof(buf_peek), MSG_PEEK); + if (res != sizeof(buf_peek)) { + fprintf(stderr, "recv(2) + MSG_PEEK, expected %zu, got %zi\n", + sizeof(buf_peek), res); + exit(EXIT_FAILURE); + } + + /* Compare partial and full peek. */ + if (memcmp(buf_half, buf_peek, sizeof(buf_half))) { + fprintf(stderr, "Partial peek data mismatch\n"); + exit(EXIT_FAILURE); + } + + if (seqpacket) { + /* This type of socket supports MSG_TRUNC flag, + * so check it with MSG_PEEK. We must get length + * of the message. + */ + res = recv(fd, buf_half, sizeof(buf_half), MSG_PEEK | + MSG_TRUNC); + if (res != sizeof(buf_peek)) { + fprintf(stderr, + "recv(2) + MSG_PEEK | MSG_TRUNC, exp %zu, got %zi\n", + sizeof(buf_half), res); + exit(EXIT_FAILURE); + } + } + + res = recv(fd, buf_normal, sizeof(buf_normal), 0); + if (res != sizeof(buf_normal)) { + fprintf(stderr, "recv(2), expected %zu, got %zi\n", + sizeof(buf_normal), res); + exit(EXIT_FAILURE); + } + + /* Compare full peek and normal read. */ + if (memcmp(buf_peek, buf_normal, sizeof(buf_peek))) { + fprintf(stderr, "Full peek data mismatch\n"); + exit(EXIT_FAILURE); + } + close(fd); } +static void test_stream_msg_peek_client(const struct test_opts *opts) +{ + return test_msg_peek_client(opts, false); +} + +static void test_stream_msg_peek_server(const struct test_opts *opts) +{ + return test_msg_peek_server(opts, false); +} + #define SOCK_BUF_SIZE (2 * 1024 * 1024) #define MAX_MSG_SIZE (32 * 1024) @@ -1053,6 +1160,16 @@ static void test_stream_virtio_skb_merge_server(const struct test_opts *opts) close(fd); } +static void test_seqpacket_msg_peek_client(const struct test_opts *opts) +{ + return test_msg_peek_client(opts, true); +} + +static void test_seqpacket_msg_peek_server(const struct test_opts *opts) +{ + return test_msg_peek_server(opts, true); +} + static struct test_case test_cases[] = { { .name = "SOCK_STREAM connection reset", @@ -1128,6 +1245,11 @@ static struct test_case test_cases[] = { .run_client = test_stream_virtio_skb_merge_client, .run_server = test_stream_virtio_skb_merge_server, }, + { + .name = "SOCK_SEQPACKET MSG_PEEK", + .run_client = test_seqpacket_msg_peek_client, + .run_server = test_seqpacket_msg_peek_server, + }, {}, }; |