diff options
760 files changed, 36146 insertions, 8872 deletions
diff --git a/.clang-format b/.clang-format index 1247d54f9e49..8d01225bfcb7 100644 --- a/.clang-format +++ b/.clang-format @@ -535,6 +535,7 @@ ForEachMacros: - 'perf_hpp_list__for_each_sort_list_safe' - 'perf_pmu__for_each_hybrid_pmu' - 'ping_portaddr_for_each_entry' + - 'ping_portaddr_for_each_entry_rcu' - 'plist_for_each' - 'plist_for_each_continue' - 'plist_for_each_entry' diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml index 2ab4642679c0..55c4f94a14d1 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml @@ -148,7 +148,7 @@ allOf: items: - const: oscclk - const: dout_clkcmu_fsys1_bus - - const: dout_clkcmu_fsys1_mmc_card + - const: gout_clkcmu_fsys1_mmc_card - const: dout_clkcmu_fsys1_usbdrd - if: diff --git a/Documentation/devicetree/bindings/net/can/fsl,flexcan.yaml b/Documentation/devicetree/bindings/net/can/fsl,flexcan.yaml index e52db841bb8c..6e59bd2a6094 100644 --- a/Documentation/devicetree/bindings/net/can/fsl,flexcan.yaml +++ b/Documentation/devicetree/bindings/net/can/fsl,flexcan.yaml @@ -17,6 +17,7 @@ properties: compatible: oneOf: - enum: + - fsl,imx93-flexcan - fsl,imx8qm-flexcan - fsl,imx8mp-flexcan - fsl,imx6q-flexcan diff --git a/Documentation/devicetree/bindings/net/can/renesas,rcar-canfd.yaml b/Documentation/devicetree/bindings/net/can/renesas,rcar-canfd.yaml index 6f71fc96bc4e..1eb98c9a1a26 100644 --- a/Documentation/devicetree/bindings/net/can/renesas,rcar-canfd.yaml +++ b/Documentation/devicetree/bindings/net/can/renesas,rcar-canfd.yaml @@ -9,9 +9,6 @@ title: Renesas R-Car CAN FD Controller maintainers: - Fabrizio Castro <fabrizio.castro.jz@renesas.com> -allOf: - - $ref: can-controller.yaml# - properties: compatible: oneOf: @@ -33,7 +30,7 @@ properties: - items: - enum: - - renesas,r9a07g043-canfd # RZ/G2UL + - renesas,r9a07g043-canfd # RZ/G2UL and RZ/Five - renesas,r9a07g044-canfd # RZ/G2{L,LC} - renesas,r9a07g054-canfd # RZ/V2L - const: renesas,rzg2l-canfd # RZ/G2L family @@ -77,12 +74,13 @@ properties: description: Maximum frequency of the CANFD clock. patternProperties: - "^channel[01]$": + "^channel[0-7]$": type: object description: - The controller supports two channels and each is represented as a child - node. Each child node supports the "status" property only, which - is used to enable/disable the respective channel. + The controller supports multiple channels and each is represented as a + child node. Each channel can be enabled/disabled individually. + + additionalProperties: false required: - compatible @@ -98,60 +96,73 @@ required: - channel0 - channel1 -if: - properties: - compatible: - contains: - enum: - - renesas,rzg2l-canfd -then: - properties: - interrupts: - items: - - description: CAN global error interrupt - - description: CAN receive FIFO interrupt - - description: CAN0 error interrupt - - description: CAN0 transmit interrupt - - description: CAN0 transmit/receive FIFO receive completion interrupt - - description: CAN1 error interrupt - - description: CAN1 transmit interrupt - - description: CAN1 transmit/receive FIFO receive completion interrupt - - interrupt-names: - items: - - const: g_err - - const: g_recc - - const: ch0_err - - const: ch0_rec - - const: ch0_trx - - const: ch1_err - - const: ch1_rec - - const: ch1_trx - - resets: - maxItems: 2 - - reset-names: - items: - - const: rstp_n - - const: rstc_n - - required: - - reset-names -else: - properties: - interrupts: - items: - - description: Channel interrupt - - description: Global interrupt - - interrupt-names: - items: - - const: ch_int - - const: g_int - - resets: - maxItems: 1 +allOf: + - $ref: can-controller.yaml# + + - if: + properties: + compatible: + contains: + enum: + - renesas,rzg2l-canfd + then: + properties: + interrupts: + items: + - description: CAN global error interrupt + - description: CAN receive FIFO interrupt + - description: CAN0 error interrupt + - description: CAN0 transmit interrupt + - description: CAN0 transmit/receive FIFO receive completion interrupt + - description: CAN1 error interrupt + - description: CAN1 transmit interrupt + - description: CAN1 transmit/receive FIFO receive completion interrupt + + interrupt-names: + items: + - const: g_err + - const: g_recc + - const: ch0_err + - const: ch0_rec + - const: ch0_trx + - const: ch1_err + - const: ch1_rec + - const: ch1_trx + + resets: + maxItems: 2 + + reset-names: + items: + - const: rstp_n + - const: rstc_n + + required: + - reset-names + else: + properties: + interrupts: + items: + - description: Channel interrupt + - description: Global interrupt + + interrupt-names: + items: + - const: ch_int + - const: g_int + + resets: + maxItems: 1 + + - if: + not: + properties: + compatible: + contains: + const: renesas,r8a779a0-canfd + then: + patternProperties: + "^channel[2-7]$": false unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml b/Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml index b2558421268a..6924aff0b2c5 100644 --- a/Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml +++ b/Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml @@ -14,7 +14,9 @@ properties: oneOf: - const: nxp,nxp-nci-i2c - items: - - const: nxp,pn547 + - enum: + - nxp,nq310 + - nxp,pn547 - const: nxp,nxp-nci-i2c enable-gpios: diff --git a/Documentation/devicetree/bindings/net/qca,ar71xx.yaml b/Documentation/devicetree/bindings/net/qca,ar71xx.yaml index 1ebf9e8c8a1d..89f94b31b546 100644 --- a/Documentation/devicetree/bindings/net/qca,ar71xx.yaml +++ b/Documentation/devicetree/bindings/net/qca,ar71xx.yaml @@ -123,7 +123,6 @@ examples: switch_port0: port@0 { reg = <0x0>; - label = "cpu"; ethernet = <ð1>; phy-mode = "gmii"; diff --git a/Documentation/loongarch/booting.rst b/Documentation/loongarch/booting.rst new file mode 100644 index 000000000000..91eccd410478 --- /dev/null +++ b/Documentation/loongarch/booting.rst @@ -0,0 +1,42 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================= +Booting Linux/LoongArch +======================= + +:Author: Yanteng Si <siyanteng@loongson.cn> +:Date: 18 Nov 2022 + +Information passed from BootLoader to kernel +============================================ + +LoongArch supports ACPI and FDT. The information that needs to be passed +to the kernel includes the memmap, the initrd, the command line, optionally +the ACPI/FDT tables, and so on. + +The kernel is passed the following arguments on `kernel_entry` : + + - a0 = efi_boot: `efi_boot` is a flag indicating whether + this boot environment is fully UEFI-compliant. + + - a1 = cmdline: `cmdline` is a pointer to the kernel command line. + + - a2 = systemtable: `systemtable` points to the EFI system table. + All pointers involved at this stage are in physical addresses. + +Header of Linux/LoongArch kernel images +======================================= + +Linux/LoongArch kernel images are EFI images. Being PE files, they have +a 64-byte header structured like:: + + u32 MZ_MAGIC /* "MZ", MS-DOS header */ + u32 res0 = 0 /* Reserved */ + u64 kernel_entry /* Kernel entry point */ + u64 _end - _text /* Kernel image effective size */ + u64 load_offset /* Kernel image load offset from start of RAM */ + u64 res1 = 0 /* Reserved */ + u64 res2 = 0 /* Reserved */ + u64 res3 = 0 /* Reserved */ + u32 LINUX_PE_MAGIC /* Magic number */ + u32 pe_header - _head /* Offset to the PE header */ diff --git a/Documentation/loongarch/index.rst b/Documentation/loongarch/index.rst index aaba648db907..c779bfa00c05 100644 --- a/Documentation/loongarch/index.rst +++ b/Documentation/loongarch/index.rst @@ -9,6 +9,7 @@ LoongArch Architecture :numbered: introduction + booting irq-chip-model features diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index 96cd7a26f3d9..adc4bf4f3c50 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -566,7 +566,8 @@ miimon link monitoring. A value of 100 is a good starting point. The use_carrier option, below, affects how the link state is determined. See the High Availability section for additional - information. The default value is 0. + information. The default value is 100 if arp_interval is not + set. min_links @@ -956,6 +957,7 @@ xmit_hash_policy hash = hash XOR source IP XOR destination IP hash = hash XOR (hash RSHIFT 16) hash = hash XOR (hash RSHIFT 8) + hash = hash RSHIFT 1 And then hash is reduced modulo slave count. If the protocol is IPv6 then the source and destination diff --git a/Documentation/networking/device_drivers/ethernet/freescale/dpaa2/mac-phy-support.rst b/Documentation/networking/device_drivers/ethernet/freescale/dpaa2/mac-phy-support.rst index 51e6624fb774..1d2f55feca24 100644 --- a/Documentation/networking/device_drivers/ethernet/freescale/dpaa2/mac-phy-support.rst +++ b/Documentation/networking/device_drivers/ethernet/freescale/dpaa2/mac-phy-support.rst @@ -181,10 +181,13 @@ when necessary using the below listed API:: - int dpaa2_mac_connect(struct dpaa2_mac *mac); - void dpaa2_mac_disconnect(struct dpaa2_mac *mac); -A phylink integration is necessary only when the partner DPMAC is not of TYPE_FIXED. -One can check for this condition using the below API:: +A phylink integration is necessary only when the partner DPMAC is not of +``TYPE_FIXED``. This means it is either of ``TYPE_PHY``, or of +``TYPE_BACKPLANE`` (the difference being the two that in the ``TYPE_BACKPLANE`` +mode, the MC firmware does not access the PCS registers). One can check for +this condition using the following helper:: - - bool dpaa2_mac_is_type_fixed(struct fsl_mc_device *dpmac_dev,struct fsl_mc_io *mc_io); + - static inline bool dpaa2_mac_is_type_phy(struct dpaa2_mac *mac); Before connection to a MAC, the caller must allocate and populate the dpaa2_mac structure with the associated net_device, a pointer to the MC portal diff --git a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst index 5edf50d7dbd5..6969652f593c 100644 --- a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst +++ b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst @@ -25,7 +25,7 @@ Enabling the driver and kconfig options | at build time via kernel Kconfig flags. | Basic features, ethernet net device rx/tx offloads and XDP, are available with the most basic flags | CONFIG_MLX5_CORE=y/m and CONFIG_MLX5_CORE_EN=y. -| For the list of advanced features please see below. +| For the list of advanced features, please see below. **CONFIG_MLX5_CORE=(y/m/n)** (module mlx5_core.ko) @@ -89,11 +89,11 @@ Enabling the driver and kconfig options **CONFIG_MLX5_EN_IPSEC=(y/n)** -| Enables `IPSec XFRM cryptography-offload accelaration <http://www.mellanox.com/related-docs/prod_software/Mellanox_Innova_IPsec_Ethernet_Adapter_Card_User_Manual.pdf>`_. +| Enables `IPSec XFRM cryptography-offload acceleration <http://www.mellanox.com/related-docs/prod_software/Mellanox_Innova_IPsec_Ethernet_Adapter_Card_User_Manual.pdf>`_. **CONFIG_MLX5_EN_TLS=(y/n)** -| TLS cryptography-offload accelaration. +| TLS cryptography-offload acceleration. **CONFIG_MLX5_INFINIBAND=(y/n/m)** (module mlx5_ib.ko) @@ -139,14 +139,14 @@ flow_steering_mode: Device flow steering mode The flow steering mode parameter controls the flow steering mode of the driver. Two modes are supported: 1. 'dmfs' - Device managed flow steering. -2. 'smfs - Software/Driver managed flow steering. +2. 'smfs' - Software/Driver managed flow steering. In DMFS mode, the HW steering entities are created and managed through the Firmware. In SMFS mode, the HW steering entities are created and managed though by -the driver directly into Hardware without firmware intervention. +the driver directly into hardware without firmware intervention. -SMFS mode is faster and provides better rule inserstion rate compared to default DMFS mode. +SMFS mode is faster and provides better rule insertion rate compared to default DMFS mode. User command examples: @@ -165,9 +165,9 @@ User command examples: enable_roce: RoCE enablement state ---------------------------------- RoCE enablement state controls driver support for RoCE traffic. -When RoCE is disabled, there is no gid table, only raw ethernet QPs are supported and traffic on the well known UDP RoCE port is handled as raw ethernet traffic. +When RoCE is disabled, there is no gid table, only raw ethernet QPs are supported and traffic on the well-known UDP RoCE port is handled as raw ethernet traffic. -To change RoCE enablement state a user must change the driverinit cmode value and run devlink reload. +To change RoCE enablement state, a user must change the driverinit cmode value and run devlink reload. User command examples: @@ -186,7 +186,7 @@ User command examples: esw_port_metadata: Eswitch port metadata state ---------------------------------------------- -When applicable, disabling Eswitch metadata can increase packet rate +When applicable, disabling eswitch metadata can increase packet rate up to 20% depending on the use case and packet sizes. Eswitch port metadata state controls whether to internally tag packets with @@ -253,26 +253,26 @@ mlx5 subfunction ================ mlx5 supports subfunction management using devlink port (see :ref:`Documentation/networking/devlink/devlink-port.rst <devlink_port>`) interface. -A Subfunction has its own function capabilities and its own resources. This +A subfunction has its own function capabilities and its own resources. This means a subfunction has its own dedicated queues (txq, rxq, cq, eq). These queues are neither shared nor stolen from the parent PCI function. -When a subfunction is RDMA capable, it has its own QP1, GID table and rdma +When a subfunction is RDMA capable, it has its own QP1, GID table, and RDMA resources neither shared nor stolen from the parent PCI function. A subfunction has a dedicated window in PCI BAR space that is not shared -with ther other subfunctions or the parent PCI function. This ensures that all -devices (netdev, rdma, vdpa etc.) of the subfunction accesses only assigned +with the other subfunctions or the parent PCI function. This ensures that all +devices (netdev, rdma, vdpa, etc.) of the subfunction accesses only assigned PCI BAR space. -A Subfunction supports eswitch representation through which it supports tc +A subfunction supports eswitch representation through which it supports tc offloads. The user configures eswitch to send/receive packets from/to the subfunction port. Subfunctions share PCI level resources such as PCI MSI-X IRQs with other subfunctions and/or with its parent PCI function. -Example mlx5 software, system and device view:: +Example mlx5 software, system, and device view:: _______ | admin | @@ -310,7 +310,7 @@ Example mlx5 software, system and device view:: | (device add/del) _____|____ ____|________ | | | subfunction | - | PCI NIC |---- activate/deactive events---->| host driver | + | PCI NIC |--- activate/deactivate events--->| host driver | |__________| | (mlx5_core) | |_____________| @@ -320,7 +320,7 @@ Subfunction is created using devlink port interface. $ devlink dev eswitch set pci/0000:06:00.0 mode switchdev -- Add a devlink port of subfunction flaovur:: +- Add a devlink port of subfunction flavour:: $ devlink port add pci/0000:06:00.0 flavour pcisf pfnum 0 sfnum 88 pci/0000:06:00.0/32768: type eth netdev eth6 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false @@ -351,46 +351,30 @@ driver. MAC address setup ----------------- -mlx5 driver provides mechanism to setup the MAC address of the PCI VF/SF. +mlx5 driver support devlink port function attr mechanism to setup MAC +address. (refer to Documentation/networking/devlink/devlink-port.rst) -The configured MAC address of the PCI VF/SF will be used by netdevice and rdma -device created for the PCI VF/SF. +RoCE capability setup +--------------------- +Not all mlx5 PCI devices/SFs require RoCE capability. -- Get the MAC address of the VF identified by its unique devlink port index:: +When RoCE capability is disabled, it saves 1 Mbytes worth of system memory per +PCI devices/SF. - $ devlink port show pci/0000:06:00.0/2 - pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 - function: - hw_addr 00:00:00:00:00:00 - -- Set the MAC address of the VF identified by its unique devlink port index:: - - $ devlink port function set pci/0000:06:00.0/2 hw_addr 00:11:22:33:44:55 - - $ devlink port show pci/0000:06:00.0/2 - pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 - function: - hw_addr 00:11:22:33:44:55 +mlx5 driver support devlink port function attr mechanism to setup RoCE +capability. (refer to Documentation/networking/devlink/devlink-port.rst) -- Get the MAC address of the SF identified by its unique devlink port index:: +migratable capability setup +--------------------------- +User who wants mlx5 PCI VFs to be able to perform live migration need to +explicitly enable the VF migratable capability. - $ devlink port show pci/0000:06:00.0/32768 - pci/0000:06:00.0/32768: type eth netdev enp6s0pf0sf88 flavour pcisf pfnum 0 sfnum 88 - function: - hw_addr 00:00:00:00:00:00 - -- Set the MAC address of the VF identified by its unique devlink port index:: - - $ devlink port function set pci/0000:06:00.0/32768 hw_addr 00:00:00:00:88:88 - - $ devlink port show pci/0000:06:00.0/32768 - pci/0000:06:00.0/32768: type eth netdev enp6s0pf0sf88 flavour pcivf pfnum 0 sfnum 88 - function: - hw_addr 00:00:00:00:88:88 +mlx5 driver support devlink port function attr mechanism to setup migratable +capability. (refer to Documentation/networking/devlink/devlink-port.rst) SF state setup -------------- -To use the SF, the user must active the SF using the SF function state +To use the SF, the user must activate the SF using the SF function state attribute. - Get the state of the SF identified by its unique devlink port index:: @@ -447,7 +431,7 @@ for it. Additionally, the SF port also gets the event when the driver attaches to the auxiliary device of the subfunction. This results in changing the operational -state of the function. This provides visiblity to the user to decide when is it +state of the function. This provides visibility to the user to decide when is it safe to delete the SF port for graceful termination of the subfunction. - Show the SF port operational state:: @@ -464,14 +448,14 @@ tx reporter ----------- The tx reporter is responsible for reporting and recovering of the following two error scenarios: -- TX timeout +- tx timeout Report on kernel tx timeout detection. Recover by searching lost interrupts. -- TX error completion +- tx error completion Report on error tx completion. - Recover by flushing the TX queue and reset it. + Recover by flushing the tx queue and reset it. -TX reporter also support on demand diagnose callback, on which it provides +tx reporter also support on demand diagnose callback, on which it provides real time information of its send queues status. User commands examples: @@ -491,32 +475,32 @@ rx reporter ----------- The rx reporter is responsible for reporting and recovering of the following two error scenarios: -- RX queues initialization (population) timeout - RX queues descriptors population on ring initialization is done in - napi context via triggering an irq, in case of a failure to get - the minimum amount of descriptors, a timeout would occur and it - could be recoverable by polling the EQ (Event Queue). -- RX completions with errors (reported by HW on interrupt context) +- rx queues' initialization (population) timeout + Population of rx queues' descriptors on ring initialization is done + in napi context via triggering an irq. In case of a failure to get + the minimum amount of descriptors, a timeout would occur, and + descriptors could be recovered by polling the EQ (Event Queue). +- rx completions with errors (reported by HW on interrupt context) Report on rx completion error. Recover (if needed) by flushing the related queue and reset it. -RX reporter also supports on demand diagnose callback, on which it -provides real time information of its receive queues status. +rx reporter also supports on demand diagnose callback, on which it +provides real time information of its receive queues' status. -- Diagnose rx queues status, and corresponding completion queue:: +- Diagnose rx queues' status and corresponding completion queue:: $ devlink health diagnose pci/0000:82:00.0 reporter rx -NOTE: This command has valid output only when interface is up, otherwise the command has empty output. +NOTE: This command has valid output only when interface is up. Otherwise, the command has empty output. - Show number of rx errors indicated, number of recover flows ended successfully, - is autorecover enabled and graceful period from last recover:: + is autorecover enabled, and graceful period from last recover:: $ devlink health show pci/0000:82:00.0 reporter rx fw reporter ----------- -The fw reporter implements diagnose and dump callbacks. +The fw reporter implements `diagnose` and `dump` callbacks. It follows symptoms of fw error such as fw syndrome by triggering fw core dump and storing it into the dump buffer. The fw reporter diagnose command can be triggered any time by the user to check @@ -537,7 +521,7 @@ running it on other PF or any VF will return "Operation not permitted". fw fatal reporter ----------------- -The fw fatal reporter implements dump and recover callbacks. +The fw fatal reporter implements `dump` and `recover` callbacks. It follows fatal errors indications by CR-space dump and recover flow. The CR-space dump uses vsc interface which is valid even if the FW command interface is not functional, which is the case in most FW fatal errors. @@ -552,7 +536,7 @@ User commands examples: $ devlink health recover pci/0000:82:00.0 reporter fw_fatal -- Read FW CR-space dump if already strored or trigger new one:: +- Read FW CR-space dump if already stored or trigger new one:: $ devlink health dump show pci/0000:82:00.1 reporter fw_fatal @@ -561,10 +545,10 @@ NOTE: This command can run only on PF. mlx5 tracepoints ================ -mlx5 driver provides internal trace points for tracking and debugging using +mlx5 driver provides internal tracepoints for tracking and debugging using kernel tracepoints interfaces (refer to Documentation/trace/ftrace.rst). -For the list of support mlx5 events check /sys/kernel/debug/tracing/events/mlx5/ +For the list of support mlx5 events, check `/sys/kernel/debug/tracing/events/mlx5/`. tc and eswitch offloads tracepoints: diff --git a/Documentation/networking/devlink/devlink-info.rst b/Documentation/networking/devlink/devlink-info.rst index 7572bf6de5c1..1242b0e6826b 100644 --- a/Documentation/networking/devlink/devlink-info.rst +++ b/Documentation/networking/devlink/devlink-info.rst @@ -198,6 +198,11 @@ fw.bundle_id Unique identifier of the entire firmware bundle. +fw.bootloader +------------- + +Version of the bootloader. + Future work =========== diff --git a/Documentation/networking/devlink/devlink-port.rst b/Documentation/networking/devlink/devlink-port.rst index 98557c2ab1c1..3da590953ce8 100644 --- a/Documentation/networking/devlink/devlink-port.rst +++ b/Documentation/networking/devlink/devlink-port.rst @@ -110,7 +110,7 @@ devlink ports for both the controllers. Function configuration ====================== -A user can configure the function attribute before enumerating the PCI +Users can configure one or more function attributes before enumerating the PCI function. Usually it means, user should configure function attribute before a bus specific device for the function is created. However, when SRIOV is enabled, virtual function devices are created on the PCI bus. @@ -119,9 +119,127 @@ function device to the driver. For subfunctions, this means user should configure port function attribute before activating the port function. A user may set the hardware address of the function using -'devlink port function set hw_addr' command. For Ethernet port function +`devlink port function set hw_addr` command. For Ethernet port function this means a MAC address. +Users may also set the RoCE capability of the function using +`devlink port function set roce` command. + +Users may also set the function as migratable using +'devlink port function set migratable' command. + +Function attributes +=================== + +MAC address setup +----------------- +The configured MAC address of the PCI VF/SF will be used by netdevice and rdma +device created for the PCI VF/SF. + +- Get the MAC address of the VF identified by its unique devlink port index:: + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:00:00:00:00:00 + +- Set the MAC address of the VF identified by its unique devlink port index:: + + $ devlink port function set pci/0000:06:00.0/2 hw_addr 00:11:22:33:44:55 + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:11:22:33:44:55 + +- Get the MAC address of the SF identified by its unique devlink port index:: + + $ devlink port show pci/0000:06:00.0/32768 + pci/0000:06:00.0/32768: type eth netdev enp6s0pf0sf88 flavour pcisf pfnum 0 sfnum 88 + function: + hw_addr 00:00:00:00:00:00 + +- Set the MAC address of the SF identified by its unique devlink port index:: + + $ devlink port function set pci/0000:06:00.0/32768 hw_addr 00:00:00:00:88:88 + + $ devlink port show pci/0000:06:00.0/32768 + pci/0000:06:00.0/32768: type eth netdev enp6s0pf0sf88 flavour pcisf pfnum 0 sfnum 88 + function: + hw_addr 00:00:00:00:88:88 + +RoCE capability setup +--------------------- +Not all PCI VFs/SFs require RoCE capability. + +When RoCE capability is disabled, it saves system memory per PCI VF/SF. + +When user disables RoCE capability for a VF/SF, user application cannot send or +receive any RoCE packets through this VF/SF and RoCE GID table for this PCI +will be empty. + +When RoCE capability is disabled in the device using port function attribute, +VF/SF driver cannot override it. + +- Get RoCE capability of the VF device:: + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:00:00:00:00:00 roce enable + +- Set RoCE capability of the VF device:: + + $ devlink port function set pci/0000:06:00.0/2 roce disable + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:00:00:00:00:00 roce disable + +migratable capability setup +--------------------------- +Live migration is the process of transferring a live virtual machine +from one physical host to another without disrupting its normal +operation. + +User who want PCI VFs to be able to perform live migration need to +explicitly enable the VF migratable capability. + +When user enables migratable capability for a VF, and the HV binds the VF to VFIO driver +with migration support, the user can migrate the VM with this VF from one HV to a +different one. + +However, when migratable capability is enable, device will disable features which cannot +be migrated. Thus migratable cap can impose limitations on a VF so let the user decide. + +Example of LM with migratable function configuration: +- Get migratable capability of the VF device:: + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:00:00:00:00:00 migratable disable + +- Set migratable capability of the VF device:: + + $ devlink port function set pci/0000:06:00.0/2 migratable enable + + $ devlink port show pci/0000:06:00.0/2 + pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0 vfnum 1 + function: + hw_addr 00:00:00:00:00:00 migratable enable + +- Bind VF to VFIO driver with migration support:: + + $ echo <pci_id> > /sys/bus/pci/devices/0000:08:00.0/driver/unbind + $ echo mlx5_vfio_pci > /sys/bus/pci/devices/0000:08:00.0/driver_override + $ echo <pci_id> > /sys/bus/pci/devices/0000:08:00.0/driver/bind + +Attach VF to the VM. +Start the VM. +Perform live migration. + Subfunction ============ diff --git a/Documentation/networking/devlink/devlink-region.rst b/Documentation/networking/devlink/devlink-region.rst index f06dca9a1eb6..9232cd7da301 100644 --- a/Documentation/networking/devlink/devlink-region.rst +++ b/Documentation/networking/devlink/devlink-region.rst @@ -31,6 +31,15 @@ in its ``devlink_region_ops`` structure. If snapshot id is not set in the ``DEVLINK_CMD_REGION_NEW`` request kernel will allocate one and send the snapshot information to user space. +Regions may optionally allow directly reading from their contents without a +snapshot. Direct read requests are not atomic. In particular a read request +of size 256 bytes or larger will be split into multiple chunks. If atomic +access is required, use a snapshot. A driver wishing to enable this for a +region should implement the ``.read`` callback in the ``devlink_region_ops`` +structure. User space can request a direct read by using the +``DEVLINK_ATTR_REGION_DIRECT`` attribute instead of specifying a snapshot +id. + example usage ------------- @@ -65,6 +74,10 @@ example usage $ devlink region read pci/0000:00:05.0/fw-health snapshot 1 address 0 length 16 0000000000000000 0014 95dc 0014 9514 0035 1670 0034 db30 + # Read from the region without a snapshot + $ devlink region read pci/0000:00:05.0/fw-health address 16 length 16 + 0000000000000010 0000 0000 ffff ff04 0029 8c00 0028 8cc8 + As regions are likely very device or driver specific, no generic regions are defined. See the driver-specific documentation files for information on the specific regions a driver supports. diff --git a/Documentation/networking/devlink/etas_es58x.rst b/Documentation/networking/devlink/etas_es58x.rst new file mode 100644 index 000000000000..3b857d82a44c --- /dev/null +++ b/Documentation/networking/devlink/etas_es58x.rst @@ -0,0 +1,36 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +etas_es58x devlink support +========================== + +This document describes the devlink features implemented by the +``etas_es58x`` device driver. + +Info versions +============= + +The ``etas_es58x`` driver reports the following versions + +.. list-table:: devlink info versions implemented + :widths: 5 5 90 + + * - Name + - Type + - Description + * - ``fw`` + - running + - Version of the firmware running on the device. Also available + through ``ethtool -i`` as the first member of the + ``firmware-version``. + * - ``fw.bootloader`` + - running + - Version of the bootloader running on the device. Also available + through ``ethtool -i`` as the second member of the + ``firmware-version``. + * - ``board.rev`` + - fixed + - The hardware revision of the device. + * - ``serial_number`` + - fixed + - The USB serial number. Also available through ``lsusb -v``. diff --git a/Documentation/networking/devlink/ice.rst b/Documentation/networking/devlink/ice.rst index 890062da7820..625efb3777d5 100644 --- a/Documentation/networking/devlink/ice.rst +++ b/Documentation/networking/devlink/ice.rst @@ -189,12 +189,21 @@ device data. * - ``nvm-flash`` - The contents of the entire flash chip, sometimes referred to as the device's Non Volatile Memory. + * - ``shadow-ram`` + - The contents of the Shadow RAM, which is loaded from the beginning + of the flash. Although the contents are primarily from the flash, + this area also contains data generated during device boot which is + not stored in flash. * - ``device-caps`` - The contents of the device firmware's capabilities buffer. Useful to determine the current state and configuration of the device. -Users can request an immediate capture of a snapshot via the -``DEVLINK_CMD_REGION_NEW`` +Both the ``nvm-flash`` and ``shadow-ram`` regions can be accessed without a +snapshot. The ``device-caps`` region requires a snapshot as the contents are +sent by firmware and can't be split into separate reads. + +Users can request an immediate capture of a snapshot for all three regions +via the ``DEVLINK_CMD_REGION_NEW`` command. .. code:: shell diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index bede24ef44fd..f10f8eb44255 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -222,6 +222,7 @@ Userspace to kernel: ``ETHTOOL_MSG_MODULE_GET`` get transceiver module parameters ``ETHTOOL_MSG_PSE_SET`` set PSE parameters ``ETHTOOL_MSG_PSE_GET`` get PSE parameters + ``ETHTOOL_MSG_RSS_GET`` get RSS settings ===================================== ================================= Kernel to userspace: @@ -263,6 +264,7 @@ Kernel to userspace: ``ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY`` PHC virtual clocks info ``ETHTOOL_MSG_MODULE_GET_REPLY`` transceiver module parameters ``ETHTOOL_MSG_PSE_GET_REPLY`` PSE parameters + ``ETHTOOL_MSG_RSS_GET_REPLY`` RSS settings ======================================== ================================= ``GET`` requests are sent by userspace applications to retrieve device @@ -1687,6 +1689,33 @@ to control PoDL PSE Admin functions. This option is implementing ``IEEE 802.3-2018`` 30.15.1.2.1 acPoDLPSEAdminControl. See ``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` for supported values. +RSS_GET +======= + +Get indirection table, hash key and hash function info associated with a +RSS context of an interface similar to ``ETHTOOL_GRSSH`` ioctl request. + +Request contents: + +===================================== ====== ========================== + ``ETHTOOL_A_RSS_HEADER`` nested request header + ``ETHTOOL_A_RSS_CONTEXT`` u32 context number +===================================== ====== ========================== + +Kernel response contents: + +===================================== ====== ========================== + ``ETHTOOL_A_RSS_HEADER`` nested reply header + ``ETHTOOL_A_RSS_HFUNC`` u32 RSS hash func + ``ETHTOOL_A_RSS_INDIR`` binary Indir table bytes + ``ETHTOOL_A_RSS_HKEY`` binary Hash key bytes +===================================== ====== ========================== + +ETHTOOL_A_RSS_HFUNC attribute is bitmap indicating the hash function +being used. Current supported options are toeplitz, xor or crc32. +ETHTOOL_A_RSS_INDIR attribute returns RSS indrection table where each byte +indicates queue number. + Request translation =================== @@ -1768,7 +1797,7 @@ are netlink only. ``ETHTOOL_GMODULEEEPROM`` ``ETHTOOL_MSG_MODULE_EEPROM_GET`` ``ETHTOOL_GEEE`` ``ETHTOOL_MSG_EEE_GET`` ``ETHTOOL_SEEE`` ``ETHTOOL_MSG_EEE_SET`` - ``ETHTOOL_GRSSH`` n/a + ``ETHTOOL_GRSSH`` ``ETHTOOL_MSG_RSS_GET`` ``ETHTOOL_SRSSH`` n/a ``ETHTOOL_GTUNABLE`` n/a ``ETHTOOL_STUNABLE`` n/a diff --git a/Documentation/networking/timestamping.rst b/Documentation/networking/timestamping.rst index be4eb1242057..f17c01834a12 100644 --- a/Documentation/networking/timestamping.rst +++ b/Documentation/networking/timestamping.rst @@ -179,7 +179,8 @@ SOF_TIMESTAMPING_OPT_ID: identifier and returns that along with the timestamp. The identifier is derived from a per-socket u32 counter (that wraps). For datagram sockets, the counter increments with each sent packet. For stream - sockets, it increments with every byte. + sockets, it increments with every byte. For stream sockets, also set + SOF_TIMESTAMPING_OPT_ID_TCP, see the section below. The counter starts at zero. It is initialized the first time that the socket option is enabled. It is reset each time the option is @@ -192,6 +193,35 @@ SOF_TIMESTAMPING_OPT_ID: among all possibly concurrently outstanding timestamp requests for that socket. +SOF_TIMESTAMPING_OPT_ID_TCP: + Pass this modifier along with SOF_TIMESTAMPING_OPT_ID for new TCP + timestamping applications. SOF_TIMESTAMPING_OPT_ID defines how the + counter increments for stream sockets, but its starting point is + not entirely trivial. This option fixes that. + + For stream sockets, if SOF_TIMESTAMPING_OPT_ID is set, this should + always be set too. On datagram sockets the option has no effect. + + A reasonable expectation is that the counter is reset to zero with + the system call, so that a subsequent write() of N bytes generates + a timestamp with counter N-1. SOF_TIMESTAMPING_OPT_ID_TCP + implements this behavior under all conditions. + + SOF_TIMESTAMPING_OPT_ID without modifier often reports the same, + especially when the socket option is set when no data is in + transmission. If data is being transmitted, it may be off by the + length of the output queue (SIOCOUTQ). + + The difference is due to being based on snd_una versus write_seq. + snd_una is the offset in the stream acknowledged by the peer. This + depends on factors outside of process control, such as network RTT. + write_seq is the last byte written by the process. This offset is + not affected by external inputs. + + The difference is subtle and unlikely to be noticed when configured + at initial socket creation, when no data is queued or sent. But + SOF_TIMESTAMPING_OPT_ID_TCP behavior is more robust regardless of + when the socket option is set. SOF_TIMESTAMPING_OPT_CMSG: Support recv() cmsg for all timestamped packets. Control messages diff --git a/Documentation/networking/xfrm_device.rst b/Documentation/networking/xfrm_device.rst index 01391dfd37d9..c43ace79e320 100644 --- a/Documentation/networking/xfrm_device.rst +++ b/Documentation/networking/xfrm_device.rst @@ -5,6 +5,7 @@ XFRM device - offloading the IPsec computations =============================================== Shannon Nelson <shannon.nelson@oracle.com> +Leon Romanovsky <leonro@nvidia.com> Overview @@ -18,10 +19,21 @@ can radically increase throughput and decrease CPU utilization. The XFRM Device interface allows NIC drivers to offer to the stack access to the hardware offload. +Right now, there are two types of hardware offload that kernel supports. + * IPsec crypto offload: + * NIC performs encrypt/decrypt + * Kernel does everything else + * IPsec packet offload: + * NIC performs encrypt/decrypt + * NIC does encapsulation + * Kernel and NIC have SA and policy in-sync + * NIC handles the SA and policies states + * The Kernel talks to the keymanager + Userland access to the offload is typically through a system such as libreswan or KAME/raccoon, but the iproute2 'ip xfrm' command set can be handy when experimenting. An example command might look something -like this:: +like this for crypto offload: ip x s add proto esp dst 14.0.0.70 src 14.0.0.52 spi 0x07 mode transport \ reqid 0x07 replay-window 32 \ @@ -29,6 +41,17 @@ like this:: sel src 14.0.0.52/24 dst 14.0.0.70/24 proto tcp \ offload dev eth4 dir in +and for packet offload + + ip x s add proto esp dst 14.0.0.70 src 14.0.0.52 spi 0x07 mode transport \ + reqid 0x07 replay-window 32 \ + aead 'rfc4106(gcm(aes))' 0x44434241343332312423222114131211f4f3f2f1 128 \ + sel src 14.0.0.52/24 dst 14.0.0.70/24 proto tcp \ + offload packet dev eth4 dir in + + ip x p add src 14.0.0.70 dst 14.0.0.52 offload packet dev eth4 dir in + tmpl src 14.0.0.70 dst 14.0.0.52 proto esp reqid 10000 mode transport + Yes, that's ugly, but that's what shell scripts and/or libreswan are for. @@ -40,17 +63,24 @@ Callbacks to implement /* from include/linux/netdevice.h */ struct xfrmdev_ops { + /* Crypto and Packet offload callbacks */ int (*xdo_dev_state_add) (struct xfrm_state *x); void (*xdo_dev_state_delete) (struct xfrm_state *x); void (*xdo_dev_state_free) (struct xfrm_state *x); bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + + /* Solely packet offload callbacks */ + void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); + int (*xdo_dev_policy_add) (struct xfrm_policy *x); + void (*xdo_dev_policy_delete) (struct xfrm_policy *x); + void (*xdo_dev_policy_free) (struct xfrm_policy *x); }; -The NIC driver offering ipsec offload will need to implement these -callbacks to make the offload available to the network stack's -XFRM subsystem. Additionally, the feature bits NETIF_F_HW_ESP and +The NIC driver offering ipsec offload will need to implement callbacks +relevant to supported offload to make the offload available to the network +stack's XFRM subsystem. Additionally, the feature bits NETIF_F_HW_ESP and NETIF_F_HW_ESP_TX_CSUM will signal the availability of the offload. @@ -79,7 +109,8 @@ and an indication of whether it is for Rx or Tx. The driver should =========== =================================== 0 success - -EOPNETSUPP offload not supported, try SW IPsec + -EOPNETSUPP offload not supported, try SW IPsec, + not applicable for packet offload mode other fail the request =========== =================================== @@ -96,6 +127,7 @@ will serviceable. This can check the packet information to be sure the offload can be supported (e.g. IPv4 or IPv6, no IPv4 options, etc) and return true of false to signify its support. +Crypto offload mode: When ready to send, the driver needs to inspect the Tx packet for the offload information, including the opaque context, and set up the packet send accordingly:: @@ -139,13 +171,25 @@ the stack in xfrm_input(). In ESN mode, xdo_dev_state_advance_esn() is called from xfrm_replay_advance_esn(). Driver will check packet seq number and update HW ESN state machine if needed. +Packet offload mode: +HW adds and deletes XFRM headers. So in RX path, XFRM stack is bypassed if HW +reported success. In TX path, the packet lefts kernel without extra header +and not encrypted, the HW is responsible to perform it. + When the SA is removed by the user, the driver's xdo_dev_state_delete() -is asked to disable the offload. Later, xdo_dev_state_free() is called -from a garbage collection routine after all reference counts to the state +and xdo_dev_policy_delete() are asked to disable the offload. Later, +xdo_dev_state_free() and xdo_dev_policy_free() are called from a garbage +collection routine after all reference counts to the state and policy have been removed and any remaining resources can be cleared for the offload state. How these are used by the driver will depend on specific hardware needs. As a netdev is set to DOWN the XFRM stack's netdev listener will call -xdo_dev_state_delete() and xdo_dev_state_free() on any remaining offloaded -states. +xdo_dev_state_delete(), xdo_dev_policy_delete(), xdo_dev_state_free() and +xdo_dev_policy_free() on any remaining offloaded states. + +Outcome of HW handling packets, the XFRM core can't count hard, soft limits. +The HW/driver are responsible to perform it and provide accurate data when +xdo_dev_state_update_curlft() is called. In case of one of these limits +occuried, the driver needs to call to xfrm_state_check_expire() to make sure +that XFRM performs rekeying sequence. diff --git a/Documentation/translations/zh_CN/loongarch/booting.rst b/Documentation/translations/zh_CN/loongarch/booting.rst new file mode 100644 index 000000000000..fb6440c438f0 --- /dev/null +++ b/Documentation/translations/zh_CN/loongarch/booting.rst @@ -0,0 +1,48 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/loongarch/booting.rst + +:翻译: + + 司延腾 Yanteng Si <siyanteng@loongson.cn> + +==================== +启动 Linux/LoongArch +==================== + +:作者: 司延腾 <siyanteng@loongson.cn> +:日期: 2022年11月18日 + +BootLoader传递给内核的信息 +========================== + +LoongArch支持ACPI和FDT启动，需要传递给内核的信息包括memmap、initrd、cmdline、可 +选的ACPI/FDT表等。 + +内核在 `kernel_entry` 入口处被传递以下参数: + + - a0 = efi_boot: `efi_boot` 是一个标志，表示这个启动环境是否完全符合UEFI + 的要求。 + + - a1 = cmdline: `cmdline` 是一个指向内核命令行的指针。 + + - a2 = systemtable: `systemtable` 指向EFI的系统表，在这个阶段涉及的所有 + 指针都是物理地址。 + +Linux/LoongArch内核镜像文件头 +============================= + +内核镜像是EFI镜像。作为PE文件，它们有一个64字节的头部结构体，如下所示:: + + u32 MZ_MAGIC /* "MZ", MS-DOS 头 */ + u32 res0 = 0 /* 保留 */ + u64 kernel_entry /* 内核入口点 */ + u64 _end - _text /* 内核镜像有效大小 */ + u64 load_offset /* 加载内核镜像相对内存起始地址的偏移量 */ + u64 res1 = 0 /* 保留 */ + u64 res2 = 0 /* 保留 */ + u64 res3 = 0 /* 保留 */ + u32 LINUX_PE_MAGIC /* 魔术数 */ + u32 pe_header - _head /* 到PE头的偏移量 */ diff --git a/Documentation/translations/zh_CN/loongarch/index.rst b/Documentation/translations/zh_CN/loongarch/index.rst index 7d23eb78379d..0273a08342f7 100644 --- a/Documentation/translations/zh_CN/loongarch/index.rst +++ b/Documentation/translations/zh_CN/loongarch/index.rst @@ -14,6 +14,7 @@ LoongArch体系结构 :numbered: introduction + booting irq-chip-model features diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index eee9f857a986..896914e3a847 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7213,14 +7213,13 @@ veto the transition. :Parameters: args[0] is the maximum poll time in nanoseconds :Returns: 0 on success; -1 on error -This capability overrides the kvm module parameter halt_poll_ns for the -target VM. - -VCPU polling allows a VCPU to poll for wakeup events instead of immediately -scheduling during guest halts. The maximum time a VCPU can spend polling is -controlled by the kvm module parameter halt_poll_ns. This capability allows -the maximum halt time to specified on a per-VM basis, effectively overriding -the module parameter for the target VM. +KVM_CAP_HALT_POLL overrides the kvm.halt_poll_ns module parameter to set the +maximum halt-polling time for all vCPUs in the target VM. This capability can +be invoked at any time and any number of times to dynamically change the +maximum halt-polling time. + +See Documentation/virt/kvm/halt-polling.rst for more information on halt +polling. 7.21 KVM_CAP_X86_USER_SPACE_MSR ------------------------------- diff --git a/Documentation/virt/kvm/x86/halt-polling.rst b/Documentation/virt/kvm/halt-polling.rst index 4922e4a15f18..3fae39b1a5ba 100644 --- a/Documentation/virt/kvm/x86/halt-polling.rst +++ b/Documentation/virt/kvm/halt-polling.rst @@ -119,6 +119,19 @@ These module parameters can be set from the debugfs files in: Note: that these module parameters are system wide values and are not able to be tuned on a per vm basis. +Any changes to these parameters will be picked up by new and existing vCPUs the +next time they halt, with the notable exception of VMs using KVM_CAP_HALT_POLL +(see next section). + +KVM_CAP_HALT_POLL +================= + +KVM_CAP_HALT_POLL is a VM capability that allows userspace to override halt_poll_ns +on a per-VM basis. VMs using KVM_CAP_HALT_POLL ignore halt_poll_ns completely (but +still obey halt_poll_ns_grow, halt_poll_ns_grow_start, and halt_poll_ns_shrink). + +See Documentation/virt/kvm/api.rst for more information on this capability. + Further Notes ============= diff --git a/Documentation/virt/kvm/index.rst b/Documentation/virt/kvm/index.rst index e0a2c74e1043..ad13ec55ddfe 100644 --- a/Documentation/virt/kvm/index.rst +++ b/Documentation/virt/kvm/index.rst @@ -17,4 +17,5 @@ KVM locking vcpu-requests + halt-polling review-checklist diff --git a/Documentation/virt/kvm/x86/index.rst b/Documentation/virt/kvm/x86/index.rst index 7ff588826b9f..9ece6b8dc817 100644 --- a/Documentation/virt/kvm/x86/index.rst +++ b/Documentation/virt/kvm/x86/index.rst @@ -10,7 +10,6 @@ KVM for x86 systems amd-memory-encryption cpuid errata - halt-polling hypercalls mmu msr diff --git a/MAINTAINERS b/MAINTAINERS index 955c1be1efb2..71f4f8776779 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7682,6 +7682,7 @@ ETAS ES58X CAN/USB DRIVER M: Vincent Mailhol <mailhol.vincent@wanadoo.fr> L: linux-can@vger.kernel.org S: Maintained +F: Documentation/networking/devlink/etas_es58x.rst F: drivers/net/can/usb/etas_es58x/ ETHERNET BRIDGE @@ -2,7 +2,7 @@ VERSION = 6 PATCHLEVEL = 1 SUBLEVEL = 0 -EXTRAVERSION = -rc7 +EXTRAVERSION = -rc8 NAME = Hurr durr I'ma ninja sloth # *DOCUMENTATION* diff --git a/arch/arm/boot/dts/at91rm9200.dtsi b/arch/arm/boot/dts/at91rm9200.dtsi index 7a113325abb9..6f9004ebf424 100644 --- a/arch/arm/boot/dts/at91rm9200.dtsi +++ b/arch/arm/boot/dts/at91rm9200.dtsi @@ -666,7 +666,7 @@ compatible = "atmel,at91rm9200-udc"; reg = <0xfffb0000 0x4000>; interrupts = <11 IRQ_TYPE_LEVEL_HIGH 2>; - clocks = <&pmc PMC_TYPE_PERIPHERAL 11>, <&pmc PMC_TYPE_SYSTEM 2>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 11>, <&pmc PMC_TYPE_SYSTEM 1>; clock-names = "pclk", "hclk"; status = "disabled"; }; diff --git a/arch/arm/mach-at91/sama5.c b/arch/arm/mach-at91/sama5.c index 67ed68fbe3a5..bf2b5c6a18c6 100644 --- a/arch/arm/mach-at91/sama5.c +++ b/arch/arm/mach-at91/sama5.c @@ -26,7 +26,7 @@ static void sama5_l2c310_write_sec(unsigned long val, unsigned reg) static void __init sama5_secure_cache_init(void) { sam_secure_init(); - if (sam_linux_is_optee_available()) + if (IS_ENABLED(CONFIG_OUTER_CACHE) && sam_linux_is_optee_available()) outer_cache.write_sec = sama5_l2c310_write_sec; } diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index d6cf535d8352..439e2bc5d5d8 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -14,16 +14,8 @@ #ifdef CONFIG_EFI extern void efi_init(void); - -bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg); #else #define efi_init() - -static inline -bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) -{ - return false; -} #endif int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); diff --git a/arch/arm64/kernel/efi-rt-wrapper.S b/arch/arm64/kernel/efi-rt-wrapper.S index 67babd5f04c2..75691a2641c1 100644 --- a/arch/arm64/kernel/efi-rt-wrapper.S +++ b/arch/arm64/kernel/efi-rt-wrapper.S @@ -6,7 +6,7 @@ #include <linux/linkage.h> SYM_FUNC_START(__efi_rt_asm_wrapper) - stp x29, x30, [sp, #-112]! + stp x29, x30, [sp, #-32]! mov x29, sp /* @@ -17,20 +17,6 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) stp x1, x18, [sp, #16] /* - * Preserve all callee saved registers and record the stack pointer - * value in a per-CPU variable so we can recover from synchronous - * exceptions occurring while running the firmware routines. - */ - stp x19, x20, [sp, #32] - stp x21, x22, [sp, #48] - stp x23, x24, [sp, #64] - stp x25, x26, [sp, #80] - stp x27, x28, [sp, #96] - - adr_this_cpu x8, __efi_rt_asm_recover_sp, x9 - str x29, [x8] - - /* * We are lucky enough that no EFI runtime services take more than * 5 arguments, so all are passed in registers rather than via the * stack. @@ -45,7 +31,7 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) ldp x1, x2, [sp, #16] cmp x2, x18 - ldp x29, x30, [sp], #112 + ldp x29, x30, [sp], #32 b.ne 0f ret 0: @@ -59,18 +45,3 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) mov x18, x2 b efi_handle_corrupted_x18 // tail call SYM_FUNC_END(__efi_rt_asm_wrapper) - -SYM_FUNC_START(__efi_rt_asm_recover) - ldr_this_cpu x8, __efi_rt_asm_recover_sp, x9 - mov sp, x8 - - ldp x0, x18, [sp, #16] - ldp x19, x20, [sp, #32] - ldp x21, x22, [sp, #48] - ldp x23, x24, [sp, #64] - ldp x25, x26, [sp, #80] - ldp x27, x28, [sp, #96] - ldp x29, x30, [sp], #112 - - b efi_handle_runtime_exception -SYM_FUNC_END(__efi_rt_asm_recover) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index ee53f2a0aa03..a908a37f0367 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -9,7 +9,6 @@ #include <linux/efi.h> #include <linux/init.h> -#include <linux/percpu.h> #include <asm/efi.h> @@ -145,28 +144,3 @@ asmlinkage efi_status_t efi_handle_corrupted_x18(efi_status_t s, const char *f) pr_err_ratelimited(FW_BUG "register x18 corrupted by EFI %s\n", f); return s; } - -asmlinkage DEFINE_PER_CPU(u64, __efi_rt_asm_recover_sp); - -asmlinkage efi_status_t __efi_rt_asm_recover(void); - -asmlinkage efi_status_t efi_handle_runtime_exception(const char *f) -{ - pr_err(FW_BUG "Synchronous exception occurred in EFI runtime service %s()\n", f); - clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); - return EFI_ABORTED; -} - -bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) -{ - /* Check whether the exception occurred while running the firmware */ - if (current_work() != &efi_rts_work.work || regs->pc >= TASK_SIZE_64) - return false; - - pr_err(FW_BUG "Unable to handle %s in EFI runtime service\n", msg); - add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); - dump_stack(); - - regs->pc = (u64)__efi_rt_asm_recover; - return true; -} diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 3cb101e8cb29..5240f6acad64 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -36,7 +36,22 @@ void arch_dma_prep_coherent(struct page *page, size_t size) { unsigned long start = (unsigned long)page_address(page); - dcache_clean_poc(start, start + size); + /* + * The architecture only requires a clean to the PoC here in order to + * meet the requirements of the DMA API. However, some vendors (i.e. + * Qualcomm) abuse the DMA API for transferring buffers from the + * non-secure to the secure world, resetting the system if a non-secure + * access shows up after the buffer has been transferred: + * + * https://lore.kernel.org/r/20221114110329.68413-1-manivannan.sadhasivam@linaro.org + * + * Using clean+invalidate appears to make this issue less likely, but + * the drivers themselves still need fixing as the CPU could issue a + * speculative read from the buffer via the linear mapping irrespective + * of the cache maintenance we use. Once the drivers are fixed, we can + * relax this to a clean operation. + */ + dcache_clean_inval_poc(start, start + size); } #ifdef CONFIG_IOMMU_DMA diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 3e9cf9826417..5b391490e045 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -30,7 +30,6 @@ #include <asm/bug.h> #include <asm/cmpxchg.h> #include <asm/cpufeature.h> -#include <asm/efi.h> #include <asm/exception.h> #include <asm/daifflags.h> #include <asm/debug-monitors.h> @@ -392,9 +391,6 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, msg = "paging request"; } - if (efi_runtime_fixup_exception(regs, msg)) - return; - die_kernel_fault(msg, addr, esr, regs); } diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index aa0e0e0d4ee5..79d5bfd913e0 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -490,6 +490,7 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } +#define pmd_young pmd_young static inline int pmd_young(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_ACCESSED); diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index 3dd172d9ffea..d82687390b4a 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -78,16 +78,6 @@ extern void calculate_cpu_foreign_map(void); */ extern void show_ipi_list(struct seq_file *p, int prec); -/* - * This function sends a 'reschedule' IPI to another CPU. - * it goes straight through and wastes no time serializing - * anything. Worst case is that we lose a reschedule ... - */ -static inline void smp_send_reschedule(int cpu) -{ - loongson_send_ipi_single(cpu, SMP_RESCHEDULE); -} - static inline void arch_send_call_function_single_ipi(int cpu) { loongson_send_ipi_single(cpu, SMP_CALL_FUNCTION); diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 6ed72f7ff278..14508d429ffa 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -149,6 +149,17 @@ void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) ipi_write_action(cpu_logical_map(i), (u32)action); } +/* + * This function sends a 'reschedule' IPI to another CPU. + * it goes straight through and wastes no time serializing + * anything. Worst case is that we lose a reschedule ... + */ +void smp_send_reschedule(int cpu) +{ + loongson_send_ipi_single(cpu, SMP_RESCHEDULE); +} +EXPORT_SYMBOL_GPL(smp_send_reschedule); + irqreturn_t loongson_ipi_interrupt(int irq, void *dev) { unsigned int action; diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index d8ee8fbc8c67..58781c6e4191 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -10,6 +10,8 @@ #include <asm/regdef.h> #include <asm/stackframe.h> +#define INVTLB_ADDR_GFALSE_AND_ASID 5 + #define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) #define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) #define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) @@ -136,13 +138,10 @@ tlb_huge_update_load: ori t0, ra, _PAGE_VALID st.d t0, t1, 0 #endif - tlbsrch - addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16) - addi.d ra, t1, 0 - csrxchg ra, t1, LOONGARCH_CSR_TLBIDX - tlbwr - - csrxchg zero, t1, LOONGARCH_CSR_TLBIDX + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the @@ -287,13 +286,11 @@ tlb_huge_update_store: ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) st.d t0, t1, 0 #endif - tlbsrch - addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16) - addi.d ra, t1, 0 - csrxchg ra, t1, LOONGARCH_CSR_TLBIDX - tlbwr + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 - csrxchg zero, t1, LOONGARCH_CSR_TLBIDX /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the @@ -436,6 +433,11 @@ tlb_huge_update_modify: ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) st.d t0, t1, 0 #endif + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 + /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the @@ -466,7 +468,7 @@ tlb_huge_update_modify: addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX - tlbwr + tlbfill /* Reset default page size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 6caec386ad2f..4678627673df 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -622,6 +622,7 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } +#define pmd_young pmd_young static inline int pmd_young(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_ACCESSED); diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index 4745bb9998bd..6d8492b6e2b8 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -602,6 +602,7 @@ ____##func(struct pt_regs *regs) /* kernel/traps.c */ DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); #ifdef CONFIG_PPC_BOOK3S_64 +DECLARE_INTERRUPT_HANDLER_RAW(machine_check_early_boot); DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async); #endif DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 43f1c76d48ce..a379b0ce19ff 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -113,23 +113,19 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) { int i; - /* First arg comes in as a 32 bits pointer. */ - EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3)); - EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0)); + /* Initialize tail_call_cnt, to be skipped if we do tail calls. */ + EMIT(PPC_RAW_LI(_R4, 0)); + +#define BPF_TAILCALL_PROLOGUE_SIZE 4 + EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx))); - /* - * Initialize tail_call_cnt in stack frame if we do tail calls. - * Otherwise, put in NOPs so that it can be skipped when we are - * invoked through a tail call. - */ if (ctx->seen & SEEN_TAILCALL) - EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_1) - 1, _R1, - bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); - else - EMIT(PPC_RAW_NOP()); + EMIT(PPC_RAW_STW(_R4, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); -#define BPF_TAILCALL_PROLOGUE_SIZE 16 + /* First arg comes in as a 32 bits pointer. */ + EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3)); + EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0)); /* * We need a stack frame, but we don't necessarily need to @@ -170,24 +166,24 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx for (i = BPF_PPC_NVR_MIN; i <= 31; i++) if (bpf_is_seen_register(ctx, i)) EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i))); -} - -void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) -{ - EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0))); - - bpf_jit_emit_common_epilogue(image, ctx); - - /* Tear down our stack frame */ if (ctx->seen & SEEN_FUNC) EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); + /* Tear down our stack frame */ EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx))); if (ctx->seen & SEEN_FUNC) EMIT(PPC_RAW_MTLR(_R0)); +} + +void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) +{ + EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0))); + + bpf_jit_emit_common_epilogue(image, ctx); + EMIT(PPC_RAW_BLR()); } @@ -244,7 +240,6 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29)); EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array)); EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs))); - EMIT(PPC_RAW_STW(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); /* * if (prog == NULL) @@ -255,19 +250,14 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o /* goto *(prog->bpf_func + prologue_size); */ EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func))); - - if (ctx->seen & SEEN_FUNC) - EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); - EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE)); - - if (ctx->seen & SEEN_FUNC) - EMIT(PPC_RAW_MTLR(_R0)); - EMIT(PPC_RAW_MTCTR(_R3)); EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1))); + /* Put tail_call_cnt in r4 */ + EMIT(PPC_RAW_MR(_R4, _R0)); + /* tear restore NVRs, ... */ bpf_jit_emit_common_epilogue(image, ctx); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index fa78595a6089..593cf09264d8 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -317,9 +317,9 @@ config SMP config NR_CPUS int "Maximum number of CPUs (2-512)" depends on SMP - range 2 512 if !SBI_V01 - range 2 32 if SBI_V01 && 32BIT - range 2 64 if SBI_V01 && 64BIT + range 2 512 if !RISCV_SBI_V01 + range 2 32 if RISCV_SBI_V01 && 32BIT + range 2 64 if RISCV_SBI_V01 && 64BIT default "32" if 32BIT default "64" if 64BIT diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 1b471ff73178..816e753de636 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -23,6 +23,7 @@ #define REG_L __REG_SEL(ld, lw) #define REG_S __REG_SEL(sd, sw) #define REG_SC __REG_SEL(sc.d, sc.w) +#define REG_AMOSWAP_AQ __REG_SEL(amoswap.d.aq, amoswap.w.aq) #define REG_ASM __REG_SEL(.dword, .word) #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) diff --git a/arch/riscv/include/asm/efi.h b/arch/riscv/include/asm/efi.h index f74879a8f1ea..e229d7be4b66 100644 --- a/arch/riscv/include/asm/efi.h +++ b/arch/riscv/include/asm/efi.h @@ -10,6 +10,7 @@ #include <asm/mmu_context.h> #include <asm/ptrace.h> #include <asm/tlbflush.h> +#include <asm/pgalloc.h> #ifdef CONFIG_EFI extern void efi_init(void); @@ -20,7 +21,10 @@ extern void efi_init(void); int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); -#define arch_efi_call_virt_setup() efi_virtmap_load() +#define arch_efi_call_virt_setup() ({ \ + sync_kernel_mappings(efi_mm.pgd); \ + efi_virtmap_load(); \ + }) #define arch_efi_call_virt_teardown() efi_virtmap_unload() #define ARCH_EFI_IRQ_FLAGS_MASK (SR_IE | SR_SPIE) diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 947f23d7b6af..59dc12b5b7e8 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -127,6 +127,13 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) #define __p4d_free_tlb(tlb, p4d, addr) p4d_free((tlb)->mm, p4d) #endif /* __PAGETABLE_PMD_FOLDED */ +static inline void sync_kernel_mappings(pgd_t *pgd) +{ + memcpy(pgd + USER_PTRS_PER_PGD, + init_mm.pgd + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); +} + static inline pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *pgd; @@ -135,9 +142,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) if (likely(pgd != NULL)) { memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); /* Copy kernel mappings */ - memcpy(pgd + USER_PTRS_PER_PGD, - init_mm.pgd + USER_PTRS_PER_PGD, - (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + sync_kernel_mappings(pgd); } return pgd; } diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 7ec936910a96..92ec2d9d7273 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -600,6 +600,7 @@ static inline int pmd_dirty(pmd_t pmd) return pte_dirty(pmd_pte(pmd)); } +#define pmd_young pmd_young static inline int pmd_young(pmd_t pmd) { return pte_young(pmd_pte(pmd)); diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index d3443be7eedc..3831b638ecab 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -50,6 +50,9 @@ void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops); /* Clear IPI for current CPU */ void riscv_clear_ipi(void); +/* Check other CPUs stop or not */ +bool smp_crash_stop_failed(void); + /* Secondary hart entry */ asmlinkage void smp_callin(void); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index b9eda3fcbd6d..186abd146eaf 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -404,6 +404,19 @@ handle_syscall_trace_exit: #ifdef CONFIG_VMAP_STACK handle_kernel_stack_overflow: + /* + * Takes the psuedo-spinlock for the shadow stack, in case multiple + * harts are concurrently overflowing their kernel stacks. We could + * store any value here, but since we're overflowing the kernel stack + * already we only have SP to use as a scratch register. So we just + * swap in the address of the spinlock, as that's definately non-zero. + * + * Pairs with a store_release in handle_bad_stack(). + */ +1: la sp, spin_shadow_stack + REG_AMOSWAP_AQ sp, sp, (sp) + bnez sp, 1b + la sp, shadow_stack addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index ee79e6839b86..2d139b724bc8 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -15,6 +15,8 @@ #include <linux/compiler.h> /* For unreachable() */ #include <linux/cpu.h> /* For cpu_down() */ #include <linux/reboot.h> +#include <linux/interrupt.h> +#include <linux/irq.h> /* * kexec_image_info - Print received image details @@ -138,20 +140,35 @@ void machine_shutdown(void) #endif } -/* Override the weak function in kernel/panic.c */ -void crash_smp_send_stop(void) +static void machine_kexec_mask_interrupts(void) { - static int cpus_stopped; + unsigned int i; + struct irq_desc *desc; - /* - * This function can be called twice in panic path, but obviously - * we execute this only once. - */ - if (cpus_stopped) - return; + for_each_irq_desc(i, desc) { + struct irq_chip *chip; + int ret; + + chip = irq_desc_get_chip(desc); + if (!chip) + continue; + + /* + * First try to remove the active state. If this + * fails, try to EOI the interrupt. + */ + ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false); + + if (ret && irqd_irq_inprogress(&desc->irq_data) && + chip->irq_eoi) + chip->irq_eoi(&desc->irq_data); - smp_send_stop(); - cpus_stopped = 1; + if (chip->irq_mask) + chip->irq_mask(&desc->irq_data); + + if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) + chip->irq_disable(&desc->irq_data); + } } /* @@ -169,6 +186,8 @@ machine_crash_shutdown(struct pt_regs *regs) crash_smp_send_stop(); crash_save_cpu(regs, smp_processor_id()); + machine_kexec_mask_interrupts(); + pr_info("Starting crashdump kernel...\n"); } @@ -195,6 +214,11 @@ machine_kexec(struct kimage *image) void *control_code_buffer = page_address(image->control_code_page); riscv_kexec_method kexec_method = NULL; +#ifdef CONFIG_SMP + WARN(smp_crash_stop_failed(), + "Some CPUs may be stale, kdump will be unreliable.\n"); +#endif + if (image->type != KEXEC_TYPE_CRASH) kexec_method = control_code_buffer; else diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 67ec1fadcfe2..86acd690d529 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -322,10 +322,11 @@ subsys_initcall(topology_init); void free_initmem(void) { - if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) - set_kernel_memory(lm_alias(__init_begin), lm_alias(__init_end), - IS_ENABLED(CONFIG_64BIT) ? - set_memory_rw : set_memory_rw_nx); + if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) { + set_kernel_memory(lm_alias(__init_begin), lm_alias(__init_end), set_memory_rw_nx); + if (IS_ENABLED(CONFIG_64BIT)) + set_kernel_memory(__init_begin, __init_end, set_memory_nx); + } free_initmem_default(POISON_FREE_INITMEM); } diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 760a64518c58..8c3b59f1f9b8 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -12,6 +12,7 @@ #include <linux/clockchips.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/kexec.h> #include <linux/profile.h> #include <linux/smp.h> #include <linux/sched.h> @@ -22,11 +23,13 @@ #include <asm/sbi.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cpu_ops.h> enum ipi_message_type { IPI_RESCHEDULE, IPI_CALL_FUNC, IPI_CPU_STOP, + IPI_CPU_CRASH_STOP, IPI_IRQ_WORK, IPI_TIMER, IPI_MAX @@ -71,6 +74,32 @@ static void ipi_stop(void) wait_for_interrupt(); } +#ifdef CONFIG_KEXEC_CORE +static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0); + +static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) +{ + crash_save_cpu(regs, cpu); + + atomic_dec(&waiting_for_crash_ipi); + + local_irq_disable(); + +#ifdef CONFIG_HOTPLUG_CPU + if (cpu_has_hotplug(cpu)) + cpu_ops[cpu]->cpu_stop(); +#endif + + for(;;) + wait_for_interrupt(); +} +#else +static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) +{ + unreachable(); +} +#endif + static const struct riscv_ipi_ops *ipi_ops __ro_after_init; void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops) @@ -124,8 +153,9 @@ void arch_irq_work_raise(void) void handle_IPI(struct pt_regs *regs) { - unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; - unsigned long *stats = ipi_data[smp_processor_id()].stats; + unsigned int cpu = smp_processor_id(); + unsigned long *pending_ipis = &ipi_data[cpu].bits; + unsigned long *stats = ipi_data[cpu].stats; riscv_clear_ipi(); @@ -154,6 +184,10 @@ void handle_IPI(struct pt_regs *regs) ipi_stop(); } + if (ops & (1 << IPI_CPU_CRASH_STOP)) { + ipi_cpu_crash_stop(cpu, get_irq_regs()); + } + if (ops & (1 << IPI_IRQ_WORK)) { stats[IPI_IRQ_WORK]++; irq_work_run(); @@ -176,6 +210,7 @@ static const char * const ipi_names[] = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNC] = "Function call interrupts", [IPI_CPU_STOP] = "CPU stop interrupts", + [IPI_CPU_CRASH_STOP] = "CPU stop (for crash dump) interrupts", [IPI_IRQ_WORK] = "IRQ work interrupts", [IPI_TIMER] = "Timer broadcast interrupts", }; @@ -235,6 +270,64 @@ void smp_send_stop(void) cpumask_pr_args(cpu_online_mask)); } +#ifdef CONFIG_KEXEC_CORE +/* + * The number of CPUs online, not counting this CPU (which may not be + * fully online and so not counted in num_online_cpus()). + */ +static inline unsigned int num_other_online_cpus(void) +{ + unsigned int this_cpu_online = cpu_online(smp_processor_id()); + + return num_online_cpus() - this_cpu_online; +} + +void crash_smp_send_stop(void) +{ + static int cpus_stopped; + cpumask_t mask; + unsigned long timeout; + + /* + * This function can be called twice in panic path, but obviously + * we execute this only once. + */ + if (cpus_stopped) + return; + + cpus_stopped = 1; + + /* + * If this cpu is the only one alive at this point in time, online or + * not, there are no stop messages to be sent around, so just back out. + */ + if (num_other_online_cpus() == 0) + return; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + atomic_set(&waiting_for_crash_ipi, num_other_online_cpus()); + + pr_crit("SMP: stopping secondary CPUs\n"); + send_ipi_mask(&mask, IPI_CPU_CRASH_STOP); + + /* Wait up to one second for other CPUs to stop */ + timeout = USEC_PER_SEC; + while ((atomic_read(&waiting_for_crash_ipi) > 0) && timeout--) + udelay(1); + + if (atomic_read(&waiting_for_crash_ipi) > 0) + pr_warn("SMP: failed to stop secondary CPUs %*pbl\n", + cpumask_pr_args(&mask)); +} + +bool smp_crash_stop_failed(void) +{ + return (atomic_read(&waiting_for_crash_ipi) > 0); +} +#endif + void smp_send_reschedule(int cpu) { send_ipi_single(cpu, IPI_RESCHEDULE); diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index f3e96d60a2ff..7abd8e4c4df6 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -221,11 +221,29 @@ asmlinkage unsigned long get_overflow_stack(void) OVERFLOW_STACK_SIZE; } +/* + * A pseudo spinlock to protect the shadow stack from being used by multiple + * harts concurrently. This isn't a real spinlock because the lock side must + * be taken without a valid stack and only a single register, it's only taken + * while in the process of panicing anyway so the performance and error + * checking a proper spinlock gives us doesn't matter. + */ +unsigned long spin_shadow_stack; + asmlinkage void handle_bad_stack(struct pt_regs *regs) { unsigned long tsk_stk = (unsigned long)current->stack; unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); + /* + * We're done with the shadow stack by this point, as we're on the + * overflow stack. Tell any other concurrent overflowing harts that + * they can proceed with panicing by releasing the pseudo-spinlock. + * + * This pairs with an amoswap.aq in handle_kernel_stack_overflow. + */ + smp_store_release(&spin_shadow_stack, 0); + console_verbose(); pr_emerg("Insufficient stack space to handle exception!\n"); diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index db6548509bb3..06e6b27f3bcc 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -17,6 +17,7 @@ vdso-syms += flush_icache obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o ccflags-y := -fno-stack-protector +ccflags-y += -DDISABLE_BRANCH_PROFILING ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index f1cb9391190d..11e901286414 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -763,6 +763,7 @@ static inline int pmd_dirty(pmd_t pmd) return (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0; } +#define pmd_young pmd_young static inline int pmd_young(pmd_t pmd) { return (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0; diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 94138f8f0c1c..ace2541ababd 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -546,8 +546,10 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_CEI)) scb_s->eca |= scb_o->eca & ECA_CEI; /* Epoch Extension */ - if (test_kvm_facility(vcpu->kvm, 139)) + if (test_kvm_facility(vcpu->kvm, 139)) { scb_s->ecd |= scb_o->ecd & ECD_MEF; + scb_s->epdx = scb_o->epdx; + } /* etoken */ if (test_kvm_facility(vcpu->kvm, 156)) diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index a779418ceba9..3bc9736bddb1 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -693,6 +693,7 @@ static inline unsigned long pmd_dirty(pmd_t pmd) return pte_dirty(pte); } +#define pmd_young pmd_young static inline unsigned long pmd_young(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index c936ce9f0c47..dfdb103ae4f6 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -321,7 +321,7 @@ static inline void indirect_branch_prediction_barrier(void) /* The Intel SPEC CTRL MSR base value cache */ extern u64 x86_spec_ctrl_base; DECLARE_PER_CPU(u64, x86_spec_ctrl_current); -extern void write_spec_ctrl_current(u64 val, bool force); +extern void update_spec_ctrl_cond(u64 val); extern u64 spec_ctrl_current(void); /* diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 5059799bebe3..286a71810f9e 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -139,6 +139,7 @@ static inline int pmd_dirty(pmd_t pmd) return pmd_flags(pmd) & _PAGE_DIRTY; } +#define pmd_young pmd_young static inline int pmd_young(pmd_t pmd) { return pmd_flags(pmd) & _PAGE_ACCESSED; @@ -1438,6 +1439,14 @@ static inline bool arch_has_hw_pte_young(void) return true; } +#ifdef CONFIG_XEN_PV +#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young +static inline bool arch_has_hw_nonleaf_pmd_young(void) +{ + return !cpu_feature_enabled(X86_FEATURE_XENPV); +} +#endif + #ifdef CONFIG_PAGE_TABLE_CHECK static inline bool pte_user_accessible_page(pte_t pte) { diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 3e3230cccaa7..6daf84229548 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -60,11 +60,18 @@ EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); static DEFINE_MUTEX(spec_ctrl_mutex); +/* Update SPEC_CTRL MSR and its cached copy unconditionally */ +static void update_spec_ctrl(u64 val) +{ + this_cpu_write(x86_spec_ctrl_current, val); + wrmsrl(MSR_IA32_SPEC_CTRL, val); +} + /* * Keep track of the SPEC_CTRL MSR value for the current task, which may differ * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). */ -void write_spec_ctrl_current(u64 val, bool force) +void update_spec_ctrl_cond(u64 val) { if (this_cpu_read(x86_spec_ctrl_current) == val) return; @@ -75,7 +82,7 @@ void write_spec_ctrl_current(u64 val, bool force) * When KERNEL_IBRS this MSR is written on return-to-user, unless * forced the update can be delayed until that time. */ - if (force || !cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) + if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) wrmsrl(MSR_IA32_SPEC_CTRL, val); } @@ -1328,7 +1335,7 @@ static void __init spec_ctrl_disable_kernel_rrsba(void) if (ia32_cap & ARCH_CAP_RRSBA) { x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; - write_spec_ctrl_current(x86_spec_ctrl_base, true); + update_spec_ctrl(x86_spec_ctrl_base); } } @@ -1450,7 +1457,7 @@ static void __init spectre_v2_select_mitigation(void) if (spectre_v2_in_ibrs_mode(mode)) { x86_spec_ctrl_base |= SPEC_CTRL_IBRS; - write_spec_ctrl_current(x86_spec_ctrl_base, true); + update_spec_ctrl(x86_spec_ctrl_base); } switch (mode) { @@ -1564,7 +1571,7 @@ static void __init spectre_v2_select_mitigation(void) static void update_stibp_msr(void * __unused) { u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); - write_spec_ctrl_current(val, true); + update_spec_ctrl(val); } /* Update x86_spec_ctrl_base in case SMT state changed. */ @@ -1797,7 +1804,7 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) x86_amd_ssb_disable(); } else { x86_spec_ctrl_base |= SPEC_CTRL_SSBD; - write_spec_ctrl_current(x86_spec_ctrl_base, true); + update_spec_ctrl(x86_spec_ctrl_base); } } @@ -2048,7 +2055,7 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) void x86_spec_ctrl_setup_ap(void) { if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) - write_spec_ctrl_current(x86_spec_ctrl_base, true); + update_spec_ctrl(x86_spec_ctrl_base); if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) x86_amd_ssb_disable(); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index c21b7347a26d..e436c9c1ef3b 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -600,7 +600,7 @@ static __always_inline void __speculation_ctrl_update(unsigned long tifp, } if (updmsr) - write_spec_ctrl_current(msr, false); + update_spec_ctrl_cond(msr); } static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2835bd796639..69227f77b201 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10574,8 +10574,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; vcpu->mmio_needed = 0; r = 0; + goto out; } - goto out; } if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { /* Page is swapped out. Do synthetic halt */ diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 23f49a2f4d14..6cceca64a6bc 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -562,17 +562,26 @@ static int initiator_cmp(void *priv, const struct list_head *a, { struct memory_initiator *ia; struct memory_initiator *ib; - unsigned long *p_nodes = priv; ia = list_entry(a, struct memory_initiator, node); ib = list_entry(b, struct memory_initiator, node); - set_bit(ia->processor_pxm, p_nodes); - set_bit(ib->processor_pxm, p_nodes); - return ia->processor_pxm - ib->processor_pxm; } +static int initiators_to_nodemask(unsigned long *p_nodes) +{ + struct memory_initiator *initiator; + + if (list_empty(&initiators)) + return -ENXIO; + + list_for_each_entry(initiator, &initiators, node) + set_bit(initiator->processor_pxm, p_nodes); + + return 0; +} + static void hmat_register_target_initiators(struct memory_target *target) { static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); @@ -609,7 +618,10 @@ static void hmat_register_target_initiators(struct memory_target *target) * initiators. */ bitmap_zero(p_nodes, MAX_NUMNODES); - list_sort(p_nodes, &initiators, initiator_cmp); + list_sort(NULL, &initiators, initiator_cmp); + if (initiators_to_nodemask(p_nodes) < 0) + return; + if (!access0done) { for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { loc = localities_types[i]; @@ -643,8 +655,9 @@ static void hmat_register_target_initiators(struct memory_target *target) /* Access 1 ignores Generic Initiators */ bitmap_zero(p_nodes, MAX_NUMNODES); - list_sort(p_nodes, &initiators, initiator_cmp); - best = 0; + if (initiators_to_nodemask(p_nodes) < 0) + return; + for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { loc = localities_types[i]; if (!loc) diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index ddf17e2d266c..b9e336bacf17 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -109,7 +109,7 @@ struct clk *ahci_platform_find_clk(struct ahci_host_priv *hpriv, const char *con int i; for (i = 0; i < hpriv->n_clks; i++) { - if (!strcmp(hpriv->clks[i].id, con_id)) + if (hpriv->clks[i].id && !strcmp(hpriv->clks[i].id, con_id)) return hpriv->clks[i].clk; } diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 271963805a38..f05018988a17 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -2056,6 +2056,11 @@ static int btusb_setup_csr(struct hci_dev *hdev) rp = (struct hci_rp_read_local_version *)skb->data; + bt_dev_info(hdev, "CSR: Setting up dongle with HCI ver=%u rev=%04x; LMP ver=%u subver=%04x; manufacturer=%u", + le16_to_cpu(rp->hci_ver), le16_to_cpu(rp->hci_rev), + le16_to_cpu(rp->lmp_ver), le16_to_cpu(rp->lmp_subver), + le16_to_cpu(rp->manufacturer)); + /* Detect a wide host of Chinese controllers that aren't CSR. * * Known fake bcdDevices: 0x0100, 0x0134, 0x1915, 0x2520, 0x7558, 0x8891 @@ -2118,6 +2123,7 @@ static int btusb_setup_csr(struct hci_dev *hdev) * without these the controller will lock up. */ set_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks); + set_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks); set_bit(HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL, &hdev->quirks); set_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks); diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index 1621ce818705..d69905233aff 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -401,13 +401,14 @@ int tpm_pm_suspend(struct device *dev) !pm_suspend_via_firmware()) goto suspended; - if (!tpm_chip_start(chip)) { + rc = tpm_try_get_ops(chip); + if (!rc) { if (chip->flags & TPM_CHIP_FLAG_TPM2) tpm2_shutdown(chip, TPM2_SU_STATE); else rc = tpm1_pm_suspend(chip, tpm_suspend_pcr); - tpm_chip_stop(chip); + tpm_put_ops(chip); } suspended: diff --git a/drivers/clk/at91/at91rm9200.c b/drivers/clk/at91/at91rm9200.c index b174f727a8ef..16870943a13e 100644 --- a/drivers/clk/at91/at91rm9200.c +++ b/drivers/clk/at91/at91rm9200.c @@ -40,7 +40,7 @@ static const struct clk_pll_characteristics rm9200_pll_characteristics = { }; static const struct sck at91rm9200_systemck[] = { - { .n = "udpck", .p = "usbck", .id = 2 }, + { .n = "udpck", .p = "usbck", .id = 1 }, { .n = "uhpck", .p = "usbck", .id = 4 }, { .n = "pck0", .p = "prog0", .id = 8 }, { .n = "pck1", .p = "prog1", .id = 9 }, diff --git a/drivers/clk/qcom/gcc-sc8280xp.c b/drivers/clk/qcom/gcc-sc8280xp.c index a18ed88f3b82..b3198784e1c3 100644 --- a/drivers/clk/qcom/gcc-sc8280xp.c +++ b/drivers/clk/qcom/gcc-sc8280xp.c @@ -5364,6 +5364,8 @@ static struct clk_branch gcc_ufs_1_card_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_ufs_1_card_clkref_clk", + .parent_data = &gcc_parent_data_tcxo, + .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -5432,6 +5434,8 @@ static struct clk_branch gcc_ufs_card_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_ufs_card_clkref_clk", + .parent_data = &gcc_parent_data_tcxo, + .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -5848,6 +5852,8 @@ static struct clk_branch gcc_ufs_ref_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(const struct clk_init_data) { .name = "gcc_ufs_ref_clkref_clk", + .parent_data = &gcc_parent_data_tcxo, + .num_parents = 1, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index 7cf5e130e92f..0f21a8a767ac 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -11,7 +11,6 @@ #include <linux/kernel.h> #include <linux/ktime.h> #include <linux/pm_domain.h> -#include <linux/pm_runtime.h> #include <linux/regmap.h> #include <linux/regulator/consumer.h> #include <linux/reset-controller.h> @@ -56,22 +55,6 @@ enum gdsc_status { GDSC_ON }; -static int gdsc_pm_runtime_get(struct gdsc *sc) -{ - if (!sc->dev) - return 0; - - return pm_runtime_resume_and_get(sc->dev); -} - -static int gdsc_pm_runtime_put(struct gdsc *sc) -{ - if (!sc->dev) - return 0; - - return pm_runtime_put_sync(sc->dev); -} - /* Returns 1 if GDSC status is status, 0 if not, and < 0 on error */ static int gdsc_check_status(struct gdsc *sc, enum gdsc_status status) { @@ -271,8 +254,9 @@ static void gdsc_retain_ff_on(struct gdsc *sc) regmap_update_bits(sc->regmap, sc->gdscr, mask, mask); } -static int _gdsc_enable(struct gdsc *sc) +static int gdsc_enable(struct generic_pm_domain *domain) { + struct gdsc *sc = domain_to_gdsc(domain); int ret; if (sc->pwrsts == PWRSTS_ON) @@ -328,22 +312,11 @@ static int _gdsc_enable(struct gdsc *sc) return 0; } -static int gdsc_enable(struct generic_pm_domain *domain) +static int gdsc_disable(struct generic_pm_domain *domain) { struct gdsc *sc = domain_to_gdsc(domain); int ret; - ret = gdsc_pm_runtime_get(sc); - if (ret) - return ret; - - return _gdsc_enable(sc); -} - -static int _gdsc_disable(struct gdsc *sc) -{ - int ret; - if (sc->pwrsts == PWRSTS_ON) return gdsc_assert_reset(sc); @@ -388,18 +361,6 @@ static int _gdsc_disable(struct gdsc *sc) return 0; } -static int gdsc_disable(struct generic_pm_domain *domain) -{ - struct gdsc *sc = domain_to_gdsc(domain); - int ret; - - ret = _gdsc_disable(sc); - - gdsc_pm_runtime_put(sc); - - return ret; -} - static int gdsc_init(struct gdsc *sc) { u32 mask, val; @@ -447,11 +408,6 @@ static int gdsc_init(struct gdsc *sc) return ret; } - /* ...and the power-domain */ - ret = gdsc_pm_runtime_get(sc); - if (ret) - goto err_disable_supply; - /* * Votable GDSCs can be ON due to Vote from other masters. * If a Votable GDSC is ON, make sure we have a Vote. @@ -459,14 +415,14 @@ static int gdsc_init(struct gdsc *sc) if (sc->flags & VOTABLE) { ret = gdsc_update_collapse_bit(sc, false); if (ret) - goto err_put_rpm; + goto err_disable_supply; } /* Turn on HW trigger mode if supported */ if (sc->flags & HW_CTRL) { ret = gdsc_hwctrl(sc, true); if (ret < 0) - goto err_put_rpm; + goto err_disable_supply; } /* @@ -496,13 +452,10 @@ static int gdsc_init(struct gdsc *sc) ret = pm_genpd_init(&sc->pd, NULL, !on); if (ret) - goto err_put_rpm; + goto err_disable_supply; return 0; -err_put_rpm: - if (on) - gdsc_pm_runtime_put(sc); err_disable_supply: if (on && sc->rsupply) regulator_disable(sc->rsupply); @@ -541,8 +494,6 @@ int gdsc_register(struct gdsc_desc *desc, for (i = 0; i < num; i++) { if (!scs[i]) continue; - if (pm_runtime_enabled(dev)) - scs[i]->dev = dev; scs[i]->regmap = regmap; scs[i]->rcdev = rcdev; ret = gdsc_init(scs[i]); diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index 981a12c8502d..803512688336 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -30,7 +30,6 @@ struct reset_controller_dev; * @resets: ids of resets associated with this gdsc * @reset_count: number of @resets * @rcdev: reset controller - * @dev: the device holding the GDSC, used for pm_runtime calls */ struct gdsc { struct generic_pm_domain pd; @@ -74,7 +73,6 @@ struct gdsc { const char *supply; struct regulator *rsupply; - struct device *dev; }; struct gdsc_desc { diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c index 273f77d54dab..e6d6cbf8c4e6 100644 --- a/drivers/clk/samsung/clk-exynos-clkout.c +++ b/drivers/clk/samsung/clk-exynos-clkout.c @@ -81,17 +81,19 @@ MODULE_DEVICE_TABLE(of, exynos_clkout_ids); static int exynos_clkout_match_parent_dev(struct device *dev, u32 *mux_mask) { const struct exynos_clkout_variant *variant; + const struct of_device_id *match; if (!dev->parent) { dev_err(dev, "not instantiated from MFD\n"); return -EINVAL; } - variant = of_device_get_match_data(dev->parent); - if (!variant) { + match = of_match_device(exynos_clkout_ids, dev->parent); + if (!match) { dev_err(dev, "cannot match parent device\n"); return -EINVAL; } + variant = match->data; *mux_mask = variant->mux_mask; diff --git a/drivers/clk/samsung/clk-exynos7885.c b/drivers/clk/samsung/clk-exynos7885.c index 62ce6814f141..0d2a950ed184 100644 --- a/drivers/clk/samsung/clk-exynos7885.c +++ b/drivers/clk/samsung/clk-exynos7885.c @@ -231,7 +231,7 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { CLK_CON_DIV_PLL_SHARED0_DIV2, 0, 1), DIV(CLK_DOUT_SHARED0_DIV3, "dout_shared0_div3", "fout_shared0_pll", CLK_CON_DIV_PLL_SHARED0_DIV3, 0, 2), - DIV(CLK_DOUT_SHARED0_DIV4, "dout_shared0_div4", "fout_shared0_pll", + DIV(CLK_DOUT_SHARED0_DIV4, "dout_shared0_div4", "dout_shared0_div2", CLK_CON_DIV_PLL_SHARED0_DIV4, 0, 1), DIV(CLK_DOUT_SHARED0_DIV5, "dout_shared0_div5", "fout_shared0_pll", CLK_CON_DIV_PLL_SHARED0_DIV5, 0, 3), @@ -239,7 +239,7 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { CLK_CON_DIV_PLL_SHARED1_DIV2, 0, 1), DIV(CLK_DOUT_SHARED1_DIV3, "dout_shared1_div3", "fout_shared1_pll", CLK_CON_DIV_PLL_SHARED1_DIV3, 0, 2), - DIV(CLK_DOUT_SHARED1_DIV4, "dout_shared1_div4", "fout_shared1_pll", + DIV(CLK_DOUT_SHARED1_DIV4, "dout_shared1_div4", "dout_shared1_div2", CLK_CON_DIV_PLL_SHARED1_DIV4, 0, 1), /* CORE */ diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index 969a552da8d2..a0d66fabf073 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -51,7 +51,7 @@ static int riscv_clock_next_event(unsigned long delta, static unsigned int riscv_clock_event_irq; static DEFINE_PER_CPU(struct clock_event_device, riscv_clock_event) = { .name = "riscv_timer_clockevent", - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP, + .features = CLOCK_EVT_FEAT_ONESHOT, .rating = 100, .set_next_event = riscv_clock_next_event, }; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c index 7503f6b18ac5..a2aba0b0d68a 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -76,10 +76,6 @@ static int otx2_cpt_devlink_info_get(struct devlink *dl, struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; int err; - err = devlink_info_driver_name_put(req, "rvu_cptpf"); - if (err) - return err; - err = otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, "fw.ae", OTX2_CPT_AE_TYPES); if (err) diff --git a/drivers/dax/hmem/device.c b/drivers/dax/hmem/device.c index 97086fab698e..903325aac991 100644 --- a/drivers/dax/hmem/device.c +++ b/drivers/dax/hmem/device.c @@ -8,6 +8,13 @@ static bool nohmem; module_param_named(disable, nohmem, bool, 0444); +static struct resource hmem_active = { + .name = "HMEM devices", + .start = 0, + .end = -1, + .flags = IORESOURCE_MEM, +}; + void hmem_register_device(int target_nid, struct resource *r) { /* define a clean / non-busy resource for the platform device */ @@ -41,6 +48,12 @@ void hmem_register_device(int target_nid, struct resource *r) goto out_pdev; } + if (!__request_region(&hmem_active, res.start, resource_size(&res), + dev_name(&pdev->dev), 0)) { + dev_dbg(&pdev->dev, "hmem range %pr already active\n", &res); + goto out_active; + } + pdev->dev.numa_node = numa_map_to_online_node(target_nid); info = (struct memregion_info) { .target_node = target_nid, @@ -66,6 +79,8 @@ void hmem_register_device(int target_nid, struct resource *r) return; out_resource: + __release_region(&hmem_active, res.start, resource_size(&res)); +out_active: platform_device_put(pdev); out_pdev: memregion_free(id); @@ -73,15 +88,6 @@ out_pdev: static __init int hmem_register_one(struct resource *res, void *data) { - /* - * If the resource is not a top-level resource it was already - * assigned to a device by the HMAT parsing. - */ - if (res->parent != &iomem_resource) { - pr_info("HMEM: skip %pr, already claimed\n", res); - return 0; - } - hmem_register_device(phys_to_target_node(res->start), res); return 0; diff --git a/drivers/gpio/gpio-amd8111.c b/drivers/gpio/gpio-amd8111.c index 14e6b3e64add..6f3ded619c8b 100644 --- a/drivers/gpio/gpio-amd8111.c +++ b/drivers/gpio/gpio-amd8111.c @@ -226,7 +226,10 @@ found: ioport_unmap(gp.pm); goto out; } + return 0; + out: + pci_dev_put(pdev); return err; } @@ -234,6 +237,7 @@ static void __exit amd_gpio_exit(void) { gpiochip_remove(&gp.chip); ioport_unmap(gp.pm); + pci_dev_put(gp.pdev); } module_init(amd_gpio_init); diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c index 870910bb9dd3..200e43a6f4b4 100644 --- a/drivers/gpio/gpio-rockchip.c +++ b/drivers/gpio/gpio-rockchip.c @@ -610,6 +610,7 @@ static int rockchip_gpiolib_register(struct rockchip_pin_bank *bank) return -ENODATA; pctldev = of_pinctrl_get(pctlnp); + of_node_put(pctlnp); if (!pctldev) return -ENODEV; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 4756ea08894f..a70522aef355 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -526,12 +526,13 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) if (ret) return ret; + /* From this point, the .release() function cleans up gpio_device */ + gdev->dev.release = gpiodevice_release; + ret = gpiochip_sysfs_register(gdev); if (ret) goto err_remove_device; - /* From this point, the .release() function cleans up gpio_device */ - gdev->dev.release = gpiodevice_release; dev_dbg(&gdev->dev, "registered GPIOs %d to %d on %s\n", gdev->base, gdev->base + gdev->ngpio - 1, gdev->chip->label ? : "generic"); @@ -597,10 +598,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct fwnode_handle *fwnode = NULL; struct gpio_device *gdev; unsigned long flags; - int base = gc->base; unsigned int i; + u32 ngpios = 0; + int base = 0; int ret = 0; - u32 ngpios; if (gc->fwnode) fwnode = gc->fwnode; @@ -647,17 +648,12 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, else gdev->owner = THIS_MODULE; - gdev->descs = kcalloc(gc->ngpio, sizeof(gdev->descs[0]), GFP_KERNEL); - if (!gdev->descs) { - ret = -ENOMEM; - goto err_free_dev_name; - } - /* * Try the device properties if the driver didn't supply the number * of GPIO lines. */ - if (gc->ngpio == 0) { + ngpios = gc->ngpio; + if (ngpios == 0) { ret = device_property_read_u32(&gdev->dev, "ngpios", &ngpios); if (ret == -ENODATA) /* @@ -668,7 +664,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, */ ngpios = 0; else if (ret) - goto err_free_descs; + goto err_free_dev_name; gc->ngpio = ngpios; } @@ -676,13 +672,19 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (gc->ngpio == 0) { chip_err(gc, "tried to insert a GPIO chip with zero lines\n"); ret = -EINVAL; - goto err_free_descs; + goto err_free_dev_name; } if (gc->ngpio > FASTPATH_NGPIO) chip_warn(gc, "line cnt %u is greater than fast path cnt %u\n", gc->ngpio, FASTPATH_NGPIO); + gdev->descs = kcalloc(gc->ngpio, sizeof(*gdev->descs), GFP_KERNEL); + if (!gdev->descs) { + ret = -ENOMEM; + goto err_free_dev_name; + } + gdev->label = kstrdup_const(gc->label ?: "unknown", GFP_KERNEL); if (!gdev->label) { ret = -ENOMEM; @@ -701,11 +703,13 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, * it may be a pipe dream. It will not happen before we get rid * of the sysfs interface anyways. */ + base = gc->base; if (base < 0) { base = gpiochip_find_base(gc->ngpio); if (base < 0) { - ret = base; spin_unlock_irqrestore(&gpio_lock, flags); + ret = base; + base = 0; goto err_free_label; } /* @@ -816,6 +820,11 @@ err_remove_of_chip: err_free_gpiochip_mask: gpiochip_remove_pin_ranges(gc); gpiochip_free_valid_mask(gc); + if (gdev->dev.release) { + /* release() has been registered by gpiochip_setup_dev() */ + put_device(&gdev->dev); + goto err_print_message; + } err_remove_from_list: spin_lock_irqsave(&gpio_lock, flags); list_del(&gdev->list); @@ -829,13 +838,14 @@ err_free_dev_name: err_free_ida: ida_free(&gpio_ida, gdev->id); err_free_gdev: + kfree(gdev); +err_print_message: /* failures here can mean systems won't boot... */ if (ret != -EPROBE_DEFER) { pr_err("%s: GPIOs %d..%d (%s) failed to register, %d\n", __func__, - gdev->base, gdev->base + gdev->ngpio - 1, + base, base + (int)ngpios - 1, gc->label ? : "generic", ret); } - kfree(gdev); return ret; } EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 0b52af415b28..ce64ca1c6e66 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -156,6 +156,9 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev) break; case IP_VERSION(3, 0, 2): fw_name = FIRMWARE_VANGOGH; + if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) && + (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)) + adev->vcn.indirect_sram = true; break; case IP_VERSION(3, 0, 16): fw_name = FIRMWARE_DIMGREY_CAVEFISH; diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 6925e0280dbe..f4f3d2665a6b 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -5,6 +5,7 @@ menu "Display Engine Configuration" config DRM_AMD_DC bool "AMD DC - Enable new display engine" default y + depends on BROKEN || !CC_IS_CLANG || X86_64 || SPARC64 || ARM64 select SND_HDA_COMPONENT if SND_HDA_CORE select DRM_AMD_DC_DCN if (X86 || PPC_LONG_DOUBLE_128) help @@ -12,6 +13,12 @@ config DRM_AMD_DC support for AMDGPU. This adds required support for Vega and Raven ASICs. + calculate_bandwidth() is presently broken on all !(X86_64 || SPARC64 || ARM64) + architectures built with Clang (all released versions), whereby the stack + frame gets blown up to well over 5k. This would cause an immediate kernel + panic on most architectures. We'll revert this when the following bug report + has been resolved: https://github.com/llvm/llvm-project/issues/41896. + config DRM_AMD_DC_DCN def_bool n help diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 461c62c88413..de77054195c6 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -3723,12 +3723,16 @@ out: static u8 bigjoiner_pipes(struct drm_i915_private *i915) { + u8 pipes; + if (DISPLAY_VER(i915) >= 12) - return BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D); + pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D); else if (DISPLAY_VER(i915) >= 11) - return BIT(PIPE_B) | BIT(PIPE_C); + pipes = BIT(PIPE_B) | BIT(PIPE_C); else - return 0; + pipes = 0; + + return pipes & RUNTIME_INFO(i915)->pipe_mask; } static bool transcoder_ddi_func_is_enabled(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index d0b03a928b9a..7caa3412a244 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -625,8 +625,13 @@ int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) return -EINTR; } - return timeout ? timeout : intel_uc_wait_for_idle(>->uc, - remaining_timeout); + if (timeout) + return timeout; + + if (remaining_timeout < 0) + remaining_timeout = 0; + + return intel_uc_wait_for_idle(>->uc, remaining_timeout); } int intel_gt_init(struct intel_gt *gt) @@ -1017,6 +1022,11 @@ static void mmio_invalidate_full(struct intel_gt *gt) if (!i915_mmio_reg_offset(rb.reg)) continue; + if (GRAPHICS_VER(i915) == 12 && (engine->class == VIDEO_DECODE_CLASS || + engine->class == VIDEO_ENHANCEMENT_CLASS || + engine->class == COMPUTE_CLASS)) + rb.bit = _MASKED_BIT_ENABLE(rb.bit); + intel_uncore_write_fw(uncore, rb.reg, rb.bit); awake |= engine->mask; } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.c b/drivers/gpu/drm/i915/gt/intel_gt_requests.c index edb881d75630..1dfd01668c79 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_requests.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.c @@ -199,7 +199,7 @@ out_active: spin_lock(&timelines->lock); if (remaining_timeout) *remaining_timeout = timeout; - return active_count ? timeout : 0; + return active_count ? timeout ?: -ETIME : 0; } static void retire_work_handler(struct work_struct *work) diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 2403ccd52c74..bba8cb6e8ae4 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -471,8 +471,7 @@ static int xelpdp_get_dram_info(struct drm_i915_private *i915) u32 val = intel_uncore_read(&i915->uncore, MTL_MEM_SS_INFO_GLOBAL); struct dram_info *dram_info = &i915->dram_info; - val = REG_FIELD_GET(MTL_DDR_TYPE_MASK, val); - switch (val) { + switch (REG_FIELD_GET(MTL_DDR_TYPE_MASK, val)) { case 0: dram_info->type = INTEL_DRAM_DDR4; break; diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 9c1d31f63f85..bd47628da6be 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -1315,6 +1315,9 @@ static s32 snto32(__u32 value, unsigned n) if (!value || !n) return 0; + if (n > 32) + n = 32; + switch (n) { case 8: return ((__s8)value); case 16: return ((__s16)value); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index dad953f66996..8f58c3c1bec3 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -274,6 +274,7 @@ #define USB_DEVICE_ID_CH_AXIS_295 0x001c #define USB_VENDOR_ID_CHERRY 0x046a +#define USB_DEVICE_ID_CHERRY_MOUSE_000C 0x000c #define USB_DEVICE_ID_CHERRY_CYMOTION 0x0023 #define USB_DEVICE_ID_CHERRY_CYMOTION_SOLAR 0x0027 @@ -917,6 +918,7 @@ #define USB_DEVICE_ID_MS_XBOX_ONE_S_CONTROLLER 0x02fd #define USB_DEVICE_ID_MS_PIXART_MOUSE 0x00cb #define USB_DEVICE_ID_8BITDO_SN30_PRO_PLUS 0x02e0 +#define USB_DEVICE_ID_MS_MOUSE_0783 0x0783 #define USB_VENDOR_ID_MOJO 0x8282 #define USB_DEVICE_ID_RETRO_ADAPTER 0x3201 @@ -1215,6 +1217,7 @@ #define USB_DEVICE_ID_SYNAPTICS_DELL_K15A 0x6e21 #define USB_DEVICE_ID_SYNAPTICS_ACER_ONE_S1002 0x73f4 #define USB_DEVICE_ID_SYNAPTICS_ACER_ONE_S1003 0x73f5 +#define USB_DEVICE_ID_SYNAPTICS_ACER_SWITCH5_017 0x73f6 #define USB_DEVICE_ID_SYNAPTICS_ACER_SWITCH5 0x81a7 #define USB_VENDOR_ID_TEXAS_INSTRUMENTS 0x2047 @@ -1381,6 +1384,7 @@ #define USB_VENDOR_ID_PRIMAX 0x0461 #define USB_DEVICE_ID_PRIMAX_MOUSE_4D22 0x4d22 +#define USB_DEVICE_ID_PRIMAX_MOUSE_4E2A 0x4e2a #define USB_DEVICE_ID_PRIMAX_KEYBOARD 0x4e05 #define USB_DEVICE_ID_PRIMAX_REZEL 0x4e72 #define USB_DEVICE_ID_PRIMAX_PIXART_MOUSE_4D0F 0x4d0f diff --git a/drivers/hid/hid-ite.c b/drivers/hid/hid-ite.c index 430fa4f52ed3..75ebfcf31889 100644 --- a/drivers/hid/hid-ite.c +++ b/drivers/hid/hid-ite.c @@ -121,6 +121,11 @@ static const struct hid_device_id ite_devices[] = { USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_ACER_ONE_S1003), .driver_data = QUIRK_TOUCHPAD_ON_OFF_REPORT }, + /* ITE8910 USB kbd ctlr, with Synaptics touchpad connected to it. */ + { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, + USB_VENDOR_ID_SYNAPTICS, + USB_DEVICE_ID_SYNAPTICS_ACER_SWITCH5_017), + .driver_data = QUIRK_TOUCHPAD_ON_OFF_REPORT }, { } }; MODULE_DEVICE_TABLE(hid, ite_devices); diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c index 5e6a0cef2a06..e3fcf1353fb3 100644 --- a/drivers/hid/hid-lg4ff.c +++ b/drivers/hid/hid-lg4ff.c @@ -872,6 +872,12 @@ static ssize_t lg4ff_alternate_modes_store(struct device *dev, struct device_att return -ENOMEM; i = strlen(lbuf); + + if (i == 0) { + kfree(lbuf); + return -EINVAL; + } + if (lbuf[i-1] == '\n') { if (i == 1) { kfree(lbuf); diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 71a9c258a20b..8a2aac18dcc5 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -4269,21 +4269,6 @@ static void hidpp_remove(struct hid_device *hdev) mutex_destroy(&hidpp->send_mutex); } -static const struct hid_device_id unhandled_hidpp_devices[] = { - /* Logitech Harmony Adapter for PS3, handled in hid-sony */ - { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_HARMONY_PS3) }, - /* Handled in hid-generic */ - { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DINOVO_EDGE_KBD) }, - {} -}; - -static bool hidpp_match(struct hid_device *hdev, - bool ignore_special_driver) -{ - /* Refuse to handle devices handled by other HID drivers */ - return !hid_match_id(hdev, unhandled_hidpp_devices); -} - #define LDJ_DEVICE(product) \ HID_DEVICE(BUS_USB, HID_GROUP_LOGITECH_DJ_DEVICE, \ USB_VENDOR_ID_LOGITECH, (product)) @@ -4367,9 +4352,15 @@ static const struct hid_device_id hidpp_devices[] = { { /* MX5500 keyboard over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb30b), .driver_data = HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS }, - - { /* And try to enable HID++ for all the Logitech Bluetooth devices */ - HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_ANY, USB_VENDOR_ID_LOGITECH, HID_ANY_ID) }, + { /* M-RCQ142 V470 Cordless Laser Mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb008) }, + { /* MX Master mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb012) }, + { /* MX Ergo trackball over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb01d) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb01e) }, + { /* MX Master 3 mouse over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb023) }, {} }; @@ -4383,7 +4374,6 @@ static const struct hid_usage_id hidpp_usages[] = { static struct hid_driver hidpp_driver = { .name = "logitech-hidpp-device", .id_table = hidpp_devices, - .match = hidpp_match, .report_fixup = hidpp_report_fixup, .probe = hidpp_probe, .remove = hidpp_remove, diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 50e1c717fc0a..0e9702c7f7d6 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -54,6 +54,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_YOKE), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_PEDALS), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_THROTTLE), HID_QUIRK_NOGET }, + { HID_USB_DEVICE(USB_VENDOR_ID_CHERRY, USB_DEVICE_ID_CHERRY_MOUSE_000C), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB_RAPIDFIRE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB), HID_QUIRK_NO_INIT_REPORTS }, @@ -122,6 +123,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C05A), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C06A), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_MCS, USB_DEVICE_ID_MCS_GAMEPADBLOCK), HID_QUIRK_MULTI_INPUT }, + { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_MOUSE_0783), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PIXART_MOUSE), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE3_COVER), HID_QUIRK_NO_INIT_REPORTS }, @@ -146,6 +148,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_MOUSE_4D22), HID_QUIRK_ALWAYS_POLL }, + { HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_MOUSE_4E2A), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_PIXART_MOUSE_4D0F), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_PIXART_MOUSE_4D65), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_PIXART_MOUSE_4E22), HID_QUIRK_ALWAYS_POLL }, diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c index 0fbc408c2607..7fa6fe04f1b2 100644 --- a/drivers/hid/hid-uclogic-core.c +++ b/drivers/hid/hid-uclogic-core.c @@ -192,6 +192,7 @@ static int uclogic_probe(struct hid_device *hdev, * than the pen, so use QUIRK_MULTI_INPUT for all tablets. */ hdev->quirks |= HID_QUIRK_MULTI_INPUT; + hdev->quirks |= HID_QUIRK_HIDINPUT_FORCE; /* Allocate and assign driver data */ drvdata = devm_kzalloc(&hdev->dev, sizeof(*drvdata), GFP_KERNEL); diff --git a/drivers/hid/hid-uclogic-rdesc.c b/drivers/hid/hid-uclogic-rdesc.c index 4bd54c4fb5b0..6b73eb0df6bd 100644 --- a/drivers/hid/hid-uclogic-rdesc.c +++ b/drivers/hid/hid-uclogic-rdesc.c @@ -1193,7 +1193,7 @@ __u8 *uclogic_rdesc_template_apply(const __u8 *template_ptr, p[sizeof(btn_head)] < param_num) { v = param_list[p[sizeof(btn_head)]]; put_unaligned((__u8)0x2A, p); /* Usage Maximum */ - put_unaligned_le16((__force u16)cpu_to_le16(v), p + 1); + put_unaligned((__force u16)cpu_to_le16(v), (s16 *)(p + 1)); p += sizeof(btn_head) + 1; } else { p++; diff --git a/drivers/hid/i2c-hid/Kconfig b/drivers/hid/i2c-hid/Kconfig index 5273ee2bb134..d65abe65ce73 100644 --- a/drivers/hid/i2c-hid/Kconfig +++ b/drivers/hid/i2c-hid/Kconfig @@ -66,6 +66,6 @@ endmenu config I2C_HID_CORE tristate - default y if I2C_HID_ACPI=y || I2C_HID_OF=y || I2C_HID_OF_GOODIX=y - default m if I2C_HID_ACPI=m || I2C_HID_OF=m || I2C_HID_OF_GOODIX=m + default y if I2C_HID_ACPI=y || I2C_HID_OF=y || I2C_HID_OF_ELAN=y || I2C_HID_OF_GOODIX=y + default m if I2C_HID_ACPI=m || I2C_HID_OF=m || I2C_HID_OF_ELAN=m || I2C_HID_OF_GOODIX=m select HID diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 81e688975c6a..a901e4e33d81 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -938,6 +938,8 @@ static int asus_ec_probe(struct platform_device *pdev) ec_data->nr_sensors = hweight_long(ec_data->board_info->sensors); ec_data->sensors = devm_kcalloc(dev, ec_data->nr_sensors, sizeof(struct ec_sensor), GFP_KERNEL); + if (!ec_data->sensors) + return -ENOMEM; status = setup_lock_data(dev); if (status) { diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 8bf32c6c85d9..9bee4d33fbdf 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -242,10 +242,13 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) */ if (host_bridge && host_bridge->vendor == PCI_VENDOR_ID_INTEL) { for (i = 0; i < ARRAY_SIZE(tjmax_pci_table); i++) { - if (host_bridge->device == tjmax_pci_table[i].device) + if (host_bridge->device == tjmax_pci_table[i].device) { + pci_dev_put(host_bridge); return tjmax_pci_table[i].tjmax; + } } } + pci_dev_put(host_bridge); for (i = 0; i < ARRAY_SIZE(tjmax_table); i++) { if (strstr(c->x86_model_id, tjmax_table[i].id)) @@ -533,6 +536,10 @@ static void coretemp_remove_core(struct platform_data *pdata, int indx) { struct temp_data *tdata = pdata->core_data[indx]; + /* if we errored on add then this is already gone */ + if (!tdata) + return; + /* Remove the sysfs attributes */ sysfs_remove_group(&pdata->hwmon_dev->kobj, &tdata->attr_group); diff --git a/drivers/hwmon/i5500_temp.c b/drivers/hwmon/i5500_temp.c index 05f68e9c9477..23b9f94fe0a9 100644 --- a/drivers/hwmon/i5500_temp.c +++ b/drivers/hwmon/i5500_temp.c @@ -117,7 +117,7 @@ static int i5500_temp_probe(struct pci_dev *pdev, u32 tstimer; s8 tsfsc; - err = pci_enable_device(pdev); + err = pcim_enable_device(pdev); if (err) { dev_err(&pdev->dev, "Failed to enable device\n"); return err; diff --git a/drivers/hwmon/ibmpex.c b/drivers/hwmon/ibmpex.c index f6ec165c0fa8..1837cccd993c 100644 --- a/drivers/hwmon/ibmpex.c +++ b/drivers/hwmon/ibmpex.c @@ -502,6 +502,7 @@ static void ibmpex_register_bmc(int iface, struct device *dev) return; out_register: + list_del(&data->list); hwmon_device_unregister(data->hwmon_dev); out_user: ipmi_destroy_user(data->user); diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c index 2a57f4b60c29..e06186986444 100644 --- a/drivers/hwmon/ina3221.c +++ b/drivers/hwmon/ina3221.c @@ -228,7 +228,7 @@ static int ina3221_read_value(struct ina3221_data *ina, unsigned int reg, * Shunt Voltage Sum register has 14-bit value with 1-bit shift * Other Shunt Voltage registers have 12 bits with 3-bit shift */ - if (reg == INA3221_SHUNT_SUM) + if (reg == INA3221_SHUNT_SUM || reg == INA3221_CRIT_SUM) *val = sign_extend32(regval >> 1, 14); else *val = sign_extend32(regval >> 3, 12); @@ -465,7 +465,7 @@ static int ina3221_write_curr(struct device *dev, u32 attr, * SHUNT_SUM: (1 / 40uV) << 1 = 1 / 20uV * SHUNT[1-3]: (1 / 40uV) << 3 = 1 / 5uV */ - if (reg == INA3221_SHUNT_SUM) + if (reg == INA3221_SHUNT_SUM || reg == INA3221_CRIT_SUM) regval = DIV_ROUND_CLOSEST(voltage_uv, 20) & 0xfffe; else regval = DIV_ROUND_CLOSEST(voltage_uv, 5) & 0xfff8; diff --git a/drivers/hwmon/ltc2947-core.c b/drivers/hwmon/ltc2947-core.c index 7404e974762f..2dbbbac9de09 100644 --- a/drivers/hwmon/ltc2947-core.c +++ b/drivers/hwmon/ltc2947-core.c @@ -396,7 +396,7 @@ static int ltc2947_read_temp(struct device *dev, const u32 attr, long *val, return ret; /* in milidegrees celcius, temp is given by: */ - *val = (__val * 204) + 550; + *val = (__val * 204) + 5500; return 0; } diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index fe0cd205502d..f58943cb1341 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -852,7 +852,8 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, CDNS_I2C_POLL_US, CDNS_I2C_TIMEOUT_US); if (ret) { ret = -EAGAIN; - i2c_recover_bus(adap); + if (id->adap.bus_recovery_info) + i2c_recover_bus(adap); goto out; } @@ -1263,8 +1264,13 @@ static int cdns_i2c_probe(struct platform_device *pdev) id->rinfo.pinctrl = devm_pinctrl_get(&pdev->dev); if (IS_ERR(id->rinfo.pinctrl)) { + int err = PTR_ERR(id->rinfo.pinctrl); + dev_info(&pdev->dev, "can't get pinctrl, bus recovery not supported\n"); - return PTR_ERR(id->rinfo.pinctrl); + if (err != -ENODEV) + return err; + } else { + id->adap.bus_recovery_info = &id->rinfo; } id->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &r_mem); @@ -1283,7 +1289,6 @@ static int cdns_i2c_probe(struct platform_device *pdev) id->adap.retries = 3; /* Default retry value. */ id->adap.algo_data = id; id->adap.dev.parent = &pdev->dev; - id->adap.bus_recovery_info = &id->rinfo; init_completion(&id->xfer_done); snprintf(id->adap.name, sizeof(id->adap.name), "Cadence I2C at %08lx", (unsigned long)r_mem->start); diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 3082183bd66a..fc70920c4dda 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1132,7 +1132,8 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, int i, result; unsigned int temp; int block_data = msgs->flags & I2C_M_RECV_LEN; - int use_dma = i2c_imx->dma && msgs->len >= DMA_THRESHOLD && !block_data; + int use_dma = i2c_imx->dma && msgs->flags & I2C_M_DMA_SAFE && + msgs->len >= DMA_THRESHOLD && !block_data; dev_dbg(&i2c_imx->adapter.dev, "<%s> write slave address: addr=0x%x\n", @@ -1298,7 +1299,8 @@ static int i2c_imx_xfer_common(struct i2c_adapter *adapter, result = i2c_imx_read(i2c_imx, &msgs[i], is_lastmsg, atomic); } else { if (!atomic && - i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD) + i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD && + msgs[i].flags & I2C_M_DMA_SAFE) result = i2c_imx_dma_write(i2c_imx, &msgs[i]); else result = i2c_imx_write(i2c_imx, &msgs[i], atomic); diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c index 0c365b57d957..83457359ec45 100644 --- a/drivers/i2c/busses/i2c-npcm7xx.c +++ b/drivers/i2c/busses/i2c-npcm7xx.c @@ -2393,8 +2393,17 @@ static struct platform_driver npcm_i2c_bus_driver = { static int __init npcm_i2c_init(void) { + int ret; + npcm_i2c_debugfs_dir = debugfs_create_dir("npcm_i2c", NULL); - return platform_driver_register(&npcm_i2c_bus_driver); + + ret = platform_driver_register(&npcm_i2c_bus_driver); + if (ret) { + debugfs_remove_recursive(npcm_i2c_debugfs_dir); + return ret; + } + + return 0; } module_init(npcm_i2c_init); diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index 84a77512614d..8fce98bb77ff 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -626,7 +626,6 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i dev_err(gi2c->se.dev, "I2C timeout gpi flags:%d addr:0x%x\n", gi2c->cur->flags, gi2c->cur->addr); gi2c->err = -ETIMEDOUT; - goto err; } if (gi2c->err) { diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 9aa7b9d9a485..13fafb74bab8 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -467,6 +467,7 @@ static int i2c_device_probe(struct device *dev) { struct i2c_client *client = i2c_verify_client(dev); struct i2c_driver *driver; + bool do_power_on; int status; if (!client) @@ -545,8 +546,8 @@ static int i2c_device_probe(struct device *dev) if (status < 0) goto err_clear_wakeup_irq; - status = dev_pm_domain_attach(&client->dev, - !i2c_acpi_waive_d0_probe(dev)); + do_power_on = !i2c_acpi_waive_d0_probe(dev); + status = dev_pm_domain_attach(&client->dev, do_power_on); if (status) goto err_clear_wakeup_irq; @@ -585,7 +586,7 @@ static int i2c_device_probe(struct device *dev) err_release_driver_resources: devres_release_group(&client->dev, client->devres_group_id); err_detach_pm_domain: - dev_pm_domain_detach(&client->dev, !i2c_acpi_waive_d0_probe(dev)); + dev_pm_domain_detach(&client->dev, do_power_on); err_clear_wakeup_irq: dev_pm_clear_wake_irq(&client->dev); device_init_wakeup(&client->dev, false); @@ -610,7 +611,7 @@ static void i2c_device_remove(struct device *dev) devres_release_group(&client->dev, client->devres_group_id); - dev_pm_domain_detach(&client->dev, !i2c_acpi_waive_d0_probe(dev)); + dev_pm_domain_detach(&client->dev, true); dev_pm_clear_wake_irq(&client->dev); device_init_wakeup(&client->dev, false); diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index bc97958818bb..e6e021af6aa9 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -230,8 +230,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, struct ib_umem_odp *umem_odp = container_of(mni, struct ib_umem_odp, notifier); struct mlx5_ib_mr *mr; - const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / - sizeof(struct mlx5_mtt)) - 1; + const u64 umr_block_mask = MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1; u64 idx = 0, blk_start_idx = 0; u64 invalidations = 0; unsigned long start; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index d5105b5c9979..029e9536ec28 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -418,7 +418,7 @@ int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd, } #define MLX5_MAX_UMR_CHUNK \ - ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_MTT_ALIGNMENT) + ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_FLEX_ALIGNMENT) #define MLX5_SPARE_UMR_CHUNK 0x10000 /* @@ -428,11 +428,11 @@ int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd, */ static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask) { - const size_t xlt_chunk_align = MLX5_UMR_MTT_ALIGNMENT / ent_size; + const size_t xlt_chunk_align = MLX5_UMR_FLEX_ALIGNMENT / ent_size; size_t size; void *res = NULL; - static_assert(PAGE_SIZE % MLX5_UMR_MTT_ALIGNMENT == 0); + static_assert(PAGE_SIZE % MLX5_UMR_FLEX_ALIGNMENT == 0); /* * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the @@ -666,7 +666,7 @@ int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) } final_size = (void *)cur_mtt - (void *)mtt; - sg.length = ALIGN(final_size, MLX5_UMR_MTT_ALIGNMENT); + sg.length = ALIGN(final_size, MLX5_UMR_FLEX_ALIGNMENT); memset(cur_mtt, 0, sg.length - final_size); mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags); @@ -690,7 +690,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) ? sizeof(struct mlx5_klm) : sizeof(struct mlx5_mtt); - const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; + const int page_align = MLX5_UMR_FLEX_ALIGNMENT / desc_size; struct mlx5_ib_dev *dev = mr_to_mdev(mr); struct device *ddev = &dev->mdev->pdev->dev; const int page_mask = page_align - 1; @@ -711,7 +711,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, if (WARN_ON(!mr->umem->is_odp)) return -EINVAL; - /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, + /* UMR copies MTTs in units of MLX5_UMR_FLEX_ALIGNMENT bytes, * so we need to align the offset and length accordingly */ if (idx & page_mask) { @@ -748,7 +748,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); - sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT); + sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); if (pages_mapped + pages_iter >= pages_to_map) mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags); diff --git a/drivers/input/touchscreen/raydium_i2c_ts.c b/drivers/input/touchscreen/raydium_i2c_ts.c index 3a4952935366..3d9c5758d8a4 100644 --- a/drivers/input/touchscreen/raydium_i2c_ts.c +++ b/drivers/input/touchscreen/raydium_i2c_ts.c @@ -211,12 +211,14 @@ static int raydium_i2c_send(struct i2c_client *client, error = raydium_i2c_xfer(client, addr, xfer, ARRAY_SIZE(xfer)); if (likely(!error)) - return 0; + goto out; msleep(RM_RETRY_DELAY_MS); } while (++tries < RM_MAX_RETRIES); dev_err(&client->dev, "%s failed: %d\n", __func__, error); +out: + kfree(tx_buf); return error; } diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 5a8f780e7ffd..bc94059a5b87 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -820,6 +820,7 @@ int __init dmar_dev_scope_init(void) info = dmar_alloc_pci_notify_info(dev, BUS_NOTIFY_ADD_DEVICE); if (!info) { + pci_dev_put(dev); return dmar_dev_scope_status; } else { dmar_pci_bus_add_dev(info); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 996a8b5ee5ee..5287efe247b1 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1396,6 +1396,24 @@ static void domain_update_iotlb(struct dmar_domain *domain) spin_unlock_irqrestore(&domain->lock, flags); } +/* + * The extra devTLB flush quirk impacts those QAT devices with PCI device + * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device() + * check because it applies only to the built-in QAT devices and it doesn't + * grant additional privileges. + */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev) +{ + if (pdev->vendor != PCI_VENDOR_ID_INTEL) + return false; + + if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK) + return false; + + return true; +} + static void iommu_enable_pci_caps(struct device_domain_info *info) { struct pci_dev *pdev; @@ -1478,6 +1496,7 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, qdep = info->ats_qdep; qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, qdep, addr, mask); + quirk_extra_dev_tlb_flush(info, addr, mask, PASID_RID2PASID, qdep); } static void iommu_flush_dev_iotlb(struct dmar_domain *domain, @@ -3854,8 +3873,10 @@ static inline bool has_external_pci(void) struct pci_dev *pdev = NULL; for_each_pci_dev(pdev) - if (pdev->external_facing) + if (pdev->external_facing) { + pci_dev_put(pdev); return true; + } return false; } @@ -4490,9 +4511,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) if (dev_is_pci(dev)) { if (ecap_dev_iotlb_support(iommu->ecap) && pci_ats_supported(pdev) && - dmar_ats_supported(pdev, iommu)) + dmar_ats_supported(pdev, iommu)) { info->ats_supported = 1; - + info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev); + } if (sm_supported(iommu)) { if (pasid_supported(iommu)) { int features = pci_pasid_features(pdev); @@ -4931,3 +4953,48 @@ static void __init check_tylersburg_isoch(void) pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", vtisochctrl); } + +/* + * Here we deal with a device TLB defect where device may inadvertently issue ATS + * invalidation completion before posted writes initiated with translated address + * that utilized translations matching the invalidation address range, violating + * the invalidation completion ordering. + * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is + * vulnerable to this defect. In other words, any dTLB invalidation initiated not + * under the control of the trusted/privileged host device driver must use this + * quirk. + * Device TLBs are invalidated under the following six conditions: + * 1. Device driver does DMA API unmap IOVA + * 2. Device driver unbind a PASID from a process, sva_unbind_device() + * 3. PASID is torn down, after PASID cache is flushed. e.g. process + * exit_mmap() due to crash + * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where + * VM has to free pages that were unmapped + * 5. Userspace driver unmaps a DMA buffer + * 6. Cache invalidation in vSVA usage (upcoming) + * + * For #1 and #2, device drivers are responsible for stopping DMA traffic + * before unmap/unbind. For #3, iommu driver gets mmu_notifier to + * invalidate TLB the same way as normal user unmap which will use this quirk. + * The dTLB invalidation after PASID cache flush does not need this quirk. + * + * As a reminder, #6 will *NEED* this quirk as we enable nested translation. + */ +void quirk_extra_dev_tlb_flush(struct device_domain_info *info, + unsigned long address, unsigned long mask, + u32 pasid, u16 qdep) +{ + u16 sid; + + if (likely(!info->dtlb_extra_inval)) + return; + + sid = PCI_DEVID(info->bus, info->devfn); + if (pasid == PASID_RID2PASID) { + qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, + qdep, address, mask); + } else { + qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid, + pasid, qdep, address, mask); + } +} diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 92023dff9513..db9df7c3790c 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -623,6 +623,7 @@ struct device_domain_info { u8 pri_enabled:1; u8 ats_supported:1; u8 ats_enabled:1; + u8 dtlb_extra_inval:1; /* Quirk for devices need extra flush */ u8 ats_qdep; struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu; /* IOMMU used by this device */ @@ -728,6 +729,9 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, u32 pasid, u16 qdep, u64 addr, unsigned int size_order); +void quirk_extra_dev_tlb_flush(struct device_domain_info *info, + unsigned long address, unsigned long pages, + u32 pasid, u16 qdep); void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu, u32 pasid); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 7d08eb034f2d..03b25358946c 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -184,10 +184,13 @@ static void __flush_svm_range_dev(struct intel_svm *svm, return; qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); - if (info->ats_enabled) + if (info->ats_enabled) { qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, svm->pasid, sdev->qdep, address, order_base_2(pages)); + quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), + svm->pasid, sdev->qdep); + } } static void intel_flush_svm_range_dev(struct intel_svm *svm, @@ -745,12 +748,16 @@ bad_req: * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ - if (!pdev || intel_svm_prq_report(iommu, &pdev->dev, req)) - handle_bad_prq_event(iommu, req, QI_RESP_INVALID); + if (!pdev) + goto bad_req; - trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], - iommu->prq_seq_number++); + if (intel_svm_prq_report(iommu, &pdev->dev, req)) + handle_bad_prq_event(iommu, req, QI_RESP_INVALID); + else + trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, + req->priv_data[0], req->priv_data[1], + iommu->prq_seq_number++); + pci_dev_put(pdev); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; } diff --git a/drivers/media/common/videobuf2/frame_vector.c b/drivers/media/common/videobuf2/frame_vector.c index 542dde9d2609..144027035892 100644 --- a/drivers/media/common/videobuf2/frame_vector.c +++ b/drivers/media/common/videobuf2/frame_vector.c @@ -35,11 +35,7 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, struct frame_vector *vec) { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - int ret_pin_user_pages_fast = 0; - int ret = 0; - int err; + int ret; if (nr_frames == 0) return 0; @@ -52,57 +48,17 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, ret = pin_user_pages_fast(start, nr_frames, FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM, (struct page **)(vec->ptrs)); - if (ret > 0) { - vec->got_ref = true; - vec->is_pfns = false; - goto out_unlocked; - } - ret_pin_user_pages_fast = ret; - - mmap_read_lock(mm); - vec->got_ref = false; - vec->is_pfns = true; - ret = 0; - do { - unsigned long *nums = frame_vector_pfns(vec); - - vma = vma_lookup(mm, start); - if (!vma) - break; - - while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { - err = follow_pfn(vma, start, &nums[ret]); - if (err) { - if (ret) - goto out; - // If follow_pfn() returns -EINVAL, then this - // is not an IO mapping or a raw PFN mapping. - // In that case, return the original error from - // pin_user_pages_fast(). Otherwise this - // function would return -EINVAL when - // pin_user_pages_fast() returned -ENOMEM, - // which makes debugging hard. - if (err == -EINVAL && ret_pin_user_pages_fast) - ret = ret_pin_user_pages_fast; - else - ret = err; - goto out; - } - start += PAGE_SIZE; - ret++; - } - /* Bail out if VMA doesn't completely cover the tail page. */ - if (start < vma->vm_end) - break; - } while (ret < nr_frames); -out: - mmap_read_unlock(mm); -out_unlocked: - if (!ret) - ret = -EFAULT; - if (ret > 0) - vec->nr_frames = ret; - return ret; + vec->got_ref = true; + vec->is_pfns = false; + vec->nr_frames = ret; + + if (likely(ret > 0)) + return ret; + + /* This used to (racily) return non-refcounted pfns. Let people know */ + WARN_ONCE(1, "get_vaddr_frames() cannot follow VM_IO mapping"); + vec->nr_frames = 0; + return ret ? ret : -EFAULT; } EXPORT_SYMBOL(get_vaddr_frames); diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c index ab9697f3b5f1..92efc4676df6 100644 --- a/drivers/media/common/videobuf2/videobuf2-core.c +++ b/drivers/media/common/videobuf2/videobuf2-core.c @@ -813,7 +813,13 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory, num_buffers = max_t(unsigned int, *count, q->min_buffers_needed); num_buffers = min_t(unsigned int, num_buffers, VB2_MAX_FRAME); memset(q->alloc_devs, 0, sizeof(q->alloc_devs)); + /* + * Set this now to ensure that drivers see the correct q->memory value + * in the queue_setup op. + */ + mutex_lock(&q->mmap_lock); q->memory = memory; + mutex_unlock(&q->mmap_lock); set_queue_coherency(q, non_coherent_mem); /* @@ -823,22 +829,27 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory, ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes, plane_sizes, q->alloc_devs); if (ret) - return ret; + goto error; /* Check that driver has set sane values */ - if (WARN_ON(!num_planes)) - return -EINVAL; + if (WARN_ON(!num_planes)) { + ret = -EINVAL; + goto error; + } for (i = 0; i < num_planes; i++) - if (WARN_ON(!plane_sizes[i])) - return -EINVAL; + if (WARN_ON(!plane_sizes[i])) { + ret = -EINVAL; + goto error; + } /* Finally, allocate buffers and video memory */ allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes); if (allocated_buffers == 0) { dprintk(q, 1, "memory allocation failed\n"); - return -ENOMEM; + ret = -ENOMEM; + goto error; } /* @@ -879,7 +890,8 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory, if (ret < 0) { /* * Note: __vb2_queue_free() will subtract 'allocated_buffers' - * from q->num_buffers. + * from q->num_buffers and it will reset q->memory to + * VB2_MEMORY_UNKNOWN. */ __vb2_queue_free(q, allocated_buffers); mutex_unlock(&q->mmap_lock); @@ -895,6 +907,12 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory, q->waiting_for_buffers = !q->is_output; return 0; + +error: + mutex_lock(&q->mmap_lock); + q->memory = VB2_MEMORY_UNKNOWN; + mutex_unlock(&q->mmap_lock); + return ret; } EXPORT_SYMBOL_GPL(vb2_core_reqbufs); @@ -906,6 +924,7 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory, unsigned int num_planes = 0, num_buffers, allocated_buffers; unsigned plane_sizes[VB2_MAX_PLANES] = { }; bool non_coherent_mem = flags & V4L2_MEMORY_FLAG_NON_COHERENT; + bool no_previous_buffers = !q->num_buffers; int ret; if (q->num_buffers == VB2_MAX_FRAME) { @@ -913,13 +932,19 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory, return -ENOBUFS; } - if (!q->num_buffers) { + if (no_previous_buffers) { if (q->waiting_in_dqbuf && *count) { dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n"); return -EBUSY; } memset(q->alloc_devs, 0, sizeof(q->alloc_devs)); + /* + * Set this now to ensure that drivers see the correct q->memory + * value in the queue_setup op. + */ + mutex_lock(&q->mmap_lock); q->memory = memory; + mutex_unlock(&q->mmap_lock); q->waiting_for_buffers = !q->is_output; set_queue_coherency(q, non_coherent_mem); } else { @@ -945,14 +970,15 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory, ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes, plane_sizes, q->alloc_devs); if (ret) - return ret; + goto error; /* Finally, allocate buffers and video memory */ allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes); if (allocated_buffers == 0) { dprintk(q, 1, "memory allocation failed\n"); - return -ENOMEM; + ret = -ENOMEM; + goto error; } /* @@ -983,7 +1009,8 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory, if (ret < 0) { /* * Note: __vb2_queue_free() will subtract 'allocated_buffers' - * from q->num_buffers. + * from q->num_buffers and it will reset q->memory to + * VB2_MEMORY_UNKNOWN. */ __vb2_queue_free(q, allocated_buffers); mutex_unlock(&q->mmap_lock); @@ -998,6 +1025,14 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory, *count = allocated_buffers; return 0; + +error: + if (no_previous_buffers) { + mutex_lock(&q->mmap_lock); + q->memory = VB2_MEMORY_UNKNOWN; + mutex_unlock(&q->mmap_lock); + } + return ret; } EXPORT_SYMBOL_GPL(vb2_core_create_bufs); @@ -2165,6 +2200,22 @@ static int __find_plane_by_offset(struct vb2_queue *q, unsigned long off, unsigned int buffer, plane; /* + * Sanity checks to ensure the lock is held, MEMORY_MMAP is + * used and fileio isn't active. + */ + lockdep_assert_held(&q->mmap_lock); + + if (q->memory != VB2_MEMORY_MMAP) { + dprintk(q, 1, "queue is not currently set up for mmap\n"); + return -EINVAL; + } + + if (vb2_fileio_is_active(q)) { + dprintk(q, 1, "file io in progress\n"); + return -EBUSY; + } + + /* * Go over all buffers and their planes, comparing the given offset * with an offset assigned to each plane. If a match is found, * return its buffer and plane numbers. @@ -2265,11 +2316,6 @@ int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma) int ret; unsigned long length; - if (q->memory != VB2_MEMORY_MMAP) { - dprintk(q, 1, "queue is not currently set up for mmap\n"); - return -EINVAL; - } - /* * Check memory area access mode. */ @@ -2291,14 +2337,9 @@ int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma) mutex_lock(&q->mmap_lock); - if (vb2_fileio_is_active(q)) { - dprintk(q, 1, "mmap: file io in progress\n"); - ret = -EBUSY; - goto unlock; - } - /* - * Find the plane corresponding to the offset passed by userspace. + * Find the plane corresponding to the offset passed by userspace. This + * will return an error if not MEMORY_MMAP or file I/O is in progress. */ ret = __find_plane_by_offset(q, off, &buffer, &plane); if (ret) @@ -2351,22 +2392,25 @@ unsigned long vb2_get_unmapped_area(struct vb2_queue *q, void *vaddr; int ret; - if (q->memory != VB2_MEMORY_MMAP) { - dprintk(q, 1, "queue is not currently set up for mmap\n"); - return -EINVAL; - } + mutex_lock(&q->mmap_lock); /* - * Find the plane corresponding to the offset passed by userspace. + * Find the plane corresponding to the offset passed by userspace. This + * will return an error if not MEMORY_MMAP or file I/O is in progress. */ ret = __find_plane_by_offset(q, off, &buffer, &plane); if (ret) - return ret; + goto unlock; vb = q->bufs[buffer]; vaddr = vb2_plane_vaddr(vb, plane); + mutex_unlock(&q->mmap_lock); return vaddr ? (unsigned long)vaddr : -EINVAL; + +unlock: + mutex_unlock(&q->mmap_lock); + return ret; } EXPORT_SYMBOL_GPL(vb2_get_unmapped_area); #endif diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index c5de202f530a..de1cc9e1ae57 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -1484,6 +1484,11 @@ void mmc_init_erase(struct mmc_card *card) card->pref_erase = 0; } +static bool is_trim_arg(unsigned int arg) +{ + return (arg & MMC_TRIM_OR_DISCARD_ARGS) && arg != MMC_DISCARD_ARG; +} + static unsigned int mmc_mmc_erase_timeout(struct mmc_card *card, unsigned int arg, unsigned int qty) { @@ -1766,7 +1771,7 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, !(card->ext_csd.sec_feature_support & EXT_CSD_SEC_ER_EN)) return -EOPNOTSUPP; - if (mmc_card_mmc(card) && (arg & MMC_TRIM_ARGS) && + if (mmc_card_mmc(card) && is_trim_arg(arg) && !(card->ext_csd.sec_feature_support & EXT_CSD_SEC_GB_CL_EN)) return -EOPNOTSUPP; @@ -1796,7 +1801,7 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, * identified by the card->eg_boundary flag. */ rem = card->erase_size - (from % card->erase_size); - if ((arg & MMC_TRIM_ARGS) && (card->eg_boundary) && (nr > rem)) { + if ((arg & MMC_TRIM_OR_DISCARD_ARGS) && card->eg_boundary && nr > rem) { err = mmc_do_erase(card, from, from + rem - 1, arg); from += rem; if ((err) || (to <= from)) diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c index 8d9bceeff986..155ce2bdfe62 100644 --- a/drivers/mmc/core/mmc_test.c +++ b/drivers/mmc/core/mmc_test.c @@ -3179,7 +3179,8 @@ static int __mmc_test_register_dbgfs_file(struct mmc_card *card, struct mmc_test_dbgfs_file *df; if (card->debugfs_root) - debugfs_create_file(name, mode, card->debugfs_root, card, fops); + file = debugfs_create_file(name, mode, card->debugfs_root, + card, fops); df = kmalloc(sizeof(*df), GFP_KERNEL); if (!df) { diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c index df941438aef5..26bc59b5a7cc 100644 --- a/drivers/mmc/host/mtk-sd.c +++ b/drivers/mmc/host/mtk-sd.c @@ -2588,13 +2588,11 @@ static int msdc_of_clock_parse(struct platform_device *pdev, return PTR_ERR(host->src_clk_cg); } - host->sys_clk_cg = devm_clk_get_optional(&pdev->dev, "sys_cg"); + /* If present, always enable for this clock gate */ + host->sys_clk_cg = devm_clk_get_optional_enabled(&pdev->dev, "sys_cg"); if (IS_ERR(host->sys_clk_cg)) host->sys_clk_cg = NULL; - /* If present, always enable for this clock gate */ - clk_prepare_enable(host->sys_clk_cg); - host->bulk_clks[0].id = "pclk_cg"; host->bulk_clks[1].id = "axi_cg"; host->bulk_clks[2].id = "ahb_cg"; diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 31ea0a2fce35..ffeb5759830f 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1512,7 +1512,7 @@ static void esdhc_cqe_enable(struct mmc_host *mmc) * system resume back. */ cqhci_writel(cq_host, 0, CQHCI_CTL); - if (cqhci_readl(cq_host, CQHCI_CTL) && CQHCI_HALT) + if (cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) dev_err(mmc_dev(host->mmc), "failed to exit halt state when enable CQE\n"); diff --git a/drivers/mmc/host/sdhci-sprd.c b/drivers/mmc/host/sdhci-sprd.c index b92a408f138d..bec3f9e3cd3f 100644 --- a/drivers/mmc/host/sdhci-sprd.c +++ b/drivers/mmc/host/sdhci-sprd.c @@ -470,7 +470,7 @@ static int sdhci_sprd_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios) } if (IS_ERR(sprd_host->pinctrl)) - return 0; + goto reset; switch (ios->signal_voltage) { case MMC_SIGNAL_VOLTAGE_180: @@ -498,6 +498,8 @@ static int sdhci_sprd_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios) /* Wait for 300 ~ 500 us for pin state stable */ usleep_range(300, 500); + +reset: sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); return 0; diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index fef03de85b99..c7ad32a75b57 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -373,6 +373,7 @@ static void sdhci_init(struct sdhci_host *host, int soft) if (soft) { /* force clock reconfiguration */ host->clock = 0; + host->reinit_uhs = true; mmc->ops->set_ios(mmc, &mmc->ios); } } @@ -2293,11 +2294,46 @@ void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing) } EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling); +static bool sdhci_timing_has_preset(unsigned char timing) +{ + switch (timing) { + case MMC_TIMING_UHS_SDR12: + case MMC_TIMING_UHS_SDR25: + case MMC_TIMING_UHS_SDR50: + case MMC_TIMING_UHS_SDR104: + case MMC_TIMING_UHS_DDR50: + case MMC_TIMING_MMC_DDR52: + return true; + }; + return false; +} + +static bool sdhci_preset_needed(struct sdhci_host *host, unsigned char timing) +{ + return !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) && + sdhci_timing_has_preset(timing); +} + +static bool sdhci_presetable_values_change(struct sdhci_host *host, struct mmc_ios *ios) +{ + /* + * Preset Values are: Driver Strength, Clock Generator and SDCLK/RCLK + * Frequency. Check if preset values need to be enabled, or the Driver + * Strength needs updating. Note, clock changes are handled separately. + */ + return !host->preset_enabled && + (sdhci_preset_needed(host, ios->timing) || host->drv_type != ios->drv_type); +} + void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) { struct sdhci_host *host = mmc_priv(mmc); + bool reinit_uhs = host->reinit_uhs; + bool turning_on_clk = false; u8 ctrl; + host->reinit_uhs = false; + if (ios->power_mode == MMC_POWER_UNDEFINED) return; @@ -2323,6 +2359,8 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) sdhci_enable_preset_value(host, false); if (!ios->clock || ios->clock != host->clock) { + turning_on_clk = ios->clock && !host->clock; + host->ops->set_clock(host, ios->clock); host->clock = ios->clock; @@ -2349,6 +2387,17 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) host->ops->set_bus_width(host, ios->bus_width); + /* + * Special case to avoid multiple clock changes during voltage + * switching. + */ + if (!reinit_uhs && + turning_on_clk && + host->timing == ios->timing && + host->version >= SDHCI_SPEC_300 && + !sdhci_presetable_values_change(host, ios)) + return; + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) { @@ -2392,6 +2441,7 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) } sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2); + host->drv_type = ios->drv_type; } else { /* * According to SDHC Spec v3.00, if the Preset Value @@ -2419,19 +2469,14 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) host->ops->set_uhs_signaling(host, ios->timing); host->timing = ios->timing; - if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) && - ((ios->timing == MMC_TIMING_UHS_SDR12) || - (ios->timing == MMC_TIMING_UHS_SDR25) || - (ios->timing == MMC_TIMING_UHS_SDR50) || - (ios->timing == MMC_TIMING_UHS_SDR104) || - (ios->timing == MMC_TIMING_UHS_DDR50) || - (ios->timing == MMC_TIMING_MMC_DDR52))) { + if (sdhci_preset_needed(host, ios->timing)) { u16 preset; sdhci_enable_preset_value(host, true); preset = sdhci_get_preset_value(host); ios->drv_type = FIELD_GET(SDHCI_PRESET_DRV_MASK, preset); + host->drv_type = ios->drv_type; } /* Re-enable SD Clock */ @@ -3768,6 +3813,7 @@ int sdhci_resume_host(struct sdhci_host *host) sdhci_init(host, 0); host->pwr = 0; host->clock = 0; + host->reinit_uhs = true; mmc->ops->set_ios(mmc, &mmc->ios); } else { sdhci_init(host, (mmc->pm_flags & MMC_PM_KEEP_POWER)); @@ -3830,6 +3876,7 @@ int sdhci_runtime_resume_host(struct sdhci_host *host, int soft_reset) /* Force clock and power re-program */ host->pwr = 0; host->clock = 0; + host->reinit_uhs = true; mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios); mmc->ops->set_ios(mmc, &mmc->ios); diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index d750c464bd1e..87a3aaa07438 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -524,6 +524,8 @@ struct sdhci_host { unsigned int clock; /* Current clock (MHz) */ u8 pwr; /* Current voltage */ + u8 drv_type; /* Current UHS-I driver type */ + bool reinit_uhs; /* Force UHS-related re-initialization */ bool runtime_suspended; /* Host is runtime suspended */ bool bus_on; /* Bus power prevents runtime suspend */ diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index c87481033995..4048876f842c 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -2524,10 +2524,10 @@ static int bond_slave_info_query(struct net_device *bond_dev, struct ifslave *in /* called with rcu_read_lock() */ static int bond_miimon_inspect(struct bonding *bond) { + bool ignore_updelay = false; int link_state, commit = 0; struct list_head *iter; struct slave *slave; - bool ignore_updelay; if (BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP) { ignore_updelay = !rcu_dereference(bond->curr_active_slave); @@ -3249,7 +3249,7 @@ static int bond_na_rcv(const struct sk_buff *skb, struct bonding *bond, goto out; saddr = &combined->ip6.saddr; - daddr = &combined->ip6.saddr; + daddr = &combined->ip6.daddr; slave_dbg(bond->dev, slave->dev, "%s: %s/%d av %d sv %d sip %pI6c tip %pI6c\n", __func__, slave->dev->name, bond_slave_state(slave), diff --git a/drivers/net/can/c_can/c_can_platform.c b/drivers/net/can/c_can/c_can_platform.c index 86e95e9d6533..03ccb7cfacaf 100644 --- a/drivers/net/can/c_can/c_can_platform.c +++ b/drivers/net/can/c_can/c_can_platform.c @@ -290,8 +290,7 @@ static int c_can_plat_probe(struct platform_device *pdev) goto exit; } - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - addr = devm_ioremap_resource(&pdev->dev, mem); + addr = devm_platform_get_and_ioremap_resource(pdev, 0, &mem); if (IS_ERR(addr)) { ret = PTR_ERR(addr); goto exit; diff --git a/drivers/net/can/can327.c b/drivers/net/can/can327.c index ed3d0b8989a0..dc7192ecb001 100644 --- a/drivers/net/can/can327.c +++ b/drivers/net/can/can327.c @@ -796,9 +796,9 @@ static int can327_netdev_close(struct net_device *dev) netif_stop_queue(dev); - /* Give UART one final chance to flush. */ - clear_bit(TTY_DO_WRITE_WAKEUP, &elm->tty->flags); - flush_work(&elm->tx_work); + /* We don't flush the UART TX queue here, as we want final stop + * commands (like the above dummy char) to be flushed out. + */ can_rx_offload_disable(&elm->offload); elm->can.state = CAN_STATE_STOPPED; @@ -1069,12 +1069,15 @@ static void can327_ldisc_close(struct tty_struct *tty) { struct can327 *elm = (struct can327 *)tty->disc_data; - /* unregister_netdev() calls .ndo_stop() so we don't have to. - * Our .ndo_stop() also flushes the TTY write wakeup handler, - * so we can safely set elm->tty = NULL after this. - */ + /* unregister_netdev() calls .ndo_stop() so we don't have to. */ unregister_candev(elm->dev); + /* Give UART one final chance to flush. + * No need to clear TTY_DO_WRITE_WAKEUP since .write_wakeup() is + * serialised against .close() and will not be called once we return. + */ + flush_work(&elm->tx_work); + /* Mark channel as dead */ spin_lock_bh(&elm->lock); tty->disc_data = NULL; diff --git a/drivers/net/can/ctucanfd/Kconfig b/drivers/net/can/ctucanfd/Kconfig index 6e2073351a8f..f52407f5c5d8 100644 --- a/drivers/net/can/ctucanfd/Kconfig +++ b/drivers/net/can/ctucanfd/Kconfig @@ -23,7 +23,7 @@ config CAN_CTUCANFD_PCI config CAN_CTUCANFD_PLATFORM tristate "CTU CAN-FD IP core platform (FPGA, SoC) driver" - depends on HAS_IOMEM && (OF || COMPILE_TEST) + depends on HAS_IOMEM && OF select CAN_CTUCANFD help The core has been tested together with OpenCores SJA1000 diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c index 9bdadd716f4e..0aeff34e5ae1 100644 --- a/drivers/net/can/flexcan/flexcan-core.c +++ b/drivers/net/can/flexcan/flexcan-core.c @@ -345,6 +345,15 @@ static struct flexcan_devtype_data fsl_imx8mp_devtype_data = { FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; +static struct flexcan_devtype_data fsl_imx93_devtype_data = { + .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | + FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX | + FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_AUTO_STOP_MODE | + FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, +}; + static const struct flexcan_devtype_data fsl_vf610_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX | @@ -532,9 +541,14 @@ static inline int flexcan_enter_stop_mode(struct flexcan_priv *priv) ret = flexcan_stop_mode_enable_scfw(priv, true); if (ret < 0) return ret; - } else { + } else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR) { regmap_update_bits(priv->stm.gpr, priv->stm.req_gpr, 1 << priv->stm.req_bit, 1 << priv->stm.req_bit); + } else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_AUTO_STOP_MODE) { + /* For the auto stop mode, software do nothing, hardware will cover + * all the operation automatically after system go into low power mode. + */ + return 0; } return flexcan_low_power_enter_ack(priv); @@ -551,7 +565,7 @@ static inline int flexcan_exit_stop_mode(struct flexcan_priv *priv) ret = flexcan_stop_mode_enable_scfw(priv, false); if (ret < 0) return ret; - } else { + } else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR) { regmap_update_bits(priv->stm.gpr, priv->stm.req_gpr, 1 << priv->stm.req_bit, 0); } @@ -560,6 +574,12 @@ static inline int flexcan_exit_stop_mode(struct flexcan_priv *priv) reg_mcr &= ~FLEXCAN_MCR_SLF_WAK; priv->write(reg_mcr, ®s->mcr); + /* For the auto stop mode, hardware will exist stop mode + * automatically after system go out of low power mode. + */ + if (priv->devtype_data.quirks & FLEXCAN_QUIRK_AUTO_STOP_MODE) + return 0; + return flexcan_low_power_exit_ack(priv); } @@ -1974,6 +1994,8 @@ static int flexcan_setup_stop_mode(struct platform_device *pdev) ret = flexcan_setup_stop_mode_scfw(pdev); else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR) ret = flexcan_setup_stop_mode_gpr(pdev); + else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_AUTO_STOP_MODE) + ret = 0; else /* return 0 directly if doesn't support stop mode feature */ return 0; @@ -1992,6 +2014,7 @@ static int flexcan_setup_stop_mode(struct platform_device *pdev) static const struct of_device_id flexcan_of_match[] = { { .compatible = "fsl,imx8qm-flexcan", .data = &fsl_imx8qm_devtype_data, }, { .compatible = "fsl,imx8mp-flexcan", .data = &fsl_imx8mp_devtype_data, }, + { .compatible = "fsl,imx93-flexcan", .data = &fsl_imx93_devtype_data, }, { .compatible = "fsl,imx6q-flexcan", .data = &fsl_imx6q_devtype_data, }, { .compatible = "fsl,imx28-flexcan", .data = &fsl_imx28_devtype_data, }, { .compatible = "fsl,imx53-flexcan", .data = &fsl_imx25_devtype_data, }, @@ -2299,8 +2322,16 @@ static int __maybe_unused flexcan_noirq_suspend(struct device *device) if (netif_running(dev)) { int err; - if (device_may_wakeup(device)) + if (device_may_wakeup(device)) { flexcan_enable_wakeup_irq(priv, true); + /* For auto stop mode, need to keep the clock on before + * system go into low power mode. After system go into + * low power mode, hardware will config the flexcan into + * stop mode, and gate off the clock automatically. + */ + if (priv->devtype_data.quirks & FLEXCAN_QUIRK_AUTO_STOP_MODE) + return 0; + } err = pm_runtime_force_suspend(device); if (err) diff --git a/drivers/net/can/flexcan/flexcan.h b/drivers/net/can/flexcan/flexcan.h index 025c3417031f..91402977780b 100644 --- a/drivers/net/can/flexcan/flexcan.h +++ b/drivers/net/can/flexcan/flexcan.h @@ -68,6 +68,8 @@ #define FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR BIT(15) /* Device supports RX via FIFO */ #define FLEXCAN_QUIRK_SUPPORT_RX_FIFO BIT(16) +/* auto enter stop mode to support wakeup */ +#define FLEXCAN_QUIRK_AUTO_STOP_MODE BIT(17) struct flexcan_devtype_data { u32 quirks; /* quirks needed for different IP cores */ diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index 0bdec28e7c85..8e83d6963d85 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -9,20 +9,20 @@ */ #include <linux/bitfield.h> +#include <linux/can/dev.h> #include <linux/ethtool.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> -#include <linux/iopoll.h> -#include <linux/can/dev.h> -#include <linux/pinctrl/consumer.h> -#include <linux/phy/phy.h> #include "m_can.h" @@ -369,9 +369,14 @@ m_can_txe_fifo_read(struct m_can_classdev *cdev, u32 fgi, u32 offset, u32 *val) return cdev->ops->read_fifo(cdev, addr_offset, val, 1); } +static inline bool _m_can_tx_fifo_full(u32 txfqs) +{ + return !!(txfqs & TXFQS_TFQF); +} + static inline bool m_can_tx_fifo_full(struct m_can_classdev *cdev) { - return !!(m_can_read(cdev, M_CAN_TXFQS) & TXFQS_TFQF); + return _m_can_tx_fifo_full(m_can_read(cdev, M_CAN_TXFQS)); } static void m_can_config_endisable(struct m_can_classdev *cdev, bool enable) @@ -472,19 +477,16 @@ static void m_can_receive_skb(struct m_can_classdev *cdev, } } -static int m_can_read_fifo(struct net_device *dev, u32 rxfs) +static int m_can_read_fifo(struct net_device *dev, u32 fgi) { struct net_device_stats *stats = &dev->stats; struct m_can_classdev *cdev = netdev_priv(dev); struct canfd_frame *cf; struct sk_buff *skb; struct id_and_dlc fifo_header; - u32 fgi; u32 timestamp = 0; int err; - /* calculate the fifo get index for where to read data */ - fgi = FIELD_GET(RXFS_FGI_MASK, rxfs); err = m_can_fifo_read(cdev, fgi, M_CAN_FIFO_ID, &fifo_header, 2); if (err) goto out_fail; @@ -528,9 +530,6 @@ static int m_can_read_fifo(struct net_device *dev, u32 rxfs) } stats->rx_packets++; - /* acknowledge rx fifo 0 */ - m_can_write(cdev, M_CAN_RXF0A, fgi); - timestamp = FIELD_GET(RX_BUF_RXTS_MASK, fifo_header.dlc) << 16; m_can_receive_skb(cdev, skb, timestamp); @@ -549,7 +548,11 @@ static int m_can_do_rx_poll(struct net_device *dev, int quota) struct m_can_classdev *cdev = netdev_priv(dev); u32 pkts = 0; u32 rxfs; - int err; + u32 rx_count; + u32 fgi; + int ack_fgi = -1; + int i; + int err = 0; rxfs = m_can_read(cdev, M_CAN_RXF0S); if (!(rxfs & RXFS_FFL_MASK)) { @@ -557,16 +560,26 @@ static int m_can_do_rx_poll(struct net_device *dev, int quota) return 0; } - while ((rxfs & RXFS_FFL_MASK) && (quota > 0)) { - err = m_can_read_fifo(dev, rxfs); + rx_count = FIELD_GET(RXFS_FFL_MASK, rxfs); + fgi = FIELD_GET(RXFS_FGI_MASK, rxfs); + + for (i = 0; i < rx_count && quota > 0; ++i) { + err = m_can_read_fifo(dev, fgi); if (err) - return err; + break; quota--; pkts++; - rxfs = m_can_read(cdev, M_CAN_RXF0S); + ack_fgi = fgi; + fgi = (++fgi >= cdev->mcfg[MRAM_RXF0].num ? 0 : fgi); } + if (ack_fgi != -1) + m_can_write(cdev, M_CAN_RXF0A, ack_fgi); + + if (err) + return err; + return pkts; } @@ -900,14 +913,12 @@ static int m_can_handle_bus_errors(struct net_device *dev, u32 irqstatus, return work_done; } -static int m_can_rx_handler(struct net_device *dev, int quota) +static int m_can_rx_handler(struct net_device *dev, int quota, u32 irqstatus) { struct m_can_classdev *cdev = netdev_priv(dev); int rx_work_or_err; int work_done = 0; - u32 irqstatus, psr; - irqstatus = cdev->irqstatus | m_can_read(cdev, M_CAN_IR); if (!irqstatus) goto end; @@ -932,13 +943,13 @@ static int m_can_rx_handler(struct net_device *dev, int quota) } } - psr = m_can_read(cdev, M_CAN_PSR); - if (irqstatus & IR_ERR_STATE) - work_done += m_can_handle_state_errors(dev, psr); + work_done += m_can_handle_state_errors(dev, + m_can_read(cdev, M_CAN_PSR)); if (irqstatus & IR_ERR_BUS_30X) - work_done += m_can_handle_bus_errors(dev, irqstatus, psr); + work_done += m_can_handle_bus_errors(dev, irqstatus, + m_can_read(cdev, M_CAN_PSR)); if (irqstatus & IR_RF0N) { rx_work_or_err = m_can_do_rx_poll(dev, (quota - work_done)); @@ -951,12 +962,12 @@ end: return work_done; } -static int m_can_rx_peripheral(struct net_device *dev) +static int m_can_rx_peripheral(struct net_device *dev, u32 irqstatus) { struct m_can_classdev *cdev = netdev_priv(dev); int work_done; - work_done = m_can_rx_handler(dev, NAPI_POLL_WEIGHT); + work_done = m_can_rx_handler(dev, NAPI_POLL_WEIGHT, irqstatus); /* Don't re-enable interrupts if the driver had a fatal error * (e.g., FIFO read failure). @@ -972,8 +983,11 @@ static int m_can_poll(struct napi_struct *napi, int quota) struct net_device *dev = napi->dev; struct m_can_classdev *cdev = netdev_priv(dev); int work_done; + u32 irqstatus; - work_done = m_can_rx_handler(dev, quota); + irqstatus = cdev->irqstatus | m_can_read(cdev, M_CAN_IR); + + work_done = m_can_rx_handler(dev, quota, irqstatus); /* Don't re-enable interrupts if the driver had a fatal error * (e.g., FIFO read failure). @@ -1014,7 +1028,9 @@ static int m_can_echo_tx_event(struct net_device *dev) u32 txe_count = 0; u32 m_can_txefs; u32 fgi = 0; + int ack_fgi = -1; int i = 0; + int err = 0; unsigned int msg_mark; struct m_can_classdev *cdev = netdev_priv(dev); @@ -1024,34 +1040,34 @@ static int m_can_echo_tx_event(struct net_device *dev) /* Get Tx Event fifo element count */ txe_count = FIELD_GET(TXEFS_EFFL_MASK, m_can_txefs); + fgi = FIELD_GET(TXEFS_EFGI_MASK, m_can_txefs); /* Get and process all sent elements */ for (i = 0; i < txe_count; i++) { u32 txe, timestamp = 0; - int err; - - /* retrieve get index */ - fgi = FIELD_GET(TXEFS_EFGI_MASK, m_can_read(cdev, M_CAN_TXEFS)); /* get message marker, timestamp */ err = m_can_txe_fifo_read(cdev, fgi, 4, &txe); if (err) { netdev_err(dev, "TXE FIFO read returned %d\n", err); - return err; + break; } msg_mark = FIELD_GET(TX_EVENT_MM_MASK, txe); timestamp = FIELD_GET(TX_EVENT_TXTS_MASK, txe) << 16; - /* ack txe element */ - m_can_write(cdev, M_CAN_TXEFA, FIELD_PREP(TXEFA_EFAI_MASK, - fgi)); + ack_fgi = fgi; + fgi = (++fgi >= cdev->mcfg[MRAM_TXE].num ? 0 : fgi); /* update stats */ m_can_tx_update_stats(cdev, msg_mark, timestamp); } - return 0; + if (ack_fgi != -1) + m_can_write(cdev, M_CAN_TXEFA, FIELD_PREP(TXEFA_EFAI_MASK, + ack_fgi)); + + return err; } static irqreturn_t m_can_isr(int irq, void *dev_id) @@ -1083,7 +1099,7 @@ static irqreturn_t m_can_isr(int irq, void *dev_id) m_can_disable_all_interrupts(cdev); if (!cdev->is_peripheral) napi_schedule(&cdev->napi); - else if (m_can_rx_peripheral(dev) < 0) + else if (m_can_rx_peripheral(dev, ir) < 0) goto out_fail; } @@ -1243,10 +1259,17 @@ static int m_can_set_bittiming(struct net_device *dev) * - setup bittiming * - configure timestamp generation */ -static void m_can_chip_config(struct net_device *dev) +static int m_can_chip_config(struct net_device *dev) { struct m_can_classdev *cdev = netdev_priv(dev); u32 cccr, test; + int err; + + err = m_can_init_ram(cdev); + if (err) { + dev_err(cdev->dev, "Message RAM configuration failed\n"); + return err; + } m_can_config_endisable(cdev, true); @@ -1370,18 +1393,25 @@ static void m_can_chip_config(struct net_device *dev) if (cdev->ops->init) cdev->ops->init(cdev); + + return 0; } -static void m_can_start(struct net_device *dev) +static int m_can_start(struct net_device *dev) { struct m_can_classdev *cdev = netdev_priv(dev); + int ret; /* basic m_can configuration */ - m_can_chip_config(dev); + ret = m_can_chip_config(dev); + if (ret) + return ret; cdev->can.state = CAN_STATE_ERROR_ACTIVE; m_can_enable_all_interrupts(cdev); + + return 0; } static int m_can_set_mode(struct net_device *dev, enum can_mode mode) @@ -1595,6 +1625,7 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) struct sk_buff *skb = cdev->tx_skb; struct id_and_dlc fifo_header; u32 cccr, fdflags; + u32 txfqs; int err; int putidx; @@ -1651,8 +1682,10 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) } else { /* Transmit routine for version >= v3.1.x */ + txfqs = m_can_read(cdev, M_CAN_TXFQS); + /* Check if FIFO full */ - if (m_can_tx_fifo_full(cdev)) { + if (_m_can_tx_fifo_full(txfqs)) { /* This shouldn't happen */ netif_stop_queue(dev); netdev_warn(dev, @@ -1668,8 +1701,7 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) } /* get put index for frame */ - putidx = FIELD_GET(TXFQS_TFQPI_MASK, - m_can_read(cdev, M_CAN_TXFQS)); + putidx = FIELD_GET(TXFQS_TFQPI_MASK, txfqs); /* Construct DLC Field, with CAN-FD configuration. * Use the put index of the fifo as the message marker, @@ -1809,7 +1841,9 @@ static int m_can_open(struct net_device *dev) } /* start the m_can controller */ - m_can_start(dev); + err = m_can_start(dev); + if (err) + goto exit_irq_fail; if (!cdev->is_peripheral) napi_enable(&cdev->napi); @@ -2068,9 +2102,13 @@ int m_can_class_resume(struct device *dev) ret = m_can_clk_start(cdev); if (ret) return ret; + ret = m_can_start(ndev); + if (ret) { + m_can_clk_stop(cdev); + + return ret; + } - m_can_init_ram(cdev); - m_can_start(ndev); netif_device_attach(ndev); netif_start_queue(ndev); } diff --git a/drivers/net/can/m_can/m_can.h b/drivers/net/can/m_can/m_can.h index 52563c048732..a839dc71dc9b 100644 --- a/drivers/net/can/m_can/m_can.h +++ b/drivers/net/can/m_can/m_can.h @@ -7,27 +7,27 @@ #define _CAN_M_CAN_H_ #include <linux/can/core.h> +#include <linux/can/dev.h> #include <linux/can/rx-offload.h> +#include <linux/clk.h> #include <linux/completion.h> +#include <linux/delay.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/freezer.h> -#include <linux/slab.h> -#include <linux/uaccess.h> -#include <linux/clk.h> -#include <linux/delay.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/netdevice.h> #include <linux/of.h> #include <linux/of_device.h> -#include <linux/pm_runtime.h> -#include <linux/iopoll.h> -#include <linux/can/dev.h> -#include <linux/pinctrl/consumer.h> #include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/uaccess.h> /* m_can lec values */ enum m_can_lec_type { diff --git a/drivers/net/can/m_can/m_can_platform.c b/drivers/net/can/m_can/m_can_platform.c index eee47bad0592..9c1dcf838006 100644 --- a/drivers/net/can/m_can/m_can_platform.c +++ b/drivers/net/can/m_can/m_can_platform.c @@ -5,8 +5,8 @@ // // Copyright (C) 2018-19 Texas Instruments Incorporated - http://www.ti.com/ -#include <linux/platform_device.h> #include <linux/phy/phy.h> +#include <linux/platform_device.h> #include "m_can.h" @@ -140,10 +140,6 @@ static int m_can_plat_probe(struct platform_device *pdev) platform_set_drvdata(pdev, mcan_class); - ret = m_can_init_ram(mcan_class); - if (ret) - goto probe_fail; - pm_runtime_enable(mcan_class->dev); ret = m_can_class_register(mcan_class); if (ret) diff --git a/drivers/net/can/m_can/tcan4x5x-core.c b/drivers/net/can/m_can/tcan4x5x-core.c index 41645a24384c..2342aa011647 100644 --- a/drivers/net/can/m_can/tcan4x5x-core.c +++ b/drivers/net/can/m_can/tcan4x5x-core.c @@ -10,7 +10,7 @@ #define TCAN4X5X_DEV_ID1 0x04 #define TCAN4X5X_REV 0x08 #define TCAN4X5X_STATUS 0x0C -#define TCAN4X5X_ERROR_STATUS 0x10 +#define TCAN4X5X_ERROR_STATUS_MASK 0x10 #define TCAN4X5X_CONTROL 0x14 #define TCAN4X5X_CONFIG 0x800 @@ -204,17 +204,7 @@ static int tcan4x5x_clear_interrupts(struct m_can_classdev *cdev) if (ret) return ret; - ret = tcan4x5x_write_tcan_reg(cdev, TCAN4X5X_MCAN_INT_REG, - TCAN4X5X_ENABLE_MCAN_INT); - if (ret) - return ret; - - ret = tcan4x5x_write_tcan_reg(cdev, TCAN4X5X_INT_FLAGS, - TCAN4X5X_CLEAR_ALL_INT); - if (ret) - return ret; - - return tcan4x5x_write_tcan_reg(cdev, TCAN4X5X_ERROR_STATUS, + return tcan4x5x_write_tcan_reg(cdev, TCAN4X5X_INT_FLAGS, TCAN4X5X_CLEAR_ALL_INT); } @@ -234,8 +224,8 @@ static int tcan4x5x_init(struct m_can_classdev *cdev) if (ret) return ret; - /* Zero out the MCAN buffers */ - ret = m_can_init_ram(cdev); + ret = tcan4x5x_write_tcan_reg(cdev, TCAN4X5X_ERROR_STATUS_MASK, + TCAN4X5X_CLEAR_ALL_INT); if (ret) return ret; diff --git a/drivers/net/can/m_can/tcan4x5x-regmap.c b/drivers/net/can/m_can/tcan4x5x-regmap.c index 26e212b8ca7a..2b218ce04e9f 100644 --- a/drivers/net/can/m_can/tcan4x5x-regmap.c +++ b/drivers/net/can/m_can/tcan4x5x-regmap.c @@ -90,16 +90,47 @@ static int tcan4x5x_regmap_read(void *context, return 0; } -static const struct regmap_range tcan4x5x_reg_table_yes_range[] = { - regmap_reg_range(0x0000, 0x002c), /* Device ID and SPI Registers */ - regmap_reg_range(0x0800, 0x083c), /* Device configuration registers and Interrupt Flags*/ +static const struct regmap_range tcan4x5x_reg_table_wr_range[] = { + /* Device ID and SPI Registers */ + regmap_reg_range(0x000c, 0x0010), + /* Device configuration registers and Interrupt Flags*/ + regmap_reg_range(0x0800, 0x080c), + regmap_reg_range(0x0814, 0x0814), + regmap_reg_range(0x0820, 0x0820), + regmap_reg_range(0x0830, 0x0830), + /* M_CAN */ + regmap_reg_range(0x100c, 0x102c), + regmap_reg_range(0x1048, 0x1048), + regmap_reg_range(0x1050, 0x105c), + regmap_reg_range(0x1080, 0x1088), + regmap_reg_range(0x1090, 0x1090), + regmap_reg_range(0x1098, 0x10a0), + regmap_reg_range(0x10a8, 0x10b0), + regmap_reg_range(0x10b8, 0x10c0), + regmap_reg_range(0x10c8, 0x10c8), + regmap_reg_range(0x10d0, 0x10d4), + regmap_reg_range(0x10e0, 0x10e4), + regmap_reg_range(0x10f0, 0x10f0), + regmap_reg_range(0x10f8, 0x10f8), + /* MRAM */ + regmap_reg_range(0x8000, 0x87fc), +}; + +static const struct regmap_range tcan4x5x_reg_table_rd_range[] = { + regmap_reg_range(0x0000, 0x0010), /* Device ID and SPI Registers */ + regmap_reg_range(0x0800, 0x0830), /* Device configuration registers and Interrupt Flags*/ regmap_reg_range(0x1000, 0x10fc), /* M_CAN */ regmap_reg_range(0x8000, 0x87fc), /* MRAM */ }; -static const struct regmap_access_table tcan4x5x_reg_table = { - .yes_ranges = tcan4x5x_reg_table_yes_range, - .n_yes_ranges = ARRAY_SIZE(tcan4x5x_reg_table_yes_range), +static const struct regmap_access_table tcan4x5x_reg_table_wr = { + .yes_ranges = tcan4x5x_reg_table_wr_range, + .n_yes_ranges = ARRAY_SIZE(tcan4x5x_reg_table_wr_range), +}; + +static const struct regmap_access_table tcan4x5x_reg_table_rd = { + .yes_ranges = tcan4x5x_reg_table_rd_range, + .n_yes_ranges = ARRAY_SIZE(tcan4x5x_reg_table_rd_range), }; static const struct regmap_config tcan4x5x_regmap = { @@ -107,8 +138,8 @@ static const struct regmap_config tcan4x5x_regmap = { .reg_stride = 4, .pad_bits = 8, .val_bits = 32, - .wr_table = &tcan4x5x_reg_table, - .rd_table = &tcan4x5x_reg_table, + .wr_table = &tcan4x5x_reg_table_wr, + .rd_table = &tcan4x5x_reg_table_rd, .max_register = TCAN4X5X_MAX_REGISTER, .cache_type = REGCACHE_NONE, .read_flag_mask = (__force unsigned long) diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 0a59eab35da7..f6fa7157b99b 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -41,12 +41,6 @@ #define RCANFD_DRV_NAME "rcar_canfd" -enum rcanfd_chip_id { - RENESAS_RCAR_GEN3 = 0, - RENESAS_RZG2L, - RENESAS_R8A779A0, -}; - /* Global register bits */ /* RSCFDnCFDGRMCFG */ @@ -522,6 +516,14 @@ enum rcar_canfd_fcanclk { struct rcar_canfd_global; +struct rcar_canfd_hw_info { + u8 max_channels; + u8 postdiv; + /* hardware features */ + unsigned shared_global_irqs:1; /* Has shared global irqs */ + unsigned multi_channel_irqs:1; /* Has multiple channel irqs */ +}; + /* Channel priv data */ struct rcar_canfd_channel { struct can_priv can; /* Must be the first member */ @@ -547,8 +549,7 @@ struct rcar_canfd_global { bool fdmode; /* CAN FD or Classical CAN only mode */ struct reset_control *rstc1; struct reset_control *rstc2; - enum rcanfd_chip_id chip_id; - u32 max_channels; + const struct rcar_canfd_hw_info *info; }; /* CAN FD mode nominal rate constants */ @@ -590,10 +591,28 @@ static const struct can_bittiming_const rcar_canfd_bittiming_const = { .brp_inc = 1, }; +static const struct rcar_canfd_hw_info rcar_gen3_hw_info = { + .max_channels = 2, + .postdiv = 2, + .shared_global_irqs = 1, +}; + +static const struct rcar_canfd_hw_info rzg2l_hw_info = { + .max_channels = 2, + .postdiv = 1, + .multi_channel_irqs = 1, +}; + +static const struct rcar_canfd_hw_info r8a779a0_hw_info = { + .max_channels = 8, + .postdiv = 2, + .shared_global_irqs = 1, +}; + /* Helper functions */ static inline bool is_v3u(struct rcar_canfd_global *gpriv) { - return gpriv->chip_id == RENESAS_R8A779A0; + return gpriv->info == &r8a779a0_hw_info; } static inline u32 reg_v3u(struct rcar_canfd_global *gpriv, @@ -721,7 +740,7 @@ static int rcar_canfd_reset_controller(struct rcar_canfd_global *gpriv) rcar_canfd_set_mode(gpriv); /* Transition all Channels to reset mode */ - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_clear_bit(gpriv->base, RCANFD_CCTR(ch), RCANFD_CCTR_CSLPR); @@ -762,7 +781,7 @@ static void rcar_canfd_configure_controller(struct rcar_canfd_global *gpriv) rcar_canfd_set_bit(gpriv->base, RCANFD_GCFG, cfg); /* Channel configuration settings */ - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_set_bit(gpriv->base, RCANFD_CCTR(ch), RCANFD_CCTR_ERRD); rcar_canfd_update_bit(gpriv->base, RCANFD_CCTR(ch), @@ -1142,7 +1161,7 @@ static irqreturn_t rcar_canfd_global_err_interrupt(int irq, void *dev_id) struct rcar_canfd_global *gpriv = dev_id; u32 ch; - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) rcar_canfd_handle_global_err(gpriv, ch); return IRQ_HANDLED; @@ -1174,7 +1193,7 @@ static irqreturn_t rcar_canfd_global_receive_fifo_interrupt(int irq, void *dev_i struct rcar_canfd_global *gpriv = dev_id; u32 ch; - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) rcar_canfd_handle_global_receive(gpriv, ch); return IRQ_HANDLED; @@ -1188,7 +1207,7 @@ static irqreturn_t rcar_canfd_global_interrupt(int irq, void *dev_id) /* Global error interrupts still indicate a condition specific * to a channel. RxFIFO interrupt is a global interrupt. */ - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_handle_global_err(gpriv, ch); rcar_canfd_handle_global_receive(gpriv, ch); } @@ -1284,7 +1303,7 @@ static irqreturn_t rcar_canfd_channel_interrupt(int irq, void *dev_id) u32 ch; /* Common FIFO is a per channel resource */ - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_handle_channel_err(gpriv, ch); rcar_canfd_handle_channel_tx(gpriv, ch); } @@ -1696,6 +1715,7 @@ static const struct ethtool_ops rcar_canfd_ethtool_ops = { static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, u32 fcan_freq) { + const struct rcar_canfd_hw_info *info = gpriv->info; struct platform_device *pdev = gpriv->pdev; struct rcar_canfd_channel *priv; struct net_device *ndev; @@ -1718,7 +1738,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, priv->can.clock.freq = fcan_freq; dev_info(&pdev->dev, "can_clk rate is %u\n", priv->can.clock.freq); - if (gpriv->chip_id == RENESAS_RZG2L) { + if (info->multi_channel_irqs) { char *irq_name; int err_irq; int tx_irq; @@ -1818,6 +1838,7 @@ static void rcar_canfd_channel_remove(struct rcar_canfd_global *gpriv, u32 ch) static int rcar_canfd_probe(struct platform_device *pdev) { + const struct rcar_canfd_hw_info *info; void __iomem *addr; u32 sts, ch, fcan_freq; struct rcar_canfd_global *gpriv; @@ -1826,18 +1847,15 @@ static int rcar_canfd_probe(struct platform_device *pdev) int err, ch_irq, g_irq; int g_err_irq, g_recc_irq; bool fdmode = true; /* CAN FD only mode - default */ - enum rcanfd_chip_id chip_id; - int max_channels; char name[9] = "channelX"; int i; - chip_id = (uintptr_t)of_device_get_match_data(&pdev->dev); - max_channels = chip_id == RENESAS_R8A779A0 ? 8 : 2; + info = of_device_get_match_data(&pdev->dev); if (of_property_read_bool(pdev->dev.of_node, "renesas,no-can-fd")) fdmode = false; /* Classical CAN only mode */ - for (i = 0; i < max_channels; ++i) { + for (i = 0; i < info->max_channels; ++i) { name[7] = '0' + i; of_child = of_get_child_by_name(pdev->dev.of_node, name); if (of_child && of_device_is_available(of_child)) @@ -1845,7 +1863,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) of_node_put(of_child); } - if (chip_id != RENESAS_RZG2L) { + if (info->shared_global_irqs) { ch_irq = platform_get_irq_byname_optional(pdev, "ch_int"); if (ch_irq < 0) { /* For backward compatibility get irq by index */ @@ -1879,8 +1897,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) gpriv->pdev = pdev; gpriv->channels_mask = channels_mask; gpriv->fdmode = fdmode; - gpriv->chip_id = chip_id; - gpriv->max_channels = max_channels; + gpriv->info = info; gpriv->rstc1 = devm_reset_control_get_optional_exclusive(&pdev->dev, "rstp_n"); @@ -1917,9 +1934,9 @@ static int rcar_canfd_probe(struct platform_device *pdev) } fcan_freq = clk_get_rate(gpriv->can_clk); - if (gpriv->fcan == RCANFD_CANFDCLK && gpriv->chip_id != RENESAS_RZG2L) + if (gpriv->fcan == RCANFD_CANFDCLK) /* CANFD clock is further divided by (1/2) within the IP */ - fcan_freq /= 2; + fcan_freq /= info->postdiv; addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(addr)) { @@ -1929,7 +1946,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) gpriv->base = addr; /* Request IRQ that's common for both channels */ - if (gpriv->chip_id != RENESAS_RZG2L) { + if (info->shared_global_irqs) { err = devm_request_irq(&pdev->dev, ch_irq, rcar_canfd_channel_interrupt, 0, "canfd.ch_int", gpriv); @@ -1995,7 +2012,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) rcar_canfd_configure_controller(gpriv); /* Configure per channel attributes */ - for_each_set_bit(ch, &gpriv->channels_mask, max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) { /* Configure Channel's Rx fifo */ rcar_canfd_configure_rx(gpriv, ch); @@ -2021,7 +2038,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) goto fail_mode; } - for_each_set_bit(ch, &gpriv->channels_mask, max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) { err = rcar_canfd_channel_probe(gpriv, ch, fcan_freq); if (err) goto fail_channel; @@ -2033,7 +2050,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) return 0; fail_channel: - for_each_set_bit(ch, &gpriv->channels_mask, max_channels) + for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) rcar_canfd_channel_remove(gpriv, ch); fail_mode: rcar_canfd_disable_global_interrupts(gpriv); @@ -2054,7 +2071,7 @@ static int rcar_canfd_remove(struct platform_device *pdev) rcar_canfd_reset_controller(gpriv); rcar_canfd_disable_global_interrupts(gpriv); - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) { + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_disable_channel_interrupts(gpriv->ch[ch]); rcar_canfd_channel_remove(gpriv, ch); } @@ -2082,9 +2099,9 @@ static SIMPLE_DEV_PM_OPS(rcar_canfd_pm_ops, rcar_canfd_suspend, rcar_canfd_resume); static const __maybe_unused struct of_device_id rcar_canfd_of_table[] = { - { .compatible = "renesas,rcar-gen3-canfd", .data = (void *)RENESAS_RCAR_GEN3 }, - { .compatible = "renesas,rzg2l-canfd", .data = (void *)RENESAS_RZG2L }, - { .compatible = "renesas,r8a779a0-canfd", .data = (void *)RENESAS_R8A779A0 }, + { .compatible = "renesas,rcar-gen3-canfd", .data = &rcar_gen3_hw_info }, + { .compatible = "renesas,rzg2l-canfd", .data = &rzg2l_hw_info }, + { .compatible = "renesas,r8a779a0-canfd", .data = &r8a779a0_hw_info }, { } }; diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c index fbb34139daa1..f4db77007c13 100644 --- a/drivers/net/can/slcan/slcan-core.c +++ b/drivers/net/can/slcan/slcan-core.c @@ -864,12 +864,14 @@ static void slcan_close(struct tty_struct *tty) { struct slcan *sl = (struct slcan *)tty->disc_data; - /* unregister_netdev() calls .ndo_stop() so we don't have to. - * Our .ndo_stop() also flushes the TTY write wakeup handler, - * so we can safely set sl->tty = NULL after this. - */ unregister_candev(sl->dev); + /* + * The netdev needn't be UP (so .ndo_stop() is not called). Hence make + * sure this is not running before freeing it up. + */ + flush_work(&sl->tx_work); + /* Mark channel as dead */ spin_lock_bh(&sl->lock); tty->disc_data = NULL; diff --git a/drivers/net/can/usb/Kconfig b/drivers/net/can/usb/Kconfig index 8c6fea661530..445504ababce 100644 --- a/drivers/net/can/usb/Kconfig +++ b/drivers/net/can/usb/Kconfig @@ -30,6 +30,7 @@ config CAN_ESD_USB config CAN_ETAS_ES58X tristate "ETAS ES58X CAN/USB interfaces" select CRC16 + select NET_DEVLINK help This driver supports the ES581.4, ES582.1 and ES584.1 interfaces from ETAS GmbH (https://www.etas.com/en/products/es58x.php). diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c index 81b88e9e5bdc..42323f5e6f3a 100644 --- a/drivers/net/can/usb/esd_usb.c +++ b/drivers/net/can/usb/esd_usb.c @@ -234,6 +234,10 @@ static void esd_usb_rx_event(struct esd_usb_net_priv *priv, u8 rxerr = msg->msg.rx.data[2]; u8 txerr = msg->msg.rx.data[3]; + netdev_dbg(priv->netdev, + "CAN_ERR_EV_EXT: dlc=%#02x state=%02x ecc=%02x rec=%02x tec=%02x\n", + msg->msg.rx.dlc, state, ecc, rxerr, txerr); + skb = alloc_can_err_skb(priv->netdev, &cf); if (skb == NULL) { stats->rx_dropped++; @@ -260,6 +264,8 @@ static void esd_usb_rx_event(struct esd_usb_net_priv *priv, break; default: priv->can.state = CAN_STATE_ERROR_ACTIVE; + txerr = 0; + rxerr = 0; break; } } else { diff --git a/drivers/net/can/usb/etas_es58x/Makefile b/drivers/net/can/usb/etas_es58x/Makefile index a129b4aa0215..d6667ebe259f 100644 --- a/drivers/net/can/usb/etas_es58x/Makefile +++ b/drivers/net/can/usb/etas_es58x/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CAN_ETAS_ES58X) += etas_es58x.o -etas_es58x-y = es58x_core.o es581_4.o es58x_fd.o +etas_es58x-y = es58x_core.o es58x_devlink.o es581_4.o es58x_fd.o diff --git a/drivers/net/can/usb/etas_es58x/es581_4.c b/drivers/net/can/usb/etas_es58x/es581_4.c index 1bcdcece5ec7..4151b18fd045 100644 --- a/drivers/net/can/usb/etas_es58x/es581_4.c +++ b/drivers/net/can/usb/etas_es58x/es581_4.c @@ -6,12 +6,12 @@ * * Copyright (c) 2019 Robert Bosch Engineering and Business Solutions. All rights reserved. * Copyright (c) 2020 ETAS K.K.. All rights reserved. - * Copyright (c) 2020, 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr> + * Copyright (c) 2020-2022 Vincent Mailhol <mailhol.vincent@wanadoo.fr> */ +#include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/units.h> -#include <asm/unaligned.h> #include "es58x_core.h" #include "es581_4.h" diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index ddb7c5735c9a..0c7f7505632c 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -7,15 +7,16 @@ * * Copyright (c) 2019 Robert Bosch Engineering and Business Solutions. All rights reserved. * Copyright (c) 2020 ETAS K.K.. All rights reserved. - * Copyright (c) 2020, 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr> + * Copyright (c) 2020-2022 Vincent Mailhol <mailhol.vincent@wanadoo.fr> */ +#include <asm/unaligned.h> +#include <linux/crc16.h> #include <linux/ethtool.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/usb.h> -#include <linux/crc16.h> -#include <asm/unaligned.h> +#include <net/devlink.h> #include "es58x_core.h" @@ -2038,10 +2039,16 @@ static int es58x_set_mode(struct net_device *netdev, enum can_mode mode) * @es58x_dev: ES58X device. * @priv: ES58X private parameters related to the network device. * @channel_idx: Index of the network device. + * + * Return: zero on success, errno if devlink port could not be + * properly registered. */ -static void es58x_init_priv(struct es58x_device *es58x_dev, - struct es58x_priv *priv, int channel_idx) +static int es58x_init_priv(struct es58x_device *es58x_dev, + struct es58x_priv *priv, int channel_idx) { + struct devlink_port_attrs attrs = { + .flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL, + }; const struct es58x_parameters *param = es58x_dev->param; struct can_priv *can = &priv->can; @@ -2060,6 +2067,10 @@ static void es58x_init_priv(struct es58x_device *es58x_dev, can->state = CAN_STATE_STOPPED; can->ctrlmode_supported = param->ctrlmode_supported; can->do_set_mode = es58x_set_mode; + + devlink_port_attrs_set(&priv->devlink_port, &attrs); + return devlink_port_register(priv_to_devlink(es58x_dev), + &priv->devlink_port, channel_idx); } /** @@ -2083,7 +2094,10 @@ static int es58x_init_netdev(struct es58x_device *es58x_dev, int channel_idx) } SET_NETDEV_DEV(netdev, dev); es58x_dev->netdev[channel_idx] = netdev; - es58x_init_priv(es58x_dev, es58x_priv(netdev), channel_idx); + ret = es58x_init_priv(es58x_dev, es58x_priv(netdev), channel_idx); + if (ret) + goto free_candev; + SET_NETDEV_DEVLINK_PORT(netdev, &es58x_priv(netdev)->devlink_port); netdev->netdev_ops = &es58x_netdev_ops; netdev->ethtool_ops = &es58x_ethtool_ops; @@ -2091,16 +2105,20 @@ static int es58x_init_netdev(struct es58x_device *es58x_dev, int channel_idx) netdev->dev_port = channel_idx; ret = register_candev(netdev); - if (ret) { - es58x_dev->netdev[channel_idx] = NULL; - free_candev(netdev); - return ret; - } + if (ret) + goto devlink_port_unregister; netdev_queue_set_dql_min_limit(netdev_get_tx_queue(netdev, 0), es58x_dev->param->dql_min_limit); return ret; + + devlink_port_unregister: + devlink_port_unregister(&es58x_priv(netdev)->devlink_port); + free_candev: + es58x_dev->netdev[channel_idx] = NULL; + free_candev(netdev); + return ret; } /** @@ -2117,54 +2135,13 @@ static void es58x_free_netdevs(struct es58x_device *es58x_dev) if (!netdev) continue; unregister_candev(netdev); + devlink_port_unregister(&es58x_priv(netdev)->devlink_port); es58x_dev->netdev[i] = NULL; free_candev(netdev); } } /** - * es58x_get_product_info() - Get the product information and print them. - * @es58x_dev: ES58X device. - * - * Do a synchronous call to get the product information. - * - * Return: zero on success, errno when any error occurs. - */ -static int es58x_get_product_info(struct es58x_device *es58x_dev) -{ - struct usb_device *udev = es58x_dev->udev; - const int es58x_prod_info_idx = 6; - /* Empirical tests show a prod_info length of maximum 83, - * below should be more than enough. - */ - const size_t prod_info_len = 127; - char *prod_info; - int ret; - - prod_info = kmalloc(prod_info_len, GFP_KERNEL); - if (!prod_info) - return -ENOMEM; - - ret = usb_string(udev, es58x_prod_info_idx, prod_info, prod_info_len); - if (ret < 0) { - dev_err(es58x_dev->dev, - "%s: Could not read the product info: %pe\n", - __func__, ERR_PTR(ret)); - goto out_free; - } - if (ret >= prod_info_len - 1) { - dev_warn(es58x_dev->dev, - "%s: Buffer is too small, result might be truncated\n", - __func__); - } - dev_info(es58x_dev->dev, "Product info: %s\n", prod_info); - - out_free: - kfree(prod_info); - return ret < 0 ? ret : 0; -} - -/** * es58x_init_es58x_dev() - Initialize the ES58X device. * @intf: USB interface. * @driver_info: Quirks of the device. @@ -2177,6 +2154,7 @@ static struct es58x_device *es58x_init_es58x_dev(struct usb_interface *intf, { struct device *dev = &intf->dev; struct es58x_device *es58x_dev; + struct devlink *devlink; const struct es58x_parameters *param; const struct es58x_operators *ops; struct usb_device *udev = interface_to_usbdev(intf); @@ -2199,11 +2177,12 @@ static struct es58x_device *es58x_init_es58x_dev(struct usb_interface *intf, ops = &es581_4_ops; } - es58x_dev = devm_kzalloc(dev, es58x_sizeof_es58x_device(param), - GFP_KERNEL); - if (!es58x_dev) + devlink = devlink_alloc(&es58x_dl_ops, es58x_sizeof_es58x_device(param), + dev); + if (!devlink) return ERR_PTR(-ENOMEM); + es58x_dev = devlink_priv(devlink); es58x_dev->param = param; es58x_dev->ops = ops; es58x_dev->dev = dev; @@ -2240,25 +2219,24 @@ static int es58x_probe(struct usb_interface *intf, const struct usb_device_id *id) { struct es58x_device *es58x_dev; - int ch_idx, ret; + int ch_idx; es58x_dev = es58x_init_es58x_dev(intf, id->driver_info); if (IS_ERR(es58x_dev)) return PTR_ERR(es58x_dev); - ret = es58x_get_product_info(es58x_dev); - if (ret) - return ret; + es58x_parse_product_info(es58x_dev); + devlink_register(priv_to_devlink(es58x_dev)); for (ch_idx = 0; ch_idx < es58x_dev->num_can_ch; ch_idx++) { - ret = es58x_init_netdev(es58x_dev, ch_idx); + int ret = es58x_init_netdev(es58x_dev, ch_idx); if (ret) { es58x_free_netdevs(es58x_dev); return ret; } } - return ret; + return 0; } /** @@ -2275,8 +2253,10 @@ static void es58x_disconnect(struct usb_interface *intf) dev_info(&intf->dev, "Disconnecting %s %s\n", es58x_dev->udev->manufacturer, es58x_dev->udev->product); + devlink_unregister(priv_to_devlink(es58x_dev)); es58x_free_netdevs(es58x_dev); es58x_free_urbs(es58x_dev); + devlink_free(priv_to_devlink(es58x_dev)); usb_set_intfdata(intf, NULL); } diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.h b/drivers/net/can/usb/etas_es58x/es58x_core.h index 640fe0a1df63..c1ba1a4e8857 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.h +++ b/drivers/net/can/usb/etas_es58x/es58x_core.h @@ -6,17 +6,18 @@ * * Copyright (c) 2019 Robert Bosch Engineering and Business Solutions. All rights reserved. * Copyright (c) 2020 ETAS K.K.. All rights reserved. - * Copyright (c) 2020, 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr> + * Copyright (c) 2020-2022 Vincent Mailhol <mailhol.vincent@wanadoo.fr> */ #ifndef __ES58X_COMMON_H__ #define __ES58X_COMMON_H__ -#include <linux/types.h> -#include <linux/usb.h> -#include <linux/netdevice.h> #include <linux/can.h> #include <linux/can/dev.h> +#include <linux/netdevice.h> +#include <linux/types.h> +#include <linux/usb.h> +#include <net/devlink.h> #include "es581_4.h" #include "es58x_fd.h" @@ -230,6 +231,7 @@ union es58x_urb_cmd { * @can: struct can_priv must be the first member (Socket CAN relies * on the fact that function netdev_priv() returns a pointer to * a struct can_priv). + * @devlink_port: devlink instance for the network interface. * @es58x_dev: pointer to the corresponding ES58X device. * @tx_urb: Used as a buffer to concatenate the TX messages and to do * a bulk send. Please refer to es58x_start_xmit() for more @@ -255,6 +257,7 @@ union es58x_urb_cmd { */ struct es58x_priv { struct can_priv can; + struct devlink_port devlink_port; struct es58x_device *es58x_dev; struct urb *tx_urb; @@ -357,6 +360,39 @@ struct es58x_operators { }; /** + * struct es58x_sw_version - Version number of the firmware or the + * bootloader. + * @major: Version major number, represented on two digits. + * @minor: Version minor number, represented on two digits. + * @revision: Version revision number, represented on two digits. + * + * The firmware and the bootloader share the same format: "xx.xx.xx" + * where 'x' is a digit. Both can be retrieved from the product + * information string. + */ +struct es58x_sw_version { + u8 major; + u8 minor; + u8 revision; +}; + +/** + * struct es58x_hw_revision - Hardware revision number. + * @letter: Revision letter. + * @major: Version major number, represented on three digits. + * @minor: Version minor number, represented on three digits. + * + * The hardware revision uses its own format: "axxx/xxx" where 'a' is + * a letter and 'x' a digit. It can be retrieved from the product + * information string. + */ +struct es58x_hw_revision { + char letter; + u16 major; + u16 minor; +}; + +/** * struct es58x_device - All information specific to an ES58X device. * @dev: Device information. * @udev: USB device information. @@ -373,6 +409,9 @@ struct es58x_operators { * queue wake/stop logic should prevent this URB from getting * empty. Please refer to es58x_get_tx_urb() for more details. * @tx_urbs_idle_cnt: number of urbs in @tx_urbs_idle. + * @firmware_version: The firmware version number. + * @bootloader_version: The bootloader version number. + * @hardware_revision: The hardware revision number. * @ktime_req_ns: kernel timestamp when es58x_set_realtime_diff_ns() * was called. * @realtime_diff_ns: difference in nanoseconds between the clocks of @@ -408,6 +447,10 @@ struct es58x_device { struct usb_anchor tx_urbs_idle; atomic_t tx_urbs_idle_cnt; + struct es58x_sw_version firmware_version; + struct es58x_sw_version bootloader_version; + struct es58x_hw_revision hardware_revision; + u64 ktime_req_ns; s64 realtime_diff_ns; @@ -674,6 +717,7 @@ static inline enum es58x_flag es58x_get_flags(const struct sk_buff *skb) return es58x_flags; } +/* es58x_core.c. */ int es58x_can_get_echo_skb(struct net_device *netdev, u32 packet_idx, u64 *tstamps, unsigned int pkts); int es58x_tx_ack_msg(struct net_device *netdev, u16 tx_free_entries, @@ -691,9 +735,15 @@ int es58x_rx_cmd_ret_u32(struct net_device *netdev, int es58x_send_msg(struct es58x_device *es58x_dev, u8 cmd_type, u8 cmd_id, const void *msg, u16 cmd_len, int channel_idx); +/* es58x_devlink.c. */ +void es58x_parse_product_info(struct es58x_device *es58x_dev); +extern const struct devlink_ops es58x_dl_ops; + +/* es581_4.c. */ extern const struct es58x_parameters es581_4_param; extern const struct es58x_operators es581_4_ops; +/* es58x_fd.c. */ extern const struct es58x_parameters es58x_fd_param; extern const struct es58x_operators es58x_fd_ops; diff --git a/drivers/net/can/usb/etas_es58x/es58x_devlink.c b/drivers/net/can/usb/etas_es58x/es58x_devlink.c new file mode 100644 index 000000000000..9fba29e2f57c --- /dev/null +++ b/drivers/net/can/usb/etas_es58x/es58x_devlink.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Driver for ETAS GmbH ES58X USB CAN(-FD) Bus Interfaces. + * + * File es58x_devlink.c: report the product information using devlink. + * + * Copyright (c) 2022 Vincent Mailhol <mailhol.vincent@wanadoo.fr> + */ + +#include <linux/ctype.h> +#include <linux/device.h> +#include <linux/usb.h> +#include <net/devlink.h> + +#include "es58x_core.h" + +/* USB descriptor index containing the product information string. */ +#define ES58X_PROD_INFO_IDX 6 + +/** + * es58x_parse_sw_version() - Extract boot loader or firmware version. + * @es58x_dev: ES58X device. + * @prod_info: USB custom string returned by the device. + * @prefix: Select which information should be parsed. Set it to "FW" + * to parse the firmware version or to "BL" to parse the + * bootloader version. + * + * The @prod_info string contains the firmware and the bootloader + * version number all prefixed by a magic string and concatenated with + * other numbers. Depending on the device, the firmware (bootloader) + * format is either "FW_Vxx.xx.xx" ("BL_Vxx.xx.xx") or "FW:xx.xx.xx" + * ("BL:xx.xx.xx") where 'x' represents a digit. @prod_info must + * contains the common part of those prefixes: "FW" or "BL". + * + * Parse @prod_info and store the version number in + * &es58x_dev.firmware_version or &es58x_dev.bootloader_version + * according to @prefix value. + * + * Return: zero on success, -EINVAL if @prefix contains an invalid + * value and -EBADMSG if @prod_info could not be parsed. + */ +static int es58x_parse_sw_version(struct es58x_device *es58x_dev, + const char *prod_info, const char *prefix) +{ + struct es58x_sw_version *version; + int major, minor, revision; + + if (!strcmp(prefix, "FW")) + version = &es58x_dev->firmware_version; + else if (!strcmp(prefix, "BL")) + version = &es58x_dev->bootloader_version; + else + return -EINVAL; + + /* Go to prefix */ + prod_info = strstr(prod_info, prefix); + if (!prod_info) + return -EBADMSG; + /* Go to beginning of the version number */ + while (!isdigit(*prod_info)) { + prod_info++; + if (!*prod_info) + return -EBADMSG; + } + + if (sscanf(prod_info, "%2u.%2u.%2u", &major, &minor, &revision) != 3) + return -EBADMSG; + + version->major = major; + version->minor = minor; + version->revision = revision; + + return 0; +} + +/** + * es58x_parse_hw_rev() - Extract hardware revision number. + * @es58x_dev: ES58X device. + * @prod_info: USB custom string returned by the device. + * + * @prod_info contains the hardware revision prefixed by a magic + * string and conquenated together with other numbers. Depending on + * the device, the hardware revision format is either + * "HW_VER:axxx/xxx" or "HR:axxx/xxx" where 'a' represents a letter + * and 'x' a digit. + * + * Parse @prod_info and store the hardware revision number in + * &es58x_dev.hardware_revision. + * + * Return: zero on success, -EBADMSG if @prod_info could not be + * parsed. + */ +static int es58x_parse_hw_rev(struct es58x_device *es58x_dev, + const char *prod_info) +{ + char letter; + int major, minor; + + /* The only occurrence of 'H' is in the hardware revision prefix. */ + prod_info = strchr(prod_info, 'H'); + if (!prod_info) + return -EBADMSG; + /* Go to beginning of the hardware revision */ + prod_info = strchr(prod_info, ':'); + if (!prod_info) + return -EBADMSG; + prod_info++; + + if (sscanf(prod_info, "%c%3u/%3u", &letter, &major, &minor) != 3) + return -EBADMSG; + + es58x_dev->hardware_revision.letter = letter; + es58x_dev->hardware_revision.major = major; + es58x_dev->hardware_revision.minor = minor; + + return 0; +} + +/** + * es58x_parse_product_info() - Parse the ES58x product information + * string. + * @es58x_dev: ES58X device. + * + * Retrieve the product information string and parse it to extract the + * firmware version, the bootloader version and the hardware + * revision. + * + * If the function fails, simply emit a log message and continue + * because product information is not critical for the driver to + * operate. + */ +void es58x_parse_product_info(struct es58x_device *es58x_dev) +{ + char *prod_info; + + prod_info = usb_cache_string(es58x_dev->udev, ES58X_PROD_INFO_IDX); + if (!prod_info) { + dev_warn(es58x_dev->dev, + "could not retrieve the product info string\n"); + return; + } + + if (es58x_parse_sw_version(es58x_dev, prod_info, "FW") || + es58x_parse_sw_version(es58x_dev, prod_info, "BL") || + es58x_parse_hw_rev(es58x_dev, prod_info)) + dev_info(es58x_dev->dev, + "could not parse product info: '%s'\n", prod_info); + + kfree(prod_info); +} + +/** + * es58x_sw_version_is_set() - Check if the version is a valid number. + * @sw_ver: Version number of either the firmware or the bootloader. + * + * If &es58x_sw_version.major, &es58x_sw_version.minor and + * &es58x_sw_version.revision are all zero, the product string could + * not be parsed and the version number is invalid. + */ +static inline bool es58x_sw_version_is_set(struct es58x_sw_version *sw_ver) +{ + return sw_ver->major || sw_ver->minor || sw_ver->revision; +} + +/** + * es58x_hw_revision_is_set() - Check if the revision is a valid number. + * @hw_rev: Revision number of the hardware. + * + * If &es58x_hw_revision.letter is the null character, the product + * string could not be parsed and the hardware revision number is + * invalid. + */ +static inline bool es58x_hw_revision_is_set(struct es58x_hw_revision *hw_rev) +{ + return hw_rev->letter != '\0'; +} + +/** + * es58x_devlink_info_get() - Report the product information. + * @devlink: Devlink. + * @req: skb wrapper where to put requested information. + * @extack: Unused. + * + * Report the firmware version, the bootloader version, the hardware + * revision and the serial number through netlink. + * + * Return: zero on success, errno when any error occurs. + */ +static int es58x_devlink_info_get(struct devlink *devlink, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct es58x_device *es58x_dev = devlink_priv(devlink); + struct es58x_sw_version *fw_ver = &es58x_dev->firmware_version; + struct es58x_sw_version *bl_ver = &es58x_dev->bootloader_version; + struct es58x_hw_revision *hw_rev = &es58x_dev->hardware_revision; + char buf[max(sizeof("xx.xx.xx"), sizeof("axxx/xxx"))]; + int ret = 0; + + if (es58x_sw_version_is_set(fw_ver)) { + snprintf(buf, sizeof(buf), "%02u.%02u.%02u", + fw_ver->major, fw_ver->minor, fw_ver->revision); + ret = devlink_info_version_running_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW, + buf); + if (ret) + return ret; + } + + if (es58x_sw_version_is_set(bl_ver)) { + snprintf(buf, sizeof(buf), "%02u.%02u.%02u", + bl_ver->major, bl_ver->minor, bl_ver->revision); + ret = devlink_info_version_running_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW_BOOTLOADER, + buf); + if (ret) + return ret; + } + + if (es58x_hw_revision_is_set(hw_rev)) { + snprintf(buf, sizeof(buf), "%c%03u/%03u", + hw_rev->letter, hw_rev->major, hw_rev->minor); + ret = devlink_info_version_fixed_put(req, + DEVLINK_INFO_VERSION_GENERIC_BOARD_REV, + buf); + if (ret) + return ret; + } + + return devlink_info_serial_number_put(req, es58x_dev->udev->serial); +} + +const struct devlink_ops es58x_dl_ops = { + .info_get = es58x_devlink_info_get, +}; diff --git a/drivers/net/can/usb/etas_es58x/es58x_fd.c b/drivers/net/can/usb/etas_es58x/es58x_fd.c index c97ffa71fd75..fa87b0b78e3e 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_fd.c +++ b/drivers/net/can/usb/etas_es58x/es58x_fd.c @@ -8,12 +8,12 @@ * * Copyright (c) 2019 Robert Bosch Engineering and Business Solutions. All rights reserved. * Copyright (c) 2020 ETAS K.K.. All rights reserved. - * Copyright (c) 2020, 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr> + * Copyright (c) 2020-2022 Vincent Mailhol <mailhol.vincent@wanadoo.fr> */ +#include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/units.h> -#include <asm/unaligned.h> #include "es58x_core.h" #include "es58x_fd.h" diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 838744d2ce34..d476c2884008 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -299,7 +299,6 @@ struct gs_can { struct net_device *netdev; struct usb_device *udev; - struct usb_interface *iface; struct can_bittiming_const bt_const, data_bt_const; unsigned int channel; /* channel number */ @@ -383,8 +382,7 @@ static int gs_cmd_reset(struct gs_can *dev) .mode = GS_CAN_MODE_RESET, }; - return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_MODE, + return usb_control_msg_send(dev->udev, 0, GS_USB_BREQ_MODE, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &dm, sizeof(dm), 1000, GFP_KERNEL); @@ -396,8 +394,7 @@ static inline int gs_usb_get_timestamp(const struct gs_can *dev, __le32 timestamp; int rc; - rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_TIMESTAMP, + rc = usb_control_msg_recv(dev->udev, 0, GS_USB_BREQ_TIMESTAMP, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, ×tamp, sizeof(timestamp), @@ -674,8 +671,7 @@ static int gs_usb_set_bittiming(struct net_device *netdev) }; /* request bit timings */ - return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_BITTIMING, + return usb_control_msg_send(dev->udev, 0, GS_USB_BREQ_BITTIMING, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &dbt, sizeof(dbt), 1000, GFP_KERNEL); @@ -698,8 +694,7 @@ static int gs_usb_set_data_bittiming(struct net_device *netdev) request = GS_USB_BREQ_QUIRK_CANTACT_PRO_DATA_BITTIMING; /* request data bit timings */ - return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - request, + return usb_control_msg_send(dev->udev, 0, request, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &dbt, sizeof(dbt), 1000, GFP_KERNEL); @@ -941,8 +936,7 @@ static int gs_can_open(struct net_device *netdev) /* finally start device */ dev->can.state = CAN_STATE_ERROR_ACTIVE; dm.flags = cpu_to_le32(flags); - rc = usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_MODE, + rc = usb_control_msg_send(dev->udev, 0, GS_USB_BREQ_MODE, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &dm, sizeof(dm), 1000, GFP_KERNEL); @@ -969,8 +963,7 @@ static int gs_usb_get_state(const struct net_device *netdev, struct gs_device_state ds; int rc; - rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_GET_STATE, + rc = usb_control_msg_recv(dev->udev, 0, GS_USB_BREQ_GET_STATE, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &ds, sizeof(ds), @@ -1064,8 +1057,7 @@ static int gs_usb_set_identify(struct net_device *netdev, bool do_identify) else imode.mode = cpu_to_le32(GS_CAN_IDENTIFY_OFF); - return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_IDENTIFY, + return usb_control_msg_send(dev->udev, 0, GS_USB_BREQ_IDENTIFY, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &imode, sizeof(imode), 100, GFP_KERNEL); @@ -1118,8 +1110,7 @@ static int gs_usb_get_termination(struct net_device *netdev, u16 *term) struct gs_device_termination_state term_state; int rc; - rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_GET_TERMINATION, + rc = usb_control_msg_recv(dev->udev, 0, GS_USB_BREQ_GET_TERMINATION, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &term_state, sizeof(term_state), 1000, @@ -1145,8 +1136,7 @@ static int gs_usb_set_termination(struct net_device *netdev, u16 term) else term_state.state = cpu_to_le32(GS_CAN_TERMINATION_STATE_OFF); - return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, - GS_USB_BREQ_SET_TERMINATION, + return usb_control_msg_send(dev->udev, 0, GS_USB_BREQ_SET_TERMINATION, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, dev->channel, 0, &term_state, sizeof(term_state), 1000, @@ -1210,7 +1200,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->bt_const.brp_inc = le32_to_cpu(bt_const.brp_inc); dev->udev = interface_to_usbdev(intf); - dev->iface = intf; dev->netdev = netdev; dev->channel = channel; diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index 3a2bfaad1406..d4c5356d5884 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -536,12 +536,11 @@ static int kvaser_usb_set_bittiming(struct net_device *netdev) struct kvaser_usb *dev = priv->dev; const struct kvaser_usb_dev_ops *ops = dev->driver_info->ops; struct can_bittiming *bt = &priv->can.bittiming; - struct kvaser_usb_busparams busparams; int tseg1 = bt->prop_seg + bt->phase_seg1; int tseg2 = bt->phase_seg2; int sjw = bt->sjw; - int err = -EOPNOTSUPP; + int err; busparams.bitrate = cpu_to_le32(bt->bitrate); busparams.sjw = (u8)sjw; @@ -581,7 +580,6 @@ static int kvaser_usb_set_data_bittiming(struct net_device *netdev) struct kvaser_usb *dev = priv->dev; const struct kvaser_usb_dev_ops *ops = dev->driver_info->ops; struct can_bittiming *dbt = &priv->can.data_bittiming; - struct kvaser_usb_busparams busparams; int tseg1 = dbt->prop_seg + dbt->phase_seg1; int tseg2 = dbt->phase_seg2; diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index ffa38f533c35..a0f7bcec719c 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -277,7 +277,6 @@ struct ucan_priv { /* linux USB device structures */ struct usb_device *udev; - struct usb_interface *intf; struct net_device *netdev; /* lock for can->echo_skb (used around @@ -1501,7 +1500,6 @@ static int ucan_probe(struct usb_interface *intf, /* initialize data */ up->udev = udev; - up->intf = intf; up->netdev = netdev; up->intf_index = iface_desc->desc.bInterfaceNumber; up->in_ep_addr = in_ep_addr; @@ -1534,9 +1532,8 @@ static int ucan_probe(struct usb_interface *intf, sizeof(union ucan_ctl_payload)); if (ret > 0) { /* copy string while ensuring zero termination */ - strncpy(firmware_str, up->ctl_msg_buffer->raw, - sizeof(union ucan_ctl_payload)); - firmware_str[sizeof(union ucan_ctl_payload)] = '\0'; + strscpy(firmware_str, up->ctl_msg_buffer->raw, + sizeof(union ucan_ctl_payload) + 1); } else { strcpy(firmware_str, "unknown"); } diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c index 951f7935c872..595a548bb0a8 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.c +++ b/drivers/net/dsa/hirschmann/hellcreek.c @@ -1176,11 +1176,6 @@ static int hellcreek_devlink_info_get(struct dsa_switch *ds, struct netlink_ext_ack *extack) { struct hellcreek *hellcreek = ds->priv; - int ret; - - ret = devlink_info_driver_name_put(req, "hellcreek"); - if (ret) - return ret; return devlink_info_version_fixed_put(req, DEVLINK_INFO_VERSION_GENERIC_ASIC_ID, diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h index 8582b4b67d98..ea05abfbd51d 100644 --- a/drivers/net/dsa/microchip/ksz8.h +++ b/drivers/net/dsa/microchip/ksz8.h @@ -57,5 +57,6 @@ int ksz8_reset_switch(struct ksz_device *dev); int ksz8_switch_detect(struct ksz_device *dev); int ksz8_switch_init(struct ksz_device *dev); void ksz8_switch_exit(struct ksz_device *dev); +int ksz8_change_mtu(struct ksz_device *dev, int port, int mtu); #endif diff --git a/drivers/net/dsa/microchip/ksz8795.c b/drivers/net/dsa/microchip/ksz8795.c index bd3b133e7085..003b0ac2854c 100644 --- a/drivers/net/dsa/microchip/ksz8795.c +++ b/drivers/net/dsa/microchip/ksz8795.c @@ -76,6 +76,57 @@ int ksz8_reset_switch(struct ksz_device *dev) return 0; } +static int ksz8863_change_mtu(struct ksz_device *dev, int frame_size) +{ + u8 ctrl2 = 0; + + if (frame_size <= KSZ8_LEGAL_PACKET_SIZE) + ctrl2 |= KSZ8863_LEGAL_PACKET_ENABLE; + else if (frame_size > KSZ8863_NORMAL_PACKET_SIZE) + ctrl2 |= KSZ8863_HUGE_PACKET_ENABLE; + + return ksz_rmw8(dev, REG_SW_CTRL_2, KSZ8863_LEGAL_PACKET_ENABLE | + KSZ8863_HUGE_PACKET_ENABLE, ctrl2); +} + +static int ksz8795_change_mtu(struct ksz_device *dev, int frame_size) +{ + u8 ctrl1 = 0, ctrl2 = 0; + int ret; + + if (frame_size > KSZ8_LEGAL_PACKET_SIZE) + ctrl2 |= SW_LEGAL_PACKET_DISABLE; + else if (frame_size > KSZ8863_NORMAL_PACKET_SIZE) + ctrl1 |= SW_HUGE_PACKET; + + ret = ksz_rmw8(dev, REG_SW_CTRL_1, SW_HUGE_PACKET, ctrl1); + if (ret) + return ret; + + return ksz_rmw8(dev, REG_SW_CTRL_2, SW_LEGAL_PACKET_DISABLE, ctrl2); +} + +int ksz8_change_mtu(struct ksz_device *dev, int port, int mtu) +{ + u16 frame_size; + + if (!dsa_is_cpu_port(dev->ds, port)) + return 0; + + frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; + + switch (dev->chip_id) { + case KSZ8795_CHIP_ID: + case KSZ8794_CHIP_ID: + case KSZ8765_CHIP_ID: + return ksz8795_change_mtu(dev, frame_size); + case KSZ8830_CHIP_ID: + return ksz8863_change_mtu(dev, frame_size); + } + + return -EOPNOTSUPP; +} + static void ksz8795_set_prio_queue(struct ksz_device *dev, int port, int queue) { u8 hi, lo; @@ -1233,8 +1284,6 @@ void ksz8_config_cpu_port(struct dsa_switch *ds) masks = dev->info->masks; regs = dev->info->regs; - /* Switch marks the maximum frame with extra byte as oversize. */ - ksz_cfg(dev, REG_SW_CTRL_2, SW_LEGAL_PACKET_DISABLE, true); ksz_cfg(dev, regs[S_TAIL_TAG_CTRL], masks[SW_TAIL_TAG_ENABLE], true); p = &dev->ports[dev->cpu_port]; @@ -1308,6 +1357,18 @@ int ksz8_setup(struct dsa_switch *ds) struct ksz_device *dev = ds->priv; int i; + ds->mtu_enforcement_ingress = true; + + /* We rely on software untagging on the CPU port, so that we + * can support both tagged and untagged VLANs + */ + ds->untag_bridge_pvid = true; + + /* VLAN filtering is partly controlled by the global VLAN + * Enable flag + */ + ds->vlan_filtering_is_global = true; + ksz_cfg(dev, S_REPLACE_VID_CTRL, SW_FLOW_CTRL, true); /* Enable automatic fast aging when link changed detected. */ @@ -1367,16 +1428,6 @@ int ksz8_switch_init(struct ksz_device *dev) dev->phy_port_cnt = dev->info->port_cnt - 1; dev->port_mask = (BIT(dev->phy_port_cnt) - 1) | dev->info->cpu_ports; - /* We rely on software untagging on the CPU port, so that we - * can support both tagged and untagged VLANs - */ - dev->ds->untag_bridge_pvid = true; - - /* VLAN filtering is partly controlled by the global VLAN - * Enable flag - */ - dev->ds->vlan_filtering_is_global = true; - return 0; } diff --git a/drivers/net/dsa/microchip/ksz8795_reg.h b/drivers/net/dsa/microchip/ksz8795_reg.h index 77487d611824..7a57c6088f80 100644 --- a/drivers/net/dsa/microchip/ksz8795_reg.h +++ b/drivers/net/dsa/microchip/ksz8795_reg.h @@ -48,6 +48,9 @@ #define NO_EXC_COLLISION_DROP BIT(3) #define SW_LEGAL_PACKET_DISABLE BIT(1) +#define KSZ8863_HUGE_PACKET_ENABLE BIT(2) +#define KSZ8863_LEGAL_PACKET_ENABLE BIT(1) + #define REG_SW_CTRL_3 0x05 #define WEIGHTED_FAIR_QUEUE_ENABLE BIT(3) diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c index 0d6b40968657..47b54ecf2c6f 100644 --- a/drivers/net/dsa/microchip/ksz9477.c +++ b/drivers/net/dsa/microchip/ksz9477.c @@ -45,24 +45,15 @@ static void ksz9477_port_cfg32(struct ksz_device *dev, int port, int offset, int ksz9477_change_mtu(struct ksz_device *dev, int port, int mtu) { - u16 frame_size, max_frame = 0; - int i; - - frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; + u16 frame_size; - /* Cache the per-port MTU setting */ - dev->ports[port].max_frame = frame_size; + if (!dsa_is_cpu_port(dev->ds, port)) + return 0; - for (i = 0; i < dev->info->port_cnt; i++) - max_frame = max(max_frame, dev->ports[i].max_frame); + frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; return regmap_update_bits(dev->regmap[1], REG_SW_MTU__2, - REG_SW_MTU_MASK, max_frame); -} - -int ksz9477_max_mtu(struct ksz_device *dev, int port) -{ - return KSZ9477_MAX_FRAME_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN; + REG_SW_MTU_MASK, frame_size); } static int ksz9477_wait_vlan_ctrl_ready(struct ksz_device *dev) @@ -1143,6 +1134,8 @@ int ksz9477_setup(struct dsa_switch *ds) struct ksz_device *dev = ds->priv; int ret = 0; + ds->mtu_enforcement_ingress = true; + /* Required for port partitioning. */ ksz9477_cfg32(dev, REG_SW_QM_CTRL__4, UNICAST_VLAN_BOUNDARY, true); diff --git a/drivers/net/dsa/microchip/ksz9477.h b/drivers/net/dsa/microchip/ksz9477.h index 00862c4cfb7f..7c5bb3032772 100644 --- a/drivers/net/dsa/microchip/ksz9477.h +++ b/drivers/net/dsa/microchip/ksz9477.h @@ -50,7 +50,6 @@ int ksz9477_mdb_add(struct ksz_device *dev, int port, int ksz9477_mdb_del(struct ksz_device *dev, int port, const struct switchdev_obj_port_mdb *mdb, struct dsa_db db); int ksz9477_change_mtu(struct ksz_device *dev, int port, int mtu); -int ksz9477_max_mtu(struct ksz_device *dev, int port); void ksz9477_config_cpu_port(struct dsa_switch *ds); int ksz9477_enable_stp_addr(struct ksz_device *dev); int ksz9477_reset_switch(struct ksz_device *dev); diff --git a/drivers/net/dsa/microchip/ksz9477_reg.h b/drivers/net/dsa/microchip/ksz9477_reg.h index 53c68d286dd3..cc457fa64939 100644 --- a/drivers/net/dsa/microchip/ksz9477_reg.h +++ b/drivers/net/dsa/microchip/ksz9477_reg.h @@ -1615,6 +1615,4 @@ #define PTP_TRIG_UNIT_M (BIT(MAX_TRIG_UNIT) - 1) #define PTP_TS_UNIT_M (BIT(MAX_TIMESTAMP_UNIT) - 1) -#define KSZ9477_MAX_FRAME_SIZE 9000 - #endif /* KSZ9477_REGS_H */ diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 8c8db315317d..423f944cc34c 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -14,6 +14,7 @@ #include <linux/phy.h> #include <linux/etherdevice.h> #include <linux/if_bridge.h> +#include <linux/if_vlan.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/of_mdio.h> @@ -69,6 +70,43 @@ struct ksz_stats_raw { u64 tx_discards; }; +struct ksz88xx_stats_raw { + u64 rx; + u64 rx_hi; + u64 rx_undersize; + u64 rx_fragments; + u64 rx_oversize; + u64 rx_jabbers; + u64 rx_symbol_err; + u64 rx_crc_err; + u64 rx_align_err; + u64 rx_mac_ctrl; + u64 rx_pause; + u64 rx_bcast; + u64 rx_mcast; + u64 rx_ucast; + u64 rx_64_or_less; + u64 rx_65_127; + u64 rx_128_255; + u64 rx_256_511; + u64 rx_512_1023; + u64 rx_1024_1522; + u64 tx; + u64 tx_hi; + u64 tx_late_col; + u64 tx_pause; + u64 tx_bcast; + u64 tx_mcast; + u64 tx_ucast; + u64 tx_deferred; + u64 tx_total_col; + u64 tx_exc_col; + u64 tx_single_col; + u64 tx_mult_col; + u64 rx_discards; + u64 tx_discards; +}; + static const struct ksz_mib_names ksz88xx_mib_names[] = { { 0x00, "rx" }, { 0x01, "rx_hi" }, @@ -155,6 +193,7 @@ static const struct ksz_dev_ops ksz8_dev_ops = { .w_phy = ksz8_w_phy, .r_mib_cnt = ksz8_r_mib_cnt, .r_mib_pkt = ksz8_r_mib_pkt, + .r_mib_stat64 = ksz88xx_r_mib_stats64, .freeze_mib = ksz8_freeze_mib, .port_init_cnt = ksz8_port_init_cnt, .fdb_dump = ksz8_fdb_dump, @@ -171,6 +210,7 @@ static const struct ksz_dev_ops ksz8_dev_ops = { .reset = ksz8_reset_switch, .init = ksz8_switch_init, .exit = ksz8_switch_exit, + .change_mtu = ksz8_change_mtu, }; static void ksz9477_phylink_mac_link_up(struct ksz_device *dev, int port, @@ -206,7 +246,6 @@ static const struct ksz_dev_ops ksz9477_dev_ops = { .mdb_add = ksz9477_mdb_add, .mdb_del = ksz9477_mdb_del, .change_mtu = ksz9477_change_mtu, - .max_mtu = ksz9477_max_mtu, .phylink_mac_link_up = ksz9477_phylink_mac_link_up, .config_cpu_port = ksz9477_config_cpu_port, .enable_stp_addr = ksz9477_enable_stp_addr, @@ -243,7 +282,6 @@ static const struct ksz_dev_ops lan937x_dev_ops = { .mdb_add = ksz9477_mdb_add, .mdb_del = ksz9477_mdb_del, .change_mtu = lan937x_change_mtu, - .max_mtu = ksz9477_max_mtu, .phylink_mac_link_up = ksz9477_phylink_mac_link_up, .config_cpu_port = lan937x_config_cpu_port, .enable_stp_addr = ksz9477_enable_stp_addr, @@ -1583,6 +1621,55 @@ void ksz_r_mib_stats64(struct ksz_device *dev, int port) spin_unlock(&mib->stats64_lock); } +void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port) +{ + struct ethtool_pause_stats *pstats; + struct rtnl_link_stats64 *stats; + struct ksz88xx_stats_raw *raw; + struct ksz_port_mib *mib; + + mib = &dev->ports[port].mib; + stats = &mib->stats64; + pstats = &mib->pause_stats; + raw = (struct ksz88xx_stats_raw *)mib->counters; + + spin_lock(&mib->stats64_lock); + + stats->rx_packets = raw->rx_bcast + raw->rx_mcast + raw->rx_ucast + + raw->rx_pause; + stats->tx_packets = raw->tx_bcast + raw->tx_mcast + raw->tx_ucast + + raw->tx_pause; + + /* HW counters are counting bytes + FCS which is not acceptable + * for rtnl_link_stats64 interface + */ + stats->rx_bytes = raw->rx + raw->rx_hi - stats->rx_packets * ETH_FCS_LEN; + stats->tx_bytes = raw->tx + raw->tx_hi - stats->tx_packets * ETH_FCS_LEN; + + stats->rx_length_errors = raw->rx_undersize + raw->rx_fragments + + raw->rx_oversize; + + stats->rx_crc_errors = raw->rx_crc_err; + stats->rx_frame_errors = raw->rx_align_err; + stats->rx_dropped = raw->rx_discards; + stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + + stats->rx_frame_errors + stats->rx_dropped; + + stats->tx_window_errors = raw->tx_late_col; + stats->tx_fifo_errors = raw->tx_discards; + stats->tx_aborted_errors = raw->tx_exc_col; + stats->tx_errors = stats->tx_window_errors + stats->tx_fifo_errors + + stats->tx_aborted_errors; + + stats->multicast = raw->rx_mcast; + stats->collisions = raw->tx_total_col; + + pstats->tx_pause_frames = raw->tx_pause; + pstats->rx_pause_frames = raw->rx_pause; + + spin_unlock(&mib->stats64_lock); +} + static void ksz_get_stats64(struct dsa_switch *ds, int port, struct rtnl_link_stats64 *s) { @@ -2500,10 +2587,29 @@ static int ksz_max_mtu(struct dsa_switch *ds, int port) { struct ksz_device *dev = ds->priv; - if (!dev->dev_ops->max_mtu) - return -EOPNOTSUPP; + switch (dev->chip_id) { + case KSZ8795_CHIP_ID: + case KSZ8794_CHIP_ID: + case KSZ8765_CHIP_ID: + return KSZ8795_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN; + case KSZ8830_CHIP_ID: + return KSZ8863_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN; + case KSZ8563_CHIP_ID: + case KSZ9477_CHIP_ID: + case KSZ9563_CHIP_ID: + case KSZ9567_CHIP_ID: + case KSZ9893_CHIP_ID: + case KSZ9896_CHIP_ID: + case KSZ9897_CHIP_ID: + case LAN9370_CHIP_ID: + case LAN9371_CHIP_ID: + case LAN9372_CHIP_ID: + case LAN9373_CHIP_ID: + case LAN9374_CHIP_ID: + return KSZ9477_MAX_FRAME_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN; + } - return dev->dev_ops->max_mtu(dev, port); + return -EOPNOTSUPP; } static void ksz_set_xmii(struct ksz_device *dev, int port, diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index c6726cbd5465..055d61ff3fb8 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -95,7 +95,6 @@ struct ksz_port { struct ksz_port_mib mib; phy_interface_t interface; - u16 max_frame; u32 rgmii_tx_val; u32 rgmii_rx_val; struct ksz_device *ksz_dev; @@ -322,7 +321,6 @@ struct ksz_dev_ops { void (*get_caps)(struct ksz_device *dev, int port, struct phylink_config *config); int (*change_mtu)(struct ksz_device *dev, int port, int mtu); - int (*max_mtu)(struct ksz_device *dev, int port); void (*freeze_mib)(struct ksz_device *dev, int port, bool freeze); void (*port_init_cnt)(struct ksz_device *dev, int port); void (*phylink_mac_config)(struct ksz_device *dev, int port, @@ -347,6 +345,7 @@ void ksz_switch_remove(struct ksz_device *dev); void ksz_init_mib_timer(struct ksz_device *dev); void ksz_r_mib_stats64(struct ksz_device *dev, int port); +void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port); void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state); bool ksz_get_gbit(struct ksz_device *dev, int port); phy_interface_t ksz_get_xmii(struct ksz_device *dev, int port, bool gbit); @@ -456,6 +455,11 @@ static inline int ksz_write64(struct ksz_device *dev, u32 reg, u64 value) return regmap_bulk_write(dev->regmap[2], reg, val, 2); } +static inline int ksz_rmw8(struct ksz_device *dev, int offset, u8 mask, u8 val) +{ + return regmap_update_bits(dev->regmap[0], offset, mask, val); +} + static inline int ksz_pread8(struct ksz_device *dev, int port, int offset, u8 *data) { @@ -588,6 +592,12 @@ static inline int is_lan937x(struct ksz_device *dev) #define PORT_SRC_PHY_INT 1 +#define KSZ8795_HUGE_PACKET_SIZE 2000 +#define KSZ8863_HUGE_PACKET_SIZE 1916 +#define KSZ8863_NORMAL_PACKET_SIZE 1536 +#define KSZ8_LEGAL_PACKET_SIZE 1518 +#define KSZ9477_MAX_FRAME_SIZE 9000 + /* Regmap tables generation */ #define KSZ_SPI_OP_RD 3 #define KSZ_SPI_OP_WR 2 diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index ccfa4751d3b7..ba4fff8690aa 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -833,10 +833,13 @@ static void mv88e6xxx_get_caps(struct dsa_switch *ds, int port, chip->info->ops->phylink_get_caps(chip, port, config); - /* Internal ports need GMII for PHYLIB */ - if (mv88e6xxx_phy_is_internal(ds, port)) + if (mv88e6xxx_phy_is_internal(ds, port)) { + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + /* Internal ports with no phy-mode need GMII for PHYLIB */ __set_bit(PHY_INTERFACE_MODE_GMII, config->supported_interfaces); + } } static void mv88e6xxx_mac_config(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/mv88e6xxx/devlink.c b/drivers/net/dsa/mv88e6xxx/devlink.c index 1266eabee086..a08dab75e0c0 100644 --- a/drivers/net/dsa/mv88e6xxx/devlink.c +++ b/drivers/net/dsa/mv88e6xxx/devlink.c @@ -821,11 +821,6 @@ int mv88e6xxx_devlink_info_get(struct dsa_switch *ds, struct netlink_ext_ack *extack) { struct mv88e6xxx_chip *chip = ds->priv; - int err; - - err = devlink_info_driver_name_put(req, "mv88e6xxx"); - if (err) - return err; return devlink_info_version_fixed_put(req, DEVLINK_INFO_VERSION_GENERIC_ASIC_ID, diff --git a/drivers/net/dsa/sja1105/sja1105_devlink.c b/drivers/net/dsa/sja1105/sja1105_devlink.c index 10c6fea1227f..30b1f1ba762f 100644 --- a/drivers/net/dsa/sja1105/sja1105_devlink.c +++ b/drivers/net/dsa/sja1105/sja1105_devlink.c @@ -95,6 +95,8 @@ static int sja1105_setup_devlink_regions(struct dsa_switch *ds) if (IS_ERR(region)) { while (--i >= 0) dsa_devlink_region_destroy(priv->regions[i]); + + kfree(priv->regions); return PTR_ERR(region); } @@ -120,16 +122,10 @@ int sja1105_devlink_info_get(struct dsa_switch *ds, struct netlink_ext_ack *extack) { struct sja1105_private *priv = ds->priv; - int rc; - - rc = devlink_info_driver_name_put(req, "sja1105"); - if (rc) - return rc; - rc = devlink_info_version_fixed_put(req, - DEVLINK_INFO_VERSION_GENERIC_ASIC_ID, - priv->info->name); - return rc; + return devlink_info_version_fixed_put(req, + DEVLINK_INFO_VERSION_GENERIC_ASIC_ID, + priv->info->name); } int sja1105_devlink_setup(struct dsa_switch *ds) diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 412666111b0c..b70dcf32a26d 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -1038,7 +1038,7 @@ static int sja1105_init_l2_policing(struct sja1105_private *priv) policing[bcast].sharindx = port; /* Only SJA1110 has multicast policers */ - if (mcast <= table->ops->max_entry_count) + if (mcast < table->ops->max_entry_count) policing[mcast].sharindx = port; } diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c index e104fb02817d..aa0d2f3aaeaa 100644 --- a/drivers/net/ethernet/aeroflex/greth.c +++ b/drivers/net/ethernet/aeroflex/greth.c @@ -258,6 +258,7 @@ static int greth_init_rings(struct greth_private *greth) if (dma_mapping_error(greth->dev, dma_addr)) { if (netif_msg_ifup(greth)) dev_err(greth->dev, "Could not create initial DMA mapping\n"); + dev_kfree_skb(skb); goto cleanup; } greth->rx_skbuff[i] = skb; diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index 55dfdb34e37b..f4ca0c6c0f51 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -71,13 +71,14 @@ config BCM63XX_ENET config BCMGENET tristate "Broadcom GENET internal MAC support" depends on HAS_IOMEM + depends on PTP_1588_CLOCK_OPTIONAL || !ARCH_BCM2835 select MII select PHYLIB select FIXED_PHY select BCM7XXX_PHY select MDIO_BCM_UNIMAC select DIMLIB - select BROADCOM_PHY if (ARCH_BCM2835 && PTP_1588_CLOCK_OPTIONAL) + select BROADCOM_PHY if ARCH_BCM2835 help This driver supports the built-in Ethernet MACs found in the Broadcom BCM7xxx Set Top Box family chipset. diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index dbe310144780..9f473854b0f4 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -3045,7 +3045,7 @@ error: dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, DMA_FROM_DEVICE); - skb = build_skb(data, 0); + skb = slab_build_skb(data); if (!skb) { kfree(data); goto error; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c index 8a6f788f6294..26913dc816d3 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c @@ -892,10 +892,6 @@ static int bnxt_dl_info_get(struct devlink *dl, struct devlink_info_req *req, u32 ver = 0; int rc; - rc = devlink_info_driver_name_put(req, DRV_MODULE_NAME); - if (rc) - return rc; - if (BNXT_PF(bp) && (bp->flags & BNXT_FLAG_DSN_VALID)) { sprintf(buf, "%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X", bp->dsn[7], bp->dsn[6], bp->dsn[5], bp->dsn[4], diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index c2f663770a7f..cbf17fcfb7ab 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -2007,6 +2007,14 @@ static void bnxt_get_fec_stats(struct net_device *dev, rx = bp->rx_port_stats_ext.sw_stats; fec_stats->corrected_bits.total = *(rx + BNXT_RX_STATS_EXT_OFFSET(rx_corrected_bits)); + + if (bp->fw_rx_stats_ext_size <= BNXT_RX_STATS_EXT_NUM_LEGACY) + return; + + fec_stats->corrected_blocks.total = + *(rx + BNXT_RX_STATS_EXT_OFFSET(rx_fec_corrected_blocks)); + fec_stats->uncorrectable_blocks.total = + *(rx + BNXT_RX_STATS_EXT_OFFSET(rx_fec_uncorrectable_blocks)); } static u32 bnxt_ethtool_forced_fec_to_fw(struct bnxt_link_info *link_info, diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index a8ce8d0cf9c4..21973046b12b 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -117,24 +117,6 @@ static inline void dmadesc_set(struct bcmgenet_priv *priv, dmadesc_set_length_status(priv, d, val); } -static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv, - void __iomem *d) -{ - dma_addr_t addr; - - addr = bcmgenet_readl(d + DMA_DESC_ADDRESS_LO); - - /* Register writes to GISB bus can take couple hundred nanoseconds - * and are done for each packet, save these expensive writes unless - * the platform is explicitly configured for 64-bits/LPAE. - */ -#ifdef CONFIG_PHYS_ADDR_T_64BIT - if (priv->hw_params->flags & GENET_HAS_40BITS) - addr |= (u64)bcmgenet_readl(d + DMA_DESC_ADDRESS_HI) << 32; -#endif - return addr; -} - #define GENET_VER_FMT "%1d.%1d EPHY: 0x%04x" #define GENET_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_console.c b/drivers/net/ethernet/cavium/liquidio/octeon_console.c index 28feabec8fbb..67c3570f875f 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_console.c +++ b/drivers/net/ethernet/cavium/liquidio/octeon_console.c @@ -247,8 +247,7 @@ static const struct cvmx_bootmem_named_block_desc struct cvmx_bootmem_named_block_desc, size)); - strncpy(desc->name, name, sizeof(desc->name)); - desc->name[sizeof(desc->name) - 1] = 0; + strscpy(desc->name, name, sizeof(desc->name)); return &oct->bootmem_named_block_desc; } else { return NULL; @@ -471,8 +470,8 @@ static void output_console_line(struct octeon_device *oct, if (line != &console_buffer[bytes_read]) { console_buffer[bytes_read] = '\0'; len = strlen(console->leftover); - strncpy(&console->leftover[len], line, - sizeof(console->leftover) - len); + strscpy(&console->leftover[len], line, + sizeof(console->leftover) - len + 1); } } diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index 98f3dc460ca7..f2f95493ec89 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -2239,7 +2239,7 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_unregister_interrupts; + goto err_destroy_workqueue; } nic->msg_enable = debug; @@ -2248,6 +2248,8 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; +err_destroy_workqueue: + destroy_workqueue(nic->nicvf_rx_mode_wq); err_unregister_interrupts: nicvf_unregister_interrupts(nic); err_free_netdev: diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 585590520076..ca21794281d6 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -283,6 +283,10 @@ static int ch_ipsec_xfrm_add_state(struct xfrm_state *x) pr_debug("Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + pr_debug("Unsupported xfrm offload\n"); + return -EINVAL; + } sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); if (!sa_entry) { diff --git a/drivers/net/ethernet/engleder/tsnep.h b/drivers/net/ethernet/engleder/tsnep.h index 09a723b827c7..f93ba48bac3f 100644 --- a/drivers/net/ethernet/engleder/tsnep.h +++ b/drivers/net/ethernet/engleder/tsnep.h @@ -18,6 +18,8 @@ #define TSNEP "tsnep" #define TSNEP_RING_SIZE 256 +#define TSNEP_RING_RX_REFILL 16 +#define TSNEP_RING_RX_REUSE (TSNEP_RING_SIZE - TSNEP_RING_SIZE / 4) #define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE) #define TSNEP_RING_PAGE_COUNT (TSNEP_RING_SIZE / TSNEP_RING_ENTRIES_PER_PAGE) @@ -110,6 +112,7 @@ struct tsnep_rx { dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT]; struct tsnep_rx_entry entry[TSNEP_RING_SIZE]; + int write; int read; u32 owner_counter; int increment_owner_counter; @@ -119,6 +122,7 @@ struct tsnep_rx { u32 bytes; u32 dropped; u32 multicast; + u32 alloc_failed; }; struct tsnep_queue { @@ -132,6 +136,8 @@ struct tsnep_queue { int irq; u32 irq_mask; + void __iomem *irq_delay_addr; + u8 irq_delay; }; struct tsnep_adapter { @@ -223,5 +229,7 @@ static inline void tsnep_ethtool_self_test(struct net_device *dev, #endif /* CONFIG_TSNEP_SELFTESTS */ void tsnep_get_system_time(struct tsnep_adapter *adapter, u64 *time); +int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs); +u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue); #endif /* _TSNEP_H */ diff --git a/drivers/net/ethernet/engleder/tsnep_ethtool.c b/drivers/net/ethernet/engleder/tsnep_ethtool.c index a713a126b227..716815dad7d2 100644 --- a/drivers/net/ethernet/engleder/tsnep_ethtool.c +++ b/drivers/net/ethernet/engleder/tsnep_ethtool.c @@ -8,6 +8,7 @@ static const char tsnep_stats_strings[][ETH_GSTRING_LEN] = { "rx_bytes", "rx_dropped", "rx_multicast", + "rx_alloc_failed", "rx_phy_errors", "rx_forwarded_phy_errors", "rx_invalid_frame_errors", @@ -21,6 +22,7 @@ struct tsnep_stats { u64 rx_bytes; u64 rx_dropped; u64 rx_multicast; + u64 rx_alloc_failed; u64 rx_phy_errors; u64 rx_forwarded_phy_errors; u64 rx_invalid_frame_errors; @@ -36,6 +38,7 @@ static const char tsnep_rx_queue_stats_strings[][ETH_GSTRING_LEN] = { "rx_%d_bytes", "rx_%d_dropped", "rx_%d_multicast", + "rx_%d_alloc_failed", "rx_%d_no_descriptor_errors", "rx_%d_buffer_too_small_errors", "rx_%d_fifo_overflow_errors", @@ -47,6 +50,7 @@ struct tsnep_rx_queue_stats { u64 rx_bytes; u64 rx_dropped; u64 rx_multicast; + u64 rx_alloc_failed; u64 rx_no_descriptor_errors; u64 rx_buffer_too_small_errors; u64 rx_fifo_overflow_errors; @@ -178,6 +182,7 @@ static void tsnep_ethtool_get_ethtool_stats(struct net_device *netdev, tsnep_stats.rx_bytes += adapter->rx[i].bytes; tsnep_stats.rx_dropped += adapter->rx[i].dropped; tsnep_stats.rx_multicast += adapter->rx[i].multicast; + tsnep_stats.rx_alloc_failed += adapter->rx[i].alloc_failed; } reg = ioread32(adapter->addr + ECM_STAT); tsnep_stats.rx_phy_errors = @@ -200,6 +205,8 @@ static void tsnep_ethtool_get_ethtool_stats(struct net_device *netdev, tsnep_rx_queue_stats.rx_bytes = adapter->rx[i].bytes; tsnep_rx_queue_stats.rx_dropped = adapter->rx[i].dropped; tsnep_rx_queue_stats.rx_multicast = adapter->rx[i].multicast; + tsnep_rx_queue_stats.rx_alloc_failed = + adapter->rx[i].alloc_failed; reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + TSNEP_RX_STATISTIC); tsnep_rx_queue_stats.rx_no_descriptor_errors = @@ -250,10 +257,10 @@ static int tsnep_ethtool_get_sset_count(struct net_device *netdev, int sset) } } -static int tsnep_ethtool_get_rxnfc(struct net_device *dev, +static int tsnep_ethtool_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, u32 *rule_locs) { - struct tsnep_adapter *adapter = netdev_priv(dev); + struct tsnep_adapter *adapter = netdev_priv(netdev); switch (cmd->cmd) { case ETHTOOL_GRXRINGS: @@ -273,10 +280,10 @@ static int tsnep_ethtool_get_rxnfc(struct net_device *dev, } } -static int tsnep_ethtool_set_rxnfc(struct net_device *dev, +static int tsnep_ethtool_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) { - struct tsnep_adapter *adapter = netdev_priv(dev); + struct tsnep_adapter *adapter = netdev_priv(netdev); switch (cmd->cmd) { case ETHTOOL_SRXCLSRLINS: @@ -288,10 +295,21 @@ static int tsnep_ethtool_set_rxnfc(struct net_device *dev, } } -static int tsnep_ethtool_get_ts_info(struct net_device *dev, +static void tsnep_ethtool_get_channels(struct net_device *netdev, + struct ethtool_channels *ch) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + ch->max_rx = adapter->num_rx_queues; + ch->max_tx = adapter->num_tx_queues; + ch->rx_count = adapter->num_rx_queues; + ch->tx_count = adapter->num_tx_queues; +} + +static int tsnep_ethtool_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) { - struct tsnep_adapter *adapter = netdev_priv(dev); + struct tsnep_adapter *adapter = netdev_priv(netdev); info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | SOF_TIMESTAMPING_RX_SOFTWARE | @@ -313,7 +331,137 @@ static int tsnep_ethtool_get_ts_info(struct net_device *dev, return 0; } +static struct tsnep_queue *tsnep_get_queue_with_tx(struct tsnep_adapter *adapter, + int index) +{ + int i; + + for (i = 0; i < adapter->num_queues; i++) { + if (adapter->queue[i].tx) { + if (index == 0) + return &adapter->queue[i]; + + index--; + } + } + + return NULL; +} + +static struct tsnep_queue *tsnep_get_queue_with_rx(struct tsnep_adapter *adapter, + int index) +{ + int i; + + for (i = 0; i < adapter->num_queues; i++) { + if (adapter->queue[i].rx) { + if (index == 0) + return &adapter->queue[i]; + + index--; + } + } + + return NULL; +} + +static int tsnep_ethtool_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct tsnep_queue *queue; + + queue = tsnep_get_queue_with_rx(adapter, 0); + if (queue) + ec->rx_coalesce_usecs = tsnep_get_irq_coalesce(queue); + + queue = tsnep_get_queue_with_tx(adapter, 0); + if (queue) + ec->tx_coalesce_usecs = tsnep_get_irq_coalesce(queue); + + return 0; +} + +static int tsnep_ethtool_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int i; + int retval; + + for (i = 0; i < adapter->num_queues; i++) { + /* RX coalesce has priority for queues with TX and RX */ + if (adapter->queue[i].rx) + retval = tsnep_set_irq_coalesce(&adapter->queue[i], + ec->rx_coalesce_usecs); + else + retval = tsnep_set_irq_coalesce(&adapter->queue[i], + ec->tx_coalesce_usecs); + if (retval != 0) + return retval; + } + + return 0; +} + +static int tsnep_ethtool_get_per_queue_coalesce(struct net_device *netdev, + u32 queue, + struct ethtool_coalesce *ec) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct tsnep_queue *queue_with_rx; + struct tsnep_queue *queue_with_tx; + + if (queue >= max(adapter->num_tx_queues, adapter->num_rx_queues)) + return -EINVAL; + + queue_with_rx = tsnep_get_queue_with_rx(adapter, queue); + if (queue_with_rx) + ec->rx_coalesce_usecs = tsnep_get_irq_coalesce(queue_with_rx); + + queue_with_tx = tsnep_get_queue_with_tx(adapter, queue); + if (queue_with_tx) + ec->tx_coalesce_usecs = tsnep_get_irq_coalesce(queue_with_tx); + + return 0; +} + +static int tsnep_ethtool_set_per_queue_coalesce(struct net_device *netdev, + u32 queue, + struct ethtool_coalesce *ec) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct tsnep_queue *queue_with_rx; + struct tsnep_queue *queue_with_tx; + int retval; + + if (queue >= max(adapter->num_tx_queues, adapter->num_rx_queues)) + return -EINVAL; + + queue_with_rx = tsnep_get_queue_with_rx(adapter, queue); + if (queue_with_rx) { + retval = tsnep_set_irq_coalesce(queue_with_rx, ec->rx_coalesce_usecs); + if (retval != 0) + return retval; + } + + /* RX coalesce has priority for queues with TX and RX */ + queue_with_tx = tsnep_get_queue_with_tx(adapter, queue); + if (queue_with_tx && !queue_with_tx->rx) { + retval = tsnep_set_irq_coalesce(queue_with_tx, ec->tx_coalesce_usecs); + if (retval != 0) + return retval; + } + + return 0; +} + const struct ethtool_ops tsnep_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS, .get_drvinfo = tsnep_ethtool_get_drvinfo, .get_regs_len = tsnep_ethtool_get_regs_len, .get_regs = tsnep_ethtool_get_regs, @@ -327,7 +475,12 @@ const struct ethtool_ops tsnep_ethtool_ops = { .get_sset_count = tsnep_ethtool_get_sset_count, .get_rxnfc = tsnep_ethtool_get_rxnfc, .set_rxnfc = tsnep_ethtool_set_rxnfc, + .get_channels = tsnep_ethtool_get_channels, .get_ts_info = tsnep_ethtool_get_ts_info, + .get_coalesce = tsnep_ethtool_get_coalesce, + .set_coalesce = tsnep_ethtool_set_coalesce, + .get_per_queue_coalesce = tsnep_ethtool_get_per_queue_coalesce, + .set_per_queue_coalesce = tsnep_ethtool_set_per_queue_coalesce, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, }; diff --git a/drivers/net/ethernet/engleder/tsnep_hw.h b/drivers/net/ethernet/engleder/tsnep_hw.h index 315dada75323..55e1caf193a6 100644 --- a/drivers/net/ethernet/engleder/tsnep_hw.h +++ b/drivers/net/ethernet/engleder/tsnep_hw.h @@ -48,6 +48,13 @@ #define ECM_COUNTER_LOW 0x0028 #define ECM_COUNTER_HIGH 0x002C +/* interrupt delay */ +#define ECM_INT_DELAY 0x0030 +#define ECM_INT_DELAY_MASK 0xF0 +#define ECM_INT_DELAY_SHIFT 4 +#define ECM_INT_DELAY_BASE_US 16 +#define ECM_INT_DELAY_OFFSET 1 + /* control and status */ #define ECM_STATUS 0x0080 #define ECM_LINK_MODE_OFF 0x01000000 diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 13d5ff4e0e02..bf0190e1d2ea 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -39,6 +39,10 @@ #endif #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) +#define TSNEP_COALESCE_USECS_DEFAULT 64 +#define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ + ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) + static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) { iowrite32(mask, adapter->addr + ECM_INT_ENABLE); @@ -83,6 +87,33 @@ static irqreturn_t tsnep_irq_txrx(int irq, void *arg) return IRQ_HANDLED; } +int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) +{ + if (usecs > TSNEP_COALESCE_USECS_MAX) + return -ERANGE; + + usecs /= ECM_INT_DELAY_BASE_US; + usecs <<= ECM_INT_DELAY_SHIFT; + usecs &= ECM_INT_DELAY_MASK; + + queue->irq_delay &= ~ECM_INT_DELAY_MASK; + queue->irq_delay |= usecs; + iowrite8(queue->irq_delay, queue->irq_delay_addr); + + return 0; +} + +u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) +{ + u32 usecs; + + usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); + usecs >>= ECM_INT_DELAY_SHIFT; + usecs *= ECM_INT_DELAY_BASE_US; + + return usecs; +} + static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) { struct tsnep_adapter *adapter = bus->priv; @@ -629,23 +660,6 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) } } -static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, - struct tsnep_rx_entry *entry) -{ - struct page *page; - - page = page_pool_dev_alloc_pages(rx->page_pool); - if (unlikely(!page)) - return -ENOMEM; - - entry->page = page; - entry->len = TSNEP_MAX_RX_BUF_SIZE; - entry->dma = page_pool_get_dma_addr(entry->page); - entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD); - - return 0; -} - static int tsnep_rx_ring_init(struct tsnep_rx *rx) { struct device *dmadev = rx->adapter->dmadev; @@ -692,10 +706,6 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx) entry = &rx->entry[i]; next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE]; entry->desc->next = __cpu_to_le64(next_entry->desc_dma); - - retval = tsnep_rx_alloc_buffer(rx, entry); - if (retval) - goto failed; } return 0; @@ -705,6 +715,45 @@ failed: return retval; } +static int tsnep_rx_desc_available(struct tsnep_rx *rx) +{ + if (rx->read <= rx->write) + return TSNEP_RING_SIZE - rx->write + rx->read - 1; + else + return rx->read - rx->write - 1; +} + +static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, + struct page *page) +{ + entry->page = page; + entry->len = TSNEP_MAX_RX_BUF_SIZE; + entry->dma = page_pool_get_dma_addr(entry->page); + entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD); +} + +static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) +{ + struct tsnep_rx_entry *entry = &rx->entry[index]; + struct page *page; + + page = page_pool_dev_alloc_pages(rx->page_pool); + if (unlikely(!page)) + return -ENOMEM; + tsnep_rx_set_page(rx, entry, page); + + return 0; +} + +static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) +{ + struct tsnep_rx_entry *entry = &rx->entry[index]; + struct tsnep_rx_entry *read = &rx->entry[rx->read]; + + tsnep_rx_set_page(rx, entry, read->page); + read->page = NULL; +} + static void tsnep_rx_activate(struct tsnep_rx *rx, int index) { struct tsnep_rx_entry *entry = &rx->entry[index]; @@ -732,6 +781,48 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index) entry->desc->properties = __cpu_to_le32(entry->properties); } +static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) +{ + int index; + bool alloc_failed = false; + bool enable = false; + int i; + int retval; + + for (i = 0; i < count && !alloc_failed; i++) { + index = (rx->write + i) % TSNEP_RING_SIZE; + + retval = tsnep_rx_alloc_buffer(rx, index); + if (unlikely(retval)) { + rx->alloc_failed++; + alloc_failed = true; + + /* reuse only if no other allocation was successful */ + if (i == 0 && reuse) + tsnep_rx_reuse_buffer(rx, index); + else + break; + } + + tsnep_rx_activate(rx, index); + + enable = true; + } + + if (enable) { + rx->write = (rx->write + i) % TSNEP_RING_SIZE; + + /* descriptor properties shall be valid before hardware is + * notified + */ + dma_wmb(); + + iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); + } + + return i; +} + static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, int length) { @@ -767,23 +858,42 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, int budget) { struct device *dmadev = rx->adapter->dmadev; + int desc_available; int done = 0; enum dma_data_direction dma_dir; struct tsnep_rx_entry *entry; - struct page *page; struct sk_buff *skb; int length; - bool enable = false; - int retval; + desc_available = tsnep_rx_desc_available(rx); dma_dir = page_pool_get_dma_dir(rx->page_pool); - while (likely(done < budget)) { + while (likely(done < budget) && (rx->read != rx->write)) { entry = &rx->entry[rx->read]; if ((__le32_to_cpu(entry->desc_wb->properties) & TSNEP_DESC_OWNER_COUNTER_MASK) != (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) break; + done++; + + if (desc_available >= TSNEP_RING_RX_REFILL) { + bool reuse = desc_available >= TSNEP_RING_RX_REUSE; + + desc_available -= tsnep_rx_refill(rx, desc_available, + reuse); + if (!entry->page) { + /* buffer has been reused for refill to prevent + * empty RX ring, thus buffer cannot be used for + * RX processing + */ + rx->read = (rx->read + 1) % TSNEP_RING_SIZE; + desc_available++; + + rx->dropped++; + + continue; + } + } /* descriptor properties shall be read first, because valid data * is signaled there @@ -795,49 +905,30 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, TSNEP_DESC_LENGTH_MASK; dma_sync_single_range_for_cpu(dmadev, entry->dma, TSNEP_SKB_PAD, length, dma_dir); - page = entry->page; - /* forward skb only if allocation is successful, otherwise - * page is reused and frame dropped - */ - retval = tsnep_rx_alloc_buffer(rx, entry); - if (!retval) { - skb = tsnep_build_skb(rx, page, length); - if (skb) { - page_pool_release_page(rx->page_pool, page); - - rx->packets++; - rx->bytes += length - - TSNEP_RX_INLINE_METADATA_SIZE; - if (skb->pkt_type == PACKET_MULTICAST) - rx->multicast++; - - napi_gro_receive(napi, skb); - } else { - page_pool_recycle_direct(rx->page_pool, page); + rx->read = (rx->read + 1) % TSNEP_RING_SIZE; + desc_available++; - rx->dropped++; - } - done++; - } else { - rx->dropped++; - } + skb = tsnep_build_skb(rx, entry->page, length); + if (skb) { + page_pool_release_page(rx->page_pool, entry->page); - tsnep_rx_activate(rx, rx->read); + rx->packets++; + rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE; + if (skb->pkt_type == PACKET_MULTICAST) + rx->multicast++; - enable = true; + napi_gro_receive(napi, skb); + } else { + page_pool_recycle_direct(rx->page_pool, entry->page); - rx->read = (rx->read + 1) % TSNEP_RING_SIZE; + rx->dropped++; + } + entry->page = NULL; } - if (enable) { - /* descriptor properties shall be valid before hardware is - * notified - */ - dma_wmb(); - - iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); - } + if (desc_available) + tsnep_rx_refill(rx, desc_available, false); return done; } @@ -846,11 +937,13 @@ static bool tsnep_rx_pending(struct tsnep_rx *rx) { struct tsnep_rx_entry *entry; - entry = &rx->entry[rx->read]; - if ((__le32_to_cpu(entry->desc_wb->properties) & - TSNEP_DESC_OWNER_COUNTER_MASK) == - (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) - return true; + if (rx->read != rx->write) { + entry = &rx->entry[rx->read]; + if ((__le32_to_cpu(entry->desc_wb->properties) & + TSNEP_DESC_OWNER_COUNTER_MASK) == + (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) + return true; + } return false; } @@ -859,7 +952,6 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, int queue_index, struct tsnep_rx *rx) { dma_addr_t dma; - int i; int retval; memset(rx, 0, sizeof(*rx)); @@ -877,13 +969,7 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, rx->owner_counter = 1; rx->increment_owner_counter = TSNEP_RING_SIZE - 1; - for (i = 0; i < TSNEP_RING_SIZE; i++) - tsnep_rx_activate(rx, i); - - /* descriptor properties shall be valid before hardware is notified */ - dma_wmb(); - - iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); + tsnep_rx_refill(rx, tsnep_rx_desc_available(rx), false); return 0; } @@ -1371,6 +1457,11 @@ static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) adapter->queue[0].tx = &adapter->tx[0]; adapter->queue[0].rx = &adapter->rx[0]; adapter->queue[0].irq_mask = irq_mask; + adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; + retval = tsnep_set_irq_coalesce(&adapter->queue[0], + TSNEP_COALESCE_USECS_DEFAULT); + if (retval < 0) + return retval; adapter->netdev->irq = adapter->queue[0].irq; @@ -1391,6 +1482,12 @@ static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) adapter->queue[i].rx = &adapter->rx[i]; adapter->queue[i].irq_mask = irq_mask << (ECM_INT_TXRX_SHIFT * i); + adapter->queue[i].irq_delay_addr = + adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; + retval = tsnep_set_irq_coalesce(&adapter->queue[i], + TSNEP_COALESCE_USECS_DEFAULT); + if (retval < 0) + return retval; } return 0; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-devlink.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-devlink.c index 5c6dd3029e2f..76f808d38066 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-devlink.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-devlink.c @@ -37,18 +37,9 @@ static int dpaa2_eth_dl_info_get(struct devlink *devlink, struct dpaa2_eth_devlink_priv *dl_priv = devlink_priv(devlink); struct dpaa2_eth_priv *priv = dl_priv->dpaa2_priv; char buf[10]; - int err; - - err = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (err) - return err; scnprintf(buf, 10, "%d.%d", priv->dpni_ver_major, priv->dpni_ver_minor); - err = devlink_info_version_running_put(req, "dpni", buf); - if (err) - return err; - - return 0; + return devlink_info_version_running_put(req, "dpni", buf); } static struct dpaa2_eth_trap_item * diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 97e1856641b4..0c35abb7d065 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -2147,8 +2147,11 @@ static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) /* When we manage the MAC/PHY using phylink there is no need * to manually update the netif_carrier. + * We can avoid locking because we are called from the "link changed" + * IRQ handler, which is the same as the "endpoint changed" IRQ handler + * (the writer to priv->mac), so we cannot race with it. */ - if (dpaa2_eth_is_type_phy(priv)) + if (dpaa2_mac_is_type_phy(priv->mac)) goto out; /* Chech link state; speed / duplex changes are not treated yet */ @@ -2179,6 +2182,8 @@ static int dpaa2_eth_open(struct net_device *net_dev) dpaa2_eth_seed_pools(priv); + mutex_lock(&priv->mac_lock); + if (!dpaa2_eth_is_type_phy(priv)) { /* We'll only start the txqs when the link is actually ready; * make sure we don't race against the link up notification, @@ -2197,14 +2202,15 @@ static int dpaa2_eth_open(struct net_device *net_dev) err = dpni_enable(priv->mc_io, 0, priv->mc_token); if (err < 0) { + mutex_unlock(&priv->mac_lock); netdev_err(net_dev, "dpni_enable() failed\n"); goto enable_err; } - if (dpaa2_eth_is_type_phy(priv)) { + if (dpaa2_eth_is_type_phy(priv)) dpaa2_mac_start(priv->mac); - phylink_start(priv->mac->phylink); - } + + mutex_unlock(&priv->mac_lock); return 0; @@ -2277,14 +2283,17 @@ static int dpaa2_eth_stop(struct net_device *net_dev) int dpni_enabled = 0; int retries = 10; + mutex_lock(&priv->mac_lock); + if (dpaa2_eth_is_type_phy(priv)) { - phylink_stop(priv->mac->phylink); dpaa2_mac_stop(priv->mac); } else { netif_tx_stop_all_queues(net_dev); netif_carrier_off(net_dev); } + mutex_unlock(&priv->mac_lock); + /* On dpni_disable(), the MC firmware will: * - stop MAC Rx and wait for all Rx frames to be enqueued to software * - cut off WRIOP dequeues from egress FQs and wait until transmission @@ -2610,12 +2619,20 @@ static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) { struct dpaa2_eth_priv *priv = netdev_priv(dev); + int err; if (cmd == SIOCSHWTSTAMP) return dpaa2_eth_ts_ioctl(dev, rq, cmd); - if (dpaa2_eth_is_type_phy(priv)) - return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_is_type_phy(priv)) { + err = phylink_mii_ioctl(priv->mac->phylink, rq, cmd); + mutex_unlock(&priv->mac_lock); + return err; + } + + mutex_unlock(&priv->mac_lock); return -EOPNOTSUPP; } @@ -3791,7 +3808,7 @@ static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", priv->dpni_ver_major, priv->dpni_ver_minor, DPNI_VER_MAJOR, DPNI_VER_MINOR); - err = -ENOTSUPP; + err = -EOPNOTSUPP; goto close; } @@ -4627,9 +4644,8 @@ static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) err = dpaa2_mac_open(mac); if (err) goto err_free_mac; - priv->mac = mac; - if (dpaa2_eth_is_type_phy(priv)) { + if (dpaa2_mac_is_type_phy(mac)) { err = dpaa2_mac_connect(mac); if (err) { if (err == -EPROBE_DEFER) @@ -4643,11 +4659,14 @@ static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) } } + mutex_lock(&priv->mac_lock); + priv->mac = mac; + mutex_unlock(&priv->mac_lock); + return 0; err_close_mac: dpaa2_mac_close(mac); - priv->mac = NULL; err_free_mac: kfree(mac); return err; @@ -4655,15 +4674,21 @@ err_free_mac: static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) { - if (dpaa2_eth_is_type_phy(priv)) - dpaa2_mac_disconnect(priv->mac); + struct dpaa2_mac *mac; - if (!dpaa2_eth_has_mac(priv)) + mutex_lock(&priv->mac_lock); + mac = priv->mac; + priv->mac = NULL; + mutex_unlock(&priv->mac_lock); + + if (!mac) return; - dpaa2_mac_close(priv->mac); - kfree(priv->mac); - priv->mac = NULL; + if (dpaa2_mac_is_type_phy(mac)) + dpaa2_mac_disconnect(mac); + + dpaa2_mac_close(mac); + kfree(mac); } static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) @@ -4673,6 +4698,7 @@ static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); struct net_device *net_dev = dev_get_drvdata(dev); struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + bool had_mac; int err; err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, @@ -4689,12 +4715,15 @@ static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); dpaa2_eth_update_tx_fqids(priv); - rtnl_lock(); - if (dpaa2_eth_has_mac(priv)) + /* We can avoid locking because the "endpoint changed" IRQ + * handler is the only one who changes priv->mac at runtime, + * so we are not racing with anyone. + */ + had_mac = !!priv->mac; + if (had_mac) dpaa2_eth_disconnect_mac(priv); else dpaa2_eth_connect_mac(priv); - rtnl_unlock(); } return IRQ_HANDLED; @@ -4792,6 +4821,8 @@ static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) priv->net_dev = net_dev; SET_NETDEV_DEVLINK_PORT(net_dev, &priv->devlink_port); + mutex_init(&priv->mac_lock); + priv->iommu_domain = iommu_get_domain_for_dev(dev); priv->tx_tstamp_type = HWTSTAMP_TX_OFF; @@ -4899,6 +4930,10 @@ static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) } #endif + err = dpaa2_eth_connect_mac(priv); + if (err) + goto err_connect_mac; + err = dpaa2_eth_setup_irqs(dpni_dev); if (err) { netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); @@ -4911,10 +4946,6 @@ static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) priv->do_link_poll = true; } - err = dpaa2_eth_connect_mac(priv); - if (err) - goto err_connect_mac; - err = dpaa2_eth_dl_alloc(priv); if (err) goto err_dl_register; @@ -4948,13 +4979,13 @@ err_dl_port_add: err_dl_trap_register: dpaa2_eth_dl_free(priv); err_dl_register: - dpaa2_eth_disconnect_mac(priv); -err_connect_mac: if (priv->do_link_poll) kthread_stop(priv->poll_thread); else fsl_mc_free_irqs(dpni_dev); err_poll_thread: + dpaa2_eth_disconnect_mac(priv); +err_connect_mac: dpaa2_eth_free_rings(priv); err_alloc_rings: err_csum: @@ -5002,9 +5033,6 @@ static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) #endif unregister_netdev(net_dev); - rtnl_lock(); - dpaa2_eth_disconnect_mac(priv); - rtnl_unlock(); dpaa2_eth_dl_port_del(priv); dpaa2_eth_dl_traps_unregister(priv); @@ -5015,6 +5043,7 @@ static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) else fsl_mc_free_irqs(ls_dev); + dpaa2_eth_disconnect_mac(priv); dpaa2_eth_free_rings(priv); free_percpu(priv->fd); free_percpu(priv->sgt_cache); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h index 5d0fc432e5b2..d56d7a13262e 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h @@ -615,6 +615,8 @@ struct dpaa2_eth_priv { #endif struct dpaa2_mac *mac; + /* Serializes changes to priv->mac */ + struct mutex mac_lock; struct workqueue_struct *dpaa2_ptp_wq; struct work_struct tx_onestep_tstamp; struct sk_buff_head tx_skbs; @@ -768,16 +770,15 @@ static inline unsigned int dpaa2_eth_rx_head_room(struct dpaa2_eth_priv *priv) static inline bool dpaa2_eth_is_type_phy(struct dpaa2_eth_priv *priv) { - if (priv->mac && - (priv->mac->attr.link_type == DPMAC_LINK_TYPE_PHY || - priv->mac->attr.link_type == DPMAC_LINK_TYPE_BACKPLANE)) - return true; + lockdep_assert_held(&priv->mac_lock); - return false; + return dpaa2_mac_is_type_phy(priv->mac); } static inline bool dpaa2_eth_has_mac(struct dpaa2_eth_priv *priv) { + lockdep_assert_held(&priv->mac_lock); + return priv->mac ? true : false; } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c index 32a38a03db57..e80e9388c71f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c @@ -85,11 +85,16 @@ static void dpaa2_eth_get_drvinfo(struct net_device *net_dev, static int dpaa2_eth_nway_reset(struct net_device *net_dev) { struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + int err = -EOPNOTSUPP; + + mutex_lock(&priv->mac_lock); if (dpaa2_eth_is_type_phy(priv)) - return phylink_ethtool_nway_reset(priv->mac->phylink); + err = phylink_ethtool_nway_reset(priv->mac->phylink); + + mutex_unlock(&priv->mac_lock); - return -EOPNOTSUPP; + return err; } static int @@ -97,10 +102,18 @@ dpaa2_eth_get_link_ksettings(struct net_device *net_dev, struct ethtool_link_ksettings *link_settings) { struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + int err; - if (dpaa2_eth_is_type_phy(priv)) - return phylink_ethtool_ksettings_get(priv->mac->phylink, - link_settings); + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_is_type_phy(priv)) { + err = phylink_ethtool_ksettings_get(priv->mac->phylink, + link_settings); + mutex_unlock(&priv->mac_lock); + return err; + } + + mutex_unlock(&priv->mac_lock); link_settings->base.autoneg = AUTONEG_DISABLE; if (!(priv->link_state.options & DPNI_LINK_OPT_HALF_DUPLEX)) @@ -115,11 +128,17 @@ dpaa2_eth_set_link_ksettings(struct net_device *net_dev, const struct ethtool_link_ksettings *link_settings) { struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + int err = -EOPNOTSUPP; - if (!dpaa2_eth_is_type_phy(priv)) - return -ENOTSUPP; + mutex_lock(&priv->mac_lock); - return phylink_ethtool_ksettings_set(priv->mac->phylink, link_settings); + if (dpaa2_eth_is_type_phy(priv)) + err = phylink_ethtool_ksettings_set(priv->mac->phylink, + link_settings); + + mutex_unlock(&priv->mac_lock); + + return err; } static void dpaa2_eth_get_pauseparam(struct net_device *net_dev, @@ -128,11 +147,16 @@ static void dpaa2_eth_get_pauseparam(struct net_device *net_dev, struct dpaa2_eth_priv *priv = netdev_priv(net_dev); u64 link_options = priv->link_state.options; + mutex_lock(&priv->mac_lock); + if (dpaa2_eth_is_type_phy(priv)) { phylink_ethtool_get_pauseparam(priv->mac->phylink, pause); + mutex_unlock(&priv->mac_lock); return; } + mutex_unlock(&priv->mac_lock); + pause->rx_pause = dpaa2_eth_rx_pause_enabled(link_options); pause->tx_pause = dpaa2_eth_tx_pause_enabled(link_options); pause->autoneg = AUTONEG_DISABLE; @@ -151,9 +175,17 @@ static int dpaa2_eth_set_pauseparam(struct net_device *net_dev, return -EOPNOTSUPP; } - if (dpaa2_eth_is_type_phy(priv)) - return phylink_ethtool_set_pauseparam(priv->mac->phylink, - pause); + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_is_type_phy(priv)) { + err = phylink_ethtool_set_pauseparam(priv->mac->phylink, + pause); + mutex_unlock(&priv->mac_lock); + return err; + } + + mutex_unlock(&priv->mac_lock); + if (pause->autoneg) return -EOPNOTSUPP; @@ -185,7 +217,6 @@ static int dpaa2_eth_set_pauseparam(struct net_device *net_dev, static void dpaa2_eth_get_strings(struct net_device *netdev, u32 stringset, u8 *data) { - struct dpaa2_eth_priv *priv = netdev_priv(netdev); u8 *p = data; int i; @@ -199,22 +230,17 @@ static void dpaa2_eth_get_strings(struct net_device *netdev, u32 stringset, strscpy(p, dpaa2_ethtool_extras[i], ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } - if (dpaa2_eth_has_mac(priv)) - dpaa2_mac_get_strings(p); + dpaa2_mac_get_strings(p); break; } } static int dpaa2_eth_get_sset_count(struct net_device *net_dev, int sset) { - int num_ss_stats = DPAA2_ETH_NUM_STATS + DPAA2_ETH_NUM_EXTRA_STATS; - struct dpaa2_eth_priv *priv = netdev_priv(net_dev); - switch (sset) { case ETH_SS_STATS: /* ethtool_get_stats(), ethtool_get_drvinfo() */ - if (dpaa2_eth_has_mac(priv)) - num_ss_stats += dpaa2_mac_get_sset_count(); - return num_ss_stats; + return DPAA2_ETH_NUM_STATS + DPAA2_ETH_NUM_EXTRA_STATS + + dpaa2_mac_get_sset_count(); default: return -EOPNOTSUPP; } @@ -315,8 +341,12 @@ static void dpaa2_eth_get_ethtool_stats(struct net_device *net_dev, } *(data + i++) = buf_cnt_total; + mutex_lock(&priv->mac_lock); + if (dpaa2_eth_has_mac(priv)) dpaa2_mac_get_ethtool_stats(priv->mac, data + i); + + mutex_unlock(&priv->mac_lock); } static int dpaa2_eth_prep_eth_rule(struct ethhdr *eth_value, struct ethhdr *eth_mask, diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index 51c9da8e1be2..c886f33f8c6f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -338,12 +338,20 @@ static void dpaa2_mac_set_supported_interfaces(struct dpaa2_mac *mac) void dpaa2_mac_start(struct dpaa2_mac *mac) { + ASSERT_RTNL(); + if (mac->serdes_phy) phy_power_on(mac->serdes_phy); + + phylink_start(mac->phylink); } void dpaa2_mac_stop(struct dpaa2_mac *mac) { + ASSERT_RTNL(); + + phylink_stop(mac->phylink); + if (mac->serdes_phy) phy_power_off(mac->serdes_phy); } @@ -422,7 +430,9 @@ int dpaa2_mac_connect(struct dpaa2_mac *mac) } mac->phylink = phylink; + rtnl_lock(); err = phylink_fwnode_phy_connect(mac->phylink, dpmac_node, 0); + rtnl_unlock(); if (err) { netdev_err(net_dev, "phylink_fwnode_phy_connect() = %d\n", err); goto err_phylink_destroy; @@ -440,10 +450,10 @@ err_pcs_destroy: void dpaa2_mac_disconnect(struct dpaa2_mac *mac) { - if (!mac->phylink) - return; - + rtnl_lock(); phylink_disconnect_phy(mac->phylink); + rtnl_unlock(); + phylink_destroy(mac->phylink); dpaa2_pcs_destroy(mac); of_phy_put(mac->serdes_phy); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h index a58cab188a99..c1ec9efd413a 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h @@ -30,8 +30,14 @@ struct dpaa2_mac { struct phy *serdes_phy; }; -bool dpaa2_mac_is_type_fixed(struct fsl_mc_device *dpmac_dev, - struct fsl_mc_io *mc_io); +static inline bool dpaa2_mac_is_type_phy(struct dpaa2_mac *mac) +{ + if (!mac) + return false; + + return mac->attr.link_type == DPMAC_LINK_TYPE_PHY || + mac->attr.link_type == DPMAC_LINK_TYPE_BACKPLANE; +} int dpaa2_mac_open(struct dpaa2_mac *mac); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c index 720c9230cab5..6bc1988be311 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c @@ -60,11 +60,18 @@ dpaa2_switch_get_link_ksettings(struct net_device *netdev, { struct ethsw_port_priv *port_priv = netdev_priv(netdev); struct dpsw_link_state state = {0}; - int err = 0; + int err; + + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_is_type_phy(port_priv)) { + err = phylink_ethtool_ksettings_get(port_priv->mac->phylink, + link_ksettings); + mutex_unlock(&port_priv->mac_lock); + return err; + } - if (dpaa2_switch_port_is_type_phy(port_priv)) - return phylink_ethtool_ksettings_get(port_priv->mac->phylink, - link_ksettings); + mutex_unlock(&port_priv->mac_lock); err = dpsw_if_get_link_state(port_priv->ethsw_data->mc_io, 0, port_priv->ethsw_data->dpsw_handle, @@ -99,9 +106,16 @@ dpaa2_switch_set_link_ksettings(struct net_device *netdev, bool if_running; int err = 0, ret; - if (dpaa2_switch_port_is_type_phy(port_priv)) - return phylink_ethtool_ksettings_set(port_priv->mac->phylink, - link_ksettings); + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_is_type_phy(port_priv)) { + err = phylink_ethtool_ksettings_set(port_priv->mac->phylink, + link_ksettings); + mutex_unlock(&port_priv->mac_lock); + return err; + } + + mutex_unlock(&port_priv->mac_lock); /* Interface needs to be down to change link settings */ if_running = netif_running(netdev); @@ -145,14 +159,9 @@ dpaa2_switch_set_link_ksettings(struct net_device *netdev, static int dpaa2_switch_ethtool_get_sset_count(struct net_device *netdev, int sset) { - struct ethsw_port_priv *port_priv = netdev_priv(netdev); - int num_ss_stats = DPAA2_SWITCH_NUM_COUNTERS; - switch (sset) { case ETH_SS_STATS: - if (port_priv->mac) - num_ss_stats += dpaa2_mac_get_sset_count(); - return num_ss_stats; + return DPAA2_SWITCH_NUM_COUNTERS + dpaa2_mac_get_sset_count(); default: return -EOPNOTSUPP; } @@ -161,7 +170,6 @@ dpaa2_switch_ethtool_get_sset_count(struct net_device *netdev, int sset) static void dpaa2_switch_ethtool_get_strings(struct net_device *netdev, u32 stringset, u8 *data) { - struct ethsw_port_priv *port_priv = netdev_priv(netdev); u8 *p = data; int i; @@ -172,8 +180,7 @@ static void dpaa2_switch_ethtool_get_strings(struct net_device *netdev, ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } - if (port_priv->mac) - dpaa2_mac_get_strings(p); + dpaa2_mac_get_strings(p); break; } } @@ -196,8 +203,12 @@ static void dpaa2_switch_ethtool_get_stats(struct net_device *netdev, dpaa2_switch_ethtool_counters[i].name, err); } - if (port_priv->mac) + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_has_mac(port_priv)) dpaa2_mac_get_ethtool_stats(port_priv->mac, data + i); + + mutex_unlock(&port_priv->mac_lock); } const struct ethtool_ops dpaa2_switch_port_ethtool_ops = { diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c index cacd454ac696..c39b866e2582 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c @@ -132,6 +132,7 @@ int dpaa2_switch_acl_entry_add(struct dpaa2_switch_filter_block *filter_block, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, acl_entry_cfg->key_iova))) { dev_err(dev, "DMA mapping failed\n"); + kfree(cmd_buff); return -EFAULT; } @@ -142,6 +143,7 @@ int dpaa2_switch_acl_entry_add(struct dpaa2_switch_filter_block *filter_block, DMA_TO_DEVICE); if (err) { dev_err(dev, "dpsw_acl_add_entry() failed %d\n", err); + kfree(cmd_buff); return err; } @@ -172,6 +174,7 @@ dpaa2_switch_acl_entry_remove(struct dpaa2_switch_filter_block *block, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, acl_entry_cfg->key_iova))) { dev_err(dev, "DMA mapping failed\n"); + kfree(cmd_buff); return -EFAULT; } @@ -182,6 +185,7 @@ dpaa2_switch_acl_entry_remove(struct dpaa2_switch_filter_block *block, DMA_TO_DEVICE); if (err) { dev_err(dev, "dpsw_acl_remove_entry() failed %d\n", err); + kfree(cmd_buff); return err; } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 14f739e04a3c..f4ae4289c41a 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -602,8 +602,11 @@ static int dpaa2_switch_port_link_state_update(struct net_device *netdev) /* When we manage the MAC/PHY using phylink there is no need * to manually update the netif_carrier. + * We can avoid locking because we are called from the "link changed" + * IRQ handler, which is the same as the "endpoint changed" IRQ handler + * (the writer to port_priv->mac), so we cannot race with it. */ - if (dpaa2_switch_port_is_type_phy(port_priv)) + if (dpaa2_mac_is_type_phy(port_priv->mac)) return 0; /* Interrupts are received even though no one issued an 'ifconfig up' @@ -683,6 +686,8 @@ static int dpaa2_switch_port_open(struct net_device *netdev) struct ethsw_core *ethsw = port_priv->ethsw_data; int err; + mutex_lock(&port_priv->mac_lock); + if (!dpaa2_switch_port_is_type_phy(port_priv)) { /* Explicitly set carrier off, otherwise * netif_carrier_ok() will return true and cause 'ip link show' @@ -696,16 +701,17 @@ static int dpaa2_switch_port_open(struct net_device *netdev) port_priv->ethsw_data->dpsw_handle, port_priv->idx); if (err) { + mutex_unlock(&port_priv->mac_lock); netdev_err(netdev, "dpsw_if_enable err %d\n", err); return err; } dpaa2_switch_enable_ctrl_if_napi(ethsw); - if (dpaa2_switch_port_is_type_phy(port_priv)) { + if (dpaa2_switch_port_is_type_phy(port_priv)) dpaa2_mac_start(port_priv->mac); - phylink_start(port_priv->mac->phylink); - } + + mutex_unlock(&port_priv->mac_lock); return 0; } @@ -716,14 +722,17 @@ static int dpaa2_switch_port_stop(struct net_device *netdev) struct ethsw_core *ethsw = port_priv->ethsw_data; int err; + mutex_lock(&port_priv->mac_lock); + if (dpaa2_switch_port_is_type_phy(port_priv)) { - phylink_stop(port_priv->mac->phylink); dpaa2_mac_stop(port_priv->mac); } else { netif_tx_stop_all_queues(netdev); netif_carrier_off(netdev); } + mutex_unlock(&port_priv->mac_lock); + err = dpsw_if_disable(port_priv->ethsw_data->mc_io, 0, port_priv->ethsw_data->dpsw_handle, port_priv->idx); @@ -1452,9 +1461,8 @@ static int dpaa2_switch_port_connect_mac(struct ethsw_port_priv *port_priv) err = dpaa2_mac_open(mac); if (err) goto err_free_mac; - port_priv->mac = mac; - if (dpaa2_switch_port_is_type_phy(port_priv)) { + if (dpaa2_mac_is_type_phy(mac)) { err = dpaa2_mac_connect(mac); if (err) { netdev_err(port_priv->netdev, @@ -1464,11 +1472,14 @@ static int dpaa2_switch_port_connect_mac(struct ethsw_port_priv *port_priv) } } + mutex_lock(&port_priv->mac_lock); + port_priv->mac = mac; + mutex_unlock(&port_priv->mac_lock); + return 0; err_close_mac: dpaa2_mac_close(mac); - port_priv->mac = NULL; err_free_mac: kfree(mac); return err; @@ -1476,15 +1487,21 @@ err_free_mac: static void dpaa2_switch_port_disconnect_mac(struct ethsw_port_priv *port_priv) { - if (dpaa2_switch_port_is_type_phy(port_priv)) - dpaa2_mac_disconnect(port_priv->mac); + struct dpaa2_mac *mac; + + mutex_lock(&port_priv->mac_lock); + mac = port_priv->mac; + port_priv->mac = NULL; + mutex_unlock(&port_priv->mac_lock); - if (!dpaa2_switch_port_has_mac(port_priv)) + if (!mac) return; - dpaa2_mac_close(port_priv->mac); - kfree(port_priv->mac); - port_priv->mac = NULL; + if (dpaa2_mac_is_type_phy(mac)) + dpaa2_mac_disconnect(mac); + + dpaa2_mac_close(mac); + kfree(mac); } static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) @@ -1494,6 +1511,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) struct ethsw_port_priv *port_priv; u32 status = ~0; int err, if_id; + bool had_mac; err = dpsw_get_irq_status(ethsw->mc_io, 0, ethsw->dpsw_handle, DPSW_IRQ_INDEX_IF, &status); @@ -1511,12 +1529,15 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) } if (status & DPSW_IRQ_EVENT_ENDPOINT_CHANGED) { - rtnl_lock(); - if (dpaa2_switch_port_has_mac(port_priv)) + /* We can avoid locking because the "endpoint changed" IRQ + * handler is the only one who changes priv->mac at runtime, + * so we are not racing with anyone. + */ + had_mac = !!port_priv->mac; + if (had_mac) dpaa2_switch_port_disconnect_mac(port_priv); else dpaa2_switch_port_connect_mac(port_priv); - rtnl_unlock(); } out: @@ -2934,9 +2955,7 @@ static void dpaa2_switch_remove_port(struct ethsw_core *ethsw, { struct ethsw_port_priv *port_priv = ethsw->ports[port_idx]; - rtnl_lock(); dpaa2_switch_port_disconnect_mac(port_priv); - rtnl_unlock(); free_netdev(port_priv->netdev); ethsw->ports[port_idx] = NULL; } @@ -3255,6 +3274,8 @@ static int dpaa2_switch_probe_port(struct ethsw_core *ethsw, port_priv->netdev = port_netdev; port_priv->ethsw_data = ethsw; + mutex_init(&port_priv->mac_lock); + port_priv->idx = port_idx; port_priv->stp_state = BR_STATE_FORWARDING; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h index 0002dca4d417..42b3ca73f55d 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h @@ -161,6 +161,8 @@ struct ethsw_port_priv { struct dpaa2_switch_filter_block *filter_block; struct dpaa2_mac *mac; + /* Protects against changes to port_priv->mac */ + struct mutex mac_lock; }; /* Switch data */ @@ -230,12 +232,7 @@ static inline bool dpaa2_switch_supports_cpu_traffic(struct ethsw_core *ethsw) static inline bool dpaa2_switch_port_is_type_phy(struct ethsw_port_priv *port_priv) { - if (port_priv->mac && - (port_priv->mac->attr.link_type == DPMAC_LINK_TYPE_PHY || - port_priv->mac->attr.link_type == DPMAC_LINK_TYPE_BACKPLANE)) - return true; - - return false; + return dpaa2_mac_is_type_phy(port_priv->mac); } static inline bool dpaa2_switch_port_has_mac(struct ethsw_port_priv *port_priv) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 9471baa11d39..5528b0af82ae 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -1216,7 +1216,8 @@ fec_restart(struct net_device *ndev) writel(0, fep->hwp + FEC_IMASK); /* Init the interrupt coalescing */ - fec_enet_itr_coal_set(ndev); + if (fep->quirks & FEC_QUIRK_HAS_COALESCE) + fec_enet_itr_coal_set(ndev); } static int fec_enet_ipc_handle_init(struct fec_enet_private *fep) diff --git a/drivers/net/ethernet/fungible/funeth/funeth_devlink.c b/drivers/net/ethernet/fungible/funeth/funeth_devlink.c index d50c222948b4..4fbeb3fd71a8 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_devlink.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_devlink.c @@ -3,14 +3,7 @@ #include "funeth.h" #include "funeth_devlink.h" -static int fun_dl_info_get(struct devlink *dl, struct devlink_info_req *req, - struct netlink_ext_ack *extack) -{ - return devlink_info_driver_name_put(req, KBUILD_MODNAME); -} - static const struct devlink_ops fun_dl_ops = { - .info_get = fun_dl_info_get, }; struct devlink *fun_devlink_alloc(struct device *dev) diff --git a/drivers/net/ethernet/hisilicon/hisi_femac.c b/drivers/net/ethernet/hisilicon/hisi_femac.c index 93846bace028..ce2571c16e43 100644 --- a/drivers/net/ethernet/hisilicon/hisi_femac.c +++ b/drivers/net/ethernet/hisilicon/hisi_femac.c @@ -283,7 +283,7 @@ static int hisi_femac_rx(struct net_device *dev, int limit) skb->protocol = eth_type_trans(skb, dev); napi_gro_receive(&priv->napi, skb); dev->stats.rx_packets++; - dev->stats.rx_bytes += skb->len; + dev->stats.rx_bytes += len; next: pos = (pos + 1) % rxq->num; if (rx_pkts_num >= limit) diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c index ffcf797dfa90..f867e9531117 100644 --- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c +++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c @@ -550,7 +550,7 @@ static int hix5hd2_rx(struct net_device *dev, int limit) skb->protocol = eth_type_trans(skb, dev); napi_gro_receive(&priv->napi, skb); dev->stats.rx_packets++; - dev->stats.rx_bytes += skb->len; + dev->stats.rx_bytes += len; next: pos = dma_ring_incr(pos, RX_DESC_NUM); } diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c index 54faf0f2d1d8..b54f3706fb97 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c @@ -644,18 +644,15 @@ static void hns_nic_get_drvinfo(struct net_device *net_dev, { struct hns_nic_priv *priv = netdev_priv(net_dev); - strncpy(drvinfo->version, HNAE_DRIVER_VERSION, + strscpy(drvinfo->version, HNAE_DRIVER_VERSION, sizeof(drvinfo->version)); - drvinfo->version[sizeof(drvinfo->version) - 1] = '\0'; - strncpy(drvinfo->driver, HNAE_DRIVER_NAME, sizeof(drvinfo->driver)); - drvinfo->driver[sizeof(drvinfo->driver) - 1] = '\0'; + strscpy(drvinfo->driver, HNAE_DRIVER_NAME, sizeof(drvinfo->driver)); - strncpy(drvinfo->bus_info, priv->dev->bus->name, + strscpy(drvinfo->bus_info, priv->dev->bus->name, sizeof(drvinfo->bus_info)); - drvinfo->bus_info[ETHTOOL_BUSINFO_LEN - 1] = '\0'; - strncpy(drvinfo->fw_version, "N/A", ETHTOOL_FWVERS_LEN); + strscpy(drvinfo->fw_version, "N/A", ETHTOOL_FWVERS_LEN); drvinfo->eedump_len = 0; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_devlink.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_devlink.c index 4c441e6a5082..3d3b69605423 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_devlink.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_devlink.c @@ -13,11 +13,6 @@ static int hclge_devlink_info_get(struct devlink *devlink, struct hclge_devlink_priv *priv = devlink_priv(devlink); char version_str[HCLGE_DEVLINK_FW_STRING_LEN]; struct hclge_dev *hdev = priv->hdev; - int ret; - - ret = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (ret) - return ret; snprintf(version_str, sizeof(version_str), "%lu.%lu.%lu.%lu", hnae3_get_field(hdev->fw_version, HNAE3_FW_VERSION_BYTE3_MASK, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_devlink.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_devlink.c index fdc19868b818..a6c3c5e8f0ab 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_devlink.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_devlink.c @@ -13,11 +13,6 @@ static int hclgevf_devlink_info_get(struct devlink *devlink, struct hclgevf_devlink_priv *priv = devlink_priv(devlink); char version_str[HCLGEVF_DEVLINK_FW_STRING_LEN]; struct hclgevf_dev *hdev = priv->hdev; - int ret; - - ret = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (ret) - return ret; snprintf(version_str, sizeof(version_str), "%lu.%lu.%lu.%lu", hnae3_get_field(hdev->fw_version, HNAE3_FW_VERSION_BYTE3_MASK, diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 36bc4fd91ef4..04acd1a992fa 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -5931,9 +5931,9 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, e1000_tx_queue(tx_ring, tx_flags, count); /* Make sure there is space in the ring for the next send. */ e1000_maybe_stop_tx(tx_ring, - (MAX_SKB_FRAGS * + ((MAX_SKB_FRAGS + 1) * DIV_ROUND_UP(PAGE_SIZE, - adapter->tx_fifo_limit) + 2)); + adapter->tx_fifo_limit) + 4)); if (!netdev_xmit_more() || netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 616d27ec3226..887a735fe2a7 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -4466,11 +4466,7 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi, return -EOPNOTSUPP; /* First 4 bytes of L4 header */ - if (usr_ip4_spec->l4_4_bytes == htonl(0xFFFFFFFF)) - new_mask |= I40E_L4_SRC_MASK | I40E_L4_DST_MASK; - else if (!usr_ip4_spec->l4_4_bytes) - new_mask &= ~(I40E_L4_SRC_MASK | I40E_L4_DST_MASK); - else + if (usr_ip4_spec->l4_4_bytes) return -EOPNOTSUPP; /* Filtering on Type of Service is not supported. */ @@ -4509,11 +4505,7 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi, else return -EOPNOTSUPP; - if (usr_ip6_spec->l4_4_bytes == htonl(0xFFFFFFFF)) - new_mask |= I40E_L4_SRC_MASK | I40E_L4_DST_MASK; - else if (!usr_ip6_spec->l4_4_bytes) - new_mask &= ~(I40E_L4_SRC_MASK | I40E_L4_DST_MASK); - else + if (usr_ip6_spec->l4_4_bytes) return -EOPNOTSUPP; /* Filtering on Traffic class is not supported. */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 6861b3e2ced3..95485b56d6c3 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -10656,6 +10656,21 @@ static int i40e_rebuild_channels(struct i40e_vsi *vsi) } /** + * i40e_clean_xps_state - clean xps state for every tx_ring + * @vsi: ptr to the VSI + **/ +static void i40e_clean_xps_state(struct i40e_vsi *vsi) +{ + int i; + + if (vsi->tx_rings) + for (i = 0; i < vsi->num_queue_pairs; i++) + if (vsi->tx_rings[i]) + clear_bit(__I40E_TX_XPS_INIT_DONE, + vsi->tx_rings[i]->state); +} + +/** * i40e_prep_for_reset - prep for the core to reset * @pf: board private structure * @@ -10679,8 +10694,10 @@ static void i40e_prep_for_reset(struct i40e_pf *pf) i40e_pf_quiesce_all_vsi(pf); for (v = 0; v < pf->num_alloc_vsi; v++) { - if (pf->vsi[v]) + if (pf->vsi[v]) { + i40e_clean_xps_state(pf->vsi[v]); pf->vsi[v]->seid = 0; + } } i40e_shutdown_adminq(&pf->hw); diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 72ddcefc45b1..635f93d60318 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -1578,6 +1578,7 @@ bool i40e_reset_vf(struct i40e_vf *vf, bool flr) i40e_cleanup_reset_vf(vf); i40e_flush(hw); + usleep_range(20000, 40000); clear_bit(I40E_VF_STATE_RESETTING, &vf->vf_states); return true; @@ -1701,6 +1702,7 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) } i40e_flush(hw); + usleep_range(20000, 40000); clear_bit(__I40E_VF_DISABLE, pf->state); return true; diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index 1d638216484d..8286e47b4bae 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -311,12 +311,6 @@ static int ice_devlink_info_get(struct devlink *devlink, } } - err = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Unable to set driver name"); - goto out_free_ctx; - } - ice_info_get_dsn(pf, ctx); err = devlink_info_serial_number_put(req, ctx->buf); @@ -1596,21 +1590,22 @@ void ice_devlink_destroy_vf_port(struct ice_vf *vf) #define ICE_DEVLINK_READ_BLK_SIZE (1024 * 1024) +static const struct devlink_region_ops ice_nvm_region_ops; +static const struct devlink_region_ops ice_sram_region_ops; + /** * ice_devlink_nvm_snapshot - Capture a snapshot of the NVM flash contents * @devlink: the devlink instance - * @ops: the devlink region being snapshotted + * @ops: the devlink region to snapshot * @extack: extended ACK response structure * @data: on exit points to snapshot data buffer * - * This function is called in response to the DEVLINK_CMD_REGION_TRIGGER for - * the nvm-flash devlink region. It captures a snapshot of the full NVM flash - * contents, including both banks of flash. This snapshot can later be viewed - * via the devlink-region interface. + * This function is called in response to a DEVLINK_CMD_REGION_NEW for either + * the nvm-flash or shadow-ram region. * - * It captures the flash using the FLASH_ONLY bit set when reading via - * firmware, so it does not read the current Shadow RAM contents. For that, - * use the shadow-ram region. + * It captures a snapshot of the NVM or Shadow RAM flash contents. This + * snapshot can then later be viewed via the DEVLINK_CMD_REGION_READ netlink + * interface. * * @returns zero on success, and updates the data pointer. Returns a non-zero * error code on failure. @@ -1622,17 +1617,27 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, struct ice_pf *pf = devlink_priv(devlink); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; + bool read_shadow_ram; u8 *nvm_data, *tmp, i; u32 nvm_size, left; s8 num_blks; int status; - nvm_size = hw->flash.flash_size; + if (ops == &ice_nvm_region_ops) { + read_shadow_ram = false; + nvm_size = hw->flash.flash_size; + } else if (ops == &ice_sram_region_ops) { + read_shadow_ram = true; + nvm_size = hw->flash.sr_words * 2u; + } else { + NL_SET_ERR_MSG_MOD(extack, "Unexpected region in snapshot function"); + return -EOPNOTSUPP; + } + nvm_data = vzalloc(nvm_size); if (!nvm_data) return -ENOMEM; - num_blks = DIV_ROUND_UP(nvm_size, ICE_DEVLINK_READ_BLK_SIZE); tmp = nvm_data; left = nvm_size; @@ -1656,7 +1661,7 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, } status = ice_read_flat_nvm(hw, i * ICE_DEVLINK_READ_BLK_SIZE, - &read_sz, tmp, false); + &read_sz, tmp, read_shadow_ram); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", read_sz, status, hw->adminq.sq_last_status); @@ -1677,62 +1682,69 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, } /** - * ice_devlink_sram_snapshot - Capture a snapshot of the Shadow RAM contents + * ice_devlink_nvm_read - Read a portion of NVM flash contents * @devlink: the devlink instance - * @ops: the devlink region being snapshotted + * @ops: the devlink region to snapshot * @extack: extended ACK response structure - * @data: on exit points to snapshot data buffer + * @offset: the offset to start at + * @size: the amount to read + * @data: the data buffer to read into * - * This function is called in response to the DEVLINK_CMD_REGION_TRIGGER for - * the shadow-ram devlink region. It captures a snapshot of the shadow ram - * contents. This snapshot can later be viewed via the devlink-region - * interface. + * This function is called in response to DEVLINK_CMD_REGION_READ to directly + * read a section of the NVM contents. + * + * It reads from either the nvm-flash or shadow-ram region contents. * * @returns zero on success, and updates the data pointer. Returns a non-zero * error code on failure. */ -static int -ice_devlink_sram_snapshot(struct devlink *devlink, - const struct devlink_region_ops __always_unused *ops, - struct netlink_ext_ack *extack, u8 **data) +static int ice_devlink_nvm_read(struct devlink *devlink, + const struct devlink_region_ops *ops, + struct netlink_ext_ack *extack, + u64 offset, u32 size, u8 *data) { struct ice_pf *pf = devlink_priv(devlink); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; - u8 *sram_data; - u32 sram_size; - int err; + bool read_shadow_ram; + u64 nvm_size; + int status; - sram_size = hw->flash.sr_words * 2u; - sram_data = vzalloc(sram_size); - if (!sram_data) - return -ENOMEM; + if (ops == &ice_nvm_region_ops) { + read_shadow_ram = false; + nvm_size = hw->flash.flash_size; + } else if (ops == &ice_sram_region_ops) { + read_shadow_ram = true; + nvm_size = hw->flash.sr_words * 2u; + } else { + NL_SET_ERR_MSG_MOD(extack, "Unexpected region in snapshot function"); + return -EOPNOTSUPP; + } - err = ice_acquire_nvm(hw, ICE_RES_READ); - if (err) { + if (offset + size >= nvm_size) { + NL_SET_ERR_MSG_MOD(extack, "Cannot read beyond the region size"); + return -ERANGE; + } + + status = ice_acquire_nvm(hw, ICE_RES_READ); + if (status) { dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - err, hw->adminq.sq_last_status); + status, hw->adminq.sq_last_status); NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - vfree(sram_data); - return err; + return -EIO; } - /* Read from the Shadow RAM, rather than directly from NVM */ - err = ice_read_flat_nvm(hw, 0, &sram_size, sram_data, true); - if (err) { + status = ice_read_flat_nvm(hw, (u32)offset, &size, data, + read_shadow_ram); + if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - sram_size, err, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, - "Failed to read Shadow RAM contents"); + size, status, hw->adminq.sq_last_status); + NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); ice_release_nvm(hw); - vfree(sram_data); - return err; + return -EIO; } - ice_release_nvm(hw); - *data = sram_data; - return 0; } @@ -1784,12 +1796,14 @@ static const struct devlink_region_ops ice_nvm_region_ops = { .name = "nvm-flash", .destructor = vfree, .snapshot = ice_devlink_nvm_snapshot, + .read = ice_devlink_nvm_read, }; static const struct devlink_region_ops ice_sram_region_ops = { .name = "shadow-ram", .destructor = vfree, - .snapshot = ice_devlink_sram_snapshot, + .snapshot = ice_devlink_nvm_snapshot, + .read = ice_devlink_nvm_read, }; static const struct devlink_region_ops ice_devcaps_region_ops = { diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 2b23b4714a26..a9a7f8b52140 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -1111,8 +1111,7 @@ ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, if (link_up == old_link && link_speed == old_link_speed) return 0; - if (!ice_is_e810(&pf->hw)) - ice_ptp_link_change(pf, pf->hw.pf_id, link_up); + ice_ptp_link_change(pf, pf->hw.pf_id, link_up); if (ice_is_dcb_active(pf)) { if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) @@ -6340,8 +6339,7 @@ static int ice_up_complete(struct ice_vsi *vsi) ice_print_link_msg(vsi, true); netif_tx_start_all_queues(vsi->netdev); netif_carrier_on(vsi->netdev); - if (!ice_is_e810(&pf->hw)) - ice_ptp_link_change(pf, pf->hw.pf_id, true); + ice_ptp_link_change(pf, pf->hw.pf_id, true); } /* Perform an initial read of the statistics registers now to @@ -6773,8 +6771,7 @@ int ice_down(struct ice_vsi *vsi) if (vsi->netdev && vsi->type == ICE_VSI_PF) { vlan_err = ice_vsi_del_vlan_zero(vsi); - if (!ice_is_e810(&vsi->back->hw)) - ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false); + ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false); netif_carrier_off(vsi->netdev); netif_tx_disable(vsi->netdev); } else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) { diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 13e75279e71c..d63161d73eb1 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -600,6 +600,23 @@ static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) } /** + * ice_ptp_is_tx_tracker_up - Check if Tx tracker is ready for new timestamps + * @tx: the PTP Tx timestamp tracker to check + * + * Check that a given PTP Tx timestamp tracker is up, i.e. that it is ready + * to accept new timestamp requests. + * + * Assumes the tx->lock spinlock is already held. + */ +static bool +ice_ptp_is_tx_tracker_up(struct ice_ptp_tx *tx) +{ + lockdep_assert_held(&tx->lock); + + return tx->init && !tx->calibrating; +} + +/** * ice_ptp_tx_tstamp - Process Tx timestamps for a port * @tx: the PTP Tx timestamp tracker * @@ -608,11 +625,13 @@ static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) * * If a given index has a valid timestamp, perform the following steps: * - * 1) copy the timestamp out of the PHY register - * 4) clear the timestamp valid bit in the PHY register - * 5) unlock the index by clearing the associated in_use bit. - * 2) extend the 40b timestamp value to get a 64bit timestamp - * 3) send that timestamp to the stack + * 1) check that the timestamp request is not stale + * 2) check that a timestamp is ready and available in the PHY memory bank + * 3) read and copy the timestamp out of the PHY register + * 4) unlock the index by clearing the associated in_use bit + * 5) check if the timestamp is stale, and discard if so + * 6) extend the 40 bit timestamp value to get a 64 bit timestamp value + * 7) send this 64 bit timestamp to the stack * * Returns true if all timestamps were handled, and false if any slots remain * without a timestamp. @@ -623,24 +642,45 @@ static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) * interrupt. In some cases hardware might not interrupt us again when the * timestamp is captured. * - * Note that we only take the tracking lock when clearing the bit and when - * checking if we need to re-queue this task. The only place where bits can be - * set is the hard xmit routine where an SKB has a request flag set. The only - * places where we clear bits are this work function, or the periodic cleanup - * thread. If the cleanup thread clears a bit we're processing we catch it - * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread - * starts a new timestamp, we might not begin processing it right away but we - * will notice it at the end when we re-queue the task. If a Tx thread starts - * a new timestamp just after this function exits without re-queuing, - * the interrupt when the timestamp finishes should trigger. Avoiding holding - * the lock for the entire function is important in order to ensure that Tx - * threads do not get blocked while waiting for the lock. + * Note that we do not hold the tracking lock while reading the Tx timestamp. + * This is because reading the timestamp requires taking a mutex that might + * sleep. + * + * The only place where we set in_use is when a new timestamp is initiated + * with a slot index. This is only called in the hard xmit routine where an + * SKB has a request flag set. The only places where we clear this bit is this + * function, or during teardown when the Tx timestamp tracker is being + * removed. A timestamp index will never be re-used until the in_use bit for + * that index is cleared. + * + * If a Tx thread starts a new timestamp, we might not begin processing it + * right away but we will notice it at the end when we re-queue the task. + * + * If a Tx thread starts a new timestamp just after this function exits, the + * interrupt for that timestamp should re-trigger this function once + * a timestamp is ready. + * + * In cases where the PTP hardware clock was directly adjusted, some + * timestamps may not be able to safely use the timestamp extension math. In + * this case, software will set the stale bit for any outstanding Tx + * timestamps when the clock is adjusted. Then this function will discard + * those captured timestamps instead of sending them to the stack. + * + * If a Tx packet has been waiting for more than 2 seconds, it is not possible + * to correctly extend the timestamp using the cached PHC time. It is + * extremely unlikely that a packet will ever take this long to timestamp. If + * we detect a Tx timestamp request that has waited for this long we assume + * the packet will never be sent by hardware and discard it without reading + * the timestamp register. */ static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) { struct ice_ptp_port *ptp_port; - bool ts_handled = true; + bool more_timestamps; struct ice_pf *pf; + struct ice_hw *hw; + u64 tstamp_ready; + int err; u8 idx; if (!tx->init) @@ -648,44 +688,86 @@ static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) ptp_port = container_of(tx, struct ice_ptp_port, tx); pf = ptp_port_to_pf(ptp_port); + hw = &pf->hw; + + /* Read the Tx ready status first */ + err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready); + if (err) + return false; for_each_set_bit(idx, tx->in_use, tx->len) { struct skb_shared_hwtstamps shhwtstamps = {}; - u8 phy_idx = idx + tx->quad_offset; - u64 raw_tstamp, tstamp; + u8 phy_idx = idx + tx->offset; + u64 raw_tstamp = 0, tstamp; + bool drop_ts = false; struct sk_buff *skb; - int err; + + /* Drop packets which have waited for more than 2 seconds */ + if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) { + drop_ts = true; + + /* Count the number of Tx timestamps that timed out */ + pf->ptp.tx_hwtstamp_timeouts++; + } + + /* Only read a timestamp from the PHY if its marked as ready + * by the tstamp_ready register. This avoids unnecessary + * reading of timestamps which are not yet valid. This is + * important as we must read all timestamps which are valid + * and only timestamps which are valid during each interrupt. + * If we do not, the hardware logic for generating a new + * interrupt can get stuck on some devices. + */ + if (!(tstamp_ready & BIT_ULL(phy_idx))) { + if (drop_ts) + goto skip_ts_read; + + continue; + } ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx); - err = ice_read_phy_tstamp(&pf->hw, tx->quad, phy_idx, - &raw_tstamp); + err = ice_read_phy_tstamp(hw, tx->block, phy_idx, &raw_tstamp); if (err) continue; ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx); - /* Check if the timestamp is invalid or stale */ - if (!(raw_tstamp & ICE_PTP_TS_VALID) || + /* For PHYs which don't implement a proper timestamp ready + * bitmap, verify that the timestamp value is different + * from the last cached timestamp. If it is not, skip this for + * now assuming it hasn't yet been captured by hardware. + */ + if (!drop_ts && tx->verify_cached && raw_tstamp == tx->tstamps[idx].cached_tstamp) continue; - /* The timestamp is valid, so we'll go ahead and clear this - * index and then send the timestamp up to the stack. - */ + /* Discard any timestamp value without the valid bit set */ + if (!(raw_tstamp & ICE_PTP_TS_VALID)) + drop_ts = true; + +skip_ts_read: spin_lock(&tx->lock); - tx->tstamps[idx].cached_tstamp = raw_tstamp; + if (tx->verify_cached && raw_tstamp) + tx->tstamps[idx].cached_tstamp = raw_tstamp; clear_bit(idx, tx->in_use); skb = tx->tstamps[idx].skb; tx->tstamps[idx].skb = NULL; + if (test_and_clear_bit(idx, tx->stale)) + drop_ts = true; spin_unlock(&tx->lock); - /* it's (unlikely but) possible we raced with the cleanup - * thread for discarding old timestamp requests. + /* It is unlikely but possible that the SKB will have been + * flushed at this point due to link change or teardown. */ if (!skb) continue; + if (drop_ts) { + dev_kfree_skb_any(skb); + continue; + } + /* Extend the timestamp using cached PHC time */ tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp); if (tstamp) { @@ -701,11 +783,10 @@ static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) * poll for remaining timestamps. */ spin_lock(&tx->lock); - if (!bitmap_empty(tx->in_use, tx->len)) - ts_handled = false; + more_timestamps = tx->init && !bitmap_empty(tx->in_use, tx->len); spin_unlock(&tx->lock); - return ts_handled; + return !more_timestamps; } /** @@ -713,26 +794,33 @@ static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) * @tx: Tx tracking structure to initialize * * Assumes that the length has already been initialized. Do not call directly, - * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead. + * use the ice_ptp_init_tx_* instead. */ static int ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx) { - tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL); - if (!tx->tstamps) - return -ENOMEM; + unsigned long *in_use, *stale; + struct ice_tx_tstamp *tstamps; + + tstamps = kcalloc(tx->len, sizeof(*tstamps), GFP_KERNEL); + in_use = bitmap_zalloc(tx->len, GFP_KERNEL); + stale = bitmap_zalloc(tx->len, GFP_KERNEL); + + if (!tstamps || !in_use || !stale) { + kfree(tstamps); + bitmap_free(in_use); + bitmap_free(stale); - tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL); - if (!tx->in_use) { - kfree(tx->tstamps); - tx->tstamps = NULL; return -ENOMEM; } - spin_lock_init(&tx->lock); - + tx->tstamps = tstamps; + tx->in_use = in_use; + tx->stale = stale; tx->init = 1; + spin_lock_init(&tx->lock); + return 0; } @@ -740,31 +828,71 @@ ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx) * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker * @pf: Board private structure * @tx: the tracker to flush + * + * Called during teardown when a Tx tracker is being removed. */ static void ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) { + struct ice_hw *hw = &pf->hw; + u64 tstamp_ready; + int err; u8 idx; - for (idx = 0; idx < tx->len; idx++) { - u8 phy_idx = idx + tx->quad_offset; + err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready); + if (err) { + dev_dbg(ice_pf_to_dev(pf), "Failed to get the Tx tstamp ready bitmap for block %u, err %d\n", + tx->block, err); + + /* If we fail to read the Tx timestamp ready bitmap just + * skip clearing the PHY timestamps. + */ + tstamp_ready = 0; + } + + for_each_set_bit(idx, tx->in_use, tx->len) { + u8 phy_idx = idx + tx->offset; + struct sk_buff *skb; + + /* In case this timestamp is ready, we need to clear it. */ + if (!hw->reset_ongoing && (tstamp_ready & BIT_ULL(phy_idx))) + ice_clear_phy_tstamp(hw, tx->block, phy_idx); spin_lock(&tx->lock); - if (tx->tstamps[idx].skb) { - dev_kfree_skb_any(tx->tstamps[idx].skb); - tx->tstamps[idx].skb = NULL; - pf->ptp.tx_hwtstamp_flushed++; - } + skb = tx->tstamps[idx].skb; + tx->tstamps[idx].skb = NULL; clear_bit(idx, tx->in_use); + clear_bit(idx, tx->stale); spin_unlock(&tx->lock); - /* Clear any potential residual timestamp in the PHY block */ - if (!pf->hw.reset_ongoing) - ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx); + /* Count the number of Tx timestamps flushed */ + pf->ptp.tx_hwtstamp_flushed++; + + /* Free the SKB after we've cleared the bit */ + dev_kfree_skb_any(skb); } } /** + * ice_ptp_mark_tx_tracker_stale - Mark unfinished timestamps as stale + * @tx: the tracker to mark + * + * Mark currently outstanding Tx timestamps as stale. This prevents sending + * their timestamp value to the stack. This is required to prevent extending + * the 40bit hardware timestamp incorrectly. + * + * This should be called when the PTP clock is modified such as after a set + * time request. + */ +static void +ice_ptp_mark_tx_tracker_stale(struct ice_ptp_tx *tx) +{ + spin_lock(&tx->lock); + bitmap_or(tx->stale, tx->stale, tx->in_use, tx->len); + spin_unlock(&tx->lock); +} + +/** * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker * @pf: Board private structure * @tx: Tx tracking structure to release @@ -774,7 +902,12 @@ ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) static void ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) { + spin_lock(&tx->lock); tx->init = 0; + spin_unlock(&tx->lock); + + /* wait for potentially outstanding interrupt to complete */ + synchronize_irq(pf->msix_entries[pf->oicr_idx].vector); ice_ptp_flush_tx_tracker(pf, tx); @@ -784,6 +917,9 @@ ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) bitmap_free(tx->in_use); tx->in_use = NULL; + bitmap_free(tx->stale); + tx->stale = NULL; + tx->len = 0; } @@ -801,9 +937,10 @@ ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) static int ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port) { - tx->quad = port / ICE_PORTS_PER_QUAD; - tx->quad_offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT; - tx->len = INDEX_PER_PORT; + tx->block = port / ICE_PORTS_PER_QUAD; + tx->offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT_E822; + tx->len = INDEX_PER_PORT_E822; + tx->verify_cached = 0; return ice_ptp_alloc_tx_tracker(tx); } @@ -819,59 +956,19 @@ ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port) static int ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx) { - tx->quad = pf->hw.port_info->lport; - tx->quad_offset = 0; - tx->len = INDEX_PER_QUAD; + tx->block = pf->hw.port_info->lport; + tx->offset = 0; + tx->len = INDEX_PER_PORT_E810; + /* The E810 PHY does not provide a timestamp ready bitmap. Instead, + * verify new timestamps against cached copy of the last read + * timestamp. + */ + tx->verify_cached = 1; return ice_ptp_alloc_tx_tracker(tx); } /** - * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped - * @pf: pointer to the PF struct - * @tx: PTP Tx tracker to clean up - * - * Loop through the Tx timestamp requests and see if any of them have been - * waiting for a long time. Discard any SKBs that have been waiting for more - * than 2 seconds. This is long enough to be reasonably sure that the - * timestamp will never be captured. This might happen if the packet gets - * discarded before it reaches the PHY timestamping block. - */ -static void ice_ptp_tx_tstamp_cleanup(struct ice_pf *pf, struct ice_ptp_tx *tx) -{ - struct ice_hw *hw = &pf->hw; - u8 idx; - - if (!tx->init) - return; - - for_each_set_bit(idx, tx->in_use, tx->len) { - struct sk_buff *skb; - u64 raw_tstamp; - - /* Check if this SKB has been waiting for too long */ - if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ)) - continue; - - /* Read tstamp to be able to use this register again */ - ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset, - &raw_tstamp); - - spin_lock(&tx->lock); - skb = tx->tstamps[idx].skb; - tx->tstamps[idx].skb = NULL; - clear_bit(idx, tx->in_use); - spin_unlock(&tx->lock); - - /* Count the number of Tx timestamps which have timed out */ - pf->ptp.tx_hwtstamp_timeouts++; - - /* Free the SKB after we've cleared the bit */ - dev_kfree_skb_any(skb); - } -} - -/** * ice_ptp_update_cached_phctime - Update the cached PHC time values * @pf: Board specific private structure * @@ -941,20 +1038,13 @@ static int ice_ptp_update_cached_phctime(struct ice_pf *pf) * @pf: Board specific private structure * * This function must be called when the cached PHC time is no longer valid, - * such as after a time adjustment. It discards any outstanding Tx timestamps, - * and updates the cached PHC time for both the PF and Rx rings. If updating - * the PHC time cannot be done immediately, a warning message is logged and - * the work item is scheduled. - * - * These steps are required in order to ensure that we do not accidentally - * report a timestamp extended by the wrong PHC cached copy. Note that we - * do not directly update the cached timestamp here because it is possible - * this might produce an error when ICE_CFG_BUSY is set. If this occurred, we - * would have to try again. During that time window, timestamps might be - * requested and returned with an invalid extension. Thus, on failure to - * immediately update the cached PHC time we would need to zero the value - * anyways. For this reason, we just zero the value immediately and queue the - * update work item. + * such as after a time adjustment. It marks any currently outstanding Tx + * timestamps as stale and updates the cached PHC time for both the PF and Rx + * rings. + * + * If updating the PHC time cannot be done immediately, a warning message is + * logged and the work item is scheduled immediately to minimize the window + * with a wrong cached timestamp. */ static void ice_ptp_reset_cached_phctime(struct ice_pf *pf) { @@ -978,8 +1068,12 @@ static void ice_ptp_reset_cached_phctime(struct ice_pf *pf) msecs_to_jiffies(10)); } - /* Flush any outstanding Tx timestamps */ - ice_ptp_flush_tx_tracker(pf, &pf->ptp.port.tx); + /* Mark any outstanding timestamps as stale, since they might have + * been captured in hardware before the time update. This could lead + * to us extending them with the wrong cached value resulting in + * incorrect timestamp values. + */ + ice_ptp_mark_tx_tracker_stale(&pf->ptp.port.tx); } /** @@ -1060,19 +1154,6 @@ static u64 ice_base_incval(struct ice_pf *pf) } /** - * ice_ptp_reset_ts_memory_quad - Reset timestamp memory for one quad - * @pf: The PF private data structure - * @quad: The quad (0-4) - */ -static void ice_ptp_reset_ts_memory_quad(struct ice_pf *pf, int quad) -{ - struct ice_hw *hw = &pf->hw; - - ice_write_quad_reg_e822(hw, quad, Q_REG_TS_CTRL, Q_REG_TS_CTRL_M); - ice_write_quad_reg_e822(hw, quad, Q_REG_TS_CTRL, ~(u32)Q_REG_TS_CTRL_M); -} - -/** * ice_ptp_check_tx_fifo - Check whether Tx FIFO is in an OK state * @port: PTP port for which Tx FIFO is checked */ @@ -1124,7 +1205,7 @@ static int ice_ptp_check_tx_fifo(struct ice_ptp_port *port) dev_dbg(ice_pf_to_dev(pf), "Port %d Tx FIFO still not empty; resetting quad %d\n", port->port_num, quad); - ice_ptp_reset_ts_memory_quad(pf, quad); + ice_ptp_reset_ts_memory_quad_e822(hw, quad); port->tx_fifo_busy_cnt = FIFO_OK; return 0; } @@ -1133,130 +1214,49 @@ static int ice_ptp_check_tx_fifo(struct ice_ptp_port *port) } /** - * ice_ptp_check_tx_offset_valid - Check if the Tx PHY offset is valid - * @port: the PTP port to check - * - * Checks whether the Tx offset for the PHY associated with this port is - * valid. Returns 0 if the offset is valid, and a non-zero error code if it is - * not. - */ -static int ice_ptp_check_tx_offset_valid(struct ice_ptp_port *port) -{ - struct ice_pf *pf = ptp_port_to_pf(port); - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - u32 val; - int err; - - err = ice_ptp_check_tx_fifo(port); - if (err) - return err; - - err = ice_read_phy_reg_e822(hw, port->port_num, P_REG_TX_OV_STATUS, - &val); - if (err) { - dev_err(dev, "Failed to read TX_OV_STATUS for port %d, err %d\n", - port->port_num, err); - return -EAGAIN; - } - - if (!(val & P_REG_TX_OV_STATUS_OV_M)) - return -EAGAIN; - - return 0; -} - -/** - * ice_ptp_check_rx_offset_valid - Check if the Rx PHY offset is valid - * @port: the PTP port to check - * - * Checks whether the Rx offset for the PHY associated with this port is - * valid. Returns 0 if the offset is valid, and a non-zero error code if it is - * not. - */ -static int ice_ptp_check_rx_offset_valid(struct ice_ptp_port *port) -{ - struct ice_pf *pf = ptp_port_to_pf(port); - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - int err; - u32 val; - - err = ice_read_phy_reg_e822(hw, port->port_num, P_REG_RX_OV_STATUS, - &val); - if (err) { - dev_err(dev, "Failed to read RX_OV_STATUS for port %d, err %d\n", - port->port_num, err); - return err; - } - - if (!(val & P_REG_RX_OV_STATUS_OV_M)) - return -EAGAIN; - - return 0; -} - -/** - * ice_ptp_check_offset_valid - Check port offset valid bit - * @port: Port for which offset valid bit is checked - * - * Returns 0 if both Tx and Rx offset are valid, and -EAGAIN if one of the - * offset is not ready. - */ -static int ice_ptp_check_offset_valid(struct ice_ptp_port *port) -{ - int tx_err, rx_err; - - /* always check both Tx and Rx offset validity */ - tx_err = ice_ptp_check_tx_offset_valid(port); - rx_err = ice_ptp_check_rx_offset_valid(port); - - if (tx_err || rx_err) - return -EAGAIN; - - return 0; -} - -/** - * ice_ptp_wait_for_offset_valid - Check for valid Tx and Rx offsets + * ice_ptp_wait_for_offsets - Check for valid Tx and Rx offsets * @work: Pointer to the kthread_work structure for this task * - * Check whether both the Tx and Rx offsets are valid for enabling the vernier - * calibration. + * Check whether hardware has completed measuring the Tx and Rx offset values + * used to configure and enable vernier timestamp calibration. + * + * Once the offset in either direction is measured, configure the associated + * registers with the calibrated offset values and enable timestamping. The Tx + * and Rx directions are configured independently as soon as their associated + * offsets are known. * - * Once we have valid offsets from hardware, update the total Tx and Rx - * offsets, and exit bypass mode. This enables more precise timestamps using - * the extra data measured during the vernier calibration process. + * This function reschedules itself until both Tx and Rx calibration have + * completed. */ -static void ice_ptp_wait_for_offset_valid(struct kthread_work *work) +static void ice_ptp_wait_for_offsets(struct kthread_work *work) { struct ice_ptp_port *port; - int err; - struct device *dev; struct ice_pf *pf; struct ice_hw *hw; + int tx_err; + int rx_err; port = container_of(work, struct ice_ptp_port, ov_work.work); pf = ptp_port_to_pf(port); hw = &pf->hw; - dev = ice_pf_to_dev(pf); - - if (ice_is_reset_in_progress(pf->state)) - return; - if (ice_ptp_check_offset_valid(port)) { - /* Offsets not ready yet, try again later */ + if (ice_is_reset_in_progress(pf->state)) { + /* wait for device driver to complete reset */ kthread_queue_delayed_work(pf->ptp.kworker, &port->ov_work, msecs_to_jiffies(100)); return; } - /* Offsets are valid, so it is safe to exit bypass mode */ - err = ice_phy_exit_bypass_e822(hw, port->port_num); - if (err) { - dev_warn(dev, "Failed to exit bypass mode for PHY port %u, err %d\n", - port->port_num, err); + tx_err = ice_ptp_check_tx_fifo(port); + if (!tx_err) + tx_err = ice_phy_cfg_tx_offset_e822(hw, port->port_num); + rx_err = ice_phy_cfg_rx_offset_e822(hw, port->port_num); + if (tx_err || rx_err) { + /* Tx and/or Rx offset not yet configured, try again later */ + kthread_queue_delayed_work(pf->ptp.kworker, + &port->ov_work, + msecs_to_jiffies(100)); return; } } @@ -1317,16 +1317,20 @@ ice_ptp_port_phy_restart(struct ice_ptp_port *ptp_port) kthread_cancel_delayed_work_sync(&ptp_port->ov_work); /* temporarily disable Tx timestamps while calibrating PHY offset */ + spin_lock(&ptp_port->tx.lock); ptp_port->tx.calibrating = true; + spin_unlock(&ptp_port->tx.lock); ptp_port->tx_fifo_busy_cnt = 0; - /* Start the PHY timer in bypass mode */ - err = ice_start_phy_timer_e822(hw, port, true); + /* Start the PHY timer in Vernier mode */ + err = ice_start_phy_timer_e822(hw, port); if (err) goto out_unlock; /* Enable Tx timestamps right away */ + spin_lock(&ptp_port->tx.lock); ptp_port->tx.calibrating = false; + spin_unlock(&ptp_port->tx.lock); kthread_queue_delayed_work(pf->ptp.kworker, &ptp_port->ov_work, 0); @@ -1341,45 +1345,33 @@ out_unlock: } /** - * ice_ptp_link_change - Set or clear port registers for timestamping + * ice_ptp_link_change - Reconfigure PTP after link status change * @pf: Board private structure * @port: Port for which the PHY start is set * @linkup: Link is up or down */ -int ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup) +void ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup) { struct ice_ptp_port *ptp_port; - if (!test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) - return 0; + if (!test_bit(ICE_FLAG_PTP, pf->flags)) + return; - if (port >= ICE_NUM_EXTERNAL_PORTS) - return -EINVAL; + if (WARN_ON_ONCE(port >= ICE_NUM_EXTERNAL_PORTS)) + return; ptp_port = &pf->ptp.port; - if (ptp_port->port_num != port) - return -EINVAL; + if (WARN_ON_ONCE(ptp_port->port_num != port)) + return; - /* Update cached link err for this port immediately */ + /* Update cached link status for this port immediately */ ptp_port->link_up = linkup; - if (!test_bit(ICE_FLAG_PTP, pf->flags)) - /* PTP is not setup */ - return -EAGAIN; - - return ice_ptp_port_phy_restart(ptp_port); -} - -/** - * ice_ptp_reset_ts_memory - Reset timestamp memory for all quads - * @pf: The PF private data structure - */ -static void ice_ptp_reset_ts_memory(struct ice_pf *pf) -{ - int quad; + /* E810 devices do not need to reconfigure the PHY */ + if (ice_is_e810(&pf->hw)) + return; - quad = pf->hw.port_info->lport / ICE_PORTS_PER_QUAD; - ice_ptp_reset_ts_memory_quad(pf, quad); + ice_ptp_port_phy_restart(ptp_port); } /** @@ -1397,7 +1389,7 @@ static int ice_ptp_tx_ena_intr(struct ice_pf *pf, bool ena, u32 threshold) int quad; u32 val; - ice_ptp_reset_ts_memory(pf); + ice_ptp_reset_ts_memory(hw); for (quad = 0; quad < ICE_MAX_QUAD; quad++) { err = ice_read_quad_reg_e822(hw, quad, Q_REG_TX_MEM_GBL_CFG, @@ -2332,11 +2324,14 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) { u8 idx; - /* Check if this tracker is initialized */ - if (!tx->init || tx->calibrating) + spin_lock(&tx->lock); + + /* Check that this tracker is accepting new timestamp requests */ + if (!ice_ptp_is_tx_tracker_up(tx)) { + spin_unlock(&tx->lock); return -1; + } - spin_lock(&tx->lock); /* Find and set the first available index */ idx = find_first_zero_bit(tx->in_use, tx->len); if (idx < tx->len) { @@ -2345,6 +2340,7 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) * requests. */ set_bit(idx, tx->in_use); + clear_bit(idx, tx->stale); tx->tstamps[idx].start = jiffies; tx->tstamps[idx].skb = skb_get(skb); skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; @@ -2359,7 +2355,7 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) if (idx >= tx->len) return -1; else - return idx + tx->quad_offset; + return idx + tx->offset; } /** @@ -2384,8 +2380,6 @@ static void ice_ptp_periodic_work(struct kthread_work *work) err = ice_ptp_update_cached_phctime(pf); - ice_ptp_tx_tstamp_cleanup(pf, &pf->ptp.port.tx); - /* Run twice a second or reschedule if phc update failed */ kthread_queue_delayed_work(ptp->kworker, &ptp->work, msecs_to_jiffies(err ? 10 : 500)); @@ -2462,7 +2456,7 @@ pfr: err = ice_ptp_init_tx_e810(pf, &ptp->port.tx); } else { kthread_init_delayed_work(&ptp->port.ov_work, - ice_ptp_wait_for_offset_valid); + ice_ptp_wait_for_offsets); err = ice_ptp_init_tx_e822(pf, &ptp->port.tx, ptp->port.port_num); } @@ -2625,7 +2619,7 @@ static int ice_ptp_init_port(struct ice_pf *pf, struct ice_ptp_port *ptp_port) return ice_ptp_init_tx_e810(pf, &ptp_port->tx); kthread_init_delayed_work(&ptp_port->ov_work, - ice_ptp_wait_for_offset_valid); + ice_ptp_wait_for_offsets); return ice_ptp_init_tx_e822(pf, &ptp_port->tx, ptp_port->port_num); } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h index 028349295b71..9cda2f43e0e5 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp.h @@ -93,9 +93,14 @@ struct ice_perout_channel { * we discard old requests that were not fulfilled within a 2 second time * window. * Timestamp values in the PHY are read only and do not get cleared except at - * hardware reset or when a new timestamp value is captured. The cached_tstamp - * field is used to detect the case where a new timestamp has not yet been - * captured, ensuring that we avoid sending stale timestamp data to the stack. + * hardware reset or when a new timestamp value is captured. + * + * Some PHY types do not provide a "ready" bitmap indicating which timestamp + * indexes are valid. In these cases, we use a cached_tstamp to keep track of + * the last timestamp we read for a given index. If the current timestamp + * value is the same as the cached value, we assume a new timestamp hasn't + * been captured. This avoids reporting stale timestamps to the stack. This is + * only done if the verify_cached flag is set in ice_ptp_tx structure. */ struct ice_tx_tstamp { struct sk_buff *skb; @@ -105,30 +110,35 @@ struct ice_tx_tstamp { /** * struct ice_ptp_tx - Tracking structure for all Tx timestamp requests on a port - * @lock: lock to prevent concurrent write to in_use bitmap + * @lock: lock to prevent concurrent access to fields of this struct * @tstamps: array of len to store outstanding requests * @in_use: bitmap of len to indicate which slots are in use - * @quad: which quad the timestamps are captured in - * @quad_offset: offset into timestamp block of the quad to get the real index + * @stale: bitmap of len to indicate slots which have stale timestamps + * @block: which memory block (quad or port) the timestamps are captured in + * @offset: offset into timestamp block to get the real index * @len: length of the tstamps and in_use fields. * @init: if true, the tracker is initialized; * @calibrating: if true, the PHY is calibrating the Tx offset. During this * window, timestamps are temporarily disabled. + * @verify_cached: if true, verify new timestamp differs from last read value */ struct ice_ptp_tx { spinlock_t lock; /* lock protecting in_use bitmap */ struct ice_tx_tstamp *tstamps; unsigned long *in_use; - u8 quad; - u8 quad_offset; + unsigned long *stale; + u8 block; + u8 offset; u8 len; - u8 init; - u8 calibrating; + u8 init : 1; + u8 calibrating : 1; + u8 verify_cached : 1; }; /* Quad and port information for initializing timestamp blocks */ #define INDEX_PER_QUAD 64 -#define INDEX_PER_PORT (INDEX_PER_QUAD / ICE_PORTS_PER_QUAD) +#define INDEX_PER_PORT_E822 16 +#define INDEX_PER_PORT_E810 64 /** * struct ice_ptp_port - data used to initialize an external port for PTP @@ -256,7 +266,7 @@ void ice_ptp_reset(struct ice_pf *pf); void ice_ptp_prepare_for_reset(struct ice_pf *pf); void ice_ptp_init(struct ice_pf *pf); void ice_ptp_release(struct ice_pf *pf); -int ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup); +void ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup); #else /* IS_ENABLED(CONFIG_PTP_1588_CLOCK) */ static inline int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr) { @@ -291,7 +301,8 @@ static inline void ice_ptp_reset(struct ice_pf *pf) { } static inline void ice_ptp_prepare_for_reset(struct ice_pf *pf) { } static inline void ice_ptp_init(struct ice_pf *pf) { } static inline void ice_ptp_release(struct ice_pf *pf) { } -static inline int ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup) -{ return 0; } +static inline void ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup) +{ +} #endif /* IS_ENABLED(CONFIG_PTP_1588_CLOCK) */ #endif /* _ICE_PTP_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 1f8dd50db524..a38614d21ea8 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -656,6 +656,32 @@ ice_clear_phy_tstamp_e822(struct ice_hw *hw, u8 quad, u8 idx) } /** + * ice_ptp_reset_ts_memory_quad_e822 - Clear all timestamps from the quad block + * @hw: pointer to the HW struct + * @quad: the quad to read from + * + * Clear all timestamps from the PHY quad block that is shared between the + * internal PHYs on the E822 devices. + */ +void ice_ptp_reset_ts_memory_quad_e822(struct ice_hw *hw, u8 quad) +{ + ice_write_quad_reg_e822(hw, quad, Q_REG_TS_CTRL, Q_REG_TS_CTRL_M); + ice_write_quad_reg_e822(hw, quad, Q_REG_TS_CTRL, ~(u32)Q_REG_TS_CTRL_M); +} + +/** + * ice_ptp_reset_ts_memory_e822 - Clear all timestamps from all quad blocks + * @hw: pointer to the HW struct + */ +static void ice_ptp_reset_ts_memory_e822(struct ice_hw *hw) +{ + unsigned int quad; + + for (quad = 0; quad < ICE_MAX_QUAD; quad++) + ice_ptp_reset_ts_memory_quad_e822(hw, quad); +} + +/** * ice_read_cgu_reg_e822 - Read a CGU register * @hw: pointer to the HW struct * @addr: Register address to read @@ -1715,21 +1741,48 @@ ice_calc_fixed_tx_offset_e822(struct ice_hw *hw, enum ice_ptp_link_spd link_spd) * adjust Tx timestamps by. This is calculated by combining some known static * latency along with the Vernier offset computations done by hardware. * - * This function must be called only after the offset registers are valid, - * i.e. after the Vernier calibration wait has passed, to ensure that the PHY - * has measured the offset. + * This function will not return successfully until the Tx offset calculations + * have been completed, which requires waiting until at least one packet has + * been transmitted by the device. It is safe to call this function + * periodically until calibration succeeds, as it will only program the offset + * once. * * To avoid overflow, when calculating the offset based on the known static * latency values, we use measurements in 1/100th of a nanosecond, and divide * the TUs per second up front. This avoids overflow while allowing * calculation of the adjustment using integer arithmetic. + * + * Returns zero on success, -EBUSY if the hardware vernier offset + * calibration has not completed, or another error code on failure. */ -static int ice_phy_cfg_tx_offset_e822(struct ice_hw *hw, u8 port) +int ice_phy_cfg_tx_offset_e822(struct ice_hw *hw, u8 port) { enum ice_ptp_link_spd link_spd; enum ice_ptp_fec_mode fec_mode; u64 total_offset, val; int err; + u32 reg; + + /* Nothing to do if we've already programmed the offset */ + err = ice_read_phy_reg_e822(hw, port, P_REG_TX_OR, ®); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read TX_OR for port %u, err %d\n", + port, err); + return err; + } + + if (reg) + return 0; + + err = ice_read_phy_reg_e822(hw, port, P_REG_TX_OV_STATUS, ®); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read TX_OV_STATUS for port %u, err %d\n", + port, err); + return err; + } + + if (!(reg & P_REG_TX_OV_STATUS_OV_M)) + return -EBUSY; err = ice_phy_get_speed_and_fec_e822(hw, port, &link_spd, &fec_mode); if (err) @@ -1783,46 +1836,8 @@ static int ice_phy_cfg_tx_offset_e822(struct ice_hw *hw, u8 port) if (err) return err; - return 0; -} - -/** - * ice_phy_cfg_fixed_tx_offset_e822 - Configure Tx offset for bypass mode - * @hw: pointer to the HW struct - * @port: the PHY port to configure - * - * Calculate and program the fixed Tx offset, and indicate that the offset is - * ready. This can be used when operating in bypass mode. - */ -static int -ice_phy_cfg_fixed_tx_offset_e822(struct ice_hw *hw, u8 port) -{ - enum ice_ptp_link_spd link_spd; - enum ice_ptp_fec_mode fec_mode; - u64 total_offset; - int err; - - err = ice_phy_get_speed_and_fec_e822(hw, port, &link_spd, &fec_mode); - if (err) - return err; - - total_offset = ice_calc_fixed_tx_offset_e822(hw, link_spd); - - /* Program the fixed Tx offset into the P_REG_TOTAL_TX_OFFSET_L - * register, then indicate that the Tx offset is ready. After this, - * timestamps will be enabled. - * - * Note that this skips including the more precise offsets generated - * by the Vernier calibration. - */ - err = ice_write_64b_phy_reg_e822(hw, port, P_REG_TOTAL_TX_OFFSET_L, - total_offset); - if (err) - return err; - - err = ice_write_phy_reg_e822(hw, port, P_REG_TX_OR, 1); - if (err) - return err; + dev_info(ice_hw_to_dev(hw), "Port=%d Tx vernier offset calibration complete\n", + port); return 0; } @@ -2026,6 +2041,11 @@ ice_calc_fixed_rx_offset_e822(struct ice_hw *hw, enum ice_ptp_link_spd link_spd) * measurements taken in hardware with some data about known fixed delay as * well as adjusting for multi-lane alignment delay. * + * This function will not return successfully until the Rx offset calculations + * have been completed, which requires waiting until at least one packet has + * been received by the device. It is safe to call this function periodically + * until calibration succeeds, as it will only program the offset once. + * * This function must be called only after the offset registers are valid, * i.e. after the Vernier calibration wait has passed, to ensure that the PHY * has measured the offset. @@ -2034,13 +2054,38 @@ ice_calc_fixed_rx_offset_e822(struct ice_hw *hw, enum ice_ptp_link_spd link_spd) * latency values, we use measurements in 1/100th of a nanosecond, and divide * the TUs per second up front. This avoids overflow while allowing * calculation of the adjustment using integer arithmetic. + * + * Returns zero on success, -EBUSY if the hardware vernier offset + * calibration has not completed, or another error code on failure. */ -static int ice_phy_cfg_rx_offset_e822(struct ice_hw *hw, u8 port) +int ice_phy_cfg_rx_offset_e822(struct ice_hw *hw, u8 port) { enum ice_ptp_link_spd link_spd; enum ice_ptp_fec_mode fec_mode; u64 total_offset, pmd, val; int err; + u32 reg; + + /* Nothing to do if we've already programmed the offset */ + err = ice_read_phy_reg_e822(hw, port, P_REG_RX_OR, ®); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read RX_OR for port %u, err %d\n", + port, err); + return err; + } + + if (reg) + return 0; + + err = ice_read_phy_reg_e822(hw, port, P_REG_RX_OV_STATUS, ®); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read RX_OV_STATUS for port %u, err %d\n", + port, err); + return err; + } + + if (!(reg & P_REG_RX_OV_STATUS_OV_M)) + return -EBUSY; err = ice_phy_get_speed_and_fec_e822(hw, port, &link_spd, &fec_mode); if (err) @@ -2101,46 +2146,8 @@ static int ice_phy_cfg_rx_offset_e822(struct ice_hw *hw, u8 port) if (err) return err; - return 0; -} - -/** - * ice_phy_cfg_fixed_rx_offset_e822 - Configure fixed Rx offset for bypass mode - * @hw: pointer to the HW struct - * @port: the PHY port to configure - * - * Calculate and program the fixed Rx offset, and indicate that the offset is - * ready. This can be used when operating in bypass mode. - */ -static int -ice_phy_cfg_fixed_rx_offset_e822(struct ice_hw *hw, u8 port) -{ - enum ice_ptp_link_spd link_spd; - enum ice_ptp_fec_mode fec_mode; - u64 total_offset; - int err; - - err = ice_phy_get_speed_and_fec_e822(hw, port, &link_spd, &fec_mode); - if (err) - return err; - - total_offset = ice_calc_fixed_rx_offset_e822(hw, link_spd); - - /* Program the fixed Rx offset into the P_REG_TOTAL_RX_OFFSET_L - * register, then indicate that the Rx offset is ready. After this, - * timestamps will be enabled. - * - * Note that this skips including the more precise offsets generated - * by Vernier calibration. - */ - err = ice_write_64b_phy_reg_e822(hw, port, P_REG_TOTAL_RX_OFFSET_L, - total_offset); - if (err) - return err; - - err = ice_write_phy_reg_e822(hw, port, P_REG_RX_OR, 1); - if (err) - return err; + dev_info(ice_hw_to_dev(hw), "Port=%d Rx vernier offset calibration complete\n", + port); return 0; } @@ -2323,20 +2330,14 @@ ice_stop_phy_timer_e822(struct ice_hw *hw, u8 port, bool soft_reset) * ice_start_phy_timer_e822 - Start the PHY clock timer * @hw: pointer to the HW struct * @port: the PHY port to start - * @bypass: if true, start the PHY in bypass mode * * Start the clock of a PHY port. This must be done as part of the flow to * re-calibrate Tx and Rx timestamping offsets whenever the clock time is * initialized or when link speed changes. * - * Bypass mode enables timestamps immediately without waiting for Vernier - * calibration to complete. Hardware will still continue taking Vernier - * measurements on Tx or Rx of packets, but they will not be applied to - * timestamps. Use ice_phy_exit_bypass_e822 to exit bypass mode once hardware - * has completed offset calculation. + * Hardware will take Vernier measurements on Tx or Rx of packets. */ -int -ice_start_phy_timer_e822(struct ice_hw *hw, u8 port, bool bypass) +int ice_start_phy_timer_e822(struct ice_hw *hw, u8 port) { u32 lo, hi, val; u64 incval; @@ -2414,110 +2415,42 @@ ice_start_phy_timer_e822(struct ice_hw *hw, u8 port, bool bypass) if (err) return err; - if (bypass) { - val |= P_REG_PS_BYPASS_MODE_M; - /* Enter BYPASS mode, enabling timestamps immediately. */ - err = ice_write_phy_reg_e822(hw, port, P_REG_PS, val); - if (err) - return err; - - /* Program the fixed Tx offset */ - err = ice_phy_cfg_fixed_tx_offset_e822(hw, port); - if (err) - return err; - - /* Program the fixed Rx offset */ - err = ice_phy_cfg_fixed_rx_offset_e822(hw, port); - if (err) - return err; - } - ice_debug(hw, ICE_DBG_PTP, "Enabled clock on PHY port %u\n", port); return 0; } /** - * ice_phy_exit_bypass_e822 - Exit bypass mode, after vernier calculations + * ice_get_phy_tx_tstamp_ready_e822 - Read Tx memory status register * @hw: pointer to the HW struct - * @port: the PHY port to configure - * - * After hardware finishes vernier calculations for the Tx and Rx offset, this - * function can be used to exit bypass mode by updating the total Tx and Rx - * offsets, and then disabling bypass. This will enable hardware to include - * the more precise offset calibrations, increasing precision of the generated - * timestamps. + * @quad: the timestamp quad to read from + * @tstamp_ready: contents of the Tx memory status register * - * This cannot be done until hardware has measured the offsets, which requires - * waiting until at least one packet has been sent and received by the device. + * Read the Q_REG_TX_MEMORY_STATUS register indicating which timestamps in + * the PHY are ready. A set bit means the corresponding timestamp is valid and + * ready to be captured from the PHY timestamp block. */ -int ice_phy_exit_bypass_e822(struct ice_hw *hw, u8 port) +static int +ice_get_phy_tx_tstamp_ready_e822(struct ice_hw *hw, u8 quad, u64 *tstamp_ready) { + u32 hi, lo; int err; - u32 val; - - err = ice_read_phy_reg_e822(hw, port, P_REG_TX_OV_STATUS, &val); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to read TX_OV_STATUS for port %u, err %d\n", - port, err); - return err; - } - - if (!(val & P_REG_TX_OV_STATUS_OV_M)) { - ice_debug(hw, ICE_DBG_PTP, "Tx offset is not yet valid for port %u\n", - port); - return -EBUSY; - } - - err = ice_read_phy_reg_e822(hw, port, P_REG_RX_OV_STATUS, &val); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to read RX_OV_STATUS for port %u, err %d\n", - port, err); - return err; - } - - if (!(val & P_REG_TX_OV_STATUS_OV_M)) { - ice_debug(hw, ICE_DBG_PTP, "Rx offset is not yet valid for port %u\n", - port); - return -EBUSY; - } - - err = ice_phy_cfg_tx_offset_e822(hw, port); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to program total Tx offset for port %u, err %d\n", - port, err); - return err; - } - - err = ice_phy_cfg_rx_offset_e822(hw, port); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to program total Rx offset for port %u, err %d\n", - port, err); - return err; - } - /* Exit bypass mode now that the offset has been updated */ - err = ice_read_phy_reg_e822(hw, port, P_REG_PS, &val); + err = ice_read_quad_reg_e822(hw, quad, Q_REG_TX_MEMORY_STATUS_U, &hi); if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to read P_REG_PS for port %u, err %d\n", - port, err); + ice_debug(hw, ICE_DBG_PTP, "Failed to read TX_MEMORY_STATUS_U for quad %u, err %d\n", + quad, err); return err; } - if (!(val & P_REG_PS_BYPASS_MODE_M)) - ice_debug(hw, ICE_DBG_PTP, "Port %u not in bypass mode\n", - port); - - val &= ~P_REG_PS_BYPASS_MODE_M; - err = ice_write_phy_reg_e822(hw, port, P_REG_PS, val); + err = ice_read_quad_reg_e822(hw, quad, Q_REG_TX_MEMORY_STATUS_L, &lo); if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to disable bypass for port %u, err %d\n", - port, err); + ice_debug(hw, ICE_DBG_PTP, "Failed to read TX_MEMORY_STATUS_L for quad %u, err %d\n", + quad, err); return err; } - dev_info(ice_hw_to_dev(hw), "Exiting bypass mode on PHY port %u\n", - port); + *tstamp_ready = (u64)hi << 32 | (u64)lo; return 0; } @@ -3196,6 +3129,22 @@ int ice_clear_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx) return ice_clear_phy_tstamp_e822(hw, block, idx); } +/** + * ice_get_phy_tx_tstamp_ready_e810 - Read Tx memory status register + * @hw: pointer to the HW struct + * @port: the PHY port to read + * @tstamp_ready: contents of the Tx memory status register + * + * E810 devices do not use a Tx memory status register. Instead simply + * indicate that all timestamps are currently ready. + */ +static int +ice_get_phy_tx_tstamp_ready_e810(struct ice_hw *hw, u8 port, u64 *tstamp_ready) +{ + *tstamp_ready = 0xFFFFFFFFFFFFFFFF; + return 0; +} + /* E810T SMA functions * * The following functions operate specifically on E810T hardware and are used @@ -3379,6 +3328,18 @@ bool ice_is_pca9575_present(struct ice_hw *hw) } /** + * ice_ptp_reset_ts_memory - Reset timestamp memory for all blocks + * @hw: pointer to the HW struct + */ +void ice_ptp_reset_ts_memory(struct ice_hw *hw) +{ + if (ice_is_e810(hw)) + return; + + ice_ptp_reset_ts_memory_e822(hw); +} + +/** * ice_ptp_init_phc - Initialize PTP hardware clock * @hw: pointer to the HW struct * @@ -3399,3 +3360,24 @@ int ice_ptp_init_phc(struct ice_hw *hw) else return ice_ptp_init_phc_e822(hw); } + +/** + * ice_get_phy_tx_tstamp_ready - Read PHY Tx memory status indication + * @hw: pointer to the HW struct + * @block: the timestamp block to check + * @tstamp_ready: storage for the PHY Tx memory status information + * + * Check the PHY for Tx timestamp memory status. This reports a 64 bit value + * which indicates which timestamps in the block may be captured. A set bit + * means the timestamp can be read. An unset bit means the timestamp is not + * ready and software should avoid reading the register. + */ +int ice_get_phy_tx_tstamp_ready(struct ice_hw *hw, u8 block, u64 *tstamp_ready) +{ + if (ice_is_e810(hw)) + return ice_get_phy_tx_tstamp_ready_e810(hw, block, + tstamp_ready); + else + return ice_get_phy_tx_tstamp_ready_e822(hw, block, + tstamp_ready); +} diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h index 2bda64c76abc..3b68cb91bd81 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h @@ -133,7 +133,9 @@ int ice_ptp_write_incval_locked(struct ice_hw *hw, u64 incval); int ice_ptp_adj_clock(struct ice_hw *hw, s32 adj); int ice_read_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx, u64 *tstamp); int ice_clear_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx); +void ice_ptp_reset_ts_memory(struct ice_hw *hw); int ice_ptp_init_phc(struct ice_hw *hw); +int ice_get_phy_tx_tstamp_ready(struct ice_hw *hw, u8 block, u64 *tstamp_ready); /* E822 family functions */ int ice_read_phy_reg_e822(struct ice_hw *hw, u8 port, u16 offset, u32 *val); @@ -141,6 +143,7 @@ int ice_write_phy_reg_e822(struct ice_hw *hw, u8 port, u16 offset, u32 val); int ice_read_quad_reg_e822(struct ice_hw *hw, u8 quad, u16 offset, u32 *val); int ice_write_quad_reg_e822(struct ice_hw *hw, u8 quad, u16 offset, u32 val); int ice_ptp_prep_port_adj_e822(struct ice_hw *hw, u8 port, s64 time); +void ice_ptp_reset_ts_memory_quad_e822(struct ice_hw *hw, u8 quad); /** * ice_e822_time_ref - Get the current TIME_REF from capabilities @@ -184,8 +187,9 @@ static inline u64 ice_e822_pps_delay(enum ice_time_ref_freq time_ref) /* E822 Vernier calibration functions */ int ice_stop_phy_timer_e822(struct ice_hw *hw, u8 port, bool soft_reset); -int ice_start_phy_timer_e822(struct ice_hw *hw, u8 port, bool bypass); -int ice_phy_exit_bypass_e822(struct ice_hw *hw, u8 port); +int ice_start_phy_timer_e822(struct ice_hw *hw, u8 port); +int ice_phy_cfg_tx_offset_e822(struct ice_hw *hw, u8 port); +int ice_phy_cfg_rx_offset_e822(struct ice_hw *hw, u8 port); /* E810 family functions */ int ice_ptp_init_phy_e810(struct ice_hw *hw); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 36acec89d3d4..7d60da1b7bf4 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -1413,6 +1413,8 @@ static int igb_intr_test(struct igb_adapter *adapter, u64 *data) *data = 1; return -1; } + wr32(E1000_IVAR_MISC, E1000_IVAR_VALID << 8); + wr32(E1000_EIMS, BIT(0)); } else if (adapter->flags & IGB_FLAG_HAS_MSI) { shared_int = false; if (request_irq(irq, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index 774de63dd93a..53a969e34883 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -585,6 +585,11 @@ static int ixgbe_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index 9984ebc62d78..c1cf540d162a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -280,6 +280,11 @@ static int ixgbevf_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index c2cb98d24f5c..f8925cac61e4 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -4270,7 +4270,7 @@ static void mvneta_percpu_elect(struct mvneta_port *pp) /* Use the cpu associated to the rxq when it is online, in all * the other cases, use the cpu 0 which can't be offline. */ - if (cpu_online(pp->rxq_def)) + if (pp->rxq_def < nr_cpu_ids && cpu_online(pp->rxq_def)) elected_cpu = pp->rxq_def; max_cpu = num_present_cpus(); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index c8724bfa86b0..b2b71fe80d61 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -64,6 +64,7 @@ static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool en); static const struct pci_device_id cgx_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_CGX) }, { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_RPM) }, { 0, } /* end of table */ }; @@ -73,12 +74,13 @@ static bool is_dev_rpm(void *cgxd) { struct cgx *cgx = cgxd; - return (cgx->pdev->device == PCI_DEVID_CN10K_RPM); + return (cgx->pdev->device == PCI_DEVID_CN10K_RPM) || + (cgx->pdev->device == PCI_DEVID_CN10KB_RPM); } bool is_lmac_valid(struct cgx *cgx, int lmac_id) { - if (!cgx || lmac_id < 0 || lmac_id >= MAX_LMAC_PER_CGX) + if (!cgx || lmac_id < 0 || lmac_id >= cgx->max_lmac_per_mac) return false; return test_bit(lmac_id, &cgx->lmac_bmap); } @@ -90,7 +92,7 @@ static int get_sequence_id_of_lmac(struct cgx *cgx, int lmac_id) { int tmp, id = 0; - for_each_set_bit(tmp, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(tmp, &cgx->lmac_bmap, cgx->max_lmac_per_mac) { if (tmp == lmac_id) break; id++; @@ -121,7 +123,7 @@ u64 cgx_read(struct cgx *cgx, u64 lmac, u64 offset) struct lmac *lmac_pdata(u8 lmac_id, struct cgx *cgx) { - if (!cgx || lmac_id >= MAX_LMAC_PER_CGX) + if (!cgx || lmac_id >= cgx->max_lmac_per_mac) return NULL; return cgx->lmac_idmap[lmac_id]; @@ -485,7 +487,7 @@ int cgx_set_pkind(void *cgxd, u8 lmac_id, int pkind) if (!is_lmac_valid(cgx, lmac_id)) return -ENODEV; - cgx_write(cgx, lmac_id, CGXX_CMRX_RX_ID_MAP, (pkind & 0x3F)); + cgx_write(cgx, lmac_id, cgx->mac_ops->rxid_map_offset, (pkind & 0x3F)); return 0; } @@ -740,6 +742,10 @@ int cgx_get_fec_stats(void *cgxd, int lmac_id, struct cgx_fec_stats_rsp *rsp) if (!cgx || lmac_id >= cgx->lmac_count) return -ENODEV; + + if (cgx->lmac_idmap[lmac_id]->link_info.fec == OTX2_FEC_NONE) + return 0; + fec_stats_count = cgx_set_fec_stats_count(&cgx->lmac_idmap[lmac_id]->link_info); if (cgx->lmac_idmap[lmac_id]->link_info.fec == OTX2_FEC_BASER) { @@ -1224,7 +1230,7 @@ static inline void link_status_user_format(u64 lstat, linfo->speed = cgx_speed_mbps[FIELD_GET(RESP_LINKSTAT_SPEED, lstat)]; linfo->an = FIELD_GET(RESP_LINKSTAT_AN, lstat); linfo->fec = FIELD_GET(RESP_LINKSTAT_FEC, lstat); - linfo->lmac_type_id = cgx_get_lmac_type(cgx, lmac_id); + linfo->lmac_type_id = FIELD_GET(RESP_LINKSTAT_LMAC_TYPE, lstat); lmac_string = cgx_lmactype_string[linfo->lmac_type_id]; strncpy(linfo->lmac_type, lmac_string, LMACTYPE_STR_LEN - 1); } @@ -1395,7 +1401,7 @@ int cgx_get_fwdata_base(u64 *base) if (!cgx) return -ENXIO; - first_lmac = find_first_bit(&cgx->lmac_bmap, MAX_LMAC_PER_CGX); + first_lmac = find_first_bit(&cgx->lmac_bmap, cgx->max_lmac_per_mac); req = FIELD_SET(CMDREG_ID, CGX_CMD_GET_FWD_BASE, req); err = cgx_fwi_cmd_generic(req, &resp, cgx, first_lmac); if (!err) @@ -1484,7 +1490,7 @@ static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool enable) static inline int cgx_fwi_read_version(u64 *resp, struct cgx *cgx) { - int first_lmac = find_first_bit(&cgx->lmac_bmap, MAX_LMAC_PER_CGX); + int first_lmac = find_first_bit(&cgx->lmac_bmap, cgx->max_lmac_per_mac); u64 req = 0; req = FIELD_SET(CMDREG_ID, CGX_CMD_GET_FW_VER, req); @@ -1522,7 +1528,7 @@ static void cgx_lmac_linkup_work(struct work_struct *work) int i, err; /* Do Link up for all the enabled lmacs */ - for_each_set_bit(i, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(i, &cgx->lmac_bmap, cgx->max_lmac_per_mac) { err = cgx_fwi_link_change(cgx, i, true); if (err) dev_info(dev, "cgx port %d:%d Link up command failed\n", @@ -1542,14 +1548,6 @@ int cgx_lmac_linkup_start(void *cgxd) return 0; } -static void cgx_lmac_get_fifolen(struct cgx *cgx) -{ - u64 cfg; - - cfg = cgx_read(cgx, 0, CGX_CONST); - cgx->mac_ops->fifo_len = FIELD_GET(CGX_CONST_RXFIFO_SIZE, cfg); -} - static int cgx_configure_interrupt(struct cgx *cgx, struct lmac *lmac, int cnt, bool req_free) { @@ -1604,17 +1602,20 @@ static int cgx_lmac_init(struct cgx *cgx) u64 lmac_list; int i, err; - cgx_lmac_get_fifolen(cgx); - - cgx->lmac_count = cgx->mac_ops->get_nr_lmacs(cgx); /* lmac_list specifies which lmacs are enabled * when bit n is set to 1, LMAC[n] is enabled */ - if (cgx->mac_ops->non_contiguous_serdes_lane) - lmac_list = cgx_read(cgx, 0, CGXX_CMRX_RX_LMACS) & 0xFULL; + if (cgx->mac_ops->non_contiguous_serdes_lane) { + if (is_dev_rpm2(cgx)) + lmac_list = + cgx_read(cgx, 0, RPM2_CMRX_RX_LMACS) & 0xFFULL; + else + lmac_list = + cgx_read(cgx, 0, CGXX_CMRX_RX_LMACS) & 0xFULL; + } - if (cgx->lmac_count > MAX_LMAC_PER_CGX) - cgx->lmac_count = MAX_LMAC_PER_CGX; + if (cgx->lmac_count > cgx->max_lmac_per_mac) + cgx->lmac_count = cgx->max_lmac_per_mac; for (i = 0; i < cgx->lmac_count; i++) { lmac = kzalloc(sizeof(struct lmac), GFP_KERNEL); @@ -1635,7 +1636,9 @@ static int cgx_lmac_init(struct cgx *cgx) lmac->cgx = cgx; lmac->mac_to_index_bmap.max = - MAX_DMAC_ENTRIES_PER_CGX / cgx->lmac_count; + cgx->mac_ops->dmac_filter_count / + cgx->lmac_count; + err = rvu_alloc_bitmap(&lmac->mac_to_index_bmap); if (err) goto err_name_free; @@ -1692,7 +1695,7 @@ static int cgx_lmac_exit(struct cgx *cgx) } /* Free all lmac related resources */ - for_each_set_bit(i, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(i, &cgx->lmac_bmap, cgx->max_lmac_per_mac) { lmac = cgx->lmac_idmap[i]; if (!lmac) continue; @@ -1708,6 +1711,12 @@ static int cgx_lmac_exit(struct cgx *cgx) static void cgx_populate_features(struct cgx *cgx) { + u64 cfg; + + cfg = cgx_read(cgx, 0, CGX_CONST); + cgx->mac_ops->fifo_len = FIELD_GET(CGX_CONST_RXFIFO_SIZE, cfg); + cgx->max_lmac_per_mac = FIELD_GET(CGX_CONST_MAX_LMACS, cfg); + if (is_dev_rpm(cgx)) cgx->hw_features = (RVU_LMAC_FEAT_DMACF | RVU_MAC_RPM | RVU_LMAC_FEAT_FC | RVU_LMAC_FEAT_PTP); @@ -1716,6 +1725,15 @@ static void cgx_populate_features(struct cgx *cgx) RVU_LMAC_FEAT_PTP | RVU_LMAC_FEAT_DMACF); } +static u8 cgx_get_rxid_mapoffset(struct cgx *cgx) +{ + if (cgx->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10KB_RPM || + is_dev_rpm2(cgx)) + return 0x80; + else + return 0x60; +} + static struct mac_ops cgx_mac_ops = { .name = "cgx", .csr_offset = 0, @@ -1728,12 +1746,14 @@ static struct mac_ops cgx_mac_ops = { .non_contiguous_serdes_lane = false, .rx_stats_cnt = 9, .tx_stats_cnt = 18, + .dmac_filter_count = 32, .get_nr_lmacs = cgx_get_nr_lmacs, .get_lmac_type = cgx_get_lmac_type, .lmac_fifo_len = cgx_get_lmac_fifo_len, .mac_lmac_intl_lbk = cgx_lmac_internal_loopback, .mac_get_rx_stats = cgx_get_rx_stats, .mac_get_tx_stats = cgx_get_tx_stats, + .get_fec_stats = cgx_get_fec_stats, .mac_enadis_rx_pause_fwding = cgx_lmac_enadis_rx_pause_fwding, .mac_get_pause_frm_status = cgx_lmac_get_pause_frm_status, .mac_enadis_pause_frm = cgx_lmac_enadis_pause_frm, @@ -1759,11 +1779,13 @@ static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, cgx); /* Use mac_ops to get MAC specific features */ - if (pdev->device == PCI_DEVID_CN10K_RPM) - cgx->mac_ops = rpm_get_mac_ops(); + if (is_dev_rpm(cgx)) + cgx->mac_ops = rpm_get_mac_ops(cgx); else cgx->mac_ops = &cgx_mac_ops; + cgx->mac_ops->rxid_map_offset = cgx_get_rxid_mapoffset(cgx); + err = pci_enable_device(pdev); if (err) { dev_err(dev, "Failed to enable PCI device\n"); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h index 0b06788b8d80..fb2d37676d84 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h @@ -18,11 +18,7 @@ /* PCI BAR nos */ #define PCI_CFG_REG_BAR_NUM 0 -#define CGX_ID_MASK 0x7 -#define MAX_LMAC_PER_CGX 4 -#define MAX_DMAC_ENTRIES_PER_CGX 32 -#define CGX_FIFO_LEN 65536 /* 64K for both Rx & Tx */ -#define CGX_OFFSET(x) ((x) * MAX_LMAC_PER_CGX) +#define CGX_ID_MASK 0xF /* Registers */ #define CGXX_CMRX_CFG 0x00 @@ -56,7 +52,8 @@ #define CGXX_SCRATCH0_REG 0x1050 #define CGXX_SCRATCH1_REG 0x1058 #define CGX_CONST 0x2000 -#define CGX_CONST_RXFIFO_SIZE GENMASK_ULL(23, 0) +#define CGX_CONST_RXFIFO_SIZE GENMASK_ULL(55, 32) +#define CGX_CONST_MAX_LMACS GENMASK_ULL(31, 24) #define CGXX_SPUX_CONTROL1 0x10000 #define CGXX_SPUX_LNX_FEC_CORR_BLOCKS 0x10700 #define CGXX_SPUX_LNX_FEC_UNCORR_BLOCKS 0x10800 diff --git a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h index 52b6016789fa..39aaf0e4467d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h @@ -75,6 +75,11 @@ struct mac_ops { /* RPM & CGX differs in number of Receive/transmit stats */ u8 rx_stats_cnt; u8 tx_stats_cnt; + /* Unlike CN10K which shares same CSR offset with CGX + * CNF10KB has different csr offset + */ + u64 rxid_map_offset; + u8 dmac_filter_count; /* Incase of RPM get number of lmacs from RPMX_CMR_RX_LMACS[LMAC_EXIST] * number of setbits in lmac_exist tells number of lmacs */ @@ -121,6 +126,9 @@ struct mac_ops { int (*mac_get_pfc_frm_cfg)(void *cgxd, int lmac_id, u8 *tx_pause, u8 *rx_pause); + /* FEC stats */ + int (*get_fec_stats)(void *cgxd, int lmac_id, + struct cgx_fec_stats_rsp *rsp); }; struct cgx { @@ -128,7 +136,10 @@ struct cgx { struct pci_dev *pdev; u8 cgx_id; u8 lmac_count; - struct lmac *lmac_idmap[MAX_LMAC_PER_CGX]; + /* number of LMACs per MAC could be 4 or 8 */ + u8 max_lmac_per_mac; +#define MAX_LMAC_COUNT 8 + struct lmac *lmac_idmap[MAX_LMAC_COUNT]; struct work_struct cgx_cmd_work; struct workqueue_struct *cgx_cmd_workq; struct list_head cgx_list; @@ -150,6 +161,6 @@ struct lmac *lmac_pdata(u8 lmac_id, struct cgx *cgx); int cgx_fwi_cmd_send(u64 req, u64 *resp, struct lmac *lmac); int cgx_fwi_cmd_generic(u64 req, u64 *resp, struct cgx *cgx, int lmac_id); bool is_lmac_valid(struct cgx *cgx, int lmac_id); -struct mac_ops *rpm_get_mac_ops(void); +struct mac_ops *rpm_get_mac_ops(struct cgx *cgx); #endif /* LMAC_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c index a70e1153fa04..de0d88dd10d6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c @@ -8,7 +8,7 @@ #include "cgx.h" #include "lmac_common.h" -static struct mac_ops rpm_mac_ops = { +static struct mac_ops rpm_mac_ops = { .name = "rpm", .csr_offset = 0x4e00, .lmac_offset = 20, @@ -20,12 +20,14 @@ static struct mac_ops rpm_mac_ops = { .non_contiguous_serdes_lane = true, .rx_stats_cnt = 43, .tx_stats_cnt = 34, + .dmac_filter_count = 32, .get_nr_lmacs = rpm_get_nr_lmacs, .get_lmac_type = rpm_get_lmac_type, .lmac_fifo_len = rpm_get_lmac_fifo_len, .mac_lmac_intl_lbk = rpm_lmac_internal_loopback, .mac_get_rx_stats = rpm_get_rx_stats, .mac_get_tx_stats = rpm_get_tx_stats, + .get_fec_stats = rpm_get_fec_stats, .mac_enadis_rx_pause_fwding = rpm_lmac_enadis_rx_pause_fwding, .mac_get_pause_frm_status = rpm_lmac_get_pause_frm_status, .mac_enadis_pause_frm = rpm_lmac_enadis_pause_frm, @@ -37,9 +39,50 @@ static struct mac_ops rpm_mac_ops = { .mac_get_pfc_frm_cfg = rpm_lmac_get_pfc_frm_cfg, }; -struct mac_ops *rpm_get_mac_ops(void) +static struct mac_ops rpm2_mac_ops = { + .name = "rpm", + .csr_offset = RPM2_CSR_OFFSET, + .lmac_offset = 20, + .int_register = RPM2_CMRX_SW_INT, + .int_set_reg = RPM2_CMRX_SW_INT_ENA_W1S, + .irq_offset = 1, + .int_ena_bit = BIT_ULL(0), + .lmac_fwi = RPM_LMAC_FWI, + .non_contiguous_serdes_lane = true, + .rx_stats_cnt = 43, + .tx_stats_cnt = 34, + .dmac_filter_count = 64, + .get_nr_lmacs = rpm2_get_nr_lmacs, + .get_lmac_type = rpm_get_lmac_type, + .lmac_fifo_len = rpm2_get_lmac_fifo_len, + .mac_lmac_intl_lbk = rpm_lmac_internal_loopback, + .mac_get_rx_stats = rpm_get_rx_stats, + .mac_get_tx_stats = rpm_get_tx_stats, + .get_fec_stats = rpm_get_fec_stats, + .mac_enadis_rx_pause_fwding = rpm_lmac_enadis_rx_pause_fwding, + .mac_get_pause_frm_status = rpm_lmac_get_pause_frm_status, + .mac_enadis_pause_frm = rpm_lmac_enadis_pause_frm, + .mac_pause_frm_config = rpm_lmac_pause_frm_config, + .mac_enadis_ptp_config = rpm_lmac_ptp_config, + .mac_rx_tx_enable = rpm_lmac_rx_tx_enable, + .mac_tx_enable = rpm_lmac_tx_enable, + .pfc_config = rpm_lmac_pfc_config, + .mac_get_pfc_frm_cfg = rpm_lmac_get_pfc_frm_cfg, +}; + +bool is_dev_rpm2(void *rpmd) +{ + rpm_t *rpm = rpmd; + + return (rpm->pdev->device == PCI_DEVID_CN10KB_RPM); +} + +struct mac_ops *rpm_get_mac_ops(rpm_t *rpm) { - return &rpm_mac_ops; + if (is_dev_rpm2(rpm)) + return &rpm2_mac_ops; + else + return &rpm_mac_ops; } static void rpm_write(rpm_t *rpm, u64 lmac, u64 offset, u64 val) @@ -52,6 +95,16 @@ static u64 rpm_read(rpm_t *rpm, u64 lmac, u64 offset) return cgx_read(rpm, lmac, offset); } +/* Read HW major version to determine RPM + * MAC type 100/USX + */ +static bool is_mac_rpmusx(void *rpmd) +{ + rpm_t *rpm = rpmd; + + return rpm_read(rpm, 0, RPMX_CONST1) & 0x700ULL; +} + int rpm_get_nr_lmacs(void *rpmd) { rpm_t *rpm = rpmd; @@ -59,6 +112,13 @@ int rpm_get_nr_lmacs(void *rpmd) return hweight8(rpm_read(rpm, 0, CGXX_CMRX_RX_LMACS) & 0xFULL); } +int rpm2_get_nr_lmacs(void *rpmd) +{ + rpm_t *rpm = rpmd; + + return hweight8(rpm_read(rpm, 0, RPM2_CMRX_RX_LMACS) & 0xFFULL); +} + int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable) { rpm_t *rpm = rpmd; @@ -222,6 +282,46 @@ static void rpm_cfg_pfc_quanta_thresh(rpm_t *rpm, int lmac_id, } } +static void rpm2_lmac_cfg_bp(rpm_t *rpm, int lmac_id, u8 tx_pause, u8 rx_pause) +{ + u64 cfg; + + cfg = rpm_read(rpm, lmac_id, RPM2_CMR_RX_OVR_BP); + if (tx_pause) { + /* Configure CL0 Pause Quanta & threshold + * for 802.3X frames + */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 1, true); + cfg &= ~RPM2_CMR_RX_OVR_BP_EN; + } else { + /* Disable all Pause Quanta & threshold values */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 0xffff, false); + cfg |= RPM2_CMR_RX_OVR_BP_EN; + cfg &= ~RPM2_CMR_RX_OVR_BP_BP; + } + rpm_write(rpm, lmac_id, RPM2_CMR_RX_OVR_BP, cfg); +} + +static void rpm_lmac_cfg_bp(rpm_t *rpm, int lmac_id, u8 tx_pause, u8 rx_pause) +{ + u64 cfg; + + cfg = rpm_read(rpm, 0, RPMX_CMR_RX_OVR_BP); + if (tx_pause) { + /* Configure CL0 Pause Quanta & threshold for + * 802.3X frames + */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 1, true); + cfg &= ~RPMX_CMR_RX_OVR_BP_EN(lmac_id); + } else { + /* Disable all Pause Quanta & threshold values */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 0xffff, false); + cfg |= RPMX_CMR_RX_OVR_BP_EN(lmac_id); + cfg &= ~RPMX_CMR_RX_OVR_BP_BP(lmac_id); + } + rpm_write(rpm, 0, RPMX_CMR_RX_OVR_BP, cfg); +} + int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause) { @@ -243,18 +343,11 @@ int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause, cfg |= tx_pause ? 0x0 : RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - cfg = rpm_read(rpm, 0, RPMX_CMR_RX_OVR_BP); - if (tx_pause) { - /* Configure CL0 Pause Quanta & threshold for 802.3X frames */ - rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 1, true); - cfg &= ~RPMX_CMR_RX_OVR_BP_EN(lmac_id); - } else { - /* Disable all Pause Quanta & threshold values */ - rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 0xffff, false); - cfg |= RPMX_CMR_RX_OVR_BP_EN(lmac_id); - cfg &= ~RPMX_CMR_RX_OVR_BP_BP(lmac_id); - } - rpm_write(rpm, 0, RPMX_CMR_RX_OVR_BP, cfg); + if (is_dev_rpm2(rpm)) + rpm2_lmac_cfg_bp(rpm, lmac_id, tx_pause, rx_pause); + else + rpm_lmac_cfg_bp(rpm, lmac_id, tx_pause, rx_pause); + return 0; } @@ -278,13 +371,16 @@ void rpm_lmac_pause_frm_config(void *rpmd, int lmac_id, bool enable) cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + /* Enable channel mask for all LMACS */ + if (is_dev_rpm2(rpm)) + rpm_write(rpm, lmac_id, RPM2_CMR_CHAN_MSK_OR, 0xffff); + else + rpm_write(rpm, 0, RPMX_CMR_CHAN_MSK_OR, ~0ULL); + /* Disable all PFC classes */ cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL); cfg = FIELD_SET(RPM_PFC_CLASS_MASK, 0, cfg); rpm_write(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL, cfg); - - /* Enable channel mask for all LMACS */ - rpm_write(rpm, 0, RPMX_CMR_CHAN_MSK_OR, ~0ULL); } int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat) @@ -292,7 +388,7 @@ int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat) rpm_t *rpm = rpmd; u64 val_lo, val_hi; - if (!rpm || lmac_id >= rpm->lmac_count) + if (!is_lmac_valid(rpm, lmac_id)) return -ENODEV; mutex_lock(&rpm->lock); @@ -320,7 +416,7 @@ int rpm_get_tx_stats(void *rpmd, int lmac_id, int idx, u64 *tx_stat) rpm_t *rpm = rpmd; u64 val_lo, val_hi; - if (!rpm || lmac_id >= rpm->lmac_count) + if (!is_lmac_valid(rpm, lmac_id)) return -ENODEV; mutex_lock(&rpm->lock); @@ -380,13 +476,71 @@ u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id) return 0; } +static int rpmusx_lmac_internal_loopback(rpm_t *rpm, int lmac_id, bool enable) +{ + u64 cfg; + + cfg = rpm_read(rpm, lmac_id, RPM2_USX_PCSX_CONTROL1); + + if (enable) + cfg |= RPM2_USX_PCS_LBK; + else + cfg &= ~RPM2_USX_PCS_LBK; + rpm_write(rpm, lmac_id, RPM2_USX_PCSX_CONTROL1, cfg); + + return 0; +} + +u32 rpm2_get_lmac_fifo_len(void *rpmd, int lmac_id) +{ + u64 hi_perf_lmac, lmac_info; + rpm_t *rpm = rpmd; + u8 num_lmacs; + u32 fifo_len; + + lmac_info = rpm_read(rpm, 0, RPM2_CMRX_RX_LMACS); + /* LMACs are divided into two groups and each group + * gets half of the FIFO + * Group0 lmac_id range {0..3} + * Group1 lmac_id range {4..7} + */ + fifo_len = rpm->mac_ops->fifo_len / 2; + + if (lmac_id < 4) { + num_lmacs = hweight8(lmac_info & 0xF); + hi_perf_lmac = (lmac_info >> 8) & 0x3ULL; + } else { + num_lmacs = hweight8(lmac_info & 0xF0); + hi_perf_lmac = (lmac_info >> 10) & 0x3ULL; + hi_perf_lmac += 4; + } + + switch (num_lmacs) { + case 1: + return fifo_len; + case 2: + return fifo_len / 2; + case 3: + /* LMAC marked as hi_perf gets half of the FIFO + * and rest 1/4th + */ + if (lmac_id == hi_perf_lmac) + return fifo_len / 2; + return fifo_len / 4; + case 4: + default: + return fifo_len / 4; + } + return 0; +} + int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable) { rpm_t *rpm = rpmd; u8 lmac_type; u64 cfg; - if (!rpm || lmac_id >= rpm->lmac_count) + if (!is_lmac_valid(rpm, lmac_id)) return -ENODEV; lmac_type = rpm->mac_ops->get_lmac_type(rpm, lmac_id); @@ -395,6 +549,9 @@ int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable) return 0; } + if (is_dev_rpm2(rpm) && is_mac_rpmusx(rpm)) + return rpmusx_lmac_internal_loopback(rpm, lmac_id, enable); + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1); if (enable) @@ -439,8 +596,8 @@ void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable) int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en) { + u64 cfg, class_en, pfc_class_mask_cfg; rpm_t *rpm = rpmd; - u64 cfg, class_en; if (!is_lmac_valid(rpm, lmac_id)) return -ENODEV; @@ -476,7 +633,10 @@ int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 p rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - rpm_write(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL, class_en); + pfc_class_mask_cfg = is_dev_rpm2(rpm) ? RPM2_CMRX_PRT_CBFC_CTL : + RPMX_CMRX_PRT_CBFC_CTL; + + rpm_write(rpm, lmac_id, pfc_class_mask_cfg, class_en); return 0; } @@ -497,3 +657,59 @@ int rpm_lmac_get_pfc_frm_cfg(void *rpmd, int lmac_id, u8 *tx_pause, u8 *rx_paus return 0; } + +int rpm_get_fec_stats(void *rpmd, int lmac_id, struct cgx_fec_stats_rsp *rsp) +{ + u64 val_lo, val_hi; + rpm_t *rpm = rpmd; + u64 cfg; + + if (!is_lmac_valid(rpm, lmac_id)) + return -ENODEV; + + if (rpm->lmac_idmap[lmac_id]->link_info.fec == OTX2_FEC_NONE) + return 0; + + if (rpm->lmac_idmap[lmac_id]->link_info.fec == OTX2_FEC_BASER) { + val_lo = rpm_read(rpm, lmac_id, RPMX_MTI_FCFECX_VL0_CCW_LO); + val_hi = rpm_read(rpm, lmac_id, RPMX_MTI_FCFECX_CW_HI); + rsp->fec_corr_blks = (val_hi << 16 | val_lo); + + val_lo = rpm_read(rpm, lmac_id, RPMX_MTI_FCFECX_VL0_NCCW_LO); + val_hi = rpm_read(rpm, lmac_id, RPMX_MTI_FCFECX_CW_HI); + rsp->fec_uncorr_blks = (val_hi << 16 | val_lo); + + /* 50G uses 2 Physical serdes lines */ + if (rpm->lmac_idmap[lmac_id]->link_info.lmac_type_id == + LMAC_MODE_50G_R) { + val_lo = rpm_read(rpm, lmac_id, + RPMX_MTI_FCFECX_VL1_CCW_LO); + val_hi = rpm_read(rpm, lmac_id, + RPMX_MTI_FCFECX_CW_HI); + rsp->fec_corr_blks += (val_hi << 16 | val_lo); + + val_lo = rpm_read(rpm, lmac_id, + RPMX_MTI_FCFECX_VL1_NCCW_LO); + val_hi = rpm_read(rpm, lmac_id, + RPMX_MTI_FCFECX_CW_HI); + rsp->fec_uncorr_blks += (val_hi << 16 | val_lo); + } + } else { + /* enable RS-FEC capture */ + cfg = rpm_read(rpm, 0, RPMX_MTI_STAT_STATN_CONTROL); + cfg |= RPMX_RSFEC_RX_CAPTURE | BIT(lmac_id); + rpm_write(rpm, 0, RPMX_MTI_STAT_STATN_CONTROL, cfg); + + val_lo = rpm_read(rpm, 0, + RPMX_MTI_RSFEC_STAT_COUNTER_CAPTURE_2); + val_hi = rpm_read(rpm, 0, RPMX_MTI_STAT_DATA_HI_CDC); + rsp->fec_corr_blks = (val_hi << 32 | val_lo); + + val_lo = rpm_read(rpm, 0, + RPMX_MTI_RSFEC_STAT_COUNTER_CAPTURE_3); + val_hi = rpm_read(rpm, 0, RPMX_MTI_STAT_DATA_HI_CDC); + rsp->fec_uncorr_blks = (val_hi << 32 | val_lo); + } + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h index 77f2ef9e1425..22147b4c2137 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h @@ -12,17 +12,19 @@ /* PCI device IDs */ #define PCI_DEVID_CN10K_RPM 0xA060 +#define PCI_SUBSYS_DEVID_CNF10KB_RPM 0xBC00 +#define PCI_DEVID_CN10KB_RPM 0xA09F /* Registers */ #define RPMX_CMRX_CFG 0x00 #define RPMX_RX_TS_PREPEND BIT_ULL(22) #define RPMX_TX_PTP_1S_SUPPORT BIT_ULL(17) +#define RPMX_CMRX_RX_ID_MAP 0x80 #define RPMX_CMRX_SW_INT 0x180 #define RPMX_CMRX_SW_INT_W1S 0x188 #define RPMX_CMRX_SW_INT_ENA_W1S 0x198 #define RPMX_CMRX_LINK_CFG 0x1070 #define RPMX_MTI_PCS100X_CONTROL1 0x20000 -#define RPMX_MTI_LPCSX_CONTROL1 0x30000 #define RPMX_MTI_PCS_LBK BIT_ULL(14) #define RPMX_MTI_LPCSX_CONTROL(id) (0x30000 | ((id) * 0x100)) @@ -76,11 +78,40 @@ #define RPMX_MTI_MAC100X_XIF_MODE 0x8100 #define RPMX_ONESTEP_ENABLE BIT_ULL(5) #define RPMX_TS_BINARY_MODE BIT_ULL(11) +#define RPMX_CONST1 0x2008 + +/* FEC stats */ +#define RPMX_MTI_STAT_STATN_CONTROL 0x10018 +#define RPMX_MTI_STAT_DATA_HI_CDC 0x10038 +#define RPMX_RSFEC_RX_CAPTURE BIT_ULL(27) +#define RPMX_MTI_RSFEC_STAT_COUNTER_CAPTURE_2 0x40050 +#define RPMX_MTI_RSFEC_STAT_COUNTER_CAPTURE_3 0x40058 +#define RPMX_MTI_FCFECX_VL0_CCW_LO 0x38618 +#define RPMX_MTI_FCFECX_VL0_NCCW_LO 0x38620 +#define RPMX_MTI_FCFECX_VL1_CCW_LO 0x38628 +#define RPMX_MTI_FCFECX_VL1_NCCW_LO 0x38630 +#define RPMX_MTI_FCFECX_CW_HI 0x38638 + +/* CN10KB CSR Declaration */ +#define RPM2_CMRX_SW_INT 0x1b0 +#define RPM2_CMRX_SW_INT_ENA_W1S 0x1b8 +#define RPM2_CMR_CHAN_MSK_OR 0x3120 +#define RPM2_CMR_RX_OVR_BP_EN BIT_ULL(2) +#define RPM2_CMR_RX_OVR_BP_BP BIT_ULL(1) +#define RPM2_CMR_RX_OVR_BP 0x3130 +#define RPM2_CSR_OFFSET 0x3e00 +#define RPM2_CMRX_PRT_CBFC_CTL 0x6510 +#define RPM2_CMRX_RX_LMACS 0x100 +#define RPM2_CMRX_RX_LOGL_XON 0x3100 +#define RPM2_CMRX_RX_STAT2 0x3010 +#define RPM2_USX_PCSX_CONTROL1 0x80000 +#define RPM2_USX_PCS_LBK BIT_ULL(14) /* Function Declarations */ int rpm_get_nr_lmacs(void *rpmd); u8 rpm_get_lmac_type(void *rpmd, int lmac_id); u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id); +u32 rpm2_get_lmac_fifo_len(void *rpmd, int lmac_id); int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable); void rpm_lmac_enadis_rx_pause_fwding(void *rpmd, int lmac_id, bool enable); int rpm_lmac_get_pause_frm_status(void *cgxd, int lmac_id, u8 *tx_pause, @@ -97,4 +128,7 @@ int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en); int rpm_lmac_get_pfc_frm_cfg(void *rpmd, int lmac_id, u8 *tx_pause, u8 *rx_pause); +int rpm2_get_nr_lmacs(void *rpmd); +bool is_dev_rpm2(void *rpmd); +int rpm_get_fec_stats(void *cgxd, int lmac_id, struct cgx_fec_stats_rsp *rsp); #endif /* RPM_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index f718cbd32a94..7f0a64731c67 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -410,9 +410,15 @@ struct rvu_fwdata { u32 ptp_ext_tstamp; #define FWDATA_RESERVED_MEM 1022 u64 reserved[FWDATA_RESERVED_MEM]; -#define CGX_MAX 5 +#define CGX_MAX 9 #define CGX_LMACS_MAX 4 - struct cgx_lmac_fwdata_s cgx_fw_data[CGX_MAX][CGX_LMACS_MAX]; +#define CGX_LMACS_USX 8 + union { + struct cgx_lmac_fwdata_s + cgx_fw_data[CGX_MAX][CGX_LMACS_MAX]; + struct cgx_lmac_fwdata_s + cgx_fw_data_usx[CGX_MAX][CGX_LMACS_USX]; + }; /* Do not add new fields below this line */ }; @@ -478,7 +484,7 @@ struct rvu { u8 cgx_mapped_pfs; u8 cgx_cnt_max; /* CGX port count max */ u8 *pf2cgxlmac_map; /* pf to cgx_lmac map */ - u16 *cgxlmac2pf_map; /* bitmap of mapped pfs for + u64 *cgxlmac2pf_map; /* bitmap of mapped pfs for * every cgx lmac port */ unsigned long pf_notify_bmap; /* Flags for PF notification */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c index addc69f4b65c..438b212fb54a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c @@ -55,8 +55,9 @@ bool is_mac_feature_supported(struct rvu *rvu, int pf, int feature) return (cgx_features_get(cgxd) & feature); } +#define CGX_OFFSET(x) ((x) * rvu->hw->lmac_per_cgx) /* Returns bitmap of mapped PFs */ -static u16 cgxlmac_to_pfmap(struct rvu *rvu, u8 cgx_id, u8 lmac_id) +static u64 cgxlmac_to_pfmap(struct rvu *rvu, u8 cgx_id, u8 lmac_id) { return rvu->cgxlmac2pf_map[CGX_OFFSET(cgx_id) + lmac_id]; } @@ -71,7 +72,8 @@ int cgxlmac_to_pf(struct rvu *rvu, int cgx_id, int lmac_id) if (!pfmap) return -ENODEV; else - return find_first_bit(&pfmap, 16); + return find_first_bit(&pfmap, + rvu->cgx_cnt_max * rvu->hw->lmac_per_cgx); } static u8 cgxlmac_id_to_bmap(u8 cgx_id, u8 lmac_id) @@ -129,14 +131,14 @@ static int rvu_map_cgx_lmac_pf(struct rvu *rvu) if (!cgx_cnt_max) return 0; - if (cgx_cnt_max > 0xF || MAX_LMAC_PER_CGX > 0xF) + if (cgx_cnt_max > 0xF || rvu->hw->lmac_per_cgx > 0xF) return -EINVAL; /* Alloc map table * An additional entry is required since PF id starts from 1 and * hence entry at offset 0 is invalid. */ - size = (cgx_cnt_max * MAX_LMAC_PER_CGX + 1) * sizeof(u8); + size = (cgx_cnt_max * rvu->hw->lmac_per_cgx + 1) * sizeof(u8); rvu->pf2cgxlmac_map = devm_kmalloc(rvu->dev, size, GFP_KERNEL); if (!rvu->pf2cgxlmac_map) return -ENOMEM; @@ -145,9 +147,10 @@ static int rvu_map_cgx_lmac_pf(struct rvu *rvu) memset(rvu->pf2cgxlmac_map, 0xFF, size); /* Reverse map table */ - rvu->cgxlmac2pf_map = devm_kzalloc(rvu->dev, - cgx_cnt_max * MAX_LMAC_PER_CGX * sizeof(u16), - GFP_KERNEL); + rvu->cgxlmac2pf_map = + devm_kzalloc(rvu->dev, + cgx_cnt_max * rvu->hw->lmac_per_cgx * sizeof(u64), + GFP_KERNEL); if (!rvu->cgxlmac2pf_map) return -ENOMEM; @@ -156,7 +159,7 @@ static int rvu_map_cgx_lmac_pf(struct rvu *rvu) if (!rvu_cgx_pdata(cgx, rvu)) continue; lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); - for_each_set_bit(iter, &lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) { lmac = cgx_get_lmacid(rvu_cgx_pdata(cgx, rvu), iter); rvu->pf2cgxlmac_map[pf] = cgxlmac_id_to_bmap(cgx, lmac); @@ -235,7 +238,8 @@ static void cgx_notify_pfs(struct cgx_link_event *event, struct rvu *rvu) pfmap = cgxlmac_to_pfmap(rvu, event->cgx_id, event->lmac_id); do { - pfid = find_first_bit(&pfmap, 16); + pfid = find_first_bit(&pfmap, + rvu->cgx_cnt_max * rvu->hw->lmac_per_cgx); clear_bit(pfid, &pfmap); /* check if notification is enabled */ @@ -310,7 +314,7 @@ static int cgx_lmac_event_handler_init(struct rvu *rvu) if (!cgxd) continue; lmac_bmap = cgx_get_lmac_bmap(cgxd); - for_each_set_bit(lmac, &lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(lmac, &lmac_bmap, rvu->hw->lmac_per_cgx) { err = cgx_lmac_evh_register(&cb, cgxd, lmac); if (err) dev_err(rvu->dev, @@ -396,7 +400,7 @@ int rvu_cgx_exit(struct rvu *rvu) if (!cgxd) continue; lmac_bmap = cgx_get_lmac_bmap(cgxd); - for_each_set_bit(lmac, &lmac_bmap, MAX_LMAC_PER_CGX) + for_each_set_bit(lmac, &lmac_bmap, rvu->hw->lmac_per_cgx) cgx_lmac_evh_unregister(cgxd, lmac); } @@ -468,6 +472,7 @@ void rvu_cgx_disable_dmac_entries(struct rvu *rvu, u16 pcifunc) { int pf = rvu_get_pf(pcifunc); int i = 0, lmac_count = 0; + struct mac_ops *mac_ops; u8 max_dmac_filters; u8 cgx_id, lmac_id; void *cgx_dev; @@ -483,7 +488,12 @@ void rvu_cgx_disable_dmac_entries(struct rvu *rvu, u16 pcifunc) rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); cgx_dev = cgx_get_pdata(cgx_id); lmac_count = cgx_get_lmac_cnt(cgx_dev); - max_dmac_filters = MAX_DMAC_ENTRIES_PER_CGX / lmac_count; + + mac_ops = get_mac_ops(cgx_dev); + if (!mac_ops) + return; + + max_dmac_filters = mac_ops->dmac_filter_count / lmac_count; for (i = 0; i < max_dmac_filters; i++) cgx_lmac_addr_del(cgx_id, lmac_id, i); @@ -569,6 +579,7 @@ int rvu_mbox_handler_cgx_fec_stats(struct rvu *rvu, struct cgx_fec_stats_rsp *rsp) { int pf = rvu_get_pf(req->hdr.pcifunc); + struct mac_ops *mac_ops; u8 cgx_idx, lmac; void *cgxd; @@ -577,7 +588,8 @@ int rvu_mbox_handler_cgx_fec_stats(struct rvu *rvu, rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_idx, &lmac); cgxd = rvu_cgx_pdata(cgx_idx, rvu); - return cgx_get_fec_stats(cgxd, lmac, rsp); + mac_ops = get_mac_ops(cgxd); + return mac_ops->get_fec_stats(cgxd, lmac, rsp); } int rvu_mbox_handler_cgx_mac_addr_set(struct rvu *rvu, @@ -1110,8 +1122,15 @@ int rvu_mbox_handler_cgx_get_aux_link_info(struct rvu *rvu, struct msg_req *req, rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); - memcpy(&rsp->fwdata, &rvu->fwdata->cgx_fw_data[cgx_id][lmac_id], - sizeof(struct cgx_lmac_fwdata_s)); + if (rvu->hw->lmac_per_cgx == CGX_LMACS_USX) + memcpy(&rsp->fwdata, + &rvu->fwdata->cgx_fw_data_usx[cgx_id][lmac_id], + sizeof(struct cgx_lmac_fwdata_s)); + else + memcpy(&rsp->fwdata, + &rvu->fwdata->cgx_fw_data[cgx_id][lmac_id], + sizeof(struct cgx_lmac_fwdata_s)); + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 0eb3085c4c21..fa280ebd3052 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -2613,7 +2613,7 @@ static void rvu_dbg_cgx_init(struct rvu *rvu) rvu->rvu_dbg.cgx = debugfs_create_dir(dname, rvu->rvu_dbg.cgx_root); - for_each_set_bit(lmac_id, &lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(lmac_id, &lmac_bmap, rvu->hw->lmac_per_cgx) { /* lmac debugfs dir */ sprintf(dname, "lmac%d", lmac_id); rvu->rvu_dbg.lmac = diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 88dee589cb21..bda1a6fa2ec4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -1547,14 +1547,7 @@ static int rvu_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, return 0; } -static int rvu_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, - struct netlink_ext_ack *extack) -{ - return devlink_info_driver_name_put(req, DRV_NAME); -} - static const struct devlink_ops rvu_devlink_ops = { - .info_get = rvu_devlink_info_get, .eswitch_mode_get = rvu_devlink_eswitch_mode_get, .eswitch_mode_set = rvu_devlink_eswitch_mode_set, }; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index a62c1b322012..6b8747ebc08c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -3197,8 +3197,12 @@ static void rvu_get_lbk_link_max_frs(struct rvu *rvu, u16 *max_mtu) static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu) { - /* RPM supports FIFO len 128 KB */ - if (rvu_cgx_get_fifolen(rvu) == 0x20000) + int fifo_size = rvu_cgx_get_fifolen(rvu); + + /* RPM supports FIFO len 128 KB and RPM2 supports double the + * FIFO len to accommodate 8 LMACS + */ + if (fifo_size == 0x20000 || fifo_size == 0x40000) *max_mtu = CN10K_LMAC_LINK_MAX_FRS; else *max_mtu = NIC_HW_MAX_FRS; @@ -4109,7 +4113,7 @@ static void nix_link_config(struct rvu *rvu, int blkaddr, /* Get LMAC id's from bitmap */ lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); - for_each_set_bit(iter, &lmac_bmap, MAX_LMAC_PER_CGX) { + for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) { lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); if (!lmac_fifo_len) { dev_err(rvu->dev, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c index 594029007f85..f69102d20c90 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c @@ -490,7 +490,7 @@ static bool rvu_npc_exact_alloc_id(struct rvu *rvu, u32 *seq_id) if (idx == table->tot_ids) { mutex_unlock(&table->lock); dev_err(rvu->dev, "%s: No space in id bitmap (%d)\n", - __func__, bitmap_weight(table->id_bmap, table->tot_ids)); + __func__, table->tot_ids); return false; } @@ -1870,12 +1870,11 @@ int rvu_npc_exact_init(struct rvu *rvu) /* Set capability to true */ rvu->hw->cap.npc_exact_match_enabled = true; - table = kmalloc(sizeof(*table), GFP_KERNEL); + table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) return -ENOMEM; dev_dbg(rvu->dev, "%s: Memory allocation for table success\n", __func__); - memset(table, 0, sizeof(*table)); rvu->hw->table = table; /* Read table size, ways and depth */ @@ -1899,24 +1898,24 @@ int rvu_npc_exact_init(struct rvu *rvu) table_size = table->mem_table.depth * table->mem_table.ways; /* Allocate bitmap for 4way 2K table */ - table->mem_table.bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(table_size), - sizeof(long), GFP_KERNEL); + table->mem_table.bmap = devm_bitmap_zalloc(rvu->dev, table_size, + GFP_KERNEL); if (!table->mem_table.bmap) return -ENOMEM; dev_dbg(rvu->dev, "%s: Allocated bitmap for 4way 2K entry table\n", __func__); /* Allocate bitmap for 32 entry mcam */ - table->cam_table.bmap = devm_kcalloc(rvu->dev, 1, sizeof(long), GFP_KERNEL); + table->cam_table.bmap = devm_bitmap_zalloc(rvu->dev, 32, GFP_KERNEL); if (!table->cam_table.bmap) return -ENOMEM; dev_dbg(rvu->dev, "%s: Allocated bitmap for 32 entry cam\n", __func__); - table->tot_ids = (table->mem_table.depth * table->mem_table.ways) + table->cam_table.depth; - table->id_bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(table->tot_ids), - table->tot_ids, GFP_KERNEL); + table->tot_ids = table_size + table->cam_table.depth; + table->id_bmap = devm_bitmap_zalloc(rvu->dev, table->tot_ids, + GFP_KERNEL); if (!table->id_bmap) return -ENOMEM; @@ -1957,7 +1956,9 @@ int rvu_npc_exact_init(struct rvu *rvu) /* Install SDP drop rule */ drop_mcam_idx = &table->num_drop_rules; - max_lmac_cnt = rvu->cgx_cnt_max * MAX_LMAC_PER_CGX + PF_CGXMAP_BASE; + max_lmac_cnt = rvu->cgx_cnt_max * rvu->hw->lmac_per_cgx + + PF_CGXMAP_BASE; + for (i = PF_CGXMAP_BASE; i < max_lmac_cnt; i++) { if (rvu->pf2cgxlmac_map[i] == 0xFF) continue; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c index 777a27047c8e..63ef7c41d18d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_devlink.c @@ -77,22 +77,7 @@ static const struct devlink_param otx2_dl_params[] = { otx2_dl_mcam_count_validate), }; -/* Devlink OPs */ -static int otx2_devlink_info_get(struct devlink *devlink, - struct devlink_info_req *req, - struct netlink_ext_ack *extack) -{ - struct otx2_devlink *otx2_dl = devlink_priv(devlink); - struct otx2_nic *pfvf = otx2_dl->pfvf; - - if (is_otx2_vf(pfvf->pcifunc)) - return devlink_info_driver_name_put(req, "rvu_nicvf"); - - return devlink_info_driver_name_put(req, "rvu_nicpf"); -} - static const struct devlink_ops otx2_devlink_ops = { - .info_get = otx2_devlink_info_get, }; int otx2_register_dl(struct otx2_nic *pfvf) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 0eb74e8c553d..0f8d1a69139f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -1268,6 +1268,39 @@ end: return err; } +static void otx2_get_fec_stats(struct net_device *netdev, + struct ethtool_fec_stats *fec_stats) +{ + struct otx2_nic *pfvf = netdev_priv(netdev); + struct cgx_fw_data *rsp; + + otx2_update_lmac_fec_stats(pfvf); + + /* Report MAC FEC stats */ + fec_stats->corrected_blocks.total = pfvf->hw.cgx_fec_corr_blks; + fec_stats->uncorrectable_blocks.total = pfvf->hw.cgx_fec_uncorr_blks; + + rsp = otx2_get_fwdata(pfvf); + if (!IS_ERR(rsp) && rsp->fwdata.phy.misc.has_fec_stats && + !otx2_get_phy_fec_stats(pfvf)) { + /* Fetch fwdata again because it's been recently populated with + * latest PHY FEC stats. + */ + rsp = otx2_get_fwdata(pfvf); + if (!IS_ERR(rsp)) { + struct fec_stats_s *p = &rsp->fwdata.phy.fec_stats; + + if (pfvf->linfo.fec == OTX2_FEC_BASER) { + fec_stats->corrected_blocks.total = p->brfec_corr_blks; + fec_stats->uncorrectable_blocks.total = p->brfec_uncorr_blks; + } else { + fec_stats->corrected_blocks.total = p->rsfec_corr_cws; + fec_stats->uncorrectable_blocks.total = p->rsfec_uncorr_cws; + } + } + } +} + static const struct ethtool_ops otx2_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | @@ -1298,6 +1331,7 @@ static const struct ethtool_ops otx2_ethtool_ops = { .get_pauseparam = otx2_get_pauseparam, .set_pauseparam = otx2_set_pauseparam, .get_ts_info = otx2_get_ts_info, + .get_fec_stats = otx2_get_fec_stats, .get_fecparam = otx2_get_fecparam, .set_fecparam = otx2_set_fecparam, .get_link_ksettings = otx2_get_link_ksettings, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index e421714524c2..044cc211424e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -1159,7 +1159,12 @@ int otx2_init_tc(struct otx2_nic *nic) return err; tc->flow_ht_params = tc_flow_ht_params; - return rhashtable_init(&tc->flow_table, &tc->flow_ht_params); + err = rhashtable_init(&tc->flow_table, &tc->flow_ht_params); + if (err) { + kfree(tc->tc_entries_bitmap); + tc->tc_entries_bitmap = NULL; + } + return err; } EXPORT_SYMBOL(otx2_init_tc); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c index 84ad05c9f12d..2a4c9df4eb79 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c @@ -355,11 +355,6 @@ static int prestera_dl_info_get(struct devlink *dl, { struct prestera_switch *sw = devlink_priv(dl); char buf[16]; - int err; - - err = devlink_info_driver_name_put(req, PRESTERA_DRV_NAME); - if (err) - return err; snprintf(buf, sizeof(buf), "%d.%d.%d", sw->dev->fw_rev.maj, diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 8b93dab79141..e3de9a53b2d9 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -4593,6 +4593,7 @@ static const struct mtk_soc_data mt7986_data = { .hw_features = MTK_HW_FEATURES, .required_clks = MT7986_CLKS_BITMAP, .required_pctl = false, + .offload_version = 2, .hash_offset = 4, .foe_entry_size = sizeof(struct mtk_foe_entry), .txrx = { diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index d041615b2bac..a6271449617f 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -174,9 +174,10 @@ mtk_wed_wo_reset(struct mtk_wed_device *dev) mtk_wdma_tx_reset(dev); mtk_wed_reset(dev, MTK_WED_RESET_WED); - mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, - MTK_WED_WO_CMD_CHANGE_STATE, &state, - sizeof(state), false); + if (mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, + MTK_WED_WO_CMD_CHANGE_STATE, &state, + sizeof(state), false)) + return; if (readx_poll_timeout(mtk_wed_wo_read_status, dev, val, val == MTK_WED_WOIF_DISABLE_DONE, @@ -576,12 +577,10 @@ mtk_wed_deinit(struct mtk_wed_device *dev) } static void -mtk_wed_detach(struct mtk_wed_device *dev) +__mtk_wed_detach(struct mtk_wed_device *dev) { struct mtk_wed_hw *hw = dev->hw; - mutex_lock(&hw_lock); - mtk_wed_deinit(dev); mtk_wdma_rx_reset(dev); @@ -590,9 +589,11 @@ mtk_wed_detach(struct mtk_wed_device *dev) mtk_wed_free_tx_rings(dev); if (mtk_wed_get_rx_capa(dev)) { - mtk_wed_wo_reset(dev); + if (hw->wed_wo) + mtk_wed_wo_reset(dev); mtk_wed_free_rx_rings(dev); - mtk_wed_wo_deinit(hw); + if (hw->wed_wo) + mtk_wed_wo_deinit(hw); } if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { @@ -612,6 +613,13 @@ mtk_wed_detach(struct mtk_wed_device *dev) module_put(THIS_MODULE); hw->wed_dev = NULL; +} + +static void +mtk_wed_detach(struct mtk_wed_device *dev) +{ + mutex_lock(&hw_lock); + __mtk_wed_detach(dev); mutex_unlock(&hw_lock); } @@ -1210,7 +1218,8 @@ mtk_wed_wdma_rx_ring_setup(struct mtk_wed_device *dev, int idx, int size, } static int -mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size) +mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size, + bool reset) { u32 desc_size = sizeof(struct mtk_wdma_desc) * dev->hw->version; struct mtk_wed_ring *wdma; @@ -1219,8 +1228,8 @@ mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size) return -EINVAL; wdma = &dev->tx_wdma[idx]; - if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, desc_size, - true)) + if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, + desc_size, true)) return -ENOMEM; wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE, @@ -1230,6 +1239,9 @@ mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size) wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0); + if (reset) + mtk_wed_ring_reset(wdma, MTK_WED_WDMA_RING_SIZE, true); + if (!idx) { wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_BASE, wdma->desc_phys); @@ -1490,8 +1502,10 @@ mtk_wed_attach(struct mtk_wed_device *dev) ret = mtk_wed_wo_init(hw); } out: - if (ret) - mtk_wed_detach(dev); + if (ret) { + dev_err(dev->hw->dev, "failed to attach wed device\n"); + __mtk_wed_detach(dev); + } unlock: mutex_unlock(&hw_lock); @@ -1569,18 +1583,20 @@ mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) } static int -mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs) +mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs, + bool reset) { struct mtk_wed_ring *ring = &dev->rx_ring[idx]; if (WARN_ON(idx >= ARRAY_SIZE(dev->rx_ring))) return -EINVAL; - if (mtk_wed_ring_alloc(dev, ring, MTK_WED_RX_RING_SIZE, - sizeof(*ring->desc), false)) + if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_RX_RING_SIZE, + sizeof(*ring->desc), false)) return -ENOMEM; - if (mtk_wed_wdma_tx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE)) + if (mtk_wed_wdma_tx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE, + reset)) return -ENOMEM; ring->reg_base = MTK_WED_RING_RX_DATA(idx); diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c index f9539e6233c9..6bad0d262f28 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c @@ -207,6 +207,9 @@ int mtk_wed_mcu_msg_update(struct mtk_wed_device *dev, int id, void *data, if (dev->hw->version == 1) return 0; + if (WARN_ON(!wo)) + return -ENODEV; + return mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, id, data, len, true); } diff --git a/drivers/net/ethernet/mediatek/mtk_wed_wo.c b/drivers/net/ethernet/mediatek/mtk_wed_wo.c index 4754b6db009e..a0a39643caf7 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_wo.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_wo.c @@ -133,17 +133,18 @@ mtk_wed_wo_dequeue(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q, u32 *len, static int mtk_wed_wo_queue_refill(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q, - gfp_t gfp, bool rx) + bool rx) { enum dma_data_direction dir = rx ? DMA_FROM_DEVICE : DMA_TO_DEVICE; int n_buf = 0; spin_lock_bh(&q->lock); while (q->queued < q->n_desc) { - void *buf = page_frag_alloc(&q->cache, q->buf_size, gfp); struct mtk_wed_wo_queue_entry *entry; dma_addr_t addr; + void *buf; + buf = page_frag_alloc(&q->cache, q->buf_size, GFP_ATOMIC); if (!buf) break; @@ -215,7 +216,7 @@ mtk_wed_wo_rx_run_queue(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) mtk_wed_mcu_rx_unsolicited_event(wo, skb); } - if (mtk_wed_wo_queue_refill(wo, q, GFP_ATOMIC, true)) { + if (mtk_wed_wo_queue_refill(wo, q, true)) { u32 index = (q->head - 1) % q->n_desc; mtk_wed_wo_queue_kick(wo, q, index); @@ -407,8 +408,10 @@ mtk_wed_wo_hardware_init(struct mtk_wed_wo *wo) return -ENODEV; wo->mmio.regs = syscon_regmap_lookup_by_phandle(np, NULL); - if (IS_ERR_OR_NULL(wo->mmio.regs)) - return PTR_ERR(wo->mmio.regs); + if (IS_ERR(wo->mmio.regs)) { + ret = PTR_ERR(wo->mmio.regs); + goto error_put; + } wo->mmio.irq = irq_of_parse_and_map(np, 0); wo->mmio.irq_mask = MTK_WED_WO_ALL_INT_MASK; @@ -432,7 +435,7 @@ mtk_wed_wo_hardware_init(struct mtk_wed_wo *wo) if (ret) goto error; - mtk_wed_wo_queue_refill(wo, &wo->q_tx, GFP_KERNEL, false); + mtk_wed_wo_queue_refill(wo, &wo->q_tx, false); mtk_wed_wo_queue_reset(wo, &wo->q_tx); regs.desc_base = MTK_WED_WO_CCIF_DUMMY5; @@ -446,7 +449,7 @@ mtk_wed_wo_hardware_init(struct mtk_wed_wo *wo) if (ret) goto error; - mtk_wed_wo_queue_refill(wo, &wo->q_rx, GFP_KERNEL, true); + mtk_wed_wo_queue_refill(wo, &wo->q_rx, true); mtk_wed_wo_queue_reset(wo, &wo->q_rx); /* rx queue irqmask */ @@ -456,7 +459,8 @@ mtk_wed_wo_hardware_init(struct mtk_wed_wo *wo) error: devm_free_irq(wo->hw->dev, wo->mmio.irq, wo); - +error_put: + of_node_put(np); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 43a4102e9c09..c5758637b7be 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -65,7 +65,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, ring->size = size; ring->size_mask = size - 1; ring->sp_stride = stride; - ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS; + ring->full_size = ring->size - HEADROOM - MLX4_MAX_DESC_TXBBS; tmp = size * sizeof(struct mlx4_en_tx_info); ring->tx_info = kvmalloc_node(tmp, GFP_KERNEL, node); @@ -77,9 +77,11 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", ring->tx_info, tmp); - ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node); + ring->bounce_buf = kmalloc_node(MLX4_TX_BOUNCE_BUFFER_SIZE, + GFP_KERNEL, node); if (!ring->bounce_buf) { - ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); + ring->bounce_buf = kmalloc(MLX4_TX_BOUNCE_BUFFER_SIZE, + GFP_KERNEL); if (!ring->bounce_buf) { err = -ENOMEM; goto err_info; @@ -909,11 +911,6 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) /* Align descriptor to TXBB size */ desc_size = ALIGN(real_size, TXBB_SIZE); nr_txbb = desc_size >> LOG_TXBB_SIZE; - if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { - if (netif_msg_tx_err(priv)) - en_warn(priv, "Oversized header or SG list\n"); - goto tx_drop_count; - } bf_ok = ring->bf_enabled; if (skb_vlan_tag_present(skb)) { @@ -941,6 +938,11 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) if (likely(index + nr_txbb <= ring->size)) tx_desc = ring->buf + (index << LOG_TXBB_SIZE); else { + if (unlikely(nr_txbb > MLX4_MAX_DESC_TXBBS)) { + if (netif_msg_tx_err(priv)) + en_warn(priv, "Oversized header or SG list\n"); + goto tx_drop_count; + } tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; bounce = true; bf_ok = false; diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index e132ff4c82f2..3d4226ddba5e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -89,9 +89,19 @@ #define MLX4_EN_FILTER_HASH_SHIFT 4 #define MLX4_EN_FILTER_EXPIRY_QUOTA 60 -/* Typical TSO descriptor with 16 gather entries is 352 bytes... */ -#define MAX_DESC_SIZE 512 -#define MAX_DESC_TXBBS (MAX_DESC_SIZE / TXBB_SIZE) +#define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg) +#define DS_SIZE sizeof(struct mlx4_wqe_data_seg) + +/* Maximal size of the bounce buffer: + * 256 bytes for LSO headers. + * CTRL_SIZE for control desc. + * DS_SIZE if skb->head contains some payload. + * MAX_SKB_FRAGS frags. + */ +#define MLX4_TX_BOUNCE_BUFFER_SIZE \ + ALIGN(256 + CTRL_SIZE + DS_SIZE + MAX_SKB_FRAGS * DS_SIZE, TXBB_SIZE) + +#define MLX4_MAX_DESC_TXBBS (MLX4_TX_BOUNCE_BUFFER_SIZE / TXBB_SIZE) /* * OS related constants and tunables @@ -217,9 +227,7 @@ struct mlx4_en_tx_info { #define MLX4_EN_BIT_DESC_OWN 0x80000000 -#define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg) #define MLX4_EN_MEMTYPE_PAD 0x100 -#define DS_SIZE sizeof(struct mlx4_wqe_data_seg) struct mlx4_en_tx_desc { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index a22c32aabf11..cd4a1ab0ea78 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -111,6 +111,7 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o steering/dr_ste_v2.o \ steering/dr_cmd.o steering/dr_fw.o \ steering/dr_action.o steering/fs_dr.o \ + steering/dr_definer.o \ steering/dr_dbg.o lib/smfs.o # # SF device diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index e7a894ba5c3e..d3ca745d107d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -37,7 +37,6 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/random.h> -#include <linux/io-mapping.h> #include <linux/mlx5/driver.h> #include <linux/mlx5/eq.h> #include <linux/debugfs.h> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index cc2ae427dcb0..ddb197970c22 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -46,10 +46,6 @@ mlx5_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, u32 running_fw, stored_fw; int err; - err = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (err) - return err; - err = devlink_info_version_fixed_put(req, "fw.psid", dev->board_id); if (err) return err; @@ -318,6 +314,10 @@ static const struct devlink_ops mlx5_devlink_ops = { .rate_node_new = mlx5_esw_devlink_rate_node_new, .rate_node_del = mlx5_esw_devlink_rate_node_del, .rate_leaf_parent_set = mlx5_esw_devlink_rate_parent_set, + .port_fn_roce_get = mlx5_devlink_port_fn_roce_get, + .port_fn_roce_set = mlx5_devlink_port_fn_roce_set, + .port_fn_migratable_get = mlx5_devlink_port_fn_migratable_get, + .port_fn_migratable_set = mlx5_devlink_port_fn_migratable_set, #endif #ifdef CONFIG_MLX5_SF_MANAGER .port_new = mlx5_devlink_sf_port_new, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c index c5bb79a4fa57..2732128e7a6e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c @@ -228,6 +228,17 @@ const char *parse_fs_hdrs(struct trace_seq *p, return ret; } +static const char +*fs_dest_range_field_to_str(enum mlx5_flow_dest_range_field field) +{ + switch (field) { + case MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN: + return "packet len"; + default: + return "unknown dest range field"; + } +} + const char *parse_fs_dst(struct trace_seq *p, const struct mlx5_flow_destination *dst, u32 counter_id) @@ -259,6 +270,11 @@ const char *parse_fs_dst(struct trace_seq *p, case MLX5_FLOW_DESTINATION_TYPE_PORT: trace_seq_printf(p, "port\n"); break; + case MLX5_FLOW_DESTINATION_TYPE_RANGE: + trace_seq_printf(p, "field=%s min=%d max=%d\n", + fs_dest_range_field_to_str(dst->range.field), + dst->range.min, dst->range.max); + break; case MLX5_FLOW_DESTINATION_TYPE_NONE: trace_seq_printf(p, "none\n"); break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ff5b302531d5..2d77fb8a8a01 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -103,11 +103,11 @@ struct page_pool; * size actually used at runtime, but it's not a problem when calculating static * array sizes. */ -#define MLX5_UMR_MAX_MTT_SPACE \ +#define MLX5_UMR_MAX_FLEX_SPACE \ (ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \ - MLX5_UMR_MTT_ALIGNMENT)) + MLX5_UMR_FLEX_ALIGNMENT)) #define MLX5_MPWRQ_MAX_PAGES_PER_WQE \ - rounddown_pow_of_two(MLX5_UMR_MAX_MTT_SPACE / sizeof(struct mlx5_mtt)) + rounddown_pow_of_two(MLX5_UMR_MAX_FLEX_SPACE / sizeof(struct mlx5_mtt)) #define MLX5E_MAX_RQ_NUM_MTTS \ (ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */ @@ -160,7 +160,7 @@ struct page_pool; (((wqe_size) - sizeof(struct mlx5e_umr_wqe)) / sizeof(struct mlx5_klm)) #define MLX5E_KLM_ENTRIES_PER_WQE(wqe_size)\ - ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_ALIGNMENT) + ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT) #define MLX5E_MAX_KLM_PER_WQE(mdev) \ MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * mlx5e_get_max_sq_aligned_wqebbs(mdev)) @@ -1245,4 +1245,5 @@ int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_t int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi); int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats); #endif +int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index bf2741eb7f9b..379c6dc9a3be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -84,7 +84,8 @@ enum { MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, #endif #ifdef CONFIG_MLX5_EN_IPSEC - MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_POL_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, #endif }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 9ec9662d1d0b..585bdc8383ee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -107,7 +107,7 @@ u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), - MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size; + MLX5_UMR_FLEX_ALIGNMENT) / umr_entry_size; max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); @@ -146,7 +146,7 @@ u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, u16 umr_wqe_sz; umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) + - ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT); + ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c index 21aab96357b5..a278f52d52b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c @@ -28,4 +28,5 @@ tc_act_parse_accept(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5e_tc_act mlx5e_tc_act_accept = { .can_offload = tc_act_can_offload_accept, .parse_action = tc_act_parse_accept, + .is_terminating_action = true, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c index 3337241cfd84..eba0c8698926 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c @@ -11,7 +11,7 @@ static struct mlx5e_tc_act *tc_acts_fdb[NUM_FLOW_ACTIONS] = { [FLOW_ACTION_DROP] = &mlx5e_tc_act_drop, [FLOW_ACTION_TRAP] = &mlx5e_tc_act_trap, [FLOW_ACTION_GOTO] = &mlx5e_tc_act_goto, - [FLOW_ACTION_REDIRECT] = &mlx5e_tc_act_mirred, + [FLOW_ACTION_REDIRECT] = &mlx5e_tc_act_redirect, [FLOW_ACTION_MIRRED] = &mlx5e_tc_act_mirred, [FLOW_ACTION_REDIRECT_INGRESS] = &mlx5e_tc_act_redirect_ingress, [FLOW_ACTION_VLAN_PUSH] = &mlx5e_tc_act_vlan, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h index e1570ff056ae..8346557eeaf6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h @@ -32,6 +32,11 @@ struct mlx5e_tc_act_parse_state { struct mlx5_tc_ct_priv *ct_priv; }; +struct mlx5e_tc_act_branch_ctrl { + enum flow_action_id act_id; + u32 extval; +}; + struct mlx5e_tc_act { bool (*can_offload)(struct mlx5e_tc_act_parse_state *parse_state, const struct flow_action_entry *act, @@ -60,6 +65,12 @@ struct mlx5e_tc_act { int (*stats_action)(struct mlx5e_priv *priv, struct flow_offload_action *fl_act); + + bool (*get_branch_ctrl)(const struct flow_action_entry *act, + struct mlx5e_tc_act_branch_ctrl *cond_true, + struct mlx5e_tc_act_branch_ctrl *cond_false); + + bool is_terminating_action; }; struct mlx5e_tc_flow_action { @@ -81,6 +92,7 @@ extern struct mlx5e_tc_act mlx5e_tc_act_vlan_mangle; extern struct mlx5e_tc_act mlx5e_tc_act_mpls_push; extern struct mlx5e_tc_act mlx5e_tc_act_mpls_pop; extern struct mlx5e_tc_act mlx5e_tc_act_mirred; +extern struct mlx5e_tc_act mlx5e_tc_act_redirect; extern struct mlx5e_tc_act mlx5e_tc_act_mirred_nic; extern struct mlx5e_tc_act mlx5e_tc_act_ct; extern struct mlx5e_tc_act mlx5e_tc_act_sample; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c index dd025a95c439..7d16aeabb119 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c @@ -27,4 +27,5 @@ tc_act_parse_drop(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5e_tc_act mlx5e_tc_act_drop = { .can_offload = tc_act_can_offload_drop, .parse_action = tc_act_parse_drop, + .is_terminating_action = true, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c index 25174f68613e..0923e6db2d0a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c @@ -121,4 +121,5 @@ struct mlx5e_tc_act mlx5e_tc_act_goto = { .can_offload = tc_act_can_offload_goto, .parse_action = tc_act_parse_goto, .post_parse = tc_act_post_parse_goto, + .is_terminating_action = true, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c index 4ac7de3f6afa..78c427b38048 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c @@ -334,4 +334,11 @@ tc_act_parse_mirred(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5e_tc_act mlx5e_tc_act_mirred = { .can_offload = tc_act_can_offload_mirred, .parse_action = tc_act_parse_mirred, + .is_terminating_action = false, +}; + +struct mlx5e_tc_act mlx5e_tc_act_redirect = { + .can_offload = tc_act_can_offload_mirred, + .parse_action = tc_act_parse_mirred, + .is_terminating_action = true, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred_nic.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred_nic.c index 90b4c1b34776..7f409692b18f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred_nic.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred_nic.c @@ -48,4 +48,5 @@ tc_act_parse_mirred_nic(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5e_tc_act mlx5e_tc_act_mirred_nic = { .can_offload = tc_act_can_offload_mirred_nic, .parse_action = tc_act_parse_mirred_nic, + .is_terminating_action = true, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c index c8e5ca65bb6e..512d43148922 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c @@ -3,6 +3,45 @@ #include "act.h" #include "en/tc_priv.h" +#include "fs_core.h" + +static bool police_act_validate_control(enum flow_action_id act_id, + struct netlink_ext_ack *extack) +{ + if (act_id != FLOW_ACTION_PIPE && + act_id != FLOW_ACTION_ACCEPT && + act_id != FLOW_ACTION_JUMP && + act_id != FLOW_ACTION_DROP) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform-exceed action is not pipe, ok, jump or drop"); + return false; + } + + return true; +} + +static int police_act_validate(const struct flow_action_entry *act, + struct netlink_ext_ack *extack) +{ + if (!police_act_validate_control(act->police.exceed.act_id, extack) || + !police_act_validate_control(act->police.notexceed.act_id, extack)) + return -EOPNOTSUPP; + + if (act->police.peakrate_bytes_ps || + act->police.avrate || act->police.overhead) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when peakrate/avrate/overhead is configured"); + return -EOPNOTSUPP; + } + + if (act->police.rate_pkt_ps) { + NL_SET_ERR_MSG_MOD(extack, + "QoS offload not support packets per second"); + return -EOPNOTSUPP; + } + + return 0; +} static bool tc_act_can_offload_police(struct mlx5e_tc_act_parse_state *parse_state, @@ -10,14 +49,10 @@ tc_act_can_offload_police(struct mlx5e_tc_act_parse_state *parse_state, int act_index, struct mlx5_flow_attr *attr) { - if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && - act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { - NL_SET_ERR_MSG_MOD(parse_state->extack, - "Offload not supported when conform action is not pipe or ok"); - return false; - } - if (mlx5e_policer_validate(parse_state->flow_action, act, - parse_state->extack)) + int err; + + err = police_act_validate(act, parse_state->extack); + if (err) return false; return !!mlx5e_get_flow_meters(parse_state->flow->priv->mdev); @@ -37,6 +72,8 @@ fill_meter_params_from_act(const struct flow_action_entry *act, params->mode = MLX5_RATE_LIMIT_PPS; params->rate = act->police.rate_pkt_ps; params->burst = act->police.burst_pkt; + } else if (act->police.mtu) { + params->mtu = act->police.mtu; } else { return -EOPNOTSUPP; } @@ -50,14 +87,25 @@ tc_act_parse_police(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5e_priv *priv, struct mlx5_flow_attr *attr) { + enum mlx5_flow_namespace_type ns = mlx5e_get_flow_namespace(parse_state->flow); + struct mlx5e_flow_meter_params *params = &attr->meter_attr.params; int err; - err = fill_meter_params_from_act(act, &attr->meter_attr.params); + err = fill_meter_params_from_act(act, params); if (err) return err; - attr->action |= MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO; - attr->exe_aso_type = MLX5_EXE_ASO_FLOW_METER; + if (params->mtu) { + if (!(mlx5_fs_get_capabilities(priv->mdev, ns) & + MLX5_FLOW_STEERING_CAP_MATCH_RANGES)) + return -EOPNOTSUPP; + + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + attr->flags |= MLX5_ATTR_FLAG_MTU; + } else { + attr->action |= MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO; + attr->exe_aso_type = MLX5_EXE_ASO_FLOW_METER; + } return 0; } @@ -79,7 +127,7 @@ tc_act_police_offload(struct mlx5e_priv *priv, struct mlx5e_flow_meter_handle *meter; int err = 0; - err = mlx5e_policer_validate(&fl_act->action, act, fl_act->extack); + err = police_act_validate(act, fl_act->extack); if (err) return err; @@ -147,6 +195,19 @@ tc_act_police_stats(struct mlx5e_priv *priv, return 0; } +static bool +tc_act_police_get_branch_ctrl(const struct flow_action_entry *act, + struct mlx5e_tc_act_branch_ctrl *cond_true, + struct mlx5e_tc_act_branch_ctrl *cond_false) +{ + cond_true->act_id = act->police.notexceed.act_id; + cond_true->extval = act->police.notexceed.extval; + + cond_false->act_id = act->police.exceed.act_id; + cond_false->extval = act->police.exceed.extval; + return true; +} + struct mlx5e_tc_act mlx5e_tc_act_police = { .can_offload = tc_act_can_offload_police, .parse_action = tc_act_parse_police, @@ -154,4 +215,5 @@ struct mlx5e_tc_act mlx5e_tc_act_police = { .offload_action = tc_act_police_offload, .destroy_action = tc_act_police_destroy, .stats_action = tc_act_police_stats, + .get_branch_ctrl = tc_act_police_get_branch_ctrl, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c index 53b270f652b9..915ce201aeb2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c @@ -3,6 +3,7 @@ #include "act.h" #include "en/tc_priv.h" +#include "eswitch.h" static bool tc_act_can_offload_trap(struct mlx5e_tc_act_parse_state *parse_state, @@ -10,13 +11,6 @@ tc_act_can_offload_trap(struct mlx5e_tc_act_parse_state *parse_state, int act_index, struct mlx5_flow_attr *attr) { - struct netlink_ext_ack *extack = parse_state->extack; - - if (parse_state->flow_action->num_entries != 1) { - NL_SET_ERR_MSG_MOD(extack, "action trap is supported as a sole action only"); - return false; - } - return true; } @@ -27,7 +21,7 @@ tc_act_parse_trap(struct mlx5e_tc_act_parse_state *parse_state, struct mlx5_flow_attr *attr) { attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - attr->flags |= MLX5_ATTR_FLAG_SLOW_PATH; + attr->dest_ft = mlx5_eswitch_get_slow_fdb(priv->mdev->priv.eswitch); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index be74e1403328..78af8a3175bf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -162,7 +162,6 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); aso_ctrl = &aso_wqe->aso_ctrl; - memset(aso_ctrl, 0, sizeof(*aso_ctrl)); aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE << 4; @@ -241,7 +240,7 @@ mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id) } static struct mlx5e_flow_meter_handle * -__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) +__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters, bool alloc_aso) { struct mlx5_core_dev *mdev = flow_meters->mdev; struct mlx5e_flow_meter_aso_obj *meters_obj; @@ -257,16 +256,19 @@ __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) counter = mlx5_fc_create(mdev, true); if (IS_ERR(counter)) { err = PTR_ERR(counter); - goto err_red_counter; + goto err_drop_counter; } - meter->red_counter = counter; + meter->drop_counter = counter; counter = mlx5_fc_create(mdev, true); if (IS_ERR(counter)) { err = PTR_ERR(counter); - goto err_green_counter; + goto err_act_counter; } - meter->green_counter = counter; + meter->act_counter = counter; + + if (!alloc_aso) + goto no_aso; meters_obj = list_first_entry_or_null(&flow_meters->partial_list, struct mlx5e_flow_meter_aso_obj, @@ -300,11 +302,12 @@ __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) } bitmap_set(meters_obj->meters_map, pos, 1); - meter->flow_meters = flow_meters; meter->meters_obj = meters_obj; meter->obj_id = meters_obj->base_id + pos / 2; meter->idx = pos % 2; +no_aso: + meter->flow_meters = flow_meters; mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n", meter->obj_id, meter->idx); @@ -313,10 +316,10 @@ __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) err_mem: mlx5e_flow_meter_destroy_aso_obj(mdev, id); err_create: - mlx5_fc_destroy(mdev, meter->green_counter); -err_green_counter: - mlx5_fc_destroy(mdev, meter->red_counter); -err_red_counter: + mlx5_fc_destroy(mdev, meter->act_counter); +err_act_counter: + mlx5_fc_destroy(mdev, meter->drop_counter); +err_drop_counter: kfree(meter); return ERR_PTR(err); } @@ -329,8 +332,11 @@ __mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) struct mlx5e_flow_meter_aso_obj *meters_obj; int n, pos; - mlx5_fc_destroy(mdev, meter->green_counter); - mlx5_fc_destroy(mdev, meter->red_counter); + mlx5_fc_destroy(mdev, meter->act_counter); + mlx5_fc_destroy(mdev, meter->drop_counter); + + if (meter->params.mtu) + goto out_no_aso; meters_obj = meter->meters_obj; pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx; @@ -345,6 +351,7 @@ __mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) list_add(&meters_obj->entry, &flow_meters->partial_list); } +out_no_aso: mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n", meter->obj_id, meter->idx); kfree(meter); @@ -409,12 +416,13 @@ mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters, { struct mlx5e_flow_meter_handle *meter; - meter = __mlx5e_flow_meter_alloc(flow_meters); + meter = __mlx5e_flow_meter_alloc(flow_meters, !params->mtu); if (IS_ERR(meter)) return meter; hash_add(flow_meters->hashtbl, &meter->hlist, params->index); meter->params.index = params->index; + meter->params.mtu = params->mtu; meter->refcnt++; return meter; @@ -575,8 +583,8 @@ mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter, u64 bytes1, packets1, lastuse1; u64 bytes2, packets2, lastuse2; - mlx5_fc_query_cached(meter->green_counter, &bytes1, &packets1, &lastuse1); - mlx5_fc_query_cached(meter->red_counter, &bytes2, &packets2, &lastuse2); + mlx5_fc_query_cached(meter->act_counter, &bytes1, &packets1, &lastuse1); + mlx5_fc_query_cached(meter->drop_counter, &bytes2, &packets2, &lastuse2); *bytes = bytes1 + bytes2; *packets = packets1 + packets2; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h index 6de6e8a16327..9b795cd106bb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h @@ -20,6 +20,7 @@ struct mlx5e_flow_meter_params { u32 index; u64 rate; u64 burst; + u32 mtu; }; struct mlx5e_flow_meter_handle { @@ -32,8 +33,8 @@ struct mlx5e_flow_meter_handle { struct hlist_node hlist; struct mlx5e_flow_meter_params params; - struct mlx5_fc *green_counter; - struct mlx5_fc *red_counter; + struct mlx5_fc *act_counter; + struct mlx5_fc *drop_counter; }; struct mlx5e_meter_attr { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c index 8b77e822810e..8d7d761482d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c @@ -8,23 +8,56 @@ #define MLX5_PACKET_COLOR_BITS MLX5_REG_MAPPING_MBITS(PACKET_COLOR_TO_REG) #define MLX5_PACKET_COLOR_MASK MLX5_REG_MAPPING_MASK(PACKET_COLOR_TO_REG) -struct mlx5e_post_meter_priv { +struct mlx5e_post_meter_rate_table { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_flow_handle *green_rule; + struct mlx5_flow_attr *green_attr; + struct mlx5_flow_handle *red_rule; + struct mlx5_flow_attr *red_attr; +}; + +struct mlx5e_post_meter_mtu_table { struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; - struct mlx5_flow_handle *fwd_green_rule; - struct mlx5_flow_handle *drop_red_rule; + struct mlx5_flow_handle *rule; + struct mlx5_flow_attr *attr; +}; + +struct mlx5e_post_meter_mtu_tables { + struct mlx5e_post_meter_mtu_table green_table; + struct mlx5e_post_meter_mtu_table red_table; +}; + +struct mlx5e_post_meter_priv { + enum mlx5e_post_meter_type type; + union { + struct mlx5e_post_meter_rate_table rate_steering_table; + struct mlx5e_post_meter_mtu_tables mtu_tables; + }; }; struct mlx5_flow_table * mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter) { - return post_meter->ft; + return post_meter->rate_steering_table.ft; } -static int +struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_true_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return post_meter->mtu_tables.green_table.ft; +} + +struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_false_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return post_meter->mtu_tables.red_table.ft; +} + +static struct mlx5_flow_table * mlx5e_post_meter_table_create(struct mlx5e_priv *priv, - enum mlx5_flow_namespace_type ns_type, - struct mlx5e_post_meter_priv *post_meter) + enum mlx5_flow_namespace_type ns_type) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_namespace *root_ns; @@ -32,7 +65,7 @@ mlx5e_post_meter_table_create(struct mlx5e_priv *priv, root_ns = mlx5_get_flow_namespace(priv->mdev, ns_type); if (!root_ns) { mlx5_core_warn(priv->mdev, "Failed to get namespace for flow meter\n"); - return -EOPNOTSUPP; + return ERR_PTR(-EOPNOTSUPP); } ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; @@ -40,19 +73,14 @@ mlx5e_post_meter_table_create(struct mlx5e_priv *priv, ft_attr.max_fte = 2; ft_attr.level = 1; - post_meter->ft = mlx5_create_flow_table(root_ns, &ft_attr); - if (IS_ERR(post_meter->ft)) { - mlx5_core_warn(priv->mdev, "Failed to create post_meter table\n"); - return PTR_ERR(post_meter->ft); - } - - return 0; + return mlx5_create_flow_table(root_ns, &ft_attr); } static int -mlx5e_post_meter_fg_create(struct mlx5e_priv *priv, - struct mlx5e_post_meter_priv *post_meter) +mlx5e_post_meter_rate_fg_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter) { + struct mlx5e_post_meter_rate_table *table = &post_meter->rate_steering_table; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); void *misc2, *match_criteria; u32 *flow_group_in; @@ -71,25 +99,58 @@ mlx5e_post_meter_fg_create(struct mlx5e_priv *priv, MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); - post_meter->fg = mlx5_create_flow_group(post_meter->ft, flow_group_in); - if (IS_ERR(post_meter->fg)) { + table->fg = mlx5_create_flow_group(table->ft, flow_group_in); + if (IS_ERR(table->fg)) { mlx5_core_warn(priv->mdev, "Failed to create post_meter flow group\n"); - err = PTR_ERR(post_meter->fg); + err = PTR_ERR(table->fg); } kvfree(flow_group_in); return err; } +static struct mlx5_flow_handle * +mlx5e_post_meter_add_rule(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5_flow_spec *spec, + struct mlx5_flow_attr *attr, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter) +{ + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + struct mlx5_flow_handle *ret; + + attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_DROP) + attr->counter = drop_counter; + else + attr->counter = act_counter; + + attr->flags |= MLX5_ATTR_FLAG_NO_IN_PORT; + attr->outer_match_level = MLX5_MATCH_NONE; + attr->chain = 0; + attr->prio = 0; + + ret = mlx5_eswitch_add_offloaded_rule(esw, spec, attr); + + /* We did not create the counter, so we can't delete it. + * Avoid freeing the counter when the attr is deleted in free_branching_attr + */ + attr->action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT; + + return ret; +} + static int -mlx5e_post_meter_rules_create(struct mlx5e_priv *priv, - struct mlx5e_post_meter_priv *post_meter, - struct mlx5e_post_act *post_act, - struct mlx5_fc *green_counter, - struct mlx5_fc *red_counter) -{ - struct mlx5_flow_destination dest[2] = {}; - struct mlx5_flow_act flow_act = {}; +mlx5e_post_meter_rate_rules_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5e_post_act *post_act, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter, + struct mlx5_flow_attr *green_attr, + struct mlx5_flow_attr *red_attr) +{ + struct mlx5e_post_meter_rate_table *table = &post_meter->rate_steering_table; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; int err; @@ -100,72 +161,242 @@ mlx5e_post_meter_rules_create(struct mlx5e_priv *priv, mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, MLX5_FLOW_METER_COLOR_RED, MLX5_PACKET_COLOR_MASK); - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[0].counter_id = mlx5_fc_id(red_counter); - - rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, dest, 1); + red_attr->ft = post_meter->rate_steering_table.ft; + rule = mlx5e_post_meter_add_rule(priv, post_meter, spec, red_attr, + act_counter, drop_counter); if (IS_ERR(rule)) { - mlx5_core_warn(priv->mdev, "Failed to create post_meter flow drop rule\n"); + mlx5_core_warn(priv->mdev, "Failed to create post_meter exceed rule\n"); err = PTR_ERR(rule); goto err_red; } - post_meter->drop_red_rule = rule; + table->red_rule = rule; + table->red_attr = red_attr; mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, MLX5_FLOW_METER_COLOR_GREEN, MLX5_PACKET_COLOR_MASK); - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest[0].ft = mlx5e_tc_post_act_get_ft(post_act); - dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[1].counter_id = mlx5_fc_id(green_counter); - - rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, dest, 2); + green_attr->ft = post_meter->rate_steering_table.ft; + rule = mlx5e_post_meter_add_rule(priv, post_meter, spec, green_attr, + act_counter, drop_counter); if (IS_ERR(rule)) { - mlx5_core_warn(priv->mdev, "Failed to create post_meter flow fwd rule\n"); + mlx5_core_warn(priv->mdev, "Failed to create post_meter notexceed rule\n"); err = PTR_ERR(rule); goto err_green; } - post_meter->fwd_green_rule = rule; + table->green_rule = rule; + table->green_attr = green_attr; kvfree(spec); return 0; err_green: - mlx5_del_flow_rules(post_meter->drop_red_rule); + mlx5_del_flow_rules(table->red_rule); err_red: kvfree(spec); return err; } static void -mlx5e_post_meter_rules_destroy(struct mlx5e_post_meter_priv *post_meter) +mlx5e_post_meter_rate_rules_destroy(struct mlx5_eswitch *esw, + struct mlx5e_post_meter_priv *post_meter) { - mlx5_del_flow_rules(post_meter->drop_red_rule); - mlx5_del_flow_rules(post_meter->fwd_green_rule); + struct mlx5e_post_meter_rate_table *rate_table = &post_meter->rate_steering_table; + + mlx5_eswitch_del_offloaded_rule(esw, rate_table->red_rule, rate_table->red_attr); + mlx5_eswitch_del_offloaded_rule(esw, rate_table->green_rule, rate_table->green_attr); } static void -mlx5e_post_meter_fg_destroy(struct mlx5e_post_meter_priv *post_meter) +mlx5e_post_meter_rate_fg_destroy(struct mlx5e_post_meter_priv *post_meter) { - mlx5_destroy_flow_group(post_meter->fg); + mlx5_destroy_flow_group(post_meter->rate_steering_table.fg); } static void -mlx5e_post_meter_table_destroy(struct mlx5e_post_meter_priv *post_meter) +mlx5e_post_meter_rate_table_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_destroy_flow_table(post_meter->rate_steering_table.ft); +} + +static void +mlx5e_post_meter_mtu_rules_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + struct mlx5e_post_meter_mtu_tables *mtu_tables = &post_meter->mtu_tables; + + mlx5_del_flow_rules(mtu_tables->green_table.rule); + mlx5_del_flow_rules(mtu_tables->red_table.rule); +} + +static void +mlx5e_post_meter_mtu_fg_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + struct mlx5e_post_meter_mtu_tables *mtu_tables = &post_meter->mtu_tables; + + mlx5_destroy_flow_group(mtu_tables->green_table.fg); + mlx5_destroy_flow_group(mtu_tables->red_table.fg); +} + +static void +mlx5e_post_meter_mtu_table_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + struct mlx5e_post_meter_mtu_tables *mtu_tables = &post_meter->mtu_tables; + + mlx5_destroy_flow_table(mtu_tables->green_table.ft); + mlx5_destroy_flow_table(mtu_tables->red_table.ft); +} + +static int +mlx5e_post_meter_rate_create(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5_flow_attr *green_attr, + struct mlx5_flow_attr *red_attr) +{ + struct mlx5_flow_table *ft; + int err; + + post_meter->type = MLX5E_POST_METER_RATE; + + ft = mlx5e_post_meter_table_create(priv, ns_type); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + mlx5_core_warn(priv->mdev, "Failed to create post_meter table\n"); + goto err_ft; + } + + post_meter->rate_steering_table.ft = ft; + + err = mlx5e_post_meter_rate_fg_create(priv, post_meter); + if (err) + goto err_fg; + + err = mlx5e_post_meter_rate_rules_create(priv, post_meter, post_act, + act_counter, drop_counter, + green_attr, red_attr); + if (err) + goto err_rules; + + return 0; + +err_rules: + mlx5e_post_meter_rate_fg_destroy(post_meter); +err_fg: + mlx5e_post_meter_rate_table_destroy(post_meter); +err_ft: + return err; +} + +static int +mlx5e_post_meter_create_mtu_table(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_meter_mtu_table *table) { - mlx5_destroy_flow_table(post_meter->ft); + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *flow_group_in; + int err; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return -ENOMEM; + + table->ft = mlx5e_post_meter_table_create(priv, ns_type); + if (IS_ERR(table->ft)) { + err = PTR_ERR(table->ft); + goto err_ft; + } + + /* create miss group */ + memset(flow_group_in, 0, inlen); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); + fg = mlx5_create_flow_group(table->ft, flow_group_in); + if (IS_ERR(fg)) { + err = PTR_ERR(fg); + goto err_miss_grp; + } + table->fg = fg; + + kvfree(flow_group_in); + return 0; + +err_miss_grp: + mlx5_destroy_flow_table(table->ft); +err_ft: + kvfree(flow_group_in); + return err; +} + +static int +mlx5e_post_meter_mtu_create(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5_flow_attr *green_attr, + struct mlx5_flow_attr *red_attr) +{ + struct mlx5e_post_meter_mtu_tables *mtu_tables = &post_meter->mtu_tables; + static struct mlx5_flow_spec zero_spec = {}; + struct mlx5_flow_handle *rule; + int err; + + post_meter->type = MLX5E_POST_METER_MTU; + + err = mlx5e_post_meter_create_mtu_table(priv, ns_type, &mtu_tables->green_table); + if (err) + goto err_green_ft; + + green_attr->ft = mtu_tables->green_table.ft; + rule = mlx5e_post_meter_add_rule(priv, post_meter, &zero_spec, green_attr, + act_counter, drop_counter); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter conform rule\n"); + err = PTR_ERR(rule); + goto err_green_rule; + } + mtu_tables->green_table.rule = rule; + mtu_tables->green_table.attr = green_attr; + + err = mlx5e_post_meter_create_mtu_table(priv, ns_type, &mtu_tables->red_table); + if (err) + goto err_red_ft; + + red_attr->ft = mtu_tables->red_table.ft; + rule = mlx5e_post_meter_add_rule(priv, post_meter, &zero_spec, red_attr, + act_counter, drop_counter); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter exceed rule\n"); + err = PTR_ERR(rule); + goto err_red_rule; + } + mtu_tables->red_table.rule = rule; + mtu_tables->red_table.attr = red_attr; + + return 0; + +err_red_rule: + mlx5_destroy_flow_table(mtu_tables->red_table.ft); +err_red_ft: + mlx5_del_flow_rules(mtu_tables->green_table.rule); +err_green_rule: + mlx5_destroy_flow_table(mtu_tables->green_table.ft); +err_green_ft: + return err; } struct mlx5e_post_meter_priv * mlx5e_post_meter_init(struct mlx5e_priv *priv, enum mlx5_flow_namespace_type ns_type, struct mlx5e_post_act *post_act, - struct mlx5_fc *green_counter, - struct mlx5_fc *red_counter) + enum mlx5e_post_meter_type type, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter, + struct mlx5_flow_attr *branch_true, + struct mlx5_flow_attr *branch_false) { struct mlx5e_post_meter_priv *post_meter; int err; @@ -174,36 +405,55 @@ mlx5e_post_meter_init(struct mlx5e_priv *priv, if (!post_meter) return ERR_PTR(-ENOMEM); - err = mlx5e_post_meter_table_create(priv, ns_type, post_meter); - if (err) - goto err_ft; - - err = mlx5e_post_meter_fg_create(priv, post_meter); - if (err) - goto err_fg; + switch (type) { + case MLX5E_POST_METER_MTU: + err = mlx5e_post_meter_mtu_create(priv, ns_type, post_act, + act_counter, drop_counter, post_meter, + branch_true, branch_false); + break; + case MLX5E_POST_METER_RATE: + err = mlx5e_post_meter_rate_create(priv, ns_type, post_act, + act_counter, drop_counter, post_meter, + branch_true, branch_false); + break; + default: + err = -EOPNOTSUPP; + } - err = mlx5e_post_meter_rules_create(priv, post_meter, post_act, green_counter, - red_counter); if (err) - goto err_rules; + goto err; return post_meter; -err_rules: - mlx5e_post_meter_fg_destroy(post_meter); -err_fg: - mlx5e_post_meter_table_destroy(post_meter); -err_ft: +err: kfree(post_meter); return ERR_PTR(err); } +static void +mlx5e_post_meter_rate_destroy(struct mlx5_eswitch *esw, struct mlx5e_post_meter_priv *post_meter) +{ + mlx5e_post_meter_rate_rules_destroy(esw, post_meter); + mlx5e_post_meter_rate_fg_destroy(post_meter); + mlx5e_post_meter_rate_table_destroy(post_meter); +} + +static void +mlx5e_post_meter_mtu_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5e_post_meter_mtu_rules_destroy(post_meter); + mlx5e_post_meter_mtu_fg_destroy(post_meter); + mlx5e_post_meter_mtu_table_destroy(post_meter); +} + void -mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter) +mlx5e_post_meter_cleanup(struct mlx5_eswitch *esw, struct mlx5e_post_meter_priv *post_meter) { - mlx5e_post_meter_rules_destroy(post_meter); - mlx5e_post_meter_fg_destroy(post_meter); - mlx5e_post_meter_table_destroy(post_meter); + if (post_meter->type == MLX5E_POST_METER_RATE) + mlx5e_post_meter_rate_destroy(esw, post_meter); + else + mlx5e_post_meter_mtu_destroy(post_meter); + kfree(post_meter); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h index 34d0e4b9fc7a..e013b77186b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h @@ -14,16 +14,49 @@ struct mlx5e_post_meter_priv; +enum mlx5e_post_meter_type { + MLX5E_POST_METER_RATE = 0, + MLX5E_POST_METER_MTU +}; + +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) + struct mlx5_flow_table * mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter); +struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_true_ft(struct mlx5e_post_meter_priv *post_meter); + +struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_false_ft(struct mlx5e_post_meter_priv *post_meter); + struct mlx5e_post_meter_priv * mlx5e_post_meter_init(struct mlx5e_priv *priv, enum mlx5_flow_namespace_type ns_type, struct mlx5e_post_act *post_act, - struct mlx5_fc *green_counter, - struct mlx5_fc *red_counter); + enum mlx5e_post_meter_type type, + struct mlx5_fc *act_counter, + struct mlx5_fc *drop_counter, + struct mlx5_flow_attr *branch_true, + struct mlx5_flow_attr *branch_false); + void -mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter); +mlx5e_post_meter_cleanup(struct mlx5_eswitch *esw, struct mlx5e_post_meter_priv *post_meter); + +#else /* CONFIG_MLX5_CLS_ACT */ + +static inline struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_true_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return NULL; +} + +static inline struct mlx5_flow_table * +mlx5e_post_meter_get_mtu_false_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return NULL; +} + +#endif #endif /* __MLX5_EN_POST_METER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index 2e42d7c5451e..2b7fd1c0e643 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -211,8 +211,4 @@ struct mlx5e_flow_meters *mlx5e_get_flow_meters(struct mlx5_core_dev *dev); void *mlx5e_get_match_headers_value(u32 flags, struct mlx5_flow_spec *spec); void *mlx5e_get_match_headers_criteria(u32 flags, struct mlx5_flow_spec *spec); -int mlx5e_policer_validate(const struct flow_action *action, - const struct flow_action_entry *act, - struct netlink_ext_ack *extack); - #endif /* __MLX5_EN_TC_PRIV_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index a715601865d3..bb9023957f74 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -45,55 +45,9 @@ static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; } -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, - unsigned int handle) +static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) { - struct mlx5e_ipsec_sa_entry *sa_entry; - struct xfrm_state *ret = NULL; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) - if (sa_entry->handle == handle) { - ret = sa_entry->x; - xfrm_state_hold(ret); - break; - } - rcu_read_unlock(); - - return ret; -} - -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - unsigned int handle = sa_entry->ipsec_obj_id; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_sa_entry *_sa_entry; - unsigned long flags; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) - if (_sa_entry->handle == handle) { - rcu_read_unlock(); - return -EEXIST; - } - rcu_read_unlock(); - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - sa_entry->handle = handle; - hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); - - return 0; -} - -static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - unsigned long flags; - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - hash_del_rcu(&sa_entry->hlist); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); + return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; } static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) @@ -129,9 +83,33 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) return false; } -static void -mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, - struct mlx5_accel_esp_xfrm_attrs *attrs) +static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct xfrm_state *x = sa_entry->x; + + attrs->hard_packet_limit = x->lft.hard_packet_limit; + if (x->lft.soft_packet_limit == XFRM_INF) + return; + + /* Hardware decrements hard_packet_limit counter through + * the operation. While fires an event when soft_packet_limit + * is reached. It emans that we need substitute the numbers + * in order to properly count soft limit. + * + * As an example: + * XFRM user sets soft limit is 2 and hard limit is 9 and + * expects to see soft event after 2 packets and hard event + * after 9 packets. In our case, the hard limit will be set + * to 9 and soft limit is comparator to 7 so user gets the + * soft event after 2 packeta + */ + attrs->soft_packet_limit = + x->lft.hard_packet_limit - x->lft.soft_packet_limit; +} + +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; @@ -157,33 +135,31 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, sizeof(aes_gcm->salt)); + attrs->authsize = crypto_aead_authsize(aead) / 4; /* in dwords */ + /* iv len */ aes_gcm->icv_len = x->aead->alg_icv_len; /* esn */ if (sa_entry->esn_state.trigger) { - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; + attrs->esn_trigger = true; attrs->esn = sa_entry->esn_state.esn; - if (sa_entry->esn_state.overlap) - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; + attrs->esn_overlap = sa_entry->esn_state.overlap; + attrs->replay_window = x->replay_esn->replay_window; } - /* action */ - attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? - MLX5_ACCEL_ESP_ACTION_ENCRYPT : - MLX5_ACCEL_ESP_ACTION_DECRYPT; - /* flags */ - attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? - MLX5_ACCEL_ESP_FLAGS_TRANSPORT : - MLX5_ACCEL_ESP_FLAGS_TUNNEL; - + attrs->dir = x->xso.dir; /* spi */ attrs->spi = be32_to_cpu(x->id.spi); /* source , destination ips */ memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); - attrs->is_ipv6 = (x->props.family != AF_INET); + attrs->family = x->props.family; + attrs->type = x->xso.type; + attrs->reqid = x->props.reqid; + + mlx5e_ipsec_init_limits(sa_entry, attrs); } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) @@ -215,11 +191,6 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); return -EINVAL; } - if (x->props.mode != XFRM_MODE_TRANSPORT && - x->props.mode != XFRM_MODE_TUNNEL) { - dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); - return -EINVAL; - } if (x->id.proto != IPPROTO_ESP) { netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); return -EINVAL; @@ -253,6 +224,67 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + switch (x->xso.type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_CRYPTO)) { + netdev_info(netdev, "Crypto offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + netdev_info(netdev, "Only transport and tunnel xfrm states may be offloaded\n"); + return -EINVAL; + } + break; + case XFRM_DEV_OFFLOAD_PACKET: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { + netdev_info(netdev, "Packet offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT) { + netdev_info(netdev, "Only transport xfrm states may be offloaded in packet mode\n"); + return -EINVAL; + } + + if (x->replay_esn && x->replay_esn->replay_window != 32 && + x->replay_esn->replay_window != 64 && + x->replay_esn->replay_window != 128 && + x->replay_esn->replay_window != 256) { + netdev_info(netdev, + "Unsupported replay window size %u\n", + x->replay_esn->replay_window); + return -EINVAL; + } + + if (!x->props.reqid) { + netdev_info(netdev, "Cannot offload without reqid\n"); + return -EINVAL; + } + + if (x->lft.hard_byte_limit != XFRM_INF || + x->lft.soft_byte_limit != XFRM_INF) { + netdev_info(netdev, + "Device doesn't support limits in bytes\n"); + return -EINVAL; + } + + if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit && + x->lft.hard_packet_limit != XFRM_INF) { + /* XFRM stack doesn't prevent such configuration :(. */ + netdev_info(netdev, + "Hard packet limit must be greater than soft one\n"); + return -EINVAL; + } + break; + default: + netdev_info(netdev, "Unsupported xfrm offload type %d\n", + x->xso.type); + return -EINVAL; + } return 0; } @@ -270,6 +302,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; struct net_device *netdev = x->xso.real_dev; + struct mlx5e_ipsec *ipsec; struct mlx5e_priv *priv; int err; @@ -277,6 +310,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (!priv->ipsec) return -EOPNOTSUPP; + ipsec = priv->ipsec; err = mlx5e_xfrm_validate_state(x); if (err) return err; @@ -288,7 +322,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) } sa_entry->x = x; - sa_entry->ipsec = priv->ipsec; + sa_entry->ipsec = ipsec; /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); @@ -299,25 +333,29 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (err) goto err_xfrm; - err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); + err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); if (err) goto err_hw_ctx; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry); - if (err) - goto err_add_rule; - } else { + /* We use *_bh() variant because xfrm_timer_handler(), which runs + * in softirq context, can reach our state delete logic and we need + * xa_erase_bh() there. + */ + err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry, + GFP_KERNEL); + if (err) + goto err_add_rule; + + if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; - } INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); x->xso.offload_handle = (unsigned long)sa_entry; - goto out; + return 0; err_add_rule: - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); err_hw_ctx: mlx5_ipsec_free_sa_ctx(sa_entry); err_xfrm: @@ -329,59 +367,69 @@ out: static void mlx5e_xfrm_del_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_sa_entry *old; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) - mlx5e_ipsec_sadb_rx_del(sa_entry); + old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id); + WARN_ON(old != sa_entry); } static void mlx5e_xfrm_free_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); - struct mlx5e_priv *priv = netdev_priv(x->xso.dev); cancel_work_sync(&sa_entry->modify_work.work); - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); mlx5_ipsec_free_sa_ctx(sa_entry); kfree(sa_entry); } -int mlx5e_ipsec_init(struct mlx5e_priv *priv) +void mlx5e_ipsec_init(struct mlx5e_priv *priv) { struct mlx5e_ipsec *ipsec; - int ret; + int ret = -ENOMEM; if (!mlx5_ipsec_device_caps(priv->mdev)) { netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); - return 0; + return; } ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL); if (!ipsec) - return -ENOMEM; + return; - hash_init(ipsec->sadb_rx); - spin_lock_init(&ipsec->sadb_rx_lock); + xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC); ipsec->mdev = priv->mdev; ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, priv->netdev->name); - if (!ipsec->wq) { - ret = -ENOMEM; + if (!ipsec->wq) goto err_wq; + + if (mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD) { + ret = mlx5e_ipsec_aso_init(ipsec); + if (ret) + goto err_aso; } ret = mlx5e_accel_ipsec_fs_init(ipsec); if (ret) goto err_fs_init; + ipsec->fs = priv->fs; priv->ipsec = ipsec; netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); - return 0; + return; err_fs_init: + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); +err_aso: destroy_workqueue(ipsec->wq); err_wq: kfree(ipsec); - return (ret != -EOPNOTSUPP) ? ret : 0; + mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret); + return; } void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) @@ -392,6 +440,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) return; mlx5e_accel_ipsec_fs_cleanup(ipsec); + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); destroy_workqueue(ipsec->wq); kfree(ipsec); priv->ipsec = NULL; @@ -427,6 +477,122 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &modify_work->work); } +static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) +{ + struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + int err; + + lockdep_assert_held(&x->lock); + + if (sa_entry->attrs.soft_packet_limit == XFRM_INF) + /* Limits are not configured, as soft limit + * must be lowever than hard limit. + */ + return; + + err = mlx5e_ipsec_aso_query(sa_entry, NULL); + if (err) + return; + + mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets); +} + +static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + + if (x->type != XFRM_POLICY_TYPE_MAIN) { + netdev_info(netdev, "Cannot offload non-main policy types\n"); + return -EINVAL; + } + + /* Please pay attention that we support only one template */ + if (x->xfrm_nr > 1) { + netdev_info(netdev, "Cannot offload more than one template\n"); + return -EINVAL; + } + + if (x->xdo.dir != XFRM_DEV_OFFLOAD_IN && + x->xdo.dir != XFRM_DEV_OFFLOAD_OUT) { + netdev_info(netdev, "Cannot offload forward policy\n"); + return -EINVAL; + } + + if (!x->xfrm_vec[0].reqid) { + netdev_info(netdev, "Cannot offload policy without reqid\n"); + return -EINVAL; + } + + if (x->xdo.type != XFRM_DEV_OFFLOAD_PACKET) { + netdev_info(netdev, "Unsupported xfrm offload type\n"); + return -EINVAL; + } + + return 0; +} + +static void +mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, + struct mlx5_accel_pol_xfrm_attrs *attrs) +{ + struct xfrm_policy *x = pol_entry->x; + struct xfrm_selector *sel; + + sel = &x->selector; + memset(attrs, 0, sizeof(*attrs)); + + memcpy(&attrs->saddr, sel->saddr.a6, sizeof(attrs->saddr)); + memcpy(&attrs->daddr, sel->daddr.a6, sizeof(attrs->daddr)); + attrs->family = sel->family; + attrs->dir = x->xdo.dir; + attrs->action = x->action; + attrs->type = XFRM_DEV_OFFLOAD_PACKET; + attrs->reqid = x->xfrm_vec[0].reqid; +} + +static int mlx5e_xfrm_add_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + struct mlx5e_ipsec_pol_entry *pol_entry; + struct mlx5e_priv *priv; + int err; + + priv = netdev_priv(netdev); + if (!priv->ipsec) + return -EOPNOTSUPP; + + err = mlx5e_xfrm_validate_policy(x); + if (err) + return err; + + pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); + if (!pol_entry) + return -ENOMEM; + + pol_entry->x = x; + pol_entry->ipsec = priv->ipsec; + + mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs); + err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); + if (err) + goto err_fs; + + x->xdo.offload_handle = (unsigned long)pol_entry; + return 0; + +err_fs: + kfree(pol_entry); + return err; +} + +static void mlx5e_xfrm_free_policy(struct xfrm_policy *x) +{ + struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x); + + mlx5e_accel_ipsec_fs_del_pol(pol_entry); + kfree(pol_entry); +} + static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_add = mlx5e_xfrm_add_state, .xdo_dev_state_delete = mlx5e_xfrm_del_state, @@ -435,6 +601,18 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, }; +static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = { + .xdo_dev_state_add = mlx5e_xfrm_add_state, + .xdo_dev_state_delete = mlx5e_xfrm_del_state, + .xdo_dev_state_free = mlx5e_xfrm_free_state, + .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, + .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, + + .xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft, + .xdo_dev_policy_add = mlx5e_xfrm_add_policy, + .xdo_dev_policy_free = mlx5e_xfrm_free_policy, +}; + void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; @@ -444,7 +622,12 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) return; mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); - netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + netdev->xfrmdev_ops = &mlx5e_ipsec_packet_xfrmdev_ops; + else + netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + netdev->features |= NETIF_F_HW_ESP; netdev->hw_enc_features |= NETIF_F_HW_ESP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 16bcceec16c4..a92e19c4c499 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -34,27 +34,14 @@ #ifndef __MLX5E_IPSEC_H__ #define __MLX5E_IPSEC_H__ -#ifdef CONFIG_MLX5_EN_IPSEC - #include <linux/mlx5/device.h> #include <net/xfrm.h> #include <linux/idr.h> +#include "lib/aso.h" #define MLX5E_IPSEC_SADB_RX_BITS 10 #define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L -enum mlx5_accel_esp_flags { - MLX5_ACCEL_ESP_FLAGS_TUNNEL = 0, /* Default */ - MLX5_ACCEL_ESP_FLAGS_TRANSPORT = 1UL << 0, - MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED = 1UL << 1, - MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP = 1UL << 2, -}; - -enum mlx5_accel_esp_action { - MLX5_ACCEL_ESP_ACTION_DECRYPT, - MLX5_ACCEL_ESP_ACTION_ENCRYPT, -}; - struct aes_gcm_keymat { u64 seq_iv; @@ -66,7 +53,6 @@ struct aes_gcm_keymat { }; struct mlx5_accel_esp_xfrm_attrs { - enum mlx5_accel_esp_action action; u32 esn; u32 spi; u32 flags; @@ -82,16 +68,37 @@ struct mlx5_accel_esp_xfrm_attrs { __be32 a6[4]; } daddr; - u8 is_ipv6; + u8 dir : 2; + u8 esn_overlap : 1; + u8 esn_trigger : 1; + u8 type : 2; + u8 family; + u32 replay_window; + u32 authsize; + u32 reqid; + u64 hard_packet_limit; + u64 soft_packet_limit; }; enum mlx5_ipsec_cap { MLX5_IPSEC_CAP_CRYPTO = 1 << 0, MLX5_IPSEC_CAP_ESN = 1 << 1, + MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2, }; struct mlx5e_priv; +struct mlx5e_ipsec_hw_stats { + u64 ipsec_rx_pkts; + u64 ipsec_rx_bytes; + u64 ipsec_rx_drop_pkts; + u64 ipsec_rx_drop_bytes; + u64 ipsec_tx_pkts; + u64 ipsec_tx_bytes; + u64 ipsec_tx_drop_pkts; + u64 ipsec_tx_drop_bytes; +}; + struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_rx_drop_sp_alloc; atomic64_t ipsec_rx_drop_sadb_miss; @@ -102,17 +109,38 @@ struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_tx_drop_trailer; }; -struct mlx5e_accel_fs_esp; +struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_work { + struct work_struct work; + struct mlx5e_ipsec *ipsec; + u32 id; +}; + +struct mlx5e_ipsec_aso { + u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; + dma_addr_t dma_addr; + struct mlx5_aso *aso; + /* IPsec ASO caches data on every query call, + * so in nested calls, we can use this boolean to save + * recursive calls to mlx5e_ipsec_aso_query() + */ + u8 use_cache : 1; +}; + struct mlx5e_ipsec { struct mlx5_core_dev *mdev; - DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); - spinlock_t sadb_rx_lock; /* Protects sadb_rx */ + struct xarray sadb; struct mlx5e_ipsec_sw_stats sw_stats; + struct mlx5e_ipsec_hw_stats hw_stats; struct workqueue_struct *wq; - struct mlx5e_accel_fs_esp *rx_fs; - struct mlx5e_ipsec_tx *tx_fs; + struct mlx5e_flow_steering *fs; + struct mlx5e_ipsec_rx *rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6; + struct mlx5e_ipsec_tx *tx; + struct mlx5e_ipsec_aso *aso; + struct notifier_block nb; }; struct mlx5e_ipsec_esn_state { @@ -123,7 +151,8 @@ struct mlx5e_ipsec_esn_state { struct mlx5e_ipsec_rule { struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *set_modify_hdr; + struct mlx5_modify_hdr *modify_hdr; + struct mlx5_pkt_reformat *pkt_reformat; }; struct mlx5e_ipsec_modify_state_work { @@ -132,9 +161,7 @@ struct mlx5e_ipsec_modify_state_work { }; struct mlx5e_ipsec_sa_entry { - struct hlist_node hlist; /* Item in SADB_RX hashtable */ struct mlx5e_ipsec_esn_state esn_state; - unsigned int handle; /* Handle in SADB_RX */ struct xfrm_state *x; struct mlx5e_ipsec *ipsec; struct mlx5_accel_esp_xfrm_attrs attrs; @@ -146,19 +173,43 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_modify_state_work modify_work; }; -int mlx5e_ipsec_init(struct mlx5e_priv *priv); +struct mlx5_accel_pol_xfrm_attrs { + union { + __be32 a4; + __be32 a6[4]; + } saddr; + + union { + __be32 a4; + __be32 a6[4]; + } daddr; + + u8 family; + u8 action; + u8 type : 2; + u8 dir : 2; + u32 reqid; +}; + +struct mlx5e_ipsec_pol_entry { + struct xfrm_policy *x; + struct mlx5e_ipsec *ipsec; + struct mlx5e_ipsec_rule ipsec_rule; + struct mlx5_accel_pol_xfrm_attrs attrs; +}; + +#ifdef CONFIG_MLX5_EN_IPSEC + +void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, - unsigned int handle); - void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry); +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry); int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); @@ -168,15 +219,33 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev); void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, const struct mlx5_accel_esp_xfrm_attrs *attrs); +int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); +void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data); +void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, + u64 *packets); + +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, + void *ipsec_stats); + +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs); static inline struct mlx5_core_dev * mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { return sa_entry->ipsec->mdev; } + +static inline struct mlx5_core_dev * +mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + return pol_entry->ipsec->mdev; +} #else -static inline int mlx5e_ipsec_init(struct mlx5e_priv *priv) +static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv) { - return 0; } static inline void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b859e4a4c744..9f19f4b59a70 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,53 +9,67 @@ #define NUM_IPSEC_FTE BIT(15) -enum accel_fs_esp_type { - ACCEL_FS_ESP4, - ACCEL_FS_ESP6, - ACCEL_FS_ESP_NUM_TYPES, +struct mlx5e_ipsec_fc { + struct mlx5_fc *cnt; + struct mlx5_fc *drop; }; -struct mlx5e_ipsec_rx_err { - struct mlx5_flow_table *ft; - struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *copy_modify_hdr; +struct mlx5e_ipsec_ft { + struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *pol; + struct mlx5_flow_table *sa; + struct mlx5_flow_table *status; + u32 refcnt; }; -struct mlx5e_accel_fs_esp_prot { - struct mlx5_flow_table *ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; - struct mlx5_flow_destination default_dest; - struct mlx5e_ipsec_rx_err rx_err; - u32 refcnt; - struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ +struct mlx5e_ipsec_miss { + struct mlx5_flow_group *group; + struct mlx5_flow_handle *rule; }; -struct mlx5e_accel_fs_esp { - struct mlx5e_accel_fs_esp_prot fs_prot[ACCEL_FS_ESP_NUM_TYPES]; +struct mlx5e_ipsec_rx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; + struct mlx5e_ipsec_miss sa; + struct mlx5e_ipsec_rule status; + struct mlx5e_ipsec_fc *fc; }; struct mlx5e_ipsec_tx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - struct mutex mutex; /* Protect IPsec TX steering */ - u32 refcnt; + struct mlx5e_ipsec_fc *fc; }; /* IPsec RX flow steering */ -static enum mlx5_traffic_types fs_esp2tt(enum accel_fs_esp_type i) +static enum mlx5_traffic_types family2tt(u32 family) { - if (i == ACCEL_FS_ESP4) + if (family == AF_INET) return MLX5_TT_IPV4_IPSEC_ESP; return MLX5_TT_IPV6_IPSEC_ESP; } -static int rx_err_add_rule(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot, - struct mlx5e_ipsec_rx_err *rx_err) +static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, + int level, int prio, + int max_num_groups) +{ + struct mlx5_flow_table_attr ft_attr = {}; + + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = max_num_groups; + ft_attr.max_fte = NUM_IPSEC_FTE; + ft_attr.level = level; + ft_attr.prio = prio; + + return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); +} + +static int ipsec_status_rule(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_act flow_act = {}; struct mlx5_modify_hdr *modify_hdr; struct mlx5_flow_handle *fte; @@ -79,26 +93,26 @@ static int rx_err_add_rule(struct mlx5e_priv *priv, if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec copy modify_header_id err=%d\n", err); + mlx5_core_err(mdev, + "fail to alloc ipsec copy modify_header_id err=%d\n", err); goto out_spec; } /* create fte */ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | - MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; flow_act.modify_hdr = modify_hdr; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, - &fs_prot->default_dest, 1); + fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); if (IS_ERR(fte)) { err = PTR_ERR(fte); - netdev_err(priv->netdev, "fail to add ipsec rx err copy rule err=%d\n", err); + mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); goto out; } kvfree(spec); - rx_err->rule = fte; - rx_err->copy_modify_hdr = modify_hdr; + rx->status.rule = fte; + rx->status.modify_hdr = modify_hdr; return 0; out: @@ -108,13 +122,12 @@ out_spec: return err; } -static int rx_fs_create(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot) +static int ipsec_miss_create(struct mlx5_core_dev *mdev, + struct mlx5_flow_table *ft, + struct mlx5e_ipsec_miss *miss, + struct mlx5_flow_destination *dest) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - struct mlx5_flow_table *ft = fs_prot->ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_spec *spec; u32 *flow_group_in; @@ -130,450 +143,888 @@ static int rx_fs_create(struct mlx5e_priv *priv, /* Create miss_group */ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); - miss_group = mlx5_create_flow_group(ft, flow_group_in); - if (IS_ERR(miss_group)) { - err = PTR_ERR(miss_group); - netdev_err(priv->netdev, "fail to create ipsec rx miss_group err=%d\n", err); + miss->group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss->group)) { + err = PTR_ERR(miss->group); + mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", + err); goto out; } - fs_prot->miss_group = miss_group; /* Create miss rule */ - miss_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); - if (IS_ERR(miss_rule)) { - mlx5_destroy_flow_group(fs_prot->miss_group); - err = PTR_ERR(miss_rule); - netdev_err(priv->netdev, "fail to create ipsec rx miss_rule err=%d\n", err); + miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); + if (IS_ERR(miss->rule)) { + mlx5_destroy_flow_group(miss->group); + err = PTR_ERR(miss->rule); + mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", + err); goto out; } - fs_prot->miss_rule = miss_rule; out: kvfree(flow_group_in); kvfree(spec); return err; } -static void rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; - - accel_esp = priv->ipsec->rx_fs; - - /* The netdev unreg already happened, so all offloaded rule are already removed */ - fs_prot = &accel_esp->fs_prot[type]; + mlx5_del_flow_rules(rx->pol.rule); + mlx5_destroy_flow_group(rx->pol.group); + mlx5_destroy_flow_table(rx->ft.pol); - mlx5_del_flow_rules(fs_prot->miss_rule); - mlx5_destroy_flow_group(fs_prot->miss_group); - mlx5_destroy_flow_table(fs_prot->ft); + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); + mlx5_destroy_flow_table(rx->ft.sa); - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + mlx5_destroy_flow_table(rx->ft.status); } -static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false); - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + struct mlx5_flow_destination dest[2]; struct mlx5_flow_table *ft; int err; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - fs_prot->default_dest = - mlx5_ttc_get_default_dest(ttc, fs_esp2tt(type)); - - ft_attr.max_fte = 1; - ft_attr.autogroup.max_num_groups = 1; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, + MLX5E_NIC_PRIO, 1); if (IS_ERR(ft)) return PTR_ERR(ft); - fs_prot->rx_err.ft = ft; - err = rx_err_add_rule(priv, fs_prot, &fs_prot->rx_err); + rx->ft.status = ft; + + dest[0] = mlx5_ttc_get_default_dest(ttc, family2tt(family)); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); + err = ipsec_status_rule(mdev, rx, dest); if (err) goto err_add; /* Create FT */ - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft_attr.autogroup.num_reserved_entries = 1; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, + 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; } - fs_prot->ft = ft; + rx->ft.sa = ft; - err = rx_fs_create(priv, fs_prot); + err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, dest); if (err) goto err_fs; + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, + 2); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto err_pol_ft; + } + rx->ft.pol = ft; + memset(dest, 0x00, 2 * sizeof(*dest)); + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[0].ft = rx->ft.sa; + err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, dest); + if (err) + goto err_pol_miss; + return 0; +err_pol_miss: + mlx5_destroy_flow_table(rx->ft.pol); +err_pol_ft: + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); err_fs: - mlx5_destroy_flow_table(fs_prot->ft); + mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); err_add: - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_destroy_flow_table(rx->ft.status); return err; } -static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec, u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5e_ipsec_rx *rx; int err = 0; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - if (fs_prot->refcnt) + if (family == AF_INET) + rx = ipsec->rx_ipv4; + else + rx = ipsec->rx_ipv6; + + mutex_lock(&rx->ft.mutex); + if (rx->ft.refcnt) goto skip; /* create FT */ - err = rx_create(priv, type); + err = rx_create(mdev, ipsec, rx, family); if (err) goto out; /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(ttc, fs_esp2tt(type), &dest); + dest.ft = rx->ft.pol; + mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest); skip: - fs_prot->refcnt++; + rx->ft.refcnt++; out: - mutex_unlock(&fs_prot->prot_mutex); - return err; + mutex_unlock(&rx->ft.mutex); + if (err) + return ERR_PTR(err); + return rx; } -static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_ft_put(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + struct mlx5e_ipsec_rx *rx; + + if (family == AF_INET) + rx = ipsec->rx_ipv4; + else + rx = ipsec->rx_ipv6; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - fs_prot->refcnt--; - if (fs_prot->refcnt) + mutex_lock(&rx->ft.mutex); + rx->ft.refcnt--; + if (rx->ft.refcnt) goto out; /* disconnect */ - mlx5_ttc_fwd_default_dest(ttc, fs_esp2tt(type)); + mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); /* remove FT */ - rx_destroy(priv, type); + rx_destroy(mdev, rx); out: - mutex_unlock(&fs_prot->prot_mutex); + mutex_unlock(&rx->ft.mutex); } /* IPsec TX flow steering */ -static int tx_create(struct mlx5e_priv *priv) +static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) { - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft; int err; - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ipsec->tx_fs->ns, &ft_attr); + ft = ipsec_ft_create(tx->ns, 1, 0, 4); + if (IS_ERR(ft)) + return PTR_ERR(ft); + + tx->ft.sa = ft; + + ft = ipsec_ft_create(tx->ns, 0, 0, 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); - netdev_err(priv->netdev, "fail to create ipsec tx ft err=%d\n", err); - return err; + goto err_pol_ft; } - ipsec->tx_fs->ft = ft; + tx->ft.pol = ft; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = tx->ft.sa; + err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); + if (err) + goto err_pol_miss; return 0; + +err_pol_miss: + mlx5_destroy_flow_table(tx->ft.pol); +err_pol_ft: + mlx5_destroy_flow_table(tx->ft.sa); + return err; } -static int tx_ft_get(struct mlx5e_priv *priv) +static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = ipsec->tx; int err = 0; - mutex_lock(&tx_fs->mutex); - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + if (tx->ft.refcnt) goto skip; - err = tx_create(priv); + err = tx_create(mdev, tx); if (err) goto out; skip: - tx_fs->refcnt++; + tx->ft.refcnt++; out: - mutex_unlock(&tx_fs->mutex); - return err; + mutex_unlock(&tx->ft.mutex); + if (err) + return ERR_PTR(err); + return tx; } -static void tx_ft_put(struct mlx5e_priv *priv) +static void tx_ft_put(struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = ipsec->tx; - mutex_lock(&tx_fs->mutex); - tx_fs->refcnt--; - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + tx->ft.refcnt--; + if (tx->ft.refcnt) goto out; - mlx5_destroy_flow_table(tx_fs->ft); + mlx5_del_flow_rules(tx->pol.rule); + mlx5_destroy_flow_group(tx->pol.group); + mlx5_destroy_flow_table(tx->ft.pol); + mlx5_destroy_flow_table(tx->ft.sa); out: - mutex_unlock(&tx_fs->mutex); + mutex_unlock(&tx->ft.mutex); } -static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 ipsec_obj_id, - struct mlx5_flow_spec *spec, - struct mlx5_flow_act *flow_act) +static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) { - u8 ip_version = attrs->is_ipv6 ? 6 : 4; - - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS; + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; - /* ip_version */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); + + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); +} - /* Non fragmented */ - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); +static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) +{ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); + + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); +} + +static void setup_fte_esp(struct mlx5_flow_spec *spec) +{ /* ESP header */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); +} +static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi) +{ /* SPI number */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); +} + +static void setup_fte_no_frags(struct mlx5_flow_spec *spec) +{ + /* Non fragmented */ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); +} + +static void setup_fte_reg_a(struct mlx5_flow_spec *spec) +{ + /* Add IPsec indicator in metadata_reg_a */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); MLX5_SET(fte_match_param, spec->match_value, - misc_parameters.outer_esp_spi, attrs->spi); - - if (ip_version == 4) { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), - &attrs->saddr.a4, 4); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), - &attrs->daddr.a4, 4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); - } else { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - &attrs->saddr.a6, 16); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - &attrs->daddr.a6, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - 0xff, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - 0xff, 16); - } + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); +} - flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; - flow_act->crypto.obj_id = ipsec_obj_id; - flow_act->flags |= FLOW_ACT_NO_APPEND; +static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) +{ + /* Pass policy check before choosing this SA */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, reqid); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.metadata_reg_c_0, reqid); } -static int rx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, + struct mlx5_flow_act *flow_act) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; + enum mlx5_flow_namespace_type ns_type; + struct mlx5_modify_hdr *modify_hdr; + + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + switch (dir) { + case XFRM_DEV_OFFLOAD_IN: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_B); + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + break; + case XFRM_DEV_OFFLOAD_OUT: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); + ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + break; + default: + return -EINVAL; + } + + MLX5_SET(set_action_in, action, data, val); + MLX5_SET(set_action_in, action, offset, 0); + MLX5_SET(set_action_in, action, length, 32); + + modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); + if (IS_ERR(modify_hdr)) { + mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", + PTR_ERR(modify_hdr)); + return PTR_ERR(modify_hdr); + } + + flow_act->modify_hdr = modify_hdr; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + return 0; +} + +static int setup_pkt_reformat(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5_flow_act *flow_act) +{ + enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_pkt_reformat *pkt_reformat; + u8 reformatbf[16] = {}; + __be32 spi; + + if (attrs->dir == XFRM_DEV_OFFLOAD_IN) { + reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + goto cmd; + } + + if (attrs->family == AF_INET) + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; + else + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; + + /* convert to network format */ + spi = htonl(attrs->spi); + memcpy(reformatbf, &spi, 4); + + reformat_params.param_0 = attrs->authsize; + reformat_params.size = sizeof(reformatbf); + reformat_params.data = &reformatbf; + +cmd: + pkt_reformat = + mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); + if (IS_ERR(pkt_reformat)) + return PTR_ERR(pkt_reformat); + + flow_act->pkt_reformat = pkt_reformat; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + return 0; +} + +static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +{ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; - u32 ipsec_obj_id = sa_entry->ipsec_obj_id; - struct mlx5_modify_hdr *modify_hdr = NULL; - struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; - enum accel_fs_esp_type type; struct mlx5_flow_spec *spec; - int err = 0; + struct mlx5e_ipsec_rx *rx; + int err; - accel_esp = priv->ipsec->rx_fs; - type = attrs->is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4; - fs_prot = &accel_esp->fs_prot[type]; - - err = rx_ft_get(priv, type); - if (err) - return err; + rx = rx_ft_get(mdev, ipsec, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out_err; + goto err_alloc; } - setup_fte_common(attrs, ipsec_obj_id, spec, &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); - /* Set bit[31] ipsec marker */ - /* Set bit[23-0] ipsec_obj_id */ - MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); - MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - MLX5_SET(set_action_in, action, data, (ipsec_obj_id | BIT(31))); - MLX5_SET(set_action_in, action, offset, 0); - MLX5_SET(set_action_in, action, length, 32); + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_no_frags(spec); - modify_hdr = mlx5_modify_header_alloc(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL, - 1, action); - if (IS_ERR(modify_hdr)) { - err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec set modify_header_id err=%d\n", err); - modify_hdr = NULL; - goto out_err; + err = setup_modify_header(mdev, sa_entry->ipsec_obj_id | BIT(31), + XFRM_DEV_OFFLOAD_IN, &flow_act); + if (err) + goto err_mod_header; + + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_PACKET: + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; } - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | - MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - flow_act.modify_hdr = modify_hdr; - dest.ft = fs_prot->rx_err.ft; - rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + dest.ft = rx->ft.status; + rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - attrs->action, err); - goto out_err; + mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); + goto err_add_flow; } + kvfree(spec); - ipsec_rule->rule = rule; - ipsec_rule->set_modify_hdr = modify_hdr; - goto out; - -out_err: - if (modify_hdr) - mlx5_modify_header_dealloc(priv->mdev, modify_hdr); - rx_ft_put(priv, type); + sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; + return 0; -out: +err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: kvfree(spec); +err_alloc: + rx_ft_put(mdev, ipsec, attrs->family); return err; } -static int tx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { + struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; int err = 0; - err = tx_ft_get(priv); - if (err) - return err; + tx = tx_ft_get(mdev, ipsec); + if (IS_ERR(tx)) + return PTR_ERR(tx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out; + goto err_alloc; } - setup_fte_common(&sa_entry->attrs, sa_entry->ipsec_obj_id, spec, - &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_reg_a(spec); + break; + case XFRM_DEV_OFFLOAD_PACKET: + setup_fte_reg_c0(spec, attrs->reqid); + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; + } - /* Add IPsec indicator in metadata_reg_a */ - spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; - MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); - MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); - - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; - rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0); + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(tx->fc->cnt); + rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - sa_entry->attrs.action, err); - goto out; + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); + goto err_add_flow; } + kvfree(spec); sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; + return 0; -out: +err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: kvfree(spec); +err_alloc: + tx_ft_put(ipsec); + return err; +} + +static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest[2] = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; + int err, dstn = 0; + + tx = tx_ft_get(mdev, pol_entry->ipsec); + if (IS_ERR(tx)) + return PTR_ERR(tx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } + + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT, + &flow_act); if (err) - tx_ft_put(priv); + goto err_mod_header; + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); + dstn++; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest[dstn].ft = tx->ft.sa; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dstn++; + rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, dest, dstn); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->ipsec_rule.rule = rule; + pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; + return 0; + +err_action: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: + kvfree(spec); +err_alloc: + tx_ft_put(pol_entry->ipsec); return err; } -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) - return tx_add_rule(priv, sa_entry); + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest[2]; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_rx *rx; + int err, dstn = 0; + + rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } - return rx_add_rule(priv, sa_entry); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); + dstn++; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[dstn].ft = rx->ft.sa; + dstn++; + rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, dest, dstn); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->ipsec_rule.rule = rule; + return 0; + +err_action: + kvfree(spec); +err_alloc: + rx_ft_put(mdev, pol_entry->ipsec, attrs->family); + return err; } -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + + mlx5_fc_destroy(mdev, tx->fc->drop); + mlx5_fc_destroy(mdev, tx->fc->cnt); + kfree(tx->fc); + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); + kfree(rx_ipv4->fc); +} + +static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_fc *fc; + struct mlx5_fc *counter; + int err; + + fc = kzalloc(sizeof(*rx_ipv4->fc), GFP_KERNEL); + if (!fc) + return -ENOMEM; + + /* Both IPv4 and IPv6 point to same flow counters struct. */ + rx_ipv4->fc = fc; + rx_ipv6->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_drop; + } + + fc->drop = counter; + fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); + if (!fc) { + err = -ENOMEM; + goto err_tx_fc; + } + + tx->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_drop; + } + + fc->drop = counter; + return 0; + +err_tx_drop: + mlx5_fc_destroy(mdev, tx->fc->cnt); +err_tx_cnt: + kfree(tx->fc); +err_tx_fc: + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); +err_rx_drop: + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); +err_rx_cnt: + kfree(rx_ipv4->fc); + return err; +} + +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_hw_stats *stats; + struct mlx5e_ipsec_fc *fc; + + stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; + + stats->ipsec_rx_pkts = 0; + stats->ipsec_rx_bytes = 0; + stats->ipsec_rx_drop_pkts = 0; + stats->ipsec_rx_drop_bytes = 0; + stats->ipsec_tx_pkts = 0; + stats->ipsec_tx_bytes = 0; + stats->ipsec_tx_drop_pkts = 0; + stats->ipsec_tx_drop_bytes = 0; + + fc = ipsec->rx_ipv4->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, + &stats->ipsec_rx_drop_bytes); + + fc = ipsec->tx->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, + &stats->ipsec_tx_drop_bytes); +} + +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +{ + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) + return tx_add_rule(sa_entry); + + return rx_add_rule(sa_entry); +} + +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); mlx5_del_flow_rules(ipsec_rule->rule); - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) { - tx_ft_put(priv); + if (ipsec_rule->pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); + + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { + tx_ft_put(sa_entry->ipsec); return; } - mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); - rx_ft_put(priv, - sa_entry->attrs.is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4); + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); + rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family); } -void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; - enum accel_fs_esp_type i; + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) + return tx_add_policy(pol_entry); - if (!ipsec->rx_fs) - return; + return rx_add_policy(pol_entry); +} + +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - mutex_destroy(&ipsec->tx_fs->mutex); - WARN_ON(ipsec->tx_fs->refcnt); - kfree(ipsec->tx_fs); + mlx5_del_flow_rules(ipsec_rule->rule); - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_destroy(&fs_prot->prot_mutex); - WARN_ON(fs_prot->refcnt); + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { + rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family); + return; } - kfree(ipsec->rx_fs); + + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); + tx_ft_put(pol_entry->ipsec); +} + +void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) +{ + if (!ipsec->tx) + return; + + ipsec_fs_destroy_counters(ipsec); + mutex_destroy(&ipsec->tx->ft.mutex); + WARN_ON(ipsec->tx->ft.refcnt); + kfree(ipsec->tx); + + mutex_destroy(&ipsec->rx_ipv4->ft.mutex); + WARN_ON(ipsec->rx_ipv4->ft.refcnt); + kfree(ipsec->rx_ipv4); + + mutex_destroy(&ipsec->rx_ipv6->ft.mutex); + WARN_ON(ipsec->rx_ipv6->ft.refcnt); + kfree(ipsec->rx_ipv6); } int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_namespace *ns; - enum accel_fs_esp_type i; int err = -ENOMEM; ns = mlx5_get_flow_namespace(ipsec->mdev, @@ -581,26 +1032,34 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) if (!ns) return -EOPNOTSUPP; - ipsec->tx_fs = kzalloc(sizeof(*ipsec->tx_fs), GFP_KERNEL); - if (!ipsec->tx_fs) + ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); + if (!ipsec->tx) return -ENOMEM; - ipsec->rx_fs = kzalloc(sizeof(*ipsec->rx_fs), GFP_KERNEL); - if (!ipsec->rx_fs) - goto err_rx; + ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); + if (!ipsec->rx_ipv4) + goto err_rx_ipv4; - mutex_init(&ipsec->tx_fs->mutex); - ipsec->tx_fs->ns = ns; + ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); + if (!ipsec->rx_ipv6) + goto err_rx_ipv6; - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_init(&fs_prot->prot_mutex); - } + err = ipsec_fs_init_counters(ipsec); + if (err) + goto err_counters; + + mutex_init(&ipsec->tx->ft.mutex); + mutex_init(&ipsec->rx_ipv4->ft.mutex); + mutex_init(&ipsec->rx_ipv6->ft.mutex); + ipsec->tx->ns = ns; return 0; -err_rx: - kfree(ipsec->tx_fs); +err_counters: + kfree(ipsec->rx_ipv6); +err_rx_ipv6: + kfree(ipsec->rx_ipv4); +err_rx_ipv4: + kfree(ipsec->tx); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 792724ce7336..8e3614218fc4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -2,9 +2,14 @@ /* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */ #include "mlx5_core.h" +#include "en.h" #include "ipsec.h" #include "lib/mlx5.h" +enum { + MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET, +}; + u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) { u32 caps = 0; @@ -31,6 +36,12 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp)) caps |= MLX5_IPSEC_CAP_CRYPTO; + if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) && + MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap)) + caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD; + if (!caps) return 0; @@ -46,6 +57,52 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) } EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps); +static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn, + struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + void *aso_ctx; + + aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso); + if (attrs->esn_trigger) { + MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1); + + if (attrs->dir == XFRM_DEV_OFFLOAD_IN) { + MLX5_SET(ipsec_aso, aso_ctx, window_sz, + attrs->replay_window / 64); + MLX5_SET(ipsec_aso, aso_ctx, mode, + MLX5_IPSEC_ASO_REPLAY_PROTECTION); + } + } + + /* ASO context */ + MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn); + MLX5_SET(ipsec_obj, obj, full_offload, 1); + MLX5_SET(ipsec_aso, aso_ctx, valid, 1); + /* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used + * in flow steering to perform matching against. Please be + * aware that this register was chosen arbitrary and can't + * be used in other places as long as IPsec packet offload + * active. + */ + MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5); + if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) + MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN); + + if (attrs->hard_packet_limit != XFRM_INF) { + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt, + lower_32_bits(attrs->hard_packet_limit)); + MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1); + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1); + } + + if (attrs->soft_packet_limit != XFRM_INF) { + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft, + lower_32_bits(attrs->soft_packet_limit)); + + MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1); + } +} + static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; @@ -54,6 +111,7 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {}; void *obj, *salt_p, *salt_iv_p; + struct mlx5e_hw_objs *res; int err; obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object); @@ -66,11 +124,10 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv); memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv)); /* esn */ - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) { + if (attrs->esn_trigger) { MLX5_SET(ipsec_obj, obj, esn_en, 1); MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn); - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) - MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap); } MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id); @@ -81,6 +138,10 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC); + res = &mdev->mlx5e_res.hw_objs; + if (attrs->type == XFRM_DEV_OFFLOAD_PACKET) + mlx5e_ipsec_packet_setup(obj, res->pdn, attrs); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) sa_entry->ipsec_obj_id = @@ -152,7 +213,7 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry, void *obj; int err; - if (!(attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED)) + if (!attrs->esn_trigger) return 0; general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); @@ -183,8 +244,7 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry, MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP | MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB); MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn); - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) - MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap); /* general object fields set */ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); @@ -203,3 +263,234 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs)); } + +static void +mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry, + const struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct mlx5_wqe_aso_ctrl_seg data = {}; + + data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6; + data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE + << 4; + data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET; + data.bitwise_data = cpu_to_be64(BIT_ULL(54)); + data.data_mask = data.bitwise_data; + + mlx5e_ipsec_aso_query(sa_entry, &data); +} + +static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, + u32 mode_param) +{ + struct mlx5_accel_esp_xfrm_attrs attrs = {}; + + if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) { + sa_entry->esn_state.esn++; + sa_entry->esn_state.overlap = 0; + } else { + sa_entry->esn_state.overlap = 1; + } + + mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); + mlx5_accel_esp_modify_xfrm(sa_entry, &attrs); + mlx5e_ipsec_aso_update_esn(sa_entry, &attrs); +} + +static void mlx5e_ipsec_handle_event(struct work_struct *_work) +{ + struct mlx5e_ipsec_work *work = + container_of(_work, struct mlx5e_ipsec_work, work); + struct mlx5_accel_esp_xfrm_attrs *attrs; + struct mlx5e_ipsec_sa_entry *sa_entry; + struct mlx5e_ipsec_aso *aso; + struct mlx5e_ipsec *ipsec; + int ret; + + sa_entry = xa_load(&work->ipsec->sadb, work->id); + if (!sa_entry) + goto out; + + ipsec = sa_entry->ipsec; + aso = ipsec->aso; + attrs = &sa_entry->attrs; + + spin_lock(&sa_entry->x->lock); + ret = mlx5e_ipsec_aso_query(sa_entry, NULL); + if (ret) + goto unlock; + + aso->use_cache = true; + if (attrs->esn_trigger && + !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) { + u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter); + + mlx5e_ipsec_update_esn_state(sa_entry, mode_param); + } + + if (attrs->soft_packet_limit != XFRM_INF) + if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable)) + xfrm_state_check_expire(sa_entry->x); + aso->use_cache = false; + +unlock: + spin_unlock(&sa_entry->x->lock); +out: + kfree(work); +} + +static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb); + struct mlx5_eqe_obj_change *object; + struct mlx5e_ipsec_work *work; + struct mlx5_eqe *eqe = data; + u16 type; + + if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE) + return NOTIFY_DONE; + + object = &eqe->data.obj_change; + type = be16_to_cpu(object->obj_type); + + if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5e_ipsec_handle_event); + work->ipsec = ipsec; + work->id = be32_to_cpu(object->obj_id); + + queue_work(ipsec->wq, &work->work); + return NOTIFY_OK; +} + +int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) +{ + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_aso *aso; + struct mlx5e_hw_objs *res; + struct device *pdev; + int err; + + aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL); + if (!aso) + return -ENOMEM; + + res = &mdev->mlx5e_res.hw_objs; + + pdev = mlx5_core_dma_dev(mdev); + aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); + err = dma_mapping_error(pdev, aso->dma_addr); + if (err) + goto err_dma; + + aso->aso = mlx5_aso_create(mdev, res->pdn); + if (IS_ERR(aso->aso)) { + err = PTR_ERR(aso->aso); + goto err_aso_create; + } + + ipsec->nb.notifier_call = mlx5e_ipsec_event; + mlx5_notifier_register(mdev, &ipsec->nb); + + ipsec->aso = aso; + return 0; + +err_aso_create: + dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); +err_dma: + kfree(aso); + return err; +} + +void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec) +{ + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_aso *aso; + struct device *pdev; + + aso = ipsec->aso; + pdev = mlx5_core_dma_dev(mdev); + + mlx5_notifier_unregister(mdev, &ipsec->nb); + mlx5_aso_destroy(aso->aso); + dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); + kfree(aso); +} + +static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl, + struct mlx5_wqe_aso_ctrl_seg *data) +{ + if (!data) + return; + + ctrl->data_mask_mode = data->data_mask_mode; + ctrl->condition_1_0_operand = data->condition_1_0_operand; + ctrl->condition_1_0_offset = data->condition_1_0_offset; + ctrl->data_offset_condition_operand = data->data_offset_condition_operand; + ctrl->condition_0_data = data->condition_0_data; + ctrl->condition_0_mask = data->condition_0_mask; + ctrl->condition_1_data = data->condition_1_data; + ctrl->condition_1_mask = data->condition_1_mask; + ctrl->bitwise_data = data->bitwise_data; + ctrl->data_mask = data->data_mask; +} + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data) +{ + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_aso *aso = ipsec->aso; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5_wqe_aso_ctrl_seg *ctrl; + struct mlx5e_hw_objs *res; + struct mlx5_aso_wqe *wqe; + u8 ds_cnt; + + lockdep_assert_held(&sa_entry->x->lock); + if (aso->use_cache) + return 0; + + res = &mdev->mlx5e_res.hw_objs; + + memset(aso->ctx, 0, sizeof(aso->ctx)); + wqe = mlx5_aso_get_wqe(aso->aso); + ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS); + mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id, + MLX5_ACCESS_ASO_OPC_MOD_IPSEC); + + ctrl = &wqe->aso_ctrl; + ctrl->va_l = + cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN); + ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr)); + ctrl->l_key = cpu_to_be32(res->mkey); + mlx5e_ipsec_aso_copy(ctrl, data); + + mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); + return mlx5_aso_poll_cq(aso->aso, false); +} + +void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, + u64 *packets) +{ + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_aso *aso = ipsec->aso; + u64 hard_cnt; + + hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt); + /* HW decresases the limit till it reaches zero to fire an avent. + * We need to fix the calculations, so the returned count is a total + * number of passed packets and not how much left. + */ + *packets = sa_entry->attrs.hard_packet_limit - hard_cnt; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 6859f1c1a831..eab5bc718771 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -312,27 +312,31 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct mlx5_cqe64 *cqe) { u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); - struct mlx5e_priv *priv; + struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_sa_entry *sa_entry; struct xfrm_offload *xo; - struct xfrm_state *xs; struct sec_path *sp; u32 sa_handle; sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data); - priv = netdev_priv(netdev); sp = secpath_set(skb); if (unlikely(!sp)) { - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sp_alloc); return; } - xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle); - if (unlikely(!xs)) { - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss); + rcu_read_lock(); + sa_entry = xa_load(&ipsec->sadb, sa_handle); + if (unlikely(!sa_entry)) { + rcu_read_unlock(); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); return; } + xfrm_state_hold(sa_entry->x); + rcu_read_unlock(); - sp->xvec[sp->len++] = xs; + sp->xvec[sp->len++] = sa_entry->x; sp->olen++; xo = xfrm_offload(skb); @@ -349,6 +353,6 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, xo->status = CRYPTO_INVALID_PACKET_SYNTAX; break; default: - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_syndrome); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c index 9de84821dafb..e0e36a09721c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c @@ -37,6 +37,17 @@ #include "en.h" #include "ipsec.h" +static const struct counter_desc mlx5e_ipsec_hw_stats_desc[] = { + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_drop_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_drop_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_drop_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_drop_bytes) }, +}; + static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sp_alloc) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sadb_miss) }, @@ -50,8 +61,48 @@ static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { #define MLX5E_READ_CTR_ATOMIC64(ptr, dsc, i) \ atomic64_read((atomic64_t *)((char *)(ptr) + (dsc)[i].offset)) +#define NUM_IPSEC_HW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_hw_stats_desc) #define NUM_IPSEC_SW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_sw_stats_desc) +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_hw) +{ + if (!priv->ipsec) + return 0; + + return NUM_IPSEC_HW_COUNTERS; +} + +static inline MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(ipsec_hw) {} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(ipsec_hw) +{ + unsigned int i; + + if (!priv->ipsec) + return idx; + + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + mlx5e_ipsec_hw_stats_desc[i].format); + + return idx; +} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_hw) +{ + int i; + + if (!priv->ipsec) + return idx; + + mlx5e_accel_ipsec_fs_read_stats(priv, &priv->ipsec->hw_stats); + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + data[idx++] = MLX5E_READ_CTR_ATOMIC64(&priv->ipsec->hw_stats, + mlx5e_ipsec_hw_stats_desc, i); + + return idx; +} + static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_sw) { return priv->ipsec ? NUM_IPSEC_SW_COUNTERS : 0; @@ -81,4 +132,5 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_sw) return idx; } +MLX5E_DEFINE_STATS_GRP(ipsec_hw, 0); MLX5E_DEFINE_STATS_GRP(ipsec_sw, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index f900709639f6..9369a580743e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -186,7 +186,7 @@ static int mlx5e_macsec_aso_reg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macs return err; } - dma_device = &mdev->pdev->dev; + dma_device = mlx5_core_dma_dev(mdev); dma_addr = dma_map_single(dma_device, umr->ctx, sizeof(umr->ctx), DMA_BIDIRECTIONAL); err = dma_mapping_error(dma_device, dma_addr); if (err) { @@ -1299,12 +1299,12 @@ static void macsec_aso_build_wqe_ctrl_seg(struct mlx5e_macsec_aso *macsec_aso, struct mlx5_wqe_aso_ctrl_seg *aso_ctrl, struct mlx5_aso_ctrl_param *param) { + struct mlx5e_macsec_umr *umr = macsec_aso->umr; + memset(aso_ctrl, 0, sizeof(*aso_ctrl)); - if (macsec_aso->umr->dma_addr) { - aso_ctrl->va_l = cpu_to_be32(macsec_aso->umr->dma_addr | ASO_CTRL_READ_EN); - aso_ctrl->va_h = cpu_to_be32((u64)macsec_aso->umr->dma_addr >> 32); - aso_ctrl->l_key = cpu_to_be32(macsec_aso->umr->mkey); - } + aso_ctrl->va_l = cpu_to_be32(umr->dma_addr | ASO_CTRL_READ_EN); + aso_ctrl->va_h = cpu_to_be32((u64)umr->dma_addr >> 32); + aso_ctrl->l_key = cpu_to_be32(umr->mkey); if (!param) return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 217c8a478977..8d36e2de53a9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -208,7 +208,7 @@ static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_ u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); u32 sz; - sz = ALIGN(entries * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT); + sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); return sz / MLX5_OCTWORD; } @@ -5238,10 +5238,6 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, } priv->fs = fs; - err = mlx5e_ipsec_init(priv); - if (err) - mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err); - err = mlx5e_ktls_init(priv); if (err) mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); @@ -5254,7 +5250,6 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) { mlx5e_health_destroy_reporters(priv); mlx5e_ktls_cleanup(priv); - mlx5e_ipsec_cleanup(priv); mlx5e_fs_cleanup(priv->fs); } @@ -5383,6 +5378,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) int err; mlx5e_fs_init_l2_addr(priv->fs, netdev); + mlx5e_ipsec_init(priv); err = mlx5e_macsec_init(priv); if (err) @@ -5446,6 +5442,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv) mlx5_lag_remove_netdev(mdev, priv->netdev); mlx5_vxlan_reset_to_default(mdev->vxlan); mlx5e_macsec_cleanup(priv); + mlx5e_ipsec_cleanup(priv); } int mlx5e_update_nic_rx(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 1b53e8852c86..75b9e1528fd2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -85,18 +85,25 @@ static const struct counter_desc sw_rep_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) }, }; -struct vport_stats { - u64 vport_rx_packets; - u64 vport_tx_packets; - u64 vport_rx_bytes; - u64 vport_tx_bytes; -}; - static const struct counter_desc vport_rep_stats_desc[] = { - { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_packets) }, - { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_bytes) }, - { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_packets) }, - { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + rx_vport_rdma_unicast_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, rx_vport_rdma_unicast_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + tx_vport_rdma_unicast_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, tx_vport_rdma_unicast_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + rx_vport_rdma_multicast_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + rx_vport_rdma_multicast_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + tx_vport_rdma_multicast_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, + tx_vport_rdma_multicast_bytes) }, }; #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc) @@ -161,33 +168,80 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep) int i; for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) - data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.vf_vport, + data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats, vport_rep_stats_desc, i); return idx; } static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep) { + struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats; + int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; - struct rtnl_link_stats64 *vport_stats; - struct ifla_vf_stats vf_stats; + u32 *out; int err; - err = mlx5_eswitch_get_vport_stats(esw, rep->vport, &vf_stats); + out = kvzalloc(outlen, GFP_KERNEL); + if (!out) + return; + + err = mlx5_core_query_vport_counter(esw->dev, 1, rep->vport - 1, 0, out); if (err) { netdev_warn(priv->netdev, "vport %d error %d reading stats\n", rep->vport, err); return; } - vport_stats = &priv->stats.vf_vport; + #define MLX5_GET_CTR(p, x) \ + MLX5_GET64(query_vport_counter_out, p, x) /* flip tx/rx as we are reporting the counters for the switch vport */ - vport_stats->rx_packets = vf_stats.tx_packets; - vport_stats->rx_bytes = vf_stats.tx_bytes; - vport_stats->tx_packets = vf_stats.rx_packets; - vport_stats->tx_bytes = vf_stats.rx_bytes; + rep_stats->vport_rx_packets = + MLX5_GET_CTR(out, transmitted_ib_unicast.packets) + + MLX5_GET_CTR(out, transmitted_eth_unicast.packets) + + MLX5_GET_CTR(out, transmitted_ib_multicast.packets) + + MLX5_GET_CTR(out, transmitted_eth_multicast.packets) + + MLX5_GET_CTR(out, transmitted_eth_broadcast.packets); + + rep_stats->vport_tx_packets = + MLX5_GET_CTR(out, received_ib_unicast.packets) + + MLX5_GET_CTR(out, received_eth_unicast.packets) + + MLX5_GET_CTR(out, received_ib_multicast.packets) + + MLX5_GET_CTR(out, received_eth_multicast.packets) + + MLX5_GET_CTR(out, received_eth_broadcast.packets); + + rep_stats->vport_rx_bytes = + MLX5_GET_CTR(out, transmitted_ib_unicast.octets) + + MLX5_GET_CTR(out, transmitted_eth_unicast.octets) + + MLX5_GET_CTR(out, transmitted_ib_multicast.octets) + + MLX5_GET_CTR(out, transmitted_eth_broadcast.octets); + + rep_stats->vport_tx_bytes = + MLX5_GET_CTR(out, received_ib_unicast.octets) + + MLX5_GET_CTR(out, received_eth_unicast.octets) + + MLX5_GET_CTR(out, received_ib_multicast.octets) + + MLX5_GET_CTR(out, received_eth_multicast.octets) + + MLX5_GET_CTR(out, received_eth_broadcast.octets); + + rep_stats->rx_vport_rdma_unicast_packets = + MLX5_GET_CTR(out, transmitted_ib_unicast.packets); + rep_stats->tx_vport_rdma_unicast_packets = + MLX5_GET_CTR(out, received_ib_unicast.packets); + rep_stats->rx_vport_rdma_unicast_bytes = + MLX5_GET_CTR(out, transmitted_ib_unicast.octets); + rep_stats->tx_vport_rdma_unicast_bytes = + MLX5_GET_CTR(out, received_ib_unicast.octets); + rep_stats->rx_vport_rdma_multicast_packets = + MLX5_GET_CTR(out, transmitted_ib_multicast.packets); + rep_stats->tx_vport_rdma_multicast_packets = + MLX5_GET_CTR(out, received_ib_multicast.packets); + rep_stats->rx_vport_rdma_multicast_bytes = + MLX5_GET_CTR(out, transmitted_ib_multicast.octets); + rep_stats->tx_vport_rdma_multicast_bytes = + MLX5_GET_CTR(out, received_ib_multicast.octets); + + kvfree(out); } static void mlx5e_rep_get_strings(struct net_device *dev, @@ -751,7 +805,6 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); - int err; priv->fs = mlx5e_fs_init(priv->profile, mdev, !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); @@ -760,10 +813,6 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, return -ENOMEM; } - err = mlx5e_ipsec_init(priv); - if (err) - mlx5_core_err(mdev, "Uplink rep IPsec initialization failed, %d\n", err); - mlx5e_vxlan_set_netdev_info(priv); mlx5e_build_rep_params(netdev); mlx5e_timestamp_init(priv); @@ -773,7 +822,6 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) { mlx5e_fs_cleanup(priv->fs); - mlx5e_ipsec_cleanup(priv); } static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) @@ -1112,6 +1160,8 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) struct mlx5_core_dev *mdev = priv->mdev; u16 max_mtu; + mlx5e_ipsec_init(priv); + netdev->min_mtu = ETH_MIN_MTU; mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); @@ -1158,6 +1208,8 @@ static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) mlx5e_rep_tc_disable(priv); mlx5_lag_remove_netdev(mdev, priv->netdev); mlx5_vxlan_reset_to_default(mdev->vxlan); + + mlx5e_ipsec_cleanup(priv); } static MLX5E_DEFINE_STATS_GRP(sw_rep, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index b1ea0b995d9c..c8820ab22169 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -593,8 +593,8 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, int headroom, i; headroom = rq->buff.headroom; - new_entries = klm_entries - (shampo->pi & (MLX5_UMR_KLM_ALIGNMENT - 1)); - entries = ALIGN(klm_entries, MLX5_UMR_KLM_ALIGNMENT); + new_entries = klm_entries - (shampo->pi & (MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT - 1)); + entries = ALIGN(klm_entries, MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT); wqe_bbs = MLX5E_KLM_UMR_WQEBBS(entries); pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs); umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); @@ -603,7 +603,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, for (i = 0; i < entries; i++, index++) { dma_info = &shampo->info[index]; if (i >= klm_entries || (index < shampo->pi && shampo->pi - index < - MLX5_UMR_KLM_ALIGNMENT)) + MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT)) goto update_klm; header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) << MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; @@ -668,8 +668,8 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) if (!klm_entries) return 0; - klm_entries += (shampo->pi & (MLX5_UMR_KLM_ALIGNMENT - 1)); - index = ALIGN_DOWN(shampo->pi, MLX5_UMR_KLM_ALIGNMENT); + klm_entries += (shampo->pi & (MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT - 1)); + index = ALIGN_DOWN(shampo->pi, MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT); entries_before = shampo->hd_per_wq - index; if (unlikely(entries_before < klm_entries)) @@ -727,6 +727,17 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) }; } + /* Pad if needed, in case the value set to ucseg->xlt_octowords + * in mlx5e_build_umr_wqe() needed alignment. + */ + if (rq->mpwqe.pages_per_wqe & (MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1)) { + int pad = ALIGN(rq->mpwqe.pages_per_wqe, MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT) - + rq->mpwqe.pages_per_wqe; + + memset(&umr_wqe->inline_mtts[rq->mpwqe.pages_per_wqe], 0, + sizeof(*umr_wqe->inline_mtts) * pad); + } + bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); wi->consumed_strides = 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 70c4ea3841d7..6687b8136e44 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -2480,6 +2480,7 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = { &MLX5E_STATS_GRP(per_prio), &MLX5E_STATS_GRP(pme), #ifdef CONFIG_MLX5_EN_IPSEC + &MLX5E_STATS_GRP(ipsec_hw), &MLX5E_STATS_GRP(ipsec_sw), #endif &MLX5E_STATS_GRP(tls), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index cbc831ca646b..375752d6546d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -463,6 +463,21 @@ struct mlx5e_ptp_cq_stats { u64 resync_event; }; +struct mlx5e_rep_stats { + u64 vport_rx_packets; + u64 vport_tx_packets; + u64 vport_rx_bytes; + u64 vport_tx_bytes; + u64 rx_vport_rdma_unicast_packets; + u64 tx_vport_rdma_unicast_packets; + u64 rx_vport_rdma_unicast_bytes; + u64 tx_vport_rdma_unicast_bytes; + u64 rx_vport_rdma_multicast_packets; + u64 tx_vport_rdma_multicast_packets; + u64 rx_vport_rdma_multicast_bytes; + u64 tx_vport_rdma_multicast_bytes; +}; + struct mlx5e_stats { struct mlx5e_sw_stats sw; struct mlx5e_qcounter_stats qcnt; @@ -471,6 +486,7 @@ struct mlx5e_stats { struct mlx5e_pport_stats pport; struct rtnl_link_stats64 vf_vport; struct mlx5e_pcie_stats pcie; + struct mlx5e_rep_stats rep_stats; }; extern mlx5e_stats_grp_t mlx5e_nic_stats_grps[]; @@ -490,6 +506,7 @@ extern MLX5E_DECLARE_STATS_GRP(per_prio); extern MLX5E_DECLARE_STATS_GRP(pme); extern MLX5E_DECLARE_STATS_GRP(channels); extern MLX5E_DECLARE_STATS_GRP(per_port_buff_congest); +extern MLX5E_DECLARE_STATS_GRP(ipsec_hw); extern MLX5E_DECLARE_STATS_GRP(ipsec_sw); extern MLX5E_DECLARE_STATS_GRP(ptp); extern MLX5E_DECLARE_STATS_GRP(macsec_hw); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 10d1609ece58..9af2aa2922f5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -132,6 +132,15 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { [PACKET_COLOR_TO_REG] = packet_color_to_reg, }; +struct mlx5e_tc_jump_state { + u32 jump_count; + bool jump_target; + struct mlx5_flow_attr *jumping_attr; + + enum flow_action_id last_id; + u32 last_index; +}; + struct mlx5e_tc_table *mlx5e_tc_table_alloc(void) { struct mlx5e_tc_table *tc; @@ -160,6 +169,7 @@ static struct lock_class_key tc_ht_lock_key; static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow); static void free_flow_post_acts(struct mlx5e_tc_flow *flow); +static void mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr); void mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, @@ -392,8 +402,9 @@ mlx5_tc_rule_delete(struct mlx5e_priv *priv, static bool is_flow_meter_action(struct mlx5_flow_attr *attr) { - return ((attr->action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && - (attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)); + return (((attr->action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && + (attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)) || + attr->flags & MLX5_ATTR_FLAG_MTU); } static int @@ -404,6 +415,7 @@ mlx5e_tc_add_flow_meter(struct mlx5e_priv *priv, struct mlx5e_post_meter_priv *post_meter; enum mlx5_flow_namespace_type ns_type; struct mlx5e_flow_meter_handle *meter; + enum mlx5e_post_meter_type type; meter = mlx5e_tc_meter_replace(priv->mdev, &attr->meter_attr.params); if (IS_ERR(meter)) { @@ -412,8 +424,11 @@ mlx5e_tc_add_flow_meter(struct mlx5e_priv *priv, } ns_type = mlx5e_tc_meter_get_namespace(meter->flow_meters); - post_meter = mlx5e_post_meter_init(priv, ns_type, post_act, meter->green_counter, - meter->red_counter); + type = meter->params.mtu ? MLX5E_POST_METER_MTU : MLX5E_POST_METER_RATE; + post_meter = mlx5e_post_meter_init(priv, ns_type, post_act, + type, + meter->act_counter, meter->drop_counter, + attr->branch_true, attr->branch_false); if (IS_ERR(post_meter)) { mlx5_core_err(priv->mdev, "Failed to init post meter\n"); goto err_meter_init; @@ -432,9 +447,9 @@ err_meter_init: } static void -mlx5e_tc_del_flow_meter(struct mlx5_flow_attr *attr) +mlx5e_tc_del_flow_meter(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) { - mlx5e_post_meter_cleanup(attr->meter_attr.post_meter); + mlx5e_post_meter_cleanup(esw, attr->meter_attr.post_meter); mlx5e_tc_meter_put(attr->meter_attr.meter); } @@ -495,7 +510,7 @@ mlx5e_tc_rule_unoffload(struct mlx5e_priv *priv, mlx5_eswitch_del_offloaded_rule(esw, rule, attr); if (attr->meter_attr.meter) - mlx5e_tc_del_flow_meter(attr); + mlx5e_tc_del_flow_meter(esw, attr); } int @@ -606,6 +621,12 @@ int mlx5e_get_flow_namespace(struct mlx5e_tc_flow *flow) MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL; } +static struct mlx5_core_dev * +get_flow_counter_dev(struct mlx5e_tc_flow *flow) +{ + return mlx5e_is_eswitch_flow(flow) ? flow->attr->esw_attr->counter_dev : flow->priv->mdev; +} + static struct mod_hdr_tbl * get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { @@ -1719,6 +1740,90 @@ clean_encap_dests(struct mlx5e_priv *priv, } static int +verify_attr_actions(u32 actions, struct netlink_ext_ack *extack) +{ + if (!(actions & + (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) { + NL_SET_ERR_MSG_MOD(extack, "Rule must have at least one forward/drop action"); + return -EOPNOTSUPP; + } + + if (!(~actions & + (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) { + NL_SET_ERR_MSG_MOD(extack, "Rule cannot support forward+drop action"); + return -EOPNOTSUPP; + } + + if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR && + actions & MLX5_FLOW_CONTEXT_ACTION_DROP) { + NL_SET_ERR_MSG_MOD(extack, "Drop with modify header action is not supported"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int +post_process_attr(struct mlx5e_tc_flow *flow, + struct mlx5_flow_attr *attr, + bool is_post_act_attr, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw = flow->priv->mdev->priv.eswitch; + bool vf_tun; + int err = 0; + + err = verify_attr_actions(attr->action, extack); + if (err) + goto err_out; + + err = set_encap_dests(flow->priv, flow, attr, extack, &vf_tun); + if (err) + goto err_out; + + if (mlx5e_is_eswitch_flow(flow)) { + err = mlx5_eswitch_add_vlan_action(esw, attr); + if (err) + goto err_out; + } + + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { + if (vf_tun || is_post_act_attr) { + err = mlx5e_tc_add_flow_mod_hdr(flow->priv, flow, attr); + if (err) + goto err_out; + } else { + err = mlx5e_attach_mod_hdr(flow->priv, flow, attr->parse_attr); + if (err) + goto err_out; + } + } + + if (attr->branch_true && + attr->branch_true->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { + err = mlx5e_tc_add_flow_mod_hdr(flow->priv, flow, attr->branch_true); + if (err) + goto err_out; + } + + if (attr->branch_false && + attr->branch_false->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { + err = mlx5e_tc_add_flow_mod_hdr(flow->priv, flow, attr->branch_false); + if (err) + goto err_out; + } + + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { + err = alloc_flow_attr_counter(get_flow_counter_dev(flow), attr); + if (err) + goto err_out; + } + +err_out: + return err; +} + +static int mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) @@ -1728,7 +1833,6 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr = flow->attr; struct mlx5_esw_flow_attr *esw_attr; u32 max_prio, max_chain; - bool vf_tun; int err = 0; parse_attr = attr->parse_attr; @@ -1818,32 +1922,10 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, esw_attr->int_port = int_port; } - err = set_encap_dests(priv, flow, attr, extack, &vf_tun); - if (err) - goto err_out; - - err = mlx5_eswitch_add_vlan_action(esw, attr); + err = post_process_attr(flow, attr, false, extack); if (err) goto err_out; - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { - if (vf_tun) { - err = mlx5e_tc_add_flow_mod_hdr(priv, flow, attr); - if (err) - goto err_out; - } else { - err = mlx5e_attach_mod_hdr(priv, flow, parse_attr); - if (err) - goto err_out; - } - } - - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { - err = alloc_flow_attr_counter(esw_attr->counter_dev, attr); - if (err) - goto err_out; - } - /* we get here if one of the following takes place: * (1) there's no error * (2) there's an encap action and we don't have valid neigh @@ -1879,6 +1961,16 @@ static bool mlx5_flow_has_geneve_opt(struct mlx5e_tc_flow *flow) return !!geneve_tlv_opt_0_data; } +static void free_branch_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr) +{ + if (!attr) + return; + + mlx5_free_flow_attr(flow, attr); + kvfree(attr->parse_attr); + kfree(attr); +} + static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { @@ -1934,6 +2026,8 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, mlx5e_detach_decap(priv, flow); free_flow_post_acts(flow); + free_branch_attr(flow, attr->branch_true); + free_branch_attr(flow, attr->branch_false); if (flow->attr->lag.count) mlx5_lag_del_mpesw_rule(esw->dev); @@ -3507,36 +3601,6 @@ actions_match_supported(struct mlx5e_priv *priv, ct_clear = flow->attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR; ct_flow = flow_flag_test(flow, CT) && !ct_clear; - if (!(actions & - (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) { - NL_SET_ERR_MSG_MOD(extack, "Rule must have at least one forward/drop action"); - return false; - } - - if (!(~actions & - (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) { - NL_SET_ERR_MSG_MOD(extack, "Rule cannot support forward+drop action"); - return false; - } - - if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR && - actions & MLX5_FLOW_CONTEXT_ACTION_DROP) { - NL_SET_ERR_MSG_MOD(extack, "Drop with modify header action is not supported"); - return false; - } - - if (!(~actions & - (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) { - NL_SET_ERR_MSG_MOD(extack, "Rule cannot support forward+drop action"); - return false; - } - - if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR && - actions & MLX5_FLOW_CONTEXT_ACTION_DROP) { - NL_SET_ERR_MSG_MOD(extack, "Drop with modify header action is not supported"); - return false; - } - if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR && !modify_header_match_supported(priv, &parse_attr->spec, flow_action, actions, ct_flow, ct_clear, extack)) @@ -3636,15 +3700,12 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr, attr2->esw_attr->split_count = 0; } + attr2->branch_true = NULL; + attr2->branch_false = NULL; + attr2->jumping_attr = NULL; return attr2; } -static struct mlx5_core_dev * -get_flow_counter_dev(struct mlx5e_tc_flow *flow) -{ - return mlx5e_is_eswitch_flow(flow) ? flow->attr->esw_attr->counter_dev : flow->priv->mdev; -} - struct mlx5_flow_attr * mlx5e_tc_get_encap_attr(struct mlx5e_tc_flow *flow) { @@ -3680,28 +3741,15 @@ mlx5e_tc_unoffload_flow_post_acts(struct mlx5e_tc_flow *flow) static void free_flow_post_acts(struct mlx5e_tc_flow *flow) { - struct mlx5_core_dev *counter_dev = get_flow_counter_dev(flow); - struct mlx5e_post_act *post_act = get_post_action(flow->priv); struct mlx5_flow_attr *attr, *tmp; - bool vf_tun; list_for_each_entry_safe(attr, tmp, &flow->attrs, list) { if (list_is_last(&attr->list, &flow->attrs)) break; - if (attr->post_act_handle) - mlx5e_tc_post_act_del(post_act, attr->post_act_handle); - - clean_encap_dests(flow->priv, flow, attr, &vf_tun); - - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) - mlx5_fc_destroy(counter_dev, attr->counter); - - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { - mlx5e_mod_hdr_dealloc(&attr->parse_attr->mod_hdr_acts); - if (attr->modify_hdr) - mlx5_modify_header_dealloc(flow->priv->mdev, attr->modify_hdr); - } + mlx5_free_flow_attr(flow, attr); + free_branch_attr(flow, attr->branch_true); + free_branch_attr(flow, attr->branch_false); list_del(&attr->list); kvfree(attr->parse_attr); @@ -3754,7 +3802,6 @@ alloc_flow_post_acts(struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) struct mlx5e_post_act *post_act = get_post_action(flow->priv); struct mlx5_flow_attr *attr, *next_attr = NULL; struct mlx5e_post_act_handle *handle; - bool vf_tun; int err; /* This is going in reverse order as needed. @@ -3764,7 +3811,9 @@ alloc_flow_post_acts(struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) if (!next_attr) { /* Set counter action on last post act rule. */ attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; - } else { + } + + if (next_attr && !(attr->flags & MLX5_ATTR_FLAG_TERMINATING)) { err = mlx5e_tc_act_set_next_post_act(flow, attr, next_attr); if (err) goto out_free; @@ -3776,26 +3825,14 @@ alloc_flow_post_acts(struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) if (list_is_last(&attr->list, &flow->attrs)) break; - err = set_encap_dests(flow->priv, flow, attr, extack, &vf_tun); + err = actions_prepare_mod_hdr_actions(flow->priv, flow, attr, extack); if (err) goto out_free; - err = actions_prepare_mod_hdr_actions(flow->priv, flow, attr, extack); + err = post_process_attr(flow, attr, true, extack); if (err) goto out_free; - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { - err = mlx5e_tc_add_flow_mod_hdr(flow->priv, flow, attr); - if (err) - goto out_free; - } - - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { - err = alloc_flow_attr_counter(get_flow_counter_dev(flow), attr); - if (err) - goto out_free; - } - handle = mlx5e_tc_post_act_add(post_act, attr); if (IS_ERR(handle)) { err = PTR_ERR(handle); @@ -3803,6 +3840,13 @@ alloc_flow_post_acts(struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) } attr->post_act_handle = handle; + + if (attr->jumping_attr) { + err = mlx5e_tc_act_set_next_post_act(flow, attr->jumping_attr, attr); + if (err) + goto out_free; + } + next_attr = attr; } @@ -3822,12 +3866,145 @@ out_free: } static int +alloc_branch_attr(struct mlx5e_tc_flow *flow, + struct mlx5e_tc_act_branch_ctrl *cond, + struct mlx5_flow_attr **cond_attr, + u32 *jump_count, + struct netlink_ext_ack *extack) +{ + struct mlx5_flow_attr *attr; + int err = 0; + + *cond_attr = mlx5e_clone_flow_attr_for_post_act(flow->attr, + mlx5e_get_flow_namespace(flow)); + if (!(*cond_attr)) + return -ENOMEM; + + attr = *cond_attr; + + switch (cond->act_id) { + case FLOW_ACTION_DROP: + attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + break; + case FLOW_ACTION_ACCEPT: + case FLOW_ACTION_PIPE: + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + attr->dest_ft = mlx5e_tc_post_act_get_ft(get_post_action(flow->priv)); + break; + case FLOW_ACTION_JUMP: + if (*jump_count) { + NL_SET_ERR_MSG_MOD(extack, "Cannot offload flows with nested jumps"); + err = -EOPNOTSUPP; + goto out_err; + } + *jump_count = cond->extval; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + attr->dest_ft = mlx5e_tc_post_act_get_ft(get_post_action(flow->priv)); + break; + default: + err = -EOPNOTSUPP; + goto out_err; + } + + return err; +out_err: + kfree(*cond_attr); + *cond_attr = NULL; + return err; +} + +static void +dec_jump_count(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act, + struct mlx5_flow_attr *attr, struct mlx5e_priv *priv, + struct mlx5e_tc_jump_state *jump_state) +{ + if (!jump_state->jump_count) + return; + + /* Single tc action can instantiate multiple offload actions (e.g. pedit) + * Jump only over a tc action + */ + if (act->id == jump_state->last_id && act->hw_index == jump_state->last_index) + return; + + jump_state->last_id = act->id; + jump_state->last_index = act->hw_index; + + /* nothing to do for intermediate actions */ + if (--jump_state->jump_count > 1) + return; + + if (jump_state->jump_count == 1) { /* last action in the jump action list */ + + /* create a new attribute after this action */ + jump_state->jump_target = true; + + if (tc_act->is_terminating_action) { /* the branch ends here */ + attr->flags |= MLX5_ATTR_FLAG_TERMINATING; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + } else { /* the branch continues executing the rest of the actions */ + struct mlx5e_post_act *post_act; + + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + post_act = get_post_action(priv); + attr->dest_ft = mlx5e_tc_post_act_get_ft(post_act); + } + } else if (jump_state->jump_count == 0) { /* first attr after the jump action list */ + /* This is the post action for the jumping attribute (either red or green) + * Use the stored jumping_attr to set the post act id on the jumping attribute + */ + attr->jumping_attr = jump_state->jumping_attr; + } +} + +static int +parse_branch_ctrl(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act, + struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr, + struct mlx5e_tc_jump_state *jump_state, + struct netlink_ext_ack *extack) +{ + struct mlx5e_tc_act_branch_ctrl cond_true, cond_false; + u32 jump_count = jump_state->jump_count; + int err; + + if (!tc_act->get_branch_ctrl) + return 0; + + tc_act->get_branch_ctrl(act, &cond_true, &cond_false); + + err = alloc_branch_attr(flow, &cond_true, + &attr->branch_true, &jump_count, extack); + if (err) + goto out_err; + + if (jump_count) + jump_state->jumping_attr = attr->branch_true; + + err = alloc_branch_attr(flow, &cond_false, + &attr->branch_false, &jump_count, extack); + if (err) + goto err_branch_false; + + if (jump_count && !jump_state->jumping_attr) + jump_state->jumping_attr = attr->branch_false; + + jump_state->jump_count = jump_count; + return 0; + +err_branch_false: + free_branch_attr(flow, attr->branch_true); +out_err: + return err; +} + +static int parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state, struct flow_action *flow_action) { struct netlink_ext_ack *extack = parse_state->extack; struct mlx5e_tc_flow_action flow_action_reorder; struct mlx5e_tc_flow *flow = parse_state->flow; + struct mlx5e_tc_jump_state jump_state = {}; struct mlx5_flow_attr *attr = flow->attr; enum mlx5_flow_namespace_type ns_type; struct mlx5e_priv *priv = flow->priv; @@ -3847,6 +4024,7 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state, list_add(&attr->list, &flow->attrs); flow_action_for_each(i, _act, &flow_action_reorder) { + jump_state.jump_target = false; act = *_act; tc_act = mlx5e_tc_act_get(act->id, ns_type); if (!tc_act) { @@ -3864,12 +4042,19 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state, if (err) goto out_free; + dec_jump_count(act, tc_act, attr, priv, &jump_state); + + err = parse_branch_ctrl(act, tc_act, flow, attr, &jump_state, extack); + if (err) + goto out_free; + parse_state->actions |= attr->action; /* Split attr for multi table act if not the last act. */ - if (tc_act->is_multi_table_act && + if (jump_state.jump_target || + (tc_act->is_multi_table_act && tc_act->is_multi_table_act(priv, act, attr) && - i < flow_action_reorder.num_entries - 1) { + i < flow_action_reorder.num_entries - 1)) { err = mlx5e_tc_act_post_parse(parse_state, flow_action, attr, ns_type); if (err) goto out_free; @@ -3951,6 +4136,10 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, if (err) return err; + err = verify_attr_actions(attr->action, extack); + if (err) + return err; + if (!actions_match_supported(priv, flow_action, parse_state->actions, parse_attr, flow, extack)) return -EOPNOTSUPP; @@ -4188,6 +4377,30 @@ mlx5_alloc_flow_attr(enum mlx5_flow_namespace_type type) return attr; } +static void +mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr) +{ + struct mlx5_core_dev *counter_dev = get_flow_counter_dev(flow); + bool vf_tun; + + if (!attr) + return; + + if (attr->post_act_handle) + mlx5e_tc_post_act_del(get_post_action(flow->priv), attr->post_act_handle); + + clean_encap_dests(flow->priv, flow, attr, &vf_tun); + + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) + mlx5_fc_destroy(counter_dev, attr->counter); + + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { + mlx5e_mod_hdr_dealloc(&attr->parse_attr->mod_hdr_acts); + if (attr->modify_hdr) + mlx5_modify_header_dealloc(flow->priv->mdev, attr->modify_hdr); + } +} + static int mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size, struct flow_cls_offload *f, unsigned long flow_flags, @@ -4730,10 +4943,17 @@ static int apply_police_params(struct mlx5e_priv *priv, u64 rate, return err; } -int mlx5e_policer_validate(const struct flow_action *action, - const struct flow_action_entry *act, - struct netlink_ext_ack *extack) +static int +tc_matchall_police_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack) { + if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not continue"); + return -EOPNOTSUPP; + } + if (act->police.exceed.act_id != FLOW_ACTION_DROP) { NL_SET_ERR_MSG_MOD(extack, "Offload not supported when exceed action is not drop"); @@ -4784,13 +5004,7 @@ static int scan_tc_matchall_fdb_actions(struct mlx5e_priv *priv, flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_POLICE: - if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE) { - NL_SET_ERR_MSG_MOD(extack, - "Offload not supported when conform action is not continue"); - return -EOPNOTSUPP; - } - - err = mlx5e_policer_validate(flow_action, act, extack); + err = tc_matchall_police_validate(flow_action, act, extack); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index 48241317a535..50af70ef22f3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -95,10 +95,13 @@ struct mlx5_flow_attr { */ bool count; } lag; + struct mlx5_flow_attr *branch_true; + struct mlx5_flow_attr *branch_false; + struct mlx5_flow_attr *jumping_attr; /* keep this union last */ union { - struct mlx5_esw_flow_attr esw_attr[0]; - struct mlx5_nic_flow_attr nic_attr[0]; + DECLARE_FLEX_ARRAY(struct mlx5_esw_flow_attr, esw_attr); + DECLARE_FLEX_ARRAY(struct mlx5_nic_flow_attr, nic_attr); }; }; @@ -110,6 +113,8 @@ enum { MLX5_ATTR_FLAG_SAMPLE = BIT(4), MLX5_ATTR_FLAG_ACCEPT = BIT(5), MLX5_ATTR_FLAG_CT = BIT(6), + MLX5_ATTR_FLAG_TERMINATING = BIT(7), + MLX5_ATTR_FLAG_MTU = BIT(8), }; /* Returns true if any of the flags that require skipping further TC/NF processing are set. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index a0242dc15741..8f7580fec193 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -19,6 +19,7 @@ #include "diag/fw_tracer.h" #include "mlx5_irq.h" #include "devlink.h" +#include "en_accel/ipsec.h" enum { MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -578,6 +579,10 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + async_event_mask |= + (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + mask[0] = async_event_mask; if (MLX5_CAP_GEN(dev, event_cap)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c index 2db13c71e88c..3d0bbcca1cb9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c @@ -12,10 +12,11 @@ enum vnic_diag_counter { MLX5_VNIC_DIAG_CQ_OVERRUN, MLX5_VNIC_DIAG_INVALID_COMMAND, MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND, + MLX5_VNIC_DIAG_RX_STEERING_DISCARD, }; static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_counter counter, - u32 *val) + u64 *val) { u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {}; @@ -57,6 +58,10 @@ static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_cou case MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND: *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, quota_exceeded_command); break; + case MLX5_VNIC_DIAG_RX_STEERING_DISCARD: + *val = MLX5_GET64(vnic_diagnostic_statistics, vnic_diag_out, + nic_receive_steering_discard); + break; } return 0; @@ -65,14 +70,14 @@ static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_cou static int __show_vnic_diag(struct seq_file *file, struct mlx5_vport *vport, enum vnic_diag_counter type) { - u32 val = 0; + u64 val = 0; int ret; ret = mlx5_esw_query_vnic_diag(vport, type, &val); if (ret) return ret; - seq_printf(file, "%d\n", val); + seq_printf(file, "%llu\n", val); return 0; } @@ -112,6 +117,11 @@ static int quota_exceeded_command_show(struct seq_file *file, void *priv) return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND); } +static int rx_steering_discard_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_RX_STEERING_DISCARD); +} + DEFINE_SHOW_ATTRIBUTE(total_q_under_processor_handle); DEFINE_SHOW_ATTRIBUTE(send_queue_priority_update_flow); DEFINE_SHOW_ATTRIBUTE(comp_eq_overrun); @@ -119,6 +129,7 @@ DEFINE_SHOW_ATTRIBUTE(async_eq_overrun); DEFINE_SHOW_ATTRIBUTE(cq_overrun); DEFINE_SHOW_ATTRIBUTE(invalid_command); DEFINE_SHOW_ATTRIBUTE(quota_exceeded_command); +DEFINE_SHOW_ATTRIBUTE(rx_steering_discard); void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num) { @@ -179,4 +190,9 @@ void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool if (MLX5_CAP_GEN(esw->dev, quota_exceeded_count)) debugfs_create_file("quota_exceeded_command", 0444, vnic_diag, vport, "a_exceeded_command_fops); + + if (MLX5_CAP_GEN(esw->dev, nic_receive_steering_discard)) + debugfs_create_file("rx_steering_discard", 0444, vnic_diag, vport, + &rx_steering_discard_fops); + } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 374e3fbdc2cf..527e4bffda8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -772,6 +772,41 @@ static void esw_vport_cleanup_acl(struct mlx5_eswitch *esw, esw_vport_destroy_offloads_acl_tables(esw, vport); } +static int mlx5_esw_vport_caps_get(struct mlx5_eswitch *esw, struct mlx5_vport *vport) +{ + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + void *query_ctx; + void *hca_caps; + int err; + + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) + return 0; + + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + if (!query_ctx) + return -ENOMEM; + + err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, + MLX5_CAP_GENERAL); + if (err) + goto out_free; + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + vport->info.roce_enabled = MLX5_GET(cmd_hca_cap, hca_caps, roce); + + memset(query_ctx, 0, query_out_sz); + err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, + MLX5_CAP_GENERAL_2); + if (err) + goto out_free; + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + vport->info.mig_enabled = MLX5_GET(cmd_hca_cap_2, hca_caps, migratable); +out_free: + kfree(query_ctx); + return err; +} + static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { u16 vport_num = vport->vport; @@ -785,6 +820,10 @@ static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) if (mlx5_esw_is_manager_vport(esw, vport_num)) return 0; + err = mlx5_esw_vport_caps_get(esw, vport); + if (err) + goto err_caps; + mlx5_modify_vport_admin_state(esw->dev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, vport_num, 1, @@ -804,6 +843,10 @@ static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) vport->info.qos, flags); return 0; + +err_caps: + esw_vport_cleanup_acl(esw, vport); + return err; } /* Don't cleanup vport->info, it's needed to restore vport configuration */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 3029bc1c0dd0..5a85a5d32be7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -153,6 +153,8 @@ struct mlx5_vport_info { u8 qos; u8 spoofchk: 1; u8 trusted: 1; + u8 roce_enabled: 1; + u8 mig_enabled: 1; }; /* Vport context events */ @@ -508,7 +510,14 @@ int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, const u8 *hw_addr, int hw_addr_len, struct netlink_ext_ack *extack); - +int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, + struct netlink_ext_ack *extack); +int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, + struct netlink_ext_ack *extack); +int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, + struct netlink_ext_ack *extack); +int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, + struct netlink_ext_ack *extack); void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type); int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, @@ -744,6 +753,11 @@ static inline int mlx5_eswitch_num_vfs(struct mlx5_eswitch *esw) return 0; } +static inline struct mlx5_flow_table * +mlx5_eswitch_get_slow_fdb(struct mlx5_eswitch *esw) +{ + return esw->fdb_table.offloads.slow_fdb; +} #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 8c6c9bcb3dc3..e455b215c708 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -50,6 +50,7 @@ #include "en/mapping.h" #include "devlink.h" #include "lag/lag.h" +#include "en/tc/post_meter.h" #define mlx5_esw_for_each_rep(esw, i, rep) \ xa_for_each(&((esw)->offloads.vport_reps), i, rep) @@ -202,6 +203,21 @@ esw_cleanup_decap_indir(struct mlx5_eswitch *esw, } static int +esw_setup_mtu_dest(struct mlx5_flow_destination *dest, + struct mlx5e_meter_attr *meter, + int i) +{ + dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; + dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; + dest[i].range.min = 0; + dest[i].range.max = meter->params.mtu; + dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); + dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); + + return 0; +} + +static int esw_setup_sampler_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, u32 sampler_id, @@ -248,7 +264,7 @@ esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_ac if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest[i].ft = esw->fdb_table.offloads.slow_fdb; + dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); } static int @@ -479,16 +495,21 @@ esw_setup_dests(struct mlx5_flow_destination *dest, esw_src_port_rewrite_supported(esw)) attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; - if (attr->flags & MLX5_ATTR_FLAG_SAMPLE && - !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) { - esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); - (*i)++; - } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { + if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { esw_setup_slow_path_dest(dest, flow_act, esw, *i); (*i)++; + goto out; + } + + if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { + esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); + (*i)++; } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { esw_setup_accept_dest(dest, flow_act, chains, *i); (*i)++; + } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { + err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); + (*i)++; } else if (esw_is_indir_table(esw, attr)) { err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i); } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { @@ -506,6 +527,7 @@ esw_setup_dests(struct mlx5_flow_destination *dest, } } +out: return err; } @@ -637,6 +659,11 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, goto err_esw_get; } + if (!i) { + kfree(dest); + dest = NULL; + } + if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, &flow_act, dest, i); @@ -1046,7 +1073,7 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, if (rep->vport == MLX5_VPORT_UPLINK) spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; - flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb, + flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", @@ -1095,7 +1122,7 @@ mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); dest.vport.num = vport_num; - flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", @@ -1248,7 +1275,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, spec, MLX5_VPORT_PF); - flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow)) { err = PTR_ERR(flow); @@ -1260,7 +1287,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, if (mlx5_ecpf_vport_exists(esw->dev)) { vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); - flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow)) { err = PTR_ERR(flow); @@ -1274,7 +1301,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, peer_dev->priv.eswitch, spec, vport->vport); - flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow)) { err = PTR_ERR(flow); @@ -1363,7 +1390,7 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) dest.vport.num = esw->manager_vport; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) { err = PTR_ERR(flow_rule); @@ -1378,7 +1405,7 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); dmac_v[0] = 0x01; - flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); if (IS_ERR(flow_rule)) { err = PTR_ERR(flow_rule); @@ -1927,7 +1954,7 @@ send_vport_err: fdb_chains_err: mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); tc_miss_table_err: - mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); + mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); slow_fdb_err: /* Holds true only as long as DMFS is the default */ mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); @@ -1938,7 +1965,7 @@ ns_err: static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) { - if (!esw->fdb_table.offloads.slow_fdb) + if (!mlx5_eswitch_get_slow_fdb(esw)) return; esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); @@ -1954,7 +1981,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) esw_chains_destroy(esw, esw_chains(esw)); mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); - mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); + mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); /* Holds true only as long as DMFS is the default */ mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, MLX5_FLOW_STEERING_MODE_DMFS); @@ -3886,7 +3913,7 @@ static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, if (!query_ctx) return -ENOMEM; - err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx); + err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); if (err) goto out_free; @@ -4019,3 +4046,212 @@ int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr); } + +static struct mlx5_vport * +mlx5_devlink_port_fn_get_vport(struct devlink_port *port, struct mlx5_eswitch *esw) +{ + u16 vport_num; + + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) + return ERR_PTR(-EOPNOTSUPP); + + vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); + if (!is_port_function_supported(esw, vport_num)) + return ERR_PTR(-EOPNOTSUPP); + + return mlx5_eswitch_get_vport(esw, vport_num); +} + +int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + int err = -EOPNOTSUPP; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + if (!MLX5_CAP_GEN(esw->dev, migration)) { + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); + return err; + } + + vport = mlx5_devlink_port_fn_get_vport(port, esw); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + if (vport->enabled) { + *is_enabled = vport->info.mig_enabled; + err = 0; + } + mutex_unlock(&esw->state_lock); + return err; +} + +int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, + struct netlink_ext_ack *extack) +{ + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + void *query_ctx; + void *hca_caps; + int err = -EOPNOTSUPP; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + if (!MLX5_CAP_GEN(esw->dev, migration)) { + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); + return err; + } + + vport = mlx5_devlink_port_fn_get_vport(port, esw); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + if (!vport->enabled) { + NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); + goto out; + } + + if (vport->info.mig_enabled == enable) { + err = 0; + goto out; + } + + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + if (!query_ctx) { + err = -ENOMEM; + goto out; + } + + err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, + MLX5_CAP_GENERAL_2); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); + goto out_free; + } + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + memcpy(hca_caps, MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability), + MLX5_UN_SZ_BYTES(hca_cap_union)); + MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, 1); + + err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); + goto out_free; + } + + vport->info.mig_enabled = enable; + +out_free: + kfree(query_ctx); +out: + mutex_unlock(&esw->state_lock); + return err; +} + +int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + int err = -EOPNOTSUPP; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + vport = mlx5_devlink_port_fn_get_vport(port, esw); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + if (vport->enabled) { + *is_enabled = vport->info.roce_enabled; + err = 0; + } + mutex_unlock(&esw->state_lock); + return err; +} + +int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, + struct netlink_ext_ack *extack) +{ + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + int err = -EOPNOTSUPP; + void *query_ctx; + void *hca_caps; + u16 vport_num; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + vport = mlx5_devlink_port_fn_get_vport(port, esw); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + vport_num = vport->vport; + + mutex_lock(&esw->state_lock); + if (!vport->enabled) { + NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); + goto out; + } + + if (vport->info.roce_enabled == enable) { + err = 0; + goto out; + } + + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + if (!query_ctx) { + err = -ENOMEM; + goto out; + } + + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, + MLX5_CAP_GENERAL); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); + goto out_free; + } + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + memcpy(hca_caps, MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability), + MLX5_UN_SZ_BYTES(hca_cap_union)); + MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); + + err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); + goto out_free; + } + + vport->info.roce_enabled = enable; + +out_free: + kfree(query_ctx); +out: + mutex_unlock(&esw->state_lock); + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c index edd910258314..3a9a6bb9158d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c @@ -210,6 +210,18 @@ static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw, return (port_mask & port_value) == MLX5_VPORT_UPLINK; } +static bool +mlx5_eswitch_is_push_vlan_no_cap(struct mlx5_eswitch *esw, + struct mlx5_flow_act *flow_act) +{ + if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH && + !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) & + MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX)) + return true; + + return false; +} + bool mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, @@ -225,10 +237,7 @@ mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port)) return false; - /* push vlan on RX */ - if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH && - !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) & - MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX)) + if (mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act)) return true; /* hairpin */ @@ -252,19 +261,31 @@ mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, struct mlx5_flow_act term_tbl_act = {}; struct mlx5_flow_handle *rule = NULL; bool term_table_created = false; + bool is_push_vlan_on_rx; int num_vport_dests = 0; int i, curr_dest; + is_push_vlan_on_rx = mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act); mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act); term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; for (i = 0; i < num_dest; i++) { struct mlx5_termtbl_handle *tt; + bool hairpin = false; /* only vport destinations can be terminated */ if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) continue; + if (attr->dests[num_vport_dests].rep && + attr->dests[num_vport_dests].rep->vport == MLX5_VPORT_UPLINK) + hairpin = true; + + if (!is_push_vlan_on_rx && !hairpin) { + num_vport_dests++; + continue; + } + if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) { term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat; @@ -312,6 +333,9 @@ revert_changes: for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) { struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl; + if (!tt) + continue; + attr->dests[curr_dest].termtbl = NULL; /* search for the destination associated with the diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index d53749248fa0..5a85d8c1e797 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -111,8 +111,8 @@ #define ETHTOOL_PRIO_NUM_LEVELS 1 #define ETHTOOL_NUM_PRIOS 11 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) -/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}} */ -#define KERNEL_NIC_PRIO_NUM_LEVELS 7 +/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy */ +#define KERNEL_NIC_PRIO_NUM_LEVELS 8 #define KERNEL_NIC_NUM_PRIOS 1 /* One more level for tc */ #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) @@ -133,7 +133,7 @@ #define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1) #define KERNEL_TX_IPSEC_NUM_PRIOS 1 -#define KERNEL_TX_IPSEC_NUM_LEVELS 1 +#define KERNEL_TX_IPSEC_NUM_LEVELS 2 #define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS) #define KERNEL_TX_MACSEC_NUM_PRIOS 1 @@ -448,7 +448,8 @@ static bool is_fwd_dest_type(enum mlx5_flow_destination_type type) type == MLX5_FLOW_DESTINATION_TYPE_UPLINK || type == MLX5_FLOW_DESTINATION_TYPE_VPORT || type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER || - type == MLX5_FLOW_DESTINATION_TYPE_TIR; + type == MLX5_FLOW_DESTINATION_TYPE_TIR || + type == MLX5_FLOW_DESTINATION_TYPE_RANGE; } static bool check_valid_spec(const struct mlx5_flow_spec *spec) @@ -1578,7 +1579,13 @@ static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1, (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM && d1->ft_num == d2->ft_num) || (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER && - d1->sampler_id == d2->sampler_id)) + d1->sampler_id == d2->sampler_id) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_RANGE && + d1->range.field == d2->range.field && + d1->range.hit_ft == d2->range.hit_ft && + d1->range.miss_ft == d2->range.miss_ft && + d1->range.min == d2->range.min && + d1->range.max == d2->range.max)) return true; } @@ -1962,6 +1969,9 @@ _mlx5_add_flow_rules(struct mlx5_flow_table *ft, if (flow_act->fg && ft->autogroup.active) return ERR_PTR(-EINVAL); + if (dest && dest_num <= 0) + return ERR_PTR(-EINVAL); + for (i = 0; i < dest_num; i++) { if (!dest_is_valid(&dest[i], flow_act, ft)) return ERR_PTR(-EINVAL); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 3af50fd04d28..f137a0611b77 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -123,6 +123,7 @@ enum mlx5_flow_steering_mode { enum mlx5_flow_steering_capabilty { MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0, MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1, + MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2, }; struct mlx5_flow_steering { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c index 0259a149a64c..d9fcb9ed726f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c @@ -118,13 +118,41 @@ struct mlx5_fib_event_work { }; }; +static struct net_device* +mlx5_lag_get_next_fib_dev(struct mlx5_lag *ldev, + struct fib_info *fi, + struct net_device *current_dev) +{ + struct net_device *fib_dev; + int i, ldev_idx, nhs; + + nhs = fib_info_num_path(fi); + i = 0; + if (current_dev) { + for (; i < nhs; i++) { + fib_dev = fib_info_nh(fi, i)->fib_nh_dev; + if (fib_dev == current_dev) { + i++; + break; + } + } + } + for (; i < nhs; i++) { + fib_dev = fib_info_nh(fi, i)->fib_nh_dev; + ldev_idx = mlx5_lag_dev_get_netdev_idx(ldev, fib_dev); + if (ldev_idx >= 0) + return ldev->pf[ldev_idx].netdev; + } + + return NULL; +} + static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, struct fib_entry_notifier_info *fen_info) { + struct net_device *nh_dev0, *nh_dev1; struct fib_info *fi = fen_info->fi; struct lag_mp *mp = &ldev->lag_mp; - struct fib_nh *fib_nh0, *fib_nh1; - unsigned int nhs; /* Handle delete event */ if (event == FIB_EVENT_ENTRY_DEL) { @@ -140,16 +168,25 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, fi->fib_priority >= mp->fib.priority) return; + nh_dev0 = mlx5_lag_get_next_fib_dev(ldev, fi, NULL); + nh_dev1 = mlx5_lag_get_next_fib_dev(ldev, fi, nh_dev0); + /* Handle add/replace event */ - nhs = fib_info_num_path(fi); - if (nhs == 1) { - if (__mlx5_lag_is_active(ldev)) { - struct fib_nh *nh = fib_info_nh(fi, 0); - struct net_device *nh_dev = nh->fib_nh_dev; - int i = mlx5_lag_dev_get_netdev_idx(ldev, nh_dev); + if (!nh_dev0) { + if (mp->fib.dst == fen_info->dst && mp->fib.dst_len == fen_info->dst_len) + mp->fib.mfi = NULL; + return; + } - if (i < 0) - return; + if (nh_dev0 == nh_dev1) { + mlx5_core_warn(ldev->pf[MLX5_LAG_P1].dev, + "Multipath offload doesn't support routes with multiple nexthops of the same device"); + return; + } + + if (!nh_dev1) { + if (__mlx5_lag_is_active(ldev)) { + int i = mlx5_lag_dev_get_netdev_idx(ldev, nh_dev0); i++; mlx5_lag_set_port_affinity(ldev, i); @@ -159,21 +196,6 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, return; } - if (nhs != 2) - return; - - /* Verify next hops are ports of the same hca */ - fib_nh0 = fib_info_nh(fi, 0); - fib_nh1 = fib_info_nh(fi, 1); - if (!(fib_nh0->fib_nh_dev == ldev->pf[MLX5_LAG_P1].netdev && - fib_nh1->fib_nh_dev == ldev->pf[MLX5_LAG_P2].netdev) && - !(fib_nh0->fib_nh_dev == ldev->pf[MLX5_LAG_P2].netdev && - fib_nh1->fib_nh_dev == ldev->pf[MLX5_LAG_P1].netdev)) { - mlx5_core_warn(ldev->pf[MLX5_LAG_P1].dev, - "Multipath offload require two ports of the same HCA\n"); - return; - } - /* First time we see multipath route */ if (!mp->fib.mfi && !__mlx5_lag_is_active(ldev)) { struct lag_tracker tracker; @@ -268,7 +290,6 @@ static int mlx5_lag_fib_event(struct notifier_block *nb, struct mlx5_fib_event_work *fib_work; struct fib_entry_notifier_info *fen_info; struct fib_nh_notifier_info *fnh_info; - struct net_device *fib_dev; struct fib_info *fi; if (info->family != AF_INET) @@ -285,11 +306,7 @@ static int mlx5_lag_fib_event(struct notifier_block *nb, fi = fen_info->fi; if (fi->nh) return NOTIFY_DONE; - fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev; - if (fib_dev != ldev->pf[MLX5_LAG_P1].netdev && - fib_dev != ldev->pf[MLX5_LAG_P2].netdev) { - return NOTIFY_DONE; - } + fib_work = mlx5_lag_init_fib_work(ldev, event); if (!fib_work) return NOTIFY_DONE; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c index c971ff04dd04..5a80fb7dbbca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -334,9 +334,6 @@ err_cq: void mlx5_aso_destroy(struct mlx5_aso *aso) { - if (IS_ERR_OR_NULL(aso)) - return; - mlx5_aso_destroy_sq(aso); mlx5_aso_destroy_cq(&aso->cq); kfree(aso); @@ -356,12 +353,15 @@ void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, cseg->general_id = cpu_to_be32(obj_id); } -void *mlx5_aso_get_wqe(struct mlx5_aso *aso) +struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso) { + struct mlx5_aso_wqe *wqe; u16 pi; pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc); - return mlx5_wq_cyc_get_wqe(&aso->wq, pi); + wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi); + memset(wqe, 0, sizeof(*wqe)); + return wqe; } void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h index 2d40dcf9d42e..afb078bbb8ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -15,6 +15,7 @@ #define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24 #define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS)) +#define ASO_CTRL_READ_EN BIT(0) struct mlx5_wqe_aso_ctrl_seg { __be32 va_h; __be32 va_l; /* include read_enable */ @@ -71,13 +72,14 @@ enum { }; enum { + MLX5_ACCESS_ASO_OPC_MOD_IPSEC = 0x0, MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2, MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5, }; struct mlx5_aso; -void *mlx5_aso_get_wqe(struct mlx5_aso *aso); +struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso); void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, struct mlx5_aso_wqe *aso_wqe, u32 obj_id, u32 opc_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 70e8dc305bec..7f5db13e3550 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -37,7 +37,6 @@ #include <linux/pci.h> #include <linux/dma-mapping.h> #include <linux/slab.h> -#include <linux/io-mapping.h> #include <linux/interrupt.h> #include <linux/delay.h> #include <linux/mlx5/driver.h> @@ -1604,8 +1603,6 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) int err; memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); - INIT_LIST_HEAD(&priv->ctx_list); - spin_lock_init(&priv->ctx_lock); lockdep_register_key(&dev->lock_key); mutex_init(&dev->intf_state_mutex); lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index a806e3de7b7c..029305a8b80a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -324,7 +324,10 @@ void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev); int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery); int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery); -int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out); +int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap, u16 function_id, + u16 opmod); +#define mlx5_vport_get_other_func_general_cap(dev, fid, out) \ + mlx5_vport_get_other_func_cap(dev, fid, out, MLX5_CAP_GENERAL) void mlx5_events_work_enqueue(struct mlx5_core_dev *dev, struct work_struct *work); static inline u32 mlx5_sriov_get_vf_total_msix(struct pci_dev *pdev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 662f1d55e30e..6bde18bcd42f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -4,6 +4,7 @@ #include <linux/interrupt.h> #include <linux/notifier.h> #include <linux/mlx5/driver.h> +#include <linux/mlx5/vport.h> #include "mlx5_core.h" #include "mlx5_irq.h" #include "pci_irq.h" @@ -101,7 +102,7 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id, goto out; } - ret = mlx5_vport_get_other_func_cap(dev, function_id, query_cap); + ret = mlx5_vport_get_other_func_general_cap(dev, function_id, query_cap); if (ret) goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index b1dfad274a39..ee104cf04392 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -44,6 +44,7 @@ static const char * const action_type_to_str[] = { [DR_ACTION_TYP_INSERT_HDR] = "DR_ACTION_TYP_INSERT_HDR", [DR_ACTION_TYP_REMOVE_HDR] = "DR_ACTION_TYP_REMOVE_HDR", [DR_ACTION_TYP_ASO_FLOW_METER] = "DR_ACTION_TYP_ASO_FLOW_METER", + [DR_ACTION_TYP_RANGE] = "DR_ACTION_TYP_RANGE", [DR_ACTION_TYP_MAX] = "DR_ACTION_UNKNOWN", }; @@ -61,6 +62,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, @@ -79,6 +81,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_DECAP, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, @@ -94,6 +97,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, @@ -103,6 +107,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, @@ -116,6 +121,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_POP_VLAN, @@ -129,6 +135,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_PUSH_VLAN, @@ -141,6 +148,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, @@ -159,6 +167,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_TERM] = { @@ -169,6 +178,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, @@ -183,6 +193,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, @@ -190,6 +201,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, @@ -197,6 +209,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, @@ -207,6 +220,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, @@ -220,6 +234,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, @@ -231,6 +246,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, @@ -250,6 +266,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_TERM] = { [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_TERM, @@ -259,6 +276,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, @@ -276,6 +294,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -291,6 +310,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, @@ -299,6 +319,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -311,6 +332,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_POP_VLAN, @@ -324,6 +346,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_PUSH_VLAN, @@ -337,6 +360,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, @@ -354,6 +378,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_ASO] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, @@ -365,6 +390,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -380,6 +406,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, @@ -388,6 +415,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -396,6 +424,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, @@ -407,6 +436,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, @@ -421,6 +451,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_PUSH_VLAN, @@ -433,6 +464,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -452,6 +484,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_RANGE] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, @@ -634,6 +667,83 @@ static void dr_action_print_sequence(struct mlx5dr_domain *dmn, actions[i]->action_type); } +static int dr_action_get_dest_fw_tbl_addr(struct mlx5dr_matcher *matcher, + struct mlx5dr_action_dest_tbl *dest_tbl, + bool is_rx_rule, + u64 *final_icm_addr) +{ + struct mlx5dr_cmd_query_flow_table_details output; + struct mlx5dr_domain *dmn = matcher->tbl->dmn; + int ret; + + if (!dest_tbl->fw_tbl.rx_icm_addr) { + ret = mlx5dr_cmd_query_flow_table(dmn->mdev, + dest_tbl->fw_tbl.type, + dest_tbl->fw_tbl.id, + &output); + if (ret) { + mlx5dr_err(dmn, + "Failed mlx5_cmd_query_flow_table ret: %d\n", + ret); + return ret; + } + + dest_tbl->fw_tbl.tx_icm_addr = output.sw_owner_icm_root_1; + dest_tbl->fw_tbl.rx_icm_addr = output.sw_owner_icm_root_0; + } + + *final_icm_addr = is_rx_rule ? dest_tbl->fw_tbl.rx_icm_addr : + dest_tbl->fw_tbl.tx_icm_addr; + return 0; +} + +static int dr_action_get_dest_sw_tbl_addr(struct mlx5dr_matcher *matcher, + struct mlx5dr_action_dest_tbl *dest_tbl, + bool is_rx_rule, + u64 *final_icm_addr) +{ + struct mlx5dr_domain *dmn = matcher->tbl->dmn; + struct mlx5dr_icm_chunk *chunk; + + if (dest_tbl->tbl->dmn != dmn) { + mlx5dr_err(dmn, + "Destination table belongs to a different domain\n"); + return -EINVAL; + } + + if (dest_tbl->tbl->level <= matcher->tbl->level) { + mlx5_core_dbg_once(dmn->mdev, + "Connecting table to a lower/same level destination table\n"); + mlx5dr_dbg(dmn, + "Connecting table at level %d to a destination table at level %d\n", + matcher->tbl->level, + dest_tbl->tbl->level); + } + + chunk = is_rx_rule ? dest_tbl->tbl->rx.s_anchor->chunk : + dest_tbl->tbl->tx.s_anchor->chunk; + + *final_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(chunk); + return 0; +} + +static int dr_action_get_dest_tbl_addr(struct mlx5dr_matcher *matcher, + struct mlx5dr_action_dest_tbl *dest_tbl, + bool is_rx_rule, + u64 *final_icm_addr) +{ + if (dest_tbl->is_fw_tbl) + return dr_action_get_dest_fw_tbl_addr(matcher, + dest_tbl, + is_rx_rule, + final_icm_addr); + + return dr_action_get_dest_sw_tbl_addr(matcher, + dest_tbl, + is_rx_rule, + final_icm_addr); +} + #define WITH_VLAN_NUM_HW_ACTIONS 6 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, @@ -661,8 +771,6 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, action_domain = dr_action_get_action_domain(dmn->type, nic_dmn->type); for (i = 0; i < num_actions; i++) { - struct mlx5dr_action_dest_tbl *dest_tbl; - struct mlx5dr_icm_chunk *chunk; struct mlx5dr_action *action; int max_actions_type = 1; u32 action_type; @@ -676,50 +784,27 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, break; case DR_ACTION_TYP_FT: dest_action = action; - dest_tbl = action->dest_tbl; - if (!dest_tbl->is_fw_tbl) { - if (dest_tbl->tbl->dmn != dmn) { - mlx5dr_err(dmn, - "Destination table belongs to a different domain\n"); - return -EINVAL; - } - if (dest_tbl->tbl->level <= matcher->tbl->level) { - mlx5_core_dbg_once(dmn->mdev, - "Connecting table to a lower/same level destination table\n"); - mlx5dr_dbg(dmn, - "Connecting table at level %d to a destination table at level %d\n", - matcher->tbl->level, - dest_tbl->tbl->level); - } - chunk = rx_rule ? dest_tbl->tbl->rx.s_anchor->chunk : - dest_tbl->tbl->tx.s_anchor->chunk; - attr.final_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(chunk); - } else { - struct mlx5dr_cmd_query_flow_table_details output; - int ret; - - /* get the relevant addresses */ - if (!action->dest_tbl->fw_tbl.rx_icm_addr) { - ret = mlx5dr_cmd_query_flow_table(dmn->mdev, - dest_tbl->fw_tbl.type, - dest_tbl->fw_tbl.id, - &output); - if (!ret) { - dest_tbl->fw_tbl.tx_icm_addr = - output.sw_owner_icm_root_1; - dest_tbl->fw_tbl.rx_icm_addr = - output.sw_owner_icm_root_0; - } else { - mlx5dr_err(dmn, - "Failed mlx5_cmd_query_flow_table ret: %d\n", - ret); - return ret; - } - } - attr.final_icm_addr = rx_rule ? - dest_tbl->fw_tbl.rx_icm_addr : - dest_tbl->fw_tbl.tx_icm_addr; - } + ret = dr_action_get_dest_tbl_addr(matcher, action->dest_tbl, + rx_rule, &attr.final_icm_addr); + if (ret) + return ret; + break; + case DR_ACTION_TYP_RANGE: + ret = dr_action_get_dest_tbl_addr(matcher, + action->range->hit_tbl_action->dest_tbl, + rx_rule, &attr.final_icm_addr); + if (ret) + return ret; + + ret = dr_action_get_dest_tbl_addr(matcher, + action->range->miss_tbl_action->dest_tbl, + rx_rule, &attr.range.miss_icm_addr); + if (ret) + return ret; + + attr.range.definer_id = action->range->definer_id; + attr.range.min = action->range->min; + attr.range.max = action->range->max; break; case DR_ACTION_TYP_QP: mlx5dr_info(dmn, "Domain doesn't support QP\n"); @@ -866,6 +951,7 @@ static unsigned int action_size[DR_ACTION_TYP_MAX] = { [DR_ACTION_TYP_REMOVE_HDR] = sizeof(struct mlx5dr_action_reformat), [DR_ACTION_TYP_SAMPLER] = sizeof(struct mlx5dr_action_sampler), [DR_ACTION_TYP_ASO_FLOW_METER] = sizeof(struct mlx5dr_action_aso_flow_meter), + [DR_ACTION_TYP_RANGE] = sizeof(struct mlx5dr_action_range), }; static struct mlx5dr_action * @@ -933,6 +1019,123 @@ dec_ref: return NULL; } +static void dr_action_range_definer_fill(u16 *format_id, + u8 *dw_selectors, + u8 *byte_selectors, + u8 *match_mask) +{ + int i; + + *format_id = MLX5_IFC_DEFINER_FORMAT_ID_SELECT; + + dw_selectors[0] = MLX5_IFC_DEFINER_FORMAT_OFFSET_OUTER_ETH_PKT_LEN / 4; + + for (i = 1; i < MLX5_IFC_DEFINER_DW_SELECTORS_NUM; i++) + dw_selectors[i] = MLX5_IFC_DEFINER_FORMAT_OFFSET_UNUSED; + + for (i = 0; i < MLX5_IFC_DEFINER_BYTE_SELECTORS_NUM; i++) + byte_selectors[i] = MLX5_IFC_DEFINER_FORMAT_OFFSET_UNUSED; + + MLX5_SET(match_definer_match_mask, match_mask, + match_dw_0, 0xffffUL << 16); +} + +static int dr_action_create_range_definer(struct mlx5dr_action *action) +{ + u8 match_mask[MLX5_FLD_SZ_BYTES(match_definer, match_mask)] = {}; + u8 byte_selectors[MLX5_IFC_DEFINER_BYTE_SELECTORS_NUM] = {}; + u8 dw_selectors[MLX5_IFC_DEFINER_DW_SELECTORS_NUM] = {}; + struct mlx5dr_domain *dmn = action->range->dmn; + u32 definer_id; + u16 format_id; + int ret; + + dr_action_range_definer_fill(&format_id, + dw_selectors, + byte_selectors, + match_mask); + + ret = mlx5dr_definer_get(dmn, format_id, + dw_selectors, byte_selectors, + match_mask, &definer_id); + if (ret) + return ret; + + action->range->definer_id = definer_id; + return 0; +} + +static void dr_action_destroy_range_definer(struct mlx5dr_action *action) +{ + mlx5dr_definer_put(action->range->dmn, action->range->definer_id); +} + +struct mlx5dr_action * +mlx5dr_action_create_dest_match_range(struct mlx5dr_domain *dmn, + u32 field, + struct mlx5_flow_table *hit_ft, + struct mlx5_flow_table *miss_ft, + u32 min, + u32 max) +{ + struct mlx5dr_action *action; + int ret; + + if (!mlx5dr_supp_match_ranges(dmn->mdev)) { + mlx5dr_dbg(dmn, "SELECT definer support is needed for match range\n"); + return NULL; + } + + if (field != MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN || + min > 0xffff || max > 0xffff) { + mlx5dr_err(dmn, "Invalid match range parameters\n"); + return NULL; + } + + action = dr_action_create_generic(DR_ACTION_TYP_RANGE); + if (!action) + return NULL; + + action->range->hit_tbl_action = + mlx5dr_is_fw_table(hit_ft) ? + mlx5dr_action_create_dest_flow_fw_table(dmn, hit_ft) : + mlx5dr_action_create_dest_table(hit_ft->fs_dr_table.dr_table); + + if (!action->range->hit_tbl_action) + goto free_action; + + action->range->miss_tbl_action = + mlx5dr_is_fw_table(miss_ft) ? + mlx5dr_action_create_dest_flow_fw_table(dmn, miss_ft) : + mlx5dr_action_create_dest_table(miss_ft->fs_dr_table.dr_table); + + if (!action->range->miss_tbl_action) + goto free_hit_tbl_action; + + action->range->min = min; + action->range->max = max; + action->range->dmn = dmn; + + ret = dr_action_create_range_definer(action); + if (ret) + goto free_miss_tbl_action; + + /* No need to increase refcount on domain for this action, + * the hit/miss table actions will do it internally. + */ + + return action; + +free_miss_tbl_action: + mlx5dr_action_destroy(action->range->miss_tbl_action); +free_hit_tbl_action: + mlx5dr_action_destroy(action->range->hit_tbl_action); +free_action: + kfree(action); + + return NULL; +} + struct mlx5dr_action * mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn, struct mlx5dr_action_dest *dests, @@ -1980,6 +2183,11 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action) case DR_ACTION_TYP_ASO_FLOW_METER: refcount_dec(&action->aso->dmn->refcount); break; + case DR_ACTION_TYP_RANGE: + dr_action_destroy_range_definer(action); + mlx5dr_action_destroy(action->range->miss_tbl_action); + mlx5dr_action_destroy(action->range->hit_tbl_action); + break; default: break; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index b4739eafc180..07b6a6dcb92f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -564,6 +564,83 @@ void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, mlx5_cmd_exec_in(mdev, dealloc_packet_reformat_context, in); } +static void dr_cmd_set_definer_format(void *ptr, u16 format_id, + u8 *dw_selectors, + u8 *byte_selectors) +{ + if (format_id != MLX5_IFC_DEFINER_FORMAT_ID_SELECT) + return; + + MLX5_SET(match_definer, ptr, format_select_dw0, dw_selectors[0]); + MLX5_SET(match_definer, ptr, format_select_dw1, dw_selectors[1]); + MLX5_SET(match_definer, ptr, format_select_dw2, dw_selectors[2]); + MLX5_SET(match_definer, ptr, format_select_dw3, dw_selectors[3]); + MLX5_SET(match_definer, ptr, format_select_dw4, dw_selectors[4]); + MLX5_SET(match_definer, ptr, format_select_dw5, dw_selectors[5]); + MLX5_SET(match_definer, ptr, format_select_dw6, dw_selectors[6]); + MLX5_SET(match_definer, ptr, format_select_dw7, dw_selectors[7]); + MLX5_SET(match_definer, ptr, format_select_dw8, dw_selectors[8]); + + MLX5_SET(match_definer, ptr, format_select_byte0, byte_selectors[0]); + MLX5_SET(match_definer, ptr, format_select_byte1, byte_selectors[1]); + MLX5_SET(match_definer, ptr, format_select_byte2, byte_selectors[2]); + MLX5_SET(match_definer, ptr, format_select_byte3, byte_selectors[3]); + MLX5_SET(match_definer, ptr, format_select_byte4, byte_selectors[4]); + MLX5_SET(match_definer, ptr, format_select_byte5, byte_selectors[5]); + MLX5_SET(match_definer, ptr, format_select_byte6, byte_selectors[6]); + MLX5_SET(match_definer, ptr, format_select_byte7, byte_selectors[7]); +} + +int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev, + u16 format_id, + u8 *dw_selectors, + u8 *byte_selectors, + u8 *match_mask, + u32 *definer_id) +{ + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; + u32 in[MLX5_ST_SZ_DW(create_match_definer_in)] = {}; + void *ptr; + int err; + + ptr = MLX5_ADDR_OF(create_match_definer_in, in, + general_obj_in_cmd_hdr); + MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type, + MLX5_OBJ_TYPE_MATCH_DEFINER); + + ptr = MLX5_ADDR_OF(create_match_definer_in, in, obj_context); + MLX5_SET(match_definer, ptr, format_id, format_id); + + dr_cmd_set_definer_format(ptr, format_id, + dw_selectors, byte_selectors); + + ptr = MLX5_ADDR_OF(match_definer, ptr, match_mask); + memcpy(ptr, match_mask, MLX5_FLD_SZ_BYTES(match_definer, match_mask)); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + *definer_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + + return 0; +} + +void +mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev, u32 definer_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_MATCH_DEFINER); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, definer_id); + + mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, u16 index, struct mlx5dr_cmd_gid_attr *attr) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c index 7adcf0eec13b..db81d881d38e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c @@ -49,7 +49,8 @@ enum dr_dump_rec_type { DR_DUMP_REC_TYPE_ACTION_POP_VLAN = 3413, DR_DUMP_REC_TYPE_ACTION_SAMPLER = 3415, DR_DUMP_REC_TYPE_ACTION_INSERT_HDR = 3420, - DR_DUMP_REC_TYPE_ACTION_REMOVE_HDR = 3421 + DR_DUMP_REC_TYPE_ACTION_REMOVE_HDR = 3421, + DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE = 3425, }; void mlx5dr_dbg_tbl_add(struct mlx5dr_table *tbl) @@ -107,6 +108,8 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, { struct mlx5dr_action *action = action_mem->action; const u64 action_id = DR_DBG_PTR_TO_ID(action); + u64 hit_tbl_ptr, miss_tbl_ptr; + u32 hit_tbl_id, miss_tbl_id; switch (action->action_type) { case DR_ACTION_TYP_DROP: @@ -198,6 +201,30 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, action->sampler->rx_icm_addr, action->sampler->tx_icm_addr); break; + case DR_ACTION_TYP_RANGE: + if (action->range->hit_tbl_action->dest_tbl->is_fw_tbl) { + hit_tbl_id = action->range->hit_tbl_action->dest_tbl->fw_tbl.id; + hit_tbl_ptr = 0; + } else { + hit_tbl_id = action->range->hit_tbl_action->dest_tbl->tbl->table_id; + hit_tbl_ptr = + DR_DBG_PTR_TO_ID(action->range->hit_tbl_action->dest_tbl->tbl); + } + + if (action->range->miss_tbl_action->dest_tbl->is_fw_tbl) { + miss_tbl_id = action->range->miss_tbl_action->dest_tbl->fw_tbl.id; + miss_tbl_ptr = 0; + } else { + miss_tbl_id = action->range->miss_tbl_action->dest_tbl->tbl->table_id; + miss_tbl_ptr = + DR_DBG_PTR_TO_ID(action->range->miss_tbl_action->dest_tbl->tbl); + } + + seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%llx,0x%x,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE, action_id, rule_id, + hit_tbl_id, hit_tbl_ptr, miss_tbl_id, miss_tbl_ptr, + action->range->definer_id); + break; default: return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_definer.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_definer.c new file mode 100644 index 000000000000..d5ea97751945 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_definer.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "dr_types.h" +#include "dr_ste.h" + +struct dr_definer_object { + u32 id; + u16 format_id; + u8 dw_selectors[MLX5_IFC_DEFINER_DW_SELECTORS_NUM]; + u8 byte_selectors[MLX5_IFC_DEFINER_BYTE_SELECTORS_NUM]; + u8 match_mask[DR_STE_SIZE_MATCH_TAG]; + refcount_t refcount; +}; + +static bool dr_definer_compare(struct dr_definer_object *definer, + u16 format_id, u8 *dw_selectors, + u8 *byte_selectors, u8 *match_mask) +{ + int i; + + if (definer->format_id != format_id) + return false; + + for (i = 0; i < MLX5_IFC_DEFINER_DW_SELECTORS_NUM; i++) + if (definer->dw_selectors[i] != dw_selectors[i]) + return false; + + for (i = 0; i < MLX5_IFC_DEFINER_BYTE_SELECTORS_NUM; i++) + if (definer->byte_selectors[i] != byte_selectors[i]) + return false; + + if (memcmp(definer->match_mask, match_mask, DR_STE_SIZE_MATCH_TAG)) + return false; + + return true; +} + +static struct dr_definer_object * +dr_definer_find_obj(struct mlx5dr_domain *dmn, u16 format_id, + u8 *dw_selectors, u8 *byte_selectors, u8 *match_mask) +{ + struct dr_definer_object *definer_obj; + unsigned long id; + + xa_for_each(&dmn->definers_xa, id, definer_obj) { + if (dr_definer_compare(definer_obj, format_id, + dw_selectors, byte_selectors, + match_mask)) + return definer_obj; + } + + return NULL; +} + +static struct dr_definer_object * +dr_definer_create_obj(struct mlx5dr_domain *dmn, u16 format_id, + u8 *dw_selectors, u8 *byte_selectors, u8 *match_mask) +{ + struct dr_definer_object *definer_obj; + int ret = 0; + + definer_obj = kzalloc(sizeof(*definer_obj), GFP_KERNEL); + if (!definer_obj) + return NULL; + + ret = mlx5dr_cmd_create_definer(dmn->mdev, + format_id, + dw_selectors, + byte_selectors, + match_mask, + &definer_obj->id); + if (ret) + goto err_free_definer_obj; + + /* Definer ID can have 32 bits, but STE format + * supports only definers with 8 bit IDs. + */ + if (definer_obj->id > 0xff) { + mlx5dr_err(dmn, "Unsupported definer ID (%d)\n", definer_obj->id); + goto err_destroy_definer; + } + + definer_obj->format_id = format_id; + memcpy(definer_obj->dw_selectors, dw_selectors, sizeof(definer_obj->dw_selectors)); + memcpy(definer_obj->byte_selectors, byte_selectors, sizeof(definer_obj->byte_selectors)); + memcpy(definer_obj->match_mask, match_mask, sizeof(definer_obj->match_mask)); + + refcount_set(&definer_obj->refcount, 1); + + ret = xa_insert(&dmn->definers_xa, definer_obj->id, definer_obj, GFP_KERNEL); + if (ret) { + mlx5dr_dbg(dmn, "Couldn't insert new definer into xarray (%d)\n", ret); + goto err_destroy_definer; + } + + return definer_obj; + +err_destroy_definer: + mlx5dr_cmd_destroy_definer(dmn->mdev, definer_obj->id); +err_free_definer_obj: + kfree(definer_obj); + + return NULL; +} + +static void dr_definer_destroy_obj(struct mlx5dr_domain *dmn, + struct dr_definer_object *definer_obj) +{ + mlx5dr_cmd_destroy_definer(dmn->mdev, definer_obj->id); + xa_erase(&dmn->definers_xa, definer_obj->id); + kfree(definer_obj); +} + +int mlx5dr_definer_get(struct mlx5dr_domain *dmn, u16 format_id, + u8 *dw_selectors, u8 *byte_selectors, + u8 *match_mask, u32 *definer_id) +{ + struct dr_definer_object *definer_obj; + int ret = 0; + + definer_obj = dr_definer_find_obj(dmn, format_id, dw_selectors, + byte_selectors, match_mask); + if (!definer_obj) { + definer_obj = dr_definer_create_obj(dmn, format_id, + dw_selectors, byte_selectors, + match_mask); + if (!definer_obj) + return -ENOMEM; + } else { + refcount_inc(&definer_obj->refcount); + } + + *definer_id = definer_obj->id; + + return ret; +} + +void mlx5dr_definer_put(struct mlx5dr_domain *dmn, u32 definer_id) +{ + struct dr_definer_object *definer_obj; + + definer_obj = xa_load(&dmn->definers_xa, definer_id); + if (!definer_obj) { + mlx5dr_err(dmn, "Definer ID %d not found\n", definer_id); + return; + } + + if (refcount_dec_and_test(&definer_obj->refcount)) + dr_definer_destroy_obj(dmn, definer_obj); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c index 9a9836218c8e..5b8bb2ca31e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c @@ -425,10 +425,11 @@ mlx5dr_domain_create(struct mlx5_core_dev *mdev, enum mlx5dr_domain_type type) refcount_set(&dmn->refcount, 1); mutex_init(&dmn->info.rx.mutex); mutex_init(&dmn->info.tx.mutex); + xa_init(&dmn->definers_xa); if (dr_domain_caps_init(mdev, dmn)) { mlx5dr_err(dmn, "Failed init domain, no caps\n"); - goto free_domain; + goto def_xa_destroy; } dmn->info.max_log_action_icm_sz = DR_CHUNK_SIZE_4K; @@ -453,7 +454,8 @@ mlx5dr_domain_create(struct mlx5_core_dev *mdev, enum mlx5dr_domain_type type) uninit_caps: dr_domain_caps_uninit(dmn); -free_domain: +def_xa_destroy: + xa_destroy(&dmn->definers_xa); kfree(dmn); return NULL; } @@ -493,6 +495,7 @@ int mlx5dr_domain_destroy(struct mlx5dr_domain *dmn) dr_domain_uninit_csum_recalc_fts(dmn); dr_domain_uninit_resources(dmn); dr_domain_caps_uninit(dmn); + xa_destroy(&dmn->definers_xa); mutex_destroy(&dmn->info.tx.mutex); mutex_destroy(&dmn->info.rx.mutex); kfree(dmn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c index 7879991048ce..74cbe53ee9db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c @@ -35,16 +35,28 @@ static int dr_rule_append_to_miss_list(struct mlx5dr_domain *dmn, return 0; } +static void dr_rule_set_last_ste_miss_addr(struct mlx5dr_matcher *matcher, + struct mlx5dr_matcher_rx_tx *nic_matcher, + u8 *hw_ste) +{ + struct mlx5dr_ste_ctx *ste_ctx = matcher->tbl->dmn->ste_ctx; + u64 icm_addr; + + if (mlx5dr_ste_is_miss_addr_set(ste_ctx, hw_ste)) + return; + + icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk); + mlx5dr_ste_set_miss_addr(ste_ctx, hw_ste, icm_addr); +} + static struct mlx5dr_ste * dr_rule_create_collision_htbl(struct mlx5dr_matcher *matcher, struct mlx5dr_matcher_rx_tx *nic_matcher, u8 *hw_ste) { struct mlx5dr_domain *dmn = matcher->tbl->dmn; - struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx; struct mlx5dr_ste_htbl *new_htbl; struct mlx5dr_ste *ste; - u64 icm_addr; /* Create new table for miss entry */ new_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool, @@ -58,8 +70,7 @@ dr_rule_create_collision_htbl(struct mlx5dr_matcher *matcher, /* One and only entry, never grows */ ste = new_htbl->chunk->ste_arr; - icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk); - mlx5dr_ste_set_miss_addr(ste_ctx, hw_ste, icm_addr); + dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste); mlx5dr_htbl_get(new_htbl); return ste; @@ -241,7 +252,6 @@ dr_rule_rehash_copy_ste(struct mlx5dr_matcher *matcher, bool use_update_list = false; u8 hw_ste[DR_STE_SIZE] = {}; struct mlx5dr_ste *new_ste; - u64 icm_addr; int new_idx; u8 sb_idx; @@ -250,9 +260,8 @@ dr_rule_rehash_copy_ste(struct mlx5dr_matcher *matcher, mlx5dr_ste_set_bit_mask(hw_ste, nic_matcher->ste_builder[sb_idx].bit_mask); /* Copy STE control and tag */ - icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk); memcpy(hw_ste, mlx5dr_ste_get_hw_ste(cur_ste), DR_STE_SIZE_REDUCED); - mlx5dr_ste_set_miss_addr(dmn->ste_ctx, hw_ste, icm_addr); + dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste); new_idx = mlx5dr_ste_calc_hash_index(hw_ste, new_htbl); new_ste = &new_htbl->chunk->ste_arr[new_idx]; @@ -773,7 +782,6 @@ static int dr_rule_handle_empty_entry(struct mlx5dr_matcher *matcher, { struct mlx5dr_domain *dmn = matcher->tbl->dmn; struct mlx5dr_ste_send_info *ste_info; - u64 icm_addr; /* Take ref on table, only on first time this ste is used */ mlx5dr_htbl_get(cur_htbl); @@ -781,8 +789,7 @@ static int dr_rule_handle_empty_entry(struct mlx5dr_matcher *matcher, /* new entry -> new branch */ list_add_tail(&ste->miss_list_node, miss_list); - icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk); - mlx5dr_ste_set_miss_addr(dmn->ste_ctx, hw_ste, icm_addr); + dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste); ste->ste_chain_location = ste_location; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c index 9e19a8dc9022..1e15f605df6e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c @@ -90,6 +90,16 @@ static void dr_ste_set_always_miss(struct dr_hw_ste_format *hw_ste) hw_ste->mask[0] = 0; } +bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, + u8 *hw_ste_p) +{ + if (!ste_ctx->is_miss_addr_set) + return false; + + /* check if miss address is already set for this type of STE */ + return ste_ctx->is_miss_addr_set(hw_ste_p); +} + void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p, u64 miss_addr) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h index 17513baff9b0..7075142bcfb6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h @@ -151,6 +151,7 @@ struct mlx5dr_ste_ctx { bool is_rx, u16 gvmi); void (*set_next_lu_type)(u8 *hw_ste_p, u16 lu_type); u16 (*get_next_lu_type)(u8 *hw_ste_p); + bool (*is_miss_addr_set)(u8 *hw_ste_p); void (*set_miss_addr)(u8 *hw_ste_p, u64 miss_addr); u64 (*get_miss_addr)(u8 *hw_ste_p); void (*set_hit_addr)(u8 *hw_ste_p, u64 icm_addr, u32 ht_size); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c index ee677a5c76be..084145f18084 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c @@ -13,6 +13,7 @@ enum dr_ste_v1_entry_format { DR_STE_V1_TYPE_BWC_BYTE = 0x0, DR_STE_V1_TYPE_BWC_DW = 0x1, DR_STE_V1_TYPE_MATCH = 0x2, + DR_STE_V1_TYPE_MATCH_RANGES = 0x7, }; /* Lookup type is built from 2B: [ Definer mode 1B ][ Definer index 1B ] */ @@ -267,6 +268,16 @@ static void dr_ste_v1_set_entry_type(u8 *hw_ste_p, u8 entry_type) MLX5_SET(ste_match_bwc_v1, hw_ste_p, entry_format, entry_type); } +bool dr_ste_v1_is_miss_addr_set(u8 *hw_ste_p) +{ + u8 entry_type = MLX5_GET(ste_match_bwc_v1, hw_ste_p, entry_format); + + /* unlike MATCH STE, for MATCH_RANGES STE both hit and miss addresses + * are part of the action, so they both set as part of STE init + */ + return entry_type == DR_STE_V1_TYPE_MATCH_RANGES; +} + void dr_ste_v1_set_miss_addr(u8 *hw_ste_p, u64 miss_addr) { u64 index = miss_addr >> 6; @@ -520,6 +531,27 @@ static void dr_ste_v1_set_aso_flow_meter(u8 *d_action, init_color); } +static void dr_ste_v1_set_match_range_pkt_len(u8 *hw_ste_p, u32 definer_id, + u32 min, u32 max) +{ + MLX5_SET(ste_match_ranges_v1, hw_ste_p, match_definer_ctx_idx, definer_id); + + /* When the STE will be sent, its mask and tags will be swapped in + * dr_ste_v1_prepare_for_postsend(). This, however, is match range STE + * which doesn't have mask, and shouldn't have mask/tag swapped. + * We're using the common utilities functions to send this STE, so need + * to allow for this swapping - place the values in the corresponding + * locations to allow flipping them when writing to ICM. + * + * min/max_value_2 corresponds to match_dw_0 in its definer. + * To allow mask/tag swapping, writing the min/max_2 to min/max_0. + * + * Pkt len is 2 bytes that are stored in the higher section of the DW. + */ + MLX5_SET(ste_match_ranges_v1, hw_ste_p, min_value_0, min << 16); + MLX5_SET(ste_match_ranges_v1, hw_ste_p, max_value_0, max << 16); +} + static void dr_ste_v1_arr_init_next_match(u8 **last_ste, u32 *added_stes, u16 gvmi) @@ -535,6 +567,14 @@ static void dr_ste_v1_arr_init_next_match(u8 **last_ste, memset(action, 0, MLX5_FLD_SZ_BYTES(ste_mask_and_match_v1, action)); } +static void dr_ste_v1_arr_init_next_match_range(u8 **last_ste, + u32 *added_stes, + u16 gvmi) +{ + dr_ste_v1_arr_init_next_match(last_ste, added_stes, gvmi); + dr_ste_v1_set_entry_type(*last_ste, DR_STE_V1_TYPE_MATCH_RANGES); +} + void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn, u8 *action_type_set, u32 actions_caps, @@ -670,6 +710,20 @@ void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn, action += DR_STE_ACTION_DOUBLE_SZ; } + if (action_type_set[DR_ACTION_TYP_RANGE]) { + /* match ranges requires a new STE of its own type */ + dr_ste_v1_arr_init_next_match_range(&last_ste, added_stes, attr->gvmi); + dr_ste_v1_set_miss_addr(last_ste, attr->range.miss_icm_addr); + + /* we do not support setting any action on the match ranges STE */ + action_sz = 0; + + dr_ste_v1_set_match_range_pkt_len(last_ste, + attr->range.definer_id, + attr->range.min, + attr->range.max); + } + dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1); } @@ -858,6 +912,20 @@ void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn, action += DR_STE_ACTION_DOUBLE_SZ; } + if (action_type_set[DR_ACTION_TYP_RANGE]) { + /* match ranges requires a new STE of its own type */ + dr_ste_v1_arr_init_next_match_range(&last_ste, added_stes, attr->gvmi); + dr_ste_v1_set_miss_addr(last_ste, attr->range.miss_icm_addr); + + /* we do not support setting any action on the match ranges STE */ + action_sz = 0; + + dr_ste_v1_set_match_range_pkt_len(last_ste, + attr->range.definer_id, + attr->range.min, + attr->range.max); + } + dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1); } @@ -2144,6 +2212,7 @@ static struct mlx5dr_ste_ctx ste_ctx_v1 = { .ste_init = &dr_ste_v1_init, .set_next_lu_type = &dr_ste_v1_set_next_lu_type, .get_next_lu_type = &dr_ste_v1_get_next_lu_type, + .is_miss_addr_set = &dr_ste_v1_is_miss_addr_set, .set_miss_addr = &dr_ste_v1_set_miss_addr, .get_miss_addr = &dr_ste_v1_get_miss_addr, .set_hit_addr = &dr_ste_v1_set_hit_addr, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h index 8a1d49790c6e..b5c0f0f8392f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h @@ -7,6 +7,7 @@ #include "dr_types.h" #include "dr_ste.h" +bool dr_ste_v1_is_miss_addr_set(u8 *hw_ste_p); void dr_ste_v1_set_miss_addr(u8 *hw_ste_p, u64 miss_addr); u64 dr_ste_v1_get_miss_addr(u8 *hw_ste_p); void dr_ste_v1_set_byte_mask(u8 *hw_ste_p, u16 byte_mask); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c index c60fddd125d2..cf1a3c9a1cf4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c @@ -202,6 +202,7 @@ static struct mlx5dr_ste_ctx ste_ctx_v2 = { .ste_init = &dr_ste_v1_init, .set_next_lu_type = &dr_ste_v1_set_next_lu_type, .get_next_lu_type = &dr_ste_v1_get_next_lu_type, + .is_miss_addr_set = &dr_ste_v1_is_miss_addr_set, .set_miss_addr = &dr_ste_v1_set_miss_addr, .get_miss_addr = &dr_ste_v1_get_miss_addr, .set_hit_addr = &dr_ste_v1_set_hit_addr, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 41a37b9ac98b..2b769dcbd453 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -81,6 +81,7 @@ mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk) enum { DR_STE_SIZE = 64, DR_STE_SIZE_CTRL = 32, + DR_STE_SIZE_MATCH_TAG = 32, DR_STE_SIZE_TAG = 16, DR_STE_SIZE_MASK = 16, DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, @@ -128,6 +129,7 @@ enum mlx5dr_action_type { DR_ACTION_TYP_REMOVE_HDR, DR_ACTION_TYP_SAMPLER, DR_ACTION_TYP_ASO_FLOW_METER, + DR_ACTION_TYP_RANGE, DR_ACTION_TYP_MAX, }; @@ -237,6 +239,7 @@ static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl) /* STE utils */ u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl); +bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p); void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste, u64 miss_addr); void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx, @@ -281,6 +284,13 @@ struct mlx5dr_ste_actions_attr { u8 dest_reg_id; u8 init_color; } aso_flow_meter; + + struct { + u64 miss_icm_addr; + u32 definer_id; + u32 min; + u32 max; + } range; }; void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, @@ -924,6 +934,7 @@ struct mlx5dr_domain { struct mlx5dr_ste_ctx *ste_ctx; struct list_head dbg_tbl_list; struct mlx5dr_dbg_dump_info dump_info; + struct xarray definers_xa; }; struct mlx5dr_table_rx_tx { @@ -1026,6 +1037,15 @@ struct mlx5dr_action_dest_tbl { }; }; +struct mlx5dr_action_range { + struct mlx5dr_domain *dmn; + struct mlx5dr_action *hit_tbl_action; + struct mlx5dr_action *miss_tbl_action; + u32 definer_id; + u32 min; + u32 max; +}; + struct mlx5dr_action_ctr { u32 ctr_id; u32 offset; @@ -1072,6 +1092,7 @@ struct mlx5dr_action { struct mlx5dr_action_push_vlan *push_vlan; struct mlx5dr_action_flow_tag *flow_tag; struct mlx5dr_action_aso_flow_meter *aso; + struct mlx5dr_action_range *range; }; }; @@ -1295,6 +1316,14 @@ int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, u32 *reformat_id); void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev, u32 reformat_id); +int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev, + u16 format_id, + u8 *dw_selectors, + u8 *byte_selectors, + u8 *match_mask, + u32 *definer_id); +void mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev, + u32 definer_id); struct mlx5dr_cmd_gid_attr { u8 gid[16]; @@ -1483,4 +1512,18 @@ int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, u32 flow_source); void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id, u32 group_id); + +static inline bool mlx5dr_is_fw_table(struct mlx5_flow_table *ft) +{ + return !ft->fs_dr_table.dr_table; +} + +static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev) +{ + return (MLX5_CAP_GEN(dev, steering_format_version) >= + MLX5_STEERING_FORMAT_CONNECTX_6DX) && + (MLX5_CAP_GEN_64(dev, match_definer_format_supported) & + (1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT)); +} + #endif /* _DR_TYPES_H_ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 13b6d4721e17..984653756779 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -7,10 +7,11 @@ #include "fs_cmd.h" #include "mlx5dr.h" #include "fs_dr.h" +#include "dr_types.h" -static bool mlx5_dr_is_fw_table(u32 flags) +static bool dr_is_fw_term_table(struct mlx5_flow_table *ft) { - if (flags & MLX5_FLOW_TABLE_TERMINATION) + if (ft->flags & MLX5_FLOW_TABLE_TERMINATION) return true; return false; @@ -69,7 +70,7 @@ static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, u32 flags; int err; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->create_flow_table(ns, ft, ft_attr, next_ft); @@ -109,7 +110,7 @@ static int mlx5_cmd_dr_destroy_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5dr_action *action = ft->fs_dr_table.miss_action; int err; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->destroy_flow_table(ns, ft); err = mlx5dr_table_destroy(ft->fs_dr_table.dr_table); @@ -134,7 +135,7 @@ static int mlx5_cmd_dr_modify_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_table *next_ft) { - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->modify_flow_table(ns, ft, next_ft); return set_miss_action(ns, ft, next_ft); @@ -153,7 +154,7 @@ static int mlx5_cmd_dr_create_flow_group(struct mlx5_flow_root_namespace *ns, match_criteria_enable); struct mlx5dr_match_parameters mask; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->create_flow_group(ns, ft, in, fg); @@ -178,7 +179,7 @@ static int mlx5_cmd_dr_destroy_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg) { - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->destroy_flow_group(ns, ft, fg); return mlx5dr_matcher_destroy(fg->fs_dr_matcher.dr_matcher); @@ -209,11 +210,22 @@ static struct mlx5dr_action *create_ft_action(struct mlx5dr_domain *domain, { struct mlx5_flow_table *dest_ft = dst->dest_attr.ft; - if (mlx5_dr_is_fw_table(dest_ft->flags)) + if (mlx5dr_is_fw_table(dest_ft)) return mlx5dr_action_create_dest_flow_fw_table(domain, dest_ft); return mlx5dr_action_create_dest_table(dest_ft->fs_dr_table.dr_table); } +static struct mlx5dr_action *create_range_action(struct mlx5dr_domain *domain, + struct mlx5_flow_rule *dst) +{ + return mlx5dr_action_create_dest_match_range(domain, + dst->dest_attr.range.field, + dst->dest_attr.range.hit_ft, + dst->dest_attr.range.miss_ft, + dst->dest_attr.range.min, + dst->dest_attr.range.max); +} + static struct mlx5dr_action *create_action_push_vlan(struct mlx5dr_domain *domain, struct mlx5_fs_vlan *vlan) { @@ -260,7 +272,7 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, int err = 0; int i; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->create_fte(ns, ft, group, fte); actions = kcalloc(MLX5_FLOW_CONTEXT_ACTION_MAX, sizeof(*actions), @@ -467,6 +479,15 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, fs_dr_actions[fs_dr_num_actions++] = tmp_action; term_actions[num_term_actions++].dest = tmp_action; break; + case MLX5_FLOW_DESTINATION_TYPE_RANGE: + tmp_action = create_range_action(domain, dst); + if (!tmp_action) { + err = -ENOMEM; + goto free_actions; + } + fs_dr_actions[fs_dr_num_actions++] = tmp_action; + term_actions[num_term_actions++].dest = tmp_action; + break; default: err = -EOPNOTSUPP; goto free_actions; @@ -702,7 +723,7 @@ static int mlx5_cmd_dr_delete_fte(struct mlx5_flow_root_namespace *ns, int err; int i; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->delete_fte(ns, ft, fte); err = mlx5dr_rule_destroy(rule->dr_rule); @@ -727,7 +748,7 @@ static int mlx5_cmd_dr_update_fte(struct mlx5_flow_root_namespace *ns, struct fs_fte fte_tmp = {}; int ret; - if (mlx5_dr_is_fw_table(ft->flags)) + if (dr_is_fw_term_table(ft)) return mlx5_fs_cmd_get_fw_cmds()->update_fte(ns, ft, group, modify_mask, fte); /* Backup current dr rule details */ @@ -780,11 +801,19 @@ static int mlx5_cmd_dr_destroy_ns(struct mlx5_flow_root_namespace *ns) static u32 mlx5_cmd_dr_get_capabilities(struct mlx5_flow_root_namespace *ns, enum fs_flow_table_type ft_type) { + u32 steering_caps = 0; + if (ft_type != FS_FT_FDB || MLX5_CAP_GEN(ns->dev, steering_format_version) == MLX5_STEERING_FORMAT_CONNECTX_5) return 0; - return MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX | MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX; + steering_caps |= MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX; + steering_caps |= MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX; + + if (mlx5dr_supp_match_ranges(ns->dev)) + steering_caps |= MLX5_FLOW_STEERING_CAP_MATCH_RANGES; + + return steering_caps; } bool mlx5_fs_dr_is_supported(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h index 34c2bd17a8b4..790a17d6207f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h @@ -165,6 +165,41 @@ struct mlx5_ifc_ste_mask_and_match_v1_bits { u8 action[0x60]; }; +struct mlx5_ifc_ste_match_ranges_v1_bits { + u8 entry_format[0x8]; + u8 counter_id[0x18]; + + u8 miss_address_63_48[0x10]; + u8 match_definer_ctx_idx[0x8]; + u8 miss_address_39_32[0x8]; + + u8 miss_address_31_6[0x1a]; + u8 reserved_at_5a[0x1]; + u8 match_polarity[0x1]; + u8 reparse[0x1]; + u8 reserved_at_5d[0x3]; + + u8 next_table_base_63_48[0x10]; + u8 hash_definer_ctx_idx[0x8]; + u8 next_table_base_39_32_size[0x8]; + + u8 next_table_base_31_5_size[0x1b]; + u8 hash_type[0x2]; + u8 hash_after_actions[0x1]; + u8 reserved_at_9e[0x2]; + + u8 action[0x60]; + + u8 max_value_0[0x20]; + u8 min_value_0[0x20]; + u8 max_value_1[0x20]; + u8 min_value_1[0x20]; + u8 max_value_2[0x20]; + u8 min_value_2[0x20]; + u8 max_value_3[0x20]; + u8 min_value_3[0x20]; +}; + struct mlx5_ifc_ste_eth_l2_src_v1_bits { u8 reserved_at_0[0x1]; u8 sx_sniffer[0x1]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 84ed77763b21..9afd268a2573 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -140,8 +140,21 @@ mlx5dr_action_create_aso(struct mlx5dr_domain *dmn, u8 init_color, u8 meter_id); +struct mlx5dr_action * +mlx5dr_action_create_dest_match_range(struct mlx5dr_domain *dmn, + u32 field, + struct mlx5_flow_table *hit_ft, + struct mlx5_flow_table *miss_ft, + u32 min, + u32 max); + int mlx5dr_action_destroy(struct mlx5dr_action *action); +int mlx5dr_definer_get(struct mlx5dr_domain *dmn, u16 format_id, + u8 *dw_selectors, u8 *byte_selectors, + u8 *match_mask, u32 *definer_id); +void mlx5dr_definer_put(struct mlx5dr_domain *dmn, u32 definer_id); + static inline bool mlx5dr_is_supported(struct mlx5_core_dev *dev) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c index 8455e79bc44a..1513112ecec8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c @@ -31,7 +31,6 @@ */ #include <linux/kernel.h> -#include <linux/io-mapping.h> #include <linux/mlx5/driver.h> #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index d5c317325030..ba7e3df22413 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -1160,14 +1160,40 @@ u64 mlx5_query_nic_system_image_guid(struct mlx5_core_dev *mdev) } EXPORT_SYMBOL_GPL(mlx5_query_nic_system_image_guid); -int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out) +int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out, + u16 opmod) { - u16 opmod = (MLX5_CAP_GENERAL << 1) | (HCA_CAP_OPMOD_GET_MAX & 0x01); u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)] = {}; + opmod = (opmod << 1) | (HCA_CAP_OPMOD_GET_MAX & 0x01); MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, in, op_mod, opmod); MLX5_SET(query_hca_cap_in, in, function_id, function_id); MLX5_SET(query_hca_cap_in, in, other_function, true); return mlx5_cmd_exec_inout(dev, query_hca_cap, in, out); } +EXPORT_SYMBOL_GPL(mlx5_vport_get_other_func_cap); + +int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap, + u16 function_id, u16 opmod) +{ + int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); + void *set_hca_cap; + void *set_ctx; + int ret; + + set_ctx = kzalloc(set_sz, GFP_KERNEL); + if (!set_ctx) + return -ENOMEM; + + MLX5_SET(set_hca_cap_in, set_ctx, opcode, MLX5_CMD_OP_SET_HCA_CAP); + MLX5_SET(set_hca_cap_in, set_ctx, op_mod, opmod << 1); + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); + memcpy(set_hca_cap, hca_cap, MLX5_ST_SZ_BYTES(cmd_hca_cap)); + MLX5_SET(set_hca_cap_in, set_ctx, function_id, function_id); + MLX5_SET(set_hca_cap_in, set_ctx, other_function, true); + ret = mlx5_cmd_exec_in(dev, set_hca_cap, set_ctx); + + kfree(set_ctx); + return ret; +} diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index a83f6bc30072..a0a06e2eff82 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1459,11 +1459,6 @@ mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, char buf[32]; int err; - err = devlink_info_driver_name_put(req, - mlxsw_core->bus_info->device_kind); - if (err) - return err; - mlxsw_reg_mgir_pack(mgir_pl); err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl); if (err) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c index a2ee695a3f17..3340b4a694c3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c @@ -363,93 +363,7 @@ static const struct mlxsw_sp_ipip_ops mlxsw_sp_ipip_gre4_ops = { }; static struct mlxsw_sp_ipip_parms -mlxsw_sp1_ipip_netdev_parms_init_gre6(const struct net_device *ol_dev) -{ - struct mlxsw_sp_ipip_parms parms = {0}; - - WARN_ON_ONCE(1); - return parms; -} - -static int -mlxsw_sp1_ipip_nexthop_update_gre6(struct mlxsw_sp *mlxsw_sp, u32 adj_index, - struct mlxsw_sp_ipip_entry *ipip_entry, - bool force, char *ratr_pl) -{ - WARN_ON_ONCE(1); - return -EINVAL; -} - -static int -mlxsw_sp1_ipip_decap_config_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry, - u32 tunnel_index) -{ - WARN_ON_ONCE(1); - return -EINVAL; -} - -static bool mlxsw_sp1_ipip_can_offload_gre6(const struct mlxsw_sp *mlxsw_sp, - const struct net_device *ol_dev) -{ - return false; -} - -static struct mlxsw_sp_rif_ipip_lb_config -mlxsw_sp1_ipip_ol_loopback_config_gre6(struct mlxsw_sp *mlxsw_sp, - const struct net_device *ol_dev) -{ - struct mlxsw_sp_rif_ipip_lb_config config = {0}; - - WARN_ON_ONCE(1); - return config; -} - -static int -mlxsw_sp1_ipip_ol_netdev_change_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry, - struct netlink_ext_ack *extack) -{ - WARN_ON_ONCE(1); - return -EINVAL; -} - -static int -mlxsw_sp1_ipip_rem_addr_set_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry) -{ - WARN_ON_ONCE(1); - return -EINVAL; -} - -static void -mlxsw_sp1_ipip_rem_addr_unset_gre6(struct mlxsw_sp *mlxsw_sp, - const struct mlxsw_sp_ipip_entry *ipip_entry) -{ - WARN_ON_ONCE(1); -} - -static const struct mlxsw_sp_ipip_ops mlxsw_sp1_ipip_gre6_ops = { - .dev_type = ARPHRD_IP6GRE, - .ul_proto = MLXSW_SP_L3_PROTO_IPV6, - .inc_parsing_depth = true, - .parms_init = mlxsw_sp1_ipip_netdev_parms_init_gre6, - .nexthop_update = mlxsw_sp1_ipip_nexthop_update_gre6, - .decap_config = mlxsw_sp1_ipip_decap_config_gre6, - .can_offload = mlxsw_sp1_ipip_can_offload_gre6, - .ol_loopback_config = mlxsw_sp1_ipip_ol_loopback_config_gre6, - .ol_netdev_change = mlxsw_sp1_ipip_ol_netdev_change_gre6, - .rem_ip_addr_set = mlxsw_sp1_ipip_rem_addr_set_gre6, - .rem_ip_addr_unset = mlxsw_sp1_ipip_rem_addr_unset_gre6, -}; - -const struct mlxsw_sp_ipip_ops *mlxsw_sp1_ipip_ops_arr[] = { - [MLXSW_SP_IPIP_TYPE_GRE4] = &mlxsw_sp_ipip_gre4_ops, - [MLXSW_SP_IPIP_TYPE_GRE6] = &mlxsw_sp1_ipip_gre6_ops, -}; - -static struct mlxsw_sp_ipip_parms -mlxsw_sp2_ipip_netdev_parms_init_gre6(const struct net_device *ol_dev) +mlxsw_sp_ipip_netdev_parms_init_gre6(const struct net_device *ol_dev) { struct __ip6_tnl_parm parms = mlxsw_sp_ipip_netdev_parms6(ol_dev); @@ -464,9 +378,9 @@ mlxsw_sp2_ipip_netdev_parms_init_gre6(const struct net_device *ol_dev) } static int -mlxsw_sp2_ipip_nexthop_update_gre6(struct mlxsw_sp *mlxsw_sp, u32 adj_index, - struct mlxsw_sp_ipip_entry *ipip_entry, - bool force, char *ratr_pl) +mlxsw_sp_ipip_nexthop_update_gre6(struct mlxsw_sp *mlxsw_sp, u32 adj_index, + struct mlxsw_sp_ipip_entry *ipip_entry, + bool force, char *ratr_pl) { u16 rif_index = mlxsw_sp_ipip_lb_rif_index(ipip_entry->ol_lb); enum mlxsw_reg_ratr_op op; @@ -482,9 +396,9 @@ mlxsw_sp2_ipip_nexthop_update_gre6(struct mlxsw_sp *mlxsw_sp, u32 adj_index, } static int -mlxsw_sp2_ipip_decap_config_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry, - u32 tunnel_index) +mlxsw_sp_ipip_decap_config_gre6(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_ipip_entry *ipip_entry, + u32 tunnel_index) { u16 rif_index = mlxsw_sp_ipip_lb_rif_index(ipip_entry->ol_lb); u16 ul_rif_id = mlxsw_sp_ipip_lb_ul_rif_id(ipip_entry->ol_lb); @@ -519,8 +433,8 @@ mlxsw_sp2_ipip_decap_config_gre6(struct mlxsw_sp *mlxsw_sp, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rtdp), rtdp_pl); } -static bool mlxsw_sp2_ipip_can_offload_gre6(const struct mlxsw_sp *mlxsw_sp, - const struct net_device *ol_dev) +static bool mlxsw_sp_ipip_can_offload_gre6(const struct mlxsw_sp *mlxsw_sp, + const struct net_device *ol_dev) { struct __ip6_tnl_parm tparm = mlxsw_sp_ipip_netdev_parms6(ol_dev); bool inherit_tos = tparm.flags & IP6_TNL_F_USE_ORIG_TCLASS; @@ -534,8 +448,8 @@ static bool mlxsw_sp2_ipip_can_offload_gre6(const struct mlxsw_sp *mlxsw_sp, } static struct mlxsw_sp_rif_ipip_lb_config -mlxsw_sp2_ipip_ol_loopback_config_gre6(struct mlxsw_sp *mlxsw_sp, - const struct net_device *ol_dev) +mlxsw_sp_ipip_ol_loopback_config_gre6(struct mlxsw_sp *mlxsw_sp, + const struct net_device *ol_dev) { struct __ip6_tnl_parm parms = mlxsw_sp_ipip_netdev_parms6(ol_dev); enum mlxsw_reg_ritr_loopback_ipip_type lb_ipipt; @@ -553,20 +467,20 @@ mlxsw_sp2_ipip_ol_loopback_config_gre6(struct mlxsw_sp *mlxsw_sp, } static int -mlxsw_sp2_ipip_ol_netdev_change_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry, - struct netlink_ext_ack *extack) +mlxsw_sp_ipip_ol_netdev_change_gre6(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_ipip_entry *ipip_entry, + struct netlink_ext_ack *extack) { struct mlxsw_sp_ipip_parms new_parms; - new_parms = mlxsw_sp2_ipip_netdev_parms_init_gre6(ipip_entry->ol_dev); + new_parms = mlxsw_sp_ipip_netdev_parms_init_gre6(ipip_entry->ol_dev); return mlxsw_sp_ipip_ol_netdev_change_gre(mlxsw_sp, ipip_entry, &new_parms, extack); } static int -mlxsw_sp2_ipip_rem_addr_set_gre6(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_ipip_entry *ipip_entry) +mlxsw_sp_ipip_rem_addr_set_gre6(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_ipip_entry *ipip_entry) { return mlxsw_sp_ipv6_addr_kvdl_index_get(mlxsw_sp, &ipip_entry->parms.daddr.addr6, @@ -574,24 +488,44 @@ mlxsw_sp2_ipip_rem_addr_set_gre6(struct mlxsw_sp *mlxsw_sp, } static void -mlxsw_sp2_ipip_rem_addr_unset_gre6(struct mlxsw_sp *mlxsw_sp, - const struct mlxsw_sp_ipip_entry *ipip_entry) +mlxsw_sp_ipip_rem_addr_unset_gre6(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_ipip_entry *ipip_entry) { mlxsw_sp_ipv6_addr_put(mlxsw_sp, &ipip_entry->parms.daddr.addr6); } +static const struct mlxsw_sp_ipip_ops mlxsw_sp1_ipip_gre6_ops = { + .dev_type = ARPHRD_IP6GRE, + .ul_proto = MLXSW_SP_L3_PROTO_IPV6, + .inc_parsing_depth = true, + .double_rif_entry = true, + .parms_init = mlxsw_sp_ipip_netdev_parms_init_gre6, + .nexthop_update = mlxsw_sp_ipip_nexthop_update_gre6, + .decap_config = mlxsw_sp_ipip_decap_config_gre6, + .can_offload = mlxsw_sp_ipip_can_offload_gre6, + .ol_loopback_config = mlxsw_sp_ipip_ol_loopback_config_gre6, + .ol_netdev_change = mlxsw_sp_ipip_ol_netdev_change_gre6, + .rem_ip_addr_set = mlxsw_sp_ipip_rem_addr_set_gre6, + .rem_ip_addr_unset = mlxsw_sp_ipip_rem_addr_unset_gre6, +}; + +const struct mlxsw_sp_ipip_ops *mlxsw_sp1_ipip_ops_arr[] = { + [MLXSW_SP_IPIP_TYPE_GRE4] = &mlxsw_sp_ipip_gre4_ops, + [MLXSW_SP_IPIP_TYPE_GRE6] = &mlxsw_sp1_ipip_gre6_ops, +}; + static const struct mlxsw_sp_ipip_ops mlxsw_sp2_ipip_gre6_ops = { .dev_type = ARPHRD_IP6GRE, .ul_proto = MLXSW_SP_L3_PROTO_IPV6, .inc_parsing_depth = true, - .parms_init = mlxsw_sp2_ipip_netdev_parms_init_gre6, - .nexthop_update = mlxsw_sp2_ipip_nexthop_update_gre6, - .decap_config = mlxsw_sp2_ipip_decap_config_gre6, - .can_offload = mlxsw_sp2_ipip_can_offload_gre6, - .ol_loopback_config = mlxsw_sp2_ipip_ol_loopback_config_gre6, - .ol_netdev_change = mlxsw_sp2_ipip_ol_netdev_change_gre6, - .rem_ip_addr_set = mlxsw_sp2_ipip_rem_addr_set_gre6, - .rem_ip_addr_unset = mlxsw_sp2_ipip_rem_addr_unset_gre6, + .parms_init = mlxsw_sp_ipip_netdev_parms_init_gre6, + .nexthop_update = mlxsw_sp_ipip_nexthop_update_gre6, + .decap_config = mlxsw_sp_ipip_decap_config_gre6, + .can_offload = mlxsw_sp_ipip_can_offload_gre6, + .ol_loopback_config = mlxsw_sp_ipip_ol_loopback_config_gre6, + .ol_netdev_change = mlxsw_sp_ipip_ol_netdev_change_gre6, + .rem_ip_addr_set = mlxsw_sp_ipip_rem_addr_set_gre6, + .rem_ip_addr_unset = mlxsw_sp_ipip_rem_addr_unset_gre6, }; const struct mlxsw_sp_ipip_ops *mlxsw_sp2_ipip_ops_arr[] = { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h index 8cc259dcc8d0..a35f009da561 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h @@ -49,6 +49,7 @@ struct mlxsw_sp_ipip_ops { int dev_type; enum mlxsw_sp_l3proto ul_proto; /* Underlay. */ bool inc_parsing_depth; + bool double_rif_entry; struct mlxsw_sp_ipip_parms (*parms_init)(const struct net_device *ol_dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 48f1fa62a4fd..c22c3ac4e2a1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -18,6 +18,7 @@ #include <linux/jhash.h> #include <linux/net_namespace.h> #include <linux/mutex.h> +#include <linux/genalloc.h> #include <net/netevent.h> #include <net/neighbour.h> #include <net/arp.h> @@ -59,6 +60,7 @@ struct mlxsw_sp_rif { int mtu; u16 rif_index; u8 mac_profile_id; + u8 rif_entries; u16 vr_id; const struct mlxsw_sp_rif_ops *ops; struct mlxsw_sp *mlxsw_sp; @@ -77,6 +79,7 @@ struct mlxsw_sp_rif_params { }; u16 vid; bool lag; + bool double_entry; }; struct mlxsw_sp_rif_subport { @@ -1068,6 +1071,7 @@ mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { .common.dev = ol_dev, .common.lag = false, + .common.double_entry = ipip_ops->double_rif_entry, .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), }; @@ -7826,18 +7830,26 @@ mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); } -static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) +static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index, + u8 rif_entries) { - int i; + *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table, + rif_entries); + if (*p_rif_index == 0) + return -ENOBUFS; + *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET; - for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { - if (!mlxsw_sp->router->rifs[i]) { - *p_rif_index = i; - return 0; - } - } + /* RIF indexes must be aligned to the allocation size. */ + WARN_ON_ONCE(*p_rif_index % rif_entries); - return -ENOBUFS; + return 0; +} + +static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index, + u8 rif_entries) +{ + gen_pool_free(mlxsw_sp->router->rifs_table, + MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries); } static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, @@ -8081,6 +8093,7 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, const struct mlxsw_sp_rif_params *params, struct netlink_ext_ack *extack) { + u8 rif_entries = params->double_entry ? 2 : 1; u32 tb_id = l3mdev_fib_table(params->dev); const struct mlxsw_sp_rif_ops *ops; struct mlxsw_sp_fid *fid = NULL; @@ -8098,7 +8111,7 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, return ERR_CAST(vr); vr->rif_count++; - err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); + err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); if (err) { NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); goto err_rif_index_alloc; @@ -8113,6 +8126,7 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, mlxsw_sp->router->rifs[rif_index] = rif; rif->mlxsw_sp = mlxsw_sp; rif->ops = ops; + rif->rif_entries = rif_entries; if (ops->fid_get) { fid = ops->fid_get(rif, extack); @@ -8146,7 +8160,7 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_rif_counters_alloc(rif); } - atomic_inc(&mlxsw_sp->router->rifs_count); + atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); return rif; err_stats_enable: @@ -8162,6 +8176,7 @@ err_fid_get: dev_put(rif->dev); kfree(rif); err_rif_alloc: + mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); err_rif_index_alloc: vr->rif_count--; mlxsw_sp_vr_put(mlxsw_sp, vr); @@ -8173,10 +8188,12 @@ static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) const struct mlxsw_sp_rif_ops *ops = rif->ops; struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; struct mlxsw_sp_fid *fid = rif->fid; + u8 rif_entries = rif->rif_entries; + u16 rif_index = rif->rif_index; struct mlxsw_sp_vr *vr; int i; - atomic_dec(&mlxsw_sp->router->rifs_count); + atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); vr = &mlxsw_sp->router->vrs[rif->vr_id]; @@ -8198,6 +8215,7 @@ static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) mlxsw_sp->router->rifs[rif->rif_index] = NULL; dev_put(rif->dev); kfree(rif); + mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); vr->rif_count--; mlxsw_sp_vr_put(mlxsw_sp, vr); } @@ -9771,42 +9789,51 @@ mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, struct netlink_ext_ack *extack) { struct mlxsw_sp_rif *ul_rif; + u8 rif_entries = 1; u16 rif_index; int err; - err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); + err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); if (err) { NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); return ERR_PTR(err); } ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); - if (!ul_rif) - return ERR_PTR(-ENOMEM); + if (!ul_rif) { + err = -ENOMEM; + goto err_rif_alloc; + } mlxsw_sp->router->rifs[rif_index] = ul_rif; ul_rif->mlxsw_sp = mlxsw_sp; + ul_rif->rif_entries = rif_entries; err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); if (err) goto ul_rif_op_err; - atomic_inc(&mlxsw_sp->router->rifs_count); + atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); return ul_rif; ul_rif_op_err: mlxsw_sp->router->rifs[rif_index] = NULL; kfree(ul_rif); +err_rif_alloc: + mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); return ERR_PTR(err); } static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) { struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; + u8 rif_entries = ul_rif->rif_entries; + u16 rif_index = ul_rif->rif_index; - atomic_dec(&mlxsw_sp->router->rifs_count); + atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; kfree(ul_rif); + mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); } static struct mlxsw_sp_rif * @@ -9940,11 +9967,43 @@ static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, }; +static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp) +{ + struct gen_pool *rifs_table; + int err; + + rifs_table = gen_pool_create(0, -1); + if (!rifs_table) + return -ENOMEM; + + gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align, + NULL); + + err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET, + MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1); + if (err) + goto err_gen_pool_add; + + mlxsw_sp->router->rifs_table = rifs_table; + + return 0; + +err_gen_pool_add: + gen_pool_destroy(rifs_table); + return err; +} + +static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp) +{ + gen_pool_destroy(mlxsw_sp->router->rifs_table); +} + static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) { u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); struct mlxsw_core *core = mlxsw_sp->core; + int err; if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) return -EIO; @@ -9957,6 +10016,10 @@ static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) if (!mlxsw_sp->router->rifs) return -ENOMEM; + err = mlxsw_sp_rifs_table_init(mlxsw_sp); + if (err) + goto err_rifs_table_init; + idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); atomic_set(&mlxsw_sp->router->rifs_count, 0); @@ -9970,6 +10033,10 @@ static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) mlxsw_sp); return 0; + +err_rifs_table_init: + kfree(mlxsw_sp->router->rifs); + return err; } static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) @@ -9986,6 +10053,7 @@ static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); + mlxsw_sp_rifs_table_fini(mlxsw_sp); kfree(mlxsw_sp->router->rifs); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h index c5dfb972b433..37d6e4c80e6a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h @@ -15,8 +15,12 @@ struct mlxsw_sp_router_nve_decap { u8 valid:1; }; +/* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ +#define MLXSW_SP_ROUTER_GENALLOC_OFFSET 0x100 + struct mlxsw_sp_router { struct mlxsw_sp *mlxsw_sp; + struct gen_pool *rifs_table; struct mlxsw_sp_rif **rifs; struct idr rif_mac_profiles_idr; atomic_t rif_mac_profiles_count; diff --git a/drivers/net/ethernet/microchip/encx24j600-regmap.c b/drivers/net/ethernet/microchip/encx24j600-regmap.c index 81a8ccca7e5e..5693784eec5b 100644 --- a/drivers/net/ethernet/microchip/encx24j600-regmap.c +++ b/drivers/net/ethernet/microchip/encx24j600-regmap.c @@ -359,7 +359,7 @@ static int regmap_encx24j600_phy_reg_read(void *context, unsigned int reg, goto err_out; usleep_range(26, 100); - while ((ret = regmap_read(ctx->regmap, MISTAT, &mistat) != 0) && + while (((ret = regmap_read(ctx->regmap, MISTAT, &mistat)) == 0) && (mistat & BUSY)) cpu_relax(); @@ -397,7 +397,7 @@ static int regmap_encx24j600_phy_reg_write(void *context, unsigned int reg, goto err_out; usleep_range(26, 100); - while ((ret = regmap_read(ctx->regmap, MISTAT, &mistat) != 0) && + while (((ret = regmap_read(ctx->regmap, MISTAT, &mistat)) == 0) && (mistat & BUSY)) cpu_relax(); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index f6092983d028..cadde20505ba 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -443,11 +443,22 @@ static int lan966x_port_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) { struct lan966x_port *port = netdev_priv(dev); + int err; + + if (cmd == SIOCSHWTSTAMP) { + err = lan966x_ptp_setup_traps(port, ifr); + if (err) + return err; + } if (!phy_has_hwtstamp(dev->phydev) && port->lan966x->ptp) { switch (cmd) { case SIOCSHWTSTAMP: - return lan966x_ptp_hwtstamp_set(port, ifr); + err = lan966x_ptp_hwtstamp_set(port, ifr); + if (err) + lan966x_ptp_del_traps(port); + + return err; case SIOCGHWTSTAMP: return lan966x_ptp_hwtstamp_get(port, ifr); } @@ -456,7 +467,11 @@ static int lan966x_port_ioctl(struct net_device *dev, struct ifreq *ifr, if (!dev->phydev) return -ENODEV; - return phy_mii_ioctl(dev->phydev, ifr, cmd); + err = phy_mii_ioctl(dev->phydev, ifr, cmd); + if (err && cmd == SIOCSHWTSTAMP) + lan966x_ptp_del_traps(port); + + return err; } static const struct net_device_ops lan966x_port_netdev_ops = { diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index f2e45da7ffd4..3491f1961835 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -88,6 +88,10 @@ #define SE_IDX_QUEUE 0 /* 0-79 : Queue scheduler elements */ #define SE_IDX_PORT 80 /* 80-89 : Port schedular elements */ +#define LAN966X_VCAP_CID_IS2_L0 VCAP_CID_INGRESS_STAGE2_L0 /* IS2 lookup 0 */ +#define LAN966X_VCAP_CID_IS2_L1 VCAP_CID_INGRESS_STAGE2_L1 /* IS2 lookup 1 */ +#define LAN966X_VCAP_CID_IS2_MAX (VCAP_CID_INGRESS_STAGE2_L2 - 1) /* IS2 Max */ + /* MAC table entry types. * ENTRYTYPE_NORMAL is subject to aging. * ENTRYTYPE_LOCKED is not subject to aging. @@ -116,6 +120,14 @@ enum lan966x_fdma_action { FDMA_REDIRECT, }; +/* Controls how PORT_MASK is applied */ +enum LAN966X_PORT_MASK_MODE { + LAN966X_PMM_NO_ACTION, + LAN966X_PMM_REPLACE, + LAN966X_PMM_FORWARDING, + LAN966X_PMM_REDIRECT, +}; + struct lan966x_port; struct lan966x_db { @@ -473,6 +485,8 @@ irqreturn_t lan966x_ptp_irq_handler(int irq, void *args); irqreturn_t lan966x_ptp_ext_irq_handler(int irq, void *args); u32 lan966x_ptp_get_period_ps(void); int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts); +int lan966x_ptp_setup_traps(struct lan966x_port *port, struct ifreq *ifr); +int lan966x_ptp_del_traps(struct lan966x_port *port); int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev); int lan966x_fdma_xmit_xdpf(struct lan966x_port *port, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c index e5a2bbe064f8..300fe4005919 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c @@ -3,6 +3,8 @@ #include <linux/ptp_classify.h> #include "lan966x_main.h" +#include "vcap_api.h" +#include "vcap_api_client.h" #define LAN966X_MAX_PTP_ID 512 @@ -18,6 +20,17 @@ #define TOD_ACC_PIN 0x7 +/* This represents the base rule ID for the PTP rules that are added in the + * VCAP to trap frames to CPU. This number needs to be bigger than the maximum + * number of entries that can exist in the VCAP. + */ +#define LAN966X_VCAP_PTP_RULE_ID 1000000 +#define LAN966X_VCAP_L2_PTP_TRAP (LAN966X_VCAP_PTP_RULE_ID + 0) +#define LAN966X_VCAP_IPV4_EV_PTP_TRAP (LAN966X_VCAP_PTP_RULE_ID + 1) +#define LAN966X_VCAP_IPV4_GEN_PTP_TRAP (LAN966X_VCAP_PTP_RULE_ID + 2) +#define LAN966X_VCAP_IPV6_EV_PTP_TRAP (LAN966X_VCAP_PTP_RULE_ID + 3) +#define LAN966X_VCAP_IPV6_GEN_PTP_TRAP (LAN966X_VCAP_PTP_RULE_ID + 4) + enum { PTP_PIN_ACTION_IDLE = 0, PTP_PIN_ACTION_LOAD, @@ -35,19 +48,228 @@ static u64 lan966x_ptp_get_nominal_value(void) return 0x304d4873ecade305; } +static int lan966x_ptp_add_trap(struct lan966x_port *port, + int (*add_ptp_key)(struct vcap_rule *vrule, + struct lan966x_port*), + u32 rule_id, + u16 proto) +{ + struct lan966x *lan966x = port->lan966x; + struct vcap_rule *vrule; + int err; + + vrule = vcap_get_rule(lan966x->vcap_ctrl, rule_id); + if (vrule) { + u32 value, mask; + + /* Just modify the ingress port mask and exit */ + vcap_rule_get_key_u32(vrule, VCAP_KF_IF_IGR_PORT_MASK, + &value, &mask); + mask &= ~BIT(port->chip_port); + vcap_rule_mod_key_u32(vrule, VCAP_KF_IF_IGR_PORT_MASK, + value, mask); + + err = vcap_mod_rule(vrule); + goto free_rule; + } + + vrule = vcap_alloc_rule(lan966x->vcap_ctrl, port->dev, + LAN966X_VCAP_CID_IS2_L0, + VCAP_USER_PTP, 0, rule_id); + if (!vrule) + return -ENOMEM; + if (IS_ERR(vrule)) + return PTR_ERR(vrule); + + err = add_ptp_key(vrule, port); + if (err) + goto free_rule; + + err = vcap_set_rule_set_actionset(vrule, VCAP_AFS_BASE_TYPE); + err |= vcap_rule_add_action_bit(vrule, VCAP_AF_CPU_COPY_ENA, VCAP_BIT_1); + err |= vcap_rule_add_action_u32(vrule, VCAP_AF_MASK_MODE, LAN966X_PMM_REPLACE); + err |= vcap_val_rule(vrule, proto); + if (err) + goto free_rule; + + err = vcap_add_rule(vrule); + +free_rule: + /* Free the local copy of the rule */ + vcap_free_rule(vrule); + return err; +} + +static int lan966x_ptp_del_trap(struct lan966x_port *port, + u32 rule_id) +{ + struct lan966x *lan966x = port->lan966x; + struct vcap_rule *vrule; + u32 value, mask; + int err; + + vrule = vcap_get_rule(lan966x->vcap_ctrl, rule_id); + if (!vrule) + return -EEXIST; + + vcap_rule_get_key_u32(vrule, VCAP_KF_IF_IGR_PORT_MASK, &value, &mask); + mask |= BIT(port->chip_port); + + /* No other port requires this trap, so it is safe to remove it */ + if (mask == GENMASK(lan966x->num_phys_ports, 0)) { + err = vcap_del_rule(lan966x->vcap_ctrl, port->dev, rule_id); + goto free_rule; + } + + vcap_rule_mod_key_u32(vrule, VCAP_KF_IF_IGR_PORT_MASK, value, mask); + err = vcap_mod_rule(vrule); + +free_rule: + vcap_free_rule(vrule); + return err; +} + +static int lan966x_ptp_add_l2_key(struct vcap_rule *vrule, + struct lan966x_port *port) +{ + return vcap_rule_add_key_u32(vrule, VCAP_KF_ETYPE, ETH_P_1588, ~0); +} + +static int lan966x_ptp_add_ip_event_key(struct vcap_rule *vrule, + struct lan966x_port *port) +{ + return vcap_rule_add_key_u32(vrule, VCAP_KF_L4_DPORT, PTP_EV_PORT, ~0) || + vcap_rule_add_key_bit(vrule, VCAP_KF_TCP_IS, VCAP_BIT_0); +} + +static int lan966x_ptp_add_ip_general_key(struct vcap_rule *vrule, + struct lan966x_port *port) +{ + return vcap_rule_add_key_u32(vrule, VCAP_KF_L4_DPORT, PTP_GEN_PORT, ~0) || + vcap_rule_add_key_bit(vrule, VCAP_KF_TCP_IS, VCAP_BIT_0); +} + +static int lan966x_ptp_add_l2_rule(struct lan966x_port *port) +{ + return lan966x_ptp_add_trap(port, lan966x_ptp_add_l2_key, + LAN966X_VCAP_L2_PTP_TRAP, ETH_P_ALL); +} + +static int lan966x_ptp_add_ipv4_rules(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_add_trap(port, lan966x_ptp_add_ip_event_key, + LAN966X_VCAP_IPV4_EV_PTP_TRAP, ETH_P_IP); + if (err) + return err; + + err = lan966x_ptp_add_trap(port, lan966x_ptp_add_ip_general_key, + LAN966X_VCAP_IPV4_GEN_PTP_TRAP, ETH_P_IP); + if (err) + lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV4_EV_PTP_TRAP); + + return err; +} + +static int lan966x_ptp_add_ipv6_rules(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_add_trap(port, lan966x_ptp_add_ip_event_key, + LAN966X_VCAP_IPV6_EV_PTP_TRAP, ETH_P_IPV6); + if (err) + return err; + + err = lan966x_ptp_add_trap(port, lan966x_ptp_add_ip_general_key, + LAN966X_VCAP_IPV6_GEN_PTP_TRAP, ETH_P_IPV6); + if (err) + lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV6_EV_PTP_TRAP); + + return err; +} + +static int lan966x_ptp_del_l2_rule(struct lan966x_port *port) +{ + return lan966x_ptp_del_trap(port, LAN966X_VCAP_L2_PTP_TRAP); +} + +static int lan966x_ptp_del_ipv4_rules(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV4_EV_PTP_TRAP); + err |= lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV4_GEN_PTP_TRAP); + + return err; +} + +static int lan966x_ptp_del_ipv6_rules(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV6_EV_PTP_TRAP); + err |= lan966x_ptp_del_trap(port, LAN966X_VCAP_IPV6_GEN_PTP_TRAP); + + return err; +} + +static int lan966x_ptp_add_traps(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_add_l2_rule(port); + if (err) + goto err_l2; + + err = lan966x_ptp_add_ipv4_rules(port); + if (err) + goto err_ipv4; + + err = lan966x_ptp_add_ipv6_rules(port); + if (err) + goto err_ipv6; + + return err; + +err_ipv6: + lan966x_ptp_del_ipv4_rules(port); +err_ipv4: + lan966x_ptp_del_l2_rule(port); +err_l2: + return err; +} + +int lan966x_ptp_del_traps(struct lan966x_port *port) +{ + int err; + + err = lan966x_ptp_del_l2_rule(port); + err |= lan966x_ptp_del_ipv4_rules(port); + err |= lan966x_ptp_del_ipv6_rules(port); + + return err; +} + +int lan966x_ptp_setup_traps(struct lan966x_port *port, struct ifreq *ifr) +{ + struct hwtstamp_config cfg; + + if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) + return -EFAULT; + + if (cfg.rx_filter == HWTSTAMP_FILTER_NONE) + return lan966x_ptp_del_traps(port); + else + return lan966x_ptp_add_traps(port); +} + int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr) { struct lan966x *lan966x = port->lan966x; struct hwtstamp_config cfg; struct lan966x_phc *phc; - /* For now don't allow to run ptp on ports that are part of a bridge, - * because in case of transparent clock the HW will still forward the - * frames, so there would be duplicate frames - */ - if (lan966x->bridge_mask & BIT(port->chip_port)) - return -EINVAL; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) return -EFAULT; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c index 04a2afd683cc..ba3fa917d6b7 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c @@ -4,14 +4,6 @@ #include "vcap_api.h" #include "vcap_api_client.h" -/* Controls how PORT_MASK is applied */ -enum LAN966X_PORT_MASK_MODE { - LAN966X_PMM_NO_ACTION, - LAN966X_PMM_REPLACE, - LAN966X_PMM_FORWARDING, - LAN966X_PMM_REDIRECT, -}; - struct lan966x_tc_flower_parse_usage { struct flow_cls_offload *f; struct flow_rule *frule; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c index 44f40d914947..d8dc9fbb81e1 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c @@ -5,10 +5,6 @@ #include "vcap_api.h" #include "vcap_api_client.h" -#define LAN966X_VCAP_CID_IS2_L0 VCAP_CID_INGRESS_STAGE2_L0 /* IS2 lookup 0 */ -#define LAN966X_VCAP_CID_IS2_L1 VCAP_CID_INGRESS_STAGE2_L1 /* IS2 lookup 1 */ -#define LAN966X_VCAP_CID_IS2_MAX (VCAP_CID_INGRESS_STAGE2_L2 - 1) /* IS2 Max */ - #define STREAMSIZE (64 * 4) #define LAN966X_IS2_LOOKUPS 2 @@ -219,9 +215,12 @@ static void lan966x_vcap_add_default_fields(struct net_device *dev, struct vcap_rule *rule) { struct lan966x_port *port = netdev_priv(dev); + u32 value, mask; - vcap_rule_add_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK, 0, - ~BIT(port->chip_port)); + if (vcap_rule_get_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK, + &value, &mask)) + vcap_rule_add_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK, 0, + ~BIT(port->chip_port)); if (lan966x_vcap_is_first_chain(rule)) vcap_rule_add_key_bit(rule, VCAP_KF_LOOKUP_FIRST_IS, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c index 66360c8c5a38..141897dfe388 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c @@ -317,7 +317,7 @@ int sparx5_fdma_xmit(struct sparx5 *sparx5, u32 *ifh, struct sk_buff *skb) next_dcb_hw = sparx5_fdma_next_dcb(tx, tx->curr_entry); db_hw = &next_dcb_hw->db[0]; if (!(db_hw->status & FDMA_DCB_STATUS_DONE)) - tx->dropped++; + return -EINVAL; db = list_first_entry(&tx->db_list, struct sparx5_db, list); list_move_tail(&db->list, &tx->db_list); next_dcb_hw->nextptr = FDMA_DCB_INVALID_DATA; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c index f8382d3124e0..d25f4f09faa0 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -897,6 +897,8 @@ static int mchp_sparx5_probe(struct platform_device *pdev) cleanup_ports: sparx5_cleanup_ports(sparx5); + if (sparx5->mact_queue) + destroy_workqueue(sparx5->mact_queue); cleanup_config: kfree(configs); cleanup_pnode: @@ -923,6 +925,7 @@ static int mchp_sparx5_remove(struct platform_device *pdev) sparx5_vcap_destroy(sparx5); /* Unregister netdevs */ sparx5_unregister_notifier_blocks(sparx5); + destroy_workqueue(sparx5->mact_queue); return 0; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c index 83c16ca5b30f..6db6ac6a3bbc 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c @@ -234,9 +234,8 @@ netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev) sparx5_set_port_ifh(ifh, port->portno); if (sparx5->ptp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { - ret = sparx5_ptp_txtstamp_request(port, skb); - if (ret) - return ret; + if (sparx5_ptp_txtstamp_request(port, skb) < 0) + return NETDEV_TX_BUSY; sparx5_set_port_ifh_rew_op(ifh, SPARX5_SKB_CB(skb)->rew_op); sparx5_set_port_ifh_pdu_type(ifh, SPARX5_SKB_CB(skb)->pdu_type); @@ -250,23 +249,31 @@ netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev) else ret = sparx5_inject(sparx5, ifh, skb, dev); - if (ret == NETDEV_TX_OK) { - stats->tx_bytes += skb->len; - stats->tx_packets++; + if (ret == -EBUSY) + goto busy; + if (ret < 0) + goto drop; - if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && - SPARX5_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) - return ret; + stats->tx_bytes += skb->len; + stats->tx_packets++; + sparx5->tx.packets++; - dev_kfree_skb_any(skb); - } else { - stats->tx_dropped++; + if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && + SPARX5_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) + return NETDEV_TX_OK; - if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && - SPARX5_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) - sparx5_ptp_txtstamp_release(port, skb); - } - return ret; + dev_consume_skb_any(skb); + return NETDEV_TX_OK; +drop: + stats->tx_dropped++; + sparx5->tx.dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +busy: + if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && + SPARX5_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) + sparx5_ptp_txtstamp_release(port, skb); + return NETDEV_TX_BUSY; } static enum hrtimer_restart sparx5_injection_timeout(struct hrtimer *tmr) diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.c b/drivers/net/ethernet/microchip/vcap/vcap_api.c index 93efa8243b02..664aae3e2acd 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api.c @@ -169,6 +169,227 @@ static void vcap_encode_typegroups(u32 *stream, int sw_width, } } +static bool vcap_bitarray_zero(int width, u8 *value) +{ + int bytes = DIV_ROUND_UP(width, BITS_PER_BYTE); + u8 total = 0, bmask = 0xff; + int rwidth = width; + int idx; + + for (idx = 0; idx < bytes; ++idx, rwidth -= BITS_PER_BYTE) { + if (rwidth && rwidth < BITS_PER_BYTE) + bmask = (1 << rwidth) - 1; + total += value[idx] & bmask; + } + return total == 0; +} + +static bool vcap_get_bit(u32 *stream, struct vcap_stream_iter *itr) +{ + u32 mask = BIT(itr->reg_bitpos); + u32 *p = &stream[itr->reg_idx]; + + return !!(*p & mask); +} + +static void vcap_decode_field(u32 *stream, struct vcap_stream_iter *itr, + int width, u8 *value) +{ + int idx; + + /* Loop over the field value bits and get the field bits and + * set them in the output value byte array + */ + for (idx = 0; idx < width; idx++) { + u8 bidx = idx & 0x7; + + /* Decode one field value bit */ + if (vcap_get_bit(stream, itr)) + *value |= 1 << bidx; + vcap_iter_next(itr); + if (bidx == 7) + value++; + } +} + +/* Verify that the type id in the stream matches the type id of the keyset */ +static bool vcap_verify_keystream_keyset(struct vcap_control *vctrl, + enum vcap_type vt, + u32 *keystream, + u32 *mskstream, + enum vcap_keyfield_set keyset) +{ + const struct vcap_info *vcap = &vctrl->vcaps[vt]; + const struct vcap_field *typefld; + const struct vcap_typegroup *tgt; + const struct vcap_field *fields; + struct vcap_stream_iter iter; + const struct vcap_set *info; + u32 value = 0; + u32 mask = 0; + + if (vcap_keyfield_count(vctrl, vt, keyset) == 0) + return false; + + info = vcap_keyfieldset(vctrl, vt, keyset); + /* Check that the keyset is valid */ + if (!info) + return false; + + /* a type_id of value -1 means that there is no type field */ + if (info->type_id == (u8)-1) + return true; + + /* Get a valid typegroup for the specific keyset */ + tgt = vcap_keyfield_typegroup(vctrl, vt, keyset); + if (!tgt) + return false; + + fields = vcap_keyfields(vctrl, vt, keyset); + if (!fields) + return false; + + typefld = &fields[VCAP_KF_TYPE]; + vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset); + vcap_decode_field(mskstream, &iter, typefld->width, (u8 *)&mask); + /* no type info if there are no mask bits */ + if (vcap_bitarray_zero(typefld->width, (u8 *)&mask)) + return false; + + /* Get the value of the type field in the stream and compare to the + * one define in the vcap keyset + */ + vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset); + vcap_decode_field(keystream, &iter, typefld->width, (u8 *)&value); + + return (value & mask) == (info->type_id & mask); +} + +/* Verify that the typegroup bits have the correct values */ +static int vcap_verify_typegroups(u32 *stream, int sw_width, + const struct vcap_typegroup *tgt, bool mask, + int sw_max) +{ + struct vcap_stream_iter iter; + int sw_cnt, idx; + + vcap_iter_set(&iter, sw_width, tgt, 0); + sw_cnt = 0; + while (iter.tg->width) { + u32 value = 0; + u32 tg_value = iter.tg->value; + + if (mask) + tg_value = (1 << iter.tg->width) - 1; + /* Set position to current typegroup bit */ + iter.offset = iter.tg->offset; + vcap_iter_update(&iter); + for (idx = 0; idx < iter.tg->width; idx++) { + /* Decode one typegroup bit */ + if (vcap_get_bit(stream, &iter)) + value |= 1 << idx; + iter.offset++; + vcap_iter_update(&iter); + } + if (value != tg_value) + return -EINVAL; + iter.tg++; /* next typegroup */ + sw_cnt++; + /* Stop checking more typegroups */ + if (sw_max && sw_cnt >= sw_max) + break; + } + return 0; +} + +/* Find the subword width of the key typegroup that matches the stream data */ +static int vcap_find_keystream_typegroup_sw(struct vcap_control *vctrl, + enum vcap_type vt, u32 *stream, + bool mask, int sw_max) +{ + const struct vcap_typegroup **tgt; + int sw_idx, res; + + tgt = vctrl->vcaps[vt].keyfield_set_typegroups; + /* Try the longest subword match first */ + for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) { + if (!tgt[sw_idx]) + continue; + + res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].sw_width, + tgt[sw_idx], mask, sw_max); + if (res == 0) + return sw_idx; + } + return -EINVAL; +} + +/* Verify that the typegroup information, subword count, keyset and type id + * are in sync and correct, return the list of matchin keysets + */ +int +vcap_find_keystream_keysets(struct vcap_control *vctrl, + enum vcap_type vt, + u32 *keystream, + u32 *mskstream, + bool mask, int sw_max, + struct vcap_keyset_list *kslist) +{ + const struct vcap_set *keyfield_set; + int sw_count, idx; + + sw_count = vcap_find_keystream_typegroup_sw(vctrl, vt, keystream, mask, + sw_max); + if (sw_count < 0) + return sw_count; + + keyfield_set = vctrl->vcaps[vt].keyfield_set; + for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) { + if (keyfield_set[idx].sw_per_item != sw_count) + continue; + + if (vcap_verify_keystream_keyset(vctrl, vt, keystream, + mskstream, idx)) + vcap_keyset_list_add(kslist, idx); + } + if (kslist->cnt > 0) + return 0; + return -EINVAL; +} +EXPORT_SYMBOL_GPL(vcap_find_keystream_keysets); + +/* Read key data from a VCAP address and discover if there are any rule keysets + * here + */ +int vcap_addr_keysets(struct vcap_control *vctrl, + struct net_device *ndev, + struct vcap_admin *admin, + int addr, + struct vcap_keyset_list *kslist) +{ + enum vcap_type vt = admin->vtype; + int keyset_sw_regs, idx; + u32 key = 0, mask = 0; + + /* Read the cache at the specified address */ + keyset_sw_regs = DIV_ROUND_UP(vctrl->vcaps[vt].sw_width, 32); + vctrl->ops->update(ndev, admin, VCAP_CMD_READ, VCAP_SEL_ALL, addr); + vctrl->ops->cache_read(ndev, admin, VCAP_SEL_ENTRY, 0, + keyset_sw_regs); + /* Skip uninitialized key/mask entries */ + for (idx = 0; idx < keyset_sw_regs; ++idx) { + key |= ~admin->cache.keystream[idx]; + mask |= admin->cache.maskstream[idx]; + } + if (key == 0 && mask == 0) + return -EINVAL; + /* Decode and locate the keysets */ + return vcap_find_keystream_keysets(vctrl, vt, admin->cache.keystream, + admin->cache.maskstream, false, 0, + kslist); +} +EXPORT_SYMBOL_GPL(vcap_addr_keysets); + /* Return the list of keyfields for the keyset */ const struct vcap_field *vcap_keyfields(struct vcap_control *vctrl, enum vcap_type vt, @@ -618,6 +839,517 @@ struct vcap_rule_internal *vcap_dup_rule(struct vcap_rule_internal *ri) return duprule; } +static void vcap_apply_width(u8 *dst, int width, int bytes) +{ + u8 bmask; + int idx; + + for (idx = 0; idx < bytes; idx++) { + if (width > 0) + if (width < 8) + bmask = (1 << width) - 1; + else + bmask = ~0; + else + bmask = 0; + dst[idx] &= bmask; + width -= 8; + } +} + +static void vcap_copy_from_w32be(u8 *dst, u8 *src, int size, int width) +{ + int idx, ridx, wstart, nidx; + int tail_bytes = (((size + 4) >> 2) << 2) - size; + + for (idx = 0, ridx = size - 1; idx < size; ++idx, --ridx) { + wstart = (idx >> 2) << 2; + nidx = wstart + 3 - (idx & 0x3); + if (nidx >= size) + nidx -= tail_bytes; + dst[nidx] = src[ridx]; + } + + vcap_apply_width(dst, width, size); +} + +static void vcap_copy_action_bit_field(struct vcap_u1_action *field, u8 *value) +{ + field->value = (*value) & 0x1; +} + +static void vcap_copy_limited_actionfield(u8 *dstvalue, u8 *srcvalue, + int width, int bytes) +{ + memcpy(dstvalue, srcvalue, bytes); + vcap_apply_width(dstvalue, width, bytes); +} + +static void vcap_copy_to_client_actionfield(struct vcap_rule_internal *ri, + struct vcap_client_actionfield *field, + u8 *value, u16 width) +{ + int field_size = actionfield_size_table[field->ctrl.type]; + + if (ri->admin->w32be) { + switch (field->ctrl.type) { + case VCAP_FIELD_BIT: + vcap_copy_action_bit_field(&field->data.u1, value); + break; + case VCAP_FIELD_U32: + vcap_copy_limited_actionfield((u8 *)&field->data.u32.value, + value, + width, field_size); + break; + case VCAP_FIELD_U48: + vcap_copy_from_w32be(field->data.u48.value, value, + field_size, width); + break; + case VCAP_FIELD_U56: + vcap_copy_from_w32be(field->data.u56.value, value, + field_size, width); + break; + case VCAP_FIELD_U64: + vcap_copy_from_w32be(field->data.u64.value, value, + field_size, width); + break; + case VCAP_FIELD_U72: + vcap_copy_from_w32be(field->data.u72.value, value, + field_size, width); + break; + case VCAP_FIELD_U112: + vcap_copy_from_w32be(field->data.u112.value, value, + field_size, width); + break; + case VCAP_FIELD_U128: + vcap_copy_from_w32be(field->data.u128.value, value, + field_size, width); + break; + }; + } else { + switch (field->ctrl.type) { + case VCAP_FIELD_BIT: + vcap_copy_action_bit_field(&field->data.u1, value); + break; + case VCAP_FIELD_U32: + vcap_copy_limited_actionfield((u8 *)&field->data.u32.value, + value, + width, field_size); + break; + case VCAP_FIELD_U48: + vcap_copy_limited_actionfield(field->data.u48.value, + value, + width, field_size); + break; + case VCAP_FIELD_U56: + vcap_copy_limited_actionfield(field->data.u56.value, + value, + width, field_size); + break; + case VCAP_FIELD_U64: + vcap_copy_limited_actionfield(field->data.u64.value, + value, + width, field_size); + break; + case VCAP_FIELD_U72: + vcap_copy_limited_actionfield(field->data.u72.value, + value, + width, field_size); + break; + case VCAP_FIELD_U112: + vcap_copy_limited_actionfield(field->data.u112.value, + value, + width, field_size); + break; + case VCAP_FIELD_U128: + vcap_copy_limited_actionfield(field->data.u128.value, + value, + width, field_size); + break; + }; + } +} + +static void vcap_copy_key_bit_field(struct vcap_u1_key *field, + u8 *value, u8 *mask) +{ + field->value = (*value) & 0x1; + field->mask = (*mask) & 0x1; +} + +static void vcap_copy_limited_keyfield(u8 *dstvalue, u8 *dstmask, + u8 *srcvalue, u8 *srcmask, + int width, int bytes) +{ + memcpy(dstvalue, srcvalue, bytes); + vcap_apply_width(dstvalue, width, bytes); + memcpy(dstmask, srcmask, bytes); + vcap_apply_width(dstmask, width, bytes); +} + +static void vcap_copy_to_client_keyfield(struct vcap_rule_internal *ri, + struct vcap_client_keyfield *field, + u8 *value, u8 *mask, u16 width) +{ + int field_size = keyfield_size_table[field->ctrl.type] / 2; + + if (ri->admin->w32be) { + switch (field->ctrl.type) { + case VCAP_FIELD_BIT: + vcap_copy_key_bit_field(&field->data.u1, value, mask); + break; + case VCAP_FIELD_U32: + vcap_copy_limited_keyfield((u8 *)&field->data.u32.value, + (u8 *)&field->data.u32.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U48: + vcap_copy_from_w32be(field->data.u48.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u48.mask, mask, + field_size, width); + break; + case VCAP_FIELD_U56: + vcap_copy_from_w32be(field->data.u56.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u56.mask, mask, + field_size, width); + break; + case VCAP_FIELD_U64: + vcap_copy_from_w32be(field->data.u64.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u64.mask, mask, + field_size, width); + break; + case VCAP_FIELD_U72: + vcap_copy_from_w32be(field->data.u72.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u72.mask, mask, + field_size, width); + break; + case VCAP_FIELD_U112: + vcap_copy_from_w32be(field->data.u112.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u112.mask, mask, + field_size, width); + break; + case VCAP_FIELD_U128: + vcap_copy_from_w32be(field->data.u128.value, value, + field_size, width); + vcap_copy_from_w32be(field->data.u128.mask, mask, + field_size, width); + break; + }; + } else { + switch (field->ctrl.type) { + case VCAP_FIELD_BIT: + vcap_copy_key_bit_field(&field->data.u1, value, mask); + break; + case VCAP_FIELD_U32: + vcap_copy_limited_keyfield((u8 *)&field->data.u32.value, + (u8 *)&field->data.u32.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U48: + vcap_copy_limited_keyfield(field->data.u48.value, + field->data.u48.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U56: + vcap_copy_limited_keyfield(field->data.u56.value, + field->data.u56.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U64: + vcap_copy_limited_keyfield(field->data.u64.value, + field->data.u64.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U72: + vcap_copy_limited_keyfield(field->data.u72.value, + field->data.u72.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U112: + vcap_copy_limited_keyfield(field->data.u112.value, + field->data.u112.mask, + value, mask, + width, field_size); + break; + case VCAP_FIELD_U128: + vcap_copy_limited_keyfield(field->data.u128.value, + field->data.u128.mask, + value, mask, + width, field_size); + break; + }; + } +} + +static void vcap_rule_alloc_keyfield(struct vcap_rule_internal *ri, + const struct vcap_field *keyfield, + enum vcap_key_field key, + u8 *value, u8 *mask) +{ + struct vcap_client_keyfield *field; + + field = kzalloc(sizeof(*field), GFP_KERNEL); + if (!field) + return; + INIT_LIST_HEAD(&field->ctrl.list); + field->ctrl.key = key; + field->ctrl.type = keyfield->type; + vcap_copy_to_client_keyfield(ri, field, value, mask, keyfield->width); + list_add_tail(&field->ctrl.list, &ri->data.keyfields); +} + +/* Read key data from a VCAP address and discover if there is a rule keyset + * here + */ +static bool +vcap_verify_actionstream_actionset(struct vcap_control *vctrl, + enum vcap_type vt, + u32 *actionstream, + enum vcap_actionfield_set actionset) +{ + const struct vcap_typegroup *tgt; + const struct vcap_field *fields; + const struct vcap_set *info; + + if (vcap_actionfield_count(vctrl, vt, actionset) == 0) + return false; + + info = vcap_actionfieldset(vctrl, vt, actionset); + /* Check that the actionset is valid */ + if (!info) + return false; + + /* a type_id of value -1 means that there is no type field */ + if (info->type_id == (u8)-1) + return true; + + /* Get a valid typegroup for the specific actionset */ + tgt = vcap_actionfield_typegroup(vctrl, vt, actionset); + if (!tgt) + return false; + + fields = vcap_actionfields(vctrl, vt, actionset); + if (!fields) + return false; + + /* Later this will be expanded with a check of the type id */ + return true; +} + +/* Find the subword width of the action typegroup that matches the stream data + */ +static int vcap_find_actionstream_typegroup_sw(struct vcap_control *vctrl, + enum vcap_type vt, u32 *stream, + int sw_max) +{ + const struct vcap_typegroup **tgt; + int sw_idx, res; + + tgt = vctrl->vcaps[vt].actionfield_set_typegroups; + /* Try the longest subword match first */ + for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) { + if (!tgt[sw_idx]) + continue; + res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].act_width, + tgt[sw_idx], false, sw_max); + if (res == 0) + return sw_idx; + } + return -EINVAL; +} + +/* Verify that the typegroup information, subword count, actionset and type id + * are in sync and correct, return the actionset + */ +static enum vcap_actionfield_set +vcap_find_actionstream_actionset(struct vcap_control *vctrl, + enum vcap_type vt, + u32 *stream, + int sw_max) +{ + const struct vcap_set *actionfield_set; + int sw_count, idx; + bool res; + + sw_count = vcap_find_actionstream_typegroup_sw(vctrl, vt, stream, + sw_max); + if (sw_count < 0) + return sw_count; + + actionfield_set = vctrl->vcaps[vt].actionfield_set; + for (idx = 0; idx < vctrl->vcaps[vt].actionfield_set_size; ++idx) { + if (actionfield_set[idx].sw_per_item != sw_count) + continue; + + res = vcap_verify_actionstream_actionset(vctrl, vt, + stream, idx); + if (res) + return idx; + } + return -EINVAL; +} + +/* Store action value in an element in a list for the client */ +static void vcap_rule_alloc_actionfield(struct vcap_rule_internal *ri, + const struct vcap_field *actionfield, + enum vcap_action_field action, + u8 *value) +{ + struct vcap_client_actionfield *field; + + field = kzalloc(sizeof(*field), GFP_KERNEL); + if (!field) + return; + INIT_LIST_HEAD(&field->ctrl.list); + field->ctrl.action = action; + field->ctrl.type = actionfield->type; + vcap_copy_to_client_actionfield(ri, field, value, actionfield->width); + list_add_tail(&field->ctrl.list, &ri->data.actionfields); +} + +static int vcap_decode_actionset(struct vcap_rule_internal *ri) +{ + struct vcap_control *vctrl = ri->vctrl; + struct vcap_admin *admin = ri->admin; + const struct vcap_field *actionfield; + enum vcap_actionfield_set actionset; + enum vcap_type vt = admin->vtype; + const struct vcap_typegroup *tgt; + struct vcap_stream_iter iter; + int idx, res, actfield_count; + u32 *actstream; + u8 value[16]; + + actstream = admin->cache.actionstream; + res = vcap_find_actionstream_actionset(vctrl, vt, actstream, 0); + if (res < 0) { + pr_err("%s:%d: could not find valid actionset: %d\n", + __func__, __LINE__, res); + return -EINVAL; + } + actionset = res; + actfield_count = vcap_actionfield_count(vctrl, vt, actionset); + actionfield = vcap_actionfields(vctrl, vt, actionset); + tgt = vcap_actionfield_typegroup(vctrl, vt, actionset); + /* Start decoding the stream */ + for (idx = 0; idx < actfield_count; ++idx) { + if (actionfield[idx].width <= 0) + continue; + /* Get the action */ + memset(value, 0, DIV_ROUND_UP(actionfield[idx].width, 8)); + vcap_iter_init(&iter, vctrl->vcaps[vt].act_width, tgt, + actionfield[idx].offset); + vcap_decode_field(actstream, &iter, actionfield[idx].width, + value); + /* Skip if no bits are set */ + if (vcap_bitarray_zero(actionfield[idx].width, value)) + continue; + vcap_rule_alloc_actionfield(ri, &actionfield[idx], idx, value); + /* Later the action id will also be checked */ + } + return vcap_set_rule_set_actionset((struct vcap_rule *)ri, actionset); +} + +static int vcap_decode_keyset(struct vcap_rule_internal *ri) +{ + struct vcap_control *vctrl = ri->vctrl; + struct vcap_stream_iter kiter, miter; + struct vcap_admin *admin = ri->admin; + enum vcap_keyfield_set keysets[10]; + const struct vcap_field *keyfield; + enum vcap_type vt = admin->vtype; + const struct vcap_typegroup *tgt; + struct vcap_keyset_list matches; + enum vcap_keyfield_set keyset; + int idx, res, keyfield_count; + u32 *maskstream; + u32 *keystream; + u8 value[16]; + u8 mask[16]; + + keystream = admin->cache.keystream; + maskstream = admin->cache.maskstream; + matches.keysets = keysets; + matches.cnt = 0; + matches.max = ARRAY_SIZE(keysets); + res = vcap_find_keystream_keysets(vctrl, vt, keystream, maskstream, + false, 0, &matches); + if (res < 0) { + pr_err("%s:%d: could not find valid keysets: %d\n", + __func__, __LINE__, res); + return -EINVAL; + } + keyset = matches.keysets[0]; + keyfield_count = vcap_keyfield_count(vctrl, vt, keyset); + keyfield = vcap_keyfields(vctrl, vt, keyset); + tgt = vcap_keyfield_typegroup(vctrl, vt, keyset); + /* Start decoding the streams */ + for (idx = 0; idx < keyfield_count; ++idx) { + if (keyfield[idx].width <= 0) + continue; + /* First get the mask */ + memset(mask, 0, DIV_ROUND_UP(keyfield[idx].width, 8)); + vcap_iter_init(&miter, vctrl->vcaps[vt].sw_width, tgt, + keyfield[idx].offset); + vcap_decode_field(maskstream, &miter, keyfield[idx].width, + mask); + /* Skip if no mask bits are set */ + if (vcap_bitarray_zero(keyfield[idx].width, mask)) + continue; + /* Get the key */ + memset(value, 0, DIV_ROUND_UP(keyfield[idx].width, 8)); + vcap_iter_init(&kiter, vctrl->vcaps[vt].sw_width, tgt, + keyfield[idx].offset); + vcap_decode_field(keystream, &kiter, keyfield[idx].width, + value); + vcap_rule_alloc_keyfield(ri, &keyfield[idx], idx, value, mask); + } + return vcap_set_rule_set_keyset((struct vcap_rule *)ri, keyset); +} + +/* Read VCAP content into the VCAP cache */ +static int vcap_read_rule(struct vcap_rule_internal *ri) +{ + struct vcap_admin *admin = ri->admin; + int sw_idx, ent_idx = 0, act_idx = 0; + u32 addr = ri->addr; + + if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) { + pr_err("%s:%d: rule is empty\n", __func__, __LINE__); + return -EINVAL; + } + vcap_erase_cache(ri); + /* Use the values in the streams to read the VCAP cache */ + for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) { + ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ, + VCAP_SEL_ALL, addr); + ri->vctrl->ops->cache_read(ri->ndev, admin, + VCAP_SEL_ENTRY, ent_idx, + ri->keyset_sw_regs); + ri->vctrl->ops->cache_read(ri->ndev, admin, + VCAP_SEL_ACTION, act_idx, + ri->actionset_sw_regs); + if (sw_idx == 0) + ri->vctrl->ops->cache_read(ri->ndev, admin, + VCAP_SEL_COUNTER, + ri->counter_id, 0); + ent_idx += ri->keyset_sw_regs; + act_idx += ri->actionset_sw_regs; + } + return 0; +} + /* Write VCAP cache content to the VCAP HW instance */ static int vcap_write_rule(struct vcap_rule_internal *ri) { @@ -985,17 +1717,12 @@ static u32 vcap_next_rule_addr(u32 addr, struct vcap_rule_internal *ri) /* Assign a unique rule id and autogenerate one if id == 0 */ static u32 vcap_set_rule_id(struct vcap_rule_internal *ri) { - u32 next_id; - if (ri->data.id != 0) return ri->data.id; - next_id = ri->vctrl->rule_id + 1; - - for (next_id = ri->vctrl->rule_id + 1; next_id < ~0; ++next_id) { + for (u32 next_id = 1; next_id < ~0; ++next_id) { if (!vcap_lookup_rule(ri->vctrl, next_id)) { ri->data.id = next_id; - ri->vctrl->rule_id = next_id; break; } } @@ -1188,6 +1915,82 @@ void vcap_free_rule(struct vcap_rule *rule) } EXPORT_SYMBOL_GPL(vcap_free_rule); +struct vcap_rule *vcap_get_rule(struct vcap_control *vctrl, u32 id) +{ + struct vcap_rule_internal *elem; + struct vcap_rule_internal *ri; + int err; + + ri = NULL; + + err = vcap_api_check(vctrl); + if (err) + return ERR_PTR(err); + elem = vcap_lookup_rule(vctrl, id); + if (!elem) + return NULL; + mutex_lock(&elem->admin->lock); + ri = vcap_dup_rule(elem); + if (IS_ERR(ri)) + goto unlock; + err = vcap_read_rule(ri); + if (err) { + ri = ERR_PTR(err); + goto unlock; + } + err = vcap_decode_keyset(ri); + if (err) { + ri = ERR_PTR(err); + goto unlock; + } + err = vcap_decode_actionset(ri); + if (err) { + ri = ERR_PTR(err); + goto unlock; + } + +unlock: + mutex_unlock(&elem->admin->lock); + return (struct vcap_rule *)ri; +} +EXPORT_SYMBOL_GPL(vcap_get_rule); + +/* Update existing rule */ +int vcap_mod_rule(struct vcap_rule *rule) +{ + struct vcap_rule_internal *ri = to_intrule(rule); + struct vcap_counter ctr; + int err; + + err = vcap_api_check(ri->vctrl); + if (err) + return err; + + if (!vcap_lookup_rule(ri->vctrl, ri->data.id)) + return -ENOENT; + + mutex_lock(&ri->admin->lock); + /* Encode the bitstreams to the VCAP cache */ + vcap_erase_cache(ri); + err = vcap_encode_rule(ri); + if (err) + goto out; + + err = vcap_write_rule(ri); + if (err) + goto out; + + memset(&ctr, 0, sizeof(ctr)); + err = vcap_write_counter(ri, &ctr); + if (err) + goto out; + +out: + mutex_unlock(&ri->admin->lock); + return err; +} +EXPORT_SYMBOL_GPL(vcap_mod_rule); + /* Return the alignment offset for a new rule address */ static int vcap_valid_rule_move(struct vcap_rule_internal *el, int offset) { @@ -1394,7 +2197,7 @@ static void vcap_copy_from_client_keyfield(struct vcap_rule *rule, vcap_copy_to_w32be(field->data.u128.value, data->u128.value, size); vcap_copy_to_w32be(field->data.u128.mask, data->u128.mask, size); break; - }; + } } /* Check if the keyfield is already in the rule */ @@ -1535,6 +2338,22 @@ int vcap_rule_add_key_u128(struct vcap_rule *rule, enum vcap_key_field key, } EXPORT_SYMBOL_GPL(vcap_rule_add_key_u128); +int vcap_rule_get_key_u32(struct vcap_rule *rule, enum vcap_key_field key, + u32 *value, u32 *mask) +{ + struct vcap_client_keyfield *ckf; + + ckf = vcap_find_keyfield(rule, key); + if (!ckf) + return -ENOENT; + + *value = ckf->data.u32.value; + *mask = ckf->data.u32.mask; + + return 0; +} +EXPORT_SYMBOL_GPL(vcap_rule_get_key_u32); + /* Find a client action field in a rule */ static struct vcap_client_actionfield * vcap_find_actionfield(struct vcap_rule *rule, enum vcap_action_field act) @@ -1584,7 +2403,7 @@ static void vcap_copy_from_client_actionfield(struct vcap_rule *rule, case VCAP_FIELD_U128: vcap_copy_to_w32be(field->data.u128.value, data->u128.value, size); break; - }; + } } /* Check if the actionfield is already in the rule */ diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.h b/drivers/net/ethernet/microchip/vcap/vcap_api.h index ca4499838306..689c7270f2a8 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api.h @@ -268,7 +268,6 @@ struct vcap_operations { /* VCAP API Client control interface */ struct vcap_control { - u32 rule_id; /* last used rule id (unique across VCAP instances) */ struct vcap_operations *ops; /* client supplied operations */ const struct vcap_info *vcaps; /* client supplied vcap models */ const struct vcap_statistics *stats; /* client supplied vcap stats */ diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_client.h b/drivers/net/ethernet/microchip/vcap/vcap_api_client.h index 93a0fcb12a81..0319866f9c94 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_client.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_client.h @@ -170,6 +170,10 @@ int vcap_add_rule(struct vcap_rule *rule); int vcap_del_rule(struct vcap_control *vctrl, struct net_device *ndev, u32 id); /* Make a full copy of an existing rule with a new rule id */ struct vcap_rule *vcap_copy_rule(struct vcap_rule *rule); +/* Get rule from a VCAP instance */ +struct vcap_rule *vcap_get_rule(struct vcap_control *vctrl, u32 id); +/* Update existing rule */ +int vcap_mod_rule(struct vcap_rule *rule); /* Update the keyset for the rule */ int vcap_set_rule_set_keyset(struct vcap_rule *rule, @@ -254,4 +258,8 @@ int vcap_rule_mod_action_u32(struct vcap_rule *rule, enum vcap_action_field action, u32 value); +/* Get a 32 bit key field value and mask from the rule */ +int vcap_rule_get_key_u32(struct vcap_rule *rule, enum vcap_key_field key, + u32 *value, u32 *mask); + #endif /* __VCAP_API_CLIENT__ */ diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c index 5df00e940333..895bfff550d2 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c @@ -18,355 +18,15 @@ struct vcap_port_debugfs_info { struct net_device *ndev; }; -static bool vcap_bitarray_zero(int width, u8 *value) -{ - int bytes = DIV_ROUND_UP(width, BITS_PER_BYTE); - u8 total = 0, bmask = 0xff; - int rwidth = width; - int idx; - - for (idx = 0; idx < bytes; ++idx, rwidth -= BITS_PER_BYTE) { - if (rwidth && rwidth < BITS_PER_BYTE) - bmask = (1 << rwidth) - 1; - total += value[idx] & bmask; - } - return total == 0; -} - -static bool vcap_get_bit(u32 *stream, struct vcap_stream_iter *itr) -{ - u32 mask = BIT(itr->reg_bitpos); - u32 *p = &stream[itr->reg_idx]; - - return !!(*p & mask); -} - -static void vcap_decode_field(u32 *stream, struct vcap_stream_iter *itr, - int width, u8 *value) -{ - int idx; - - /* Loop over the field value bits and get the field bits and - * set them in the output value byte array - */ - for (idx = 0; idx < width; idx++) { - u8 bidx = idx & 0x7; - - /* Decode one field value bit */ - if (vcap_get_bit(stream, itr)) - *value |= 1 << bidx; - vcap_iter_next(itr); - if (bidx == 7) - value++; - } -} - -/* Verify that the typegroup bits have the correct values */ -static int vcap_verify_typegroups(u32 *stream, int sw_width, - const struct vcap_typegroup *tgt, bool mask, - int sw_max) -{ - struct vcap_stream_iter iter; - int sw_cnt, idx; - - vcap_iter_set(&iter, sw_width, tgt, 0); - sw_cnt = 0; - while (iter.tg->width) { - u32 value = 0; - u32 tg_value = iter.tg->value; - - if (mask) - tg_value = (1 << iter.tg->width) - 1; - /* Set position to current typegroup bit */ - iter.offset = iter.tg->offset; - vcap_iter_update(&iter); - for (idx = 0; idx < iter.tg->width; idx++) { - /* Decode one typegroup bit */ - if (vcap_get_bit(stream, &iter)) - value |= 1 << idx; - iter.offset++; - vcap_iter_update(&iter); - } - if (value != tg_value) - return -EINVAL; - iter.tg++; /* next typegroup */ - sw_cnt++; - /* Stop checking more typegroups */ - if (sw_max && sw_cnt >= sw_max) - break; - } - return 0; -} - -/* Find the subword width of the key typegroup that matches the stream data */ -static int vcap_find_keystream_typegroup_sw(struct vcap_control *vctrl, - enum vcap_type vt, u32 *stream, - bool mask, int sw_max) -{ - const struct vcap_typegroup **tgt; - int sw_idx, res; - - tgt = vctrl->vcaps[vt].keyfield_set_typegroups; - /* Try the longest subword match first */ - for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) { - if (!tgt[sw_idx]) - continue; - - res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].sw_width, - tgt[sw_idx], mask, sw_max); - if (res == 0) - return sw_idx; - } - return -EINVAL; -} - -/* Find the subword width of the action typegroup that matches the stream data - */ -static int vcap_find_actionstream_typegroup_sw(struct vcap_control *vctrl, - enum vcap_type vt, u32 *stream, - int sw_max) -{ - const struct vcap_typegroup **tgt; - int sw_idx, res; - - tgt = vctrl->vcaps[vt].actionfield_set_typegroups; - /* Try the longest subword match first */ - for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) { - if (!tgt[sw_idx]) - continue; - res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].act_width, - tgt[sw_idx], false, sw_max); - if (res == 0) - return sw_idx; - } - return -EINVAL; -} - -/* Verify that the type id in the stream matches the type id of the keyset */ -static bool vcap_verify_keystream_keyset(struct vcap_control *vctrl, - enum vcap_type vt, - u32 *keystream, - u32 *mskstream, - enum vcap_keyfield_set keyset) -{ - const struct vcap_info *vcap = &vctrl->vcaps[vt]; - const struct vcap_field *typefld; - const struct vcap_typegroup *tgt; - const struct vcap_field *fields; - struct vcap_stream_iter iter; - const struct vcap_set *info; - u32 value = 0; - u32 mask = 0; - - if (vcap_keyfield_count(vctrl, vt, keyset) == 0) - return false; - - info = vcap_keyfieldset(vctrl, vt, keyset); - /* Check that the keyset is valid */ - if (!info) - return false; - - /* a type_id of value -1 means that there is no type field */ - if (info->type_id == (u8)-1) - return true; - - /* Get a valid typegroup for the specific keyset */ - tgt = vcap_keyfield_typegroup(vctrl, vt, keyset); - if (!tgt) - return false; - - fields = vcap_keyfields(vctrl, vt, keyset); - if (!fields) - return false; - - typefld = &fields[VCAP_KF_TYPE]; - vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset); - vcap_decode_field(mskstream, &iter, typefld->width, (u8 *)&mask); - /* no type info if there are no mask bits */ - if (vcap_bitarray_zero(typefld->width, (u8 *)&mask)) - return false; - - /* Get the value of the type field in the stream and compare to the - * one define in the vcap keyset - */ - vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset); - vcap_decode_field(keystream, &iter, typefld->width, (u8 *)&value); - - return (value & mask) == (info->type_id & mask); -} - -/* Verify that the typegroup information, subword count, keyset and type id - * are in sync and correct, return the list of matching keysets - */ -static int -vcap_find_keystream_keysets(struct vcap_control *vctrl, - enum vcap_type vt, - u32 *keystream, - u32 *mskstream, - bool mask, int sw_max, - struct vcap_keyset_list *kslist) -{ - const struct vcap_set *keyfield_set; - int sw_count, idx; - - sw_count = vcap_find_keystream_typegroup_sw(vctrl, vt, keystream, mask, - sw_max); - if (sw_count < 0) - return sw_count; - - keyfield_set = vctrl->vcaps[vt].keyfield_set; - for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) { - if (keyfield_set[idx].sw_per_item != sw_count) - continue; - - if (vcap_verify_keystream_keyset(vctrl, vt, keystream, - mskstream, idx)) - vcap_keyset_list_add(kslist, idx); - } - if (kslist->cnt > 0) - return 0; - return -EINVAL; -} - -/* Read key data from a VCAP address and discover if there is a rule keyset - * here - */ -static bool -vcap_verify_actionstream_actionset(struct vcap_control *vctrl, - enum vcap_type vt, - u32 *actionstream, - enum vcap_actionfield_set actionset) -{ - const struct vcap_typegroup *tgt; - const struct vcap_field *fields; - const struct vcap_set *info; - - if (vcap_actionfield_count(vctrl, vt, actionset) == 0) - return false; - - info = vcap_actionfieldset(vctrl, vt, actionset); - /* Check that the actionset is valid */ - if (!info) - return false; - - /* a type_id of value -1 means that there is no type field */ - if (info->type_id == (u8)-1) - return true; - - /* Get a valid typegroup for the specific actionset */ - tgt = vcap_actionfield_typegroup(vctrl, vt, actionset); - if (!tgt) - return false; - - fields = vcap_actionfields(vctrl, vt, actionset); - if (!fields) - return false; - - /* Later this will be expanded with a check of the type id */ - return true; -} - -/* Verify that the typegroup information, subword count, actionset and type id - * are in sync and correct, return the actionset - */ -static enum vcap_actionfield_set -vcap_find_actionstream_actionset(struct vcap_control *vctrl, - enum vcap_type vt, - u32 *stream, - int sw_max) -{ - const struct vcap_set *actionfield_set; - int sw_count, idx; - bool res; - - sw_count = vcap_find_actionstream_typegroup_sw(vctrl, vt, stream, - sw_max); - if (sw_count < 0) - return sw_count; - - actionfield_set = vctrl->vcaps[vt].actionfield_set; - for (idx = 0; idx < vctrl->vcaps[vt].actionfield_set_size; ++idx) { - if (actionfield_set[idx].sw_per_item != sw_count) - continue; - - res = vcap_verify_actionstream_actionset(vctrl, vt, - stream, idx); - if (res) - return idx; - } - return -EINVAL; -} - -/* Read key data from a VCAP address and discover if there are any rule keysets - * here - */ -static int vcap_addr_keysets(struct vcap_control *vctrl, - struct net_device *ndev, - struct vcap_admin *admin, - int addr, - struct vcap_keyset_list *kslist) -{ - enum vcap_type vt = admin->vtype; - int keyset_sw_regs, idx; - u32 key = 0, mask = 0; - - /* Read the cache at the specified address */ - keyset_sw_regs = DIV_ROUND_UP(vctrl->vcaps[vt].sw_width, 32); - vctrl->ops->update(ndev, admin, VCAP_CMD_READ, VCAP_SEL_ALL, addr); - vctrl->ops->cache_read(ndev, admin, VCAP_SEL_ENTRY, 0, - keyset_sw_regs); - /* Skip uninitialized key/mask entries */ - for (idx = 0; idx < keyset_sw_regs; ++idx) { - key |= ~admin->cache.keystream[idx]; - mask |= admin->cache.maskstream[idx]; - } - if (key == 0 && mask == 0) - return -EINVAL; - /* Decode and locate the keysets */ - return vcap_find_keystream_keysets(vctrl, vt, admin->cache.keystream, - admin->cache.maskstream, false, 0, - kslist); -} - -static int vcap_read_rule(struct vcap_rule_internal *ri) -{ - struct vcap_admin *admin = ri->admin; - int sw_idx, ent_idx = 0, act_idx = 0; - u32 addr = ri->addr; - - if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) { - pr_err("%s:%d: rule is empty\n", __func__, __LINE__); - return -EINVAL; - } - vcap_erase_cache(ri); - /* Use the values in the streams to read the VCAP cache */ - for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) { - ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ, - VCAP_SEL_ALL, addr); - ri->vctrl->ops->cache_read(ri->ndev, admin, - VCAP_SEL_ENTRY, ent_idx, - ri->keyset_sw_regs); - ri->vctrl->ops->cache_read(ri->ndev, admin, - VCAP_SEL_ACTION, act_idx, - ri->actionset_sw_regs); - if (sw_idx == 0) - ri->vctrl->ops->cache_read(ri->ndev, admin, - VCAP_SEL_COUNTER, - ri->counter_id, 0); - ent_idx += ri->keyset_sw_regs; - act_idx += ri->actionset_sw_regs; - } - return 0; -} - /* Dump the keyfields value and mask values */ static void vcap_debugfs_show_rule_keyfield(struct vcap_control *vctrl, struct vcap_output_print *out, enum vcap_key_field key, const struct vcap_field *keyfield, - u8 *value, u8 *mask) + struct vcap_client_keyfield_data *data) { bool hex = false; + u8 *value, *mask; int idx, bytes; out->prf(out->dst, " %s: W%d: ", vcap_keyfield_name(vctrl, key), @@ -374,40 +34,62 @@ static void vcap_debugfs_show_rule_keyfield(struct vcap_control *vctrl, switch (keyfield[key].type) { case VCAP_FIELD_BIT: - out->prf(out->dst, "%d/%d", value[0], mask[0]); + out->prf(out->dst, "%d/%d", data->u1.value, data->u1.mask); break; case VCAP_FIELD_U32: + value = (u8 *)(&data->u32.value); + mask = (u8 *)(&data->u32.mask); + if (key == VCAP_KF_L3_IP4_SIP || key == VCAP_KF_L3_IP4_DIP) { - out->prf(out->dst, "%pI4h/%pI4h", value, mask); + out->prf(out->dst, "%pI4h/%pI4h", &data->u32.value, + &data->u32.mask); } else if (key == VCAP_KF_ETYPE || key == VCAP_KF_IF_IGR_PORT_MASK) { hex = true; } else { u32 fmsk = (1 << keyfield[key].width) - 1; - u32 val = *(u32 *)value; - u32 msk = *(u32 *)mask; - out->prf(out->dst, "%u/%u", val & fmsk, msk & fmsk); + out->prf(out->dst, "%u/%u", data->u32.value & fmsk, + data->u32.mask & fmsk); } break; case VCAP_FIELD_U48: + value = data->u48.value; + mask = data->u48.mask; if (key == VCAP_KF_L2_SMAC || key == VCAP_KF_L2_DMAC) - out->prf(out->dst, "%pMR/%pMR", value, mask); + out->prf(out->dst, "%pMR/%pMR", data->u48.value, + data->u48.mask); else hex = true; break; case VCAP_FIELD_U56: + value = data->u56.value; + mask = data->u56.mask; + hex = true; + break; case VCAP_FIELD_U64: + value = data->u64.value; + mask = data->u64.mask; + hex = true; + break; case VCAP_FIELD_U72: + value = data->u72.value; + mask = data->u72.mask; + hex = true; + break; case VCAP_FIELD_U112: + value = data->u112.value; + mask = data->u112.mask; hex = true; break; case VCAP_FIELD_U128: if (key == VCAP_KF_L3_IP6_SIP || key == VCAP_KF_L3_IP6_DIP) { u8 nvalue[16], nmask[16]; - vcap_netbytes_copy(nvalue, value, sizeof(nvalue)); - vcap_netbytes_copy(nmask, mask, sizeof(nmask)); + vcap_netbytes_copy(nvalue, data->u128.value, + sizeof(nvalue)); + vcap_netbytes_copy(nmask, data->u128.mask, + sizeof(nmask)); out->prf(out->dst, "%pI6/%pI6", nvalue, nmask); } else { hex = true; @@ -472,19 +154,15 @@ static int vcap_debugfs_show_rule_keyset(struct vcap_rule_internal *ri, struct vcap_output_print *out) { struct vcap_control *vctrl = ri->vctrl; - struct vcap_stream_iter kiter, miter; struct vcap_admin *admin = ri->admin; enum vcap_keyfield_set keysets[10]; const struct vcap_field *keyfield; enum vcap_type vt = admin->vtype; - const struct vcap_typegroup *tgt; + struct vcap_client_keyfield *ckf; struct vcap_keyset_list matches; - enum vcap_keyfield_set keyset; - int idx, res, keyfield_count; u32 *maskstream; u32 *keystream; - u8 value[16]; - u8 mask[16]; + int res; keystream = admin->cache.keystream; maskstream = admin->cache.maskstream; @@ -498,39 +176,20 @@ static int vcap_debugfs_show_rule_keyset(struct vcap_rule_internal *ri, __func__, __LINE__, res); return -EINVAL; } - keyset = matches.keysets[0]; out->prf(out->dst, " keysets:"); - for (idx = 0; idx < matches.cnt; ++idx) + for (int idx = 0; idx < matches.cnt; ++idx) out->prf(out->dst, " %s", vcap_keyset_name(vctrl, matches.keysets[idx])); out->prf(out->dst, "\n"); out->prf(out->dst, " keyset_sw: %d\n", ri->keyset_sw); out->prf(out->dst, " keyset_sw_regs: %d\n", ri->keyset_sw_regs); - keyfield_count = vcap_keyfield_count(vctrl, vt, keyset); - keyfield = vcap_keyfields(vctrl, vt, keyset); - tgt = vcap_keyfield_typegroup(vctrl, vt, keyset); - /* Start decoding the streams */ - for (idx = 0; idx < keyfield_count; ++idx) { - if (keyfield[idx].width <= 0) - continue; - /* First get the mask */ - memset(mask, 0, DIV_ROUND_UP(keyfield[idx].width, 8)); - vcap_iter_init(&miter, vctrl->vcaps[vt].sw_width, tgt, - keyfield[idx].offset); - vcap_decode_field(maskstream, &miter, keyfield[idx].width, - mask); - /* Skip if no mask bits are set */ - if (vcap_bitarray_zero(keyfield[idx].width, mask)) - continue; - /* Get the key */ - memset(value, 0, DIV_ROUND_UP(keyfield[idx].width, 8)); - vcap_iter_init(&kiter, vctrl->vcaps[vt].sw_width, tgt, - keyfield[idx].offset); - vcap_decode_field(keystream, &kiter, keyfield[idx].width, - value); - vcap_debugfs_show_rule_keyfield(vctrl, out, idx, keyfield, - value, mask); + + list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) { + keyfield = vcap_keyfields(vctrl, admin->vtype, ri->data.keyset); + vcap_debugfs_show_rule_keyfield(vctrl, out, ckf->ctrl.key, + keyfield, &ckf->data); } + return 0; } @@ -540,48 +199,21 @@ static int vcap_debugfs_show_rule_actionset(struct vcap_rule_internal *ri, struct vcap_control *vctrl = ri->vctrl; struct vcap_admin *admin = ri->admin; const struct vcap_field *actionfield; - enum vcap_actionfield_set actionset; - enum vcap_type vt = admin->vtype; - const struct vcap_typegroup *tgt; - struct vcap_stream_iter iter; - int idx, res, actfield_count; - u32 *actstream; - u8 value[16]; - bool no_bits; - - actstream = admin->cache.actionstream; - res = vcap_find_actionstream_actionset(vctrl, vt, actstream, 0); - if (res < 0) { - pr_err("%s:%d: could not find valid actionset: %d\n", - __func__, __LINE__, res); - return -EINVAL; - } - actionset = res; + struct vcap_client_actionfield *caf; + out->prf(out->dst, " actionset: %s\n", vcap_actionset_name(vctrl, ri->data.actionset)); out->prf(out->dst, " actionset_sw: %d\n", ri->actionset_sw); out->prf(out->dst, " actionset_sw_regs: %d\n", ri->actionset_sw_regs); - actfield_count = vcap_actionfield_count(vctrl, vt, actionset); - actionfield = vcap_actionfields(vctrl, vt, actionset); - tgt = vcap_actionfield_typegroup(vctrl, vt, actionset); - /* Start decoding the stream */ - for (idx = 0; idx < actfield_count; ++idx) { - if (actionfield[idx].width <= 0) - continue; - /* Get the action */ - memset(value, 0, DIV_ROUND_UP(actionfield[idx].width, 8)); - vcap_iter_init(&iter, vctrl->vcaps[vt].act_width, tgt, - actionfield[idx].offset); - vcap_decode_field(actstream, &iter, actionfield[idx].width, - value); - /* Skip if no bits are set */ - no_bits = vcap_bitarray_zero(actionfield[idx].width, value); - if (no_bits) - continue; - /* Later the action id will also be checked */ - vcap_debugfs_show_rule_actionfield(vctrl, out, idx, actionfield, - value); + + list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) { + actionfield = vcap_actionfields(vctrl, admin->vtype, + ri->data.actionset); + vcap_debugfs_show_rule_actionfield(vctrl, out, caf->ctrl.action, + actionfield, + &caf->data.u1.value); } + return 0; } @@ -632,25 +264,22 @@ static int vcap_show_admin(struct vcap_control *vctrl, struct vcap_admin *admin, struct vcap_output_print *out) { - struct vcap_rule_internal *elem, *ri; + struct vcap_rule_internal *elem; + struct vcap_rule *vrule; int ret = 0; vcap_show_admin_info(vctrl, admin, out); - mutex_lock(&admin->lock); list_for_each_entry(elem, &admin->rules, list) { - ri = vcap_dup_rule(elem); - if (IS_ERR(ri)) - goto free_rule; - /* Read data from VCAP */ - ret = vcap_read_rule(ri); - if (ret) - goto free_rule; + vrule = vcap_get_rule(vctrl, elem->data.id); + if (IS_ERR_OR_NULL(vrule)) { + ret = PTR_ERR(vrule); + break; + } + out->prf(out->dst, "\n"); - vcap_show_admin_rule(vctrl, admin, out, ri); -free_rule: - vcap_free_rule((struct vcap_rule *)ri); + vcap_show_admin_rule(vctrl, admin, out, to_intrule(vrule)); + vcap_free_rule(vrule); } - mutex_unlock(&admin->lock); return ret; } diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_private.h b/drivers/net/ethernet/microchip/vcap/vcap_api_private.h index 9ac1b1d55f22..4fd21da97679 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_private.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_private.h @@ -96,4 +96,18 @@ const char *vcap_actionset_name(struct vcap_control *vctrl, const char *vcap_actionfield_name(struct vcap_control *vctrl, enum vcap_action_field action); +/* Read key data from a VCAP address and discover if there are any rule keysets + * here + */ +int vcap_addr_keysets(struct vcap_control *vctrl, struct net_device *ndev, + struct vcap_admin *admin, int addr, + struct vcap_keyset_list *kslist); + +/* Verify that the typegroup information, subword count, keyset and type id + * are in sync and correct, return the list of matchin keysets + */ +int vcap_find_keystream_keysets(struct vcap_control *vctrl, enum vcap_type vt, + u32 *keystream, u32 *mskstream, bool mask, + int sw_max, struct vcap_keyset_list *kslist); + #endif /* __VCAP_API_PRIVATE__ */ diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index ad1277ac7f0d..2f6a048dee90 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1363,10 +1363,11 @@ static void mana_poll_rx_cq(struct mana_cq *cq) xdp_do_flush(); } -static void mana_cq_handler(void *context, struct gdma_queue *gdma_queue) +static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) { struct mana_cq *cq = context; u8 arm_bit; + int w; WARN_ON_ONCE(cq->gdma_cq != gdma_queue); @@ -1375,26 +1376,31 @@ static void mana_cq_handler(void *context, struct gdma_queue *gdma_queue) else mana_poll_tx_cq(cq); - if (cq->work_done < cq->budget && - napi_complete_done(&cq->napi, cq->work_done)) { + w = cq->work_done; + + if (w < cq->budget && + napi_complete_done(&cq->napi, w)) { arm_bit = SET_ARM_BIT; } else { arm_bit = 0; } mana_gd_ring_cq(gdma_queue, arm_bit); + + return w; } static int mana_poll(struct napi_struct *napi, int budget) { struct mana_cq *cq = container_of(napi, struct mana_cq, napi); + int w; cq->work_done = 0; cq->budget = budget; - mana_cq_handler(cq, cq->gdma_cq); + w = mana_cq_handler(cq, cq->gdma_cq); - return min(cq->work_done, budget); + return min(w, budget); } static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 9063e2e22cd5..8073d7a90a26 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -552,8 +552,7 @@ myri10ge_validate_firmware(struct myri10ge_priv *mgp, } /* save firmware version for ethtool */ - strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); - mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; + strscpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, &mgp->fw_ver_minor, &mgp->fw_ver_tiny); diff --git a/drivers/net/ethernet/netronome/nfp/ccm_mbox.c b/drivers/net/ethernet/netronome/nfp/ccm_mbox.c index 4247bca09807..aa8aba4ff7aa 100644 --- a/drivers/net/ethernet/netronome/nfp/ccm_mbox.c +++ b/drivers/net/ethernet/netronome/nfp/ccm_mbox.c @@ -503,7 +503,7 @@ nfp_ccm_mbox_msg_prepare(struct nfp_net *nn, struct sk_buff *skb, max_len = max(max_reply_size, round_up(skb->len, 4)); if (max_len > mbox_max) { nn_dp_warn(&nn->dp, - "message too big for tha mailbox: %u/%u vs %u\n", + "message too big for the mailbox: %u/%u vs %u\n", skb->len, max_reply_size, mbox_max); return -EMSGSIZE; } diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c index 3728870d8e9c..4632268695cb 100644 --- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c +++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c @@ -302,6 +302,11 @@ static int nfp_net_xfrm_add_state(struct xfrm_state *x) return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + nn_err(nn, "Unsupported xfrm offload tyoe\n"); + return -EINVAL; + } + cfg->spi = ntohl(x->id.spi); /* Hash/Authentication */ diff --git a/drivers/net/ethernet/netronome/nfp/nfdk/dp.c b/drivers/net/ethernet/netronome/nfp/nfdk/dp.c index 2b427d8ccb2f..ccacb6ab6c39 100644 --- a/drivers/net/ethernet/netronome/nfp/nfdk/dp.c +++ b/drivers/net/ethernet/netronome/nfp/nfdk/dp.c @@ -282,7 +282,7 @@ netdev_tx_t nfp_nfdk_tx(struct sk_buff *skb, struct net_device *netdev) dma_len = skb_headlen(skb); if (skb_is_gso(skb)) type = NFDK_DESC_TX_TYPE_TSO; - else if (!nr_frags && dma_len < NFDK_TX_MAX_DATA_PER_HEAD) + else if (!nr_frags && dma_len <= NFDK_TX_MAX_DATA_PER_HEAD) type = NFDK_DESC_TX_TYPE_SIMPLE; else type = NFDK_DESC_TX_TYPE_GATHER; @@ -927,7 +927,7 @@ nfp_nfdk_tx_xdp_buf(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring, dma_len = pkt_len; dma_addr = rxbuf->dma_addr + dma_off; - if (dma_len < NFDK_TX_MAX_DATA_PER_HEAD) + if (dma_len <= NFDK_TX_MAX_DATA_PER_HEAD) type = NFDK_DESC_TX_TYPE_SIMPLE; else type = NFDK_DESC_TX_TYPE_GATHER; @@ -1325,7 +1325,7 @@ nfp_nfdk_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec, txbuf = &tx_ring->ktxbufs[wr_idx]; dma_len = skb_headlen(skb); - if (dma_len < NFDK_TX_MAX_DATA_PER_HEAD) + if (dma_len <= NFDK_TX_MAX_DATA_PER_HEAD) type = NFDK_DESC_TX_TYPE_SIMPLE; else type = NFDK_DESC_TX_TYPE_GATHER; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c index 784f23602a8a..bf6bae557158 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c @@ -239,10 +239,6 @@ nfp_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, char *buf = NULL; int err; - err = devlink_info_driver_name_put(req, "nfp"); - if (err) - return err; - vendor = nfp_hwinfo_lookup(pf->hwinfo, "assembly.vendor"); part = nfp_hwinfo_lookup(pf->hwinfo, "assembly.partno"); sn = nfp_hwinfo_lookup(pf->hwinfo, "assembly.serial"); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h index 6c83e47d8b3d..da33f09facb9 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h @@ -88,6 +88,9 @@ #define NFP_NET_FL_BATCH 16 /* Add freelist in this Batch size */ #define NFP_NET_XDP_MAX_COMPLETE 2048 /* XDP bufs to reclaim in NAPI poll */ +/* MC definitions */ +#define NFP_NET_CFG_MAC_MC_MAX 1024 /* The maximum number of MC address per port*/ + /* Offload definitions */ #define NFP_NET_N_VXLAN_PORTS (NFP_NET_CFG_VXLAN_SZ / sizeof(__be16)) @@ -476,6 +479,7 @@ struct nfp_stat_pair { * @rx_dma_off: Offset at which DMA packets (for XDP headroom) * @rx_offset: Offset in the RX buffers where packet data starts * @ctrl: Local copy of the control register/word. + * @ctrl_w1: Local copy of the control register/word1. * @fl_bufsz: Currently configured size of the freelist buffers * @xdp_prog: Installed XDP program * @tx_rings: Array of pre-allocated TX ring structures @@ -508,6 +512,7 @@ struct nfp_net_dp { u32 rx_dma_off; u32 ctrl; + u32 ctrl_w1; u32 fl_bufsz; struct bpf_prog *xdp_prog; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 682a9198fb54..2314cf55e821 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -1007,6 +1007,7 @@ static int nfp_net_set_config_and_enable(struct nfp_net *nn) new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); + nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, nn->dp.ctrl_w1); err = nfp_net_reconfig(nn, update); if (err) { nfp_net_clear_config_and_disable(nn); @@ -1333,18 +1334,59 @@ err_unlock: return err; } +static int nfp_net_mc_cfg(struct net_device *netdev, const unsigned char *addr, const u32 cmd) +{ + struct nfp_net *nn = netdev_priv(netdev); + int ret; + + ret = nfp_net_mbox_lock(nn, NFP_NET_CFG_MULTICAST_SZ); + if (ret) + return ret; + + nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_HI, + get_unaligned_be32(addr)); + nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_LO, + get_unaligned_be16(addr + 4)); + + return nfp_net_mbox_reconfig_and_unlock(nn, cmd); +} + +static int nfp_net_mc_sync(struct net_device *netdev, const unsigned char *addr) +{ + struct nfp_net *nn = netdev_priv(netdev); + + if (netdev_mc_count(netdev) > NFP_NET_CFG_MAC_MC_MAX) { + nn_err(nn, "Requested number of MC addresses (%d) exceeds maximum (%d).\n", + netdev_mc_count(netdev), NFP_NET_CFG_MAC_MC_MAX); + return -EINVAL; + } + + return nfp_net_mc_cfg(netdev, addr, NFP_NET_CFG_MBOX_CMD_MULTICAST_ADD); +} + +static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr) +{ + return nfp_net_mc_cfg(netdev, addr, NFP_NET_CFG_MBOX_CMD_MULTICAST_DEL); +} + static void nfp_net_set_rx_mode(struct net_device *netdev) { struct nfp_net *nn = netdev_priv(netdev); - u32 new_ctrl; + u32 new_ctrl, new_ctrl_w1; new_ctrl = nn->dp.ctrl; + new_ctrl_w1 = nn->dp.ctrl_w1; if (!netdev_mc_empty(netdev) || netdev->flags & IFF_ALLMULTI) new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_L2MC; else new_ctrl &= ~NFP_NET_CFG_CTRL_L2MC; + if (netdev->flags & IFF_ALLMULTI) + new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_MCAST_FILTER; + else + new_ctrl_w1 |= nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER; + if (netdev->flags & IFF_PROMISC) { if (nn->cap & NFP_NET_CFG_CTRL_PROMISC) new_ctrl |= NFP_NET_CFG_CTRL_PROMISC; @@ -1354,13 +1396,21 @@ static void nfp_net_set_rx_mode(struct net_device *netdev) new_ctrl &= ~NFP_NET_CFG_CTRL_PROMISC; } - if (new_ctrl == nn->dp.ctrl) + if ((nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER) && + __dev_mc_sync(netdev, nfp_net_mc_sync, nfp_net_mc_unsync)) + netdev_err(netdev, "Sync mc address failed\n"); + + if (new_ctrl == nn->dp.ctrl && new_ctrl_w1 == nn->dp.ctrl_w1) return; - nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); + if (new_ctrl != nn->dp.ctrl) + nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl); + if (new_ctrl_w1 != nn->dp.ctrl_w1) + nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1); nfp_net_reconfig_post(nn, NFP_NET_CFG_UPDATE_GEN); nn->dp.ctrl = new_ctrl; + nn->dp.ctrl_w1 = new_ctrl_w1; } static void nfp_net_rss_init_itbl(struct nfp_net *nn) @@ -2092,7 +2142,7 @@ void nfp_net_info(struct nfp_net *nn) nn->fw_ver.extend, nn->fw_ver.class, nn->fw_ver.major, nn->fw_ver.minor, nn->max_mtu); - nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", + nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", nn->cap, nn->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", nn->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", @@ -2120,6 +2170,7 @@ void nfp_net_info(struct nfp_net *nn) nn->cap & NFP_NET_CFG_CTRL_CSUM_COMPLETE ? "RXCSUM_COMPLETE " : "", nn->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "", + nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER ? "MULTICAST_FILTER " : "", nfp_app_extra_cap(nn->app, nn)); } @@ -2548,6 +2599,9 @@ int nfp_net_init(struct nfp_net *nn) if (nn->cap & NFP_NET_CFG_CTRL_TXRWB) nn->dp.ctrl |= NFP_NET_CFG_CTRL_TXRWB; + if (nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER) + nn->dp.ctrl_w1 |= NFP_NET_CFG_CTRL_MCAST_FILTER; + /* Stash the re-configuration queue away. First odd queue in TX Bar */ nn->qcp_cfg = nn->tx_bar + NFP_QCP_QUEUE_ADDR_SZ; @@ -2555,6 +2609,7 @@ int nfp_net_init(struct nfp_net *nn) nn_writel(nn, NFP_NET_CFG_CTRL, 0); nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0); nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0); + nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, 0); err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_RING | NFP_NET_CFG_UPDATE_GEN); if (err) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h index cc11b3dc1252..51124309ae1f 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h @@ -267,6 +267,7 @@ #define NFP_NET_CFG_CTRL_WORD1 0x0098 #define NFP_NET_CFG_CTRL_PKT_TYPE (0x1 << 0) /* Pkttype offload */ #define NFP_NET_CFG_CTRL_IPSEC (0x1 << 1) /* IPsec offload */ +#define NFP_NET_CFG_CTRL_MCAST_FILTER (0x1 << 2) /* Multicast Filter */ #define NFP_NET_CFG_CAP_WORD1 0x00a4 @@ -413,6 +414,9 @@ #define NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET 5 #define NFP_NET_CFG_MBOX_CMD_TLV_CMSG 6 +#define NFP_NET_CFG_MBOX_CMD_MULTICAST_ADD 8 +#define NFP_NET_CFG_MBOX_CMD_MULTICAST_DEL 9 + /* VLAN filtering using general use mailbox * %NFP_NET_CFG_VLAN_FILTER: Base address of VLAN filter mailbox * %NFP_NET_CFG_VLAN_FILTER_VID: VLAN ID to filter @@ -424,6 +428,17 @@ #define NFP_NET_CFG_VLAN_FILTER_PROTO (NFP_NET_CFG_VLAN_FILTER + 2) #define NFP_NET_CFG_VLAN_FILTER_SZ 0x0004 +/* Multicast filtering using general use mailbox + * %NFP_NET_CFG_MULTICAST: Base address of Multicast filter mailbox + * %NFP_NET_CFG_MULTICAST_MAC_HI: High 32-bits of Multicast MAC address + * %NFP_NET_CFG_MULTICAST_MAC_LO: Low 16-bits of Multicast MAC address + * %NFP_NET_CFG_MULTICAST_SZ: Size of the Multicast filter mailbox in bytes + */ +#define NFP_NET_CFG_MULTICAST NFP_NET_CFG_MBOX_SIMPLE_VAL +#define NFP_NET_CFG_MULTICAST_MAC_HI NFP_NET_CFG_MULTICAST +#define NFP_NET_CFG_MULTICAST_MAC_LO (NFP_NET_CFG_MULTICAST + 6) +#define NFP_NET_CFG_MULTICAST_SZ 0x0006 + /* TLV capabilities * %NFP_NET_CFG_TLV_TYPE: Offset of type within the TLV * %NFP_NET_CFG_TLV_TYPE_REQUIRED: Driver must be able to parse the TLV diff --git a/drivers/net/ethernet/pensando/ionic/ionic_devlink.c b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c index 567f778433e2..e6ff757895ab 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_devlink.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c @@ -26,10 +26,6 @@ static int ionic_dl_info_get(struct devlink *dl, struct devlink_info_req *req, char buf[16]; int err = 0; - err = devlink_info_driver_name_put(req, IONIC_DRV_NAME); - if (err) - return err; - err = devlink_info_version_running_put(req, DEVLINK_INFO_VERSION_GENERIC_FW, idev->dev_info.fw_version); diff --git a/drivers/net/ethernet/qlogic/qed/qed_devlink.c b/drivers/net/ethernet/qlogic/qed/qed_devlink.c index 6bb4e165b592..922c47797af6 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_devlink.c +++ b/drivers/net/ethernet/qlogic/qed/qed_devlink.c @@ -162,10 +162,6 @@ static int qed_devlink_info_get(struct devlink *devlink, dev_info = &cdev->common_dev_info; - err = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (err) - return err; - memcpy(buf, cdev->hwfns[0].hw_info.part_num, sizeof(cdev->hwfns[0].hw_info.part_num)); buf[sizeof(cdev->hwfns[0].hw_info.part_num)] = 0; diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c index ed274f033626..e5116a86cfbc 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c @@ -200,7 +200,7 @@ static void qed_ll2b_complete_rx_packet(void *cxt, dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr, cdev->ll2->rx_size, DMA_FROM_DEVICE); - skb = build_skb(buffer->data, 0); + skb = slab_build_skb(buffer->data); if (!skb) { DP_INFO(cdev, "Failed to build SKB\n"); kfree(buffer->data); diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 5bc1181f829b..a9dcc98b6af1 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -4559,12 +4559,13 @@ static int rtl8169_poll(struct napi_struct *napi, int budget) static void r8169_phylink_handler(struct net_device *ndev) { struct rtl8169_private *tp = netdev_priv(ndev); + struct device *d = tp_to_dev(tp); if (netif_carrier_ok(ndev)) { rtl_link_chg_patch(tp); - pm_request_resume(&tp->pci_dev->dev); + pm_request_resume(d); } else { - pm_runtime_idle(&tp->pci_dev->dev); + pm_runtime_idle(d); } phy_print_status(tp->phydev); @@ -5282,6 +5283,8 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->hw_features |= NETIF_F_RXALL; dev->hw_features |= NETIF_F_RXFCS; + netdev_sw_irq_coalesce_default_on(dev); + /* configure chip for default features */ rtl8169_set_features(dev, dev->features); diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 6bc923326268..33f723a9f471 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -841,7 +841,7 @@ static bool ravb_rx_gbeth(struct net_device *ndev, int *quota, int q) napi_gro_receive(&priv->napi[q], priv->rx_1st_skb); stats->rx_packets++; - stats->rx_bytes += priv->rx_1st_skb->len; + stats->rx_bytes += pkt_len; break; } } diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index c2224e41a694..cc30524c2fe4 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -1164,7 +1164,7 @@ static ssize_t mcdi_logging_show(struct device *dev, struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); - return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); + return sysfs_emit(buf, "%d\n", mcdi->logging_enabled); } static ssize_t mcdi_logging_store(struct device *dev, diff --git a/drivers/net/ethernet/sfc/siena/efx_common.c b/drivers/net/ethernet/sfc/siena/efx_common.c index 1fd396b00bfb..e4b294b8e9ac 100644 --- a/drivers/net/ethernet/sfc/siena/efx_common.c +++ b/drivers/net/ethernet/sfc/siena/efx_common.c @@ -1178,7 +1178,7 @@ static ssize_t mcdi_logging_show(struct device *dev, struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); - return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); + return sysfs_emit(buf, "%d\n", mcdi->logging_enabled); } static ssize_t mcdi_logging_store(struct device *dev, diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 31ff35174034..f77511fe4e87 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -235,6 +235,15 @@ config DWMAC_INTEL_PLAT the stmmac device driver. This driver is used for the Intel Keem Bay SoC. +config DWMAC_TEGRA + tristate "NVIDIA Tegra MGBE support" + depends on ARCH_TEGRA || COMPILE_TEST + help + This selects the Multi-GigaBit Ethernet (MGBE) Controller that is + found on the NVIDIA Tegra SoC devices. This driver provides the glue + layer on top of the stmmac driver required for these NVIDIA Tegra SoC + devices. + config DWMAC_VISCONTI tristate "Toshiba Visconti DWMAC support" default ARCH_VISCONTI diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index d4e12e9ace4f..057e4bab5c08 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_DWMAC_DWC_QOS_ETH) += dwmac-dwc-qos-eth.o obj-$(CONFIG_DWMAC_INTEL_PLAT) += dwmac-intel-plat.o obj-$(CONFIG_DWMAC_GENERIC) += dwmac-generic.o obj-$(CONFIG_DWMAC_IMX8) += dwmac-imx.o +obj-$(CONFIG_DWMAC_TEGRA) += dwmac-tegra.o obj-$(CONFIG_DWMAC_VISCONTI) += dwmac-visconti.o stmmac-platform-objs:= stmmac_platform.o dwmac-altr-socfpga-objs := altr_tse_pcs.o dwmac-socfpga.o diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c new file mode 100644 index 000000000000..bdf990cf2f31 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/platform_device.h> +#include <linux/of_device.h> +#include <linux/module.h> +#include <linux/stmmac.h> +#include <linux/clk.h> + +#include "stmmac_platform.h" + +static const char *const mgbe_clks[] = { + "rx-pcs", "tx", "tx-pcs", "mac-divider", "mac", "mgbe", "ptp-ref", "mac" +}; + +struct tegra_mgbe { + struct device *dev; + + struct clk_bulk_data *clks; + + struct reset_control *rst_mac; + struct reset_control *rst_pcs; + + void __iomem *hv; + void __iomem *regs; + void __iomem *xpcs; + + struct mii_bus *mii; +}; + +#define XPCS_WRAP_UPHY_RX_CONTROL 0x801c +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_SW_OVRD BIT(31) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_PCS_PHY_RDY BIT(10) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_CDR_RESET BIT(9) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_CAL_EN BIT(8) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_SLEEP (BIT(7) | BIT(6)) +#define XPCS_WRAP_UPHY_RX_CONTROL_AUX_RX_IDDQ BIT(5) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_IDDQ BIT(4) +#define XPCS_WRAP_UPHY_RX_CONTROL_RX_DATA_EN BIT(0) +#define XPCS_WRAP_UPHY_HW_INIT_CTRL 0x8020 +#define XPCS_WRAP_UPHY_HW_INIT_CTRL_TX_EN BIT(0) +#define XPCS_WRAP_UPHY_HW_INIT_CTRL_RX_EN BIT(2) +#define XPCS_WRAP_UPHY_STATUS 0x8044 +#define XPCS_WRAP_UPHY_STATUS_TX_P_UP BIT(0) +#define XPCS_WRAP_IRQ_STATUS 0x8050 +#define XPCS_WRAP_IRQ_STATUS_PCS_LINK_STS BIT(6) + +#define XPCS_REG_ADDR_SHIFT 10 +#define XPCS_REG_ADDR_MASK 0x1fff +#define XPCS_ADDR 0x3fc + +#define MGBE_WRAP_COMMON_INTR_ENABLE 0x8704 +#define MAC_SBD_INTR BIT(2) +#define MGBE_WRAP_AXI_ASID0_CTRL 0x8400 +#define MGBE_SID 0x6 + +static int __maybe_unused tegra_mgbe_suspend(struct device *dev) +{ + struct tegra_mgbe *mgbe = get_stmmac_bsp_priv(dev); + int err; + + err = stmmac_suspend(dev); + if (err) + return err; + + clk_bulk_disable_unprepare(ARRAY_SIZE(mgbe_clks), mgbe->clks); + + return reset_control_assert(mgbe->rst_mac); +} + +static int __maybe_unused tegra_mgbe_resume(struct device *dev) +{ + struct tegra_mgbe *mgbe = get_stmmac_bsp_priv(dev); + u32 value; + int err; + + err = clk_bulk_prepare_enable(ARRAY_SIZE(mgbe_clks), mgbe->clks); + if (err < 0) + return err; + + err = reset_control_deassert(mgbe->rst_mac); + if (err < 0) + return err; + + /* Enable common interrupt at wrapper level */ + writel(MAC_SBD_INTR, mgbe->regs + MGBE_WRAP_COMMON_INTR_ENABLE); + + /* Program SID */ + writel(MGBE_SID, mgbe->hv + MGBE_WRAP_AXI_ASID0_CTRL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_STATUS); + if ((value & XPCS_WRAP_UPHY_STATUS_TX_P_UP) == 0) { + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL); + value |= XPCS_WRAP_UPHY_HW_INIT_CTRL_TX_EN; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL); + } + + err = readl_poll_timeout(mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL, value, + (value & XPCS_WRAP_UPHY_HW_INIT_CTRL_TX_EN) == 0, + 500, 500 * 2000); + if (err < 0) { + dev_err(mgbe->dev, "timeout waiting for TX lane to become enabled\n"); + clk_bulk_disable_unprepare(ARRAY_SIZE(mgbe_clks), mgbe->clks); + return err; + } + + err = stmmac_resume(dev); + if (err < 0) + clk_bulk_disable_unprepare(ARRAY_SIZE(mgbe_clks), mgbe->clks); + + return err; +} + +static int mgbe_uphy_lane_bringup_serdes_up(struct net_device *ndev, void *mgbe_data) +{ + struct tegra_mgbe *mgbe = (struct tegra_mgbe *)mgbe_data; + u32 value; + int err; + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_SW_OVRD; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value &= ~XPCS_WRAP_UPHY_RX_CONTROL_RX_IDDQ; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value &= ~XPCS_WRAP_UPHY_RX_CONTROL_AUX_RX_IDDQ; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value &= ~XPCS_WRAP_UPHY_RX_CONTROL_RX_SLEEP; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_CAL_EN; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + err = readl_poll_timeout(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL, value, + (value & XPCS_WRAP_UPHY_RX_CONTROL_RX_CAL_EN) == 0, + 1000, 1000 * 2000); + if (err < 0) { + dev_err(mgbe->dev, "timeout waiting for RX calibration to become enabled\n"); + return err; + } + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_DATA_EN; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_CDR_RESET; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value &= ~XPCS_WRAP_UPHY_RX_CONTROL_RX_CDR_RESET; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_PCS_PHY_RDY; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + err = readl_poll_timeout(mgbe->xpcs + XPCS_WRAP_IRQ_STATUS, value, + value & XPCS_WRAP_IRQ_STATUS_PCS_LINK_STS, + 500, 500 * 2000); + if (err < 0) { + dev_err(mgbe->dev, "timeout waiting for link to become ready\n"); + return err; + } + + /* clear status */ + writel(value, mgbe->xpcs + XPCS_WRAP_IRQ_STATUS); + + return 0; +} + +static void mgbe_uphy_lane_bringup_serdes_down(struct net_device *ndev, void *mgbe_data) +{ + struct tegra_mgbe *mgbe = (struct tegra_mgbe *)mgbe_data; + u32 value; + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_SW_OVRD; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value &= ~XPCS_WRAP_UPHY_RX_CONTROL_RX_DATA_EN; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_SLEEP; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_AUX_RX_IDDQ; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); + value |= XPCS_WRAP_UPHY_RX_CONTROL_RX_IDDQ; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_RX_CONTROL); +} + +static int tegra_mgbe_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat; + struct stmmac_resources res; + struct tegra_mgbe *mgbe; + int irq, err, i; + u32 value; + + mgbe = devm_kzalloc(&pdev->dev, sizeof(*mgbe), GFP_KERNEL); + if (!mgbe) + return -ENOMEM; + + mgbe->dev = &pdev->dev; + + memset(&res, 0, sizeof(res)); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + mgbe->hv = devm_platform_ioremap_resource_byname(pdev, "hypervisor"); + if (IS_ERR(mgbe->hv)) + return PTR_ERR(mgbe->hv); + + mgbe->regs = devm_platform_ioremap_resource_byname(pdev, "mac"); + if (IS_ERR(mgbe->regs)) + return PTR_ERR(mgbe->regs); + + mgbe->xpcs = devm_platform_ioremap_resource_byname(pdev, "xpcs"); + if (IS_ERR(mgbe->xpcs)) + return PTR_ERR(mgbe->xpcs); + + res.addr = mgbe->regs; + res.irq = irq; + + mgbe->clks = devm_kzalloc(&pdev->dev, sizeof(*mgbe->clks), GFP_KERNEL); + if (!mgbe->clks) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(mgbe_clks); i++) + mgbe->clks[i].id = mgbe_clks[i]; + + err = devm_clk_bulk_get(mgbe->dev, ARRAY_SIZE(mgbe_clks), mgbe->clks); + if (err < 0) + return err; + + err = clk_bulk_prepare_enable(ARRAY_SIZE(mgbe_clks), mgbe->clks); + if (err < 0) + return err; + + /* Perform MAC reset */ + mgbe->rst_mac = devm_reset_control_get(&pdev->dev, "mac"); + if (IS_ERR(mgbe->rst_mac)) { + err = PTR_ERR(mgbe->rst_mac); + goto disable_clks; + } + + err = reset_control_assert(mgbe->rst_mac); + if (err < 0) + goto disable_clks; + + usleep_range(2000, 4000); + + err = reset_control_deassert(mgbe->rst_mac); + if (err < 0) + goto disable_clks; + + /* Perform PCS reset */ + mgbe->rst_pcs = devm_reset_control_get(&pdev->dev, "pcs"); + if (IS_ERR(mgbe->rst_pcs)) { + err = PTR_ERR(mgbe->rst_pcs); + goto disable_clks; + } + + err = reset_control_assert(mgbe->rst_pcs); + if (err < 0) + goto disable_clks; + + usleep_range(2000, 4000); + + err = reset_control_deassert(mgbe->rst_pcs); + if (err < 0) + goto disable_clks; + + plat = stmmac_probe_config_dt(pdev, res.mac); + if (IS_ERR(plat)) { + err = PTR_ERR(plat); + goto disable_clks; + } + + plat->has_xgmac = 1; + plat->tso_en = 1; + plat->pmt = 1; + plat->bsp_priv = mgbe; + + if (!plat->mdio_node) + plat->mdio_node = of_get_child_by_name(pdev->dev.of_node, "mdio"); + + if (!plat->mdio_bus_data) { + plat->mdio_bus_data = devm_kzalloc(&pdev->dev, sizeof(*plat->mdio_bus_data), + GFP_KERNEL); + if (!plat->mdio_bus_data) { + err = -ENOMEM; + goto remove; + } + } + + plat->mdio_bus_data->needs_reset = true; + + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_STATUS); + if ((value & XPCS_WRAP_UPHY_STATUS_TX_P_UP) == 0) { + value = readl(mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL); + value |= XPCS_WRAP_UPHY_HW_INIT_CTRL_TX_EN; + writel(value, mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL); + } + + err = readl_poll_timeout(mgbe->xpcs + XPCS_WRAP_UPHY_HW_INIT_CTRL, value, + (value & XPCS_WRAP_UPHY_HW_INIT_CTRL_TX_EN) == 0, + 500, 500 * 2000); + if (err < 0) { + dev_err(mgbe->dev, "timeout waiting for TX lane to become enabled\n"); + goto remove; + } + + plat->serdes_powerup = mgbe_uphy_lane_bringup_serdes_up; + plat->serdes_powerdown = mgbe_uphy_lane_bringup_serdes_down; + + /* Tx FIFO Size - 128KB */ + plat->tx_fifo_size = 131072; + /* Rx FIFO Size - 192KB */ + plat->rx_fifo_size = 196608; + + /* Enable common interrupt at wrapper level */ + writel(MAC_SBD_INTR, mgbe->regs + MGBE_WRAP_COMMON_INTR_ENABLE); + + /* Program SID */ + writel(MGBE_SID, mgbe->hv + MGBE_WRAP_AXI_ASID0_CTRL); + + plat->serdes_up_after_phy_linkup = 1; + + err = stmmac_dvr_probe(&pdev->dev, plat, &res); + if (err < 0) + goto remove; + + return 0; + +remove: + stmmac_remove_config_dt(pdev, plat); +disable_clks: + clk_bulk_disable_unprepare(ARRAY_SIZE(mgbe_clks), mgbe->clks); + + return err; +} + +static int tegra_mgbe_remove(struct platform_device *pdev) +{ + struct tegra_mgbe *mgbe = get_stmmac_bsp_priv(&pdev->dev); + + clk_bulk_disable_unprepare(ARRAY_SIZE(mgbe_clks), mgbe->clks); + + stmmac_pltfr_remove(pdev); + + return 0; +} + +static const struct of_device_id tegra_mgbe_match[] = { + { .compatible = "nvidia,tegra234-mgbe", }, + { } +}; +MODULE_DEVICE_TABLE(of, tegra_mgbe_match); + +static SIMPLE_DEV_PM_OPS(tegra_mgbe_pm_ops, tegra_mgbe_suspend, tegra_mgbe_resume); + +static struct platform_driver tegra_mgbe_driver = { + .probe = tegra_mgbe_probe, + .remove = tegra_mgbe_remove, + .driver = { + .name = "tegra-mgbe", + .pm = &tegra_mgbe_pm_ops, + .of_match_table = tegra_mgbe_match, + }, +}; +module_platform_driver(tegra_mgbe_driver); + +MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra MGBE driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 18c7ca29da2c..f36590d0c830 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -988,6 +988,9 @@ static void stmmac_mac_link_up(struct phylink_config *config, struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); u32 old_ctrl, ctrl; + if (priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) + priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); + old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); ctrl = old_ctrl & ~priv->hw->link.speed_mask; @@ -3809,7 +3812,7 @@ static int __stmmac_open(struct net_device *dev, stmmac_reset_queues_param(priv); - if (priv->plat->serdes_powerup) { + if (!priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) { ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); if (ret < 0) { netdev_err(priv->dev, "%s: Serdes powerup failed\n", @@ -7518,7 +7521,7 @@ int stmmac_resume(struct device *dev) stmmac_mdio_reset(priv->mii); } - if (priv->plat->serdes_powerup) { + if (!priv->plat->serdes_up_after_phy_linkup && priv->plat->serdes_powerup) { ret = priv->plat->serdes_powerup(ndev, priv->plat->bsp_priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 50f6b4a14be4..eb6d9cd8e93f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -108,10 +108,10 @@ static struct stmmac_axi *stmmac_axi_setup(struct platform_device *pdev) axi->axi_lpi_en = of_property_read_bool(np, "snps,lpi_en"); axi->axi_xit_frm = of_property_read_bool(np, "snps,xit_frm"); - axi->axi_kbbe = of_property_read_bool(np, "snps,axi_kbbe"); - axi->axi_fb = of_property_read_bool(np, "snps,axi_fb"); - axi->axi_mb = of_property_read_bool(np, "snps,axi_mb"); - axi->axi_rb = of_property_read_bool(np, "snps,axi_rb"); + axi->axi_kbbe = of_property_read_bool(np, "snps,kbbe"); + axi->axi_fb = of_property_read_bool(np, "snps,fb"); + axi->axi_mb = of_property_read_bool(np, "snps,mb"); + axi->axi_rb = of_property_read_bool(np, "snps,rb"); if (of_property_read_u32(np, "snps,wr_osr_lmt", &axi->axi_wr_osr_lmt)) axi->axi_wr_osr_lmt = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 773e415cc2de..2cfb18cef1d4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -926,6 +926,9 @@ static int tc_setup_taprio(struct stmmac_priv *priv, int i, ret = 0; u64 ctr; + if (qopt->base_time < 0) + return -ERANGE; + if (!priv->dma_cap.estsel) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 51c37e99d086..de112ab3195c 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -133,11 +133,6 @@ NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \ NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) -static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common); -static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common); -static void am65_cpsw_nuss_free_tx_chns(struct am65_cpsw_common *common); -static void am65_cpsw_nuss_free_rx_chns(struct am65_cpsw_common *common); - static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave, const u8 *dev_addr) { @@ -379,20 +374,6 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) if (common->usage_count) return 0; - /* init tx/rx channels */ - ret = am65_cpsw_nuss_init_tx_chns(common); - if (ret) { - dev_err(common->dev, "init_tx_chns failed\n"); - return ret; - } - - ret = am65_cpsw_nuss_init_rx_chns(common); - if (ret) { - dev_err(common->dev, "init_rx_chns failed\n"); - am65_cpsw_nuss_free_tx_chns(common); - return ret; - } - /* Control register */ writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE | AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD, @@ -421,7 +402,6 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) /* disable priority elevation */ writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE); - cpsw_ale_control_set(common->ale, 0, ALE_CLEAR, 1); cpsw_ale_start(common->ale); /* limit to one RX flow only */ @@ -453,8 +433,7 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) GFP_KERNEL); if (!skb) { dev_err(common->dev, "cannot allocate skb\n"); - ret = -ENOMEM; - goto err; + return -ENOMEM; } ret = am65_cpsw_nuss_rx_push(common, skb); @@ -463,7 +442,7 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) "cannot submit skb to channel rx, error %d\n", ret); kfree_skb(skb); - goto err; + return ret; } kmemleak_not_leak(skb); } @@ -472,7 +451,7 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) for (i = 0; i < common->tx_ch_num; i++) { ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn); if (ret) - goto err; + return ret; napi_enable(&common->tx_chns[i].napi_tx); } @@ -484,12 +463,6 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) dev_dbg(common->dev, "cpsw_nuss started\n"); return 0; - -err: - am65_cpsw_nuss_free_tx_chns(common); - am65_cpsw_nuss_free_rx_chns(common); - - return ret; } static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma); @@ -543,9 +516,6 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) writel(0, common->cpsw_base + AM65_CPSW_REG_CTL); writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); - am65_cpsw_nuss_free_tx_chns(common); - am65_cpsw_nuss_free_rx_chns(common); - dev_dbg(common->dev, "cpsw_nuss stopped\n"); return 0; } @@ -587,7 +557,6 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) struct am65_cpsw_port *port = am65_ndev_to_port(ndev); int ret, i; u32 reg; - int tmo; ret = pm_runtime_resume_and_get(common->dev); if (ret < 0) @@ -595,19 +564,17 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) /* Idle MAC port */ cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); - - tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); - dev_info(common->dev, "down msc_sl %08x tmo %d\n", - cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo); - + cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); cpsw_sl_ctl_reset(port->slave.mac_sl); /* soft reset MAC */ cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1); mdelay(1); reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET); - if (reg) - dev_info(common->dev, "mac reset not yet done\n"); + if (reg) { + dev_err(common->dev, "soft RESET didn't complete\n"); + return -ETIMEDOUT; + } /* Notify the stack of the actual queue counts. */ ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num); @@ -1495,7 +1462,7 @@ static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy if (speed == SPEED_1000) mac_control |= CPSW_SL_CTL_GIG; - if (speed == SPEED_10 && interface == PHY_INTERFACE_MODE_RGMII) + if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface)) /* Can be used with in band mode only */ mac_control |= CPSW_SL_CTL_EXT_EN; if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII) @@ -1539,9 +1506,9 @@ static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port) cpsw_sl_ctl_reset(port->slave.mac_sl); } -static void am65_cpsw_nuss_free_tx_chns(struct am65_cpsw_common *common) +static void am65_cpsw_nuss_free_tx_chns(void *data) { - struct device *dev = common->dev; + struct am65_cpsw_common *common = data; int i; for (i = 0; i < common->tx_ch_num; i++) { @@ -1553,11 +1520,7 @@ static void am65_cpsw_nuss_free_tx_chns(struct am65_cpsw_common *common) if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) k3_udma_glue_release_tx_chn(tx_chn->tx_chn); - /* Don't clear tx_chn memory as we need to preserve - * data between suspend/resume - */ - if (!(tx_chn->irq < 0)) - devm_free_irq(dev, tx_chn->irq, tx_chn); + memset(tx_chn, 0, sizeof(*tx_chn)); } } @@ -1566,10 +1529,12 @@ void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common) struct device *dev = common->dev; int i; + devm_remove_action(dev, am65_cpsw_nuss_free_tx_chns, common); + for (i = 0; i < common->tx_ch_num; i++) { struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; - if (!(tx_chn->irq < 0)) + if (tx_chn->irq) devm_free_irq(dev, tx_chn->irq, tx_chn); netif_napi_del(&tx_chn->napi_tx); @@ -1584,6 +1549,32 @@ void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common) } } +static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common) +{ + struct device *dev = common->dev; + int i, ret = 0; + + for (i = 0; i < common->tx_ch_num; i++) { + struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; + + netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx, + am65_cpsw_nuss_tx_poll); + + ret = devm_request_irq(dev, tx_chn->irq, + am65_cpsw_nuss_tx_irq, + IRQF_TRIGGER_HIGH, + tx_chn->tx_chn_name, tx_chn); + if (ret) { + dev_err(dev, "failure requesting tx%u irq %u, %d\n", + tx_chn->id, tx_chn->irq, ret); + goto err; + } + } + +err: + return ret; +} + static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) { u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS); @@ -1639,7 +1630,7 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) } tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); - if (tx_chn->irq < 0) { + if (tx_chn->irq <= 0) { dev_err(dev, "Failed to get tx dma irq %d\n", tx_chn->irq); goto err; @@ -1648,41 +1639,59 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) snprintf(tx_chn->tx_chn_name, sizeof(tx_chn->tx_chn_name), "%s-tx%d", dev_name(dev), tx_chn->id); - - ret = devm_request_irq(dev, tx_chn->irq, - am65_cpsw_nuss_tx_irq, - IRQF_TRIGGER_HIGH, - tx_chn->tx_chn_name, tx_chn); - if (ret) { - dev_err(dev, "failure requesting tx%u irq %u, %d\n", - tx_chn->id, tx_chn->irq, ret); - tx_chn->irq = -EINVAL; - goto err; - } } - return 0; + ret = am65_cpsw_nuss_ndev_add_tx_napi(common); + if (ret) { + dev_err(dev, "Failed to add tx NAPI %d\n", ret); + goto err; + } err: - am65_cpsw_nuss_free_tx_chns(common); + i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common); + if (i) { + dev_err(dev, "Failed to add free_tx_chns action %d\n", i); + return i; + } return ret; } -static void am65_cpsw_nuss_free_rx_chns(struct am65_cpsw_common *common) +static void am65_cpsw_nuss_free_rx_chns(void *data) { + struct am65_cpsw_common *common = data; struct am65_cpsw_rx_chn *rx_chn; rx_chn = &common->rx_chns; + if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) + k3_cppi_desc_pool_destroy(rx_chn->desc_pool); + + if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) + k3_udma_glue_release_rx_chn(rx_chn->rx_chn); +} + +static void am65_cpsw_nuss_remove_rx_chns(void *data) +{ + struct am65_cpsw_common *common = data; + struct am65_cpsw_rx_chn *rx_chn; + struct device *dev = common->dev; + + rx_chn = &common->rx_chns; + devm_remove_action(dev, am65_cpsw_nuss_free_rx_chns, common); + if (!(rx_chn->irq < 0)) - devm_free_irq(common->dev, rx_chn->irq, common); + devm_free_irq(dev, rx_chn->irq, common); + + netif_napi_del(&common->napi_rx); if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) k3_cppi_desc_pool_destroy(rx_chn->desc_pool); if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) k3_udma_glue_release_rx_chn(rx_chn->rx_chn); + + common->rx_flow_id_base = -1; } static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) @@ -1700,7 +1709,7 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE; rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS; - rx_cfg.flow_id_base = -1; + rx_cfg.flow_id_base = common->rx_flow_id_base; /* init all flows */ rx_chn->dev = dev; @@ -1772,20 +1781,24 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) } } + netif_napi_add(common->dma_ndev, &common->napi_rx, + am65_cpsw_nuss_rx_poll); + ret = devm_request_irq(dev, rx_chn->irq, am65_cpsw_nuss_rx_irq, IRQF_TRIGGER_HIGH, dev_name(dev), common); if (ret) { dev_err(dev, "failure requesting rx irq %u, %d\n", rx_chn->irq, ret); - rx_chn->irq = -EINVAL; goto err; } - return 0; - err: - am65_cpsw_nuss_free_rx_chns(common); + i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common); + if (i) { + dev_err(dev, "Failed to add free_rx_chns action %d\n", i); + return i; + } return ret; } @@ -2105,26 +2118,9 @@ static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common) return ret; } - netif_napi_add(common->dma_ndev, &common->napi_rx, - am65_cpsw_nuss_rx_poll); - return ret; } -static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common) -{ - int i; - - for (i = 0; i < common->tx_ch_num; i++) { - struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; - - netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx, - am65_cpsw_nuss_tx_poll); - } - - return 0; -} - static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common) { struct am65_cpsw_port *port; @@ -2587,7 +2583,11 @@ static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common) struct am65_cpsw_port *port; int ret = 0, i; - ret = am65_cpsw_nuss_ndev_add_tx_napi(common); + /* init tx channels */ + ret = am65_cpsw_nuss_init_tx_chns(common); + if (ret) + return ret; + ret = am65_cpsw_nuss_init_rx_chns(common); if (ret) return ret; @@ -2634,10 +2634,8 @@ int am65_cpsw_nuss_update_tx_chns(struct am65_cpsw_common *common, int num_tx) common->tx_ch_num = num_tx; ret = am65_cpsw_nuss_init_tx_chns(common); - if (ret) - return ret; - return am65_cpsw_nuss_ndev_add_tx_napi(common); + return ret; } struct am65_cpsw_soc_pdata { @@ -2745,6 +2743,7 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev) if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS) return -ENOENT; + common->rx_flow_id_base = -1; init_completion(&common->tdown_complete); common->tx_ch_num = 1; common->pf_p0_rx_ptype_rrobin = false; @@ -2878,10 +2877,10 @@ static int am65_cpsw_nuss_remove(struct platform_device *pdev) static int am65_cpsw_nuss_suspend(struct device *dev) { struct am65_cpsw_common *common = dev_get_drvdata(dev); + struct am65_cpsw_host *host_p = am65_common_get_host(common); struct am65_cpsw_port *port; struct net_device *ndev; int i, ret; - struct am65_cpsw_host *host_p = am65_common_get_host(common); cpsw_ale_dump(common->ale, common->ale_context); host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); @@ -2907,6 +2906,9 @@ static int am65_cpsw_nuss_suspend(struct device *dev) am65_cpts_suspend(common->cpts); + am65_cpsw_nuss_remove_rx_chns(common); + am65_cpsw_nuss_remove_tx_chns(common); + return 0; } @@ -2918,6 +2920,17 @@ static int am65_cpsw_nuss_resume(struct device *dev) int i, ret; struct am65_cpsw_host *host_p = am65_common_get_host(common); + ret = am65_cpsw_nuss_init_tx_chns(common); + if (ret) + return ret; + ret = am65_cpsw_nuss_init_rx_chns(common); + if (ret) + return ret; + + /* If RX IRQ was disabled before suspend, keep it disabled */ + if (common->rx_irq_disabled) + disable_irq(common->rx_chns.irq); + am65_cpts_resume(common->cpts); for (i = 0; i < common->port_num; i++) { diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index 450b16ad40a4..e1a569b99e4a 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -885,7 +885,7 @@ static int ca8210_spi_transfer( dev_dbg(&spi->dev, "%s called\n", __func__); - cas_ctl = kmalloc(sizeof(*cas_ctl), GFP_ATOMIC); + cas_ctl = kzalloc(sizeof(*cas_ctl), GFP_ATOMIC); if (!cas_ctl) return -ENOMEM; diff --git a/drivers/net/ieee802154/cc2520.c b/drivers/net/ieee802154/cc2520.c index c69b87d3837d..edc769daad07 100644 --- a/drivers/net/ieee802154/cc2520.c +++ b/drivers/net/ieee802154/cc2520.c @@ -970,7 +970,7 @@ static int cc2520_hw_init(struct cc2520_private *priv) if (timeout-- <= 0) { dev_err(&priv->spi->dev, "oscillator start failed!\n"); - return ret; + return -ETIMEDOUT; } udelay(1); } while (!(status & CC2520_STATUS_XOSC32M_STABLE)); diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c index 5cbc15a971f9..14bd2f903045 100644 --- a/drivers/net/ipa/ipa_sysfs.c +++ b/drivers/net/ipa/ipa_sysfs.c @@ -46,7 +46,7 @@ version_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ipa *ipa = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%s\n", ipa_version_string(ipa)); + return sysfs_emit(buf, "%s\n", ipa_version_string(ipa)); } static DEVICE_ATTR_RO(version); @@ -70,7 +70,7 @@ static ssize_t rx_offload_show(struct device *dev, { struct ipa *ipa = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%s\n", ipa_offload_string(ipa)); + return sysfs_emit(buf, "%s\n", ipa_offload_string(ipa)); } static DEVICE_ATTR_RO(rx_offload); @@ -80,7 +80,7 @@ static ssize_t tx_offload_show(struct device *dev, { struct ipa *ipa = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%s\n", ipa_offload_string(ipa)); + return sysfs_emit(buf, "%s\n", ipa_offload_string(ipa)); } static DEVICE_ATTR_RO(tx_offload); diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index d73b9d535b7a..937f5b1f04ff 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -3698,6 +3698,7 @@ static const struct nla_policy macsec_rtnl_policy[IFLA_MACSEC_MAX + 1] = { [IFLA_MACSEC_SCB] = { .type = NLA_U8 }, [IFLA_MACSEC_REPLAY_PROTECT] = { .type = NLA_U8 }, [IFLA_MACSEC_VALIDATION] = { .type = NLA_U8 }, + [IFLA_MACSEC_OFFLOAD] = { .type = NLA_U8 }, }; static void macsec_free_netdev(struct net_device *dev) diff --git a/drivers/net/mdio/fwnode_mdio.c b/drivers/net/mdio/fwnode_mdio.c index eb344f6d4a7b..b782c35c4ac1 100644 --- a/drivers/net/mdio/fwnode_mdio.c +++ b/drivers/net/mdio/fwnode_mdio.c @@ -98,6 +98,7 @@ int fwnode_mdiobus_phy_device_register(struct mii_bus *mdio, */ rc = phy_device_register(phy); if (rc) { + device_set_node(&phy->mdio.dev, NULL); fwnode_handle_put(child); return rc; } @@ -153,7 +154,8 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus, /* All data is now stored in the phy struct, so register it */ rc = phy_device_register(phy); if (rc) { - fwnode_handle_put(phy->mdio.dev.fwnode); + phy->mdio.dev.fwnode = NULL; + fwnode_handle_put(child); goto clean_phy; } } else if (is_of_node(child)) { diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 796e9c7857d0..510822d6d0d9 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -68,8 +68,9 @@ static int of_mdiobus_register_device(struct mii_bus *mdio, /* All data is now stored in the mdiodev struct; register it. */ rc = mdio_device_register(mdiodev); if (rc) { + device_set_node(&mdiodev->dev, NULL); + fwnode_handle_put(fwnode); mdio_device_free(mdiodev); - of_node_put(child); return rc; } diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index e14686594a71..b962fc8e1397 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -994,9 +994,6 @@ static int nsim_dev_info_get(struct devlink *devlink, { int err; - err = devlink_info_driver_name_put(req, DRV_NAME); - if (err) - return err; err = devlink_info_version_stored_put_ext(req, "fw.mgmt", "10.20.30", DEVLINK_INFO_VERSION_TYPE_COMPONENT); if (err) diff --git a/drivers/net/netdevsim/ipsec.c b/drivers/net/netdevsim/ipsec.c index 386336a38f34..b93baf5c8bee 100644 --- a/drivers/net/netdevsim/ipsec.c +++ b/drivers/net/netdevsim/ipsec.c @@ -149,6 +149,11 @@ static int nsim_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + /* find the first unused index */ ret = nsim_ipsec_find_empty_idx(ipsec); if (ret < 0) { diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig index af00cf44cd97..1327290decab 100644 --- a/drivers/net/phy/Kconfig +++ b/drivers/net/phy/Kconfig @@ -47,7 +47,6 @@ config LED_TRIGGER_PHY config FIXED_PHY tristate "MDIO Bus/PHY emulation with fixed speed/link PHYs" - depends on PHYLIB select SWPHY help Adds the platform "fixed" MDIO Bus to cover the boards that use @@ -112,7 +111,6 @@ config BROADCOM_PHY config BCM54140_PHY tristate "Broadcom BCM54140 PHY" - depends on PHYLIB depends on HWMON || HWMON=n select BCM_NET_PHYLIB help @@ -137,7 +135,6 @@ config BCM7XXX_PHY config BCM84881_PHY tristate "Broadcom BCM84881 PHY" - depends on PHYLIB help Support the Broadcom BCM84881 PHY. diff --git a/drivers/net/phy/mdio_device.c b/drivers/net/phy/mdio_device.c index 250742ffdfd9..044828d081d2 100644 --- a/drivers/net/phy/mdio_device.c +++ b/drivers/net/phy/mdio_device.c @@ -21,6 +21,7 @@ #include <linux/slab.h> #include <linux/string.h> #include <linux/unistd.h> +#include <linux/property.h> void mdio_device_free(struct mdio_device *mdiodev) { @@ -30,6 +31,7 @@ EXPORT_SYMBOL(mdio_device_free); static void mdio_device_release(struct device *dev) { + fwnode_handle_put(dev->fwnode); kfree(to_mdio_device(dev)); } diff --git a/drivers/net/phy/mxl-gpy.c b/drivers/net/phy/mxl-gpy.c index 27c0f161623e..147d7a5a9b35 100644 --- a/drivers/net/phy/mxl-gpy.c +++ b/drivers/net/phy/mxl-gpy.c @@ -9,6 +9,7 @@ #include <linux/module.h> #include <linux/bitfield.h> #include <linux/hwmon.h> +#include <linux/mutex.h> #include <linux/phy.h> #include <linux/polynomial.h> #include <linux/netdevice.h> @@ -78,8 +79,16 @@ VSPEC1_SGMII_CTRL_ANRS) /* Temperature sensor */ -#define VPSPEC1_TEMP_STA 0x0E -#define VPSPEC1_TEMP_STA_DATA GENMASK(9, 0) +#define VSPEC1_TEMP_STA 0x0E +#define VSPEC1_TEMP_STA_DATA GENMASK(9, 0) + +/* Mailbox */ +#define VSPEC1_MBOX_DATA 0x5 +#define VSPEC1_MBOX_ADDRLO 0x6 +#define VSPEC1_MBOX_CMD 0x7 +#define VSPEC1_MBOX_CMD_ADDRHI GENMASK(7, 0) +#define VSPEC1_MBOX_CMD_RD (0 << 8) +#define VSPEC1_MBOX_CMD_READY BIT(15) /* WoL */ #define VPSPEC2_WOL_CTL 0x0E06 @@ -88,7 +97,13 @@ #define VPSPEC2_WOL_AD45 0x0E0A #define WOL_EN BIT(0) +/* Internal registers, access via mbox */ +#define REG_GPIO0_OUT 0xd3ce00 + struct gpy_priv { + /* serialize mailbox acesses */ + struct mutex mbox_lock; + u8 fw_major; u8 fw_minor; }; @@ -140,14 +155,14 @@ static int gpy_hwmon_read(struct device *dev, struct phy_device *phydev = dev_get_drvdata(dev); int ret; - ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, VPSPEC1_TEMP_STA); + ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, VSPEC1_TEMP_STA); if (ret < 0) return ret; if (!ret) return -ENODATA; *value = polynomial_calc(&poly_N_to_temp, - FIELD_GET(VPSPEC1_TEMP_STA_DATA, ret)); + FIELD_GET(VSPEC1_TEMP_STA_DATA, ret)); return 0; } @@ -198,6 +213,45 @@ static int gpy_hwmon_register(struct phy_device *phydev) } #endif +static int gpy_mbox_read(struct phy_device *phydev, u32 addr) +{ + struct gpy_priv *priv = phydev->priv; + int val, ret; + u16 cmd; + + mutex_lock(&priv->mbox_lock); + + ret = phy_write_mmd(phydev, MDIO_MMD_VEND1, VSPEC1_MBOX_ADDRLO, + addr); + if (ret) + goto out; + + cmd = VSPEC1_MBOX_CMD_RD; + cmd |= FIELD_PREP(VSPEC1_MBOX_CMD_ADDRHI, addr >> 16); + + ret = phy_write_mmd(phydev, MDIO_MMD_VEND1, VSPEC1_MBOX_CMD, cmd); + if (ret) + goto out; + + /* The mbox read is used in the interrupt workaround. It was observed + * that a read might take up to 2.5ms. This is also the time for which + * the interrupt line is stuck low. To be on the safe side, poll the + * ready bit for 10ms. + */ + ret = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1, + VSPEC1_MBOX_CMD, val, + (val & VSPEC1_MBOX_CMD_READY), + 500, 10000, false); + if (ret) + goto out; + + ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, VSPEC1_MBOX_DATA); + +out: + mutex_unlock(&priv->mbox_lock); + return ret; +} + static int gpy_config_init(struct phy_device *phydev) { int ret; @@ -212,6 +266,13 @@ static int gpy_config_init(struct phy_device *phydev) return ret < 0 ? ret : 0; } +static bool gpy_has_broken_mdint(struct phy_device *phydev) +{ + /* At least these PHYs are known to have broken interrupt handling */ + return phydev->drv->phy_id == PHY_ID_GPY215B || + phydev->drv->phy_id == PHY_ID_GPY215C; +} + static int gpy_probe(struct phy_device *phydev) { struct device *dev = &phydev->mdio.dev; @@ -229,6 +290,7 @@ static int gpy_probe(struct phy_device *phydev) if (!priv) return -ENOMEM; phydev->priv = priv; + mutex_init(&priv->mbox_lock); fw_version = phy_read(phydev, PHY_FWV); if (fw_version < 0) @@ -574,6 +636,29 @@ static irqreturn_t gpy_handle_interrupt(struct phy_device *phydev) if (!(reg & PHY_IMASK_MASK)) return IRQ_NONE; + /* The PHY might leave the interrupt line asserted even after PHY_ISTAT + * is read. To avoid interrupt storms, delay the interrupt handling as + * long as the PHY drives the interrupt line. An internal bus read will + * stall as long as the interrupt line is asserted, thus just read a + * random register here. + * Because we cannot access the internal bus at all while the interrupt + * is driven by the PHY, there is no way to make the interrupt line + * unstuck (e.g. by changing the pinmux to GPIO input) during that time + * frame. Therefore, polling is the best we can do and won't do any more + * harm. + * It was observed that this bug happens on link state and link speed + * changes on a GPY215B and GYP215C independent of the firmware version + * (which doesn't mean that this list is exhaustive). + */ + if (gpy_has_broken_mdint(phydev) && + (reg & (PHY_IMASK_LSTC | PHY_IMASK_LSPC))) { + reg = gpy_mbox_read(phydev, REG_GPIO0_OUT); + if (reg < 0) { + phy_error(phydev); + return IRQ_NONE; + } + } + phy_trigger_machine(phydev); return IRQ_HANDLED; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 8cff61dbc4b5..716870a4499c 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1512,6 +1512,15 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, phy_resume(phydev); phy_led_triggers_register(phydev); + /** + * If the external phy used by current mac interface is managed by + * another mac interface, so we should create a device link between + * phy dev and mac dev. + */ + if (phydev->mdio.bus->parent && dev->dev.parent != phydev->mdio.bus->parent) + phydev->devlink = device_link_add(dev->dev.parent, &phydev->mdio.dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); + return err; error: @@ -1750,6 +1759,9 @@ void phy_detach(struct phy_device *phydev) struct module *ndev_owner = NULL; struct mii_bus *bus; + if (phydev->devlink) + device_link_del(phydev->devlink); + if (phydev->sysfs_links) { if (dev) sysfs_remove_link(&dev->dev.kobj, "phydev"); diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 39fd1811375c..83b99d95b278 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -2642,10 +2642,46 @@ static void sfp_cleanup(void *data) kfree(sfp); } +static int sfp_i2c_get(struct sfp *sfp) +{ + struct acpi_handle *acpi_handle; + struct fwnode_handle *h; + struct i2c_adapter *i2c; + struct device_node *np; + int err; + + h = fwnode_find_reference(dev_fwnode(sfp->dev), "i2c-bus", 0); + if (IS_ERR(h)) { + dev_err(sfp->dev, "missing 'i2c-bus' property\n"); + return -ENODEV; + } + + if (is_acpi_device_node(h)) { + acpi_handle = ACPI_HANDLE_FWNODE(h); + i2c = i2c_acpi_find_adapter_by_handle(acpi_handle); + } else if ((np = to_of_node(h)) != NULL) { + i2c = of_find_i2c_adapter_by_node(np); + } else { + err = -EINVAL; + goto put; + } + + if (!i2c) { + err = -EPROBE_DEFER; + goto put; + } + + err = sfp_i2c_configure(sfp, i2c); + if (err) + i2c_put_adapter(i2c); +put: + fwnode_handle_put(h); + return err; +} + static int sfp_probe(struct platform_device *pdev) { const struct sff_data *sff; - struct i2c_adapter *i2c; char *sfp_irq_name; struct sfp *sfp; int err, i; @@ -2663,51 +2699,20 @@ static int sfp_probe(struct platform_device *pdev) sff = sfp->type = &sfp_data; if (pdev->dev.of_node) { - struct device_node *node = pdev->dev.of_node; const struct of_device_id *id; - struct device_node *np; - id = of_match_node(sfp_of_match, node); + id = of_match_node(sfp_of_match, pdev->dev.of_node); if (WARN_ON(!id)) return -EINVAL; sff = sfp->type = id->data; - - np = of_parse_phandle(node, "i2c-bus", 0); - if (!np) { - dev_err(sfp->dev, "missing 'i2c-bus' property\n"); - return -ENODEV; - } - - i2c = of_find_i2c_adapter_by_node(np); - of_node_put(np); - } else if (has_acpi_companion(&pdev->dev)) { - struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); - struct fwnode_handle *fw = acpi_fwnode_handle(adev); - struct fwnode_reference_args args; - struct acpi_handle *acpi_handle; - int ret; - - ret = acpi_node_get_property_reference(fw, "i2c-bus", 0, &args); - if (ret || !is_acpi_device_node(args.fwnode)) { - dev_err(&pdev->dev, "missing 'i2c-bus' property\n"); - return -ENODEV; - } - - acpi_handle = ACPI_HANDLE_FWNODE(args.fwnode); - i2c = i2c_acpi_find_adapter_by_handle(acpi_handle); - } else { + } else if (!has_acpi_companion(&pdev->dev)) { return -EINVAL; } - if (!i2c) - return -EPROBE_DEFER; - - err = sfp_i2c_configure(sfp, i2c); - if (err < 0) { - i2c_put_adapter(i2c); + err = sfp_i2c_get(sfp); + if (err) return err; - } for (i = 0; i < GPIO_MAX; i++) if (sff->gpios & BIT(i)) { diff --git a/drivers/net/plip/plip.c b/drivers/net/plip/plip.c index c8791e9b451d..40ce8abe6999 100644 --- a/drivers/net/plip/plip.c +++ b/drivers/net/plip/plip.c @@ -450,12 +450,12 @@ plip_bh_timeout_error(struct net_device *dev, struct net_local *nl, } rcv->state = PLIP_PK_DONE; if (rcv->skb) { - kfree_skb(rcv->skb); + dev_kfree_skb_irq(rcv->skb); rcv->skb = NULL; } snd->state = PLIP_PK_DONE; if (snd->skb) { - dev_kfree_skb(snd->skb); + dev_consume_skb_irq(snd->skb); snd->skb = NULL; } spin_unlock_irq(&nl->lock); diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 9e75ed3f08ce..a2be1994b389 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -957,6 +957,10 @@ static int set_offload(struct tap_queue *q, unsigned long arg) if (arg & TUN_F_TSO6) feature_mask |= NETIF_F_TSO6; } + + /* TODO: for now USO4 and USO6 should work simultaneously */ + if ((arg & (TUN_F_USO4 | TUN_F_USO6)) == (TUN_F_USO4 | TUN_F_USO6)) + features |= NETIF_F_GSO_UDP_L4; } /* tun/tap driver inverts the usage for TSO offloads, where @@ -967,7 +971,8 @@ static int set_offload(struct tap_queue *q, unsigned long arg) * When user space turns off TSO, we turn off GSO/LRO so that * user-space will not receive TSO frames. */ - if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6)) + if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6) || + (feature_mask & (TUN_F_USO4 | TUN_F_USO6)) == (TUN_F_USO4 | TUN_F_USO6)) features |= RX_OFFLOADS; else features &= ~RX_OFFLOADS; @@ -1091,7 +1096,8 @@ static long tap_ioctl(struct file *file, unsigned int cmd, case TUNSETOFFLOAD: /* let the user check for future flags */ if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | - TUN_F_TSO_ECN | TUN_F_UFO)) + TUN_F_TSO_ECN | TUN_F_UFO | + TUN_F_USO4 | TUN_F_USO6)) return -EINVAL; rtnl_lock(); diff --git a/drivers/net/thunderbolt.c b/drivers/net/thunderbolt.c index b20cd370b7f2..990484776f2d 100644 --- a/drivers/net/thunderbolt.c +++ b/drivers/net/thunderbolt.c @@ -58,10 +58,10 @@ * supported then @frame_id is filled, otherwise it stays %0. */ struct thunderbolt_ip_frame_header { - u32 frame_size; - u16 frame_index; - u16 frame_id; - u32 frame_count; + __le32 frame_size; + __le16 frame_index; + __le16 frame_id; + __le32 frame_count; }; enum thunderbolt_ip_frame_pdf { @@ -914,6 +914,7 @@ static int tbnet_open(struct net_device *dev) eof_mask, tbnet_start_poll, net); if (!ring) { netdev_err(dev, "failed to allocate Rx ring\n"); + tb_xdomain_release_out_hopid(xd, hopid); tb_ring_free(net->tx_ring.ring); net->tx_ring.ring = NULL; return -ENOMEM; @@ -1319,7 +1320,7 @@ static void tbnet_shutdown(struct tb_service *svc) tbnet_tear_down(tb_service_get_drvdata(svc), true); } -static int __maybe_unused tbnet_suspend(struct device *dev) +static int tbnet_suspend(struct device *dev) { struct tb_service *svc = tb_to_service(dev); struct tbnet *net = tb_service_get_drvdata(svc); @@ -1334,7 +1335,7 @@ static int __maybe_unused tbnet_suspend(struct device *dev) return 0; } -static int __maybe_unused tbnet_resume(struct device *dev) +static int tbnet_resume(struct device *dev) { struct tb_service *svc = tb_to_service(dev); struct tbnet *net = tb_service_get_drvdata(svc); @@ -1350,9 +1351,7 @@ static int __maybe_unused tbnet_resume(struct device *dev) return 0; } -static const struct dev_pm_ops tbnet_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(tbnet_suspend, tbnet_resume) -}; +static DEFINE_SIMPLE_DEV_PM_OPS(tbnet_pm_ops, tbnet_suspend, tbnet_resume); static const struct tb_service_id tbnet_ids[] = { { TB_SERVICE("network", 1) }, @@ -1364,7 +1363,7 @@ static struct tb_service_driver tbnet_driver = { .driver = { .owner = THIS_MODULE, .name = "thunderbolt-net", - .pm = &tbnet_pm_ops, + .pm = pm_sleep_ptr(&tbnet_pm_ops), }, .probe = tbnet_probe, .remove = tbnet_remove, diff --git a/drivers/net/tun.c b/drivers/net/tun.c index b4baa2001a63..a7d17c680f4a 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -185,7 +185,7 @@ struct tun_struct { struct net_device *dev; netdev_features_t set_features; #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \ - NETIF_F_TSO6) + NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4) int align; int vnet_hdr_sz; @@ -2878,6 +2878,12 @@ static int set_offload(struct tun_struct *tun, unsigned long arg) } arg &= ~TUN_F_UFO; + + /* TODO: for now USO4 and USO6 should work simultaneously */ + if (arg & TUN_F_USO4 && arg & TUN_F_USO6) { + features |= NETIF_F_GSO_UDP_L4; + arg &= ~(TUN_F_USO4 | TUN_F_USO6); + } } /* This gives the user a way to test for new features in future by diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c index 0fe3773c5bca..743cbf5d662c 100644 --- a/drivers/net/usb/asix_devices.c +++ b/drivers/net/usb/asix_devices.c @@ -1350,6 +1350,20 @@ static const struct driver_info ax88772b_info = { .data = FLAG_EEPROM_MAC, }; +static const struct driver_info lxausb_t1l_info = { + .description = "Linux Automation GmbH USB 10Base-T1L", + .bind = ax88772_bind, + .unbind = ax88772_unbind, + .status = asix_status, + .reset = ax88772_reset, + .stop = ax88772_stop, + .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | + FLAG_MULTI_PACKET, + .rx_fixup = asix_rx_fixup_common, + .tx_fixup = asix_tx_fixup, + .data = FLAG_EEPROM_MAC, +}; + static const struct driver_info ax88178_info = { .description = "ASIX AX88178 USB 2.0 Ethernet", .bind = ax88178_bind, @@ -1538,6 +1552,10 @@ static const struct usb_device_id products [] = { */ USB_DEVICE(0x066b, 0x20f9), .driver_info = (unsigned long) &hg20f9_info, +}, { + // Linux Automation GmbH USB 10Base-T1L + USB_DEVICE(0x33f7, 0x0004), + .driver_info = (unsigned long) &lxausb_t1l_info, }, { }, // END }; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 19eee0655b99..7723b2a49d8e 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -60,13 +60,17 @@ static const unsigned long guest_offloads[] = { VIRTIO_NET_F_GUEST_TSO6, VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, - VIRTIO_NET_F_GUEST_CSUM + VIRTIO_NET_F_GUEST_CSUM, + VIRTIO_NET_F_GUEST_USO4, + VIRTIO_NET_F_GUEST_USO6 }; #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ - (1ULL << VIRTIO_NET_F_GUEST_UFO)) + (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ + (1ULL << VIRTIO_NET_F_GUEST_USO4) | \ + (1ULL << VIRTIO_NET_F_GUEST_USO6)) struct virtnet_stat_desc { char desc[ETH_GSTRING_LEN]; @@ -3085,7 +3089,9 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || - virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) { + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); return -EOPNOTSUPP; } @@ -3690,7 +3696,9 @@ static bool virtnet_check_guest_gso(const struct virtnet_info *vi) return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || - virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO); + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || + (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); } static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) @@ -3759,6 +3767,8 @@ static int virtnet_probe(struct virtio_device *vdev) dev->hw_features |= NETIF_F_TSO6; if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) dev->hw_features |= NETIF_F_TSO_ECN; + if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO)) + dev->hw_features |= NETIF_F_GSO_UDP_L4; dev->features |= NETIF_F_GSO_ROBUST; @@ -4036,6 +4046,7 @@ static struct virtio_device_id id_table[] = { VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ + VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \ VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index d3e7b27eb933..6f1e560fb15c 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -75,8 +75,14 @@ vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) for (i = 0; i < adapter->intr.num_intrs; i++) vmxnet3_enable_intr(adapter, i); - adapter->shared->devRead.intrConf.intrCtrl &= + if (!VMXNET3_VERSION_GE_6(adapter) || + !adapter->queuesExtEnabled) { + adapter->shared->devRead.intrConf.intrCtrl &= + cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); + } else { + adapter->shared->devReadExt.intrConfExt.intrCtrl &= cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); + } } @@ -85,8 +91,14 @@ vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) { int i; - adapter->shared->devRead.intrConf.intrCtrl |= + if (!VMXNET3_VERSION_GE_6(adapter) || + !adapter->queuesExtEnabled) { + adapter->shared->devRead.intrConf.intrCtrl |= + cpu_to_le32(VMXNET3_IC_DISABLE_ALL); + } else { + adapter->shared->devReadExt.intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); + } for (i = 0; i < adapter->intr.num_intrs; i++) vmxnet3_disable_intr(adapter, i); } @@ -1396,6 +1408,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, }; u32 num_pkts = 0; bool skip_page_frags = false; + bool encap_lro = false; struct Vmxnet3_RxCompDesc *rcd; struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; u16 segCnt = 0, mss = 0; @@ -1556,13 +1569,18 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, if (VMXNET3_VERSION_GE_2(adapter) && rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { struct Vmxnet3_RxCompDescExt *rcdlro; + union Vmxnet3_GenericDesc *gdesc; + rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; + gdesc = (union Vmxnet3_GenericDesc *)rcd; segCnt = rcdlro->segCnt; WARN_ON_ONCE(segCnt == 0); mss = rcdlro->mss; if (unlikely(segCnt <= 1)) segCnt = 0; + encap_lro = (le32_to_cpu(gdesc->dword[0]) & + (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); } else { segCnt = 0; } @@ -1630,7 +1648,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, vmxnet3_rx_csum(adapter, skb, (union Vmxnet3_GenericDesc *)rcd); skb->protocol = eth_type_trans(skb, adapter->netdev); - if (!rcd->tcp || + if ((!rcd->tcp && !encap_lro) || !(adapter->netdev->features & NETIF_F_LRO)) goto not_lro; @@ -1639,7 +1657,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, SKB_GSO_TCPV4 : SKB_GSO_TCPV6; skb_shinfo(skb)->gso_size = mss; skb_shinfo(skb)->gso_segs = segCnt; - } else if (segCnt != 0 || skb->len > mtu) { + } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { u32 hlen; hlen = vmxnet3_get_hdr_len(adapter, skb, @@ -1668,6 +1686,7 @@ not_lro: napi_gro_receive(&rq->napi, skb); ctx->skb = NULL; + encap_lro = false; num_pkts++; } diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig index ca007b800f75..e6ea884cafc1 100644 --- a/drivers/net/wireless/ath/ath10k/Kconfig +++ b/drivers/net/wireless/ath/ath10k/Kconfig @@ -44,6 +44,7 @@ config ATH10K_SNOC tristate "Qualcomm ath10k SNOC support" depends on ATH10K depends on ARCH_QCOM || COMPILE_TEST + depends on QCOM_SMEM select QCOM_SCM select QCOM_QMI_HELPERS help diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c index e76aab973320..438b0caaceb7 100644 --- a/drivers/net/wireless/ath/ath10k/htt_rx.c +++ b/drivers/net/wireless/ath/ath10k/htt_rx.c @@ -1379,7 +1379,7 @@ static void ath10k_process_rx(struct ath10k *ar, struct sk_buff *skb) ath10k_get_tid(hdr, tid, sizeof(tid)), is_multicast_ether_addr(ieee80211_get_DA(hdr)) ? "mcast" : "ucast", - (__le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_SEQ) >> 4, + IEEE80211_SEQ_TO_SN(__le16_to_cpu(hdr->seq_ctrl)), (status->encoding == RX_ENC_LEGACY) ? "legacy" : "", (status->encoding == RX_ENC_HT) ? "ht" : "", (status->encoding == RX_ENC_VHT) ? "vht" : "", @@ -1844,15 +1844,14 @@ static void ath10k_htt_rx_h_csum_offload(struct ath10k_hw_params *hw, } static u64 ath10k_htt_rx_h_get_pn(struct ath10k *ar, struct sk_buff *skb, - u16 offset, enum htt_rx_mpdu_encrypt_type enctype) { struct ieee80211_hdr *hdr; u64 pn = 0; u8 *ehdr; - hdr = (struct ieee80211_hdr *)(skb->data + offset); - ehdr = skb->data + offset + ieee80211_hdrlen(hdr->frame_control); + hdr = (struct ieee80211_hdr *)skb->data; + ehdr = skb->data + ieee80211_hdrlen(hdr->frame_control); if (enctype == HTT_RX_MPDU_ENCRYPT_AES_CCM_WPA2) { pn = ehdr[0]; @@ -1866,19 +1865,17 @@ static u64 ath10k_htt_rx_h_get_pn(struct ath10k *ar, struct sk_buff *skb, } static bool ath10k_htt_rx_h_frag_multicast_check(struct ath10k *ar, - struct sk_buff *skb, - u16 offset) + struct sk_buff *skb) { struct ieee80211_hdr *hdr; - hdr = (struct ieee80211_hdr *)(skb->data + offset); + hdr = (struct ieee80211_hdr *)skb->data; return !is_multicast_ether_addr(hdr->addr1); } static bool ath10k_htt_rx_h_frag_pn_check(struct ath10k *ar, struct sk_buff *skb, u16 peer_id, - u16 offset, enum htt_rx_mpdu_encrypt_type enctype) { struct ath10k_peer *peer; @@ -1893,16 +1890,16 @@ static bool ath10k_htt_rx_h_frag_pn_check(struct ath10k *ar, return false; } - hdr = (struct ieee80211_hdr *)(skb->data + offset); + hdr = (struct ieee80211_hdr *)skb->data; if (ieee80211_is_data_qos(hdr->frame_control)) tid = ieee80211_get_tid(hdr); else tid = ATH10K_TXRX_NON_QOS_TID; last_pn = &peer->frag_tids_last_pn[tid]; - new_pn.pn48 = ath10k_htt_rx_h_get_pn(ar, skb, offset, enctype); + new_pn.pn48 = ath10k_htt_rx_h_get_pn(ar, skb, enctype); frag_number = le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG; - seq = (__le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_SEQ) >> 4; + seq = IEEE80211_SEQ_TO_SN(__le16_to_cpu(hdr->seq_ctrl)); if (frag_number == 0) { last_pn->pn48 = new_pn.pn48; @@ -2059,13 +2056,11 @@ static void ath10k_htt_rx_h_mpdu(struct ath10k *ar, frag_pn_check = ath10k_htt_rx_h_frag_pn_check(ar, msdu, peer_id, - 0, enctype); if (frag) multicast_check = ath10k_htt_rx_h_frag_multicast_check(ar, - msdu, - 0); + msdu); if (!frag_pn_check || !multicast_check) { /* Discard the fragment with invalid PN or multicast DA @@ -2824,7 +2819,7 @@ static bool ath10k_htt_rx_proc_rx_frag_ind_hl(struct ath10k_htt *htt, hdr_space = ieee80211_hdrlen(hdr->frame_control); sc = __le16_to_cpu(hdr->seq_ctrl); - seq = (sc & IEEE80211_SCTL_SEQ) >> 4; + seq = IEEE80211_SEQ_TO_SN(sc); frag = sc & IEEE80211_SCTL_FRAG; sec_index = MS(rx_desc_info, HTT_RX_DESC_HL_INFO_MCAST_BCAST) ? diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c index e56c6a6b1379..728d607289c3 100644 --- a/drivers/net/wireless/ath/ath10k/pci.c +++ b/drivers/net/wireless/ath/ath10k/pci.c @@ -3792,18 +3792,22 @@ static struct pci_driver ath10k_pci_driver = { static int __init ath10k_pci_init(void) { - int ret; + int ret1, ret2; - ret = pci_register_driver(&ath10k_pci_driver); - if (ret) + ret1 = pci_register_driver(&ath10k_pci_driver); + if (ret1) printk(KERN_ERR "failed to register ath10k pci driver: %d\n", - ret); + ret1); - ret = ath10k_ahb_init(); - if (ret) - printk(KERN_ERR "ahb init failed: %d\n", ret); + ret2 = ath10k_ahb_init(); + if (ret2) + printk(KERN_ERR "ahb init failed: %d\n", ret2); - return ret; + if (ret1 && ret2) + return ret1; + + /* registered to at least one bus */ + return 0; } module_init(ath10k_pci_init); diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c index 66cb7a1e628a..3f94fbf83702 100644 --- a/drivers/net/wireless/ath/ath10k/qmi.c +++ b/drivers/net/wireless/ath/ath10k/qmi.c @@ -14,6 +14,7 @@ #include <linux/net.h> #include <linux/platform_device.h> #include <linux/qcom_scm.h> +#include <linux/soc/qcom/smem.h> #include <linux/string.h> #include <net/sock.h> @@ -22,6 +23,10 @@ #define ATH10K_QMI_CLIENT_ID 0x4b4e454c #define ATH10K_QMI_TIMEOUT 30 +#define SMEM_IMAGE_VERSION_TABLE 469 +#define SMEM_IMAGE_TABLE_CNSS_INDEX 13 +#define SMEM_IMAGE_VERSION_ENTRY_SIZE 128 +#define SMEM_IMAGE_VERSION_NAME_SIZE 75 static int ath10k_qmi_map_msa_permission(struct ath10k_qmi *qmi, struct ath10k_msa_mem_info *mem_info) @@ -536,6 +541,33 @@ int ath10k_qmi_wlan_disable(struct ath10k *ar) return ath10k_qmi_mode_send_sync_msg(ar, QMI_WLFW_OFF_V01); } +static void ath10k_qmi_add_wlan_ver_smem(struct ath10k *ar, const char *fw_build_id) +{ + u8 *table_ptr; + size_t smem_item_size; + const u32 smem_img_idx_wlan = SMEM_IMAGE_TABLE_CNSS_INDEX * + SMEM_IMAGE_VERSION_ENTRY_SIZE; + + table_ptr = qcom_smem_get(QCOM_SMEM_HOST_ANY, + SMEM_IMAGE_VERSION_TABLE, + &smem_item_size); + + if (IS_ERR(table_ptr)) { + ath10k_err(ar, "smem image version table not found\n"); + return; + } + + if (smem_img_idx_wlan + SMEM_IMAGE_VERSION_ENTRY_SIZE > + smem_item_size) { + ath10k_err(ar, "smem block size too small: %zu\n", + smem_item_size); + return; + } + + strscpy(table_ptr + smem_img_idx_wlan, fw_build_id, + SMEM_IMAGE_VERSION_NAME_SIZE); +} + static int ath10k_qmi_cap_send_sync_msg(struct ath10k_qmi *qmi) { struct wlfw_cap_resp_msg_v01 *resp; @@ -606,6 +638,9 @@ static int ath10k_qmi_cap_send_sync_msg(struct ath10k_qmi *qmi) qmi->fw_version, qmi->fw_build_timestamp, qmi->fw_build_id); } + if (resp->fw_build_id_valid) + ath10k_qmi_add_wlan_ver_smem(ar, qmi->fw_build_id); + kfree(resp); return 0; @@ -618,7 +653,7 @@ static int ath10k_qmi_host_cap_send_sync(struct ath10k_qmi *qmi) { struct wlfw_host_cap_resp_msg_v01 resp = {}; struct wlfw_host_cap_req_msg_v01 req = {}; - struct qmi_elem_info *req_ei; + const struct qmi_elem_info *req_ei; struct ath10k *ar = qmi->ar; struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar); struct qmi_txn txn; diff --git a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c index 86fcf4e1de5f..1c81e454f943 100644 --- a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c +++ b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c @@ -7,7 +7,7 @@ #include <linux/types.h> #include "qmi_wlfw_v01.h" -static struct qmi_elem_info wlfw_ce_tgt_pipe_cfg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_ce_tgt_pipe_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -56,7 +56,7 @@ static struct qmi_elem_info wlfw_ce_tgt_pipe_cfg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_ce_svc_pipe_cfg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_ce_svc_pipe_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -87,7 +87,7 @@ static struct qmi_elem_info wlfw_ce_svc_pipe_cfg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_shadow_reg_cfg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_shadow_reg_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_2_BYTE, .elem_len = 1, @@ -109,7 +109,7 @@ static struct qmi_elem_info wlfw_shadow_reg_cfg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_shadow_reg_v2_cfg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_shadow_reg_v2_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -122,7 +122,7 @@ static struct qmi_elem_info wlfw_shadow_reg_v2_cfg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_memory_region_info_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_memory_region_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -153,7 +153,7 @@ static struct qmi_elem_info wlfw_memory_region_info_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_mem_cfg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_mem_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -184,7 +184,7 @@ static struct qmi_elem_info wlfw_mem_cfg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_mem_seg_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_mem_seg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -225,7 +225,7 @@ static struct qmi_elem_info wlfw_mem_seg_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_mem_seg_resp_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_mem_seg_resp_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -256,7 +256,7 @@ static struct qmi_elem_info wlfw_mem_seg_resp_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_rf_chip_info_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_rf_chip_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -278,7 +278,7 @@ static struct qmi_elem_info wlfw_rf_chip_info_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_rf_board_info_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_rf_board_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -291,7 +291,7 @@ static struct qmi_elem_info wlfw_rf_board_info_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_soc_info_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_soc_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -304,7 +304,7 @@ static struct qmi_elem_info wlfw_soc_info_s_v01_ei[] = { {} }; -static struct qmi_elem_info wlfw_fw_version_info_s_v01_ei[] = { +static const struct qmi_elem_info wlfw_fw_version_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -326,7 +326,7 @@ static struct qmi_elem_info wlfw_fw_version_info_s_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_ind_register_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_ind_register_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -528,7 +528,7 @@ struct qmi_elem_info wlfw_ind_register_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_ind_register_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_ind_register_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -560,15 +560,15 @@ struct qmi_elem_info wlfw_ind_register_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_fw_ready_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_fw_ready_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_msa_ready_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_msa_ready_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_pin_connect_result_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_pin_connect_result_ind_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -626,7 +626,7 @@ struct qmi_elem_info wlfw_pin_connect_result_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_wlan_mode_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_wlan_mode_req_msg_v01_ei[] = { { .data_type = QMI_SIGNED_4_BYTE_ENUM, .elem_len = 1, @@ -657,7 +657,7 @@ struct qmi_elem_info wlfw_wlan_mode_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_wlan_mode_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_wlan_mode_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -671,7 +671,7 @@ struct qmi_elem_info wlfw_wlan_mode_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_wlan_cfg_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_wlan_cfg_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -805,7 +805,7 @@ struct qmi_elem_info wlfw_wlan_cfg_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_wlan_cfg_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_wlan_cfg_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -819,11 +819,11 @@ struct qmi_elem_info wlfw_wlan_cfg_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cap_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cap_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cap_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cap_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -949,7 +949,7 @@ struct qmi_elem_info wlfw_cap_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_bdf_download_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_bdf_download_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, .elem_len = 1, @@ -1079,7 +1079,7 @@ struct qmi_elem_info wlfw_bdf_download_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_bdf_download_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_bdf_download_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1093,7 +1093,7 @@ struct qmi_elem_info wlfw_bdf_download_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_report_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_report_req_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -1133,7 +1133,7 @@ struct qmi_elem_info wlfw_cal_report_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_report_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_report_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1147,7 +1147,7 @@ struct qmi_elem_info wlfw_cal_report_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_initiate_cal_download_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_initiate_cal_download_ind_msg_v01_ei[] = { { .data_type = QMI_SIGNED_4_BYTE_ENUM, .elem_len = 1, @@ -1160,7 +1160,7 @@ struct qmi_elem_info wlfw_initiate_cal_download_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_download_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_download_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, .elem_len = 1, @@ -1272,7 +1272,7 @@ struct qmi_elem_info wlfw_cal_download_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_download_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_download_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1286,7 +1286,7 @@ struct qmi_elem_info wlfw_cal_download_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_initiate_cal_update_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_initiate_cal_update_ind_msg_v01_ei[] = { { .data_type = QMI_SIGNED_4_BYTE_ENUM, .elem_len = 1, @@ -1308,7 +1308,7 @@ struct qmi_elem_info wlfw_initiate_cal_update_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_update_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_update_req_msg_v01_ei[] = { { .data_type = QMI_SIGNED_4_BYTE_ENUM, .elem_len = 1, @@ -1330,7 +1330,7 @@ struct qmi_elem_info wlfw_cal_update_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_cal_update_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_cal_update_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1443,7 +1443,7 @@ struct qmi_elem_info wlfw_cal_update_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_msa_info_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_msa_info_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -1465,7 +1465,7 @@ struct qmi_elem_info wlfw_msa_info_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_msa_info_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_msa_info_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1498,11 +1498,11 @@ struct qmi_elem_info wlfw_msa_info_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_msa_ready_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_msa_ready_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_msa_ready_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_msa_ready_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1516,7 +1516,7 @@ struct qmi_elem_info wlfw_msa_ready_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_ini_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_ini_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -1538,7 +1538,7 @@ struct qmi_elem_info wlfw_ini_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_ini_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_ini_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1552,7 +1552,7 @@ struct qmi_elem_info wlfw_ini_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_athdiag_read_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_athdiag_read_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1583,7 +1583,7 @@ struct qmi_elem_info wlfw_athdiag_read_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_athdiag_read_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_athdiag_read_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1624,7 +1624,7 @@ struct qmi_elem_info wlfw_athdiag_read_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_athdiag_write_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_athdiag_write_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1664,7 +1664,7 @@ struct qmi_elem_info wlfw_athdiag_write_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_athdiag_write_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_athdiag_write_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1678,7 +1678,7 @@ struct qmi_elem_info wlfw_athdiag_write_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_vbatt_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_vbatt_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -1691,7 +1691,7 @@ struct qmi_elem_info wlfw_vbatt_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_vbatt_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_vbatt_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1705,7 +1705,7 @@ struct qmi_elem_info wlfw_vbatt_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_mac_addr_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_mac_addr_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -1727,7 +1727,7 @@ struct qmi_elem_info wlfw_mac_addr_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_mac_addr_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_mac_addr_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1741,7 +1741,7 @@ struct qmi_elem_info wlfw_mac_addr_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_host_cap_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_host_cap_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -1988,7 +1988,7 @@ struct qmi_elem_info wlfw_host_cap_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_host_cap_8bit_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_host_cap_8bit_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -2010,7 +2010,7 @@ struct qmi_elem_info wlfw_host_cap_8bit_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_host_cap_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_host_cap_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -2024,7 +2024,7 @@ struct qmi_elem_info wlfw_host_cap_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_request_mem_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_request_mem_ind_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -2047,7 +2047,7 @@ struct qmi_elem_info wlfw_request_mem_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_respond_mem_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_respond_mem_req_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -2070,7 +2070,7 @@ struct qmi_elem_info wlfw_respond_mem_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_respond_mem_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_respond_mem_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -2084,15 +2084,15 @@ struct qmi_elem_info wlfw_respond_mem_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_mem_ready_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_mem_ready_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_fw_init_done_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_fw_init_done_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_rejuvenate_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_rejuvenate_ind_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -2168,11 +2168,11 @@ struct qmi_elem_info wlfw_rejuvenate_ind_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_rejuvenate_ack_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_rejuvenate_ack_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_rejuvenate_ack_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_rejuvenate_ack_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -2186,7 +2186,7 @@ struct qmi_elem_info wlfw_rejuvenate_ack_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_dynamic_feature_mask_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_dynamic_feature_mask_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -2208,7 +2208,7 @@ struct qmi_elem_info wlfw_dynamic_feature_mask_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_dynamic_feature_mask_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_dynamic_feature_mask_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -2258,7 +2258,7 @@ struct qmi_elem_info wlfw_dynamic_feature_mask_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_m3_info_req_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_m3_info_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -2280,7 +2280,7 @@ struct qmi_elem_info wlfw_m3_info_req_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_m3_info_resp_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_m3_info_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -2294,7 +2294,7 @@ struct qmi_elem_info wlfw_m3_info_resp_msg_v01_ei[] = { {} }; -struct qmi_elem_info wlfw_xo_cal_ind_msg_v01_ei[] = { +const struct qmi_elem_info wlfw_xo_cal_ind_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, .elem_len = 1, diff --git a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h index 4d107e1364a8..f0db991408dc 100644 --- a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h +++ b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h @@ -215,7 +215,7 @@ struct wlfw_ind_register_req_msg_v01 { }; #define WLFW_IND_REGISTER_REQ_MSG_V01_MAX_MSG_LEN 50 -extern struct qmi_elem_info wlfw_ind_register_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_ind_register_req_msg_v01_ei[]; struct wlfw_ind_register_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -224,21 +224,21 @@ struct wlfw_ind_register_resp_msg_v01 { }; #define WLFW_IND_REGISTER_RESP_MSG_V01_MAX_MSG_LEN 18 -extern struct qmi_elem_info wlfw_ind_register_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_ind_register_resp_msg_v01_ei[]; struct wlfw_fw_ready_ind_msg_v01 { char placeholder; }; #define WLFW_FW_READY_IND_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_fw_ready_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_fw_ready_ind_msg_v01_ei[]; struct wlfw_msa_ready_ind_msg_v01 { char placeholder; }; #define WLFW_MSA_READY_IND_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_msa_ready_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_msa_ready_ind_msg_v01_ei[]; struct wlfw_pin_connect_result_ind_msg_v01 { u8 pwr_pin_result_valid; @@ -250,7 +250,7 @@ struct wlfw_pin_connect_result_ind_msg_v01 { }; #define WLFW_PIN_CONNECT_RESULT_IND_MSG_V01_MAX_MSG_LEN 21 -extern struct qmi_elem_info wlfw_pin_connect_result_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_pin_connect_result_ind_msg_v01_ei[]; struct wlfw_wlan_mode_req_msg_v01 { enum wlfw_driver_mode_enum_v01 mode; @@ -259,14 +259,14 @@ struct wlfw_wlan_mode_req_msg_v01 { }; #define WLFW_WLAN_MODE_REQ_MSG_V01_MAX_MSG_LEN 11 -extern struct qmi_elem_info wlfw_wlan_mode_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_wlan_mode_req_msg_v01_ei[]; struct wlfw_wlan_mode_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_WLAN_MODE_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_wlan_mode_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_wlan_mode_resp_msg_v01_ei[]; struct wlfw_wlan_cfg_req_msg_v01 { u8 host_version_valid; @@ -286,21 +286,21 @@ struct wlfw_wlan_cfg_req_msg_v01 { }; #define WLFW_WLAN_CFG_REQ_MSG_V01_MAX_MSG_LEN 803 -extern struct qmi_elem_info wlfw_wlan_cfg_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_wlan_cfg_req_msg_v01_ei[]; struct wlfw_wlan_cfg_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_WLAN_CFG_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_wlan_cfg_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_wlan_cfg_resp_msg_v01_ei[]; struct wlfw_cap_req_msg_v01 { char placeholder; }; #define WLFW_CAP_REQ_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_cap_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cap_req_msg_v01_ei[]; struct wlfw_cap_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -319,7 +319,7 @@ struct wlfw_cap_resp_msg_v01 { }; #define WLFW_CAP_RESP_MSG_V01_MAX_MSG_LEN 207 -extern struct qmi_elem_info wlfw_cap_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cap_resp_msg_v01_ei[]; struct wlfw_bdf_download_req_msg_v01 { u8 valid; @@ -339,14 +339,14 @@ struct wlfw_bdf_download_req_msg_v01 { }; #define WLFW_BDF_DOWNLOAD_REQ_MSG_V01_MAX_MSG_LEN 6182 -extern struct qmi_elem_info wlfw_bdf_download_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_bdf_download_req_msg_v01_ei[]; struct wlfw_bdf_download_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_BDF_DOWNLOAD_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_bdf_download_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_bdf_download_resp_msg_v01_ei[]; struct wlfw_cal_report_req_msg_v01 { u32 meta_data_len; @@ -356,21 +356,21 @@ struct wlfw_cal_report_req_msg_v01 { }; #define WLFW_CAL_REPORT_REQ_MSG_V01_MAX_MSG_LEN 28 -extern struct qmi_elem_info wlfw_cal_report_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_report_req_msg_v01_ei[]; struct wlfw_cal_report_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_CAL_REPORT_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_cal_report_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_report_resp_msg_v01_ei[]; struct wlfw_initiate_cal_download_ind_msg_v01 { enum wlfw_cal_temp_id_enum_v01 cal_id; }; #define WLFW_INITIATE_CAL_DOWNLOAD_IND_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_initiate_cal_download_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_initiate_cal_download_ind_msg_v01_ei[]; struct wlfw_cal_download_req_msg_v01 { u8 valid; @@ -388,14 +388,14 @@ struct wlfw_cal_download_req_msg_v01 { }; #define WLFW_CAL_DOWNLOAD_REQ_MSG_V01_MAX_MSG_LEN 6178 -extern struct qmi_elem_info wlfw_cal_download_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_download_req_msg_v01_ei[]; struct wlfw_cal_download_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_CAL_DOWNLOAD_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_cal_download_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_download_resp_msg_v01_ei[]; struct wlfw_initiate_cal_update_ind_msg_v01 { enum wlfw_cal_temp_id_enum_v01 cal_id; @@ -403,7 +403,7 @@ struct wlfw_initiate_cal_update_ind_msg_v01 { }; #define WLFW_INITIATE_CAL_UPDATE_IND_MSG_V01_MAX_MSG_LEN 14 -extern struct qmi_elem_info wlfw_initiate_cal_update_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_initiate_cal_update_ind_msg_v01_ei[]; struct wlfw_cal_update_req_msg_v01 { enum wlfw_cal_temp_id_enum_v01 cal_id; @@ -411,7 +411,7 @@ struct wlfw_cal_update_req_msg_v01 { }; #define WLFW_CAL_UPDATE_REQ_MSG_V01_MAX_MSG_LEN 14 -extern struct qmi_elem_info wlfw_cal_update_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_update_req_msg_v01_ei[]; struct wlfw_cal_update_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -429,7 +429,7 @@ struct wlfw_cal_update_resp_msg_v01 { }; #define WLFW_CAL_UPDATE_RESP_MSG_V01_MAX_MSG_LEN 6181 -extern struct qmi_elem_info wlfw_cal_update_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_cal_update_resp_msg_v01_ei[]; struct wlfw_msa_info_req_msg_v01 { u64 msa_addr; @@ -437,7 +437,7 @@ struct wlfw_msa_info_req_msg_v01 { }; #define WLFW_MSA_INFO_REQ_MSG_V01_MAX_MSG_LEN 18 -extern struct qmi_elem_info wlfw_msa_info_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_msa_info_req_msg_v01_ei[]; struct wlfw_msa_info_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -446,21 +446,21 @@ struct wlfw_msa_info_resp_msg_v01 { }; #define WLFW_MSA_INFO_RESP_MSG_V01_MAX_MSG_LEN 37 -extern struct qmi_elem_info wlfw_msa_info_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_msa_info_resp_msg_v01_ei[]; struct wlfw_msa_ready_req_msg_v01 { char placeholder; }; #define WLFW_MSA_READY_REQ_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_msa_ready_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_msa_ready_req_msg_v01_ei[]; struct wlfw_msa_ready_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_MSA_READY_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_msa_ready_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_msa_ready_resp_msg_v01_ei[]; struct wlfw_ini_req_msg_v01 { u8 enablefwlog_valid; @@ -468,14 +468,14 @@ struct wlfw_ini_req_msg_v01 { }; #define WLFW_INI_REQ_MSG_V01_MAX_MSG_LEN 4 -extern struct qmi_elem_info wlfw_ini_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_ini_req_msg_v01_ei[]; struct wlfw_ini_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_INI_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_ini_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_ini_resp_msg_v01_ei[]; struct wlfw_athdiag_read_req_msg_v01 { u32 offset; @@ -484,7 +484,7 @@ struct wlfw_athdiag_read_req_msg_v01 { }; #define WLFW_ATHDIAG_READ_REQ_MSG_V01_MAX_MSG_LEN 21 -extern struct qmi_elem_info wlfw_athdiag_read_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_athdiag_read_req_msg_v01_ei[]; struct wlfw_athdiag_read_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -494,7 +494,7 @@ struct wlfw_athdiag_read_resp_msg_v01 { }; #define WLFW_ATHDIAG_READ_RESP_MSG_V01_MAX_MSG_LEN 6156 -extern struct qmi_elem_info wlfw_athdiag_read_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_athdiag_read_resp_msg_v01_ei[]; struct wlfw_athdiag_write_req_msg_v01 { u32 offset; @@ -504,28 +504,28 @@ struct wlfw_athdiag_write_req_msg_v01 { }; #define WLFW_ATHDIAG_WRITE_REQ_MSG_V01_MAX_MSG_LEN 6163 -extern struct qmi_elem_info wlfw_athdiag_write_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_athdiag_write_req_msg_v01_ei[]; struct wlfw_athdiag_write_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_ATHDIAG_WRITE_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_athdiag_write_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_athdiag_write_resp_msg_v01_ei[]; struct wlfw_vbatt_req_msg_v01 { u64 voltage_uv; }; #define WLFW_VBATT_REQ_MSG_V01_MAX_MSG_LEN 11 -extern struct qmi_elem_info wlfw_vbatt_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_vbatt_req_msg_v01_ei[]; struct wlfw_vbatt_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_VBATT_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_vbatt_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_vbatt_resp_msg_v01_ei[]; struct wlfw_mac_addr_req_msg_v01 { u8 mac_addr_valid; @@ -533,14 +533,14 @@ struct wlfw_mac_addr_req_msg_v01 { }; #define WLFW_MAC_ADDR_REQ_MSG_V01_MAX_MSG_LEN 9 -extern struct qmi_elem_info wlfw_mac_addr_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_mac_addr_req_msg_v01_ei[]; struct wlfw_mac_addr_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_MAC_ADDR_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_mac_addr_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_mac_addr_resp_msg_v01_ei[]; #define QMI_WLFW_MAX_NUM_GPIO_V01 32 struct wlfw_host_cap_req_msg_v01 { @@ -574,15 +574,15 @@ struct wlfw_host_cap_req_msg_v01 { }; #define WLFW_HOST_CAP_REQ_MSG_V01_MAX_MSG_LEN 189 -extern struct qmi_elem_info wlfw_host_cap_req_msg_v01_ei[]; -extern struct qmi_elem_info wlfw_host_cap_8bit_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_host_cap_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_host_cap_8bit_req_msg_v01_ei[]; struct wlfw_host_cap_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_HOST_CAP_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_host_cap_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_host_cap_resp_msg_v01_ei[]; struct wlfw_request_mem_ind_msg_v01 { u32 mem_seg_len; @@ -590,7 +590,7 @@ struct wlfw_request_mem_ind_msg_v01 { }; #define WLFW_REQUEST_MEM_IND_MSG_V01_MAX_MSG_LEN 564 -extern struct qmi_elem_info wlfw_request_mem_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_request_mem_ind_msg_v01_ei[]; struct wlfw_respond_mem_req_msg_v01 { u32 mem_seg_len; @@ -598,28 +598,28 @@ struct wlfw_respond_mem_req_msg_v01 { }; #define WLFW_RESPOND_MEM_REQ_MSG_V01_MAX_MSG_LEN 260 -extern struct qmi_elem_info wlfw_respond_mem_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_respond_mem_req_msg_v01_ei[]; struct wlfw_respond_mem_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_RESPOND_MEM_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_respond_mem_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_respond_mem_resp_msg_v01_ei[]; struct wlfw_mem_ready_ind_msg_v01 { char placeholder; }; #define WLFW_MEM_READY_IND_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_mem_ready_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_mem_ready_ind_msg_v01_ei[]; struct wlfw_fw_init_done_ind_msg_v01 { char placeholder; }; #define WLFW_FW_INIT_DONE_IND_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_fw_init_done_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_fw_init_done_ind_msg_v01_ei[]; struct wlfw_rejuvenate_ind_msg_v01 { u8 cause_for_rejuvenation_valid; @@ -633,21 +633,21 @@ struct wlfw_rejuvenate_ind_msg_v01 { }; #define WLFW_REJUVENATE_IND_MSG_V01_MAX_MSG_LEN 144 -extern struct qmi_elem_info wlfw_rejuvenate_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_rejuvenate_ind_msg_v01_ei[]; struct wlfw_rejuvenate_ack_req_msg_v01 { char placeholder; }; #define WLFW_REJUVENATE_ACK_REQ_MSG_V01_MAX_MSG_LEN 0 -extern struct qmi_elem_info wlfw_rejuvenate_ack_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_rejuvenate_ack_req_msg_v01_ei[]; struct wlfw_rejuvenate_ack_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_REJUVENATE_ACK_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_rejuvenate_ack_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_rejuvenate_ack_resp_msg_v01_ei[]; struct wlfw_dynamic_feature_mask_req_msg_v01 { u8 mask_valid; @@ -655,7 +655,7 @@ struct wlfw_dynamic_feature_mask_req_msg_v01 { }; #define WLFW_DYNAMIC_FEATURE_MASK_REQ_MSG_V01_MAX_MSG_LEN 11 -extern struct qmi_elem_info wlfw_dynamic_feature_mask_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_dynamic_feature_mask_req_msg_v01_ei[]; struct wlfw_dynamic_feature_mask_resp_msg_v01 { struct qmi_response_type_v01 resp; @@ -666,7 +666,7 @@ struct wlfw_dynamic_feature_mask_resp_msg_v01 { }; #define WLFW_DYNAMIC_FEATURE_MASK_RESP_MSG_V01_MAX_MSG_LEN 29 -extern struct qmi_elem_info wlfw_dynamic_feature_mask_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_dynamic_feature_mask_resp_msg_v01_ei[]; struct wlfw_m3_info_req_msg_v01 { u64 addr; @@ -674,20 +674,20 @@ struct wlfw_m3_info_req_msg_v01 { }; #define WLFW_M3_INFO_REQ_MSG_V01_MAX_MSG_LEN 18 -extern struct qmi_elem_info wlfw_m3_info_req_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_m3_info_req_msg_v01_ei[]; struct wlfw_m3_info_resp_msg_v01 { struct qmi_response_type_v01 resp; }; #define WLFW_M3_INFO_RESP_MSG_V01_MAX_MSG_LEN 7 -extern struct qmi_elem_info wlfw_m3_info_resp_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_m3_info_resp_msg_v01_ei[]; struct wlfw_xo_cal_ind_msg_v01 { u8 xo_cal_data; }; #define WLFW_XO_CAL_IND_MSG_V01_MAX_MSG_LEN 4 -extern struct qmi_elem_info wlfw_xo_cal_ind_msg_v01_ei[]; +extern const struct qmi_elem_info wlfw_xo_cal_ind_msg_v01_ei[]; #endif diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index 0ab2f190f1ec..edf78df9b12f 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -195,6 +195,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = true, .tx_ring_size = DP_TCL_DATA_RING_SIZE, .smp2p_wow_exit = false, + .support_fw_mac_sequence = false, }, { .name = "qca6390 hw2.0", @@ -277,6 +278,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = true, .tx_ring_size = DP_TCL_DATA_RING_SIZE, .smp2p_wow_exit = false, + .support_fw_mac_sequence = true, }, { .name = "qcn9074 hw1.0", @@ -356,6 +358,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = true, .tx_ring_size = DP_TCL_DATA_RING_SIZE, .smp2p_wow_exit = false, + .support_fw_mac_sequence = false, }, { .name = "wcn6855 hw2.0", @@ -438,6 +441,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = true, .tx_ring_size = DP_TCL_DATA_RING_SIZE, .smp2p_wow_exit = false, + .support_fw_mac_sequence = true, }, { .name = "wcn6855 hw2.1", @@ -519,6 +523,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = true, .tx_ring_size = DP_TCL_DATA_RING_SIZE, .smp2p_wow_exit = false, + .support_fw_mac_sequence = true, }, { .name = "wcn6750 hw1.0", @@ -597,6 +602,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .tcl_ring_retry = false, .tx_ring_size = DP_TCL_DATA_RING_SIZE_WCN6750, .smp2p_wow_exit = true, + .support_fw_mac_sequence = true, }, }; diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h index 8a3f24862edc..0c5ef8a526d8 100644 --- a/drivers/net/wireless/ath/ath11k/hw.h +++ b/drivers/net/wireless/ath/ath11k/hw.h @@ -219,6 +219,7 @@ struct ath11k_hw_params { bool tcl_ring_retry; u32 tx_ring_size; bool smp2p_wow_exit; + bool support_fw_mac_sequence; }; struct ath11k_hw_ops { diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 6426a38f8961..9e923ecb0891 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -8010,6 +8010,7 @@ ath11k_mac_op_reconfig_complete(struct ieee80211_hw *hw, struct ath11k *ar = hw->priv; struct ath11k_base *ab = ar->ab; int recovery_count; + struct ath11k_vif *arvif; if (reconfig_type != IEEE80211_RECONFIG_TYPE_RESTART) return; @@ -8045,6 +8046,12 @@ ath11k_mac_op_reconfig_complete(struct ieee80211_hw *hw, ath11k_dbg(ab, ATH11K_DBG_BOOT, "reset success\n"); } } + if (ar->ab->hw_params.support_fw_mac_sequence) { + list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif->is_up && arvif->vdev_type == WMI_VDEV_TYPE_STA) + ieee80211_hw_restart_disconnect(arvif->vif); + } + } } mutex_unlock(&ar->conf_mutex); diff --git a/drivers/net/wireless/ath/ath11k/mac.h b/drivers/net/wireless/ath/ath11k/mac.h index 2a0d3afb0c99..0231783ad754 100644 --- a/drivers/net/wireless/ath/ath11k/mac.h +++ b/drivers/net/wireless/ath/ath11k/mac.h @@ -163,7 +163,7 @@ void ath11k_mac_drain_tx(struct ath11k *ar); void ath11k_mac_peer_cleanup_all(struct ath11k *ar); int ath11k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx); u8 ath11k_mac_bw_to_mac80211_bw(u8 bw); -u32 ath11k_mac_he_gi_to_nl80211_he_gi(u8 sgi); +enum nl80211_he_gi ath11k_mac_he_gi_to_nl80211_he_gi(u8 sgi); enum nl80211_he_ru_alloc ath11k_mac_phy_he_ru_to_nl80211_he_ru_alloc(u16 ru_phy); enum nl80211_he_ru_alloc ath11k_mac_he_ru_tones_to_nl80211_he_ru_alloc(u16 ru_tones); enum ath11k_supported_bw ath11k_mac_mac80211_bw_to_ath11k_bw(enum rate_info_bw bw); diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index 145f20a681bd..ab923e24b0a9 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -29,7 +29,7 @@ module_param_named(cold_boot_cal, ath11k_cold_boot_cal, bool, 0644); MODULE_PARM_DESC(cold_boot_cal, "Decrease the channel switch time but increase the driver load time (Default: true)"); -static struct qmi_elem_info qmi_wlanfw_host_cap_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_host_cap_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -280,7 +280,7 @@ static struct qmi_elem_info qmi_wlanfw_host_cap_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_host_cap_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_host_cap_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -297,7 +297,7 @@ static struct qmi_elem_info qmi_wlanfw_host_cap_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_ind_register_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_ind_register_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -522,7 +522,7 @@ static struct qmi_elem_info qmi_wlanfw_ind_register_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_ind_register_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_ind_register_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -558,7 +558,7 @@ static struct qmi_elem_info qmi_wlanfw_ind_register_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_mem_cfg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_mem_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -590,7 +590,7 @@ static struct qmi_elem_info qmi_wlanfw_mem_cfg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_mem_seg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_mem_seg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -632,7 +632,7 @@ static struct qmi_elem_info qmi_wlanfw_mem_seg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_request_mem_ind_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_request_mem_ind_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -659,7 +659,7 @@ static struct qmi_elem_info qmi_wlanfw_request_mem_ind_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_mem_seg_resp_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_mem_seg_resp_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -699,7 +699,7 @@ static struct qmi_elem_info qmi_wlanfw_mem_seg_resp_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_respond_mem_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_respond_mem_req_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -726,7 +726,7 @@ static struct qmi_elem_info qmi_wlanfw_respond_mem_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_respond_mem_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_respond_mem_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -744,7 +744,7 @@ static struct qmi_elem_info qmi_wlanfw_respond_mem_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_cap_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_cap_req_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, @@ -752,7 +752,7 @@ static struct qmi_elem_info qmi_wlanfw_cap_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_device_info_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_device_info_req_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, @@ -760,7 +760,7 @@ static struct qmi_elem_info qmi_wlanfw_device_info_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlfw_device_info_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlfw_device_info_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -814,7 +814,7 @@ static struct qmi_elem_info qmi_wlfw_device_info_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_rf_chip_info_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_rf_chip_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -840,7 +840,7 @@ static struct qmi_elem_info qmi_wlanfw_rf_chip_info_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_rf_board_info_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_rf_board_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -857,7 +857,7 @@ static struct qmi_elem_info qmi_wlanfw_rf_board_info_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_soc_info_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_soc_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -873,7 +873,7 @@ static struct qmi_elem_info qmi_wlanfw_soc_info_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_fw_version_info_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_fw_version_info_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -899,7 +899,7 @@ static struct qmi_elem_info qmi_wlanfw_fw_version_info_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_cap_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_cap_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1100,7 +1100,7 @@ static struct qmi_elem_info qmi_wlanfw_cap_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_bdf_download_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_bdf_download_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, .elem_len = 1, @@ -1235,7 +1235,7 @@ static struct qmi_elem_info qmi_wlanfw_bdf_download_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_bdf_download_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_bdf_download_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1253,7 +1253,7 @@ static struct qmi_elem_info qmi_wlanfw_bdf_download_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_m3_info_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_m3_info_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_8_BYTE, .elem_len = 1, @@ -1277,7 +1277,7 @@ static struct qmi_elem_info qmi_wlanfw_m3_info_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_m3_info_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_m3_info_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1294,7 +1294,7 @@ static struct qmi_elem_info qmi_wlanfw_m3_info_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_ce_tgt_pipe_cfg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_ce_tgt_pipe_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1347,7 +1347,7 @@ static struct qmi_elem_info qmi_wlanfw_ce_tgt_pipe_cfg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_ce_svc_pipe_cfg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_ce_svc_pipe_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1382,7 +1382,7 @@ static struct qmi_elem_info qmi_wlanfw_ce_svc_pipe_cfg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_shadow_reg_cfg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_shadow_reg_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_2_BYTE, .elem_len = 1, @@ -1406,7 +1406,7 @@ static struct qmi_elem_info qmi_wlanfw_shadow_reg_cfg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_shadow_reg_v2_cfg_s_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_shadow_reg_v2_cfg_s_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1423,7 +1423,7 @@ static struct qmi_elem_info qmi_wlanfw_shadow_reg_v2_cfg_s_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_mode_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_mode_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, .elem_len = 1, @@ -1458,7 +1458,7 @@ static struct qmi_elem_info qmi_wlanfw_wlan_mode_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_mode_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_mode_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1476,7 +1476,7 @@ static struct qmi_elem_info qmi_wlanfw_wlan_mode_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -1615,7 +1615,7 @@ static struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_cfg_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1632,28 +1632,28 @@ static struct qmi_elem_info qmi_wlanfw_wlan_cfg_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_mem_ready_ind_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_mem_ready_ind_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, }, }; -static struct qmi_elem_info qmi_wlanfw_fw_ready_ind_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_fw_ready_ind_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, }, }; -static struct qmi_elem_info qmi_wlanfw_cold_boot_cal_done_ind_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_cold_boot_cal_done_ind_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_ini_req_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_ini_req_msg_v01_ei[] = { { .data_type = QMI_OPT_FLAG, .elem_len = 1, @@ -1679,7 +1679,7 @@ static struct qmi_elem_info qmi_wlanfw_wlan_ini_req_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlanfw_wlan_ini_resp_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlanfw_wlan_ini_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -1697,7 +1697,7 @@ static struct qmi_elem_info qmi_wlanfw_wlan_ini_resp_msg_v01_ei[] = { }, }; -static struct qmi_elem_info qmi_wlfw_fw_init_done_ind_msg_v01_ei[] = { +static const struct qmi_elem_info qmi_wlfw_fw_init_done_ind_msg_v01_ei[] = { { .data_type = QMI_EOTI, .array_type = NO_ARRAY, @@ -3090,6 +3090,9 @@ static const struct qmi_msg_handler ath11k_qmi_msg_handlers[] = { sizeof(struct qmi_wlfw_fw_init_done_ind_msg_v01), .fn = ath11k_qmi_msg_fw_init_done_cb, }, + + /* end of list */ + {}, }; static int ath11k_qmi_ops_new_server(struct qmi_handle *qmi_hdl, diff --git a/drivers/net/wireless/ath/ath9k/ar9003_mci.c b/drivers/net/wireless/ath/ath9k/ar9003_mci.c index 9899661f9a60..8d7efd80d97a 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_mci.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_mci.c @@ -585,7 +585,7 @@ static u32 ar9003_mci_wait_for_gpm(struct ath_hw *ah, u8 gpm_type, { struct ath_common *common = ath9k_hw_common(ah); struct ath9k_hw_mci *mci = &ah->btcoex_hw.mci; - u32 *p_gpm = NULL, mismatch = 0, more_data; + u32 *p_gpm = NULL, more_data; u32 offset; u8 recv_type = 0, recv_opcode = 0; bool b_is_bt_cal_done = (gpm_type == MCI_GPM_BT_CAL_DONE); @@ -656,7 +656,6 @@ static u32 ar9003_mci_wait_for_gpm(struct ath_hw *ah, u8 gpm_type, } else { ath_dbg(common, MCI, "MCI GPM subtype not match 0x%x\n", *(p_gpm + 1)); - mismatch++; ar9003_mci_process_gpm_extra(ah, recv_type, recv_opcode, p_gpm); } diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index ba271a10d4ab..39abb59d8771 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -1678,7 +1678,6 @@ void ath9k_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_tx_info *info; struct list_head bf_q; struct ath_buf *bf_tail = NULL, *bf = NULL; - int sent = 0; int i, ret; INIT_LIST_HEAD(&bf_q); @@ -1707,7 +1706,6 @@ void ath9k_release_buffered_frames(struct ieee80211_hw *hw, bf_tail = bf; nframes--; - sent++; TX_STAT_INC(sc, txq->axq_qnum, a_queued_hw); if (an->sta && skb_queue_empty(&tid->retry_q)) diff --git a/drivers/net/wireless/ath/carl9170/fwcmd.h b/drivers/net/wireless/ath/carl9170/fwcmd.h index 4a500095555c..ff4b3b50250c 100644 --- a/drivers/net/wireless/ath/carl9170/fwcmd.h +++ b/drivers/net/wireless/ath/carl9170/fwcmd.h @@ -118,10 +118,10 @@ struct carl9170_reg_list { } __packed; struct carl9170_write_reg { - struct { + DECLARE_FLEX_ARRAY(struct { __le32 addr; __le32 val; - } regs[0] __packed; + } __packed, regs); } __packed; struct carl9170_write_reg_byte { diff --git a/drivers/net/wireless/broadcom/b43/main.c b/drivers/net/wireless/broadcom/b43/main.c index 298febbdffc0..92ca0b2ca286 100644 --- a/drivers/net/wireless/broadcom/b43/main.c +++ b/drivers/net/wireless/broadcom/b43/main.c @@ -5784,15 +5784,12 @@ void b43_controller_restart(struct b43_wldev *dev, const char *reason) static void b43_print_driverinfo(void) { - const char *feat_pci = "", *feat_pcmcia = "", *feat_nphy = "", + const char *feat_pci = "", *feat_nphy = "", *feat_leds = "", *feat_sdio = ""; #ifdef CONFIG_B43_PCI_AUTOSELECT feat_pci = "P"; #endif -#ifdef CONFIG_B43_PCMCIA - feat_pcmcia = "M"; -#endif #ifdef CONFIG_B43_PHY_N feat_nphy = "N"; #endif @@ -5803,9 +5800,8 @@ static void b43_print_driverinfo(void) feat_sdio = "S"; #endif printk(KERN_INFO "Broadcom 43xx driver loaded " - "[ Features: %s%s%s%s%s ]\n", - feat_pci, feat_pcmcia, feat_nphy, - feat_leds, feat_sdio); + "[ Features: %s%s%s%s ]\n", + feat_pci, feat_nphy, feat_leds, feat_sdio); } static int __init b43_init(void) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ae9507dec74a..bff3128c2f26 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -101,6 +101,9 @@ #define BRCMF_ASSOC_PARAMS_FIXED_SIZE \ (sizeof(struct brcmf_assoc_params_le) - sizeof(u16)) +#define BRCMF_MAX_CHANSPEC_LIST \ + (BRCMF_DCMD_MEDLEN / sizeof(__le32) - 1) + struct brcmf_dump_survey { u32 obss; u32 ibss; @@ -6840,6 +6843,13 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, band->channels[i].flags = IEEE80211_CHAN_DISABLED; total = le32_to_cpu(list->count); + if (total > BRCMF_MAX_CHANSPEC_LIST) { + bphy_err(drvr, "Invalid count of channel Spec. (%u)\n", + total); + err = -EINVAL; + goto fail_pbuf; + } + for (i = 0; i < total; i++) { ch.chspec = (u16)le32_to_cpu(list->element[i]); cfg->d11inf.decchspec(&ch); @@ -6985,6 +6995,13 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) band = cfg_to_wiphy(cfg)->bands[NL80211_BAND_2GHZ]; list = (struct brcmf_chanspec_list *)pbuf; num_chan = le32_to_cpu(list->count); + if (num_chan > BRCMF_MAX_CHANSPEC_LIST) { + bphy_err(drvr, "Invalid count of channel Spec. (%u)\n", + num_chan); + kfree(pbuf); + return -EINVAL; + } + for (i = 0; i < num_chan; i++) { ch.chspec = (u16)le32_to_cpu(list->element[i]); cfg->d11inf.decchspec(&ch); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 22344e68fd59..4a309e5a5707 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -110,9 +110,9 @@ static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, dload_buf->dload_type = cpu_to_le16(DL_TYPE_CLM); dload_buf->len = cpu_to_le32(len); dload_buf->crc = cpu_to_le32(0); - len = sizeof(*dload_buf) + len - 1; - err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, len); + err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, + struct_size(dload_buf, data, len)); return err; } @@ -139,7 +139,8 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) return 0; } - chunk_buf = kzalloc(sizeof(*chunk_buf) + MAX_CHUNK_LEN - 1, GFP_KERNEL); + chunk_buf = kzalloc(struct_size(chunk_buf, data, MAX_CHUNK_LEN), + GFP_KERNEL); if (!chunk_buf) { err = -ENOMEM; goto done; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index f518e025d6e4..04e1beedfd81 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -943,7 +943,7 @@ struct brcmf_dload_data_le { __le16 dload_type; __le32 len; __le32 crc; - u8 data[1]; + u8 data[]; }; /** @@ -1049,7 +1049,7 @@ struct brcmf_gscan_config { u8 count_of_channel_buckets; u8 retry_threshold; __le16 lost_ap_window; - struct brcmf_gscan_bucket_config bucket[1]; + struct brcmf_gscan_bucket_config bucket[]; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index 170c61c8136c..05f66ab13bed 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -405,7 +405,7 @@ static int brcmf_pno_config_sched_scans(struct brcmf_if *ifp) if (n_buckets < 0) return n_buckets; - gsz = sizeof(*gscan_cfg) + (n_buckets - 1) * sizeof(*buckets); + gsz = struct_size(gscan_cfg, bucket, n_buckets); gscan_cfg = kzalloc(gsz, GFP_KERNEL); if (!gscan_cfg) { err = -ENOMEM; @@ -434,8 +434,8 @@ static int brcmf_pno_config_sched_scans(struct brcmf_if *ifp) gscan_cfg->flags = BRCMF_GSCAN_CFG_ALL_BUCKETS_IN_1ST_SCAN; gscan_cfg->count_of_channel_buckets = n_buckets; - memcpy(&gscan_cfg->bucket[0], buckets, - n_buckets * sizeof(*buckets)); + memcpy(gscan_cfg->bucket, buckets, + array_size(n_buckets, sizeof(*buckets))); err = brcmf_fil_iovar_data_set(ifp, "pfn_gscan_cfg", gscan_cfg, gsz); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h index 9b7caf968346..e3eda251c728 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2021 Intel Corporation + * Copyright (C) 2012-2014, 2018-2022 Intel Corporation * Copyright (C) 2017 Intel Deutschland GmbH */ #ifndef __iwl_fw_api_mac_h__ @@ -398,7 +398,7 @@ struct iwl_he_backoff_conf { * @IWL_HE_PKT_EXT_64QAM: 64-QAM * @IWL_HE_PKT_EXT_256QAM: 256-QAM * @IWL_HE_PKT_EXT_1024QAM: 1024-QAM - * @IWL_HE_PKT_EXT_RESERVED: reserved value + * @IWL_HE_PKT_EXT_4096QAM: 4096-QAM, for EHT only * @IWL_HE_PKT_EXT_NONE: not defined */ enum iwl_he_pkt_ext_constellations { @@ -408,7 +408,7 @@ enum iwl_he_pkt_ext_constellations { IWL_HE_PKT_EXT_64QAM, IWL_HE_PKT_EXT_256QAM, IWL_HE_PKT_EXT_1024QAM, - IWL_HE_PKT_EXT_RESERVED, + IWL_HE_PKT_EXT_4096QAM, IWL_HE_PKT_EXT_NONE, }; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h b/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h index e66f77924f83..2f7d8558becd 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h @@ -13,10 +13,12 @@ #define PHY_BAND_6 (2) /* Supported channel width, vary if there is VHT support */ -#define PHY_VHT_CHANNEL_MODE20 (0x0) -#define PHY_VHT_CHANNEL_MODE40 (0x1) -#define PHY_VHT_CHANNEL_MODE80 (0x2) -#define PHY_VHT_CHANNEL_MODE160 (0x3) +#define IWL_PHY_CHANNEL_MODE20 0x0 +#define IWL_PHY_CHANNEL_MODE40 0x1 +#define IWL_PHY_CHANNEL_MODE80 0x2 +#define IWL_PHY_CHANNEL_MODE160 0x3 +/* and 320 MHz for EHT */ +#define IWL_PHY_CHANNEL_MODE320 0x4 /* * Control channel position: @@ -24,20 +26,17 @@ * For VHT - bit-2 marks if the control is lower/upper relative to center-freq * bits-1:0 mark the distance from the center freq. for 20Mhz, offset is 0. * center_freq - * | - * 40Mhz |_______|_______| - * 80Mhz |_______|_______|_______|_______| - * 160Mhz |_______|_______|_______|_______|_______|_______|_______|_______| - * code 011 010 001 000 | 100 101 110 111 + * For EHT - bit-3 is used for extended distance + * | + * 40Mhz |____|____| + * 80Mhz |____|____|____|____| + * 160Mhz |____|____|____|____|____|____|____|____| + * 320MHz |____|____|____|____|____|____|____|____|____|____|____|____|____|____|____|____| + * code 1011 1010 1001 1000 0011 0010 0001 0000 0100 0101 0110 0111 1100 1101 1110 1111 */ -#define PHY_VHT_CTRL_POS_1_BELOW (0x0) -#define PHY_VHT_CTRL_POS_2_BELOW (0x1) -#define PHY_VHT_CTRL_POS_3_BELOW (0x2) -#define PHY_VHT_CTRL_POS_4_BELOW (0x3) -#define PHY_VHT_CTRL_POS_1_ABOVE (0x4) -#define PHY_VHT_CTRL_POS_2_ABOVE (0x5) -#define PHY_VHT_CTRL_POS_3_ABOVE (0x6) -#define PHY_VHT_CTRL_POS_4_ABOVE (0x7) +#define IWL_PHY_CTRL_POS_ABOVE 0x4 +#define IWL_PHY_CTRL_POS_OFFS_EXT 0x8 +#define IWL_PHY_CTRL_POS_OFFS_MSK 0x3 /* * struct iwl_fw_channel_info_v1 - channel information diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h index 687f804c46b7..ddacd5b45aea 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h @@ -36,14 +36,14 @@ enum iwl_tlc_mng_cfg_flags { * @IWL_TLC_MNG_CH_WIDTH_40MHZ: 40MHZ channel * @IWL_TLC_MNG_CH_WIDTH_80MHZ: 80MHZ channel * @IWL_TLC_MNG_CH_WIDTH_160MHZ: 160MHZ channel - * @IWL_TLC_MNG_CH_WIDTH_LAST: maximum value + * @IWL_TLC_MNG_CH_WIDTH_320MHZ: 320MHZ channel */ enum iwl_tlc_mng_cfg_cw { IWL_TLC_MNG_CH_WIDTH_20MHZ, IWL_TLC_MNG_CH_WIDTH_40MHZ, IWL_TLC_MNG_CH_WIDTH_80MHZ, IWL_TLC_MNG_CH_WIDTH_160MHZ, - IWL_TLC_MNG_CH_WIDTH_LAST = IWL_TLC_MNG_CH_WIDTH_160MHZ, + IWL_TLC_MNG_CH_WIDTH_320MHZ, }; /** @@ -64,8 +64,7 @@ enum iwl_tlc_mng_cfg_chains { * @IWL_TLC_MNG_MODE_HT: enable HT * @IWL_TLC_MNG_MODE_VHT: enable VHT * @IWL_TLC_MNG_MODE_HE: enable HE - * @IWL_TLC_MNG_MODE_INVALID: invalid value - * @IWL_TLC_MNG_MODE_NUM: a count of possible modes + * @IWL_TLC_MNG_MODE_EHT: enable EHT */ enum iwl_tlc_mng_cfg_mode { IWL_TLC_MNG_MODE_CCK = 0, @@ -74,8 +73,7 @@ enum iwl_tlc_mng_cfg_mode { IWL_TLC_MNG_MODE_HT, IWL_TLC_MNG_MODE_VHT, IWL_TLC_MNG_MODE_HE, - IWL_TLC_MNG_MODE_INVALID, - IWL_TLC_MNG_MODE_NUM = IWL_TLC_MNG_MODE_INVALID, + IWL_TLC_MNG_MODE_EHT, }; /** diff --git a/drivers/net/wireless/intel/iwlwifi/fw/rs.c b/drivers/net/wireless/intel/iwlwifi/fw/rs.c index a835214611ce..e128d2e07f38 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/rs.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/rs.c @@ -218,6 +218,8 @@ int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate) type = "HT"; else if (format == RATE_MCS_HE_MSK) type = "HE"; + else if (format == RATE_MCS_EHT_MSK) + type = "EHT"; else type = "Unknown"; /* shouldn't happen */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c index a2203f661321..ab7065c93826 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c @@ -1971,3 +1971,6 @@ MODULE_PARM_DESC(remove_when_gone, module_param_named(disable_11ax, iwlwifi_mod_params.disable_11ax, bool, S_IRUGO); MODULE_PARM_DESC(disable_11ax, "Disable HE capabilities (default: false)"); + +module_param_named(disable_11be, iwlwifi_mod_params.disable_11be, bool, 0444); +MODULE_PARM_DESC(disable_11be, "Disable EHT capabilities (default: false)"); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h index 0efffb6eeb1e..baa643386018 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2005-2014, 2018 Intel Corporation + * Copyright (C) 2005-2014, 2018, 2020-2022 Intel Corporation * Copyright (C) 2015 Intel Mobile Communications GmbH */ #ifndef __iwl_eeprom_parse_h__ @@ -31,6 +31,7 @@ struct iwl_nvm_data { bool sku_cap_amt_enable; bool sku_cap_ipan_enable; bool sku_cap_mimo_disabled; + bool sku_cap_11be_enable; u16 radio_cfg_type; u8 radio_cfg_step; diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h index d0b4d02bdab9..1cf26ab4f488 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h @@ -62,6 +62,7 @@ enum iwl_uapsd_disable { * @disable_11ac: disable VHT capabilities, default = false. * @remove_when_gone: remove an inaccessible device from the PCIe bus. * @enable_ini: enable new FW debug infratructure (INI TLVs) + * @disable_11be: disable EHT capabilities, default = false. */ struct iwl_mod_params { int swcrypto; diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c index 9040da3dcce3..476b90f32626 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2005-2014, 2018-2021 Intel Corporation + * Copyright (C) 2005-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -546,7 +546,7 @@ static const u8 iwl_vendor_caps[] = { 0x00 }; -static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { +static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = { { .types_mask = BIT(NL80211_IFTYPE_STATION), .he_cap = { @@ -571,10 +571,6 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { IEEE80211_HE_MAC_CAP5_UL_2x996_TONE_RU | IEEE80211_HE_MAC_CAP5_HE_DYNAMIC_SM_PS | IEEE80211_HE_MAC_CAP5_HT_VHT_TRIG_FRAME_RX, - .phy_cap_info[0] = - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G, .phy_cap_info[1] = IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK | IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | @@ -631,6 +627,78 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { */ .ppe_thres = {0x61, 0x1c, 0xc7, 0x71}, }, + .eht_cap = { + .has_eht = true, + .eht_cap_elem = { + .mac_cap_info[0] = + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_OM_CONTROL | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2, + .phy_cap_info[0] = + IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ | + IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI | + IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE | + IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK, + .phy_cap_info[1] = + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK | + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK | + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK, + .phy_cap_info[3] = + IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK, + + .phy_cap_info[4] = + IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | + IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP | + IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI, + .phy_cap_info[5] = + IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK | + IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT, + .phy_cap_info[6] = + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK | + IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP, + .phy_cap_info[8] = + IEEE80211_EHT_PHY_CAP8_RX_1024QAM_WIDER_BW_DL_OFDMA | + IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA, + }, + + /* For all MCS and bandwidth, set 2 NSS for both Tx and + * Rx - note we don't set the only_20mhz, but due to this + * being a union, it gets set correctly anyway. + */ + .eht_mcs_nss_supp = { + .bw._80 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + .bw._160 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + .bw._320 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + }, + + /* + * PPE thresholds for NSS = 2, and RU index bitmap set + * to 0xc. + */ + .eht_ppe_thres = {0xc1, 0x0e, 0xe0 } + }, }, { .types_mask = BIT(NL80211_IFTYPE_AP), @@ -644,9 +712,6 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8, .mac_cap_info[3] = IEEE80211_HE_MAC_CAP3_OMI_CONTROL, - .phy_cap_info[0] = - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G, .phy_cap_info[1] = IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD, .phy_cap_info[2] = @@ -687,6 +752,49 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { */ .ppe_thres = {0x61, 0x1c, 0xc7, 0x71}, }, + .eht_cap = { + .has_eht = true, + .eht_cap_elem = { + .mac_cap_info[0] = + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_OM_CONTROL | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2, + .phy_cap_info[0] = + IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ | + IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI, + .phy_cap_info[5] = + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT, + }, + + /* For all MCS and bandwidth, set 2 NSS for both Tx and + * Rx - note we don't set the only_20mhz, but due to this + * being a union, it gets set correctly anyway. + */ + .eht_mcs_nss_supp = { + .bw._80 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + .bw._160 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + .bw._320 = { + .rx_tx_mcs9_max_nss = 0x22, + .rx_tx_mcs11_max_nss = 0x22, + .rx_tx_mcs13_max_nss = 0x22, + }, + }, + + /* + * PPE thresholds for NSS = 2, and RU index bitmap set + * to 0xc. + */ + .eht_ppe_thres = {0xc1, 0x0e, 0xe0 } + }, }, }; @@ -738,6 +846,7 @@ static void iwl_init_he_6ghz_capa(struct iwl_trans *trans, static void iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, + struct iwl_nvm_data *data, struct ieee80211_supported_band *sband, struct ieee80211_sband_iftype_data *iftype_data, u8 tx_chains, u8 rx_chains, @@ -745,6 +854,9 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, { bool is_ap = iftype_data->types_mask & BIT(NL80211_IFTYPE_AP); + if (!data->sku_cap_11be_enable || iwlwifi_mod_params.disable_11be) + iftype_data->eht_cap.has_eht = false; + /* Advertise an A-MPDU exponent extension based on * operating band */ @@ -755,9 +867,30 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, iftype_data->he_cap.he_cap_elem.mac_cap_info[3] |= IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3; - if (is_ap && iwlwifi_mod_params.nvm_file) + switch (sband->band) { + case NL80211_BAND_2GHZ: iftype_data->he_cap.he_cap_elem.phy_cap_info[0] |= - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G; + iftype_data->eht_cap.eht_cap_elem.mac_cap_info[0] |= + u8_encode_bits(IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454, + IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK); + break; + case NL80211_BAND_6GHZ: + if (!is_ap || iwlwifi_mod_params.nvm_file) + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[0] |= + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + fallthrough; + case NL80211_BAND_5GHZ: + iftype_data->he_cap.he_cap_elem.phy_cap_info[0] |= + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G; + if (!is_ap || iwlwifi_mod_params.nvm_file) + iftype_data->he_cap.he_cap_elem.phy_cap_info[0] |= + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + break; + default: + WARN_ON(1); + break; + } if ((tx_chains & rx_chains) == ANT_AB) { iftype_data->he_cap.he_cap_elem.phy_cap_info[2] |= @@ -765,19 +898,44 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, iftype_data->he_cap.he_cap_elem.phy_cap_info[5] |= IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2 | IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_2; - if (!is_ap) + if (!is_ap) { iftype_data->he_cap.he_cap_elem.phy_cap_info[7] |= IEEE80211_HE_PHY_CAP7_MAX_NC_2; - } else if (!is_ap) { - /* If not 2x2, we need to indicate 1x1 in the - * Midamble RX Max NSTS - but not for AP mode - */ - iftype_data->he_cap.he_cap_elem.phy_cap_info[1] &= - ~IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS; - iftype_data->he_cap.he_cap_elem.phy_cap_info[2] &= - ~IEEE80211_HE_PHY_CAP2_MIDAMBLE_RX_TX_MAX_NSTS; - iftype_data->he_cap.he_cap_elem.phy_cap_info[7] |= - IEEE80211_HE_PHY_CAP7_MAX_NC_1; + + if (iftype_data->eht_cap.has_eht) { + /* + * Set the number of sounding dimensions for each + * bandwidth to 1 to indicate the maximal supported + * value of TXVECTOR parameter NUM_STS of 2 + */ + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[2] |= 0x49; + + /* + * Set the MAX NC to 1 to indicate sounding feedback of + * 2 supported by the beamfomee. + */ + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[4] |= 0x10; + } + } + } else { + if (iftype_data->eht_cap.has_eht) { + struct ieee80211_eht_mcs_nss_supp *mcs_nss = + &iftype_data->eht_cap.eht_mcs_nss_supp; + + memset(mcs_nss, 0x11, sizeof(*mcs_nss)); + } + + if (!is_ap) { + /* If not 2x2, we need to indicate 1x1 in the + * Midamble RX Max NSTS - but not for AP mode + */ + iftype_data->he_cap.he_cap_elem.phy_cap_info[1] &= + ~IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS; + iftype_data->he_cap.he_cap_elem.phy_cap_info[2] &= + ~IEEE80211_HE_PHY_CAP2_MIDAMBLE_RX_TX_MAX_NSTS; + iftype_data->he_cap.he_cap_elem.phy_cap_info[7] |= + IEEE80211_HE_PHY_CAP7_MAX_NC_1; + } } switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) { @@ -792,6 +950,29 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, break; } + if (CSR_HW_REV_TYPE(trans->hw_rev) == IWL_CFG_MAC_TYPE_GL && + iftype_data->eht_cap.has_eht) { + iftype_data->eht_cap.eht_cap_elem.mac_cap_info[0] &= + ~(IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2); + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[3] &= + ~(IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO | + IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK); + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[4] &= + ~(IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | + IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP); + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[5] &= + ~IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK; + iftype_data->eht_cap.eht_cap_elem.phy_cap_info[6] &= + ~(IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK | + IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP); + } + if (fw_has_capa(&fw->ucode_capa, IWL_UCODE_TLV_CAPA_BROADCAST_TWT)) iftype_data->he_cap.he_cap_elem.mac_cap_info[2] |= IEEE80211_HE_MAC_CAP2_BCAST_TWT; @@ -816,8 +997,8 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans, if (WARN_ON(sband->iftype_data)) return; - BUILD_BUG_ON(sizeof(data->iftd.low) != sizeof(iwl_he_capa)); - BUILD_BUG_ON(sizeof(data->iftd.high) != sizeof(iwl_he_capa)); + BUILD_BUG_ON(sizeof(data->iftd.low) != sizeof(iwl_he_eht_capa)); + BUILD_BUG_ON(sizeof(data->iftd.high) != sizeof(iwl_he_eht_capa)); switch (sband->band) { case NL80211_BAND_2GHZ: @@ -832,13 +1013,13 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans, return; } - memcpy(iftype_data, iwl_he_capa, sizeof(iwl_he_capa)); + memcpy(iftype_data, iwl_he_eht_capa, sizeof(iwl_he_eht_capa)); sband->iftype_data = iftype_data; - sband->n_iftype_data = ARRAY_SIZE(iwl_he_capa); + sband->n_iftype_data = ARRAY_SIZE(iwl_he_eht_capa); for (i = 0; i < sband->n_iftype_data; i++) - iwl_nvm_fixup_sband_iftd(trans, sband, &iftype_data[i], + iwl_nvm_fixup_sband_iftd(trans, data, sband, &iftype_data[i], tx_chains, rx_chains, fw); iwl_init_he_6ghz_capa(trans, data, sband, tx_chains, rx_chains); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h index 82cf904e0b6d..62ce116d3783 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h @@ -368,6 +368,7 @@ enum { #define CNVR_AUX_MISC_CHIP 0xA2B800 #define CNVR_SCU_SD_REGS_SD_REG_DIG_DCDC_VTRIM 0xA29890 #define CNVR_SCU_SD_REGS_SD_REG_ACTIVE_VDIG_MIRROR 0xA29938 +#define CNVI_SCU_SEQ_DATA_DW9 0xA27488 #define PREG_AUX_BUS_WPROT_0 0xA04CC0 diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index d659ccd065f7..32bd7f19f1d5 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -1542,5 +1542,6 @@ void iwl_trans_free(struct iwl_trans *trans); ******************************************************/ int __must_check iwl_pci_register_driver(void); void iwl_pci_unregister_driver(void); +void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan); #endif /* __iwl_trans_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index ef43f6971cd9..0a41cd0ab243 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -354,6 +354,20 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm, */ ret = iwl_wait_notification(&mvm->notif_wait, &alive_wait, MVM_UCODE_ALIVE_TIMEOUT); + + if (mvm->trans->trans_cfg->device_family == + IWL_DEVICE_FAMILY_AX210) { + /* print these registers regardless of alive fail/success */ + IWL_INFO(mvm, "WFPM_UMAC_PD_NOTIFICATION: 0x%x\n", + iwl_read_umac_prph(mvm->trans, WFPM_ARC1_PD_NOTIFICATION)); + IWL_INFO(mvm, "WFPM_LMAC2_PD_NOTIFICATION: 0x%x\n", + iwl_read_umac_prph(mvm->trans, WFPM_LMAC2_PD_NOTIFICATION)); + IWL_INFO(mvm, "WFPM_AUTH_KEY_0: 0x%x\n", + iwl_read_umac_prph(mvm->trans, SB_MODIFY_CFG_FLAG)); + IWL_INFO(mvm, "CNVI_SCU_SEQ_DATA_DW9: 0x%x\n", + iwl_read_prph(mvm->trans, CNVI_SCU_SEQ_DATA_DW9)); + } + if (ret) { struct iwl_trans *trans = mvm->trans; @@ -390,7 +404,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm, UREG_LMAC2_CURRENT_PC)); } - if (ret == -ETIMEDOUT) + if (ret == -ETIMEDOUT && !mvm->pldr_sync) iwl_fw_dbg_error_collect(&mvm->fwrt, FW_DBG_TRIGGER_ALIVE_TIMEOUT); @@ -404,7 +418,8 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm, return -EIO; } - iwl_mei_alive_notif(!ret); + /* if reached this point, Alive notification was received */ + iwl_mei_alive_notif(true); ret = iwl_pnvm_load(mvm->trans, &mvm->notif_wait); if (ret) { @@ -1467,18 +1482,22 @@ int iwl_mvm_up(struct iwl_mvm *mvm) return ret; sb_cfg = iwl_read_umac_prph(mvm->trans, SB_MODIFY_CFG_FLAG); - if (!(sb_cfg & SB_CFG_RESIDES_IN_OTP_MASK) && iwl_mei_pldr_req()) - return ret; + mvm->pldr_sync = !(sb_cfg & SB_CFG_RESIDES_IN_OTP_MASK); + if (mvm->pldr_sync && iwl_mei_pldr_req()) + return -EBUSY; ret = iwl_mvm_load_rt_fw(mvm); if (ret) { IWL_ERR(mvm, "Failed to start RT ucode: %d\n", ret); - if (ret != -ERFKILL) + if (ret != -ERFKILL && !mvm->pldr_sync) iwl_fw_dbg_error_collect(&mvm->fwrt, FW_DBG_TRIGGER_DRIVER); goto error; } + /* FW loaded successfully */ + mvm->pldr_sync = false; + iwl_get_shared_mem_conf(&mvm->fwrt); ret = iwl_mvm_sf_update(mvm, NULL, false); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c index a81dd488caa8..5273ade71117 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c @@ -1068,6 +1068,16 @@ static int iwl_mvm_mac_start(struct ieee80211_hw *hw) if (!ret) break; + /* + * In PLDR sync PCI re-enumeration is needed. no point to retry + * mac start before that. + */ + if (mvm->pldr_sync) { + iwl_mei_alive_notif(false); + iwl_trans_pcie_remove(mvm->trans, true); + break; + } + IWL_ERR(mvm, "mac start retry %d\n", retry); } clear_bit(IWL_MVM_STATUS_STARTING, &mvm->status); @@ -1825,7 +1835,8 @@ static u8 iwl_mvm_he_get_ppe_val(u8 *ppe, u8 ppe_pos_bit) static void iwl_mvm_parse_ppe(struct iwl_mvm *mvm, struct iwl_he_pkt_ext_v2 *pkt_ext, u8 nss, - u8 ru_index_bitmap, u8 *ppe, u8 ppe_pos_bit) + u8 ru_index_bitmap, u8 *ppe, u8 ppe_pos_bit, + bool inheritance) { int i; @@ -1851,14 +1862,25 @@ static void iwl_mvm_parse_ppe(struct iwl_mvm *mvm, bw++) { ru_index_tmp >>= 1; - if (!(ru_index_tmp & 1)) - continue; + /* + * According to the 11be spec, if for a specific BW the PPE Thresholds + * isn't present - it should inherit the thresholds from the last + * BW for which we had PPE Thresholds. In 11ax though, we don't have + * this inheritance - continue in this case + */ + if (!(ru_index_tmp & 1)) { + if (inheritance) + goto set_thresholds; + else + continue; + } high_th = iwl_mvm_he_get_ppe_val(ppe, ppe_pos_bit); ppe_pos_bit += IEEE80211_PPE_THRES_INFO_PPET_SIZE; low_th = iwl_mvm_he_get_ppe_val(ppe, ppe_pos_bit); ppe_pos_bit += IEEE80211_PPE_THRES_INFO_PPET_SIZE; +set_thresholds: pkt_ext->pkt_ext_qam_th[i][bw][0] = low_th; pkt_ext->pkt_ext_qam_th[i][bw][1] = high_th; } @@ -1867,7 +1889,8 @@ static void iwl_mvm_parse_ppe(struct iwl_mvm *mvm, static void iwl_mvm_set_pkt_ext_from_he_ppe(struct iwl_mvm *mvm, struct ieee80211_sta *sta, - struct iwl_he_pkt_ext_v2 *pkt_ext) + struct iwl_he_pkt_ext_v2 *pkt_ext, + bool inheritance) { u8 nss = (sta->deflink.he_cap.ppe_thres[0] & IEEE80211_PPE_THRES_NSS_MASK) + 1; u8 *ppe = &sta->deflink.he_cap.ppe_thres[0]; @@ -1877,7 +1900,8 @@ static void iwl_mvm_set_pkt_ext_from_he_ppe(struct iwl_mvm *mvm, /* Starting after PPE header */ u8 ppe_pos_bit = IEEE80211_HE_PPE_THRES_INFO_HEADER_SIZE; - iwl_mvm_parse_ppe(mvm, pkt_ext, nss, ru_index_bitmap, ppe, ppe_pos_bit); + iwl_mvm_parse_ppe(mvm, pkt_ext, nss, ru_index_bitmap, ppe, ppe_pos_bit, + inheritance); } static void iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *pkt_ext, @@ -1888,16 +1912,18 @@ static void iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *p int high_th = -1; int i; + /* all the macros are the same for EHT and HE */ switch (nominal_padding) { - case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US: + case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_0US: low_th = IWL_HE_PKT_EXT_NONE; high_th = IWL_HE_PKT_EXT_NONE; break; - case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US: + case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US: low_th = IWL_HE_PKT_EXT_BPSK; high_th = IWL_HE_PKT_EXT_NONE; break; - case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US: + case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_16US: + case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_20US: low_th = IWL_HE_PKT_EXT_NONE; high_th = IWL_HE_PKT_EXT_BPSK; break; @@ -1920,6 +1946,31 @@ static void iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *p } } +static void iwl_mvm_get_optimal_ppe_info(struct iwl_he_pkt_ext_v2 *pkt_ext, + u8 nominal_padding) +{ + int i; + + for (i = 0; i < MAX_HE_SUPP_NSS; i++) { + u8 bw; + + for (bw = 0; bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]); + bw++) { + u8 *qam_th = &pkt_ext->pkt_ext_qam_th[i][bw][0]; + + if (nominal_padding > + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US && + qam_th[1] == IWL_HE_PKT_EXT_NONE) + qam_th[1] = IWL_HE_PKT_EXT_4096QAM; + else if (nominal_padding == + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US && + qam_th[0] == IWL_HE_PKT_EXT_NONE && + qam_th[1] == IWL_HE_PKT_EXT_NONE) + qam_th[0] = IWL_HE_PKT_EXT_4096QAM; + } + } +} + static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif, u8 sta_id) { @@ -1943,6 +1994,7 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm, struct ieee80211_chanctx_conf *chanctx_conf; const struct ieee80211_supported_band *sband; void *cmd; + u8 nominal_padding; if (!fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_MBSSID_HE)) ver = 1; @@ -2032,22 +2084,96 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm, memset(&sta_ctxt_cmd.pkt_ext, IWL_HE_PKT_EXT_NONE, sizeof(sta_ctxt_cmd.pkt_ext)); - /* If PPE Thresholds exist, parse them into a FW-familiar format. */ - if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[6] & - IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) { - iwl_mvm_set_pkt_ext_from_he_ppe(mvm, sta, - &sta_ctxt_cmd.pkt_ext); - flags |= STA_CTXT_HE_PACKET_EXT; - /* PPE Thresholds doesn't exist - set the API PPE values - * according to Common Nominal Packet Padding fiels. */ - } else { - u8 nominal_padding = - u8_get_bits(sta->deflink.he_cap.he_cap_elem.phy_cap_info[9], - IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); - if (nominal_padding != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED) + if (sta->deflink.eht_cap.has_eht) { + nominal_padding = + u8_get_bits(sta->deflink.eht_cap.eht_cap_elem.phy_cap_info[5], + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK); + + /* If PPE Thresholds exists, parse them into a FW-familiar format. */ + if (sta->deflink.eht_cap.eht_cap_elem.phy_cap_info[5] & + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) { + u8 nss = (sta->deflink.eht_cap.eht_ppe_thres[0] & + IEEE80211_EHT_PPE_THRES_NSS_MASK) + 1; + u8 *ppe = &sta->deflink.eht_cap.eht_ppe_thres[0]; + u8 ru_index_bitmap = + u16_get_bits(*ppe, + IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK); + /* Starting after PPE header */ + u8 ppe_pos_bit = IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE; + + iwl_mvm_parse_ppe(mvm, + &sta_ctxt_cmd.pkt_ext, + nss, ru_index_bitmap, ppe, + ppe_pos_bit, true); + flags |= STA_CTXT_HE_PACKET_EXT; + /* EHT PPE Thresholds doesn't exist - set the API according to HE PPE Tresholds*/ + } else if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[6] & + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) { + struct iwl_he_pkt_ext_v2 *pkt_ext = + &sta_ctxt_cmd.pkt_ext; + + /* + * Even though HE Capabilities IE doesn't contain PPE + * Thresholds for BW 320Mhz, thresholds for this BW will + * be filled in with the same values as 160Mhz, due to + * the inheritance, as required. + */ + iwl_mvm_set_pkt_ext_from_he_ppe(mvm, sta, pkt_ext, + true); + + /* + * According to the requirements, for MCSs 12-13 the maximum value between + * HE PPE Threshold and Common Nominal Packet Padding needs to be taken + */ + iwl_mvm_get_optimal_ppe_info(pkt_ext, nominal_padding); + + flags |= STA_CTXT_HE_PACKET_EXT; + + /* + * if PPE Thresholds doesn't present in both EHT IE and HE IE - + * take the Thresholds from Common Nominal Packet Padding field + */ + } else { iwl_mvm_set_pkt_ext_from_nominal_padding(&sta_ctxt_cmd.pkt_ext, nominal_padding, &flags); + } + } else if (sta->deflink.he_cap.has_he) { + /* If PPE Thresholds exist, parse them into a FW-familiar format. */ + if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[6] & + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) { + iwl_mvm_set_pkt_ext_from_he_ppe(mvm, sta, + &sta_ctxt_cmd.pkt_ext, + false); + flags |= STA_CTXT_HE_PACKET_EXT; + /* + * PPE Thresholds doesn't exist - set the API PPE values + * according to Common Nominal Packet Padding field. + */ + } else { + nominal_padding = + u8_get_bits(sta->deflink.he_cap.he_cap_elem.phy_cap_info[9], + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); + if (nominal_padding != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED) + iwl_mvm_set_pkt_ext_from_nominal_padding(&sta_ctxt_cmd.pkt_ext, + nominal_padding, + &flags); + } + } + + for (i = 0; i < MAX_HE_SUPP_NSS; i++) { + int bw; + + for (bw = 0; + bw < ARRAY_SIZE(sta_ctxt_cmd.pkt_ext.pkt_ext_qam_th[i]); + bw++) { + u8 *qam_th = + &sta_ctxt_cmd.pkt_ext.pkt_ext_qam_th[i][bw][0]; + + IWL_DEBUG_HT(mvm, + "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n", + i, bw, qam_th[0], qam_th[1]); + } } if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[2] & @@ -2195,8 +2321,10 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm, * interface was added. */ if (changes & BSS_CHANGED_ASSOC && vif->cfg.assoc) { - if (vif->bss_conf.he_support && - !iwlwifi_mod_params.disable_11ax) + if ((vif->bss_conf.he_support && + !iwlwifi_mod_params.disable_11ax) || + (vif->bss_conf.eht_support && + !iwlwifi_mod_params.disable_11be)) iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->ap_sta_id); iwl_mvm_mac_ctxt_recalc_tsf_id(mvm, vif); @@ -2204,8 +2332,11 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm, /* Update MU EDCA params */ if (changes & BSS_CHANGED_QOS && mvmvif->associated && - vif->cfg.assoc && vif->bss_conf.he_support && - !iwlwifi_mod_params.disable_11ax) + vif->cfg.assoc && + ((vif->bss_conf.he_support && + !iwlwifi_mod_params.disable_11ax) || + (vif->bss_conf.eht_support && + !iwlwifi_mod_params.disable_11be))) iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->ap_sta_id); /* @@ -3218,8 +3349,10 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw, vif->bss_conf.he_support = sta->deflink.he_cap.has_he; mvmvif->ap_assoc_sta_count++; iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL); - if (vif->bss_conf.he_support && - !iwlwifi_mod_params.disable_11ax) + if ((vif->bss_conf.he_support && + !iwlwifi_mod_params.disable_11ax) || + (vif->bss_conf.eht_support && + !iwlwifi_mod_params.disable_11be)) iwl_mvm_cfg_he_sta(mvm, vif, mvm_sta->sta_id); } else if (vif->type == NL80211_IFTYPE_STATION) { vif->bss_conf.he_support = sta->deflink.he_cap.has_he; @@ -3777,7 +3910,7 @@ static int iwl_mvm_send_aux_roc_cmd(struct iwl_mvm *mvm, /* Set the channel info data */ iwl_mvm_set_chan_info(mvm, &aux_roc_req.channel_info, channel->hw_value, iwl_mvm_phy_band_from_nl80211(channel->band), - PHY_VHT_CHANNEL_MODE20, + IWL_PHY_CHANNEL_MODE20, 0); /* Set the time and duration */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index 962e304fc2b1..ce6b701f3f4c 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -1105,6 +1105,8 @@ struct iwl_mvm { unsigned long last_reset_or_resume_time_jiffies; bool sta_remove_requires_queue_remove; + + bool pldr_sync; }; /* Extract MVM priv from op_mode and _hw */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index 9699433137bc..e78f5beaa2d0 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -1077,6 +1077,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, static const u8 no_reclaim_cmds[] = { TX_CMD, }; + u32 max_agg; size_t scan_size; u32 min_backoff; struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused; @@ -1098,12 +1099,17 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, if (!hw) return NULL; - hw->max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; + if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) + max_agg = IEEE80211_MAX_AMPDU_BUF_EHT; + else + max_agg = IEEE80211_MAX_AMPDU_BUF_HE; + + hw->max_rx_aggregation_subframes = max_agg; if (cfg->max_tx_agg_size) hw->max_tx_aggregation_subframes = cfg->max_tx_agg_size; else - hw->max_tx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; + hw->max_tx_aggregation_subframes = max_agg; op_mode = hw->priv; @@ -1882,6 +1888,9 @@ static void iwl_mvm_nic_error(struct iwl_op_mode *op_mode, bool sync) { struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode); + if (mvm->pldr_sync) + return; + if (!test_bit(STATUS_TRANS_DEAD, &mvm->trans->status) && !test_and_clear_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE, &mvm->status)) diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c index a3cefbc43e80..06f4203fb989 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c @@ -14,16 +14,18 @@ u8 iwl_mvm_get_channel_width(struct cfg80211_chan_def *chandef) switch (chandef->width) { case NL80211_CHAN_WIDTH_20_NOHT: case NL80211_CHAN_WIDTH_20: - return PHY_VHT_CHANNEL_MODE20; + return IWL_PHY_CHANNEL_MODE20; case NL80211_CHAN_WIDTH_40: - return PHY_VHT_CHANNEL_MODE40; + return IWL_PHY_CHANNEL_MODE40; case NL80211_CHAN_WIDTH_80: - return PHY_VHT_CHANNEL_MODE80; + return IWL_PHY_CHANNEL_MODE80; case NL80211_CHAN_WIDTH_160: - return PHY_VHT_CHANNEL_MODE160; + return IWL_PHY_CHANNEL_MODE160; + case NL80211_CHAN_WIDTH_320: + return IWL_PHY_CHANNEL_MODE320; default: WARN(1, "Invalid channel width=%u", chandef->width); - return PHY_VHT_CHANNEL_MODE20; + return IWL_PHY_CHANNEL_MODE20; } } @@ -33,34 +35,32 @@ u8 iwl_mvm_get_channel_width(struct cfg80211_chan_def *chandef) */ u8 iwl_mvm_get_ctrl_pos(struct cfg80211_chan_def *chandef) { - switch (chandef->chan->center_freq - chandef->center_freq1) { - case -70: - return PHY_VHT_CTRL_POS_4_BELOW; - case -50: - return PHY_VHT_CTRL_POS_3_BELOW; - case -30: - return PHY_VHT_CTRL_POS_2_BELOW; - case -10: - return PHY_VHT_CTRL_POS_1_BELOW; - case 10: - return PHY_VHT_CTRL_POS_1_ABOVE; - case 30: - return PHY_VHT_CTRL_POS_2_ABOVE; - case 50: - return PHY_VHT_CTRL_POS_3_ABOVE; - case 70: - return PHY_VHT_CTRL_POS_4_ABOVE; - default: - WARN(1, "Invalid channel definition"); - fallthrough; - case 0: + int offs = chandef->chan->center_freq - chandef->center_freq1; + int abs_offs = abs(offs); + u8 ret; + + if (offs == 0) { /* * The FW is expected to check the control channel position only * when in HT/VHT and the channel width is not 20MHz. Return * this value as the default one. */ - return PHY_VHT_CTRL_POS_1_BELOW; + return 0; } + + /* this results in a value 0-7, i.e. fitting into 0b0111 */ + ret = (abs_offs - 10) / 20; + /* + * But we need the value to be in 0b1011 because 0b0100 is + * IWL_PHY_CTRL_POS_ABOVE, so shift bit 2 up to land in + * IWL_PHY_CTRL_POS_OFFS_EXT (0b1000) + */ + ret = (ret & IWL_PHY_CTRL_POS_OFFS_MSK) | + ((ret & BIT(2)) << 1); + /* and add the above bit */ + ret |= (offs > 0) * IWL_PHY_CTRL_POS_ABOVE; + + return ret; } /* diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c index 2e9081cb6627..f30eeab5505b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c @@ -9,9 +9,11 @@ #include "iwl-op-mode.h" #include "mvm.h" -static u8 rs_fw_bw_from_sta_bw(struct ieee80211_sta *sta) +static u8 rs_fw_bw_from_sta_bw(const struct ieee80211_sta *sta) { switch (sta->deflink.bandwidth) { + case IEEE80211_STA_RX_BW_320: + return IWL_TLC_MNG_CH_WIDTH_320MHZ; case IEEE80211_STA_RX_BW_160: return IWL_TLC_MNG_CH_WIDTH_160MHZ; case IEEE80211_STA_RX_BW_80: @@ -238,6 +240,122 @@ rs_fw_he_set_enabled_rates(const struct ieee80211_sta *sta, } } +static u8 rs_fw_eht_max_nss(u8 rx_nss, u8 tx_nss) +{ + u8 tx = u8_get_bits(tx_nss, IEEE80211_EHT_MCS_NSS_TX); + u8 rx = u8_get_bits(rx_nss, IEEE80211_EHT_MCS_NSS_RX); + /* the max nss that can be used, + * is the min with our tx capa and the peer rx capa. + */ + return min(tx, rx); +} + +#define MAX_NSS_MCS(mcs_num, rx, tx) \ + rs_fw_eht_max_nss((rx)->rx_tx_mcs ##mcs_num## _max_nss, \ + (tx)->rx_tx_mcs ##mcs_num## _max_nss) + +static void rs_fw_set_eht_mcs_nss(__le16 ht_rates[][3], + enum IWL_TLC_MCS_PER_BW bw, + u8 max_nss, u16 mcs_msk) +{ + if (max_nss >= 2) + ht_rates[IWL_TLC_NSS_2][bw] |= cpu_to_le16(mcs_msk); + + if (max_nss >= 1) + ht_rates[IWL_TLC_NSS_1][bw] |= cpu_to_le16(mcs_msk); +} + +static const +struct ieee80211_eht_mcs_nss_supp_bw * +rs_fw_rs_mcs2eht_mcs(enum IWL_TLC_MCS_PER_BW bw, + const struct ieee80211_eht_mcs_nss_supp *eht_mcs) +{ + switch (bw) { + case IWL_TLC_MCS_PER_BW_80: + return &eht_mcs->bw._80; + case IWL_TLC_MCS_PER_BW_160: + return &eht_mcs->bw._160; + case IWL_TLC_MCS_PER_BW_320: + return &eht_mcs->bw._320; + default: + return NULL; + } +} + +static void rs_fw_eht_set_enabled_rates(const struct ieee80211_sta *sta, + struct ieee80211_supported_band *sband, + struct iwl_tlc_config_cmd_v4 *cmd) +{ + /* peer RX mcs capa */ + const struct ieee80211_eht_mcs_nss_supp *eht_rx_mcs = + &sta->deflink.eht_cap.eht_mcs_nss_supp; + /* our TX mcs capa */ + const struct ieee80211_eht_mcs_nss_supp *eht_tx_mcs = + &sband->iftype_data->eht_cap.eht_mcs_nss_supp; + + enum IWL_TLC_MCS_PER_BW bw; + struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_rx_20; + struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_tx_20; + + /* peer is 20Mhz only */ + if (!(sta->deflink.he_cap.he_cap_elem.phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) { + mcs_rx_20 = eht_rx_mcs->only_20mhz; + } else { + mcs_rx_20.rx_tx_mcs7_max_nss = eht_rx_mcs->bw._80.rx_tx_mcs9_max_nss; + mcs_rx_20.rx_tx_mcs9_max_nss = eht_rx_mcs->bw._80.rx_tx_mcs9_max_nss; + mcs_rx_20.rx_tx_mcs11_max_nss = eht_rx_mcs->bw._80.rx_tx_mcs11_max_nss; + mcs_rx_20.rx_tx_mcs13_max_nss = eht_rx_mcs->bw._80.rx_tx_mcs13_max_nss; + } + + /* nic is 20Mhz only */ + if (!(sband->iftype_data->he_cap.he_cap_elem.phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) { + mcs_tx_20 = eht_tx_mcs->only_20mhz; + } else { + mcs_tx_20.rx_tx_mcs7_max_nss = eht_tx_mcs->bw._80.rx_tx_mcs9_max_nss; + mcs_tx_20.rx_tx_mcs9_max_nss = eht_tx_mcs->bw._80.rx_tx_mcs9_max_nss; + mcs_tx_20.rx_tx_mcs11_max_nss = eht_tx_mcs->bw._80.rx_tx_mcs11_max_nss; + mcs_tx_20.rx_tx_mcs13_max_nss = eht_tx_mcs->bw._80.rx_tx_mcs13_max_nss; + } + + /* rates for 20/40/80 bw */ + bw = IWL_TLC_MCS_PER_BW_80; + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(7, &mcs_rx_20, &mcs_tx_20), GENMASK(7, 0)); + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(9, &mcs_rx_20, &mcs_tx_20), GENMASK(9, 8)); + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(11, &mcs_rx_20, &mcs_tx_20), GENMASK(11, 10)); + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(13, &mcs_rx_20, &mcs_tx_20), GENMASK(13, 12)); + + /* rate for 160/320 bw */ + for (bw = IWL_TLC_MCS_PER_BW_160; bw <= IWL_TLC_MCS_PER_BW_320; bw++) { + const struct ieee80211_eht_mcs_nss_supp_bw *mcs_rx = + rs_fw_rs_mcs2eht_mcs(bw, eht_rx_mcs); + const struct ieee80211_eht_mcs_nss_supp_bw *mcs_tx = + rs_fw_rs_mcs2eht_mcs(bw, eht_tx_mcs); + + /* got unsuppored index for bw */ + if (!mcs_rx || !mcs_tx) + continue; + + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(9, mcs_rx, mcs_tx), GENMASK(9, 0)); + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(11, mcs_rx, mcs_tx), GENMASK(11, 10)); + rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw, + MAX_NSS_MCS(13, mcs_rx, mcs_tx), GENMASK(13, 12)); + } + + /* the station support only a single receive chain */ + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC || + sta->deflink.rx_nss < 2) + memset(cmd->ht_rates[IWL_TLC_NSS_2], 0, + sizeof(cmd->ht_rates[IWL_TLC_NSS_2])); +} + static void rs_fw_set_supp_rates(struct ieee80211_sta *sta, struct ieee80211_supported_band *sband, struct iwl_tlc_config_cmd_v4 *cmd) @@ -258,7 +376,10 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta, cmd->mode = IWL_TLC_MNG_MODE_NON_HT; /* HT/VHT rates */ - if (he_cap->has_he) { + if (sta->deflink.eht_cap.has_eht) { + cmd->mode = IWL_TLC_MNG_MODE_EHT; + rs_fw_eht_set_enabled_rates(sta, sband, cmd); + } else if (he_cap->has_he) { cmd->mode = IWL_TLC_MNG_MODE_HE; rs_fw_he_set_enabled_rates(sta, sband, cmd); } else if (vht_cap->vht_supported) { diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c index f460332333a7..fadaa683a416 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c @@ -1215,6 +1215,7 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb, struct sk_buff_head mpdus_skbs; unsigned int payload_len; int ret; + struct sk_buff *orig_skb = skb; if (WARN_ON_ONCE(!mvmsta)) return -1; @@ -1247,8 +1248,17 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb, ret = iwl_mvm_tx_mpdu(mvm, skb, &info, sta); if (ret) { + /* Free skbs created as part of TSO logic that have not yet been dequeued */ __skb_queue_purge(&mpdus_skbs); - return ret; + /* skb here is not necessarily same as skb that entered this method, + * so free it explicitly. + */ + if (skb == orig_skb) + ieee80211_free_txskb(mvm->hw, skb); + else + kfree_skb(skb); + /* there was error, but we consumed skb one way or another, so return 0 */ + return 0; } } diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index bd50f52a1aad..0a9af1ad1f20 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -2052,6 +2052,7 @@ static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state) struct iwl_trans_pcie_removal { struct pci_dev *pdev; struct work_struct work; + bool rescan; }; static void iwl_trans_pcie_removal_wk(struct work_struct *wk) @@ -2060,18 +2061,61 @@ static void iwl_trans_pcie_removal_wk(struct work_struct *wk) container_of(wk, struct iwl_trans_pcie_removal, work); struct pci_dev *pdev = removal->pdev; static char *prop[] = {"EVENT=INACCESSIBLE", NULL}; + struct pci_bus *bus = pdev->bus; dev_err(&pdev->dev, "Device gone - attempting removal\n"); kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop); pci_lock_rescan_remove(); pci_dev_put(pdev); pci_stop_and_remove_bus_device(pdev); + if (removal->rescan) + pci_rescan_bus(bus->parent); pci_unlock_rescan_remove(); kfree(removal); module_put(THIS_MODULE); } +void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan) +{ + struct iwl_trans_pcie_removal *removal; + + if (test_bit(STATUS_TRANS_DEAD, &trans->status)) + return; + + IWL_ERR(trans, "Device gone - scheduling removal!\n"); + + /* + * get a module reference to avoid doing this + * while unloading anyway and to avoid + * scheduling a work with code that's being + * removed. + */ + if (!try_module_get(THIS_MODULE)) { + IWL_ERR(trans, + "Module is being unloaded - abort\n"); + return; + } + + removal = kzalloc(sizeof(*removal), GFP_ATOMIC); + if (!removal) { + module_put(THIS_MODULE); + return; + } + /* + * we don't need to clear this flag, because + * the trans will be freed and reallocated. + */ + set_bit(STATUS_TRANS_DEAD, &trans->status); + + removal->pdev = to_pci_dev(trans->dev); + removal->rescan = rescan; + INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk); + pci_dev_get(removal->pdev); + schedule_work(&removal->work); +} +EXPORT_SYMBOL(iwl_trans_pcie_remove); + /* * This version doesn't disable BHs but rather assumes they're * already disabled. @@ -2131,47 +2175,12 @@ bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans) iwl_trans_pcie_dump_regs(trans); - if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) { - struct iwl_trans_pcie_removal *removal; - - if (test_bit(STATUS_TRANS_DEAD, &trans->status)) - goto err; - - IWL_ERR(trans, "Device gone - scheduling removal!\n"); - - /* - * get a module reference to avoid doing this - * while unloading anyway and to avoid - * scheduling a work with code that's being - * removed. - */ - if (!try_module_get(THIS_MODULE)) { - IWL_ERR(trans, - "Module is being unloaded - abort\n"); - goto err; - } - - removal = kzalloc(sizeof(*removal), GFP_ATOMIC); - if (!removal) { - module_put(THIS_MODULE); - goto err; - } - /* - * we don't need to clear this flag, because - * the trans will be freed and reallocated. - */ - set_bit(STATUS_TRANS_DEAD, &trans->status); - - removal->pdev = to_pci_dev(trans->dev); - INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk); - pci_dev_get(removal->pdev); - schedule_work(&removal->work); - } else { + if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) + iwl_trans_pcie_remove(trans, false); + else iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_FORCE_NMI); - } -err: spin_unlock(&trans_pcie->reg_lock); return false; } diff --git a/drivers/net/wireless/intersil/p54/eeprom.h b/drivers/net/wireless/intersil/p54/eeprom.h index 1d0aaf54389a..641c4e79879e 100644 --- a/drivers/net/wireless/intersil/p54/eeprom.h +++ b/drivers/net/wireless/intersil/p54/eeprom.h @@ -108,10 +108,10 @@ struct pda_country { } __packed; struct pda_antenna_gain { - struct { + DECLARE_FLEX_ARRAY(struct { u8 gain_5GHz; /* 0.25 dBi units */ u8 gain_2GHz; /* 0.25 dBi units */ - } __packed antenna[0]; + } __packed, antenna); } __packed; struct pda_custom_wrapper { diff --git a/drivers/net/wireless/mediatek/mt76/Kconfig b/drivers/net/wireless/mediatek/mt76/Kconfig index 9ff43f1fc50d..d7f90a0eb21e 100644 --- a/drivers/net/wireless/mediatek/mt76/Kconfig +++ b/drivers/net/wireless/mediatek/mt76/Kconfig @@ -34,3 +34,4 @@ source "drivers/net/wireless/mediatek/mt76/mt7603/Kconfig" source "drivers/net/wireless/mediatek/mt76/mt7615/Kconfig" source "drivers/net/wireless/mediatek/mt76/mt7915/Kconfig" source "drivers/net/wireless/mediatek/mt76/mt7921/Kconfig" +source "drivers/net/wireless/mediatek/mt76/mt7996/Kconfig" diff --git a/drivers/net/wireless/mediatek/mt76/Makefile b/drivers/net/wireless/mediatek/mt76/Makefile index c78ae4b89761..84c99b7e57f9 100644 --- a/drivers/net/wireless/mediatek/mt76/Makefile +++ b/drivers/net/wireless/mediatek/mt76/Makefile @@ -35,3 +35,4 @@ obj-$(CONFIG_MT7603E) += mt7603/ obj-$(CONFIG_MT7615_COMMON) += mt7615/ obj-$(CONFIG_MT7915E) += mt7915/ obj-$(CONFIG_MT7921_COMMON) += mt7921/ +obj-$(CONFIG_MT7996E) += mt7996/ diff --git a/drivers/net/wireless/mediatek/mt76/debugfs.c b/drivers/net/wireless/mediatek/mt76/debugfs.c index 47e9911ee9fe..11b0b3d62f29 100644 --- a/drivers/net/wireless/mediatek/mt76/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/debugfs.c @@ -100,23 +100,6 @@ void mt76_seq_puts_array(struct seq_file *file, const char *str, } EXPORT_SYMBOL_GPL(mt76_seq_puts_array); -static int mt76_read_rate_txpower(struct seq_file *s, void *data) -{ - struct mt76_dev *dev = dev_get_drvdata(s->private); - - mt76_seq_puts_array(s, "CCK", dev->rate_power.cck, - ARRAY_SIZE(dev->rate_power.cck)); - mt76_seq_puts_array(s, "OFDM", dev->rate_power.ofdm, - ARRAY_SIZE(dev->rate_power.ofdm)); - mt76_seq_puts_array(s, "STBC", dev->rate_power.stbc, - ARRAY_SIZE(dev->rate_power.stbc)); - mt76_seq_puts_array(s, "HT", dev->rate_power.ht, - ARRAY_SIZE(dev->rate_power.ht)); - mt76_seq_puts_array(s, "VHT", dev->rate_power.vht, - ARRAY_SIZE(dev->rate_power.vht)); - return 0; -} - struct dentry * mt76_register_debugfs_fops(struct mt76_phy *phy, const struct file_operations *ops) @@ -137,8 +120,6 @@ mt76_register_debugfs_fops(struct mt76_phy *phy, debugfs_create_blob("eeprom", 0400, dir, &dev->eeprom); if (dev->otp.data) debugfs_create_blob("otp", 0400, dir, &dev->otp); - debugfs_create_devm_seqfile(dev->dev, "rate_txpower", dir, - mt76_read_rate_txpower); debugfs_create_devm_seqfile(dev->dev, "rx-queues", dir, mt76_rx_queues_read); diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index 28cbf07046c3..f795548562f5 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -60,6 +60,19 @@ mt76_alloc_txwi(struct mt76_dev *dev) } static struct mt76_txwi_cache * +mt76_alloc_rxwi(struct mt76_dev *dev) +{ + struct mt76_txwi_cache *t; + + t = kzalloc(L1_CACHE_ALIGN(sizeof(*t)), GFP_ATOMIC); + if (!t) + return NULL; + + t->ptr = NULL; + return t; +} + +static struct mt76_txwi_cache * __mt76_get_txwi(struct mt76_dev *dev) { struct mt76_txwi_cache *t = NULL; @@ -76,6 +89,22 @@ __mt76_get_txwi(struct mt76_dev *dev) } static struct mt76_txwi_cache * +__mt76_get_rxwi(struct mt76_dev *dev) +{ + struct mt76_txwi_cache *t = NULL; + + spin_lock(&dev->wed_lock); + if (!list_empty(&dev->rxwi_cache)) { + t = list_first_entry(&dev->rxwi_cache, struct mt76_txwi_cache, + list); + list_del(&t->list); + } + spin_unlock(&dev->wed_lock); + + return t; +} + +static struct mt76_txwi_cache * mt76_get_txwi(struct mt76_dev *dev) { struct mt76_txwi_cache *t = __mt76_get_txwi(dev); @@ -86,6 +115,18 @@ mt76_get_txwi(struct mt76_dev *dev) return mt76_alloc_txwi(dev); } +struct mt76_txwi_cache * +mt76_get_rxwi(struct mt76_dev *dev) +{ + struct mt76_txwi_cache *t = __mt76_get_rxwi(dev); + + if (t) + return t; + + return mt76_alloc_rxwi(dev); +} +EXPORT_SYMBOL_GPL(mt76_get_rxwi); + void mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) { @@ -98,6 +139,18 @@ mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) } EXPORT_SYMBOL_GPL(mt76_put_txwi); +void +mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) +{ + if (!t) + return; + + spin_lock(&dev->wed_lock); + list_add(&t->list, &dev->rxwi_cache); + spin_unlock(&dev->wed_lock); +} +EXPORT_SYMBOL_GPL(mt76_put_rxwi); + static void mt76_free_pending_txwi(struct mt76_dev *dev) { @@ -113,6 +166,20 @@ mt76_free_pending_txwi(struct mt76_dev *dev) } static void +mt76_free_pending_rxwi(struct mt76_dev *dev) +{ + struct mt76_txwi_cache *t; + + local_bh_disable(); + while ((t = __mt76_get_rxwi(dev)) != NULL) { + if (t->ptr) + skb_free_frag(t->ptr); + kfree(t); + } + local_bh_enable(); +} + +static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { Q_WRITE(dev, q, desc_base, q->desc_dma); @@ -148,11 +215,6 @@ mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, u32 ctrl; int i, idx = -1; - if (txwi) { - q->entry[q->head].txwi = DMA_DUMMY_DATA; - q->entry[q->head].skip_buf0 = true; - } - for (i = 0; i < nbufs; i += 2, buf += 2) { u32 buf0 = buf[0].addr, buf1 = 0; @@ -162,28 +224,48 @@ mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, desc = &q->desc[idx]; entry = &q->entry[idx]; - if (buf[0].skip_unmap) - entry->skip_buf0 = true; - entry->skip_buf1 = i == nbufs - 1; - - entry->dma_addr[0] = buf[0].addr; - entry->dma_len[0] = buf[0].len; - - ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); - if (i < nbufs - 1) { - entry->dma_addr[1] = buf[1].addr; - entry->dma_len[1] = buf[1].len; - buf1 = buf[1].addr; - ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); - if (buf[1].skip_unmap) - entry->skip_buf1 = true; + if ((q->flags & MT_QFLAG_WED) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) { + struct mt76_txwi_cache *t = txwi; + int rx_token; + + if (!t) + return -ENOMEM; + + rx_token = mt76_rx_token_consume(dev, (void *)skb, t, + buf[0].addr); + buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token); + ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len) | + MT_DMA_CTL_TO_HOST; + } else { + if (txwi) { + q->entry[q->head].txwi = DMA_DUMMY_DATA; + q->entry[q->head].skip_buf0 = true; + } + + if (buf[0].skip_unmap) + entry->skip_buf0 = true; + entry->skip_buf1 = i == nbufs - 1; + + entry->dma_addr[0] = buf[0].addr; + entry->dma_len[0] = buf[0].len; + + ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); + if (i < nbufs - 1) { + entry->dma_addr[1] = buf[1].addr; + entry->dma_len[1] = buf[1].len; + buf1 = buf[1].addr; + ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); + if (buf[1].skip_unmap) + entry->skip_buf1 = true; + } + + if (i == nbufs - 1) + ctrl |= MT_DMA_CTL_LAST_SEC0; + else if (i == nbufs - 2) + ctrl |= MT_DMA_CTL_LAST_SEC1; } - if (i == nbufs - 1) - ctrl |= MT_DMA_CTL_LAST_SEC0; - else if (i == nbufs - 2) - ctrl |= MT_DMA_CTL_LAST_SEC1; - WRITE_ONCE(desc->buf0, cpu_to_le32(buf0)); WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); WRITE_ONCE(desc->info, cpu_to_le32(info)); @@ -272,33 +354,60 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) static void * mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, - int *len, u32 *info, bool *more) + int *len, u32 *info, bool *more, bool *drop) { struct mt76_queue_entry *e = &q->entry[idx]; struct mt76_desc *desc = &q->desc[idx]; - dma_addr_t buf_addr; - void *buf = e->buf; - int buf_len = SKB_WITH_OVERHEAD(q->buf_size); + void *buf; - buf_addr = e->dma_addr[0]; if (len) { - u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl)); - *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl); - *more = !(ctl & MT_DMA_CTL_LAST_SEC0); + u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); + *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl); + *more = !(ctrl & MT_DMA_CTL_LAST_SEC0); } if (info) *info = le32_to_cpu(desc->info); - dma_unmap_single(dev->dma_dev, buf_addr, buf_len, DMA_FROM_DEVICE); - e->buf = NULL; + if ((q->flags & MT_QFLAG_WED) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) { + u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, + le32_to_cpu(desc->buf1)); + struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token); + + if (!t) + return NULL; + + dma_unmap_single(dev->dma_dev, t->dma_addr, + SKB_WITH_OVERHEAD(q->buf_size), + DMA_FROM_DEVICE); + + buf = t->ptr; + t->dma_addr = 0; + t->ptr = NULL; + + mt76_put_rxwi(dev, t); + + if (drop) { + u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); + + *drop = !!(ctrl & (MT_DMA_CTL_TO_HOST_A | + MT_DMA_CTL_DROP)); + } + } else { + buf = e->buf; + e->buf = NULL; + dma_unmap_single(dev->dma_dev, e->dma_addr[0], + SKB_WITH_OVERHEAD(q->buf_size), + DMA_FROM_DEVICE); + } return buf; } static void * mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, - int *len, u32 *info, bool *more) + int *len, u32 *info, bool *more, bool *drop) { int idx = q->tail; @@ -314,7 +423,7 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->tail = (q->tail + 1) % q->ndesc; q->queued--; - return mt76_dma_get_buf(dev, q, idx, len, info, more); + return mt76_dma_get_buf(dev, q, idx, len, info, more, drop); } static int @@ -441,14 +550,26 @@ free_skb: return ret; } +static struct page_frag_cache * +mt76_dma_rx_get_frag_cache(struct mt76_dev *dev, struct mt76_queue *q) +{ + struct page_frag_cache *rx_page = &q->rx_page; + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + if ((q->flags & MT_QFLAG_WED) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) + rx_page = &dev->mmio.wed.rx_buf_ring.rx_page; +#endif + return rx_page; +} + static int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q) { - dma_addr_t addr; - void *buf; - int frames = 0; + struct page_frag_cache *rx_page = mt76_dma_rx_get_frag_cache(dev, q); int len = SKB_WITH_OVERHEAD(q->buf_size); - int offset = q->buf_offset; + int frames = 0, offset = q->buf_offset; + dma_addr_t addr; if (!q->ndesc) return 0; @@ -456,9 +577,18 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q) spin_lock_bh(&q->lock); while (q->queued < q->ndesc - 1) { + struct mt76_txwi_cache *t = NULL; struct mt76_queue_buf qbuf; + void *buf = NULL; - buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC); + if ((q->flags & MT_QFLAG_WED) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) { + t = mt76_get_rxwi(dev); + if (!t) + break; + } + + buf = page_frag_alloc(rx_page, q->buf_size, GFP_ATOMIC); if (!buf) break; @@ -471,7 +601,7 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q) qbuf.addr = addr + offset; qbuf.len = len - offset; qbuf.skip_unmap = false; - mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL); + mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, t); frames++; } @@ -517,6 +647,11 @@ mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q) if (!ret) q->wed_regs = wed->txfree_ring.reg_base; break; + case MT76_WED_Q_RX: + ret = mtk_wed_device_rx_ring_setup(wed, ring, q->regs, false); + if (!ret) + q->wed_regs = wed->rx_ring[ring].reg_base; + break; default: ret = -EINVAL; } @@ -574,7 +709,7 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) spin_lock_bh(&q->lock); do { - buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more); + buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL); if (!buf) break; @@ -615,7 +750,7 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) static void mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, - int len, bool more) + int len, bool more, u32 info) { struct sk_buff *skb = q->rx_head; struct skb_shared_info *shinfo = skb_shinfo(skb); @@ -635,7 +770,7 @@ mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, q->rx_head = NULL; if (nr_frags < ARRAY_SIZE(shinfo->frags)) - dev->drv->rx_skb(dev, q - dev->q_rx, skb); + dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); else dev_kfree_skb(skb); } @@ -656,6 +791,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) } while (done < budget) { + bool drop = false; u32 info; if (check_ddone) { @@ -666,10 +802,14 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) break; } - data = mt76_dma_dequeue(dev, q, false, &len, &info, &more); + data = mt76_dma_dequeue(dev, q, false, &len, &info, &more, + &drop); if (!data) break; + if (drop) + goto free_frag; + if (q->rx_head) data_len = q->buf_size; else @@ -682,7 +822,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) } if (q->rx_head) { - mt76_add_fragment(dev, q, data, len, more); + mt76_add_fragment(dev, q, data, len, more, info); continue; } @@ -706,7 +846,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) continue; } - dev->drv->rx_skb(dev, q - dev->q_rx, skb); + dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); continue; free_frag: @@ -803,11 +943,15 @@ void mt76_dma_cleanup(struct mt76_dev *dev) mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true); mt76_for_each_q_rx(dev, i) { + struct mt76_queue *q = &dev->q_rx[i]; + netif_napi_del(&dev->napi[i]); - mt76_dma_rx_cleanup(dev, &dev->q_rx[i]); + if (FIELD_GET(MT_QFLAG_WED_TYPE, q->flags)) + mt76_dma_rx_cleanup(dev, q); } mt76_free_pending_txwi(dev); + mt76_free_pending_rxwi(dev); if (mtk_wed_device_active(&dev->mmio.wed)) mtk_wed_device_detach(&dev->mmio.wed); diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index fdf786f975ea..53c6ce2528b2 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -15,6 +15,14 @@ #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) +#define MT_DMA_CTL_TO_HOST BIT(8) +#define MT_DMA_CTL_TO_HOST_A BIT(12) +#define MT_DMA_CTL_DROP BIT(14) +#define MT_DMA_CTL_TOKEN GENMASK(31, 16) + +#define MT_DMA_PPE_CPU_REASON GENMASK(15, 11) +#define MT_DMA_PPE_ENTRY GENMASK(30, 16) +#define MT_DMA_INFO_PPE_VLD BIT(31) #define MT_DMA_HDR_LEN 4 #define MT_RX_INFO_LEN 4 diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 6de13d641438..fc608b369b3c 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -443,8 +443,12 @@ mt76_phy_init(struct mt76_phy *phy, struct ieee80211_hw *hw) ieee80211_hw_set(hw, SUPPORTS_CLONED_SKBS); ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU); ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER); - ieee80211_hw_set(hw, TX_AMSDU); - ieee80211_hw_set(hw, TX_FRAG_LIST); + + if (!(dev->drv->drv_flags & MT_DRV_AMSDU_OFFLOAD)) { + ieee80211_hw_set(hw, TX_AMSDU); + ieee80211_hw_set(hw, TX_FRAG_LIST); + } + ieee80211_hw_set(hw, MFP_CAPABLE); ieee80211_hw_set(hw, AP_LINK_PS); ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS); @@ -568,6 +572,7 @@ mt76_alloc_device(struct device *pdev, unsigned int size, spin_lock_init(&dev->lock); spin_lock_init(&dev->cc_lock); spin_lock_init(&dev->status_lock); + spin_lock_init(&dev->wed_lock); mutex_init(&dev->mutex); init_waitqueue_head(&dev->tx_wait); @@ -590,9 +595,13 @@ mt76_alloc_device(struct device *pdev, unsigned int size, spin_lock_init(&dev->token_lock); idr_init(&dev->token); + spin_lock_init(&dev->rx_token_lock); + idr_init(&dev->rx_token); + INIT_LIST_HEAD(&dev->wcid_list); INIT_LIST_HEAD(&dev->txwi_cache); + INIT_LIST_HEAD(&dev->rxwi_cache); dev->token_size = dev->drv->token_size; for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) @@ -947,14 +956,12 @@ void mt76_wcid_key_setup(struct mt76_dev *dev, struct mt76_wcid *wcid, } EXPORT_SYMBOL(mt76_wcid_key_setup); -static int -mt76_rx_signal(struct mt76_rx_status *status) +int mt76_rx_signal(u8 chain_mask, s8 *chain_signal) { - s8 *chain_signal = status->chain_signal; int signal = -128; u8 chains; - for (chains = status->chains; chains; chains >>= 1, chain_signal++) { + for (chains = chain_mask; chains; chains >>= 1, chain_signal++) { int cur, diff; cur = *chain_signal; @@ -976,6 +983,7 @@ mt76_rx_signal(struct mt76_rx_status *status) return signal; } +EXPORT_SYMBOL(mt76_rx_signal); static void mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb, @@ -1005,7 +1013,7 @@ mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb, status->ampdu_reference = mstat.ampdu_ref; status->device_timestamp = mstat.timestamp; status->mactime = mstat.timestamp; - status->signal = mt76_rx_signal(&mstat); + status->signal = mt76_rx_signal(mstat.chains, mstat.chain_signal); if (status->signal <= -128) status->flag |= RX_FLAG_NO_SIGNAL_VAL; @@ -1289,7 +1297,10 @@ void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q, while ((skb = __skb_dequeue(&dev->rx_skb[q])) != NULL) { mt76_check_sta(dev, skb); - mt76_rx_aggr_reorder(skb, &frames); + if (mtk_wed_device_active(&dev->mmio.wed)) + __skb_queue_tail(&frames, skb); + else + mt76_rx_aggr_reorder(skb, &frames); } mt76_rx_complete(dev, &frames, napi); diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 87db9498dea4..32a77a0ae9da 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -35,6 +35,7 @@ FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ FIELD_PREP(MT_QFLAG_WED_RING, _n)) #define MT_WED_Q_TX(_n) __MT_WED_Q(MT76_WED_Q_TX, _n) +#define MT_WED_Q_RX(_n) __MT_WED_Q(MT76_WED_Q_RX, _n) #define MT_WED_Q_TXFREE __MT_WED_Q(MT76_WED_Q_TXFREE, 0) struct mt76_dev; @@ -56,6 +57,7 @@ enum mt76_bus_type { enum mt76_wed_type { MT76_WED_Q_TX, MT76_WED_Q_TXFREE, + MT76_WED_Q_RX, }; struct mt76_bus_ops { @@ -271,9 +273,15 @@ struct mt76_sta_stats { u64 tx_nss[4]; /* 1, 2, 3, 4 */ u64 tx_mcs[16]; /* mcs idx */ u64 tx_bytes; + /* WED TX */ u32 tx_packets; u32 tx_retries; u32 tx_failed; + /* WED RX */ + u64 rx_bytes; + u32 rx_packets; + u32 rx_errors; + u32 rx_drops; }; enum mt76_wcid_flags { @@ -339,7 +347,10 @@ struct mt76_txwi_cache { struct list_head list; dma_addr_t dma_addr; - struct sk_buff *skb; + union { + struct sk_buff *skb; + void *ptr; + }; }; struct mt76_rx_tid { @@ -415,6 +426,7 @@ struct mt76_hw_cap { #define MT_DRV_SW_RX_AIRTIME BIT(2) #define MT_DRV_RX_DMA_HDR BIT(3) #define MT_DRV_HW_MGMT_TXQ BIT(4) +#define MT_DRV_AMSDU_OFFLOAD BIT(5) struct mt76_driver_ops { u32 drv_flags; @@ -438,7 +450,7 @@ struct mt76_driver_ops { bool (*rx_check)(struct mt76_dev *dev, void *data, int len); void (*rx_skb)(struct mt76_dev *dev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); void (*rx_poll_complete)(struct mt76_dev *dev, enum mt76_rxq_id q); @@ -470,19 +482,6 @@ struct mt76_sband { struct mt76_channel_state *chan; }; -struct mt76_rate_power { - union { - struct { - s8 cck[4]; - s8 ofdm[8]; - s8 stbc[10]; - s8 ht[16]; - s8 vht[10]; - }; - s8 all[48]; - }; -}; - /* addr req mask */ #define MT_VEND_TYPE_EEPROM BIT(31) #define MT_VEND_TYPE_CFG BIT(30) @@ -705,6 +704,8 @@ struct mt76_phy { enum mt76_dfs_state dfs_state; ktime_t survey_time; + u32 aggr_stats[32]; + struct mt76_hw_cap cap; struct mt76_sband sband_2g; struct mt76_sband sband_5g; @@ -738,6 +739,7 @@ struct mt76_dev { struct ieee80211_hw *hw; + spinlock_t wed_lock; spinlock_t lock; spinlock_t cc_lock; @@ -764,6 +766,7 @@ struct mt76_dev { struct sk_buff_head rx_skb[__MT_RXQ_MAX]; struct list_head txwi_cache; + struct list_head rxwi_cache; struct mt76_queue *q_mcu[__MT_MCUQ_MAX]; struct mt76_queue q_rx[__MT_RXQ_MAX]; const struct mt76_queue_ops *queue_ops; @@ -778,6 +781,10 @@ struct mt76_dev { u16 token_count; u16 token_size; + spinlock_t rx_token_lock; + struct idr rx_token; + u16 rx_token_size; + wait_queue_head_t tx_wait; /* spinclock used to protect wcid pktid linked list */ spinlock_t status_lock; @@ -793,8 +800,6 @@ struct mt76_dev { u32 rev; - u32 aggr_stats[32]; - struct tasklet_struct pre_tbtt_tasklet; int beacon_int; u8 beacon_mask; @@ -802,8 +807,6 @@ struct mt76_dev { struct debugfs_blob_wrapper eeprom; struct debugfs_blob_wrapper otp; - struct mt76_rate_power rate_power; - char alpha2[3]; enum nl80211_dfs_regions region; @@ -1107,8 +1110,9 @@ static inline bool mt76_is_skb_pktid(u8 pktid) static inline u8 mt76_tx_power_nss_delta(u8 nss) { static const u8 nss_delta[4] = { 0, 6, 9, 12 }; + u8 idx = nss - 1; - return nss_delta[nss - 1]; + return (idx < ARRAY_SIZE(nss_delta)) ? nss_delta[idx] : 0; } static inline bool mt76_testmode_enabled(struct mt76_phy *phy) @@ -1163,6 +1167,7 @@ void mt76_update_survey(struct mt76_phy *phy); void mt76_update_survey_active_time(struct mt76_phy *phy, ktime_t time); int mt76_get_survey(struct ieee80211_hw *hw, int idx, struct survey_info *survey); +int mt76_rx_signal(u8 chain_mask, s8 *chain_signal); void mt76_set_stream_caps(struct mt76_phy *phy, bool vht); int mt76_rx_aggr_start(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tid, @@ -1260,6 +1265,8 @@ mt76_tx_status_get_hw(struct mt76_dev *dev, struct sk_buff *skb) } void mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t); +void mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t); +struct mt76_txwi_cache *mt76_get_rxwi(struct mt76_dev *dev); void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames, struct napi_struct *napi); void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q, @@ -1404,6 +1411,9 @@ struct mt76_txwi_cache * mt76_token_release(struct mt76_dev *dev, int token, bool *wake); int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi); void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked); +struct mt76_txwi_cache *mt76_rx_token_release(struct mt76_dev *dev, int token); +int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, + struct mt76_txwi_cache *r, dma_addr_t phys); static inline void mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7603/debugfs.c index f52165dff422..3967f2f05774 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/debugfs.c @@ -85,7 +85,7 @@ mt7603_ampdu_stat_show(struct seq_file *file, void *data) bound[i], bound[i + 1]); seq_puts(file, "\nCount: "); for (i = 0; i < ARRAY_SIZE(bound); i++) - seq_printf(file, "%8d | ", dev->mt76.aggr_stats[i]); + seq_printf(file, "%8d | ", dev->mphy.aggr_stats[i]); seq_puts(file, "\n"); return 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/dma.c b/drivers/net/wireless/mediatek/mt76/mt7603/dma.c index f9e5857850e7..03ba11a61c90 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/dma.c @@ -69,7 +69,7 @@ free: } void mt7603_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) + struct sk_buff *skb, u32 *info) { struct mt7603_dev *dev = container_of(mdev, struct mt7603_dev, mt76); __le32 *rxd = (__le32 *)skb->data; diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c index 49a511ae8161..70a7f84af028 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c @@ -39,7 +39,7 @@ void mt7603_mac_reset_counters(struct mt7603_dev *dev) for (i = 0; i < 2; i++) mt76_rr(dev, MT_TX_AGG_CNT(i)); - memset(dev->mt76.aggr_stats, 0, sizeof(dev->mt76.aggr_stats)); + memset(dev->mphy.aggr_stats, 0, sizeof(dev->mphy.aggr_stats)); } void mt7603_mac_set_timing(struct mt7603_dev *dev) @@ -1827,8 +1827,8 @@ void mt7603_mac_work(struct work_struct *work) for (i = 0, idx = 0; i < 2; i++) { u32 val = mt76_rr(dev, MT_TX_AGG_CNT(i)); - dev->mt76.aggr_stats[idx++] += val & 0xffff; - dev->mt76.aggr_stats[idx++] += val >> 16; + dev->mphy.aggr_stats[idx++] += val & 0xffff; + dev->mphy.aggr_stats[idx++] += val >> 16; } if (dev->mphy.mac_work_count == 10) diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/mt7603.h b/drivers/net/wireless/mediatek/mt76/mt7603/mt7603.h index 0fd46d907638..7c3be596da09 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/mt7603.h +++ b/drivers/net/wireless/mediatek/mt76/mt7603/mt7603.h @@ -244,7 +244,7 @@ int mt7603_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, void mt7603_tx_complete_skb(struct mt76_dev *mdev, struct mt76_queue_entry *e); void mt7603_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); void mt7603_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q); void mt7603_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); int mt7603_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c index c26b45a09923..2a6d317db5e0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c @@ -278,7 +278,6 @@ mt7615_ampdu_stat_read_phy(struct mt7615_phy *phy, { struct mt7615_dev *dev = file->private; u32 reg = is_mt7663(&dev->mt76) ? MT_MIB_ARNG(0) : MT_AGG_ASRCR0; - bool ext_phy = phy != &dev->phy; int bound[7], i, range; if (!phy) @@ -292,7 +291,7 @@ mt7615_ampdu_stat_read_phy(struct mt7615_phy *phy, for (i = 0; i < 3; i++) bound[i + 4] = MT_AGG_ASRCR_RANGE(range, i) + 1; - seq_printf(file, "\nPhy %d\n", ext_phy); + seq_printf(file, "\nPhy %d\n", phy != &dev->phy); seq_printf(file, "Length: %8d | ", bound[0]); for (i = 0; i < ARRAY_SIZE(bound) - 1; i++) @@ -300,9 +299,8 @@ mt7615_ampdu_stat_read_phy(struct mt7615_phy *phy, bound[i], bound[i + 1]); seq_puts(file, "\nCount: "); - range = ext_phy ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0; i < ARRAY_SIZE(bound); i++) - seq_printf(file, "%8d | ", dev->mt76.aggr_stats[i + range]); + seq_printf(file, "%8d | ", phy->mt76->aggr_stats[i]); seq_puts(file, "\n"); seq_printf(file, "BA miss count: %d\n", phy->mib.ba_miss_cnt); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index 2ce1705c0f43..a95602473359 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -107,9 +107,9 @@ static struct mt76_wcid *mt7615_rx_get_wcid(struct mt7615_dev *dev, return &sta->vif->sta.wcid; } -void mt7615_mac_reset_counters(struct mt7615_dev *dev) +void mt7615_mac_reset_counters(struct mt7615_phy *phy) { - struct mt76_phy *mphy_ext = dev->mt76.phys[MT_BAND1]; + struct mt7615_dev *dev = phy->dev; int i; for (i = 0; i < 4; i++) { @@ -117,10 +117,8 @@ void mt7615_mac_reset_counters(struct mt7615_dev *dev) mt76_rr(dev, MT_TX_AGG_CNT(1, i)); } - memset(dev->mt76.aggr_stats, 0, sizeof(dev->mt76.aggr_stats)); - dev->mt76.phy.survey_time = ktime_get_boottime(); - if (mphy_ext) - mphy_ext->survey_time = ktime_get_boottime(); + memset(phy->mt76->aggr_stats, 0, sizeof(phy->mt76->aggr_stats)); + phy->mt76->survey_time = ktime_get_boottime(); /* reset airtime counters */ mt76_rr(dev, MT_MIB_SDR9(0)); @@ -1177,6 +1175,21 @@ void mt7615_mac_set_rates(struct mt7615_phy *phy, struct mt7615_sta *sta, } EXPORT_SYMBOL_GPL(mt7615_mac_set_rates); +void mt7615_mac_enable_rtscts(struct mt7615_dev *dev, + struct ieee80211_vif *vif, bool enable) +{ + struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; + u32 addr; + + addr = mt7615_mac_wtbl_addr(dev, mvif->sta.wcid.idx) + 3 * 4; + + if (enable) + mt76_set(dev, addr, MT_WTBL_W3_RTS); + else + mt76_clear(dev, addr, MT_WTBL_W3_RTS); +} +EXPORT_SYMBOL_GPL(mt7615_mac_enable_rtscts); + static int mt7615_mac_wtbl_update_key(struct mt7615_dev *dev, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, @@ -1653,7 +1666,7 @@ bool mt7615_rx_check(struct mt76_dev *mdev, void *data, int len) EXPORT_SYMBOL_GPL(mt7615_rx_check); void mt7615_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) + struct sk_buff *skb, u32 *info) { struct mt7615_dev *dev = container_of(mdev, struct mt7615_dev, mt76); __le32 *rxd = (__le32 *)skb->data; @@ -1999,7 +2012,7 @@ mt7615_mac_update_mib_stats(struct mt7615_phy *phy) struct mt7615_dev *dev = phy->dev; struct mib_stats *mib = &phy->mib; bool ext_phy = phy != &dev->phy; - int i, aggr; + int i, aggr = 0; u32 val, val2; mib->fcs_err_cnt += mt76_get_field(dev, MT_MIB_SDR3(ext_phy), @@ -2013,7 +2026,6 @@ mt7615_mac_update_mib_stats(struct mt7615_phy *phy) mib->aggr_per = 1000 * (val - val2) / val; } - aggr = ext_phy ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0; i < 4; i++) { val = mt76_rr(dev, MT_MIB_MB_SDR1(ext_phy, i)); mib->ba_miss_cnt += FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); @@ -2026,8 +2038,8 @@ mt7615_mac_update_mib_stats(struct mt7615_phy *phy) val); val = mt76_rr(dev, MT_TX_AGG_CNT(ext_phy, i)); - dev->mt76.aggr_stats[aggr++] += val & 0xffff; - dev->mt76.aggr_stats[aggr++] += val >> 16; + phy->mt76->aggr_stats[aggr++] += val & 0xffff; + phy->mt76->aggr_stats[aggr++] += val >> 16; } } diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index 8d4733f87cda..ab4c1b4478aa 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -83,7 +83,7 @@ static int mt7615_start(struct ieee80211_hw *hw) ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, timeout); if (!running) - mt7615_mac_reset_counters(dev); + mt7615_mac_reset_counters(phy); out: mt7615_mutex_release(dev); @@ -320,7 +320,7 @@ int mt7615_set_channel(struct mt7615_phy *phy) if (ret) goto out; - mt7615_mac_reset_counters(dev); + mt7615_mac_reset_counters(phy); phy->noise = 0; phy->chfreq = mt76_rr(dev, MT_CHFREQ(ext_phy)); @@ -572,6 +572,9 @@ static void mt7615_bss_info_changed(struct ieee80211_hw *hw, } } + if (changed & BSS_CHANGED_ERP_CTS_PROT) + mt7615_mac_enable_rtscts(dev, vif, info->use_cts_prot); + if (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon) { mt7615_mcu_add_bss_info(phy, vif, NULL, true); mt7615_mcu_sta_add(phy, vif, NULL, true); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c index 3dac76e6df4d..83f30305414d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c @@ -1119,7 +1119,7 @@ mt7615_mcu_uni_add_bss(struct mt7615_phy *phy, struct ieee80211_vif *vif, struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; return mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, - enable); + enable, NULL); } static inline int diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h index 060d52c81d9e..087d4886162e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h @@ -469,10 +469,12 @@ void mt7615_init_work(struct mt7615_dev *dev); int mt7615_mcu_restart(struct mt76_dev *dev); void mt7615_update_channel(struct mt76_phy *mphy); bool mt7615_mac_wtbl_update(struct mt7615_dev *dev, int idx, u32 mask); -void mt7615_mac_reset_counters(struct mt7615_dev *dev); +void mt7615_mac_reset_counters(struct mt7615_phy *phy); void mt7615_mac_cca_stats_reset(struct mt7615_phy *phy); void mt7615_mac_set_scs(struct mt7615_phy *phy, bool enable); void mt7615_mac_enable_nf(struct mt7615_dev *dev, bool ext_phy); +void mt7615_mac_enable_rtscts(struct mt7615_dev *dev, + struct ieee80211_vif *vif, bool enable); void mt7615_mac_sta_poll(struct mt7615_dev *dev); int mt7615_mac_write_txwi(struct mt7615_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, @@ -511,7 +513,7 @@ void mt7615_tx_worker(struct mt76_worker *w); void mt7615_tx_token_put(struct mt7615_dev *dev); bool mt7615_rx_check(struct mt76_dev *mdev, void *data, int len); void mt7615_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); void mt7615_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); int mt7615_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h index 6712ad9faeaa..fa1b9b26b399 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h @@ -446,6 +446,8 @@ enum mt7615_reg_base { #define MT_WTBL_RIUCR3_RATE6 GENMASK(19, 8) #define MT_WTBL_RIUCR3_RATE7 GENMASK(31, 20) +#define MT_WTBL_W3_RTS BIT(22) + #define MT_WTBL_W5_CHANGE_BW_RATE GENMASK(7, 5) #define MT_WTBL_W5_SHORT_GI_20 BIT(8) #define MT_WTBL_W5_SHORT_GI_40 BIT(9) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 635192c878cb..8ba883b03e50 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -187,6 +187,11 @@ static inline bool is_mt7986(struct mt76_dev *dev) return mt76_chip(dev) == 0x7986; } +static inline bool is_mt7996(struct mt76_dev *dev) +{ + return mt76_chip(dev) == 0x7990; +} + static inline bool is_mt7622(struct mt76_dev *dev) { if (!IS_ENABLED(CONFIG_MT7622_WMAC)) @@ -261,6 +266,17 @@ mt76_connac_txwi_to_txp(struct mt76_dev *dev, struct mt76_txwi_cache *t) return (void *)(txwi + MT_TXD_SIZE); } +static inline u8 mt76_connac_spe_idx(u8 antenna_mask) +{ + static const u8 ant_to_spe[] = {0, 0, 1, 0, 3, 2, 4, 0, + 9, 8, 6, 10, 16, 12, 18, 0}; + + if (antenna_mask >= sizeof(ant_to_spe)) + return 0; + + return ant_to_spe[antenna_mask]; +} + int mt76_connac_pm_wake(struct mt76_phy *phy, struct mt76_connac_pm *pm); void mt76_connac_power_save_sched(struct mt76_phy *phy, struct mt76_connac_pm *pm); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 34ac3d81a510..fd60123fb284 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -417,9 +417,6 @@ mt76_connac2_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, if (ieee80211_is_beacon(fc)) { txwi[3] &= ~cpu_to_le32(MT_TXD3_SW_POWER_MGMT); txwi[3] |= cpu_to_le32(MT_TXD3_REM_TX_COUNT); - if (!is_mt7921(dev)) - txwi[7] |= cpu_to_le32(FIELD_PREP(MT_TXD7_SPE_IDX, - 0x18)); } if (info->flags & IEEE80211_TX_CTL_INJECTED) { @@ -550,6 +547,14 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, val |= FIELD_PREP(MT_TXD6_TX_RATE, rate); txwi[6] |= cpu_to_le32(val); txwi[3] |= cpu_to_le32(MT_TXD3_BA_DISABLE); + + if (!is_mt7921(dev)) { + u8 spe_idx = mt76_connac_spe_idx(mphy->antenna_mask); + + if (!spe_idx) + spe_idx = 24 + phy_idx; + txwi[7] |= cpu_to_le32(FIELD_PREP(MT_TXD7_SPE_IDX, spe_idx)); + } } } EXPORT_SYMBOL_GPL(mt76_connac2_mac_write_txwi); @@ -562,7 +567,7 @@ bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid, struct mt76_phy *mphy; struct rate_info rate = {}; bool cck = false; - u32 txrate, txs, mode; + u32 txrate, txs, mode, stbc; txs = le32_to_cpu(txs_data[0]); @@ -582,6 +587,10 @@ bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid, rate.mcs = FIELD_GET(MT_TX_RATE_IDX, txrate); rate.nss = FIELD_GET(MT_TX_RATE_NSS, txrate) + 1; + stbc = FIELD_GET(MT_TX_RATE_STBC, txrate); + + if (stbc && rate.nss > 1) + rate.nss >>= 1; if (rate.nss - 1 < ARRAY_SIZE(stats->tx_nss)) stats->tx_nss[rate.nss - 1]++; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 011fc9729b38..5a047e630860 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -65,7 +65,8 @@ int mt76_connac_mcu_init_download(struct mt76_dev *dev, u32 addr, u32 len, int cmd; if ((!is_connac_v1(dev) && addr == MCU_PATCH_ADDRESS) || - (is_mt7921(dev) && addr == 0x900000)) + (is_mt7921(dev) && addr == 0x900000) || + (is_mt7996(dev) && addr == 0x900000)) cmd = MCU_CMD(PATCH_START_REQ); else cmd = MCU_CMD(TARGET_ADDRESS_LEN_REQ); @@ -744,6 +745,39 @@ mt76_connac_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) he->pkt_ext = 2; } +static void +mt76_connac_mcu_sta_he_tlv_v2(struct sk_buff *skb, struct ieee80211_sta *sta) +{ + struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap; + struct ieee80211_he_cap_elem *elem = &he_cap->he_cap_elem; + struct sta_rec_he_v2 *he; + struct tlv *tlv; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HE_V2, sizeof(*he)); + + he = (struct sta_rec_he_v2 *)tlv; + memcpy(he->he_phy_cap, elem->phy_cap_info, sizeof(he->he_phy_cap)); + memcpy(he->he_mac_cap, elem->mac_cap_info, sizeof(he->he_mac_cap)); + + switch (sta->deflink.bandwidth) { + case IEEE80211_STA_RX_BW_160: + if (elem->phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G) + he->max_nss_mcs[CMD_HE_MCS_BW8080] = + he_cap->he_mcs_nss_supp.rx_mcs_80p80; + + he->max_nss_mcs[CMD_HE_MCS_BW160] = + he_cap->he_mcs_nss_supp.rx_mcs_160; + fallthrough; + default: + he->max_nss_mcs[CMD_HE_MCS_BW80] = + he_cap->he_mcs_nss_supp.rx_mcs_80; + break; + } + + he->pkt_ext = IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US; +} + static u8 mt76_connac_get_phy_mode_v2(struct mt76_phy *mphy, struct ieee80211_vif *vif, enum nl80211_band band, struct ieee80211_sta *sta) @@ -838,6 +872,7 @@ void mt76_connac_mcu_sta_tlv(struct mt76_phy *mphy, struct sk_buff *skb, /* starec he */ if (sta->deflink.he_cap.has_he) { mt76_connac_mcu_sta_he_tlv(skb, sta); + mt76_connac_mcu_sta_he_tlv_v2(skb, sta); if (band == NL80211_BAND_6GHZ && sta_state == MT76_STA_INFO_STATE_ASSOC) { struct sta_rec_he_6g_capa *he_6g_capa; @@ -1184,6 +1219,16 @@ void mt76_connac_mcu_sta_ba_tlv(struct sk_buff *skb, } EXPORT_SYMBOL_GPL(mt76_connac_mcu_sta_ba_tlv); +int mt76_connac_mcu_sta_wed_update(struct mt76_dev *dev, struct sk_buff *skb) +{ + if (!mtk_wed_device_active(&dev->mmio.wed)) + return 0; + + return mtk_wed_device_update_msg(&dev->mmio.wed, WED_WO_STA_REC, + skb->data, skb->len); +} +EXPORT_SYMBOL_GPL(mt76_connac_mcu_sta_wed_update); + int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, struct ieee80211_ampdu_params *params, int cmd, bool enable, bool tx) @@ -1209,6 +1254,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, mt76_connac_mcu_wtbl_ba_tlv(dev, skb, params, enable, tx, sta_wtbl, wtbl_hdr); + ret = mt76_connac_mcu_sta_wed_update(dev, skb); + if (ret) + return ret; + ret = mt76_mcu_skb_send_msg(dev, skb, cmd, true); if (ret) return ret; @@ -1219,6 +1268,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, mt76_connac_mcu_sta_ba_tlv(skb, params, enable, tx); + ret = mt76_connac_mcu_sta_wed_update(dev, skb); + if (ret) + return ret; + return mt76_mcu_skb_send_msg(dev, skb, cmd, true); } EXPORT_SYMBOL_GPL(mt76_connac_mcu_sta_ba); @@ -1313,13 +1366,10 @@ mt76_connac_mcu_uni_bss_he_tlv(struct mt76_phy *phy, struct ieee80211_vif *vif, he->max_nss_mcs[CMD_HE_MCS_BW8080] = cap->he_mcs_nss_supp.tx_mcs_80p80; } -int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, - struct ieee80211_vif *vif, - struct mt76_wcid *wcid, - bool enable) +int mt76_connac_mcu_uni_set_chctx(struct mt76_phy *phy, struct mt76_vif *mvif, + struct ieee80211_chanctx_conf *ctx) { - struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; - struct cfg80211_chan_def *chandef = &phy->chandef; + struct cfg80211_chan_def *chandef = ctx ? &ctx->def : &phy->chandef; int freq1 = chandef->center_freq1, freq2 = chandef->center_freq2; enum nl80211_band band = chandef->chan->band; struct mt76_dev *mdev = phy->dev; @@ -1328,34 +1378,6 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, u8 bss_idx; u8 pad[3]; } __packed hdr; - struct mt76_connac_bss_basic_tlv basic; - struct mt76_connac_bss_qos_tlv qos; - } basic_req = { - .hdr = { - .bss_idx = mvif->idx, - }, - .basic = { - .tag = cpu_to_le16(UNI_BSS_INFO_BASIC), - .len = cpu_to_le16(sizeof(struct mt76_connac_bss_basic_tlv)), - .bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int), - .dtim_period = vif->bss_conf.dtim_period, - .omac_idx = mvif->omac_idx, - .band_idx = mvif->band_idx, - .wmm_idx = mvif->wmm_idx, - .active = true, /* keep bss deactivated */ - .phymode = mt76_connac_get_phy_mode(phy, vif, band, NULL), - }, - .qos = { - .tag = cpu_to_le16(UNI_BSS_INFO_QBSS), - .len = cpu_to_le16(sizeof(struct mt76_connac_bss_qos_tlv)), - .qos = vif->bss_conf.qos, - }, - }; - struct { - struct { - u8 bss_idx; - u8 pad[3]; - } __packed hdr; struct rlm_tlv { __le16 tag; __le16 len; @@ -1388,6 +1410,82 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, .band = band, }, }; + + switch (chandef->width) { + case NL80211_CHAN_WIDTH_40: + rlm_req.rlm.bw = CMD_CBW_40MHZ; + break; + case NL80211_CHAN_WIDTH_80: + rlm_req.rlm.bw = CMD_CBW_80MHZ; + break; + case NL80211_CHAN_WIDTH_80P80: + rlm_req.rlm.bw = CMD_CBW_8080MHZ; + break; + case NL80211_CHAN_WIDTH_160: + rlm_req.rlm.bw = CMD_CBW_160MHZ; + break; + case NL80211_CHAN_WIDTH_5: + rlm_req.rlm.bw = CMD_CBW_5MHZ; + break; + case NL80211_CHAN_WIDTH_10: + rlm_req.rlm.bw = CMD_CBW_10MHZ; + break; + case NL80211_CHAN_WIDTH_20_NOHT: + case NL80211_CHAN_WIDTH_20: + default: + rlm_req.rlm.bw = CMD_CBW_20MHZ; + rlm_req.rlm.ht_op_info = 0; + break; + } + + if (rlm_req.rlm.control_channel < rlm_req.rlm.center_chan) + rlm_req.rlm.sco = 1; /* SCA */ + else if (rlm_req.rlm.control_channel > rlm_req.rlm.center_chan) + rlm_req.rlm.sco = 3; /* SCB */ + + return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(BSS_INFO_UPDATE), &rlm_req, + sizeof(rlm_req), true); +} +EXPORT_SYMBOL_GPL(mt76_connac_mcu_uni_set_chctx); + +int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, + struct ieee80211_vif *vif, + struct mt76_wcid *wcid, + bool enable, + struct ieee80211_chanctx_conf *ctx) +{ + struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; + struct cfg80211_chan_def *chandef = ctx ? &ctx->def : &phy->chandef; + enum nl80211_band band = chandef->chan->band; + struct mt76_dev *mdev = phy->dev; + struct { + struct { + u8 bss_idx; + u8 pad[3]; + } __packed hdr; + struct mt76_connac_bss_basic_tlv basic; + struct mt76_connac_bss_qos_tlv qos; + } basic_req = { + .hdr = { + .bss_idx = mvif->idx, + }, + .basic = { + .tag = cpu_to_le16(UNI_BSS_INFO_BASIC), + .len = cpu_to_le16(sizeof(struct mt76_connac_bss_basic_tlv)), + .bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int), + .dtim_period = vif->bss_conf.dtim_period, + .omac_idx = mvif->omac_idx, + .band_idx = mvif->band_idx, + .wmm_idx = mvif->wmm_idx, + .active = true, /* keep bss deactivated */ + .phymode = mt76_connac_get_phy_mode(phy, vif, band, NULL), + }, + .qos = { + .tag = cpu_to_le16(UNI_BSS_INFO_QBSS), + .len = cpu_to_le16(sizeof(struct mt76_connac_bss_qos_tlv)), + .qos = vif->bss_conf.qos, + }, + }; int err, conn_type; u8 idx, basic_phy; @@ -1474,40 +1572,7 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, return err; } - switch (chandef->width) { - case NL80211_CHAN_WIDTH_40: - rlm_req.rlm.bw = CMD_CBW_40MHZ; - break; - case NL80211_CHAN_WIDTH_80: - rlm_req.rlm.bw = CMD_CBW_80MHZ; - break; - case NL80211_CHAN_WIDTH_80P80: - rlm_req.rlm.bw = CMD_CBW_8080MHZ; - break; - case NL80211_CHAN_WIDTH_160: - rlm_req.rlm.bw = CMD_CBW_160MHZ; - break; - case NL80211_CHAN_WIDTH_5: - rlm_req.rlm.bw = CMD_CBW_5MHZ; - break; - case NL80211_CHAN_WIDTH_10: - rlm_req.rlm.bw = CMD_CBW_10MHZ; - break; - case NL80211_CHAN_WIDTH_20_NOHT: - case NL80211_CHAN_WIDTH_20: - default: - rlm_req.rlm.bw = CMD_CBW_20MHZ; - rlm_req.rlm.ht_op_info = 0; - break; - } - - if (rlm_req.rlm.control_channel < rlm_req.rlm.center_chan) - rlm_req.rlm.sco = 1; /* SCA */ - else if (rlm_req.rlm.control_channel > rlm_req.rlm.center_chan) - rlm_req.rlm.sco = 3; /* SCB */ - - return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(BSS_INFO_UPDATE), &rlm_req, - sizeof(rlm_req), true); + return mt76_connac_mcu_uni_set_chctx(phy, mvif, ctx); } EXPORT_SYMBOL_GPL(mt76_connac_mcu_uni_add_bss); @@ -1525,6 +1590,9 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif, struct mt76_connac_hw_scan_req *req; struct sk_buff *skb; + if (test_bit(MT76_HW_SCANNING, &phy->state)) + return -EBUSY; + skb = mt76_mcu_msg_alloc(mdev, NULL, sizeof(*req)); if (!skb) return -ENOMEM; @@ -2646,6 +2714,10 @@ int mt76_connac_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, if (ret) return ret; + ret = mt76_connac_mcu_sta_wed_update(dev, skb); + if (ret) + return ret; + return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } EXPORT_SYMBOL_GPL(mt76_connac_mcu_add_key); @@ -2834,6 +2906,9 @@ mt76_connac_mcu_send_ram_firmware(struct mt76_dev *dev, len = le32_to_cpu(region->len); addr = le32_to_cpu(region->addr); + if (region->feature_set & FW_FEATURE_NON_DL) + goto next; + if (region->feature_set & FW_FEATURE_OVERRIDE_ADDR) override = addr; @@ -2850,6 +2925,7 @@ mt76_connac_mcu_send_ram_firmware(struct mt76_dev *dev, return err; } +next: offset += len; } diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index 718f427d8f6b..f1e942b9a887 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -63,7 +63,7 @@ struct mt76_connac2_mcu_txd { } __packed __aligned(4); /** - * struct mt76_connac2_mcu_uni_txd - mcu command descriptor for firmware v3 + * struct mt76_connac2_mcu_uni_txd - mcu command descriptor for connac2 and connac3 * @txd: hardware descriptor * @len: total length not including txd * @cid: command identifier @@ -121,11 +121,13 @@ struct mt76_connac2_mcu_rxd { u8 eid; u8 seq; - u8 rsv[2]; - + u8 option; + u8 rsv; u8 ext_eid; u8 rsv1[2]; u8 s2d_index; + + u8 tlv[0]; }; struct mt76_connac2_patch_hdr { @@ -354,6 +356,16 @@ struct sta_rec_he { u8 rsv2[2]; } __packed; +struct sta_rec_he_v2 { + __le16 tag; + __le16 len; + u8 he_mac_cap[6]; + u8 he_phy_cap[11]; + u8 pkt_ext; + /* 0: BW80, 1: BW160, 2: BW8080 */ + __le16 max_nss_mcs[CMD_HE_MCS_BW_NUM]; +} __packed; + struct sta_rec_amsdu { __le16 tag; __le16 len; @@ -391,7 +403,8 @@ struct sta_rec_phy { u8 ampdu; u8 rts_policy; u8 rcpi; - u8 rsv[2]; + u8 max_ampdu_len; /* connac3 */ + u8 rsv[1]; } __packed; struct sta_rec_he_6g_capa { @@ -452,8 +465,8 @@ struct sta_rec_bf { u8 ibf_dbw; u8 ibf_ncol; u8 ibf_nrow; - u8 nrow_bw160; - u8 ncol_bw160; + u8 nrow_gt_bw80; + u8 ncol_gt_bw80; u8 ru_start_idx; u8 ru_end_idx; @@ -580,7 +593,7 @@ struct sta_rec_ra_fixed { struct sta_phy phy; - u8 spe_en; + u8 spe_idx; u8 short_preamble; u8 is_5g; u8 mmps_mode; @@ -779,6 +792,9 @@ enum { STA_REC_BFEE, STA_REC_PHY = 0x15, STA_REC_HE_6G = 0x17, + STA_REC_HE_V2 = 0x19, + STA_REC_HDRT = 0x28, + STA_REC_HDR_TRANS = 0x2B, STA_REC_MAX_NUM }; @@ -946,6 +962,9 @@ enum { DEV_INFO_MAX_NUM }; +#define MCU_UNI_CMD_EVENT BIT(1) +#define MCU_UNI_CMD_UNSOLICITED_EVENT BIT(2) + /* event table */ enum { MCU_EVENT_TARGET_ADDRESS_LEN = 0x01, @@ -981,6 +1000,17 @@ enum { MCU_EXT_EVENT_MURU_CTRL = 0x9f, }; +/* unified event table */ +enum { + MCU_UNI_EVENT_RESULT = 0x01, + MCU_UNI_EVENT_FW_LOG_2_HOST = 0x04, + MCU_UNI_EVENT_IE_COUNTDOWN = 0x09, + MCU_UNI_EVENT_RDD_REPORT = 0x11, +}; + +#define MCU_UNI_CMD_EVENT BIT(1) +#define MCU_UNI_CMD_UNSOLICITED_EVENT BIT(2) + enum { MCU_Q_QUERY, MCU_Q_SET, @@ -1063,10 +1093,11 @@ enum { #define MCU_CMD_ACK BIT(0) #define MCU_CMD_UNI BIT(1) -#define MCU_CMD_QUERY BIT(2) +#define MCU_CMD_SET BIT(2) #define MCU_CMD_UNI_EXT_ACK (MCU_CMD_ACK | MCU_CMD_UNI | \ - MCU_CMD_QUERY) + MCU_CMD_SET) +#define MCU_CMD_UNI_QUERY_ACK (MCU_CMD_ACK | MCU_CMD_UNI) #define __MCU_CMD_FIELD_ID GENMASK(7, 0) #define __MCU_CMD_FIELD_EXT_ID GENMASK(15, 8) @@ -1074,6 +1105,7 @@ enum { #define __MCU_CMD_FIELD_UNI BIT(17) #define __MCU_CMD_FIELD_CE BIT(18) #define __MCU_CMD_FIELD_WA BIT(19) +#define __MCU_CMD_FIELD_WM BIT(20) #define MCU_CMD(_t) FIELD_PREP(__MCU_CMD_FIELD_ID, \ MCU_CMD_##_t) @@ -1095,6 +1127,16 @@ enum { FIELD_PREP(__MCU_CMD_FIELD_EXT_ID, \ MCU_WA_PARAM_CMD_##_t)) +#define MCU_WM_UNI_CMD(_t) (MCU_UNI_CMD(_t) | \ + __MCU_CMD_FIELD_WM) +#define MCU_WM_UNI_CMD_QUERY(_t) (MCU_UNI_CMD(_t) | \ + __MCU_CMD_FIELD_QUERY | \ + __MCU_CMD_FIELD_WM) +#define MCU_WA_UNI_CMD(_t) (MCU_UNI_CMD(_t) | \ + __MCU_CMD_FIELD_WA) +#define MCU_WMWA_UNI_CMD(_t) (MCU_WM_UNI_CMD(_t) | \ + __MCU_CMD_FIELD_WA) + enum { MCU_EXT_CMD_EFUSE_ACCESS = 0x01, MCU_EXT_CMD_RF_REG_ACCESS = 0x02, @@ -1148,10 +1190,33 @@ enum { MCU_UNI_CMD_DEV_INFO_UPDATE = 0x01, MCU_UNI_CMD_BSS_INFO_UPDATE = 0x02, MCU_UNI_CMD_STA_REC_UPDATE = 0x03, + MCU_UNI_CMD_EDCA_UPDATE = 0x04, MCU_UNI_CMD_SUSPEND = 0x05, MCU_UNI_CMD_OFFLOAD = 0x06, MCU_UNI_CMD_HIF_CTRL = 0x07, + MCU_UNI_CMD_BAND_CONFIG = 0x08, + MCU_UNI_CMD_REPT_MUAR = 0x09, + MCU_UNI_CMD_WSYS_CONFIG = 0x0b, + MCU_UNI_CMD_REG_ACCESS = 0x0d, + MCU_UNI_CMD_POWER_CREL = 0x0f, + MCU_UNI_CMD_RX_HDR_TRANS = 0x12, + MCU_UNI_CMD_SER = 0x13, + MCU_UNI_CMD_TWT = 0x14, + MCU_UNI_CMD_RDD_CTRL = 0x19, + MCU_UNI_CMD_GET_MIB_INFO = 0x22, MCU_UNI_CMD_SNIFFER = 0x24, + MCU_UNI_CMD_SR = 0x25, + MCU_UNI_CMD_ROC = 0x27, + MCU_UNI_CMD_TXPOWER = 0x2b, + MCU_UNI_CMD_EFUSE_CTRL = 0x2d, + MCU_UNI_CMD_RA = 0x2f, + MCU_UNI_CMD_MURU = 0x31, + MCU_UNI_CMD_BF = 0x33, + MCU_UNI_CMD_CHANNEL_SWITCH = 0x34, + MCU_UNI_CMD_THERMAL = 0x35, + MCU_UNI_CMD_VOW = 0x37, + MCU_UNI_CMD_RRO = 0x57, + MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58, }; enum { @@ -1201,14 +1266,23 @@ enum { enum { UNI_BSS_INFO_BASIC = 0, + UNI_BSS_INFO_RA = 1, UNI_BSS_INFO_RLM = 2, UNI_BSS_INFO_BSS_COLOR = 4, UNI_BSS_INFO_HE_BASIC = 5, UNI_BSS_INFO_BCN_CONTENT = 7, + UNI_BSS_INFO_BCN_CSA = 8, + UNI_BSS_INFO_BCN_BCC = 9, + UNI_BSS_INFO_BCN_MBSSID = 10, + UNI_BSS_INFO_RATE = 11, UNI_BSS_INFO_QBSS = 15, + UNI_BSS_INFO_SEC = 16, + UNI_BSS_INFO_TXCMD = 18, UNI_BSS_INFO_UAPSD = 19, UNI_BSS_INFO_PS = 21, UNI_BSS_INFO_BCNFT = 22, + UNI_BSS_INFO_OFFLOAD = 25, + UNI_BSS_INFO_MLD = 26, }; enum { @@ -1736,10 +1810,14 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy, int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, struct ieee80211_ampdu_params *params, int cmd, bool enable, bool tx); +int mt76_connac_mcu_uni_set_chctx(struct mt76_phy *phy, + struct mt76_vif *vif, + struct ieee80211_chanctx_conf *ctx); int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, struct ieee80211_vif *vif, struct mt76_wcid *wcid, - bool enable); + bool enable, + struct ieee80211_chanctx_conf *ctx); int mt76_connac_mcu_sta_cmd(struct mt76_phy *phy, struct mt76_sta_cmd_info *info); void mt76_connac_mcu_beacon_loss_iter(void *priv, u8 *mac, @@ -1813,6 +1891,7 @@ int mt76_connac_mcu_set_pm(struct mt76_dev *dev, int band, int enter); int mt76_connac_mcu_restart(struct mt76_dev *dev); int mt76_connac_mcu_rdd_cmd(struct mt76_dev *dev, int cmd, u8 index, u8 rx_sel, u8 val); +int mt76_connac_mcu_sta_wed_update(struct mt76_dev *dev, struct sk_buff *skb); int mt76_connac2_load_ram(struct mt76_dev *dev, const char *fw_wm, const char *fw_wa); int mt76_connac2_load_patch(struct mt76_dev *dev, const char *fw_name); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c index da2ca2563ac9..c3a392a1a659 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c @@ -151,7 +151,7 @@ static s8 mt76x0_get_delta(struct mt76x02_dev *dev) void mt76x0_get_tx_power_per_rate(struct mt76x02_dev *dev, struct ieee80211_channel *chan, - struct mt76_rate_power *t) + struct mt76x02_rate_power *t) { bool is_2ghz = chan->band == NL80211_BAND_2GHZ; u16 val, addr; @@ -179,31 +179,19 @@ void mt76x0_get_tx_power_per_rate(struct mt76x02_dev *dev, /* ht-vht mcs 1ss 0, 1, 2, 3 */ addr = is_2ghz ? MT_EE_TX_POWER_BYRATE_BASE + 6 : 0x124; val = mt76x02_eeprom_get(dev, addr); - t->ht[0] = t->ht[1] = t->vht[0] = t->vht[1] = s6_to_s8(val); - t->ht[2] = t->ht[3] = t->vht[2] = t->vht[3] = s6_to_s8(val >> 8); + t->ht[0] = t->ht[1] = s6_to_s8(val); + t->ht[2] = t->ht[3] = s6_to_s8(val >> 8); /* ht-vht mcs 1ss 4, 5, 6 */ addr = is_2ghz ? MT_EE_TX_POWER_BYRATE_BASE + 8 : 0x126; val = mt76x02_eeprom_get(dev, addr); - t->ht[4] = t->ht[5] = t->vht[4] = t->vht[5] = s6_to_s8(val); - t->ht[6] = t->ht[7] = t->vht[6] = t->vht[7] = s6_to_s8(val >> 8); - - /* ht-vht mcs 1ss 0, 1, 2, 3 stbc */ - addr = is_2ghz ? MT_EE_TX_POWER_BYRATE_BASE + 14 : 0xec; - val = mt76x02_eeprom_get(dev, addr); - t->stbc[0] = t->stbc[1] = s6_to_s8(val); - t->stbc[2] = t->stbc[3] = s6_to_s8(val >> 8); - - /* ht-vht mcs 1ss 4, 5, 6 stbc */ - addr = is_2ghz ? MT_EE_TX_POWER_BYRATE_BASE + 16 : 0xee; - val = mt76x02_eeprom_get(dev, addr); - t->stbc[4] = t->stbc[5] = s6_to_s8(val); - t->stbc[6] = t->stbc[7] = s6_to_s8(val >> 8); + t->ht[4] = t->ht[5] = s6_to_s8(val); + t->ht[6] = t->ht[7] = s6_to_s8(val >> 8); /* vht mcs 8, 9 5GHz */ val = mt76x02_eeprom_get(dev, 0x12c); - t->vht[8] = s6_to_s8(val); - t->vht[9] = s6_to_s8(val >> 8); + t->vht[0] = s6_to_s8(val); + t->vht[1] = s6_to_s8(val >> 8); delta = mt76x0_tssi_enabled(dev) ? 0 : mt76x0_get_delta(dev); mt76x02_add_rate_power_offset(t, delta); @@ -235,7 +223,7 @@ void mt76x0_get_power_info(struct mt76x02_dev *dev, data = mt76x02_eeprom_get(dev, MT_EE_5G_TARGET_POWER); else data = mt76x02_eeprom_get(dev, MT_EE_2G_TARGET_POWER); - target_power = (data & 0xff) - dev->mt76.rate_power.ofdm[7]; + target_power = (data & 0xff) - dev->rate_power.ofdm[7]; *tp = target_power + mt76x0_get_delta(dev); return; diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.h index 15540ce8db87..08f1b10bf3ba 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.h @@ -19,7 +19,7 @@ int mt76x0_eeprom_init(struct mt76x02_dev *dev); void mt76x0_read_rx_gain(struct mt76x02_dev *dev); void mt76x0_get_tx_power_per_rate(struct mt76x02_dev *dev, struct ieee80211_channel *chan, - struct mt76_rate_power *t); + struct mt76x02_rate_power *t); void mt76x0_get_power_info(struct mt76x02_dev *dev, struct ieee80211_channel *chan, s8 *tp); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/init.c b/drivers/net/wireless/mediatek/mt76/mt76x0/init.c index 66d47c70111a..6257460f8de5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/init.c @@ -217,7 +217,7 @@ mt76x0_init_txpower(struct mt76x02_dev *dev, struct ieee80211_supported_band *sband) { struct ieee80211_channel *chan; - struct mt76_rate_power t; + struct mt76x02_rate_power t; s8 tp; int i; diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/phy.c b/drivers/net/wireless/mediatek/mt76/mt76x0/phy.c index e91c314cdfac..6c6c8ada7943 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/phy.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/phy.c @@ -595,10 +595,7 @@ mt76x0_phy_get_target_power(struct mt76x02_dev *dev, u8 tx_mode, case 0: /* cck rates */ tx_rate = (info[0] & 0x60) >> 5; - if (tx_rate > 3) - return -EINVAL; - - *target_power = cur_power + dev->mt76.rate_power.cck[tx_rate]; + *target_power = cur_power + dev->rate_power.cck[tx_rate]; *target_pa_power = mt76x0_phy_get_rf_pa_mode(dev, 0, tx_rate); break; case 1: { @@ -635,7 +632,7 @@ mt76x0_phy_get_target_power(struct mt76x02_dev *dev, u8 tx_mode, return -EINVAL; } - *target_power = cur_power + dev->mt76.rate_power.ofdm[index]; + *target_power = cur_power + dev->rate_power.ofdm[index]; *target_pa_power = mt76x0_phy_get_rf_pa_mode(dev, 0, index + 4); break; } @@ -645,7 +642,7 @@ mt76x0_phy_get_target_power(struct mt76x02_dev *dev, u8 tx_mode, if (tx_rate > 9) return -EINVAL; - *target_power = cur_power + dev->mt76.rate_power.vht[tx_rate]; + *target_power = cur_power + dev->rate_power.vht[tx_rate]; *target_pa_power = mt76x0_phy_get_rf_pa_mode(dev, 1, tx_rate); break; default: @@ -654,7 +651,7 @@ mt76x0_phy_get_target_power(struct mt76x02_dev *dev, u8 tx_mode, if (tx_rate > 9) return -EINVAL; - *target_power = cur_power + dev->mt76.rate_power.ht[tx_rate]; + *target_power = cur_power + dev->rate_power.ht[tx_rate]; *target_pa_power = mt76x0_phy_get_rf_pa_mode(dev, 1, tx_rate); break; } @@ -841,7 +838,7 @@ static void mt76x0_phy_tssi_calibrate(struct mt76x02_dev *dev) void mt76x0_phy_set_txpower(struct mt76x02_dev *dev) { - struct mt76_rate_power *t = &dev->mt76.rate_power; + struct mt76x02_rate_power *t = &dev->rate_power; s8 info; mt76x0_get_tx_power_per_rate(dev, dev->mphy.chandef.chan, t); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02.h b/drivers/net/wireless/mediatek/mt76/mt76x02.h index 50eaeff11af3..4cd63bacd742 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02.h +++ b/drivers/net/wireless/mediatek/mt76/mt76x02.h @@ -72,6 +72,18 @@ struct mt76x02_beacon_ops { #define mt76x02_pre_tbtt_enable(dev, enable) \ (dev)->beacon_ops->pre_tbtt_enable(dev, enable) +struct mt76x02_rate_power { + union { + struct { + s8 cck[4]; + s8 ofdm[8]; + s8 ht[16]; + s8 vht[2]; + }; + s8 all[30]; + }; +}; + struct mt76x02_dev { union { /* must be first */ struct mt76_dev mt76; @@ -107,6 +119,8 @@ struct mt76x02_dev { u8 beacon_hang_check; u8 mcu_timeout; + struct mt76x02_rate_power rate_power; + struct mt76x02_calibration cal; int txpower_conf; @@ -174,7 +188,7 @@ int mt76x02_set_rts_threshold(struct ieee80211_hw *hw, u32 val); void mt76x02_remove_hdr_pad(struct sk_buff *skb, int len); bool mt76x02_tx_status_data(struct mt76_dev *mdev, u8 *update); void mt76x02_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); void mt76x02_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q); irqreturn_t mt76x02_irq_handler(int irq, void *dev_instance); void mt76x02_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_debugfs.c b/drivers/net/wireless/mediatek/mt76/mt76x02_debugfs.c index c4fe1c436aaa..8ce4bf44733d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_debugfs.c @@ -20,7 +20,7 @@ mt76x02_ampdu_stat_show(struct seq_file *file, void *data) seq_puts(file, "Count: "); for (j = 0; j < 8; j++) seq_printf(file, "%8d | ", - dev->mt76.aggr_stats[i * 8 + j]); + dev->mphy.aggr_stats[i * 8 + j]); seq_puts(file, "\n"); seq_puts(file, "--------"); for (j = 0; j < 8; j++) @@ -114,6 +114,21 @@ mt76_edcca_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(fops_edcca, mt76_edcca_get, mt76_edcca_set, "%lld\n"); +static int mt76x02_read_rate_txpower(struct seq_file *s, void *data) +{ + struct mt76x02_dev *dev = dev_get_drvdata(s->private); + + mt76_seq_puts_array(s, "CCK", dev->rate_power.cck, + ARRAY_SIZE(dev->rate_power.cck)); + mt76_seq_puts_array(s, "OFDM", dev->rate_power.ofdm, + ARRAY_SIZE(dev->rate_power.ofdm)); + mt76_seq_puts_array(s, "HT", dev->rate_power.ht, + ARRAY_SIZE(dev->rate_power.ht)); + mt76_seq_puts_array(s, "VHT", dev->rate_power.vht, + ARRAY_SIZE(dev->rate_power.vht)); + return 0; +} + void mt76x02_init_debugfs(struct mt76x02_dev *dev) { struct dentry *dir; @@ -133,6 +148,8 @@ void mt76x02_init_debugfs(struct mt76x02_dev *dev) debugfs_create_devm_seqfile(dev->mt76.dev, "txpower", dir, read_txpower); + debugfs_create_devm_seqfile(dev->mt76.dev, "rate_txpower", dir, + mt76x02_read_rate_txpower); debugfs_create_devm_seqfile(dev->mt76.dev, "agc", dir, read_agc); debugfs_create_u32("tx_hang_reset", 0400, dir, &dev->tx_hang_reset); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.h b/drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.h index 99941a4700f3..13fa70853b0d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.h @@ -62,8 +62,6 @@ enum mt76x02_eeprom_field { MT_EE_TX_POWER_HT_MCS4 = 0x0a8, MT_EE_TX_POWER_HT_MCS8 = 0x0aa, MT_EE_TX_POWER_HT_MCS12 = 0x0ac, - MT_EE_TX_POWER_VHT_MCS0 = 0x0ba, - MT_EE_TX_POWER_VHT_MCS4 = 0x0bc, MT_EE_TX_POWER_VHT_MCS8 = 0x0be, MT_EE_2G_TARGET_POWER = 0x0d0, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c index 93d96739f802..d3f74473e6fb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c @@ -25,7 +25,7 @@ void mt76x02_mac_reset_counters(struct mt76x02_dev *dev) for (i = 0; i < 16; i++) mt76_rr(dev, MT_TX_STAT_FIFO); - memset(dev->mt76.aggr_stats, 0, sizeof(dev->mt76.aggr_stats)); + memset(dev->mphy.aggr_stats, 0, sizeof(dev->mphy.aggr_stats)); } EXPORT_SYMBOL_GPL(mt76x02_mac_reset_counters); @@ -1191,8 +1191,8 @@ void mt76x02_mac_work(struct work_struct *work) for (i = 0, idx = 0; i < 16; i++) { u32 val = mt76_rr(dev, MT_TX_AGG_CNT(i)); - dev->mt76.aggr_stats[idx++] += val & 0xffff; - dev->mt76.aggr_stats[idx++] += val >> 16; + dev->mphy.aggr_stats[idx++] += val & 0xffff; + dev->mphy.aggr_stats[idx++] += val >> 16; } mt76x02_check_mac_err(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_phy.c b/drivers/net/wireless/mediatek/mt76/mt76x02_phy.c index 2e53b0c1afdd..cbe7e6f0c29a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_phy.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_phy.c @@ -59,7 +59,7 @@ mt76x02_tx_power_mask(u8 v1, u8 v2, u8 v3, u8 v4) return val; } -int mt76x02_get_max_rate_power(struct mt76_rate_power *r) +int mt76x02_get_max_rate_power(struct mt76x02_rate_power *r) { s8 ret = 0; int i; @@ -71,7 +71,7 @@ int mt76x02_get_max_rate_power(struct mt76_rate_power *r) } EXPORT_SYMBOL_GPL(mt76x02_get_max_rate_power); -void mt76x02_limit_rate_power(struct mt76_rate_power *r, int limit) +void mt76x02_limit_rate_power(struct mt76x02_rate_power *r, int limit) { int i; @@ -81,7 +81,7 @@ void mt76x02_limit_rate_power(struct mt76_rate_power *r, int limit) } EXPORT_SYMBOL_GPL(mt76x02_limit_rate_power); -void mt76x02_add_rate_power_offset(struct mt76_rate_power *r, int offset) +void mt76x02_add_rate_power_offset(struct mt76x02_rate_power *r, int offset) { int i; @@ -92,7 +92,7 @@ EXPORT_SYMBOL_GPL(mt76x02_add_rate_power_offset); void mt76x02_phy_set_txpower(struct mt76x02_dev *dev, int txp_0, int txp_1) { - struct mt76_rate_power *t = &dev->mt76.rate_power; + struct mt76x02_rate_power *t = &dev->rate_power; mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_0, txp_0); mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_1, txp_1); @@ -107,17 +107,17 @@ void mt76x02_phy_set_txpower(struct mt76x02_dev *dev, int txp_0, int txp_1) mt76x02_tx_power_mask(t->ht[4], t->ht[6], t->ht[8], t->ht[10])); mt76_wr(dev, MT_TX_PWR_CFG_3, - mt76x02_tx_power_mask(t->ht[12], t->ht[14], t->stbc[0], - t->stbc[2])); + mt76x02_tx_power_mask(t->ht[12], t->ht[14], t->ht[0], + t->ht[2])); mt76_wr(dev, MT_TX_PWR_CFG_4, - mt76x02_tx_power_mask(t->stbc[4], t->stbc[6], 0, 0)); + mt76x02_tx_power_mask(t->ht[4], t->ht[6], 0, 0)); mt76_wr(dev, MT_TX_PWR_CFG_7, - mt76x02_tx_power_mask(t->ofdm[7], t->vht[8], t->ht[7], - t->vht[9])); + mt76x02_tx_power_mask(t->ofdm[7], t->vht[0], t->ht[7], + t->vht[1])); mt76_wr(dev, MT_TX_PWR_CFG_8, - mt76x02_tx_power_mask(t->ht[14], 0, t->vht[8], t->vht[9])); + mt76x02_tx_power_mask(t->ht[14], 0, t->vht[0], t->vht[1])); mt76_wr(dev, MT_TX_PWR_CFG_9, - mt76x02_tx_power_mask(t->ht[7], 0, t->stbc[8], t->stbc[9])); + mt76x02_tx_power_mask(t->ht[7], 0, t->vht[0], t->vht[1])); } EXPORT_SYMBOL_GPL(mt76x02_phy_set_txpower); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_phy.h b/drivers/net/wireless/mediatek/mt76/mt76x02_phy.h index 1def25bf735a..84d8a6138b3e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_phy.h +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_phy.h @@ -34,10 +34,10 @@ mt76x02_get_low_rssi_gain_thresh(struct mt76x02_dev *dev) } } -void mt76x02_add_rate_power_offset(struct mt76_rate_power *r, int offset); +void mt76x02_add_rate_power_offset(struct mt76x02_rate_power *r, int offset); void mt76x02_phy_set_txpower(struct mt76x02_dev *dev, int txp_0, int txp_2); -void mt76x02_limit_rate_power(struct mt76_rate_power *r, int limit); -int mt76x02_get_max_rate_power(struct mt76_rate_power *r); +void mt76x02_limit_rate_power(struct mt76x02_rate_power *r, int limit); +int mt76x02_get_max_rate_power(struct mt76x02_rate_power *r); void mt76x02_phy_set_rxpath(struct mt76x02_dev *dev); void mt76x02_phy_set_txdac(struct mt76x02_dev *dev); void mt76x02_phy_set_bw(struct mt76x02_dev *dev, int width, u8 ctrl); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_txrx.c b/drivers/net/wireless/mediatek/mt76/mt76x02_txrx.c index 96fdf423a348..d8bc4ae185f5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_txrx.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_txrx.c @@ -33,7 +33,7 @@ void mt76x02_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control, EXPORT_SYMBOL_GPL(mt76x02_tx); void mt76x02_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) + struct sk_buff *skb, u32 *info) { struct mt76x02_dev *dev = container_of(mdev, struct mt76x02_dev, mt76); void *rxwi = skb->data; @@ -62,23 +62,23 @@ s8 mt76x02_tx_get_max_txpwr_adj(struct mt76x02_dev *dev, u8 mcs = ieee80211_rate_get_vht_mcs(rate); if (mcs == 8 || mcs == 9) { - max_txpwr = dev->mt76.rate_power.vht[8]; + max_txpwr = dev->rate_power.vht[0]; } else { u8 nss, idx; nss = ieee80211_rate_get_vht_nss(rate); idx = ((nss - 1) << 3) + mcs; - max_txpwr = dev->mt76.rate_power.ht[idx & 0xf]; + max_txpwr = dev->rate_power.ht[idx & 0xf]; } } else if (rate->flags & IEEE80211_TX_RC_MCS) { - max_txpwr = dev->mt76.rate_power.ht[rate->idx & 0xf]; + max_txpwr = dev->rate_power.ht[rate->idx & 0xf]; } else { enum nl80211_band band = dev->mphy.chandef.chan->band; if (band == NL80211_BAND_2GHZ) { const struct ieee80211_rate *r; struct wiphy *wiphy = dev->mt76.hw->wiphy; - struct mt76_rate_power *rp = &dev->mt76.rate_power; + struct mt76x02_rate_power *rp = &dev->rate_power; r = &wiphy->bands[band]->bitrates[rate->idx]; if (r->flags & IEEE80211_RATE_SHORT_PREAMBLE) @@ -86,7 +86,7 @@ s8 mt76x02_tx_get_max_txpwr_adj(struct mt76x02_dev *dev, else max_txpwr = rp->ofdm[r->hw_value & 0x7]; } else { - max_txpwr = dev->mt76.rate_power.ofdm[rate->idx & 0x7]; + max_txpwr = dev->rate_power.ofdm[rate->idx & 0x7]; } } @@ -112,7 +112,7 @@ void mt76x02_tx_set_txpwr_auto(struct mt76x02_dev *dev, s8 txpwr) s8 txpwr_adj; txpwr_adj = mt76x02_tx_get_txpwr_adj(dev, txpwr, - dev->mt76.rate_power.ofdm[4]); + dev->rate_power.ofdm[4]); mt76_rmw_field(dev, MT_PROT_AUTO_TX_CFG, MT_PROT_AUTO_TX_CFG_PROT_PADJ, txpwr_adj); mt76_rmw_field(dev, MT_PROT_AUTO_TX_CFG, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.c index c57e05a5c65e..d5809408d1d3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.c @@ -280,7 +280,7 @@ void mt76x2_read_rx_gain(struct mt76x02_dev *dev) } EXPORT_SYMBOL_GPL(mt76x2_read_rx_gain); -void mt76x2_get_rate_power(struct mt76x02_dev *dev, struct mt76_rate_power *t, +void mt76x2_get_rate_power(struct mt76x02_dev *dev, struct mt76x02_rate_power *t, struct ieee80211_channel *chan) { bool is_5ghz; @@ -324,22 +324,10 @@ void mt76x2_get_rate_power(struct mt76x02_dev *dev, struct mt76_rate_power *t, t->ht[12] = t->ht[13] = mt76x02_rate_power_val(val); t->ht[14] = t->ht[15] = mt76x02_rate_power_val(val >> 8); - val = mt76x02_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS0); - t->vht[0] = t->vht[1] = mt76x02_rate_power_val(val); - t->vht[2] = t->vht[3] = mt76x02_rate_power_val(val >> 8); - - val = mt76x02_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS4); - t->vht[4] = t->vht[5] = mt76x02_rate_power_val(val); - t->vht[6] = t->vht[7] = mt76x02_rate_power_val(val >> 8); - val = mt76x02_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS8); if (!is_5ghz) val >>= 8; - t->vht[8] = t->vht[9] = mt76x02_rate_power_val(val >> 8); - - memcpy(t->stbc, t->ht, sizeof(t->stbc[0]) * 8); - t->stbc[8] = t->vht[8]; - t->stbc[9] = t->vht[9]; + t->vht[0] = t->vht[1] = mt76x02_rate_power_val(val >> 8); } EXPORT_SYMBOL_GPL(mt76x2_get_rate_power); diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.h index 3755632e6494..43430ef98b11 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.h @@ -40,7 +40,7 @@ struct mt76x2_temp_comp { unsigned int low_slope; /* J / dB */ }; -void mt76x2_get_rate_power(struct mt76x02_dev *dev, struct mt76_rate_power *t, +void mt76x2_get_rate_power(struct mt76x02_dev *dev, struct mt76x02_rate_power *t, struct ieee80211_channel *chan); void mt76x2_get_power_info(struct mt76x02_dev *dev, struct mt76x2_tx_power_info *t, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/init.c b/drivers/net/wireless/mediatek/mt76/mt76x2/init.c index 7b01a06d7f8d..19c139290adb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/init.c @@ -182,7 +182,7 @@ void mt76x2_init_txpower(struct mt76x02_dev *dev, { struct ieee80211_channel *chan; struct mt76x2_tx_power_info txp; - struct mt76_rate_power t = {}; + struct mt76x02_rate_power t = {}; int i; for (i = 0; i < sband->n_channels; i++) { diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/phy.c b/drivers/net/wireless/mediatek/mt76/mt76x2/phy.c index ed2dcb05d614..f84517d932dc 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/phy.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/phy.c @@ -116,7 +116,7 @@ void mt76x2_phy_set_txpower_regs(struct mt76x02_dev *dev, EXPORT_SYMBOL_GPL(mt76x2_phy_set_txpower_regs); static int -mt76x2_get_min_rate_power(struct mt76_rate_power *r) +mt76x2_get_min_rate_power(struct mt76x02_rate_power *r) { int i; s8 ret = 0; @@ -140,7 +140,7 @@ void mt76x2_phy_set_txpower(struct mt76x02_dev *dev) struct ieee80211_channel *chan = dev->mphy.chandef.chan; struct mt76x2_tx_power_info txp; int txp_0, txp_1, delta = 0; - struct mt76_rate_power t = {}; + struct mt76x02_rate_power t = {}; int base_power, gain; mt76x2_get_power_info(dev, &txp, chan); @@ -175,7 +175,7 @@ void mt76x2_phy_set_txpower(struct mt76x02_dev *dev) dev->target_power = txp.target_power; dev->target_power_delta[0] = txp_0 - txp.chain[0].target_power; dev->target_power_delta[1] = txp_1 - txp.chain[0].target_power; - dev->mt76.rate_power = t; + dev->rate_power = t; mt76x02_phy_set_txpower(dev, txp_0, txp_1); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/Kconfig b/drivers/net/wireless/mediatek/mt76/mt7915/Kconfig index f21282cea845..d710726d47bf 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/Kconfig +++ b/drivers/net/wireless/mediatek/mt76/mt7915/Kconfig @@ -2,6 +2,7 @@ config MT7915E tristate "MediaTek MT7915E (PCIe) support" select MT76_CONNAC_LIB + select WANT_DEV_COREDUMP depends on MAC80211 depends on PCI select RELAY diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/Makefile b/drivers/net/wireless/mediatek/mt76/mt7915/Makefile index b794ceb79c37..797ae49805c3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/Makefile +++ b/drivers/net/wireless/mediatek/mt76/mt7915/Makefile @@ -6,4 +6,5 @@ mt7915e-y := pci.o init.o dma.o eeprom.o main.o mcu.o mac.o \ debugfs.o mmio.o mt7915e-$(CONFIG_NL80211_TESTMODE) += testmode.o -mt7915e-$(CONFIG_MT7986_WMAC) += soc.o
\ No newline at end of file +mt7915e-$(CONFIG_MT7986_WMAC) += soc.o +mt7915e-$(CONFIG_DEV_COREDUMP) += coredump.o diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/coredump.c b/drivers/net/wireless/mediatek/mt76/mt7915/coredump.c new file mode 100644 index 000000000000..d097a56dd33d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7915/coredump.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: ISC +/* Copyright (C) 2022 MediaTek Inc. */ + +#include <linux/devcoredump.h> +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/utsname.h> +#include "coredump.h" + +static bool coredump_memdump; +module_param(coredump_memdump, bool, 0644); +MODULE_PARM_DESC(coredump_memdump, "Optional ability to dump firmware memory"); + +static const struct mt7915_mem_region mt7915_mem_regions[] = { + { + .start = 0xe003b400, + .len = 0x00003bff, + .name = "CRAM", + }, +}; + +static const struct mt7915_mem_region mt7916_mem_regions[] = { + { + .start = 0x00800000, + .len = 0x0005ffff, + .name = "ROM", + }, + { + .start = 0x00900000, + .len = 0x00013fff, + .name = "ULM1", + }, + { + .start = 0x02200000, + .len = 0x0004ffff, + .name = "ULM2", + }, + { + .start = 0x02300000, + .len = 0x0004ffff, + .name = "ULM3", + }, + { + .start = 0x00400000, + .len = 0x00027fff, + .name = "SRAM", + }, + { + .start = 0xe0000000, + .len = 0x00157fff, + .name = "CRAM", + }, +}; + +static const struct mt7915_mem_region mt7986_mem_regions[] = { + { + .start = 0x00800000, + .len = 0x0005ffff, + .name = "ROM", + }, + { + .start = 0x00900000, + .len = 0x0000ffff, + .name = "ULM1", + }, + { + .start = 0x02200000, + .len = 0x0004ffff, + .name = "ULM2", + }, + { + .start = 0x02300000, + .len = 0x0004ffff, + .name = "ULM3", + }, + { + .start = 0x00400000, + .len = 0x00017fff, + .name = "SRAM", + }, + { + .start = 0xe0000000, + .len = 0x00113fff, + .name = "CRAM", + }, +}; + +const struct mt7915_mem_region* +mt7915_coredump_get_mem_layout(struct mt7915_dev *dev, u32 *num) +{ + switch (mt76_chip(&dev->mt76)) { + case 0x7915: + *num = ARRAY_SIZE(mt7915_mem_regions); + return &mt7915_mem_regions[0]; + case 0x7986: + *num = ARRAY_SIZE(mt7986_mem_regions); + return &mt7986_mem_regions[0]; + case 0x7916: + *num = ARRAY_SIZE(mt7916_mem_regions); + return &mt7916_mem_regions[0]; + default: + return NULL; + } +} + +static int mt7915_coredump_get_mem_size(struct mt7915_dev *dev) +{ + const struct mt7915_mem_region *mem_region; + size_t size = 0; + u32 num; + int i; + + mem_region = mt7915_coredump_get_mem_layout(dev, &num); + if (!mem_region) + return 0; + + for (i = 0; i < num; i++) { + size += mem_region->len; + mem_region++; + } + + /* reserve space for the headers */ + size += num * sizeof(struct mt7915_mem_hdr); + /* make sure it is aligned 4 bytes for debug message print out */ + size = ALIGN(size, 4); + + return size; +} + +struct mt7915_crash_data *mt7915_coredump_new(struct mt7915_dev *dev) +{ + struct mt7915_crash_data *crash_data = dev->coredump.crash_data; + + lockdep_assert_held(&dev->dump_mutex); + + guid_gen(&crash_data->guid); + ktime_get_real_ts64(&crash_data->timestamp); + + return crash_data; +} + +static void +mt7915_coredump_fw_state(struct mt7915_dev *dev, struct mt7915_coredump *dump, + bool *exception) +{ + u32 state, count, type; + + type = (u32)mt76_get_field(dev, MT_FW_EXCEPT_TYPE, GENMASK(7, 0)); + state = (u32)mt76_get_field(dev, MT_FW_ASSERT_STAT, GENMASK(7, 0)); + count = is_mt7915(&dev->mt76) ? + (u32)mt76_get_field(dev, MT_FW_EXCEPT_COUNT, GENMASK(15, 8)) : + (u32)mt76_get_field(dev, MT_FW_EXCEPT_COUNT, GENMASK(7, 0)); + + /* normal mode: driver can manually trigger assert for detail info */ + if (!count) + strscpy(dump->fw_state, "normal", sizeof(dump->fw_state)); + else if (state > 1 && (count == 1) && type == 5) + strscpy(dump->fw_state, "assert", sizeof(dump->fw_state)); + else if ((state > 1 && count == 1) || count > 1) + strscpy(dump->fw_state, "exception", sizeof(dump->fw_state)); + + *exception = !!count; +} + +static void +mt7915_coredump_fw_trace(struct mt7915_dev *dev, struct mt7915_coredump *dump, + bool exception) +{ + u32 n, irq, sch, base = MT_FW_EINT_INFO; + + /* trap or run? */ + dump->last_msg_id = mt76_rr(dev, MT_FW_LAST_MSG_ID); + + n = is_mt7915(&dev->mt76) ? + (u32)mt76_get_field(dev, base, GENMASK(7, 0)) : + (u32)mt76_get_field(dev, base, GENMASK(15, 8)); + dump->eint_info_idx = n; + + irq = mt76_rr(dev, base + 0x8); + n = is_mt7915(&dev->mt76) ? + FIELD_GET(GENMASK(7, 0), irq) : FIELD_GET(GENMASK(23, 16), irq); + dump->irq_info_idx = n; + + sch = mt76_rr(dev, MT_FW_SCHED_INFO); + n = is_mt7915(&dev->mt76) ? + FIELD_GET(GENMASK(7, 0), sch) : FIELD_GET(GENMASK(15, 8), sch); + dump->sched_info_idx = n; + + if (exception) { + u32 i, y; + + /* sched trace */ + n = is_mt7915(&dev->mt76) ? + FIELD_GET(GENMASK(15, 8), sch) : FIELD_GET(GENMASK(7, 0), sch); + n = n > 60 ? 60 : n; + + strscpy(dump->trace_sched, "(sched_info) id, time", + sizeof(dump->trace_sched)); + + for (y = dump->sched_info_idx, i = 0; i < n; i++, y++) { + mt7915_memcpy_fromio(dev, dump->sched, base + 0xc + y * 12, + sizeof(dump->sched)); + y = y >= n ? 0 : y; + } + + /* irq trace */ + n = is_mt7915(&dev->mt76) ? + FIELD_GET(GENMASK(15, 8), irq) : FIELD_GET(GENMASK(7, 0), irq); + n = n > 60 ? 60 : n; + + strscpy(dump->trace_irq, "(irq_info) id, time", + sizeof(dump->trace_irq)); + + for (y = dump->irq_info_idx, i = 0; i < n; i++, y++) { + mt7915_memcpy_fromio(dev, dump->irq, base + 0x4 + y * 16, + sizeof(dump->irq)); + y = y >= n ? 0 : y; + } + } +} + +static void +mt7915_coredump_fw_stack(struct mt7915_dev *dev, struct mt7915_coredump *dump, + bool exception) +{ + u32 oldest, i, idx; + + /* stop call stack record */ + if (!exception) + mt76_clear(dev, 0x89050200, BIT(0)); + + oldest = (u32)mt76_get_field(dev, 0x89050200, GENMASK(20, 16)) + 2; + for (i = 0; i < 16; i++) { + idx = ((oldest + 2 * i + 1) % 32); + dump->call_stack[i] = mt76_rr(dev, 0x89050204 + idx * 4); + } + + /* start call stack record */ + if (!exception) + mt76_set(dev, 0x89050200, BIT(0)); +} + +static void +mt7915_coredump_fw_task(struct mt7915_dev *dev, struct mt7915_coredump *dump) +{ + u32 offs = is_mt7915(&dev->mt76) ? 0xe0 : 0x170; + + strscpy(dump->task_qid, "(task queue id) read, write", + sizeof(dump->task_qid)); + + dump->taskq[0].read = mt76_rr(dev, MT_FW_TASK_QID1); + dump->taskq[0].write = mt76_rr(dev, MT_FW_TASK_QID1 - 4); + dump->taskq[1].read = mt76_rr(dev, MT_FW_TASK_QID2); + dump->taskq[1].write = mt76_rr(dev, MT_FW_TASK_QID2 - 4); + + strscpy(dump->task_info, "(task stack) start, end, size", + sizeof(dump->task_info)); + + dump->taski[0].start = mt76_rr(dev, MT_FW_TASK_START); + dump->taski[0].end = mt76_rr(dev, MT_FW_TASK_END); + dump->taski[0].size = mt76_rr(dev, MT_FW_TASK_SIZE); + dump->taski[1].start = mt76_rr(dev, MT_FW_TASK_START + offs); + dump->taski[1].end = mt76_rr(dev, MT_FW_TASK_END + offs); + dump->taski[1].size = mt76_rr(dev, MT_FW_TASK_SIZE + offs); +} + +static void +mt7915_coredump_fw_context(struct mt7915_dev *dev, struct mt7915_coredump *dump) +{ + u32 count, idx, id; + + count = mt76_rr(dev, MT_FW_CIRQ_COUNT); + + /* current context */ + if (!count) { + strscpy(dump->fw_context, "(context) interrupt", + sizeof(dump->fw_context)); + + idx = is_mt7915(&dev->mt76) ? + (u32)mt76_get_field(dev, MT_FW_CIRQ_IDX, GENMASK(31, 16)) : + (u32)mt76_get_field(dev, MT_FW_CIRQ_IDX, GENMASK(15, 0)); + dump->context.idx = idx; + dump->context.handler = mt76_rr(dev, MT_FW_CIRQ_LISR); + } else { + idx = mt76_rr(dev, MT_FW_TASK_IDX); + id = mt76_rr(dev, MT_FW_TASK_ID); + + if (!id && idx == 3) { + strscpy(dump->fw_context, "(context) idle", + sizeof(dump->fw_context)); + } else if (id && idx != 3) { + strscpy(dump->fw_context, "(context) task", + sizeof(dump->fw_context)); + + dump->context.idx = idx; + dump->context.handler = id; + } + } +} + +static struct mt7915_coredump *mt7915_coredump_build(struct mt7915_dev *dev) +{ + struct mt7915_crash_data *crash_data = dev->coredump.crash_data; + struct mt7915_coredump *dump; + struct mt7915_coredump_mem *dump_mem; + size_t len, sofar = 0, hdr_len = sizeof(*dump); + unsigned char *buf; + bool exception; + + len = hdr_len; + + if (coredump_memdump && crash_data->memdump_buf_len) + len += sizeof(*dump_mem) + crash_data->memdump_buf_len; + + sofar += hdr_len; + + /* this is going to get big when we start dumping memory and such, + * so go ahead and use vmalloc. + */ + buf = vzalloc(len); + if (!buf) + return NULL; + + mutex_lock(&dev->dump_mutex); + + dump = (struct mt7915_coredump *)(buf); + dump->len = len; + + /* plain text */ + strscpy(dump->magic, "mt76-crash-dump", sizeof(dump->magic)); + strscpy(dump->kernel, init_utsname()->release, sizeof(dump->kernel)); + strscpy(dump->fw_ver, dev->mt76.hw->wiphy->fw_version, + sizeof(dump->fw_ver)); + + guid_copy(&dump->guid, &crash_data->guid); + dump->tv_sec = crash_data->timestamp.tv_sec; + dump->tv_nsec = crash_data->timestamp.tv_nsec; + dump->device_id = mt76_chip(&dev->mt76); + + mt7915_coredump_fw_state(dev, dump, &exception); + mt7915_coredump_fw_trace(dev, dump, exception); + mt7915_coredump_fw_task(dev, dump); + mt7915_coredump_fw_context(dev, dump); + mt7915_coredump_fw_stack(dev, dump, exception); + + /* gather memory content */ + dump_mem = (struct mt7915_coredump_mem *)(buf + sofar); + dump_mem->len = crash_data->memdump_buf_len; + if (coredump_memdump && crash_data->memdump_buf_len) + memcpy(dump_mem->data, crash_data->memdump_buf, + crash_data->memdump_buf_len); + + mutex_unlock(&dev->dump_mutex); + + return dump; +} + +int mt7915_coredump_submit(struct mt7915_dev *dev) +{ + struct mt7915_coredump *dump; + + dump = mt7915_coredump_build(dev); + if (!dump) { + dev_warn(dev->mt76.dev, "no crash dump data found\n"); + return -ENODATA; + } + + dev_coredumpv(dev->mt76.dev, dump, dump->len, GFP_KERNEL); + + return 0; +} + +int mt7915_coredump_register(struct mt7915_dev *dev) +{ + struct mt7915_crash_data *crash_data; + + crash_data = vzalloc(sizeof(*dev->coredump.crash_data)); + if (!crash_data) + return -ENOMEM; + + dev->coredump.crash_data = crash_data; + + if (coredump_memdump) { + crash_data->memdump_buf_len = mt7915_coredump_get_mem_size(dev); + if (!crash_data->memdump_buf_len) + /* no memory content */ + return 0; + + crash_data->memdump_buf = vzalloc(crash_data->memdump_buf_len); + if (!crash_data->memdump_buf) { + vfree(crash_data); + return -ENOMEM; + } + } + + return 0; +} + +void mt7915_coredump_unregister(struct mt7915_dev *dev) +{ + if (dev->coredump.crash_data->memdump_buf) { + vfree(dev->coredump.crash_data->memdump_buf); + dev->coredump.crash_data->memdump_buf = NULL; + dev->coredump.crash_data->memdump_buf_len = 0; + } + + vfree(dev->coredump.crash_data); + dev->coredump.crash_data = NULL; +} + diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/coredump.h b/drivers/net/wireless/mediatek/mt76/mt7915/coredump.h new file mode 100644 index 000000000000..709f8e9c795c --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7915/coredump.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: ISC */ +/* Copyright (C) 2022 MediaTek Inc. */ + +#ifndef _COREDUMP_H_ +#define _COREDUMP_H_ + +#include "mt7915.h" + +struct trace { + u32 id; + u32 timestamp; +}; + +struct mt7915_coredump { + char magic[16]; + + u32 len; + + guid_t guid; + + /* time-of-day stamp */ + u64 tv_sec; + /* time-of-day stamp, nano-seconds */ + u64 tv_nsec; + /* kernel version */ + char kernel[64]; + /* firmware version */ + char fw_ver[ETHTOOL_FWVERS_LEN]; + + u32 device_id; + + /* exception state */ + char fw_state[12]; + + u32 last_msg_id; + u32 eint_info_idx; + u32 irq_info_idx; + u32 sched_info_idx; + + /* schedule info */ + char trace_sched[32]; + struct { + struct trace t; + u32 pc; + } sched[60]; + + /* irq info */ + char trace_irq[32]; + struct trace irq[60]; + + /* task queue status */ + char task_qid[32]; + struct { + u32 read; + u32 write; + } taskq[2]; + + /* task stack info */ + char task_info[32]; + struct { + u32 start; + u32 end; + u32 size; + } taski[2]; + + /* firmware context */ + char fw_context[24]; + struct { + u32 idx; + u32 handler; + } context; + + /* link registers calltrace */ + u32 call_stack[16]; + + /* memory content */ + u8 data[]; +} __packed; + +struct mt7915_coredump_mem { + u32 len; + u8 data[]; +} __packed; + +struct mt7915_mem_hdr { + u32 start; + u32 len; + u8 data[]; +}; + +struct mt7915_mem_region { + u32 start; + size_t len; + + const char *name; +}; + +#ifdef CONFIG_DEV_COREDUMP + +const struct mt7915_mem_region * +mt7915_coredump_get_mem_layout(struct mt7915_dev *dev, u32 *num); +struct mt7915_crash_data *mt7915_coredump_new(struct mt7915_dev *dev); +int mt7915_coredump_submit(struct mt7915_dev *dev); +int mt7915_coredump_register(struct mt7915_dev *dev); +void mt7915_coredump_unregister(struct mt7915_dev *dev); + +#else /* CONFIG_DEV_COREDUMP */ + +static inline const struct mt7915_mem_region * +mt7915_coredump_get_mem_layout(struct mt7915_dev *dev, u32 *num) +{ + return NULL; +} + +static inline int mt7915_coredump_submit(struct mt7915_dev *dev) +{ + return 0; +} + +static inline struct mt7915_crash_data *mt7915_coredump_new(struct mt7915_dev *dev) +{ + return NULL; +} + +static inline int mt7915_coredump_register(struct mt7915_dev *dev) +{ + return 0; +} + +static inline void mt7915_coredump_unregister(struct mt7915_dev *dev) +{ +} + +#endif /* CONFIG_DEV_COREDUMP */ + +#endif /* _COREDUMP_H_ */ diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c index 6ef3431cad64..fb46c2c1784f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c @@ -46,12 +46,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(fops_implicit_txbf, mt7915_implicit_txbf_get, /* test knob of system error recovery */ static ssize_t -mt7915_fw_ser_set(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) +mt7915_sys_recovery_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) { struct mt7915_phy *phy = file->private_data; struct mt7915_dev *dev = phy->dev; - bool ext_phy = phy != &dev->phy; + bool band = phy->mt76->band_idx; char buf[16]; int ret = 0; u16 val; @@ -71,9 +71,19 @@ mt7915_fw_ser_set(struct file *file, const char __user *user_buf, return -EINVAL; switch (val) { + /* + * 0: grab firmware current SER state. + * 1: trigger & enable system error L1 recovery. + * 2: trigger & enable system error L2 recovery. + * 3: trigger & enable system error L3 rx abort. + * 4: trigger & enable system error L3 tx abort + * 5: trigger & enable system error L3 tx disable. + * 6: trigger & enable system error L3 bf recovery. + * 7: trigger & enable system error full recovery. + * 8: trigger firmware crash. + */ case SER_QUERY: - /* grab firmware SER stats */ - ret = mt7915_mcu_set_ser(dev, 0, 0, ext_phy); + ret = mt7915_mcu_set_ser(dev, 0, 0, band); break; case SER_SET_RECOVER_L1: case SER_SET_RECOVER_L2: @@ -81,11 +91,28 @@ mt7915_fw_ser_set(struct file *file, const char __user *user_buf, case SER_SET_RECOVER_L3_TX_ABORT: case SER_SET_RECOVER_L3_TX_DISABLE: case SER_SET_RECOVER_L3_BF: - ret = mt7915_mcu_set_ser(dev, SER_ENABLE, BIT(val), ext_phy); + ret = mt7915_mcu_set_ser(dev, SER_ENABLE, BIT(val), band); + if (ret) + return ret; + + ret = mt7915_mcu_set_ser(dev, SER_RECOVER, val, band); + break; + + /* enable full chip reset */ + case SER_SET_RECOVER_FULL: + mt76_set(dev, MT_WFDMA0_MCU_HOST_INT_ENA, MT_MCU_CMD_WDT_MASK); + ret = mt7915_mcu_set_ser(dev, 1, 3, band); if (ret) return ret; - ret = mt7915_mcu_set_ser(dev, SER_RECOVER, val, ext_phy); + dev->recovery.state |= MT_MCU_CMD_WDT_MASK; + mt7915_reset(dev); + break; + + /* WARNING: trigger firmware crash */ + case SER_SET_SYSTEM_ASSERT: + mt76_wr(dev, MT_MCU_WM_CIRQ_EINT_MASK_CLR_ADDR, BIT(18)); + mt76_wr(dev, MT_MCU_WM_CIRQ_EINT_SOFT_ADDR, BIT(18)); break; default: break; @@ -95,20 +122,45 @@ mt7915_fw_ser_set(struct file *file, const char __user *user_buf, } static ssize_t -mt7915_fw_ser_get(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) +mt7915_sys_recovery_get(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) { struct mt7915_phy *phy = file->private_data; struct mt7915_dev *dev = phy->dev; char *buff; int desc = 0; ssize_t ret; - static const size_t bufsz = 400; + static const size_t bufsz = 1024; buff = kmalloc(bufsz, GFP_KERNEL); if (!buff) return -ENOMEM; + /* HELP */ + desc += scnprintf(buff + desc, bufsz - desc, + "Please echo the correct value ...\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "0: grab firmware transient SER state\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "1: trigger system error L1 recovery\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "2: trigger system error L2 recovery\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "3: trigger system error L3 rx abort\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "4: trigger system error L3 tx abort\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "5: trigger system error L3 tx disable\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "6: trigger system error L3 bf recovery\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "7: trigger system error full recovery\n"); + desc += scnprintf(buff + desc, bufsz - desc, + "8: trigger firmware crash\n"); + + /* SER statistics */ + desc += scnprintf(buff + desc, bufsz - desc, + "\nlet's dump firmware SER statistics...\n"); desc += scnprintf(buff + desc, bufsz - desc, "::E R , SER_STATUS = 0x%08x\n", mt76_rr(dev, MT_SWDEF_SER_STATS)); @@ -139,15 +191,19 @@ mt7915_fw_ser_get(struct file *file, char __user *user_buf, desc += scnprintf(buff + desc, bufsz - desc, "::E R , SER_LMAC_WISR7_B1 = 0x%08x\n", mt76_rr(dev, MT_SWDEF_LAMC_WISR7_BN1_STATS)); + desc += scnprintf(buff + desc, bufsz - desc, + "\nSYS_RESET_COUNT: WM %d, WA %d\n", + dev->recovery.wm_reset_count, + dev->recovery.wa_reset_count); ret = simple_read_from_buffer(user_buf, count, ppos, buff, desc); kfree(buff); return ret; } -static const struct file_operations mt7915_fw_ser_ops = { - .write = mt7915_fw_ser_set, - .read = mt7915_fw_ser_get, +static const struct file_operations mt7915_sys_recovery_ops = { + .write = mt7915_sys_recovery_set, + .read = mt7915_sys_recovery_get, .open = simple_open, .llseek = default_llseek, }; @@ -598,10 +654,6 @@ mt7915_fw_util_wm_show(struct seq_file *file, void *data) struct mt7915_dev *dev = file->private; seq_printf(file, "Program counter: 0x%x\n", mt76_rr(dev, MT_WM_MCU_PC)); - seq_printf(file, "Exception state: 0x%x\n", - is_mt7915(&dev->mt76) ? - (u32)mt76_get_field(dev, MT_FW_EXCEPTION, GENMASK(15, 8)) : - (u32)mt76_get_field(dev, MT_FW_EXCEPTION, GENMASK(7, 0))); if (dev->fw.debug_wm) { seq_printf(file, "Busy: %u%% Peak busy: %u%%\n", @@ -639,16 +691,17 @@ mt7915_ampdu_stat_read_phy(struct mt7915_phy *phy, { struct mt7915_dev *dev = phy->dev; bool ext_phy = phy != &dev->phy; - int bound[15], range[4], i, n; + int bound[15], range[4], i; + u8 band = phy->mt76->band_idx; /* Tx ampdu stat */ for (i = 0; i < ARRAY_SIZE(range); i++) - range[i] = mt76_rr(dev, MT_MIB_ARNG(phy->band_idx, i)); + range[i] = mt76_rr(dev, MT_MIB_ARNG(band, i)); for (i = 0; i < ARRAY_SIZE(bound); i++) bound[i] = MT_MIB_ARNCR_RANGE(range[i / 4], i % 4) + 1; - seq_printf(file, "\nPhy %d, Phy band %d\n", ext_phy, phy->band_idx); + seq_printf(file, "\nPhy %d, Phy band %d\n", ext_phy, band); seq_printf(file, "Length: %8d | ", bound[0]); for (i = 0; i < ARRAY_SIZE(bound) - 1; i++) @@ -656,9 +709,8 @@ mt7915_ampdu_stat_read_phy(struct mt7915_phy *phy, bound[i] + 1, bound[i + 1]); seq_puts(file, "\nCount: "); - n = phy->band_idx ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0; i < ARRAY_SIZE(bound); i++) - seq_printf(file, "%8d | ", dev->mt76.aggr_stats[i + n]); + seq_printf(file, "%8d | ", phy->mt76->aggr_stats[i]); seq_puts(file, "\n"); seq_printf(file, "BA miss count: %d\n", phy->mib.ba_miss_cnt); @@ -906,35 +958,199 @@ mt7915_xmit_queues_show(struct seq_file *file, void *data) DEFINE_SHOW_ATTRIBUTE(mt7915_xmit_queues); -static int -mt7915_rate_txpower_show(struct seq_file *file, void *data) +#define mt7915_txpower_puts(prefix, rate) \ +({ \ + len += scnprintf(buf + len, sz - len, "%-16s:", #prefix " (tmac)"); \ + for (i = 0; i < mt7915_sku_group_len[rate]; i++, offs++) \ + len += scnprintf(buf + len, sz - len, " %6d", txpwr[offs]); \ + len += scnprintf(buf + len, sz - len, "\n"); \ +}) + +#define mt7915_txpower_sets(rate, pwr, flag) \ +({ \ + offs += len; \ + len = mt7915_sku_group_len[rate]; \ + if (mode == flag) { \ + for (i = 0; i < len; i++) \ + req.txpower_sku[offs + i] = pwr; \ + } \ +}) + +static ssize_t +mt7915_rate_txpower_get(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct mt7915_phy *phy = file->private_data; + struct mt7915_dev *dev = phy->dev; + s8 txpwr[MT7915_SKU_RATE_NUM]; + static const size_t sz = 2048; + u8 band = phy->mt76->band_idx; + int i, offs = 0, len = 0; + ssize_t ret; + char *buf; + u32 reg; + + buf = kzalloc(sz, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = mt7915_mcu_get_txpower_sku(phy, txpwr, sizeof(txpwr)); + if (ret) + return ret; + + /* Txpower propagation path: TMAC -> TXV -> BBP */ + len += scnprintf(buf + len, sz - len, + "\nPhy%d Tx power table (channel %d)\n", + phy != &dev->phy, phy->mt76->chandef.chan->hw_value); + len += scnprintf(buf + len, sz - len, "%-16s %6s %6s %6s %6s\n", + " ", "1m", "2m", "5m", "11m"); + mt7915_txpower_puts(CCK, SKU_CCK); + + len += scnprintf(buf + len, sz - len, + "%-16s %6s %6s %6s %6s %6s %6s %6s %6s\n", + " ", "6m", "9m", "12m", "18m", "24m", "36m", "48m", + "54m"); + mt7915_txpower_puts(OFDM, SKU_OFDM); + + len += scnprintf(buf + len, sz - len, + "%-16s %6s %6s %6s %6s %6s %6s %6s %6s\n", + " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4", + "mcs5", "mcs6", "mcs7"); + mt7915_txpower_puts(HT20, SKU_HT_BW20); + + len += scnprintf(buf + len, sz - len, + "%-16s %6s %6s %6s %6s %6s %6s %6s %6s %6s\n", + " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4", "mcs5", + "mcs6", "mcs7", "mcs32"); + mt7915_txpower_puts(HT40, SKU_HT_BW40); + + len += scnprintf(buf + len, sz - len, + "%-16s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s\n", + " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4", "mcs5", + "mcs6", "mcs7", "mcs8", "mcs9", "mcs10", "mcs11"); + mt7915_txpower_puts(VHT20, SKU_VHT_BW20); + mt7915_txpower_puts(VHT40, SKU_VHT_BW40); + mt7915_txpower_puts(VHT80, SKU_VHT_BW80); + mt7915_txpower_puts(VHT160, SKU_VHT_BW160); + mt7915_txpower_puts(HE26, SKU_HE_RU26); + mt7915_txpower_puts(HE52, SKU_HE_RU52); + mt7915_txpower_puts(HE106, SKU_HE_RU106); + mt7915_txpower_puts(HE242, SKU_HE_RU242); + mt7915_txpower_puts(HE484, SKU_HE_RU484); + mt7915_txpower_puts(HE996, SKU_HE_RU996); + mt7915_txpower_puts(HE996x2, SKU_HE_RU2x996); + + reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_TPC_CTRL_STAT(band) : + MT_WF_PHY_TPC_CTRL_STAT_MT7916(band); + + len += scnprintf(buf + len, sz - len, "\nTx power (bbp) : %6ld\n", + mt76_get_field(dev, reg, MT_WF_PHY_TPC_POWER)); + + ret = simple_read_from_buffer(user_buf, count, ppos, buf, len); + kfree(buf); + return ret; +} + +static ssize_t +mt7915_rate_txpower_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) { - static const char * const sku_group_name[] = { - "CCK", "OFDM", "HT20", "HT40", - "VHT20", "VHT40", "VHT80", "VHT160", - "RU26", "RU52", "RU106", "RU242/SU20", - "RU484/SU40", "RU996/SU80", "RU2x996/SU160" + struct mt7915_phy *phy = file->private_data; + struct mt7915_dev *dev = phy->dev; + struct mt76_phy *mphy = phy->mt76; + struct mt7915_mcu_txpower_sku req = { + .format_id = TX_POWER_LIMIT_TABLE, + .band_idx = phy->mt76->band_idx, }; - struct mt7915_phy *phy = file->private; - s8 txpower[MT7915_SKU_RATE_NUM], *buf; - int i; + char buf[100]; + int i, ret, pwr160 = 0, pwr80 = 0, pwr40 = 0, pwr20 = 0; + enum mac80211_rx_encoding mode; + u32 offs = 0, len = 0; - seq_printf(file, "\nBand %d\n", phy != &phy->dev->phy); - mt7915_mcu_get_txpower_sku(phy, txpower, sizeof(txpower)); - for (i = 0, buf = txpower; i < ARRAY_SIZE(mt7915_sku_group_len); i++) { - u8 mcs_num = mt7915_sku_group_len[i]; + if (count >= sizeof(buf)) + return -EINVAL; - if (i >= SKU_VHT_BW20 && i <= SKU_VHT_BW160) - mcs_num = 10; + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; - mt76_seq_puts_array(file, sku_group_name[i], buf, mcs_num); - buf += mt7915_sku_group_len[i]; + if (count && buf[count - 1] == '\n') + buf[count - 1] = '\0'; + else + buf[count] = '\0'; + + if (sscanf(buf, "%u %u %u %u %u", + &mode, &pwr160, &pwr80, &pwr40, &pwr20) != 5) { + dev_warn(dev->mt76.dev, + "per bandwidth power limit: Mode BW160 BW80 BW40 BW20"); + return -EINVAL; } - return 0; + if (mode > RX_ENC_HE) + return -EINVAL; + + if (pwr160) + pwr160 = mt7915_get_power_bound(phy, pwr160); + if (pwr80) + pwr80 = mt7915_get_power_bound(phy, pwr80); + if (pwr40) + pwr40 = mt7915_get_power_bound(phy, pwr40); + if (pwr20) + pwr20 = mt7915_get_power_bound(phy, pwr20); + + if (pwr160 < 0 || pwr80 < 0 || pwr40 < 0 || pwr20 < 0) + return -EINVAL; + + mutex_lock(&dev->mt76.mutex); + ret = mt7915_mcu_get_txpower_sku(phy, req.txpower_sku, + sizeof(req.txpower_sku)); + if (ret) + goto out; + + mt7915_txpower_sets(SKU_CCK, pwr20, RX_ENC_LEGACY); + mt7915_txpower_sets(SKU_OFDM, pwr20, RX_ENC_LEGACY); + if (mode == RX_ENC_LEGACY) + goto skip; + + mt7915_txpower_sets(SKU_HT_BW20, pwr20, RX_ENC_HT); + mt7915_txpower_sets(SKU_HT_BW40, pwr40, RX_ENC_HT); + if (mode == RX_ENC_HT) + goto skip; + + mt7915_txpower_sets(SKU_VHT_BW20, pwr20, RX_ENC_VHT); + mt7915_txpower_sets(SKU_VHT_BW40, pwr40, RX_ENC_VHT); + mt7915_txpower_sets(SKU_VHT_BW80, pwr80, RX_ENC_VHT); + mt7915_txpower_sets(SKU_VHT_BW160, pwr160, RX_ENC_VHT); + if (mode == RX_ENC_VHT) + goto skip; + + mt7915_txpower_sets(SKU_HE_RU26, pwr20, RX_ENC_HE + 1); + mt7915_txpower_sets(SKU_HE_RU52, pwr20, RX_ENC_HE + 1); + mt7915_txpower_sets(SKU_HE_RU106, pwr20, RX_ENC_HE + 1); + mt7915_txpower_sets(SKU_HE_RU242, pwr20, RX_ENC_HE); + mt7915_txpower_sets(SKU_HE_RU484, pwr40, RX_ENC_HE); + mt7915_txpower_sets(SKU_HE_RU996, pwr80, RX_ENC_HE); + mt7915_txpower_sets(SKU_HE_RU2x996, pwr160, RX_ENC_HE); +skip: + ret = mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), + &req, sizeof(req), true); + if (ret) + goto out; + + mphy->txpower_cur = max(mphy->txpower_cur, + max(pwr160, max(pwr80, max(pwr40, pwr20)))); +out: + mutex_unlock(&dev->mt76.mutex); + + return ret ? ret : count; } -DEFINE_SHOW_ATTRIBUTE(mt7915_rate_txpower); +static const struct file_operations mt7915_rate_txpower_fops = { + .write = mt7915_rate_txpower_set, + .read = mt7915_rate_txpower_get, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; static int mt7915_twt_stats(struct seq_file *s, void *data) @@ -963,7 +1179,7 @@ mt7915_twt_stats(struct seq_file *s, void *data) } /* The index of RF registers use the generic regidx, combined with two parts: - * WF selection [31:28] and offset [27:0]. + * WF selection [31:24] and offset [23:0]. */ static int mt7915_rf_regval_get(void *data, u64 *val) @@ -1010,7 +1226,8 @@ int mt7915_init_debugfs(struct mt7915_phy *phy) debugfs_create_file("xmit-queues", 0400, dir, phy, &mt7915_xmit_queues_fops); debugfs_create_file("tx_stats", 0400, dir, phy, &mt7915_tx_stats_fops); - debugfs_create_file("fw_ser", 0600, dir, phy, &mt7915_fw_ser_ops); + debugfs_create_file("sys_recovery", 0600, dir, phy, + &mt7915_sys_recovery_ops); debugfs_create_file("fw_debug_wm", 0600, dir, dev, &fops_fw_debug_wm); debugfs_create_file("fw_debug_wa", 0600, dir, dev, &fops_fw_debug_wa); debugfs_create_file("fw_debug_bin", 0600, dir, dev, &fops_fw_debug_bin); @@ -1026,7 +1243,7 @@ int mt7915_init_debugfs(struct mt7915_phy *phy) mt7915_twt_stats); debugfs_create_file("rf_regval", 0600, dir, dev, &fops_rf_regval); - if (!dev->dbdc_support || phy->band_idx) { + if (!dev->dbdc_support || phy->mt76->band_idx) { debugfs_create_u32("dfs_hw_pattern", 0400, dir, &dev->hw_pattern); debugfs_create_file("radar_trigger", 0200, dir, dev, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c index 00aafc2422f3..e3fa064918bf 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c @@ -11,7 +11,11 @@ mt7915_init_tx_queues(struct mt7915_phy *phy, int idx, int n_desc, int ring_base struct mt7915_dev *dev = phy->dev; if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) { - ring_base = MT_WED_TX_RING_BASE; + if (is_mt7986(&dev->mt76)) + ring_base += MT_TXQ_ID(0) * MT_RING_SIZE; + else + ring_base = MT_WED_TX_RING_BASE; + idx -= MT_TXQ_ID(0); } @@ -46,29 +50,65 @@ static void mt7915_dma_config(struct mt7915_dev *dev) #define TXQ_CONFIG(q, wfdma, int, id) Q_CONFIG(__TXQ(q), (wfdma), (int), (id)) if (is_mt7915(&dev->mt76)) { - RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, MT7915_RXQ_BAND0); - RXQ_CONFIG(MT_RXQ_MCU, WFDMA1, MT_INT_RX_DONE_WM, MT7915_RXQ_MCU_WM); - RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA1, MT_INT_RX_DONE_WA, MT7915_RXQ_MCU_WA); - RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7915_RXQ_BAND1); - RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA1, MT_INT_RX_DONE_WA_EXT, MT7915_RXQ_MCU_WA_EXT); - RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA1, MT_INT_RX_DONE_WA_MAIN, MT7915_RXQ_MCU_WA); + RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, + MT7915_RXQ_BAND0); + RXQ_CONFIG(MT_RXQ_MCU, WFDMA1, MT_INT_RX_DONE_WM, + MT7915_RXQ_MCU_WM); + RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA1, MT_INT_RX_DONE_WA, + MT7915_RXQ_MCU_WA); + RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, + MT7915_RXQ_BAND1); + RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA1, MT_INT_RX_DONE_WA_EXT, + MT7915_RXQ_MCU_WA_EXT); + RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA1, MT_INT_RX_DONE_WA_MAIN, + MT7915_RXQ_MCU_WA); TXQ_CONFIG(0, WFDMA1, MT_INT_TX_DONE_BAND0, MT7915_TXQ_BAND0); TXQ_CONFIG(1, WFDMA1, MT_INT_TX_DONE_BAND1, MT7915_TXQ_BAND1); - MCUQ_CONFIG(MT_MCUQ_WM, WFDMA1, MT_INT_TX_DONE_MCU_WM, MT7915_TXQ_MCU_WM); - MCUQ_CONFIG(MT_MCUQ_WA, WFDMA1, MT_INT_TX_DONE_MCU_WA, MT7915_TXQ_MCU_WA); - MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA1, MT_INT_TX_DONE_FWDL, MT7915_TXQ_FWDL); + MCUQ_CONFIG(MT_MCUQ_WM, WFDMA1, MT_INT_TX_DONE_MCU_WM, + MT7915_TXQ_MCU_WM); + MCUQ_CONFIG(MT_MCUQ_WA, WFDMA1, MT_INT_TX_DONE_MCU_WA, + MT7915_TXQ_MCU_WA); + MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA1, MT_INT_TX_DONE_FWDL, + MT7915_TXQ_FWDL); } else { - RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0_MT7916, MT7916_RXQ_BAND0); - RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, MT7916_RXQ_MCU_WM); - RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, MT7916_RXQ_MCU_WA); - RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1_MT7916, MT7916_RXQ_BAND1); - RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT_MT7916, MT7916_RXQ_MCU_WA_EXT); - RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN_MT7916, MT7916_RXQ_MCU_WA_MAIN); - TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7915_TXQ_BAND0); - TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7915_TXQ_BAND1); - MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7915_TXQ_MCU_WM); - MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA_MT7916, MT7915_TXQ_MCU_WA); - MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, MT7915_TXQ_FWDL); + RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, + MT7916_RXQ_MCU_WM); + RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT_MT7916, + MT7916_RXQ_MCU_WA_EXT); + MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, + MT7915_TXQ_MCU_WM); + MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA_MT7916, + MT7915_TXQ_MCU_WA); + MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, + MT7915_TXQ_FWDL); + + if (is_mt7916(&dev->mt76) && mtk_wed_device_active(&dev->mt76.mmio.wed)) { + RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_WED_RX_DONE_BAND0_MT7916, + MT7916_RXQ_BAND0); + RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_WED_RX_DONE_WA_MT7916, + MT7916_RXQ_MCU_WA); + RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_WED_RX_DONE_BAND1_MT7916, + MT7916_RXQ_BAND1); + RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_WED_RX_DONE_WA_MAIN_MT7916, + MT7916_RXQ_MCU_WA_MAIN); + TXQ_CONFIG(0, WFDMA0, MT_INT_WED_TX_DONE_BAND0, + MT7915_TXQ_BAND0); + TXQ_CONFIG(1, WFDMA0, MT_INT_WED_TX_DONE_BAND1, + MT7915_TXQ_BAND1); + } else { + RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0_MT7916, + MT7916_RXQ_BAND0); + RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, + MT7916_RXQ_MCU_WA); + RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1_MT7916, + MT7916_RXQ_BAND1); + RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN_MT7916, + MT7916_RXQ_MCU_WA_MAIN); + TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, + MT7915_TXQ_BAND0); + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, + MT7915_TXQ_BAND1); + } } } @@ -313,17 +353,26 @@ static int mt7915_dma_enable(struct mt7915_dev *dev) MT_INT_TX_DONE_MCU | MT_INT_MCU_CMD; - if (!dev->phy.band_idx) + if (!dev->phy.mt76->band_idx) irq_mask |= MT_INT_BAND0_RX_DONE; - if (dev->dbdc_support || dev->phy.band_idx) + if (dev->dbdc_support || dev->phy.mt76->band_idx) irq_mask |= MT_INT_BAND1_RX_DONE; if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { u32 wed_irq_mask = irq_mask; + int ret; wed_irq_mask |= MT_INT_TX_DONE_BAND0 | MT_INT_TX_DONE_BAND1; - mt76_wr(dev, MT_INT_WED_MASK_CSR, wed_irq_mask); + if (!is_mt7986(&dev->mt76)) + mt76_wr(dev, MT_INT_WED_MASK_CSR, wed_irq_mask); + else + mt76_wr(dev, MT_INT_MASK_CSR, wed_irq_mask); + + ret = mt7915_mcu_wed_enable_rx_stats(dev); + if (ret) + return ret; + mtk_wed_device_start(&dev->mt76.mmio.wed, wed_irq_mask); } @@ -348,20 +397,28 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) mt7915_dma_disable(dev, true); - if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { - mt76_set(dev, MT_WFDMA_HOST_CONFIG, MT_WFDMA_HOST_CONFIG_WED); - - mt76_wr(dev, MT_WFDMA_WED_RING_CONTROL, - FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX0, 18) | - FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX1, 19) | - FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_RX1, 1)); + if (mtk_wed_device_active(&mdev->mmio.wed)) { + if (!is_mt7986(mdev)) { + u8 wed_control_rx1 = is_mt7915(mdev) ? 1 : 2; + + mt76_set(dev, MT_WFDMA_HOST_CONFIG, + MT_WFDMA_HOST_CONFIG_WED); + mt76_wr(dev, MT_WFDMA_WED_RING_CONTROL, + FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX0, 18) | + FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX1, 19) | + FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_RX1, + wed_control_rx1)); + if (is_mt7915(mdev)) + mt76_rmw(dev, MT_WFDMA0_EXT0_CFG, MT_WFDMA0_EXT0_RXWB_KEEP, + MT_WFDMA0_EXT0_RXWB_KEEP); + } } else { mt76_clear(dev, MT_WFDMA_HOST_CONFIG, MT_WFDMA_HOST_CONFIG_WED); } /* init tx queue */ ret = mt7915_init_tx_queues(&dev->phy, - MT_TXQ_ID(dev->phy.band_idx), + MT_TXQ_ID(dev->phy.mt76->band_idx), MT7915_TX_RING_SIZE, MT_TXQ_RING_BASE(0)); if (ret) @@ -369,7 +426,7 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) if (phy2) { ret = mt7915_init_tx_queues(phy2, - MT_TXQ_ID(phy2->band_idx), + MT_TXQ_ID(phy2->mt76->band_idx), MT7915_TX_RING_SIZE, MT_TXQ_RING_BASE(1)); if (ret) @@ -410,7 +467,7 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) return ret; /* event from WA */ - if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { + if (mtk_wed_device_active(&mdev->mmio.wed) && is_mt7915(mdev)) { wa_rx_base = MT_WED_RX_RING_BASE; wa_rx_idx = MT7915_RXQ_MCU_WA; dev->mt76.q_rx[MT_RXQ_MCU_WA].flags = MT_WED_Q_TXFREE; @@ -425,7 +482,14 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) return ret; /* rx data queue for band0 */ - if (!dev->phy.band_idx) { + if (!dev->phy.mt76->band_idx) { + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) { + dev->mt76.q_rx[MT_RXQ_MAIN].flags = + MT_WED_Q_RX(MT7915_RXQ_BAND0); + dev->mt76.rx_token_size += MT7915_RX_RING_SIZE; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN], MT_RXQ_ID(MT_RXQ_MAIN), MT7915_RX_RING_SIZE, @@ -437,16 +501,32 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) /* tx free notify event from WA for band0 */ if (!is_mt7915(mdev)) { + wa_rx_base = MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA); + wa_rx_idx = MT_RXQ_ID(MT_RXQ_MAIN_WA); + + if (mtk_wed_device_active(&mdev->mmio.wed)) { + mdev->q_rx[MT_RXQ_MAIN_WA].flags = MT_WED_Q_TXFREE; + if (is_mt7916(mdev)) { + wa_rx_base = MT_WED_RX_RING_BASE; + wa_rx_idx = MT7915_RXQ_MCU_WA; + } + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA], - MT_RXQ_ID(MT_RXQ_MAIN_WA), - MT7915_RX_MCU_RING_SIZE, - MT_RX_BUF_SIZE, - MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA)); + wa_rx_idx, MT7915_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, wa_rx_base); if (ret) return ret; } - if (dev->dbdc_support || dev->phy.band_idx) { + if (dev->dbdc_support || dev->phy.mt76->band_idx) { + if (mtk_wed_device_active(&mdev->mmio.wed) && + mtk_wed_get_rx_capa(&mdev->mmio.wed)) { + dev->mt76.q_rx[MT_RXQ_BAND1].flags = + MT_WED_Q_RX(MT7915_RXQ_BAND1); + dev->mt76.rx_token_size += MT7915_RX_RING_SIZE; + } + /* rx data queue for band1 */ ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1], MT_RXQ_ID(MT_RXQ_BAND1), @@ -479,6 +559,53 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) return 0; } +int mt7915_dma_reset(struct mt7915_dev *dev, bool force) +{ + struct mt76_phy *mphy_ext = dev->mt76.phys[MT_BAND1]; + int i; + + /* clean up hw queues */ + for (i = 0; i < ARRAY_SIZE(dev->mt76.phy.q_tx); i++) { + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true); + if (mphy_ext) + mt76_queue_tx_cleanup(dev, mphy_ext->q_tx[i], true); + } + + for (i = 0; i < ARRAY_SIZE(dev->mt76.q_mcu); i++) + mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true); + + mt76_for_each_q_rx(&dev->mt76, i) + mt76_queue_rx_cleanup(dev, &dev->mt76.q_rx[i]); + + /* reset wfsys */ + if (force) + mt7915_wfsys_reset(dev); + + mt7915_dma_disable(dev, force); + + /* reset hw queues */ + for (i = 0; i < __MT_TXQ_MAX; i++) { + mt76_queue_reset(dev, dev->mphy.q_tx[i]); + if (mphy_ext) + mt76_queue_reset(dev, mphy_ext->q_tx[i]); + } + + for (i = 0; i < __MT_MCUQ_MAX; i++) + mt76_queue_reset(dev, dev->mt76.q_mcu[i]); + + mt76_for_each_q_rx(&dev->mt76, i) + mt76_queue_reset(dev, &dev->mt76.q_rx[i]); + + mt76_tx_status_check(&dev->mt76, true); + + mt7915_dma_enable(dev); + + mt76_for_each_q_rx(&dev->mt76, i) + mt76_queue_rx_reset(dev, i); + + return 0; +} + void mt7915_dma_cleanup(struct mt7915_dev *dev) { mt7915_dma_disable(dev, true); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index 4b1a9811646f..59069fb86414 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -131,9 +131,10 @@ static void mt7915_eeprom_parse_band_config(struct mt7915_phy *phy) { struct mt7915_dev *dev = phy->dev; u8 *eeprom = dev->mt76.eeprom.data; + u8 band = phy->mt76->band_idx; u32 val; - val = eeprom[MT_EE_WIFI_CONF + phy->band_idx]; + val = eeprom[MT_EE_WIFI_CONF + band]; val = FIELD_GET(MT_EE_WIFI_CONF0_BAND_SEL, val); if (!is_mt7915(&dev->mt76)) { @@ -153,7 +154,7 @@ static void mt7915_eeprom_parse_band_config(struct mt7915_phy *phy) return; } } else if (val == MT_EE_BAND_SEL_DEFAULT && dev->dbdc_support) { - val = phy->band_idx ? MT_EE_BAND_SEL_5GHZ : MT_EE_BAND_SEL_2GHZ; + val = band ? MT_EE_BAND_SEL_5GHZ : MT_EE_BAND_SEL_2GHZ; } switch (val) { @@ -173,60 +174,51 @@ static void mt7915_eeprom_parse_band_config(struct mt7915_phy *phy) void mt7915_eeprom_parse_hw_cap(struct mt7915_dev *dev, struct mt7915_phy *phy) { - u8 nss, nss_band, nss_band_max, *eeprom = dev->mt76.eeprom.data; + u8 path, nss, nss_max = 4, *eeprom = dev->mt76.eeprom.data; struct mt76_phy *mphy = phy->mt76; - bool ext_phy = phy != &dev->phy; + u8 band = phy->mt76->band_idx; mt7915_eeprom_parse_band_config(phy); - /* read tx/rx mask from eeprom */ + /* read tx/rx path from eeprom */ if (is_mt7915(&dev->mt76)) { - nss = FIELD_GET(MT_EE_WIFI_CONF0_TX_PATH, - eeprom[MT_EE_WIFI_CONF]); + path = FIELD_GET(MT_EE_WIFI_CONF0_TX_PATH, + eeprom[MT_EE_WIFI_CONF]); } else { - nss = FIELD_GET(MT_EE_WIFI_CONF0_TX_PATH, - eeprom[MT_EE_WIFI_CONF + phy->band_idx]); + path = FIELD_GET(MT_EE_WIFI_CONF0_TX_PATH, + eeprom[MT_EE_WIFI_CONF + band]); } - if (!nss || nss > 4) - nss = 4; + if (!path || path > 4) + path = 4; /* read tx/rx stream */ - nss_band = nss; - + nss = path; if (dev->dbdc_support) { if (is_mt7915(&dev->mt76)) { - nss_band = FIELD_GET(MT_EE_WIFI_CONF3_TX_PATH_B0, - eeprom[MT_EE_WIFI_CONF + 3]); - if (phy->band_idx) - nss_band = FIELD_GET(MT_EE_WIFI_CONF3_TX_PATH_B1, - eeprom[MT_EE_WIFI_CONF + 3]); + path = min_t(u8, path, 2); + nss = FIELD_GET(MT_EE_WIFI_CONF3_TX_PATH_B0, + eeprom[MT_EE_WIFI_CONF + 3]); + if (band) + nss = FIELD_GET(MT_EE_WIFI_CONF3_TX_PATH_B1, + eeprom[MT_EE_WIFI_CONF + 3]); } else { - nss_band = FIELD_GET(MT_EE_WIFI_CONF_STREAM_NUM, - eeprom[MT_EE_WIFI_CONF + 2 + phy->band_idx]); + nss = FIELD_GET(MT_EE_WIFI_CONF_STREAM_NUM, + eeprom[MT_EE_WIFI_CONF + 2 + band]); } - nss_band_max = is_mt7986(&dev->mt76) ? - MT_EE_NSS_MAX_DBDC_MA7986 : MT_EE_NSS_MAX_DBDC_MA7915; - } else { - nss_band_max = is_mt7986(&dev->mt76) ? - MT_EE_NSS_MAX_MA7986 : MT_EE_NSS_MAX_MA7915; + if (!is_mt7986(&dev->mt76)) + nss_max = 2; } - if (!nss_band || nss_band > nss_band_max) - nss_band = nss_band_max; - - if (nss_band > nss) { - dev_warn(dev->mt76.dev, - "nss mismatch, nss(%d) nss_band(%d) band(%d) ext_phy(%d)\n", - nss, nss_band, phy->band_idx, ext_phy); - nss = nss_band; - } + if (!nss) + nss = nss_max; + nss = min_t(u8, min_t(u8, nss_max, nss), path); - mphy->chainmask = BIT(nss) - 1; - if (ext_phy) + mphy->chainmask = BIT(path) - 1; + if (band) mphy->chainmask <<= dev->chainshift; - mphy->antenna_mask = BIT(nss_band) - 1; + mphy->antenna_mask = BIT(nss) - 1; dev->chainmask |= mphy->chainmask; dev->chainshift = hweight8(dev->mphy.chainmask); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h index 7578ac6d0be6..f3e56817d36e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h @@ -58,11 +58,6 @@ enum mt7915_eeprom_field { #define MT_EE_RATE_DELTA_SIGN BIT(6) #define MT_EE_RATE_DELTA_EN BIT(7) -#define MT_EE_NSS_MAX_MA7915 4 -#define MT_EE_NSS_MAX_DBDC_MA7915 2 -#define MT_EE_NSS_MAX_MA7986 4 -#define MT_EE_NSS_MAX_DBDC_MA7986 4 - enum mt7915_adie_sku { MT7976_ONE_ADIE_DBDC = 0x7, MT7975_ONE_ADIE = 0x8, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index cc2aac86bcfb..c810c31fbd6e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -8,6 +8,7 @@ #include "mt7915.h" #include "mac.h" #include "mcu.h" +#include "coredump.h" #include "eeprom.h" static const struct ieee80211_iface_limit if_limits[] = { @@ -262,9 +263,8 @@ static void mt7915_led_set_brightness(struct led_classdev *led_cdev, mt7915_led_set_config(led_cdev, 0xff, 0); } -static void -mt7915_init_txpower(struct mt7915_dev *dev, - struct ieee80211_supported_band *sband) +void mt7915_init_txpower(struct mt7915_dev *dev, + struct ieee80211_supported_band *sband) { int i, n_chains = hweight8(dev->mphy.antenna_mask); int nss_delta = mt76_tx_power_nss_delta(n_chains); @@ -353,6 +353,10 @@ mt7915_init_wiphy(struct ieee80211_hw *hw) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_HE); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_DISCOVERY); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT); + + if (!is_mt7915(&dev->mt76)) + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_STA_TX_PWR); if (!mdev->dev->of_node || !of_property_read_bool(mdev->dev->of_node, @@ -444,9 +448,32 @@ mt7915_mac_init_band(struct mt7915_dev *dev, u8 band) /* mt7915: disable rx rate report by default due to hw issues */ mt76_clear(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN); + + /* clear estimated value of EIFS for Rx duration & OBSS time */ + mt76_wr(dev, MT_WF_RMAC_RSVD0(band), MT_WF_RMAC_RSVD0_EIFS_CLR); + + /* clear backoff time for Rx duration */ + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME1(band), + MT_WF_RMAC_MIB_NONQOSD_BACKOFF); + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME3(band), + MT_WF_RMAC_MIB_QOS01_BACKOFF); + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME4(band), + MT_WF_RMAC_MIB_QOS23_BACKOFF); + + /* clear backoff time and set software compensation for OBSS time */ + mask = MT_WF_RMAC_MIB_OBSS_BACKOFF | MT_WF_RMAC_MIB_ED_OFFSET; + set = FIELD_PREP(MT_WF_RMAC_MIB_OBSS_BACKOFF, 0) | + FIELD_PREP(MT_WF_RMAC_MIB_ED_OFFSET, 4); + mt76_rmw(dev, MT_WF_RMAC_MIB_AIRTIME0(band), mask, set); + + /* filter out non-resp frames and get instanstaeous signal reporting */ + mask = MT_WTBLOFF_TOP_RSCR_RCPI_MODE | MT_WTBLOFF_TOP_RSCR_RCPI_PARAM; + set = FIELD_PREP(MT_WTBLOFF_TOP_RSCR_RCPI_MODE, 0) | + FIELD_PREP(MT_WTBLOFF_TOP_RSCR_RCPI_PARAM, 0x3); + mt76_rmw(dev, MT_WTBLOFF_TOP_RSCR(band), mask, set); } -static void mt7915_mac_init(struct mt7915_dev *dev) +void mt7915_mac_init(struct mt7915_dev *dev) { int i; u32 rx_len = is_mt7915(&dev->mt76) ? 0x400 : 0x680; @@ -476,7 +503,7 @@ static void mt7915_mac_init(struct mt7915_dev *dev) } } -static int mt7915_txbf_init(struct mt7915_dev *dev) +int mt7915_txbf_init(struct mt7915_dev *dev) { int ret; @@ -513,7 +540,7 @@ mt7915_alloc_ext_phy(struct mt7915_dev *dev) phy->mt76 = mphy; /* Bind main phy to band0 and ext_phy to band1 for dbdc case */ - phy->band_idx = 1; + phy->mt76->band_idx = 1; return phy; } @@ -633,7 +660,7 @@ static bool mt7915_band_config(struct mt7915_dev *dev) { bool ret = true; - dev->phy.band_idx = 0; + dev->phy.mt76->band_idx = 0; if (is_mt7986(&dev->mt76)) { u32 sku = mt7915_check_adie(dev, true); @@ -644,7 +671,7 @@ static bool mt7915_band_config(struct mt7915_dev *dev) * dbdc is disabled. */ if (sku == MT7975_ONE_ADIE || sku == MT7976_ONE_ADIE) { - dev->phy.band_idx = 1; + dev->phy.mt76->band_idx = 1; ret = false; } } else { @@ -700,45 +727,49 @@ mt7915_init_hardware(struct mt7915_dev *dev, struct mt7915_phy *phy2) void mt7915_set_stream_vht_txbf_caps(struct mt7915_phy *phy) { - int nss; + int sts; u32 *cap; if (!phy->mt76->cap.has_5ghz) return; - nss = hweight8(phy->mt76->chainmask); + sts = hweight8(phy->mt76->chainmask); cap = &phy->mt76->sband_5g.sband.vht_cap.cap; *cap |= IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE | IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE | - (3 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT); + FIELD_PREP(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK, + sts - 1); *cap &= ~(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK | IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE | IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE); - if (nss < 2) + if (sts < 2) return; *cap |= IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE | IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE | FIELD_PREP(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK, - nss - 1); + sts - 1); } static void -mt7915_set_stream_he_txbf_caps(struct mt7915_dev *dev, - struct ieee80211_sta_he_cap *he_cap, - int vif, int nss) +mt7915_set_stream_he_txbf_caps(struct mt7915_phy *phy, + struct ieee80211_sta_he_cap *he_cap, int vif) { + struct mt7915_dev *dev = phy->dev; struct ieee80211_he_cap_elem *elem = &he_cap->he_cap_elem; - u8 c, nss_160; + int sts = hweight8(phy->mt76->chainmask); + u8 c, sts_160 = sts; - /* Can do 1/2 of NSS streams in 160Mhz mode for mt7915 */ - if (is_mt7915(&dev->mt76) && !dev->dbdc_support) - nss_160 = nss / 2; - else - nss_160 = nss; + /* Can do 1/2 of STS in 160Mhz mode for mt7915 */ + if (is_mt7915(&dev->mt76)) { + if (!dev->dbdc_support) + sts_160 /= 2; + else + sts_160 = 0; + } #ifdef CONFIG_MAC80211_MESH if (vif == NL80211_IFTYPE_MESH_POINT) @@ -748,8 +779,9 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_dev *dev, elem->phy_cap_info[3] &= ~IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; elem->phy_cap_info[4] &= ~IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER; - c = IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK | - IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK; + c = IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK; + if (sts_160) + c |= IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK; elem->phy_cap_info[5] &= ~c; c = IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | @@ -765,8 +797,9 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_dev *dev, elem->phy_cap_info[2] |= c; c = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | - IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4 | - IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4; + if (sts_160) + c |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; elem->phy_cap_info[4] |= c; /* do not support NG16 due to spec D4.0 changes subcarrier idx */ @@ -778,11 +811,11 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_dev *dev, elem->phy_cap_info[6] |= c; - if (nss < 2) + if (sts < 2) return; /* the maximum cap is 4 x 3, (Nr, Nc) = (3, 2) */ - elem->phy_cap_info[7] |= min_t(int, nss - 1, 2) << 3; + elem->phy_cap_info[7] |= min_t(int, sts - 1, 2) << 3; if (vif != NL80211_IFTYPE_AP) return; @@ -791,12 +824,13 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_dev *dev, elem->phy_cap_info[4] |= IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER; /* num_snd_dim - * for mt7915, max supported nss is 2 for bw > 80MHz + * for mt7915, max supported sts is 2 for bw > 80MHz and 0 if dbdc */ c = FIELD_PREP(IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK, - nss - 1) | - FIELD_PREP(IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK, - nss_160 - 1); + sts - 1); + if (sts_160) + c |= FIELD_PREP(IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK, + sts_160 - 1); elem->phy_cap_info[5] |= c; c = IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | @@ -836,16 +870,19 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band, struct ieee80211_sband_iftype_data *data) { struct mt7915_dev *dev = phy->dev; - int i, idx = 0, nss = hweight8(phy->mt76->chainmask); + int i, idx = 0, nss = hweight8(phy->mt76->antenna_mask); u16 mcs_map = 0; u16 mcs_map_160 = 0; u8 nss_160; - /* Can do 1/2 of NSS streams in 160Mhz mode for mt7915 */ - if (is_mt7915(&dev->mt76) && !dev->dbdc_support) + if (!is_mt7915(&dev->mt76)) + nss_160 = nss; + else if (!dev->dbdc_support) + /* Can do 1/2 of NSS streams in 160Mhz mode for mt7915 */ nss_160 = nss / 2; else - nss_160 = nss; + /* Can't do 160MHz with mt7915 dbdc */ + nss_160 = 0; for (i = 0; i < 8; i++) { if (i < nss) @@ -891,11 +928,14 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band, if (band == NL80211_BAND_2GHZ) he_cap_elem->phy_cap_info[0] = IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G; - else + else if (nss_160) he_cap_elem->phy_cap_info[0] = IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G; + else + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G; he_cap_elem->phy_cap_info[1] = IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD; @@ -949,9 +989,11 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band, IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI; he_cap_elem->phy_cap_info[8] |= IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G | - IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | - IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU | IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_484; + if (nss_160) + he_cap_elem->phy_cap_info[8] |= + IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; he_cap_elem->phy_cap_info[9] |= IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM | IEEE80211_HE_PHY_CAP9_NON_TRIGGERED_CQI_FEEDBACK | @@ -969,7 +1011,7 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band, he_mcs->rx_mcs_80p80 = cpu_to_le16(mcs_map_160); he_mcs->tx_mcs_80p80 = cpu_to_le16(mcs_map_160); - mt7915_set_stream_he_txbf_caps(dev, he_cap, i, nss); + mt7915_set_stream_he_txbf_caps(phy, he_cap, i); memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres)); if (he_cap_elem->phy_cap_info[6] & @@ -1078,6 +1120,8 @@ int mt7915_register_device(struct mt7915_dev *dev) init_waitqueue_head(&dev->reset_wait); INIT_WORK(&dev->reset_work, mt7915_mac_reset_work); + INIT_WORK(&dev->dump_work, mt7915_mac_dump_work); + mutex_init(&dev->dump_mutex); dev->dbdc_support = mt7915_band_config(dev); @@ -1118,7 +1162,15 @@ int mt7915_register_device(struct mt7915_dev *dev) goto unreg_thermal; } - mt7915_init_debugfs(&dev->phy); + dev->recovery.hw_init_done = true; + + ret = mt7915_init_debugfs(&dev->phy); + if (ret) + goto unreg_thermal; + + ret = mt7915_coredump_register(dev); + if (ret) + goto unreg_thermal; return 0; @@ -1137,6 +1189,7 @@ free_phy2: void mt7915_unregister_device(struct mt7915_dev *dev) { mt7915_unregister_ext_phy(dev); + mt7915_coredump_unregister(dev); mt7915_unregister_thermal(&dev->phy); mt76_unregister_device(&dev->mt76); mt7915_stop_hardware(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index a4bcc617c1a3..f0d5a3603902 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -3,12 +3,13 @@ #include <linux/etherdevice.h> #include <linux/timekeeping.h> +#include "coredump.h" #include "mt7915.h" #include "../dma.h" #include "mac.h" #include "mcu.h" -#define to_rssi(field, rxv) ((FIELD_GET(field, rxv) - 220) / 2) +#define to_rssi(field, rcpi) ((FIELD_GET(field, rcpi) - 220) / 2) static const struct mt7915_dfs_radar_spec etsi_radar_specs = { .pulse_th = { 110, -10, -80, 40, 5200, 128, 5200 }, @@ -118,6 +119,7 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) bool clear = false; u32 addr, val; u16 idx; + s8 rssi[4]; u8 bw; spin_lock_bh(&dev->sta_poll_lock); @@ -131,6 +133,8 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) spin_unlock_bh(&dev->sta_poll_lock); idx = msta->wcid.idx; + + /* refresh peer's airtime reporting */ addr = mt7915_mac_wtbl_lmac_addr(dev, idx, 20); for (i = 0; i < IEEE80211_NUM_ACS; i++) { @@ -161,9 +165,9 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) sta = container_of((void *)msta, struct ieee80211_sta, drv_priv); for (i = 0; i < IEEE80211_NUM_ACS; i++) { - u8 q = mt76_connac_lmac_mapping(i); - u32 tx_cur = tx_time[q]; - u32 rx_cur = rx_time[q]; + u8 queue = mt76_connac_lmac_mapping(i); + u32 tx_cur = tx_time[queue]; + u32 rx_cur = rx_time[queue]; u8 tid = ac_to_tid[i]; if (!tx_cur && !rx_cur) @@ -209,13 +213,69 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) else rate->flags &= ~RATE_INFO_FLAGS_SHORT_GI; } + + /* get signal strength of resp frames (CTS/BA/ACK) */ + addr = mt7915_mac_wtbl_lmac_addr(dev, idx, 30); + val = mt76_rr(dev, addr); + + rssi[0] = to_rssi(GENMASK(7, 0), val); + rssi[1] = to_rssi(GENMASK(15, 8), val); + rssi[2] = to_rssi(GENMASK(23, 16), val); + rssi[3] = to_rssi(GENMASK(31, 14), val); + + msta->ack_signal = + mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); + + ewma_avg_signal_add(&msta->avg_ack_signal, -msta->ack_signal); } rcu_read_unlock(); } +void mt7915_mac_enable_rtscts(struct mt7915_dev *dev, + struct ieee80211_vif *vif, bool enable) +{ + struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; + u32 addr; + + addr = mt7915_mac_wtbl_lmac_addr(dev, mvif->sta.wcid.idx, 5); + if (enable) + mt76_set(dev, addr, BIT(5)); + else + mt76_clear(dev, addr, BIT(5)); +} + +static void +mt7915_wed_check_ppe(struct mt7915_dev *dev, struct mt76_queue *q, + struct mt7915_sta *msta, struct sk_buff *skb, + u32 info) +{ + struct ieee80211_vif *vif; + struct wireless_dev *wdev; + + if (!msta || !msta->vif) + return; + + if (!(q->flags & MT_QFLAG_WED) || + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) != MT76_WED_Q_RX) + return; + + if (!(info & MT_DMA_INFO_PPE_VLD)) + return; + + vif = container_of((void *)msta->vif, struct ieee80211_vif, + drv_priv); + wdev = ieee80211_vif_to_wdev(vif); + skb->dev = wdev->netdev; + + mtk_wed_device_ppe_check(&dev->mt76.mmio.wed, skb, + FIELD_GET(MT_DMA_PPE_CPU_REASON, info), + FIELD_GET(MT_DMA_PPE_ENTRY, info)); +} + static int -mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) +mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb, + enum mt76_rxq_id q, u32 *info) { struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct mt76_phy *mphy = &dev->mt76.phy; @@ -242,7 +302,7 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) memset(status, 0, sizeof(*status)); - if ((rxd1 & MT_RXD1_NORMAL_BAND_IDX) && !phy->band_idx) { + if ((rxd1 & MT_RXD1_NORMAL_BAND_IDX) && !phy->mt76->band_idx) { mphy = dev->mt76.phys[MT_BAND1]; if (!mphy) return -EINVAL; @@ -482,6 +542,8 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) } } else { status->flag |= RX_FLAG_8023; + mt7915_wed_check_ppe(dev, &dev->mt76.q_rx[q], msta, skb, + *info); } if (rxv && mode >= MT_PHY_TYPE_HE_SU && !(status->flag & RX_FLAG_8023)) @@ -513,7 +575,7 @@ mt7915_mac_fill_rx_vector(struct mt7915_dev *dev, struct sk_buff *skb) int i; band_idx = le32_get_bits(rxv_hdr[1], MT_RXV_HDR_BAND_IDX); - if (band_idx && !phy->band_idx) { + if (band_idx && !phy->mt76->band_idx) { phy = mt7915_ext_phy(dev); if (!phy) goto out; @@ -905,17 +967,19 @@ mt7915_mac_tx_free(struct mt7915_dev *dev, void *data, int len) total = le16_get_bits(free->ctrl, MT_TX_FREE_MSDU_CNT); v3 = (FIELD_GET(MT_TX_FREE_VER, txd) == 0x4); - if (WARN_ON_ONCE((void *)&tx_info[total >> v3] > end)) - return; for (cur_info = tx_info; count < total; cur_info++) { - u32 msdu, info = le32_to_cpu(*cur_info); + u32 msdu, info; u8 i; + if (WARN_ON_ONCE((void *)cur_info >= end)) + return; + /* * 1'b1: new wcid pair. * 1'b0: msdu_id with the same 'wcid pair' as above. */ + info = le32_to_cpu(*cur_info); if (info & MT_TX_FREE_PAIR) { struct mt7915_sta *msta; struct mt76_wcid *wcid; @@ -1063,7 +1127,7 @@ bool mt7915_rx_check(struct mt76_dev *mdev, void *data, int len) } void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) + struct sk_buff *skb, u32 *info) { struct mt7915_dev *dev = container_of(mdev, struct mt7915_dev, mt76); __le32 *rxd = (__le32 *)skb->data; @@ -1097,7 +1161,7 @@ void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, dev_kfree_skb(skb); break; case PKT_TYPE_NORMAL: - if (!mt7915_mac_fill_rx(dev, skb)) { + if (!mt7915_mac_fill_rx(dev, skb, q, info)) { mt76_rx(&dev->mt76, q, skb); return; } @@ -1111,7 +1175,7 @@ void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, void mt7915_mac_cca_stats_reset(struct mt7915_phy *phy) { struct mt7915_dev *dev = phy->dev; - u32 reg = MT_WF_PHY_RX_CTRL1(phy->band_idx); + u32 reg = MT_WF_PHY_RX_CTRL1(phy->mt76->band_idx); mt76_clear(dev, reg, MT_WF_PHY_RX_CTRL1_STSCNT_EN); mt76_set(dev, reg, BIT(11) | BIT(9)); @@ -1123,19 +1187,15 @@ void mt7915_mac_reset_counters(struct mt7915_phy *phy) int i; for (i = 0; i < 4; i++) { - mt76_rr(dev, MT_TX_AGG_CNT(phy->band_idx, i)); - mt76_rr(dev, MT_TX_AGG_CNT2(phy->band_idx, i)); + mt76_rr(dev, MT_TX_AGG_CNT(phy->mt76->band_idx, i)); + mt76_rr(dev, MT_TX_AGG_CNT2(phy->mt76->band_idx, i)); } - i = 0; phy->mt76->survey_time = ktime_get_boottime(); - if (phy->band_idx) - i = ARRAY_SIZE(dev->mt76.aggr_stats) / 2; - - memset(&dev->mt76.aggr_stats[i], 0, sizeof(dev->mt76.aggr_stats) / 2); + memset(phy->mt76->aggr_stats, 0, sizeof(phy->mt76->aggr_stats)); /* reset airtime counters */ - mt76_set(dev, MT_WF_RMAC_MIB_AIRTIME0(phy->band_idx), + mt76_set(dev, MT_WF_RMAC_MIB_AIRTIME0(phy->mt76->band_idx), MT_WF_RMAC_MIB_RXTIME_CLR); mt7915_mcu_get_chan_mib_info(phy, true); @@ -1151,7 +1211,8 @@ void mt7915_mac_set_timing(struct mt7915_phy *phy) FIELD_PREP(MT_TIMEOUT_VAL_CCA, 48); u32 ofdm = FIELD_PREP(MT_TIMEOUT_VAL_PLCP, 60) | FIELD_PREP(MT_TIMEOUT_VAL_CCA, 28); - int offset; + u8 band = phy->mt76->band_idx; + int eifs_ofdm = 360, sifs = 10, offset; bool a_band = !(phy->mt76->chandef.chan->band == NL80211_BAND_2GHZ); if (!test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) @@ -1161,7 +1222,7 @@ void mt7915_mac_set_timing(struct mt7915_phy *phy) coverage_class = max_t(s16, dev->phy.coverage_class, ext_phy->coverage_class); - mt76_set(dev, MT_ARB_SCR(phy->band_idx), + mt76_set(dev, MT_ARB_SCR(band), MT_ARB_SCR_TX_DISABLE | MT_ARB_SCR_RX_DISABLE); udelay(1); @@ -1169,39 +1230,48 @@ void mt7915_mac_set_timing(struct mt7915_phy *phy) reg_offset = FIELD_PREP(MT_TIMEOUT_VAL_PLCP, offset) | FIELD_PREP(MT_TIMEOUT_VAL_CCA, offset); - mt76_wr(dev, MT_TMAC_CDTR(phy->band_idx), cck + reg_offset); - mt76_wr(dev, MT_TMAC_ODTR(phy->band_idx), ofdm + reg_offset); - mt76_wr(dev, MT_TMAC_ICR0(phy->band_idx), - FIELD_PREP(MT_IFS_EIFS_OFDM, a_band ? 84 : 78) | + if (!is_mt7915(&dev->mt76)) { + if (!a_band) { + mt76_wr(dev, MT_TMAC_ICR1(band), + FIELD_PREP(MT_IFS_EIFS_CCK, 314)); + eifs_ofdm = 78; + } else { + eifs_ofdm = 84; + } + } else if (a_band) { + sifs = 16; + } + + mt76_wr(dev, MT_TMAC_CDTR(band), cck + reg_offset); + mt76_wr(dev, MT_TMAC_ODTR(band), ofdm + reg_offset); + mt76_wr(dev, MT_TMAC_ICR0(band), + FIELD_PREP(MT_IFS_EIFS_OFDM, eifs_ofdm) | FIELD_PREP(MT_IFS_RIFS, 2) | - FIELD_PREP(MT_IFS_SIFS, 10) | + FIELD_PREP(MT_IFS_SIFS, sifs) | FIELD_PREP(MT_IFS_SLOT, phy->slottime)); - mt76_wr(dev, MT_TMAC_ICR1(phy->band_idx), - FIELD_PREP(MT_IFS_EIFS_CCK, 314)); - if (phy->slottime < 20 || a_band) val = MT7915_CFEND_RATE_DEFAULT; else val = MT7915_CFEND_RATE_11B; - mt76_rmw_field(dev, MT_AGG_ACR0(phy->band_idx), MT_AGG_ACR_CFEND_RATE, val); - mt76_clear(dev, MT_ARB_SCR(phy->band_idx), + mt76_rmw_field(dev, MT_AGG_ACR0(band), MT_AGG_ACR_CFEND_RATE, val); + mt76_clear(dev, MT_ARB_SCR(band), MT_ARB_SCR_TX_DISABLE | MT_ARB_SCR_RX_DISABLE); } -void mt7915_mac_enable_nf(struct mt7915_dev *dev, bool ext_phy) +void mt7915_mac_enable_nf(struct mt7915_dev *dev, bool band) { u32 reg; - reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_RXTD12(ext_phy) : - MT_WF_PHY_RXTD12_MT7916(ext_phy); + reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_RXTD12(band) : + MT_WF_PHY_RXTD12_MT7916(band); mt76_set(dev, reg, MT_WF_PHY_RXTD12_IRPI_SW_CLR_ONLY | MT_WF_PHY_RXTD12_IRPI_SW_CLR); - reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_RX_CTRL1(ext_phy) : - MT_WF_PHY_RX_CTRL1_MT7916(ext_phy); + reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_RX_CTRL1(band) : + MT_WF_PHY_RX_CTRL1_MT7916(band); mt76_set(dev, reg, FIELD_PREP(MT_WF_PHY_RX_CTRL1_IPI_EN, 0x5)); } @@ -1239,7 +1309,7 @@ void mt7915_update_channel(struct mt76_phy *mphy) mt7915_mcu_get_chan_mib_info(phy, false); - nf = mt7915_phy_get_nf(phy, phy->band_idx); + nf = mt7915_phy_get_nf(phy, phy->mt76->band_idx); if (!phy->noise) phy->noise = nf << 4; else if (nf) @@ -1254,7 +1324,7 @@ mt7915_wait_reset_state(struct mt7915_dev *dev, u32 state) bool ret; ret = wait_event_timeout(dev->reset_wait, - (READ_ONCE(dev->reset_state) & state), + (READ_ONCE(dev->recovery.state) & state), MT7915_RESET_TIMEOUT); WARN(!ret, "Timeout waiting for MCU reset state %x\n", state); @@ -1295,85 +1365,180 @@ mt7915_update_beacons(struct mt7915_dev *dev) mt7915_update_vif_beacon, mphy_ext->hw); } -static void -mt7915_dma_reset(struct mt7915_dev *dev) +void mt7915_tx_token_put(struct mt7915_dev *dev) { - struct mt76_phy *mphy_ext = dev->mt76.phys[MT_BAND1]; - u32 hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); - int i; + struct mt76_txwi_cache *txwi; + int id; - mt76_clear(dev, MT_WFDMA0_GLO_CFG, - MT_WFDMA0_GLO_CFG_TX_DMA_EN | - MT_WFDMA0_GLO_CFG_RX_DMA_EN); + spin_lock_bh(&dev->mt76.token_lock); + idr_for_each_entry(&dev->mt76.token, txwi, id) { + mt7915_txwi_free(dev, txwi, NULL, NULL); + dev->mt76.token_count--; + } + spin_unlock_bh(&dev->mt76.token_lock); + idr_destroy(&dev->mt76.token); +} + +static int +mt7915_mac_restart(struct mt7915_dev *dev) +{ + struct mt7915_phy *phy2; + struct mt76_phy *ext_phy; + struct mt76_dev *mdev = &dev->mt76; + int i, ret; + + ext_phy = dev->mt76.phys[MT_BAND1]; + phy2 = ext_phy ? ext_phy->priv : NULL; - if (is_mt7915(&dev->mt76)) - mt76_clear(dev, MT_WFDMA1_GLO_CFG, - MT_WFDMA1_GLO_CFG_TX_DMA_EN | - MT_WFDMA1_GLO_CFG_RX_DMA_EN); if (dev->hif2) { - mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, - MT_WFDMA0_GLO_CFG_TX_DMA_EN | - MT_WFDMA0_GLO_CFG_RX_DMA_EN); + mt76_wr(dev, MT_INT1_MASK_CSR, 0x0); + mt76_wr(dev, MT_INT1_SOURCE_CSR, ~0); + } - if (is_mt7915(&dev->mt76)) - mt76_clear(dev, MT_WFDMA1_GLO_CFG + hif1_ofs, - MT_WFDMA1_GLO_CFG_TX_DMA_EN | - MT_WFDMA1_GLO_CFG_RX_DMA_EN); + if (dev_is_pci(mdev->dev)) { + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0x0); + if (dev->hif2) + mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0x0); + } + + set_bit(MT76_RESET, &dev->mphy.state); + set_bit(MT76_MCU_RESET, &dev->mphy.state); + wake_up(&dev->mt76.mcu.wait); + if (ext_phy) { + set_bit(MT76_RESET, &ext_phy->state); + set_bit(MT76_MCU_RESET, &ext_phy->state); } - usleep_range(1000, 2000); + /* lock/unlock all queues to ensure that no tx is pending */ + mt76_txq_schedule_all(&dev->mphy); + if (ext_phy) + mt76_txq_schedule_all(ext_phy); - for (i = 0; i < __MT_TXQ_MAX; i++) { - mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true); - if (mphy_ext) - mt76_queue_tx_cleanup(dev, mphy_ext->q_tx[i], true); + /* disable all tx/rx napi */ + mt76_worker_disable(&dev->mt76.tx_worker); + mt76_for_each_q_rx(mdev, i) { + if (mdev->q_rx[i].ndesc) + napi_disable(&dev->mt76.napi[i]); } + napi_disable(&dev->mt76.tx_napi); - for (i = 0; i < __MT_MCUQ_MAX; i++) - mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true); + /* token reinit */ + mt7915_tx_token_put(dev); + idr_init(&dev->mt76.token); - mt76_for_each_q_rx(&dev->mt76, i) - mt76_queue_rx_reset(dev, i); + mt7915_dma_reset(dev, true); - mt76_tx_status_check(&dev->mt76, true); + local_bh_disable(); + mt76_for_each_q_rx(mdev, i) { + if (mdev->q_rx[i].ndesc) { + napi_enable(&dev->mt76.napi[i]); + napi_schedule(&dev->mt76.napi[i]); + } + } + local_bh_enable(); + clear_bit(MT76_MCU_RESET, &dev->mphy.state); + clear_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); - /* re-init prefetch settings after reset */ - mt7915_dma_prefetch(dev); + mt76_wr(dev, MT_INT_MASK_CSR, dev->mt76.mmio.irqmask); + mt76_wr(dev, MT_INT_SOURCE_CSR, ~0); - mt76_set(dev, MT_WFDMA0_GLO_CFG, - MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN); - if (is_mt7915(&dev->mt76)) - mt76_set(dev, MT_WFDMA1_GLO_CFG, - MT_WFDMA1_GLO_CFG_TX_DMA_EN | - MT_WFDMA1_GLO_CFG_RX_DMA_EN | - MT_WFDMA1_GLO_CFG_OMIT_TX_INFO | - MT_WFDMA1_GLO_CFG_OMIT_RX_INFO); if (dev->hif2) { - mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, - MT_WFDMA0_GLO_CFG_TX_DMA_EN | - MT_WFDMA0_GLO_CFG_RX_DMA_EN); + mt76_wr(dev, MT_INT1_MASK_CSR, dev->mt76.mmio.irqmask); + mt76_wr(dev, MT_INT1_SOURCE_CSR, ~0); + } + if (dev_is_pci(mdev->dev)) { + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); + if (dev->hif2) + mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0xff); + } + + /* load firmware */ + ret = mt7915_mcu_init_firmware(dev); + if (ret) + goto out; + + /* set the necessary init items */ + ret = mt7915_mcu_set_eeprom(dev); + if (ret) + goto out; + + mt7915_mac_init(dev); + mt7915_init_txpower(dev, &dev->mphy.sband_2g.sband); + mt7915_init_txpower(dev, &dev->mphy.sband_5g.sband); + ret = mt7915_txbf_init(dev); - if (is_mt7915(&dev->mt76)) - mt76_set(dev, MT_WFDMA1_GLO_CFG + hif1_ofs, - MT_WFDMA1_GLO_CFG_TX_DMA_EN | - MT_WFDMA1_GLO_CFG_RX_DMA_EN | - MT_WFDMA1_GLO_CFG_OMIT_TX_INFO | - MT_WFDMA1_GLO_CFG_OMIT_RX_INFO); + if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) { + ret = mt7915_run(dev->mphy.hw); + if (ret) + goto out; + } + + if (ext_phy && test_bit(MT76_STATE_RUNNING, &ext_phy->state)) { + ret = mt7915_run(ext_phy->hw); + if (ret) + goto out; } + +out: + /* reset done */ + clear_bit(MT76_RESET, &dev->mphy.state); + if (phy2) + clear_bit(MT76_RESET, &phy2->mt76->state); + + local_bh_disable(); + napi_enable(&dev->mt76.tx_napi); + napi_schedule(&dev->mt76.tx_napi); + local_bh_enable(); + + mt76_worker_enable(&dev->mt76.tx_worker); + + return ret; } -void mt7915_tx_token_put(struct mt7915_dev *dev) +static void +mt7915_mac_full_reset(struct mt7915_dev *dev) { - struct mt76_txwi_cache *txwi; - int id; + struct mt76_phy *ext_phy; + int i; - spin_lock_bh(&dev->mt76.token_lock); - idr_for_each_entry(&dev->mt76.token, txwi, id) { - mt7915_txwi_free(dev, txwi, NULL, NULL); - dev->mt76.token_count--; + ext_phy = dev->mt76.phys[MT_BAND1]; + + dev->recovery.hw_full_reset = true; + + wake_up(&dev->mt76.mcu.wait); + ieee80211_stop_queues(mt76_hw(dev)); + if (ext_phy) + ieee80211_stop_queues(ext_phy->hw); + + cancel_delayed_work_sync(&dev->mphy.mac_work); + if (ext_phy) + cancel_delayed_work_sync(&ext_phy->mac_work); + + mutex_lock(&dev->mt76.mutex); + for (i = 0; i < 10; i++) { + if (!mt7915_mac_restart(dev)) + break; } - spin_unlock_bh(&dev->mt76.token_lock); - idr_destroy(&dev->mt76.token); + mutex_unlock(&dev->mt76.mutex); + + if (i == 10) + dev_err(dev->mt76.dev, "chip full reset failed\n"); + + ieee80211_restart_hw(mt76_hw(dev)); + if (ext_phy) + ieee80211_restart_hw(ext_phy->hw); + + ieee80211_wake_queues(mt76_hw(dev)); + if (ext_phy) + ieee80211_wake_queues(ext_phy->hw); + + dev->recovery.hw_full_reset = false; + ieee80211_queue_delayed_work(mt76_hw(dev), &dev->mphy.mac_work, + MT7915_WATCHDOG_TIME); + if (ext_phy) + ieee80211_queue_delayed_work(ext_phy->hw, + &ext_phy->mac_work, + MT7915_WATCHDOG_TIME); } /* system error recovery */ @@ -1388,7 +1553,33 @@ void mt7915_mac_reset_work(struct work_struct *work) ext_phy = dev->mt76.phys[MT_BAND1]; phy2 = ext_phy ? ext_phy->priv : NULL; - if (!(READ_ONCE(dev->reset_state) & MT_MCU_CMD_STOP_DMA)) + /* chip full reset */ + if (dev->recovery.restart) { + /* disable WA/WM WDT */ + mt76_clear(dev, MT_WFDMA0_MCU_HOST_INT_ENA, + MT_MCU_CMD_WDT_MASK); + + if (READ_ONCE(dev->recovery.state) & MT_MCU_CMD_WA_WDT) + dev->recovery.wa_reset_count++; + else + dev->recovery.wm_reset_count++; + + mt7915_mac_full_reset(dev); + + /* enable mcu irq */ + mt7915_irq_enable(dev, MT_INT_MCU_CMD); + mt7915_irq_disable(dev, 0); + + /* enable WA/WM WDT */ + mt76_set(dev, MT_WFDMA0_MCU_HOST_INT_ENA, MT_MCU_CMD_WDT_MASK); + + dev->recovery.state = MT_MCU_CMD_NORMAL_STATE; + dev->recovery.restart = false; + return; + } + + /* chip partial reset */ + if (!(READ_ONCE(dev->recovery.state) & MT_MCU_CMD_STOP_DMA)) return; ieee80211_stop_queues(mt76_hw(dev)); @@ -1413,7 +1604,7 @@ void mt7915_mac_reset_work(struct work_struct *work) mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7915_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { - mt7915_dma_reset(dev); + mt7915_dma_reset(dev, false); mt7915_tx_token_put(dev); idr_init(&dev->mt76.token); @@ -1462,132 +1653,227 @@ void mt7915_mac_reset_work(struct work_struct *work) MT7915_WATCHDOG_TIME); } +/* firmware coredump */ +void mt7915_mac_dump_work(struct work_struct *work) +{ + const struct mt7915_mem_region *mem_region; + struct mt7915_crash_data *crash_data; + struct mt7915_dev *dev; + struct mt7915_mem_hdr *hdr; + size_t buf_len; + int i; + u32 num; + u8 *buf; + + dev = container_of(work, struct mt7915_dev, dump_work); + + mutex_lock(&dev->dump_mutex); + + crash_data = mt7915_coredump_new(dev); + if (!crash_data) { + mutex_unlock(&dev->dump_mutex); + goto skip_coredump; + } + + mem_region = mt7915_coredump_get_mem_layout(dev, &num); + if (!mem_region || !crash_data->memdump_buf_len) { + mutex_unlock(&dev->dump_mutex); + goto skip_memdump; + } + + buf = crash_data->memdump_buf; + buf_len = crash_data->memdump_buf_len; + + /* dumping memory content... */ + memset(buf, 0, buf_len); + for (i = 0; i < num; i++) { + if (mem_region->len > buf_len) { + dev_warn(dev->mt76.dev, "%s len %lu is too large\n", + mem_region->name, + (unsigned long)mem_region->len); + break; + } + + /* reserve space for the header */ + hdr = (void *)buf; + buf += sizeof(*hdr); + buf_len -= sizeof(*hdr); + + mt7915_memcpy_fromio(dev, buf, mem_region->start, + mem_region->len); + + hdr->start = mem_region->start; + hdr->len = mem_region->len; + + if (!mem_region->len) + /* note: the header remains, just with zero length */ + break; + + buf += mem_region->len; + buf_len -= mem_region->len; + + mem_region++; + } + + mutex_unlock(&dev->dump_mutex); + +skip_memdump: + mt7915_coredump_submit(dev); +skip_coredump: + queue_work(dev->mt76.wq, &dev->reset_work); +} + +void mt7915_reset(struct mt7915_dev *dev) +{ + if (!dev->recovery.hw_init_done) + return; + + if (dev->recovery.hw_full_reset) + return; + + /* wm/wa exception: do full recovery */ + if (READ_ONCE(dev->recovery.state) & MT_MCU_CMD_WDT_MASK) { + dev->recovery.restart = true; + dev_info(dev->mt76.dev, + "%s indicated firmware crash, attempting recovery\n", + wiphy_name(dev->mt76.hw->wiphy)); + + mt7915_irq_disable(dev, MT_INT_MCU_CMD); + queue_work(dev->mt76.wq, &dev->dump_work); + return; + } + + queue_work(dev->mt76.wq, &dev->reset_work); + wake_up(&dev->reset_wait); +} + void mt7915_mac_update_stats(struct mt7915_phy *phy) { struct mt7915_dev *dev = phy->dev; struct mib_stats *mib = &phy->mib; - int i, aggr0, aggr1, cnt; + int i, aggr0 = 0, aggr1, cnt; + u8 band = phy->mt76->band_idx; u32 val; - cnt = mt76_rr(dev, MT_MIB_SDR3(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR3(band)); mib->fcs_err_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR3_FCS_ERR_MASK, cnt) : FIELD_GET(MT_MIB_SDR3_FCS_ERR_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR4(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR4(band)); mib->rx_fifo_full_cnt += FIELD_GET(MT_MIB_SDR4_RX_FIFO_FULL_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR5(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR5(band)); mib->rx_mpdu_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDR6(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR6(band)); mib->channel_idle_cnt += FIELD_GET(MT_MIB_SDR6_CHANNEL_IDL_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR7(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR7(band)); mib->rx_vector_mismatch_cnt += FIELD_GET(MT_MIB_SDR7_RX_VECTOR_MISMATCH_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR8(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR8(band)); mib->rx_delimiter_fail_cnt += FIELD_GET(MT_MIB_SDR8_RX_DELIMITER_FAIL_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR10(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR10(band)); mib->rx_mrdy_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR10_MRDY_COUNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR10_MRDY_COUNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR11(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR11(band)); mib->rx_len_mismatch_cnt += FIELD_GET(MT_MIB_SDR11_RX_LEN_MISMATCH_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR12(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR12(band)); mib->tx_ampdu_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDR13(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR13(band)); mib->tx_stop_q_empty_cnt += FIELD_GET(MT_MIB_SDR13_TX_STOP_Q_EMPTY_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR14(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR14(band)); mib->tx_mpdu_attempts_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR14_TX_MPDU_ATTEMPTS_CNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR14_TX_MPDU_ATTEMPTS_CNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR15(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR15(band)); mib->tx_mpdu_success_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR15_TX_MPDU_SUCCESS_CNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR15_TX_MPDU_SUCCESS_CNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR16(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR16(band)); mib->primary_cca_busy_time += FIELD_GET(MT_MIB_SDR16_PRIMARY_CCA_BUSY_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR17(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR17(band)); mib->secondary_cca_busy_time += FIELD_GET(MT_MIB_SDR17_SECONDARY_CCA_BUSY_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR18(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR18(band)); mib->primary_energy_detect_time += FIELD_GET(MT_MIB_SDR18_PRIMARY_ENERGY_DETECT_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR19(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR19(band)); mib->cck_mdrdy_time += FIELD_GET(MT_MIB_SDR19_CCK_MDRDY_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR20(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR20(band)); mib->ofdm_mdrdy_time += FIELD_GET(MT_MIB_SDR20_OFDM_VHT_MDRDY_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR21(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR21(band)); mib->green_mdrdy_time += FIELD_GET(MT_MIB_SDR21_GREEN_MDRDY_TIME_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR22(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR22(band)); mib->rx_ampdu_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDR23(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR23(band)); mib->rx_ampdu_bytes_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDR24(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR24(band)); mib->rx_ampdu_valid_subframe_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR24_RX_AMPDU_SF_CNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR24_RX_AMPDU_SF_CNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR25(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR25(band)); mib->rx_ampdu_valid_subframe_bytes_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDR27(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR27(band)); mib->tx_rwp_fail_cnt += FIELD_GET(MT_MIB_SDR27_TX_RWP_FAIL_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR28(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR28(band)); mib->tx_rwp_need_cnt += FIELD_GET(MT_MIB_SDR28_TX_RWP_NEED_CNT_MASK, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR29(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR29(band)); mib->rx_pfdrop_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR29_RX_PFDROP_CNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR29_RX_PFDROP_CNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDRVEC(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDRVEC(band)); mib->rx_vec_queue_overflow_drop_cnt += is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR30_RX_VEC_QUEUE_OVERFLOW_DROP_CNT_MASK, cnt) : FIELD_GET(MT_MIB_SDR30_RX_VEC_QUEUE_OVERFLOW_DROP_CNT_MASK_MT7916, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR31(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR31(band)); mib->rx_ba_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_SDRMUBF(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDRMUBF(band)); mib->tx_bf_cnt += FIELD_GET(MT_MIB_MU_BF_TX_CNT, cnt); - cnt = mt76_rr(dev, MT_MIB_DR8(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_DR8(band)); mib->tx_mu_mpdu_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_DR9(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_DR9(band)); mib->tx_mu_acked_mpdu_cnt += cnt; - cnt = mt76_rr(dev, MT_MIB_DR11(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_DR11(band)); mib->tx_su_acked_mpdu_cnt += cnt; - cnt = mt76_rr(dev, MT_ETBF_PAR_RPT0(phy->band_idx)); + cnt = mt76_rr(dev, MT_ETBF_PAR_RPT0(band)); mib->tx_bf_rx_fb_bw = FIELD_GET(MT_ETBF_PAR_RPT0_FB_BW, cnt); mib->tx_bf_rx_fb_nc_cnt += FIELD_GET(MT_ETBF_PAR_RPT0_FB_NC, cnt); mib->tx_bf_rx_fb_nr_cnt += FIELD_GET(MT_ETBF_PAR_RPT0_FB_NR, cnt); @@ -1598,44 +1884,43 @@ void mt7915_mac_update_stats(struct mt7915_phy *phy) mib->tx_amsdu_cnt += cnt; } - aggr0 = phy->band_idx ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; if (is_mt7915(&dev->mt76)) { - for (i = 0, aggr1 = aggr0 + 4; i < 4; i++) { - val = mt76_rr(dev, MT_MIB_MB_SDR1(phy->band_idx, (i << 4))); + for (i = 0, aggr1 = aggr0 + 8; i < 4; i++) { + val = mt76_rr(dev, MT_MIB_MB_SDR1(band, (i << 4))); mib->ba_miss_cnt += FIELD_GET(MT_MIB_BA_MISS_COUNT_MASK, val); mib->ack_fail_cnt += FIELD_GET(MT_MIB_ACK_FAIL_COUNT_MASK, val); - val = mt76_rr(dev, MT_MIB_MB_SDR0(phy->band_idx, (i << 4))); + val = mt76_rr(dev, MT_MIB_MB_SDR0(band, (i << 4))); mib->rts_cnt += FIELD_GET(MT_MIB_RTS_COUNT_MASK, val); mib->rts_retries_cnt += FIELD_GET(MT_MIB_RTS_RETRIES_COUNT_MASK, val); - val = mt76_rr(dev, MT_TX_AGG_CNT(phy->band_idx, i)); - dev->mt76.aggr_stats[aggr0++] += val & 0xffff; - dev->mt76.aggr_stats[aggr0++] += val >> 16; + val = mt76_rr(dev, MT_TX_AGG_CNT(band, i)); + phy->mt76->aggr_stats[aggr0++] += val & 0xffff; + phy->mt76->aggr_stats[aggr0++] += val >> 16; - val = mt76_rr(dev, MT_TX_AGG_CNT2(phy->band_idx, i)); - dev->mt76.aggr_stats[aggr1++] += val & 0xffff; - dev->mt76.aggr_stats[aggr1++] += val >> 16; + val = mt76_rr(dev, MT_TX_AGG_CNT2(band, i)); + phy->mt76->aggr_stats[aggr1++] += val & 0xffff; + phy->mt76->aggr_stats[aggr1++] += val >> 16; } - cnt = mt76_rr(dev, MT_MIB_SDR32(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR32(band)); mib->tx_pkt_ebf_cnt += FIELD_GET(MT_MIB_SDR32_TX_PKT_EBF_CNT, cnt); - cnt = mt76_rr(dev, MT_MIB_SDR33(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR33(band)); mib->tx_pkt_ibf_cnt += FIELD_GET(MT_MIB_SDR33_TX_PKT_IBF_CNT, cnt); - cnt = mt76_rr(dev, MT_ETBF_TX_APP_CNT(phy->band_idx)); + cnt = mt76_rr(dev, MT_ETBF_TX_APP_CNT(band)); mib->tx_bf_ibf_ppdu_cnt += FIELD_GET(MT_ETBF_TX_IBF_CNT, cnt); mib->tx_bf_ebf_ppdu_cnt += FIELD_GET(MT_ETBF_TX_EBF_CNT, cnt); - cnt = mt76_rr(dev, MT_ETBF_TX_NDP_BFRP(phy->band_idx)); + cnt = mt76_rr(dev, MT_ETBF_TX_NDP_BFRP(band)); mib->tx_bf_fb_cpl_cnt += FIELD_GET(MT_ETBF_TX_FB_CPL, cnt); mib->tx_bf_fb_trig_cnt += FIELD_GET(MT_ETBF_TX_FB_TRI, cnt); - cnt = mt76_rr(dev, MT_ETBF_RX_FB_CNT(phy->band_idx)); + cnt = mt76_rr(dev, MT_ETBF_RX_FB_CNT(band)); mib->tx_bf_rx_fb_all_cnt += FIELD_GET(MT_ETBF_RX_FB_ALL, cnt); mib->tx_bf_rx_fb_he_cnt += FIELD_GET(MT_ETBF_RX_FB_HE, cnt); mib->tx_bf_rx_fb_vht_cnt += FIELD_GET(MT_ETBF_RX_FB_VHT, cnt); @@ -1643,51 +1928,51 @@ void mt7915_mac_update_stats(struct mt7915_phy *phy) } else { for (i = 0; i < 2; i++) { /* rts count */ - val = mt76_rr(dev, MT_MIB_MB_SDR0(phy->band_idx, (i << 2))); + val = mt76_rr(dev, MT_MIB_MB_SDR0(band, (i << 2))); mib->rts_cnt += FIELD_GET(GENMASK(15, 0), val); mib->rts_cnt += FIELD_GET(GENMASK(31, 16), val); /* rts retry count */ - val = mt76_rr(dev, MT_MIB_MB_SDR1(phy->band_idx, (i << 2))); + val = mt76_rr(dev, MT_MIB_MB_SDR1(band, (i << 2))); mib->rts_retries_cnt += FIELD_GET(GENMASK(15, 0), val); mib->rts_retries_cnt += FIELD_GET(GENMASK(31, 16), val); /* ba miss count */ - val = mt76_rr(dev, MT_MIB_MB_SDR2(phy->band_idx, (i << 2))); + val = mt76_rr(dev, MT_MIB_MB_SDR2(band, (i << 2))); mib->ba_miss_cnt += FIELD_GET(GENMASK(15, 0), val); mib->ba_miss_cnt += FIELD_GET(GENMASK(31, 16), val); /* ack fail count */ - val = mt76_rr(dev, MT_MIB_MB_BFTF(phy->band_idx, (i << 2))); + val = mt76_rr(dev, MT_MIB_MB_BFTF(band, (i << 2))); mib->ack_fail_cnt += FIELD_GET(GENMASK(15, 0), val); mib->ack_fail_cnt += FIELD_GET(GENMASK(31, 16), val); } for (i = 0; i < 8; i++) { - val = mt76_rr(dev, MT_TX_AGG_CNT(phy->band_idx, i)); - dev->mt76.aggr_stats[aggr0++] += FIELD_GET(GENMASK(15, 0), val); - dev->mt76.aggr_stats[aggr0++] += FIELD_GET(GENMASK(31, 16), val); + val = mt76_rr(dev, MT_TX_AGG_CNT(band, i)); + phy->mt76->aggr_stats[aggr0++] += FIELD_GET(GENMASK(15, 0), val); + phy->mt76->aggr_stats[aggr0++] += FIELD_GET(GENMASK(31, 16), val); } - cnt = mt76_rr(dev, MT_MIB_SDR32(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR32(band)); mib->tx_pkt_ibf_cnt += FIELD_GET(MT_MIB_SDR32_TX_PKT_IBF_CNT, cnt); mib->tx_bf_ibf_ppdu_cnt += FIELD_GET(MT_MIB_SDR32_TX_PKT_IBF_CNT, cnt); mib->tx_pkt_ebf_cnt += FIELD_GET(MT_MIB_SDR32_TX_PKT_EBF_CNT, cnt); mib->tx_bf_ebf_ppdu_cnt += FIELD_GET(MT_MIB_SDR32_TX_PKT_EBF_CNT, cnt); - cnt = mt76_rr(dev, MT_MIB_BFCR7(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_BFCR7(band)); mib->tx_bf_fb_cpl_cnt += FIELD_GET(MT_MIB_BFCR7_BFEE_TX_FB_CPL, cnt); - cnt = mt76_rr(dev, MT_MIB_BFCR2(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_BFCR2(band)); mib->tx_bf_fb_trig_cnt += FIELD_GET(MT_MIB_BFCR2_BFEE_TX_FB_TRIG, cnt); - cnt = mt76_rr(dev, MT_MIB_BFCR0(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_BFCR0(band)); mib->tx_bf_rx_fb_vht_cnt += FIELD_GET(MT_MIB_BFCR0_RX_FB_VHT, cnt); mib->tx_bf_rx_fb_all_cnt += FIELD_GET(MT_MIB_BFCR0_RX_FB_VHT, cnt); mib->tx_bf_rx_fb_ht_cnt += FIELD_GET(MT_MIB_BFCR0_RX_FB_HT, cnt); mib->tx_bf_rx_fb_all_cnt += FIELD_GET(MT_MIB_BFCR0_RX_FB_HT, cnt); - cnt = mt76_rr(dev, MT_MIB_BFCR1(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_BFCR1(band)); mib->tx_bf_rx_fb_he_cnt += FIELD_GET(MT_MIB_BFCR1_RX_FB_HE, cnt); mib->tx_bf_rx_fb_all_cnt += FIELD_GET(MT_MIB_BFCR1_RX_FB_HE, cnt); } @@ -1696,7 +1981,6 @@ void mt7915_mac_update_stats(struct mt7915_phy *phy) static void mt7915_mac_severe_check(struct mt7915_phy *phy) { struct mt7915_dev *dev = phy->dev; - bool ext_phy = phy != &dev->phy; u32 trb; if (!phy->omac_mask) @@ -1706,7 +1990,7 @@ static void mt7915_mac_severe_check(struct mt7915_phy *phy) * stopping Rx, so check status periodically to see if TRB hardware * requires minimal recovery. */ - trb = mt76_rr(dev, MT_TRB_RXPSR0(phy->band_idx)); + trb = mt76_rr(dev, MT_TRB_RXPSR0(phy->mt76->band_idx)); if ((FIELD_GET(MT_TRB_RXPSR0_RX_RMAC_PTR, trb) != FIELD_GET(MT_TRB_RXPSR0_RX_WTBL_PTR, trb)) && @@ -1714,7 +1998,7 @@ static void mt7915_mac_severe_check(struct mt7915_phy *phy) FIELD_GET(MT_TRB_RXPSR0_RX_WTBL_PTR, phy->trb_ts)) && trb == phy->trb_ts) mt7915_mcu_set_ser(dev, SER_RECOVER, SER_SET_RECOVER_L3_RX_ABORT, - ext_phy); + phy->mt76->band_idx); phy->trb_ts = trb; } @@ -1816,6 +2100,13 @@ static int mt7915_dfs_start_rdd(struct mt7915_dev *dev, int chain) if (err < 0) return err; + if (is_mt7915(&dev->mt76)) { + err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_SET_WF_ANT, chain, + 0, dev->dbdc_support ? 2 : 0); + if (err < 0) + return err; + } + return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_DET_MODE, chain, MT_RX_SEL0, 1); } @@ -1827,16 +2118,16 @@ static int mt7915_dfs_start_radar_detector(struct mt7915_phy *phy) int err; /* start CAC */ - err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_START, phy->band_idx, - MT_RX_SEL0, 0); + err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_START, + phy->mt76->band_idx, MT_RX_SEL0, 0); if (err < 0) return err; - err = mt7915_dfs_start_rdd(dev, phy->band_idx); + err = mt7915_dfs_start_rdd(dev, phy->mt76->band_idx); if (err < 0) return err; - phy->rdd_state |= BIT(phy->band_idx); + phy->rdd_state |= BIT(phy->mt76->band_idx); if (!is_mt7915(&dev->mt76)) return 0; @@ -1921,7 +2212,7 @@ int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy) return 0; err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_END, - phy->band_idx, MT_RX_SEL0, 0); + phy->mt76->band_idx, MT_RX_SEL0, 0); if (err < 0) { phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN; return err; @@ -1932,10 +2223,18 @@ int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy) stop: err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_NORMAL_START, - phy->band_idx, MT_RX_SEL0, 0); + phy->mt76->band_idx, MT_RX_SEL0, 0); if (err < 0) return err; + if (is_mt7915(&dev->mt76)) { + err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_SET_WF_ANT, + phy->mt76->band_idx, 0, + dev->dbdc_support ? 2 : 0); + if (err < 0) + return err; + } + mt7915_dfs_stop_radar_detector(phy); phy->mt76->dfs_state = MT_DFS_STATE_DISABLED; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 89b519cfd14c..0511d6a505b0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -20,45 +20,45 @@ static bool mt7915_dev_running(struct mt7915_dev *dev) return phy && test_bit(MT76_STATE_RUNNING, &phy->mt76->state); } -static int mt7915_start(struct ieee80211_hw *hw) +int mt7915_run(struct ieee80211_hw *hw) { struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); bool running; int ret; - flush_work(&dev->init_work); - - mutex_lock(&dev->mt76.mutex); - running = mt7915_dev_running(dev); if (!running) { - ret = mt76_connac_mcu_set_pm(&dev->mt76, 0, 0); + ret = mt76_connac_mcu_set_pm(&dev->mt76, + dev->phy.mt76->band_idx, 0); if (ret) goto out; - ret = mt7915_mcu_set_mac(dev, 0, true, true); + ret = mt7915_mcu_set_mac(dev, dev->phy.mt76->band_idx, + true, true); if (ret) goto out; - mt7915_mac_enable_nf(dev, 0); + mt7915_mac_enable_nf(dev, dev->phy.mt76->band_idx); } - if (phy != &dev->phy || phy->band_idx) { - ret = mt76_connac_mcu_set_pm(&dev->mt76, 1, 0); + if (phy != &dev->phy) { + ret = mt76_connac_mcu_set_pm(&dev->mt76, + phy->mt76->band_idx, 0); if (ret) goto out; - ret = mt7915_mcu_set_mac(dev, 1, true, true); + ret = mt7915_mcu_set_mac(dev, phy->mt76->band_idx, + true, true); if (ret) goto out; - mt7915_mac_enable_nf(dev, 1); + mt7915_mac_enable_nf(dev, phy->mt76->band_idx); } ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, 0x92b, - phy != &dev->phy); + phy->mt76->band_idx); if (ret) goto out; @@ -80,6 +80,18 @@ static int mt7915_start(struct ieee80211_hw *hw) mt7915_mac_reset_counters(phy); out: + return ret; +} + +static int mt7915_start(struct ieee80211_hw *hw) +{ + struct mt7915_dev *dev = mt7915_hw_dev(hw); + int ret; + + flush_work(&dev->init_work); + + mutex_lock(&dev->mt76.mutex); + ret = mt7915_run(hw); mutex_unlock(&dev->mt76.mutex); return ret; @@ -99,13 +111,13 @@ static void mt7915_stop(struct ieee80211_hw *hw) clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); if (phy != &dev->phy) { - mt76_connac_mcu_set_pm(&dev->mt76, 1, 1); - mt7915_mcu_set_mac(dev, 1, false, false); + mt76_connac_mcu_set_pm(&dev->mt76, phy->mt76->band_idx, 1); + mt7915_mcu_set_mac(dev, phy->mt76->band_idx, false, false); } if (!mt7915_dev_running(dev)) { - mt76_connac_mcu_set_pm(&dev->mt76, 0, 1); - mt7915_mcu_set_mac(dev, 0, false, false); + mt76_connac_mcu_set_pm(&dev->mt76, dev->phy.mt76->band_idx, 1); + mt7915_mcu_set_mac(dev, dev->phy.mt76->band_idx, false, false); } mutex_unlock(&dev->mt76.mutex); @@ -209,7 +221,7 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, } mvif->mt76.omac_idx = idx; mvif->phy = phy; - mvif->mt76.band_idx = phy->band_idx; + mvif->mt76.band_idx = phy->mt76->band_idx; mvif->mt76.wmm_idx = vif->type != NL80211_IFTYPE_AP; if (ext_phy) @@ -432,7 +444,6 @@ static int mt7915_config(struct ieee80211_hw *hw, u32 changed) { struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); - bool band = phy != &dev->phy; int ret; if (changed & IEEE80211_CONF_CHANGE_CHANNEL) { @@ -460,6 +471,7 @@ static int mt7915_config(struct ieee80211_hw *hw, u32 changed) if (changed & IEEE80211_CONF_CHANGE_MONITOR) { bool enabled = !!(hw->conf.flags & IEEE80211_CONF_MONITOR); + bool band = phy->mt76->band_idx; if (!enabled) phy->rxfilter |= MT_WF_RFCR_DROP_OTHER_UC; @@ -498,7 +510,7 @@ static void mt7915_configure_filter(struct ieee80211_hw *hw, { struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); - bool band = phy != &dev->phy; + bool band = phy->mt76->band_idx; u32 ctl_flags = MT_WF_RFCR1_DROP_ACK | MT_WF_RFCR1_DROP_BF_POLL | MT_WF_RFCR1_DROP_BA | @@ -593,10 +605,11 @@ static void mt7915_bss_info_changed(struct ieee80211_hw *hw, mt7915_mcu_add_sta(dev, vif, NULL, join); } - if (changed & BSS_CHANGED_ASSOC) { + if (changed & BSS_CHANGED_ASSOC) mt7915_mcu_add_bss_info(phy, vif, vif->cfg.assoc); - mt7915_mcu_add_obss_spr(dev, vif, info->he_obss_pd.enable); - } + + if (changed & BSS_CHANGED_ERP_CTS_PROT) + mt7915_mac_enable_rtscts(dev, vif, info->use_cts_prot); if (changed & BSS_CHANGED_ERP_SLOT) { int slottime = info->use_short_slot ? 9 : 20; @@ -617,7 +630,7 @@ static void mt7915_bss_info_changed(struct ieee80211_hw *hw, mt7915_mcu_set_tx(dev, vif); if (changed & BSS_CHANGED_HE_OBSS_PD) - mt7915_mcu_add_obss_spr(dev, vif, info->he_obss_pd.enable); + mt7915_mcu_add_obss_spr(phy, vif, &info->he_obss_pd); if (changed & BSS_CHANGED_HE_BSS_COLOR) mt7915_update_bss_color(hw, vif, &info->he_bss_color); @@ -665,6 +678,8 @@ int mt7915_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, msta->wcid.tx_info |= MT_WCID_TX_INFO_SET; msta->jiffies = jiffies; + ewma_avg_signal_init(&msta->avg_ack_signal); + mt7915_mac_wtbl_update(dev, idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); @@ -732,7 +747,8 @@ static int mt7915_set_rts_threshold(struct ieee80211_hw *hw, u32 val) int ret; mutex_lock(&dev->mt76.mutex); - ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, val, phy != &dev->phy); + ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, val, + phy->mt76->band_idx); mutex_unlock(&dev->mt76.mutex); return ret; @@ -835,7 +851,7 @@ u64 __mt7915_get_tsf(struct ieee80211_hw *hw, struct mt7915_vif *mvif) { struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); - bool band = phy != &dev->phy; + bool band = phy->mt76->band_idx; union { u64 t64; u32 t32[2]; @@ -880,7 +896,7 @@ mt7915_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); - bool band = phy != &dev->phy; + bool band = phy->mt76->band_idx; union { u64 t64; u32 t32[2]; @@ -911,7 +927,7 @@ mt7915_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); - bool band = phy != &dev->phy; + bool band = phy->mt76->band_idx; union { u64 t64; u32 t32[2]; @@ -953,22 +969,21 @@ mt7915_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); int max_nss = hweight8(hw->wiphy->available_antennas_tx); - bool ext_phy = phy != &dev->phy; + u8 chainshift = dev->chainshift; + u8 band = phy->mt76->band_idx; if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) return -EINVAL; - if ((BIT(hweight8(tx_ant)) - 1) != tx_ant) - tx_ant = BIT(ffs(tx_ant) - 1) - 1; - mutex_lock(&dev->mt76.mutex); phy->mt76->antenna_mask = tx_ant; - if (ext_phy) - tx_ant <<= dev->chainshift; - - phy->mt76->chainmask = tx_ant; + /* handle a variant of mt7916 which has 3T3R but nss2 on 5 GHz band */ + if (is_mt7916(&dev->mt76) && band && hweight8(tx_ant) == max_nss) + phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; + else + phy->mt76->chainmask = tx_ant << (chainshift * band); mt76_set_stream_caps(phy->mt76, true); mt7915_set_stream_vht_txbf_caps(phy); @@ -1026,7 +1041,21 @@ static void mt7915_sta_statistics(struct ieee80211_hw *hw, sinfo->tx_retries = msta->wcid.stats.tx_retries; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); + + if (mtk_wed_get_rx_capa(&phy->dev->mt76.mmio.wed)) { + sinfo->rx_bytes = msta->wcid.stats.rx_bytes; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64); + + sinfo->rx_packets = msta->wcid.stats.rx_packets; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS); + } } + + sinfo->ack_signal = (s8)msta->ack_signal; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL); + + sinfo->avg_ack_signal = -(s8)ewma_avg_signal_read(&msta->avg_ack_signal); + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG); } static void mt7915_sta_rc_work(void *data, struct ieee80211_sta *sta) @@ -1111,6 +1140,39 @@ static void mt7915_sta_set_decap_offload(struct ieee80211_hw *hw, mt76_connac_mcu_wtbl_update_hdr_trans(&dev->mt76, vif, sta); } +static int mt7915_sta_set_txpwr(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7915_phy *phy = mt7915_hw_phy(hw); + struct mt7915_dev *dev = mt7915_hw_dev(hw); + s16 txpower = sta->deflink.txpwr.power; + int ret; + + if (sta->deflink.txpwr.type == NL80211_TX_POWER_AUTOMATIC) + txpower = 0; + + mutex_lock(&dev->mt76.mutex); + + /* NOTE: temporarily use 0 as minimum limit, which is a + * global setting and will be applied to all stations. + */ + ret = mt7915_mcu_set_txpower_frame_min(phy, 0); + if (ret) + goto out; + + /* This only applies to data frames while pushing traffic, + * whereas the management frames or other packets that are + * using fixed rate can be configured via TxD. + */ + ret = mt7915_mcu_set_txpower_frame(phy, vif, sta, txpower); + +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + static const char mt7915_gstrings_stats[][ETH_GSTRING_LEN] = { "tx_ampdu_cnt", "tx_stop_q_empty_cnt", @@ -1258,7 +1320,7 @@ void mt7915_get_et_stats(struct ieee80211_hw *hw, }; struct mib_stats *mib = &phy->mib; /* See mt7915_ampdu_stat_read_phy, etc */ - int i, n, ei = 0; + int i, ei = 0; mutex_lock(&dev->mt76.mutex); @@ -1274,9 +1336,8 @@ void mt7915_get_et_stats(struct ieee80211_hw *hw, data[ei++] = mib->tx_pkt_ibf_cnt; /* Tx ampdu stat */ - n = phy->band_idx ? ARRAY_SIZE(dev->mt76.aggr_stats) / 2 : 0; for (i = 0; i < 15 /*ARRAY_SIZE(bound)*/; i++) - data[ei++] = dev->mt76.aggr_stats[i + n]; + data[ei++] = phy->mt76->aggr_stats[i]; data[ei++] = phy->mib.ba_miss_cnt; @@ -1431,7 +1492,7 @@ mt7915_net_fill_forward_path(struct ieee80211_hw *hw, path->dev = ctx->dev; path->mtk_wdma.wdma_idx = wed->wdma_idx; path->mtk_wdma.bss = mvif->mt76.idx; - path->mtk_wdma.wcid = msta->wcid.idx; + path->mtk_wdma.wcid = is_mt7915(&dev->mt76) ? msta->wcid.idx : 0x3ff; path->mtk_wdma.queue = phy != &dev->phy; ctx->dev = NULL; @@ -1477,6 +1538,7 @@ const struct ieee80211_ops mt7915_ops = { .set_bitrate_mask = mt7915_set_bitrate_mask, .set_coverage_class = mt7915_set_coverage_class, .sta_statistics = mt7915_sta_statistics, + .sta_set_txpwr = mt7915_sta_set_txpwr, .sta_set_4addr = mt7915_sta_set_4addr, .sta_set_decap_offload = mt7915_sta_set_decap_offload, .add_twt_setup = mt7915_mac_add_twt_setup, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index 8d297e4aa7d4..b2652de082ba 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -32,6 +32,10 @@ #define HE_PHY(p, c) u8_get_bits(c, IEEE80211_HE_PHY_##p) #define HE_MAC(m, c) u8_get_bits(c, IEEE80211_HE_MAC_##m) +static bool sr_scene_detect = true; +module_param(sr_scene_detect, bool, 0644); +MODULE_PARM_DESC(sr_scene_detect, "Enable firmware scene detection algorithm"); + static u8 mt7915_mcu_get_sta_nss(u16 mcs_map) { @@ -228,7 +232,8 @@ mt7915_mcu_rx_csa_notify(struct mt7915_dev *dev, struct sk_buff *skb) c = (struct mt7915_mcu_csa_notify *)skb->data; - if ((c->band_idx && !dev->phy.band_idx) && dev->mt76.phys[MT_BAND1]) + if ((c->band_idx && !dev->phy.mt76->band_idx) && + dev->mt76.phys[MT_BAND1]) mphy = dev->mt76.phys[MT_BAND1]; ieee80211_iterate_active_interfaces_atomic(mphy->hw, @@ -247,7 +252,8 @@ mt7915_mcu_rx_thermal_notify(struct mt7915_dev *dev, struct sk_buff *skb) if (t->ctrl.ctrl_id != THERMAL_PROTECT_ENABLE) return; - if ((t->ctrl.band_idx && !dev->phy.band_idx) && dev->mt76.phys[MT_BAND1]) + if ((t->ctrl.band_idx && !dev->phy.mt76->band_idx) && + dev->mt76.phys[MT_BAND1]) mphy = dev->mt76.phys[MT_BAND1]; phy = (struct mt7915_phy *)mphy->priv; @@ -262,7 +268,8 @@ mt7915_mcu_rx_radar_detected(struct mt7915_dev *dev, struct sk_buff *skb) r = (struct mt7915_mcu_rdd_report *)skb->data; - if ((r->band_idx && !dev->phy.band_idx) && dev->mt76.phys[MT_BAND1]) + if ((r->band_idx && !dev->phy.mt76->band_idx) && + dev->mt76.phys[MT_BAND1]) mphy = dev->mt76.phys[MT_BAND1]; if (r->band_idx == MT_RX_SEL2) @@ -319,7 +326,7 @@ mt7915_mcu_rx_bcc_notify(struct mt7915_dev *dev, struct sk_buff *skb) b = (struct mt7915_mcu_bcc_notify *)skb->data; - if ((b->band_idx && !dev->phy.band_idx) && dev->mt76.phys[MT_BAND1]) + if ((b->band_idx && !dev->phy.mt76->band_idx) && dev->mt76.phys[MT_BAND1]) mphy = dev->mt76.phys[MT_BAND1]; ieee80211_iterate_active_interfaces_atomic(mphy->hw, @@ -485,7 +492,7 @@ static void mt7915_mcu_bss_ra_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, struct mt7915_phy *phy) { - int max_nss = hweight8(phy->mt76->chainmask); + int max_nss = hweight8(phy->mt76->antenna_mask); struct bss_info_ra *ra; struct tlv *tlv; @@ -595,7 +602,7 @@ mt7915_mcu_muar_config(struct mt7915_phy *phy, struct ieee80211_vif *vif, .mode = !!mask || enable, .entry_count = 1, .write = 1, - .band = phy != &dev->phy, + .band = phy->mt76->band_idx, .index = idx * 2 + bssid, }; @@ -1131,7 +1138,7 @@ mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif, mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_160); nss_mcs = mt7915_mcu_get_sta_nss(mcs_map); - bf->ncol_bw160 = nss_mcs; + bf->ncol_gt_bw80 = nss_mcs; } if (pe->phy_cap_info[0] & @@ -1139,10 +1146,10 @@ mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif, mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80p80); nss_mcs = mt7915_mcu_get_sta_nss(mcs_map); - if (bf->ncol_bw160) - bf->ncol_bw160 = min_t(u8, bf->ncol_bw160, nss_mcs); + if (bf->ncol_gt_bw80) + bf->ncol_gt_bw80 = min_t(u8, bf->ncol_gt_bw80, nss_mcs); else - bf->ncol_bw160 = nss_mcs; + bf->ncol_gt_bw80 = nss_mcs; } snd_dim = HE_PHY(CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK, @@ -1150,7 +1157,7 @@ mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif, sts = HE_PHY(CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_MASK, pe->phy_cap_info[4]); - bf->nrow_bw160 = min_t(int, snd_dim, sts); + bf->nrow_gt_bw80 = min_t(int, snd_dim, sts); } static void @@ -1306,6 +1313,9 @@ int mt7915_mcu_set_fixed_rate_ctrl(struct mt7915_dev *dev, case RATE_PARAM_MMPS_UPDATE: ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->deflink.smps_mode); break; + case RATE_PARAM_SPE_UPDATE: + ra->spe_idx = *(u8 *)data; + break; default: break; } @@ -1349,6 +1359,18 @@ int mt7915_mcu_add_smps(struct mt7915_dev *dev, struct ieee80211_vif *vif, } static int +mt7915_mcu_set_spe_idx(struct mt7915_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; + struct mt76_phy *mphy = mvif->phy->mt76; + u8 spe_idx = mt76_connac_spe_idx(mphy->antenna_mask); + + return mt7915_mcu_set_fixed_rate_ctrl(dev, vif, sta, &spe_idx, + RATE_PARAM_SPE_UPDATE); +} + +static int mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) @@ -1435,7 +1457,7 @@ mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev, return ret; } - return 0; + return mt7915_mcu_set_spe_idx(dev, vif, sta); } static void @@ -1662,10 +1684,32 @@ int mt7915_mcu_add_sta(struct mt7915_dev *dev, struct ieee80211_vif *vif, return ret; } out: + ret = mt76_connac_mcu_sta_wed_update(&dev->mt76, skb); + if (ret) + return ret; + return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_EXT_CMD(STA_REC_UPDATE), true); } +int mt7915_mcu_wed_enable_rx_stats(struct mt7915_dev *dev) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + struct { + __le32 args[2]; + } req = { + .args[0] = cpu_to_le32(1), + .args[1] = cpu_to_le32(6), + }; + + return mtk_wed_device_update_msg(wed, MTK_WED_WO_CMD_RXCNT_CTRL, + &req, sizeof(req)); +#else + return 0; +#endif +} + int mt7915_mcu_add_dev_info(struct mt7915_phy *phy, struct ieee80211_vif *vif, bool enable) { @@ -1674,7 +1718,7 @@ int mt7915_mcu_add_dev_info(struct mt7915_phy *phy, struct { struct req_hdr { u8 omac_idx; - u8 dbdc_idx; + u8 band_idx; __le16 tlv_num; u8 is_tlv_append; u8 rsv[3]; @@ -1683,13 +1727,13 @@ int mt7915_mcu_add_dev_info(struct mt7915_phy *phy, __le16 tag; __le16 len; u8 active; - u8 dbdc_idx; + u8 band_idx; u8 omac_addr[ETH_ALEN]; } __packed tlv; } data = { .hdr = { .omac_idx = mvif->mt76.omac_idx, - .dbdc_idx = mvif->mt76.band_idx, + .band_idx = mvif->mt76.band_idx, .tlv_num = cpu_to_le16(1), .is_tlv_append = 1, }, @@ -1697,7 +1741,7 @@ int mt7915_mcu_add_dev_info(struct mt7915_phy *phy, .tag = cpu_to_le16(DEV_INFO_ACTIVE), .len = cpu_to_le16(sizeof(struct req_tlv)), .active = enable, - .dbdc_idx = mvif->mt76.band_idx, + .band_idx = mvif->mt76.band_idx, }, }; @@ -2151,7 +2195,7 @@ int mt7915_mcu_muru_debug_get(struct mt7915_phy *phy, void *ms) u8 band_idx; } req = { .cmd = cpu_to_le32(MURU_GET_TXC_TX_STATS), - .band_idx = phy->band_idx, + .band_idx = phy->mt76->band_idx, }; ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_EXT_CMD(MURU_CTRL), @@ -2234,18 +2278,10 @@ mt7915_mcu_init_rx_airtime(struct mt7915_dev *dev) sizeof(req), true); } -int mt7915_mcu_init(struct mt7915_dev *dev) +int mt7915_mcu_init_firmware(struct mt7915_dev *dev) { - static const struct mt76_mcu_ops mt7915_mcu_ops = { - .headroom = sizeof(struct mt76_connac2_mcu_txd), - .mcu_skb_send_msg = mt7915_mcu_send_message, - .mcu_parse_response = mt7915_mcu_parse_response, - .mcu_restart = mt76_connac_mcu_restart, - }; int ret; - dev->mt76.mcu_ops = &mt7915_mcu_ops; - /* force firmware operation mode into normal state, * which should be set before firmware download stage. */ @@ -2274,7 +2310,7 @@ int mt7915_mcu_init(struct mt7915_dev *dev) if (ret) return ret; - if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && is_mt7915(&dev->mt76)) mt7915_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(CAPABILITY), 0, 0, 0); ret = mt7915_mcu_set_mwds(dev, 1); @@ -2294,6 +2330,20 @@ int mt7915_mcu_init(struct mt7915_dev *dev) MCU_WA_PARAM_RED, 0, 0); } +int mt7915_mcu_init(struct mt7915_dev *dev) +{ + static const struct mt76_mcu_ops mt7915_mcu_ops = { + .headroom = sizeof(struct mt76_connac2_mcu_txd), + .mcu_skb_send_msg = mt7915_mcu_send_message, + .mcu_parse_response = mt7915_mcu_parse_response, + .mcu_restart = mt76_connac_mcu_restart, + }; + + dev->mt76.mcu_ops = &mt7915_mcu_ops; + + return mt7915_mcu_init_firmware(dev); +} + void mt7915_mcu_exit(struct mt7915_dev *dev) { __mt76_mcu_restart(&dev->mt76); @@ -2538,7 +2588,7 @@ mt7915_mcu_background_chain_ctrl(struct mt7915_phy *phy, req.monitor_central_chan = ieee80211_frequency_to_channel(chandef->center_freq1); req.monitor_bw = mt76_connac_chan_bw(chandef); - req.band_idx = phy != &dev->phy; + req.band_idx = phy->mt76->band_idx; req.scan_mode = 1; break; } @@ -2546,7 +2596,7 @@ mt7915_mcu_background_chain_ctrl(struct mt7915_phy *phy, req.monitor_chan = chandef->chan->hw_value; req.monitor_central_chan = ieee80211_frequency_to_channel(chandef->center_freq1); - req.band_idx = phy != &dev->phy; + req.band_idx = phy->mt76->band_idx; req.scan_mode = 2; break; case CH_SWITCH_BACKGROUND_SCAN_STOP: @@ -2613,12 +2663,13 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) struct mt7915_dev *dev = phy->dev; struct cfg80211_chan_def *chandef = &phy->mt76->chandef; int freq1 = chandef->center_freq1; + u8 band = phy->mt76->band_idx; struct { u8 control_ch; u8 center_ch; u8 bw; - u8 tx_streams_num; - u8 rx_streams; /* mask or num */ + u8 tx_path_num; + u8 rx_path; /* mask or num */ u8 switch_reason; u8 band_idx; u8 center_ch2; /* for 80+80 only */ @@ -2634,25 +2685,23 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) .control_ch = chandef->chan->hw_value, .center_ch = ieee80211_frequency_to_channel(freq1), .bw = mt76_connac_chan_bw(chandef), - .tx_streams_num = hweight8(phy->mt76->antenna_mask), - .rx_streams = phy->mt76->antenna_mask, - .band_idx = phy->band_idx, + .tx_path_num = hweight16(phy->mt76->chainmask), + .rx_path = phy->mt76->chainmask >> (dev->chainshift * band), + .band_idx = band, .channel_band = ch_band[chandef->chan->band], }; #ifdef CONFIG_NL80211_TESTMODE if (phy->mt76->test.tx_antenna_mask && - (phy->mt76->test.state == MT76_TM_STATE_TX_FRAMES || - phy->mt76->test.state == MT76_TM_STATE_RX_FRAMES || - phy->mt76->test.state == MT76_TM_STATE_TX_CONT)) { - req.tx_streams_num = fls(phy->mt76->test.tx_antenna_mask); - req.rx_streams = phy->mt76->test.tx_antenna_mask; - - if (phy != &dev->phy) - req.rx_streams >>= dev->chainshift; + mt76_testmode_enabled(phy->mt76)) { + req.tx_path_num = fls(phy->mt76->test.tx_antenna_mask); + req.rx_path = phy->mt76->test.tx_antenna_mask; } #endif + if (mt76_connac_spe_idx(phy->mt76->antenna_mask)) + req.tx_path_num = fls(phy->mt76->antenna_mask); + if (cmd == MCU_EXT_CMD(SET_RX_PATH) || dev->mt76.hw->conf.flags & IEEE80211_CONF_MONITOR) req.switch_reason = CH_SWITCH_NORMAL; @@ -2665,7 +2714,7 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) req.switch_reason = CH_SWITCH_NORMAL; if (cmd == MCU_EXT_CMD(CHANNEL_SWITCH)) - req.rx_streams = hweight8(req.rx_streams); + req.rx_path = hweight8(req.rx_path); if (chandef->width == NL80211_CHAN_WIDTH_80P80) { int freq2 = chandef->center_freq2; @@ -2927,25 +2976,36 @@ int mt7915_mcu_get_chan_mib_info(struct mt7915_phy *phy, bool chan_switch) { /* strict order */ static const u32 offs[] = { - MIB_BUSY_TIME, MIB_TX_TIME, MIB_RX_TIME, MIB_OBSS_AIRTIME, - MIB_BUSY_TIME_V2, MIB_TX_TIME_V2, MIB_RX_TIME_V2, + MIB_NON_WIFI_TIME, + MIB_TX_TIME, + MIB_RX_TIME, + MIB_OBSS_AIRTIME, + MIB_TXOP_INIT_COUNT, + /* v2 */ + MIB_NON_WIFI_TIME_V2, + MIB_TX_TIME_V2, + MIB_RX_TIME_V2, MIB_OBSS_AIRTIME_V2 }; struct mt76_channel_state *state = phy->mt76->chan_state; struct mt76_channel_state *state_ts = &phy->state_ts; struct mt7915_dev *dev = phy->dev; - struct mt7915_mcu_mib *res, req[4]; + struct mt7915_mcu_mib *res, req[5]; struct sk_buff *skb; int i, ret, start = 0, ofs = 20; + u64 cc_tx; if (!is_mt7915(&dev->mt76)) { - start = 4; + start = 5; ofs = 0; } - for (i = 0; i < 4; i++) { - req[i].band = cpu_to_le32(phy != &dev->phy); + for (i = 0; i < 5; i++) { + req[i].band = cpu_to_le32(phy->mt76->band_idx); req[i].offs = cpu_to_le32(offs[i + start]); + + if (!is_mt7915(&dev->mt76) && i == 3) + break; } ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_EXT_CMD(GET_MIB_INFO), @@ -2955,20 +3015,24 @@ int mt7915_mcu_get_chan_mib_info(struct mt7915_phy *phy, bool chan_switch) res = (struct mt7915_mcu_mib *)(skb->data + ofs); +#define __res_u64(s) le64_to_cpu(res[s].data) + /* subtract Tx backoff time from Tx duration */ + cc_tx = is_mt7915(&dev->mt76) ? __res_u64(1) - __res_u64(4) : __res_u64(1); + if (chan_switch) goto out; -#define __res_u64(s) le64_to_cpu(res[s].data) - state->cc_busy += __res_u64(0) - state_ts->cc_busy; - state->cc_tx += __res_u64(1) - state_ts->cc_tx; + state->cc_tx += cc_tx - state_ts->cc_tx; state->cc_bss_rx += __res_u64(2) - state_ts->cc_bss_rx; state->cc_rx += __res_u64(2) + __res_u64(3) - state_ts->cc_rx; + state->cc_busy += __res_u64(0) + cc_tx + __res_u64(2) + __res_u64(3) - + state_ts->cc_busy; out: - state_ts->cc_busy = __res_u64(0); - state_ts->cc_tx = __res_u64(1); + state_ts->cc_tx = cc_tx; state_ts->cc_bss_rx = __res_u64(2); state_ts->cc_rx = __res_u64(2) + __res_u64(3); + state_ts->cc_busy = __res_u64(0) + cc_tx + __res_u64(2) + __res_u64(3); #undef __res_u64 dev_kfree_skb(skb); @@ -2982,11 +3046,11 @@ int mt7915_mcu_get_temperature(struct mt7915_phy *phy) struct { u8 ctrl_id; u8 action; - u8 dbdc_idx; + u8 band_idx; u8 rsv[5]; } req = { .ctrl_id = THERMAL_SENSOR_TEMP_QUERY, - .dbdc_idx = phy != &dev->phy, + .band_idx = phy->mt76->band_idx, }; return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(THERMAL_CTRL), &req, @@ -3005,7 +3069,7 @@ int mt7915_mcu_set_thermal_throttling(struct mt7915_phy *phy, u8 state) u8 rsv[2]; } __packed req = { .ctrl = { - .band_idx = phy->band_idx, + .band_idx = phy->mt76->band_idx, }, }; int level; @@ -3045,28 +3109,103 @@ out: &req, sizeof(req), false); } -int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy) +int mt7915_mcu_set_txpower_frame_min(struct mt7915_phy *phy, s8 txpower) { struct mt7915_dev *dev = phy->dev; + struct { + u8 format_id; + u8 rsv; + u8 band_idx; + s8 txpower_min; + } __packed req = { + .format_id = TX_POWER_LIMIT_FRAME_MIN, + .band_idx = phy->mt76->band_idx, + .txpower_min = txpower * 2, /* 0.5db */ + }; + + return mt76_mcu_send_msg(&dev->mt76, + MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), &req, + sizeof(req), true); +} + +int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, s8 txpower) +{ + struct mt7915_sta *msta = (struct mt7915_sta *)sta->drv_priv; + struct mt7915_dev *dev = phy->dev; struct mt76_phy *mphy = phy->mt76; - struct ieee80211_hw *hw = mphy->hw; - struct mt7915_sku_val { + struct { u8 format_id; - u8 limit_type; - u8 dbdc_idx; - s8 val[MT7915_SKU_RATE_NUM]; + u8 rsv[3]; + u8 band_idx; + s8 txpower_max; + __le16 wcid; + s8 txpower_offs[48]; } __packed req = { - .format_id = 4, - .dbdc_idx = phy != &dev->phy, + .format_id = TX_POWER_LIMIT_FRAME, + .band_idx = phy->mt76->band_idx, + .txpower_max = DIV_ROUND_UP(mphy->txpower_cur, 2), + .wcid = cpu_to_le16(msta->wcid.idx), + }; + int ret; + s8 txpower_sku[MT7915_SKU_RATE_NUM]; + + ret = mt7915_mcu_get_txpower_sku(phy, txpower_sku, sizeof(txpower_sku)); + if (ret) + return ret; + + txpower = mt7915_get_power_bound(phy, txpower); + if (txpower > mphy->txpower_cur || txpower < 0) + return -EINVAL; + + if (txpower) { + u32 offs, len, i; + + if (sta->deflink.ht_cap.ht_supported) { + const u8 *sku_len = mt7915_sku_group_len; + + offs = sku_len[SKU_CCK] + sku_len[SKU_OFDM]; + len = sku_len[SKU_HT_BW20] + sku_len[SKU_HT_BW40]; + + if (sta->deflink.vht_cap.vht_supported) { + offs += len; + len = sku_len[SKU_VHT_BW20] * 4; + + if (sta->deflink.he_cap.has_he) { + offs += len + sku_len[SKU_HE_RU26] * 3; + len = sku_len[SKU_HE_RU242] * 4; + } + } + } else { + return -EINVAL; + } + + for (i = 0; i < len; i++, offs++) + req.txpower_offs[i] = + DIV_ROUND_UP(txpower - txpower_sku[offs], 2); + } + + return mt76_mcu_send_msg(&dev->mt76, + MCU_EXT_CMD(TX_POWER_FEATURE_CTRL), &req, + sizeof(req), true); +} + +int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy) +{ + struct mt7915_dev *dev = phy->dev; + struct mt76_phy *mphy = phy->mt76; + struct ieee80211_hw *hw = mphy->hw; + struct mt7915_mcu_txpower_sku req = { + .format_id = TX_POWER_LIMIT_TABLE, + .band_idx = phy->mt76->band_idx, }; struct mt76_power_limits limits_array; s8 *la = (s8 *)&limits_array; - int i, idx, n_chains = hweight8(mphy->antenna_mask); - int tx_power = hw->conf.power_level * 2; + int i, idx; + int tx_power; - tx_power = mt76_get_sar_power(mphy, mphy->chandef.chan, - tx_power); - tx_power -= mt76_tx_power_nss_delta(n_chains); + tx_power = mt7915_get_power_bound(phy, hw->conf.power_level); tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan, &limits_array, tx_power); mphy->txpower_cur = tx_power; @@ -3085,7 +3224,7 @@ int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy) } for (j = 0; j < min_t(u8, mcs_num, len); j++) - req.val[idx + j] = la[j]; + req.txpower_sku[idx + j] = la[j]; la += mcs_num; idx += len; @@ -3103,14 +3242,14 @@ int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len) struct { u8 format_id; u8 category; - u8 band; + u8 band_idx; u8 _rsv; } __packed req = { - .format_id = 7, + .format_id = TX_POWER_LIMIT_INFO, .category = RATE_POWER_INFO, - .band = phy != &dev->phy, + .band_idx = phy->mt76->band_idx, }; - s8 res[MT7915_SKU_RATE_NUM][2]; + s8 txpower_sku[MT7915_SKU_RATE_NUM][2]; struct sk_buff *skb; int ret, i; @@ -3120,9 +3259,9 @@ int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len) if (ret) return ret; - memcpy(res, skb->data + 4, sizeof(res)); + memcpy(txpower_sku, skb->data + 4, sizeof(txpower_sku)); for (i = 0; i < len; i++) - txpower[i] = res[i][req.band]; + txpower[i] = txpower_sku[i][req.band_idx]; dev_kfree_skb(skb); @@ -3157,11 +3296,11 @@ int mt7915_mcu_set_sku_en(struct mt7915_phy *phy, bool enable) struct mt7915_sku { u8 format_id; u8 sku_enable; - u8 dbdc_idx; + u8 band_idx; u8 rsv; } __packed req = { - .format_id = 0, - .dbdc_idx = phy != &dev->phy, + .format_id = TX_POWER_LIMIT_ENABLE, + .band_idx = phy->mt76->band_idx, .sku_enable = enable, }; @@ -3236,31 +3375,193 @@ int mt7915_mcu_set_txbf(struct mt7915_dev *dev, u8 action) sizeof(req), true); } -int mt7915_mcu_add_obss_spr(struct mt7915_dev *dev, struct ieee80211_vif *vif, - bool enable) +static int +mt7915_mcu_enable_obss_spr(struct mt7915_phy *phy, u8 action, u8 val) +{ + struct mt7915_dev *dev = phy->dev; + struct mt7915_mcu_sr_ctrl req = { + .action = action, + .argnum = 1, + .band_idx = phy->mt76->band_idx, + .val = cpu_to_le32(val), + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(SET_SPR), &req, + sizeof(req), true); +} + +static int +mt7915_mcu_set_obss_spr_pd(struct mt7915_phy *phy, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct mt7915_dev *dev = phy->dev; + struct { + struct mt7915_mcu_sr_ctrl ctrl; + struct { + u8 pd_th_non_srg; + u8 pd_th_srg; + u8 period_offs; + u8 rcpi_src; + __le16 obss_pd_min; + __le16 obss_pd_min_srg; + u8 resp_txpwr_mode; + u8 txpwr_restrict_mode; + u8 txpwr_ref; + u8 rsv[3]; + } __packed param; + } __packed req = { + .ctrl = { + .action = SPR_SET_PARAM, + .argnum = 9, + .band_idx = phy->mt76->band_idx, + }, + }; + int ret; + u8 max_th = 82, non_srg_max_th = 62; + + /* disable firmware dynamical PD asjustment */ + ret = mt7915_mcu_enable_obss_spr(phy, SPR_ENABLE_DPD, false); + if (ret) + return ret; + + if (he_obss_pd->sr_ctrl & + IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED) + req.param.pd_th_non_srg = max_th; + else if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT) + req.param.pd_th_non_srg = max_th - he_obss_pd->non_srg_max_offset; + else + req.param.pd_th_non_srg = non_srg_max_th; + + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT) + req.param.pd_th_srg = max_th - he_obss_pd->max_offset; + + req.param.obss_pd_min = cpu_to_le16(82); + req.param.obss_pd_min_srg = cpu_to_le16(82); + req.param.txpwr_restrict_mode = 2; + req.param.txpwr_ref = 21; + + return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(SET_SPR), &req, + sizeof(req), true); +} + +static int +mt7915_mcu_set_obss_spr_siga(struct mt7915_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd) { -#define MT_SPR_ENABLE 1 struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; + struct mt7915_dev *dev = phy->dev; + u8 omac = mvif->mt76.omac_idx; struct { - u8 action; - u8 arg_num; - u8 band_idx; - u8 status; - u8 drop_tx_idx; - u8 sta_idx; /* 256 sta */ - u8 rsv[2]; - __le32 val; + struct mt7915_mcu_sr_ctrl ctrl; + struct { + u8 omac; + u8 rsv[3]; + u8 flag[20]; + } __packed siga; + } __packed req = { + .ctrl = { + .action = SPR_SET_SIGA, + .argnum = 1, + .band_idx = phy->mt76->band_idx, + }, + .siga = { + .omac = omac > HW_BSSID_MAX ? omac - 12 : omac, + }, + }; + int ret; + + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED) + req.siga.flag[req.siga.omac] = 0xf; + else + return 0; + + /* switch to normal AP mode */ + ret = mt7915_mcu_enable_obss_spr(phy, SPR_ENABLE_MODE, 0); + if (ret) + return ret; + + return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(SET_SPR), &req, + sizeof(req), true); +} + +static int +mt7915_mcu_set_obss_spr_bitmap(struct mt7915_phy *phy, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct mt7915_dev *dev = phy->dev; + struct { + struct mt7915_mcu_sr_ctrl ctrl; + struct { + __le32 color_l[2]; + __le32 color_h[2]; + __le32 bssid_l[2]; + __le32 bssid_h[2]; + } __packed bitmap; } __packed req = { - .action = MT_SPR_ENABLE, - .arg_num = 1, - .band_idx = mvif->mt76.band_idx, - .val = cpu_to_le32(enable), + .ctrl = { + .action = SPR_SET_SRG_BITMAP, + .argnum = 4, + .band_idx = phy->mt76->band_idx, + }, }; + u32 bitmap; + + memcpy(&bitmap, he_obss_pd->bss_color_bitmap, sizeof(bitmap)); + req.bitmap.color_l[req.ctrl.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->bss_color_bitmap + 4, sizeof(bitmap)); + req.bitmap.color_h[req.ctrl.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->partial_bssid_bitmap, sizeof(bitmap)); + req.bitmap.bssid_l[req.ctrl.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->partial_bssid_bitmap + 4, sizeof(bitmap)); + req.bitmap.bssid_h[req.ctrl.band_idx] = cpu_to_le32(bitmap); return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(SET_SPR), &req, sizeof(req), true); } +int mt7915_mcu_add_obss_spr(struct mt7915_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + int ret; + + /* enable firmware scene detection algorithms */ + ret = mt7915_mcu_enable_obss_spr(phy, SPR_ENABLE_SD, sr_scene_detect); + if (ret) + return ret; + + /* firmware dynamically adjusts PD threshold so skip manual control */ + if (sr_scene_detect && !he_obss_pd->enable) + return 0; + + /* enable spatial reuse */ + ret = mt7915_mcu_enable_obss_spr(phy, SPR_ENABLE, he_obss_pd->enable); + if (ret) + return ret; + + if (sr_scene_detect || !he_obss_pd->enable) + return 0; + + ret = mt7915_mcu_enable_obss_spr(phy, SPR_ENABLE_TX, true); + if (ret) + return ret; + + /* set SRG/non-SRG OBSS PD threshold */ + ret = mt7915_mcu_set_obss_spr_pd(phy, he_obss_pd); + if (ret) + return ret; + + /* Set SR prohibit */ + ret = mt7915_mcu_set_obss_spr_siga(phy, vif, he_obss_pd); + if (ret) + return ret; + + /* set SRG BSS color/BSSID bitmap */ + return mt7915_mcu_set_obss_spr_bitmap(phy, he_obss_pd); +} + int mt7915_mcu_get_rx_rate(struct mt7915_phy *phy, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct rate_info *rate) { @@ -3447,8 +3748,8 @@ int mt7915_mcu_rf_regval(struct mt7915_dev *dev, u32 regidx, u32 *val, bool set) __le32 ofs; __le32 data; } __packed req = { - .idx = cpu_to_le32(u32_get_bits(regidx, GENMASK(31, 28))), - .ofs = cpu_to_le32(u32_get_bits(regidx, GENMASK(27, 0))), + .idx = cpu_to_le32(u32_get_bits(regidx, GENMASK(31, 24))), + .ofs = cpu_to_le32(u32_get_bits(regidx, GENMASK(23, 0))), .data = set ? cpu_to_le32(*val) : 0, }; struct sk_buff *skb; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index cd1edf553fc1..29b5434bfdb8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -129,6 +129,17 @@ struct mt7915_mcu_background_chain_ctrl { u8 rsv[2]; } __packed; +struct mt7915_mcu_sr_ctrl { + u8 action; + u8 argnum; + u8 band_idx; + u8 status; + u8 drop_ta_idx; + u8 sta_idx; /* 256 sta */ + u8 rsv[2]; + __le32 val; +} __packed; + struct mt7915_mcu_eeprom { u8 buffer_mode; u8 format; @@ -160,17 +171,26 @@ struct mt7915_mcu_mib { enum mt7915_chan_mib_offs { /* mt7915 */ - MIB_BUSY_TIME = 14, MIB_TX_TIME = 81, MIB_RX_TIME, MIB_OBSS_AIRTIME = 86, + MIB_NON_WIFI_TIME, + MIB_TXOP_INIT_COUNT, + /* mt7916 */ - MIB_BUSY_TIME_V2 = 0, MIB_TX_TIME_V2 = 6, MIB_RX_TIME_V2 = 8, - MIB_OBSS_AIRTIME_V2 = 490 + MIB_OBSS_AIRTIME_V2 = 490, + MIB_NON_WIFI_TIME_V2 }; +struct mt7915_mcu_txpower_sku { + u8 format_id; + u8 limit_type; + u8 band_idx; + s8 txpower_sku[MT7915_SKU_RATE_NUM]; +} __packed; + struct edca { u8 queue; u8 set; @@ -394,6 +414,7 @@ enum { RATE_PARAM_FIXED_MCS, RATE_PARAM_FIXED_GI = 11, RATE_PARAM_AUTO = 20, + RATE_PARAM_SPE_UPDATE = 22, }; #define RATE_CFG_MCS GENMASK(3, 0) @@ -406,6 +427,25 @@ enum { #define RATE_CFG_HE_LTF GENMASK(31, 28) enum { + TX_POWER_LIMIT_ENABLE, + TX_POWER_LIMIT_TABLE = 0x4, + TX_POWER_LIMIT_INFO = 0x7, + TX_POWER_LIMIT_FRAME = 0x11, + TX_POWER_LIMIT_FRAME_MIN = 0x12, +}; + +enum { + SPR_ENABLE = 0x1, + SPR_ENABLE_SD = 0x3, + SPR_ENABLE_MODE = 0x5, + SPR_ENABLE_DPD = 0x23, + SPR_ENABLE_TX = 0x25, + SPR_SET_SRG_BITMAP = 0x80, + SPR_SET_PARAM = 0xc2, + SPR_SET_SIGA = 0xdc, +}; + +enum { THERMAL_PROTECT_PARAMETER_CTRL, THERMAL_PROTECT_BASIC_INFO, THERMAL_PROTECT_ENABLE, @@ -447,6 +487,8 @@ enum { SER_SET_RECOVER_L3_TX_ABORT, SER_SET_RECOVER_L3_TX_DISABLE, SER_SET_RECOVER_L3_BF, + SER_SET_RECOVER_FULL, + SER_SET_SYSTEM_ASSERT, /* action */ SER_ENABLE = 2, SER_RECOVER @@ -474,4 +516,16 @@ enum { sizeof(struct bss_info_bcn_cont) + \ sizeof(struct bss_info_inband_discovery)) +static inline s8 +mt7915_get_power_bound(struct mt7915_phy *phy, s8 txpower) +{ + struct mt76_phy *mphy = phy->mt76; + int n_chains = hweight8(mphy->antenna_mask); + + txpower = mt76_get_sar_power(mphy, mphy->chandef.chan, txpower * 2); + txpower -= mt76_tx_power_nss_delta(n_chains); + + return txpower; +} + #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c index 7bd5f6725d7b..0a95c3da241b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c @@ -9,53 +9,112 @@ #include "mt7915.h" #include "mac.h" #include "../trace.h" +#include "../dma.h" + +static bool wed_enable; +module_param(wed_enable, bool, 0644); +MODULE_PARM_DESC(wed_enable, "Enable Wireless Ethernet Dispatch support"); static const u32 mt7915_reg[] = { - [INT_SOURCE_CSR] = 0xd7010, - [INT_MASK_CSR] = 0xd7014, - [INT1_SOURCE_CSR] = 0xd7088, - [INT1_MASK_CSR] = 0xd708c, - [INT_MCU_CMD_SOURCE] = 0xd51f0, - [INT_MCU_CMD_EVENT] = 0x3108, - [WFDMA0_ADDR] = 0xd4000, - [WFDMA0_PCIE1_ADDR] = 0xd8000, - [WFDMA_EXT_CSR_ADDR] = 0xd7000, - [CBTOP1_PHY_END] = 0x77ffffff, - [INFRA_MCU_ADDR_END] = 0x7c3fffff, - [FW_EXCEPTION_ADDR] = 0x219848, - [SWDEF_BASE_ADDR] = 0x41f200, + [INT_SOURCE_CSR] = 0xd7010, + [INT_MASK_CSR] = 0xd7014, + [INT1_SOURCE_CSR] = 0xd7088, + [INT1_MASK_CSR] = 0xd708c, + [INT_MCU_CMD_SOURCE] = 0xd51f0, + [INT_MCU_CMD_EVENT] = 0x3108, + [WFDMA0_ADDR] = 0xd4000, + [WFDMA0_PCIE1_ADDR] = 0xd8000, + [WFDMA_EXT_CSR_ADDR] = 0xd7000, + [CBTOP1_PHY_END] = 0x77ffffff, + [INFRA_MCU_ADDR_END] = 0x7c3fffff, + [FW_ASSERT_STAT_ADDR] = 0x219848, + [FW_EXCEPT_TYPE_ADDR] = 0x21987c, + [FW_EXCEPT_COUNT_ADDR] = 0x219848, + [FW_CIRQ_COUNT_ADDR] = 0x216f94, + [FW_CIRQ_IDX_ADDR] = 0x216ef8, + [FW_CIRQ_LISR_ADDR] = 0x2170ac, + [FW_TASK_ID_ADDR] = 0x216f90, + [FW_TASK_IDX_ADDR] = 0x216f9c, + [FW_TASK_QID1_ADDR] = 0x219680, + [FW_TASK_QID2_ADDR] = 0x219760, + [FW_TASK_START_ADDR] = 0x219558, + [FW_TASK_END_ADDR] = 0x219554, + [FW_TASK_SIZE_ADDR] = 0x219560, + [FW_LAST_MSG_ID_ADDR] = 0x216f70, + [FW_EINT_INFO_ADDR] = 0x219818, + [FW_SCHED_INFO_ADDR] = 0x219828, + [SWDEF_BASE_ADDR] = 0x41f200, + [TXQ_WED_RING_BASE] = 0xd7300, + [RXQ_WED_RING_BASE] = 0xd7410, + [RXQ_WED_DATA_RING_BASE] = 0xd4500, }; static const u32 mt7916_reg[] = { - [INT_SOURCE_CSR] = 0xd4200, - [INT_MASK_CSR] = 0xd4204, - [INT1_SOURCE_CSR] = 0xd8200, - [INT1_MASK_CSR] = 0xd8204, - [INT_MCU_CMD_SOURCE] = 0xd41f0, - [INT_MCU_CMD_EVENT] = 0x2108, - [WFDMA0_ADDR] = 0xd4000, - [WFDMA0_PCIE1_ADDR] = 0xd8000, - [WFDMA_EXT_CSR_ADDR] = 0xd7000, - [CBTOP1_PHY_END] = 0x7fffffff, - [INFRA_MCU_ADDR_END] = 0x7c085fff, - [FW_EXCEPTION_ADDR] = 0x022050bc, - [SWDEF_BASE_ADDR] = 0x411400, + [INT_SOURCE_CSR] = 0xd4200, + [INT_MASK_CSR] = 0xd4204, + [INT1_SOURCE_CSR] = 0xd8200, + [INT1_MASK_CSR] = 0xd8204, + [INT_MCU_CMD_SOURCE] = 0xd41f0, + [INT_MCU_CMD_EVENT] = 0x2108, + [WFDMA0_ADDR] = 0xd4000, + [WFDMA0_PCIE1_ADDR] = 0xd8000, + [WFDMA_EXT_CSR_ADDR] = 0xd7000, + [CBTOP1_PHY_END] = 0x7fffffff, + [INFRA_MCU_ADDR_END] = 0x7c085fff, + [FW_ASSERT_STAT_ADDR] = 0x02204c14, + [FW_EXCEPT_TYPE_ADDR] = 0x022051a4, + [FW_EXCEPT_COUNT_ADDR] = 0x022050bc, + [FW_CIRQ_COUNT_ADDR] = 0x022001ac, + [FW_CIRQ_IDX_ADDR] = 0x02204f84, + [FW_CIRQ_LISR_ADDR] = 0x022050d0, + [FW_TASK_ID_ADDR] = 0x0220406c, + [FW_TASK_IDX_ADDR] = 0x0220500c, + [FW_TASK_QID1_ADDR] = 0x022028c8, + [FW_TASK_QID2_ADDR] = 0x02202a38, + [FW_TASK_START_ADDR] = 0x0220286c, + [FW_TASK_END_ADDR] = 0x02202870, + [FW_TASK_SIZE_ADDR] = 0x02202878, + [FW_LAST_MSG_ID_ADDR] = 0x02204fe8, + [FW_EINT_INFO_ADDR] = 0x0220525c, + [FW_SCHED_INFO_ADDR] = 0x0220516c, + [SWDEF_BASE_ADDR] = 0x411400, + [TXQ_WED_RING_BASE] = 0xd7300, + [RXQ_WED_RING_BASE] = 0xd7410, + [RXQ_WED_DATA_RING_BASE] = 0xd4540, }; static const u32 mt7986_reg[] = { - [INT_SOURCE_CSR] = 0x24200, - [INT_MASK_CSR] = 0x24204, - [INT1_SOURCE_CSR] = 0x28200, - [INT1_MASK_CSR] = 0x28204, - [INT_MCU_CMD_SOURCE] = 0x241f0, - [INT_MCU_CMD_EVENT] = 0x54000108, - [WFDMA0_ADDR] = 0x24000, - [WFDMA0_PCIE1_ADDR] = 0x28000, - [WFDMA_EXT_CSR_ADDR] = 0x27000, - [CBTOP1_PHY_END] = 0x7fffffff, - [INFRA_MCU_ADDR_END] = 0x7c085fff, - [FW_EXCEPTION_ADDR] = 0x02204ffc, - [SWDEF_BASE_ADDR] = 0x411400, + [INT_SOURCE_CSR] = 0x24200, + [INT_MASK_CSR] = 0x24204, + [INT1_SOURCE_CSR] = 0x28200, + [INT1_MASK_CSR] = 0x28204, + [INT_MCU_CMD_SOURCE] = 0x241f0, + [INT_MCU_CMD_EVENT] = 0x54000108, + [WFDMA0_ADDR] = 0x24000, + [WFDMA0_PCIE1_ADDR] = 0x28000, + [WFDMA_EXT_CSR_ADDR] = 0x27000, + [CBTOP1_PHY_END] = 0x7fffffff, + [INFRA_MCU_ADDR_END] = 0x7c085fff, + [FW_ASSERT_STAT_ADDR] = 0x02204b54, + [FW_EXCEPT_TYPE_ADDR] = 0x022050dc, + [FW_EXCEPT_COUNT_ADDR] = 0x02204ffc, + [FW_CIRQ_COUNT_ADDR] = 0x022001ac, + [FW_CIRQ_IDX_ADDR] = 0x02204ec4, + [FW_CIRQ_LISR_ADDR] = 0x02205010, + [FW_TASK_ID_ADDR] = 0x02204fac, + [FW_TASK_IDX_ADDR] = 0x02204f4c, + [FW_TASK_QID1_ADDR] = 0x02202814, + [FW_TASK_QID2_ADDR] = 0x02202984, + [FW_TASK_START_ADDR] = 0x022027b8, + [FW_TASK_END_ADDR] = 0x022027bc, + [FW_TASK_SIZE_ADDR] = 0x022027c4, + [FW_LAST_MSG_ID_ADDR] = 0x02204f28, + [FW_EINT_INFO_ADDR] = 0x02205194, + [FW_SCHED_INFO_ADDR] = 0x022051a4, + [SWDEF_BASE_ADDR] = 0x411400, + [TXQ_WED_RING_BASE] = 0x24420, + [RXQ_WED_RING_BASE] = 0x24520, + [RXQ_WED_DATA_RING_BASE] = 0x24540, }; static const u32 mt7915_offs[] = { @@ -448,6 +507,14 @@ static u32 __mt7915_reg_addr(struct mt7915_dev *dev, u32 addr) return mt7915_reg_map_l2(dev, addr); } +void mt7915_memcpy_fromio(struct mt7915_dev *dev, void *buf, u32 offset, + size_t len) +{ + u32 addr = __mt7915_reg_addr(dev, offset); + + memcpy_fromio(buf, dev->mt76.mmio.regs + addr, len); +} + static u32 mt7915_rr(struct mt76_dev *mdev, u32 offset) { struct mt7915_dev *dev = container_of(mdev, struct mt7915_dev, mt76); @@ -472,6 +539,257 @@ static u32 mt7915_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) return dev->bus_ops->rmw(mdev, addr, mask, val); } +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +static int mt7915_mmio_wed_offload_enable(struct mtk_wed_device *wed) +{ + struct mt7915_dev *dev; + struct mt7915_phy *phy; + int ret; + + dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); + + spin_lock_bh(&dev->mt76.token_lock); + dev->mt76.token_size = wed->wlan.token_start; + spin_unlock_bh(&dev->mt76.token_lock); + + ret = wait_event_timeout(dev->mt76.tx_wait, + !dev->mt76.wed_token_count, HZ); + if (!ret) + return -EAGAIN; + + phy = &dev->phy; + mt76_set(dev, MT_AGG_ACR4(phy->mt76->band_idx), MT_AGG_ACR_PPDU_TXS2H); + + phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; + if (phy) + mt76_set(dev, MT_AGG_ACR4(phy->mt76->band_idx), + MT_AGG_ACR_PPDU_TXS2H); + + return 0; +} + +static void mt7915_mmio_wed_offload_disable(struct mtk_wed_device *wed) +{ + struct mt7915_dev *dev; + struct mt7915_phy *phy; + + dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); + + spin_lock_bh(&dev->mt76.token_lock); + dev->mt76.token_size = MT7915_TOKEN_SIZE; + spin_unlock_bh(&dev->mt76.token_lock); + + /* MT_TXD5_TX_STATUS_HOST (MPDU format) has higher priority than + * MT_AGG_ACR_PPDU_TXS2H (PPDU format) even though ACR bit is set. + */ + phy = &dev->phy; + mt76_clear(dev, MT_AGG_ACR4(phy->mt76->band_idx), MT_AGG_ACR_PPDU_TXS2H); + + phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; + if (phy) + mt76_clear(dev, MT_AGG_ACR4(phy->mt76->band_idx), + MT_AGG_ACR_PPDU_TXS2H); +} + +static void mt7915_mmio_wed_release_rx_buf(struct mtk_wed_device *wed) +{ + struct mt7915_dev *dev; + struct page *page; + int i; + + dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); + for (i = 0; i < dev->mt76.rx_token_size; i++) { + struct mt76_txwi_cache *t; + + t = mt76_rx_token_release(&dev->mt76, i); + if (!t || !t->ptr) + continue; + + dma_unmap_single(dev->mt76.dma_dev, t->dma_addr, + wed->wlan.rx_size, DMA_FROM_DEVICE); + skb_free_frag(t->ptr); + t->ptr = NULL; + + mt76_put_rxwi(&dev->mt76, t); + } + + if (!wed->rx_buf_ring.rx_page.va) + return; + + page = virt_to_page(wed->rx_buf_ring.rx_page.va); + __page_frag_cache_drain(page, wed->rx_buf_ring.rx_page.pagecnt_bias); + memset(&wed->rx_buf_ring.rx_page, 0, sizeof(wed->rx_buf_ring.rx_page)); +} + +static u32 mt7915_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size) +{ + struct mtk_rxbm_desc *desc = wed->rx_buf_ring.desc; + struct mt7915_dev *dev; + u32 length; + int i; + + dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); + length = SKB_DATA_ALIGN(NET_SKB_PAD + wed->wlan.rx_size + + sizeof(struct skb_shared_info)); + + for (i = 0; i < size; i++) { + struct mt76_txwi_cache *t = mt76_get_rxwi(&dev->mt76); + dma_addr_t phy_addr; + int token; + void *ptr; + + ptr = page_frag_alloc(&wed->rx_buf_ring.rx_page, length, + GFP_KERNEL); + if (!ptr) + goto unmap; + + phy_addr = dma_map_single(dev->mt76.dma_dev, ptr, + wed->wlan.rx_size, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev->mt76.dev, phy_addr))) { + skb_free_frag(ptr); + goto unmap; + } + + desc->buf0 = cpu_to_le32(phy_addr); + token = mt76_rx_token_consume(&dev->mt76, ptr, t, phy_addr); + desc->token |= cpu_to_le32(FIELD_PREP(MT_DMA_CTL_TOKEN, + token)); + desc++; + } + + return 0; + +unmap: + mt7915_mmio_wed_release_rx_buf(wed); + return -ENOMEM; +} + +static void mt7915_mmio_wed_update_rx_stats(struct mtk_wed_device *wed, + struct mtk_wed_wo_rx_stats *stats) +{ + int idx = le16_to_cpu(stats->wlan_idx); + struct mt7915_dev *dev; + struct mt76_wcid *wcid; + + dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); + + if (idx >= mt7915_wtbl_size(dev)) + return; + + rcu_read_lock(); + + wcid = rcu_dereference(dev->mt76.wcid[idx]); + if (wcid) { + wcid->stats.rx_bytes += le32_to_cpu(stats->rx_byte_cnt); + wcid->stats.rx_packets += le32_to_cpu(stats->rx_pkt_cnt); + wcid->stats.rx_errors += le32_to_cpu(stats->rx_err_cnt); + wcid->stats.rx_drops += le32_to_cpu(stats->rx_drop_cnt); + } + + rcu_read_unlock(); +} +#endif + +int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, + bool pci, int *irq) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + int ret; + + if (!wed_enable) + return 0; + + if (pci) { + struct pci_dev *pci_dev = pdev_ptr; + + wed->wlan.pci_dev = pci_dev; + wed->wlan.bus_type = MTK_WED_BUS_PCIE; + wed->wlan.base = devm_ioremap(dev->mt76.dev, + pci_resource_start(pci_dev, 0), + pci_resource_len(pci_dev, 0)); + wed->wlan.phy_base = pci_resource_start(pci_dev, 0); + wed->wlan.wpdma_int = pci_resource_start(pci_dev, 0) + + MT_INT_WED_SOURCE_CSR; + wed->wlan.wpdma_mask = pci_resource_start(pci_dev, 0) + + MT_INT_WED_MASK_CSR; + wed->wlan.wpdma_phys = pci_resource_start(pci_dev, 0) + + MT_WFDMA_EXT_CSR_BASE; + wed->wlan.wpdma_tx = pci_resource_start(pci_dev, 0) + + MT_TXQ_WED_RING_BASE; + wed->wlan.wpdma_txfree = pci_resource_start(pci_dev, 0) + + MT_RXQ_WED_RING_BASE; + wed->wlan.wpdma_rx_glo = pci_resource_start(pci_dev, 0) + + MT_WPDMA_GLO_CFG; + wed->wlan.wpdma_rx = pci_resource_start(pci_dev, 0) + + MT_RXQ_WED_DATA_RING_BASE; + } else { + struct platform_device *plat_dev = pdev_ptr; + struct resource *res; + + res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0); + if (!res) + return -ENOMEM; + + wed->wlan.platform_dev = plat_dev; + wed->wlan.bus_type = MTK_WED_BUS_AXI; + wed->wlan.base = devm_ioremap(dev->mt76.dev, res->start, + resource_size(res)); + wed->wlan.phy_base = res->start; + wed->wlan.wpdma_int = res->start + MT_INT_SOURCE_CSR; + wed->wlan.wpdma_mask = res->start + MT_INT_MASK_CSR; + wed->wlan.wpdma_tx = res->start + MT_TXQ_WED_RING_BASE; + wed->wlan.wpdma_txfree = res->start + MT_RXQ_WED_RING_BASE; + wed->wlan.wpdma_rx_glo = res->start + MT_WPDMA_GLO_CFG; + wed->wlan.wpdma_rx = res->start + MT_RXQ_WED_DATA_RING_BASE; + } + wed->wlan.nbuf = 4096; + wed->wlan.tx_tbit[0] = is_mt7915(&dev->mt76) ? 4 : 30; + wed->wlan.tx_tbit[1] = is_mt7915(&dev->mt76) ? 5 : 31; + wed->wlan.txfree_tbit = is_mt7986(&dev->mt76) ? 2 : 1; + wed->wlan.token_start = MT7915_TOKEN_SIZE - wed->wlan.nbuf; + wed->wlan.wcid_512 = !is_mt7915(&dev->mt76); + + wed->wlan.rx_nbuf = 65536; + wed->wlan.rx_npkt = MT7915_WED_RX_TOKEN_SIZE; + wed->wlan.rx_size = SKB_WITH_OVERHEAD(MT_RX_BUF_SIZE); + if (is_mt7915(&dev->mt76)) { + wed->wlan.rx_tbit[0] = 16; + wed->wlan.rx_tbit[1] = 17; + } else if (is_mt7986(&dev->mt76)) { + wed->wlan.rx_tbit[0] = 22; + wed->wlan.rx_tbit[1] = 23; + } else { + wed->wlan.rx_tbit[0] = 18; + wed->wlan.rx_tbit[1] = 19; + } + + wed->wlan.init_buf = mt7915_wed_init_buf; + wed->wlan.offload_enable = mt7915_mmio_wed_offload_enable; + wed->wlan.offload_disable = mt7915_mmio_wed_offload_disable; + wed->wlan.init_rx_buf = mt7915_mmio_wed_init_rx_buf; + wed->wlan.release_rx_buf = mt7915_mmio_wed_release_rx_buf; + wed->wlan.update_wo_rx_stats = mt7915_mmio_wed_update_rx_stats; + + dev->mt76.rx_token_size = wed->wlan.rx_npkt; + + if (mtk_wed_device_attach(wed)) + return 0; + + *irq = wed->irq; + dev->mt76.dma_dev = wed->dev; + + ret = dma_set_mask(wed->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + return 1; +#else + return 0; +#endif +} + static int mt7915_mmio_init(struct mt76_dev *mdev, void __iomem *mem_base, u32 device_id) @@ -536,7 +854,11 @@ void mt7915_dual_hif_set_irq_mask(struct mt7915_dev *dev, mdev->mmio.irqmask |= set; if (write_reg) { - mt76_wr(dev, MT_INT_MASK_CSR, mdev->mmio.irqmask); + if (mtk_wed_device_active(&mdev->mmio.wed)) + mtk_wed_device_irq_set_mask(&mdev->mmio.wed, + mdev->mmio.irqmask); + else + mt76_wr(dev, MT_INT_MASK_CSR, mdev->mmio.irqmask); mt76_wr(dev, MT_INT1_MASK_CSR, mdev->mmio.irqmask); } @@ -560,6 +882,8 @@ static void mt7915_irq_tasklet(struct tasklet_struct *t) if (mtk_wed_device_active(wed)) { mtk_wed_device_irq_set_mask(wed, 0); + if (dev->hif2) + mt76_wr(dev, MT_INT1_MASK_CSR, 0); intr = mtk_wed_device_irq_get(wed, dev->mt76.mmio.irqmask); } else { mt76_wr(dev, MT_INT_MASK_CSR, 0); @@ -613,10 +937,9 @@ static void mt7915_irq_tasklet(struct tasklet_struct *t) u32 val = mt76_rr(dev, MT_MCU_CMD); mt76_wr(dev, MT_MCU_CMD, val); - if (val & MT_MCU_CMD_ERROR_MASK) { - dev->reset_state = val; - queue_work(dev->mt76.wq, &dev->reset_work); - wake_up(&dev->reset_wait); + if (val & (MT_MCU_CMD_ERROR_MASK | MT_MCU_CMD_WDT_MASK)) { + dev->recovery.state = val; + mt7915_reset(dev); } } } @@ -648,7 +971,8 @@ struct mt7915_dev *mt7915_mmio_probe(struct device *pdev, static const struct mt76_driver_ops drv_ops = { /* txwi_size = txd size + txp size */ .txwi_size = MT_TXD_SIZE + sizeof(struct mt76_connac_fw_txp), - .drv_flags = MT_DRV_TXWI_NO_FREE | MT_DRV_HW_MGMT_TXQ, + .drv_flags = MT_DRV_TXWI_NO_FREE | MT_DRV_HW_MGMT_TXQ | + MT_DRV_AMSDU_OFFLOAD, .survey_flags = SURVEY_INFO_TIME_TX | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_BSS_RX, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index 1eb11617a625..6351feba6bdf 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -68,6 +68,8 @@ #define MT7915_MIN_TWT_DUR 64 #define MT7915_MAX_QUEUE (MT_RXQ_BAND2 + __MT_MCUQ_MAX + 2) +#define MT7915_WED_RX_TOKEN_SIZE 12288 + struct mt7915_vif; struct mt7915_sta; struct mt7915_dfs_pulse; @@ -114,6 +116,8 @@ struct mt7915_twt_flow { u8 sched:1; }; +DECLARE_EWMA(avg_signal, 10, 8) + struct mt7915_sta { struct mt76_wcid wcid; /* must be first */ @@ -123,10 +127,12 @@ struct mt7915_sta { struct list_head rc_list; u32 airtime_ac[8]; + int ack_signal; + struct ewma_avg_signal avg_ack_signal; + unsigned long changed; unsigned long jiffies; unsigned long ampdu_state; - struct mt76_connac_sta_key_conf bip; struct { @@ -220,6 +226,15 @@ struct mib_stats { u32 tx_amsdu_cnt; }; +/* crash-dump */ +struct mt7915_crash_data { + guid_t guid; + struct timespec64 timestamp; + + u8 *memdump_buf; + size_t memdump_buf_len; +}; + struct mt7915_hif { struct list_head list; @@ -243,7 +258,6 @@ struct mt7915_phy { u32 rxfilter; u64 omac_mask; - u8 band_idx; u16 noise; @@ -301,9 +315,26 @@ struct mt7915_dev { struct work_struct init_work; struct work_struct rc_work; + struct work_struct dump_work; struct work_struct reset_work; wait_queue_head_t reset_wait; - u32 reset_state; + + struct { + u32 state; + u32 wa_reset_count; + u32 wm_reset_count; + bool hw_full_reset:1; + bool hw_init_done:1; + bool restart:1; + } recovery; + + /* protects coredump data */ + struct mutex dump_mutex; +#ifdef CONFIG_DEV_COREDUMP + struct { + struct mt7915_crash_data *crash_data; + } coredump; +#endif struct list_head sta_rc_list; struct list_head sta_poll_list; @@ -357,6 +388,7 @@ enum mt7915_rdd_cmd { RDD_DET_MODE, RDD_RADAR_EMULATE, RDD_START_TXQ = 20, + RDD_SET_WF_ANT = 30, RDD_CAC_START = 50, RDD_CAC_END, RDD_NORMAL_START, @@ -442,7 +474,14 @@ s8 mt7915_eeprom_get_power_delta(struct mt7915_dev *dev, int band); int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2); void mt7915_dma_prefetch(struct mt7915_dev *dev); void mt7915_dma_cleanup(struct mt7915_dev *dev); +int mt7915_dma_reset(struct mt7915_dev *dev, bool force); +int mt7915_txbf_init(struct mt7915_dev *dev); +void mt7915_init_txpower(struct mt7915_dev *dev, + struct ieee80211_supported_band *sband); +void mt7915_reset(struct mt7915_dev *dev); +int mt7915_run(struct ieee80211_hw *hw); int mt7915_mcu_init(struct mt7915_dev *dev); +int mt7915_mcu_init_firmware(struct mt7915_dev *dev); int mt7915_mcu_twt_agrt_update(struct mt7915_dev *dev, struct mt7915_vif *mvif, struct mt7915_twt_flow *flow, @@ -463,8 +502,8 @@ int mt7915_mcu_update_bss_color(struct mt7915_dev *dev, struct ieee80211_vif *vi struct cfg80211_he_bss_color *he_bss_color); int mt7915_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, int enable, u32 changed); -int mt7915_mcu_add_obss_spr(struct mt7915_dev *dev, struct ieee80211_vif *vif, - bool enable); +int mt7915_mcu_add_obss_spr(struct mt7915_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd); int mt7915_mcu_add_rate_ctrl(struct mt7915_dev *dev, struct ieee80211_vif *vif, struct ieee80211_sta *sta, bool changed); int mt7915_mcu_add_smps(struct mt7915_dev *dev, struct ieee80211_vif *vif, @@ -488,6 +527,10 @@ int mt7915_mcu_set_ser(struct mt7915_dev *dev, u8 action, u8 set, u8 band); int mt7915_mcu_set_sku_en(struct mt7915_phy *phy, bool enable); int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy); int mt7915_mcu_get_txpower_sku(struct mt7915_phy *phy, s8 *txpower, int len); +int mt7915_mcu_set_txpower_frame_min(struct mt7915_phy *phy, s8 txpower); +int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, s8 txpower); int mt7915_mcu_set_txbf(struct mt7915_dev *dev, u8 action); int mt7915_mcu_set_fcc5_lpn(struct mt7915_dev *dev, int val); int mt7915_mcu_set_pulse_th(struct mt7915_dev *dev, @@ -542,11 +585,17 @@ static inline void mt7915_irq_disable(struct mt7915_dev *dev, u32 mask) mt76_set_irq_mask(&dev->mt76, MT_INT_MASK_CSR, mask, 0); } +void mt7915_memcpy_fromio(struct mt7915_dev *dev, void *buf, u32 offset, + size_t len); + +void mt7915_mac_init(struct mt7915_dev *dev); u32 mt7915_mac_wtbl_lmac_addr(struct mt7915_dev *dev, u16 wcid, u8 dw); bool mt7915_mac_wtbl_update(struct mt7915_dev *dev, int idx, u32 mask); void mt7915_mac_reset_counters(struct mt7915_phy *phy); void mt7915_mac_cca_stats_reset(struct mt7915_phy *phy); void mt7915_mac_enable_nf(struct mt7915_dev *dev, bool ext_phy); +void mt7915_mac_enable_rtscts(struct mt7915_dev *dev, + struct ieee80211_vif *vif, bool enable); void mt7915_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, int pid, struct ieee80211_key_conf *key, @@ -558,6 +607,7 @@ void mt7915_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); void mt7915_mac_work(struct work_struct *work); void mt7915_mac_reset_work(struct work_struct *work); +void mt7915_mac_dump_work(struct work_struct *work); void mt7915_mac_sta_rc_work(struct work_struct *work); void mt7915_mac_update_stats(struct mt7915_phy *phy); void mt7915_mac_twt_teardown_flow(struct mt7915_dev *dev, @@ -572,7 +622,7 @@ int mt7915_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, struct mt76_tx_info *tx_info); void mt7915_tx_token_put(struct mt7915_dev *dev); void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); bool mt7915_rx_check(struct mt76_dev *mdev, void *data, int len); void mt7915_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); void mt7915_stats_work(struct work_struct *work); @@ -583,6 +633,7 @@ void mt7915_set_stream_vht_txbf_caps(struct mt7915_phy *phy); void mt7915_update_channel(struct mt76_phy *mphy); int mt7915_mcu_muru_debug_set(struct mt7915_dev *dev, bool enable); int mt7915_mcu_muru_debug_get(struct mt7915_phy *phy, void *ms); +int mt7915_mcu_wed_enable_rx_stats(struct mt7915_dev *dev); int mt7915_init_debugfs(struct mt7915_phy *phy); void mt7915_debugfs_rx_fw_monitor(struct mt7915_dev *dev, const void *data, int len); bool mt7915_debugfs_rx_log(struct mt7915_dev *dev, const void *data, int len); @@ -590,5 +641,7 @@ bool mt7915_debugfs_rx_log(struct mt7915_dev *dev, const void *data, int len); void mt7915_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct dentry *dir); #endif +int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, + bool pci, int *irq); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c index 728a879c3b00..39132894e8ea 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c @@ -12,9 +12,6 @@ #include "mac.h" #include "../trace.h" -static bool wed_enable = false; -module_param(wed_enable, bool, 0644); - static LIST_HEAD(hif_list); static DEFINE_SPINLOCK(hif_lock); static u32 hif_idx; @@ -65,10 +62,17 @@ static void mt7915_put_hif2(struct mt7915_hif *hif) static struct mt7915_hif *mt7915_pci_init_hif2(struct pci_dev *pdev) { + struct pci_dev *tmp_pdev; + hif_idx++; - if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7916, NULL) && - !pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x790a, NULL)) - return NULL; + + tmp_pdev = pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7916, NULL); + if (!tmp_pdev) { + tmp_pdev = pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x790a, NULL); + if (!tmp_pdev) + return NULL; + } + pci_dev_put(tmp_pdev); writel(hif_idx | MT_PCIE_RECOG_ID_SEM, pcim_iomap_table(pdev)[0] + MT_PCIE_RECOG_ID); @@ -95,94 +99,6 @@ static int mt7915_pci_hif2_probe(struct pci_dev *pdev) return 0; } -#ifdef CONFIG_NET_MEDIATEK_SOC_WED -static int mt7915_wed_offload_enable(struct mtk_wed_device *wed) -{ - struct mt7915_dev *dev; - struct mt7915_phy *phy; - int ret; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - - spin_lock_bh(&dev->mt76.token_lock); - dev->mt76.token_size = wed->wlan.token_start; - spin_unlock_bh(&dev->mt76.token_lock); - - ret = wait_event_timeout(dev->mt76.tx_wait, - !dev->mt76.wed_token_count, HZ); - if (!ret) - return -EAGAIN; - - phy = &dev->phy; - mt76_set(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H); - - phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; - if (phy) - mt76_set(dev, MT_AGG_ACR4(phy->band_idx), - MT_AGG_ACR_PPDU_TXS2H); - - return 0; -} - -static void mt7915_wed_offload_disable(struct mtk_wed_device *wed) -{ - struct mt7915_dev *dev; - struct mt7915_phy *phy; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - - spin_lock_bh(&dev->mt76.token_lock); - dev->mt76.token_size = MT7915_TOKEN_SIZE; - spin_unlock_bh(&dev->mt76.token_lock); - - /* MT_TXD5_TX_STATUS_HOST (MPDU format) has higher priority than - * MT_AGG_ACR_PPDU_TXS2H (PPDU format) even though ACR bit is set. - */ - phy = &dev->phy; - mt76_clear(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H); - - phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; - if (phy) - mt76_clear(dev, MT_AGG_ACR4(phy->band_idx), - MT_AGG_ACR_PPDU_TXS2H); -} -#endif - -static int -mt7915_pci_wed_init(struct mt7915_dev *dev, struct pci_dev *pdev, int *irq) -{ -#ifdef CONFIG_NET_MEDIATEK_SOC_WED - struct mtk_wed_device *wed = &dev->mt76.mmio.wed; - int ret; - - if (!wed_enable) - return 0; - - wed->wlan.pci_dev = pdev; - wed->wlan.wpdma_phys = pci_resource_start(pdev, 0) + - MT_WFDMA_EXT_CSR_BASE; - wed->wlan.nbuf = 4096; - wed->wlan.token_start = MT7915_TOKEN_SIZE - wed->wlan.nbuf; - wed->wlan.init_buf = mt7915_wed_init_buf; - wed->wlan.offload_enable = mt7915_wed_offload_enable; - wed->wlan.offload_disable = mt7915_wed_offload_disable; - - if (mtk_wed_device_attach(wed) != 0) - return 0; - - *irq = wed->irq; - dev->mt76.dma_dev = wed->dev; - - ret = dma_set_mask(wed->dev, DMA_BIT_MASK(32)); - if (ret) - return ret; - - return 1; -#else - return 0; -#endif -} - static int mt7915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { @@ -220,7 +136,7 @@ static int mt7915_pci_probe(struct pci_dev *pdev, mt7915_wfsys_reset(dev); hif2 = mt7915_pci_init_hif2(pdev); - ret = mt7915_pci_wed_init(dev, pdev, &irq); + ret = mt7915_mmio_wed_init(dev, pdev, true, &irq); if (ret < 0) goto free_wed_or_irq_vector; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h index 5920e705835a..aca1b2f1e9e3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h @@ -24,8 +24,26 @@ enum reg_rev { WFDMA_EXT_CSR_ADDR, CBTOP1_PHY_END, INFRA_MCU_ADDR_END, - FW_EXCEPTION_ADDR, + FW_ASSERT_STAT_ADDR, + FW_EXCEPT_TYPE_ADDR, + FW_EXCEPT_COUNT_ADDR, + FW_CIRQ_COUNT_ADDR, + FW_CIRQ_IDX_ADDR, + FW_CIRQ_LISR_ADDR, + FW_TASK_ID_ADDR, + FW_TASK_IDX_ADDR, + FW_TASK_QID1_ADDR, + FW_TASK_QID2_ADDR, + FW_TASK_START_ADDR, + FW_TASK_END_ADDR, + FW_TASK_SIZE_ADDR, + FW_LAST_MSG_ID_ADDR, + FW_EINT_INFO_ADDR, + FW_SCHED_INFO_ADDR, SWDEF_BASE_ADDR, + TXQ_WED_RING_BASE, + RXQ_WED_RING_BASE, + RXQ_WED_DATA_RING_BASE, __MT_REG_MAX, }; @@ -224,6 +242,14 @@ enum offs_rev { #define MT_DMA_DCR0_MAX_RX_LEN GENMASK(15, 3) #define MT_DMA_DCR0_RXD_G5_EN BIT(23) +/* WTBLOFF TOP: band 0(0x820e9000),band 1(0x820f9000) */ +#define MT_WTBLOFF_TOP_BASE(_band) ((_band) ? 0x820f9000 : 0x820e9000) +#define MT_WTBLOFF_TOP(_band, ofs) (MT_WTBLOFF_TOP_BASE(_band) + (ofs)) + +#define MT_WTBLOFF_TOP_RSCR(_band) MT_WTBLOFF_TOP(_band, 0x008) +#define MT_WTBLOFF_TOP_RSCR_RCPI_MODE GENMASK(31, 30) +#define MT_WTBLOFF_TOP_RSCR_RCPI_PARAM GENMASK(25, 24) + /* ETBF: band 0(0x820ea000), band 1(0x820fa000) */ #define MT_WF_ETBF_BASE(_band) ((_band) ? 0x820fa000 : 0x820ea000) #define MT_WF_ETBF(_band, ofs) (MT_WF_ETBF_BASE(_band) + (ofs)) @@ -523,8 +549,22 @@ enum offs_rev { #define MT_WF_RFCR1_DROP_CFEND BIT(7) #define MT_WF_RFCR1_DROP_CFACK BIT(8) +#define MT_WF_RMAC_RSVD0(_band) MT_WF_RMAC(_band, 0x02e0) +#define MT_WF_RMAC_RSVD0_EIFS_CLR BIT(21) + #define MT_WF_RMAC_MIB_AIRTIME0(_band) MT_WF_RMAC(_band, 0x0380) #define MT_WF_RMAC_MIB_RXTIME_CLR BIT(31) +#define MT_WF_RMAC_MIB_OBSS_BACKOFF GENMASK(15, 0) +#define MT_WF_RMAC_MIB_ED_OFFSET GENMASK(20, 16) + +#define MT_WF_RMAC_MIB_AIRTIME1(_band) MT_WF_RMAC(_band, 0x0384) +#define MT_WF_RMAC_MIB_NONQOSD_BACKOFF GENMASK(31, 16) + +#define MT_WF_RMAC_MIB_AIRTIME3(_band) MT_WF_RMAC(_band, 0x038c) +#define MT_WF_RMAC_MIB_QOS01_BACKOFF GENMASK(31, 0) + +#define MT_WF_RMAC_MIB_AIRTIME4(_band) MT_WF_RMAC(_band, 0x0390) +#define MT_WF_RMAC_MIB_QOS23_BACKOFF GENMASK(31, 0) /* WFDMA0 */ #define MT_WFDMA0_BASE __REG(WFDMA0_ADDR) @@ -539,6 +579,8 @@ enum offs_rev { #define MT_WFDMA0_BUSY_ENA_TX_FIFO1 BIT(1) #define MT_WFDMA0_BUSY_ENA_RX_FIFO BIT(2) +#define MT_WFDMA0_MCU_HOST_INT_ENA MT_WFDMA0(0x1f4) + #define MT_WFDMA0_GLO_CFG MT_WFDMA0(0x208) #define MT_WFDMA0_GLO_CFG_TX_DMA_EN BIT(0) #define MT_WFDMA0_GLO_CFG_RX_DMA_EN BIT(2) @@ -547,9 +589,14 @@ enum offs_rev { #define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2 BIT(21) #define MT_WFDMA0_RST_DTX_PTR MT_WFDMA0(0x20c) + +#define MT_WFDMA0_EXT0_CFG MT_WFDMA0(0x2b0) +#define MT_WFDMA0_EXT0_RXWB_KEEP BIT(10) + #define MT_WFDMA0_PRI_DLY_INT_CFG0 MT_WFDMA0(0x2f0) #define MT_WFDMA0_PRI_DLY_INT_CFG1 MT_WFDMA0(0x2f4) #define MT_WFDMA0_PRI_DLY_INT_CFG2 MT_WFDMA0(0x2f8) +#define MT_WPDMA_GLO_CFG MT_WFDMA0(0x208) /* WFDMA1 */ #define MT_WFDMA1_BASE 0xd5000 @@ -596,6 +643,7 @@ enum offs_rev { #define MT_PCIE_RECOG_ID_MASK GENMASK(30, 0) #define MT_PCIE_RECOG_ID_SEM BIT(31) +#define MT_INT_WED_SOURCE_CSR MT_WFDMA_EXT_CSR(0x200) #define MT_INT_WED_MASK_CSR MT_WFDMA_EXT_CSR(0x204) #define MT_WED_TX_RING_BASE MT_WFDMA_EXT_CSR(0x300) @@ -642,6 +690,10 @@ enum offs_rev { #define MT_TXQ_EXT_CTRL(q) (MT_Q_BASE(__TXQ(q)) + 0x600 + \ MT_TXQ_ID(q)* 0x4) +#define MT_TXQ_WED_RING_BASE __REG(TXQ_WED_RING_BASE) +#define MT_RXQ_WED_RING_BASE __REG(RXQ_WED_RING_BASE) +#define MT_RXQ_WED_DATA_RING_BASE __REG(RXQ_WED_DATA_RING_BASE) + #define MT_INT_SOURCE_CSR __REG(INT_SOURCE_CSR) #define MT_INT_MASK_CSR __REG(INT_MASK_CSR) @@ -660,6 +712,11 @@ enum offs_rev { #define MT_INT_RX_DONE_WA_MAIN_MT7916 BIT(2) #define MT_INT_RX_DONE_WA_EXT_MT7916 BIT(3) +#define MT_INT_WED_RX_DONE_BAND0_MT7916 BIT(18) +#define MT_INT_WED_RX_DONE_BAND1_MT7916 BIT(19) +#define MT_INT_WED_RX_DONE_WA_MAIN_MT7916 BIT(1) +#define MT_INT_WED_RX_DONE_WA_MT7916 BIT(17) + #define MT_INT_RX(q) (dev->q_int_mask[__RXQ(q)]) #define MT_INT_TX_MCU(q) (dev->q_int_mask[(q)]) @@ -683,6 +740,8 @@ enum offs_rev { #define MT_INT_TX_DONE_BAND0 BIT(30) #define MT_INT_TX_DONE_BAND1 BIT(31) #define MT_INT_TX_DONE_MCU_WA_MT7916 BIT(25) +#define MT_INT_WED_TX_DONE_BAND0 BIT(4) +#define MT_INT_WED_TX_DONE_BAND1 BIT(5) #define MT_INT_TX_DONE_MCU (MT_INT_TX_MCU(MT_MCUQ_WA) | \ MT_INT_TX_MCU(MT_MCUQ_WM) | \ @@ -696,6 +755,10 @@ enum offs_rev { #define MT_MCU_CMD_NORMAL_STATE BIT(5) #define MT_MCU_CMD_ERROR_MASK GENMASK(5, 1) +#define MT_MCU_CMD_WA_WDT BIT(31) +#define MT_MCU_CMD_WM_WDT BIT(30) +#define MT_MCU_CMD_WDT_MASK GENMASK(31, 30) + /* TOP RGU */ #define MT_TOP_RGU_BASE 0x18000000 #define MT_TOP_PWR_CTRL (MT_TOP_RGU_BASE + (0x0)) @@ -938,7 +1001,22 @@ enum offs_rev { #define MT_ADIE_TYPE_MASK BIT(1) /* FW MODE SYNC */ -#define MT_FW_EXCEPTION __REG(FW_EXCEPTION_ADDR) +#define MT_FW_ASSERT_STAT __REG(FW_ASSERT_STAT_ADDR) +#define MT_FW_EXCEPT_TYPE __REG(FW_EXCEPT_TYPE_ADDR) +#define MT_FW_EXCEPT_COUNT __REG(FW_EXCEPT_COUNT_ADDR) +#define MT_FW_CIRQ_COUNT __REG(FW_CIRQ_COUNT_ADDR) +#define MT_FW_CIRQ_IDX __REG(FW_CIRQ_IDX_ADDR) +#define MT_FW_CIRQ_LISR __REG(FW_CIRQ_LISR_ADDR) +#define MT_FW_TASK_ID __REG(FW_TASK_ID_ADDR) +#define MT_FW_TASK_IDX __REG(FW_TASK_IDX_ADDR) +#define MT_FW_TASK_QID1 __REG(FW_TASK_QID1_ADDR) +#define MT_FW_TASK_QID2 __REG(FW_TASK_QID2_ADDR) +#define MT_FW_TASK_START __REG(FW_TASK_START_ADDR) +#define MT_FW_TASK_END __REG(FW_TASK_END_ADDR) +#define MT_FW_TASK_SIZE __REG(FW_TASK_SIZE_ADDR) +#define MT_FW_LAST_MSG_ID __REG(FW_LAST_MSG_ID_ADDR) +#define MT_FW_EINT_INFO __REG(FW_EINT_INFO_ADDR) +#define MT_FW_SCHED_INFO __REG(FW_SCHED_INFO_ADDR) #define MT_SWDEF_BASE __REG(SWDEF_BASE_ADDR) @@ -1108,9 +1186,15 @@ enum offs_rev { #define MT_WF_PHY_RXTD12_IRPI_SW_CLR_ONLY BIT(18) #define MT_WF_PHY_RXTD12_IRPI_SW_CLR BIT(29) +#define MT_WF_PHY_TPC_CTRL_STAT(_phy) MT_WF_PHY(0xe7a0 + ((_phy) << 16)) +#define MT_WF_PHY_TPC_CTRL_STAT_MT7916(_phy) MT_WF_PHY(0xe7a0 + ((_phy) << 20)) +#define MT_WF_PHY_TPC_POWER GENMASK(15, 8) + #define MT_MCU_WM_CIRQ_BASE 0x89010000 #define MT_MCU_WM_CIRQ(ofs) (MT_MCU_WM_CIRQ_BASE + (ofs)) #define MT_MCU_WM_CIRQ_IRQ_MASK_CLR_ADDR MT_MCU_WM_CIRQ(0x80) #define MT_MCU_WM_CIRQ_IRQ_SOFT_ADDR MT_MCU_WM_CIRQ(0xc0) +#define MT_MCU_WM_CIRQ_EINT_MASK_CLR_ADDR MT_MCU_WM_CIRQ(0x108) +#define MT_MCU_WM_CIRQ_EINT_SOFT_ADDR MT_MCU_WM_CIRQ(0x118) #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c index c74afa746251..c06c56a0270d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c @@ -1172,10 +1172,6 @@ static int mt7986_wmac_probe(struct platform_device *pdev) chip_id = (uintptr_t)of_device_get_match_data(&pdev->dev); - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - mem_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mem_base)) { dev_err(&pdev->dev, "Failed to get memory resource\n"); @@ -1187,6 +1183,18 @@ static int mt7986_wmac_probe(struct platform_device *pdev) return PTR_ERR(dev); mdev = &dev->mt76; + ret = mt7915_mmio_wed_init(dev, pdev, false, &irq); + if (ret < 0) + goto free_device; + + if (!ret) { + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + ret = irq; + goto free_device; + } + } + ret = devm_request_irq(mdev->dev, irq, mt7915_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); if (ret) @@ -1206,9 +1214,10 @@ static int mt7986_wmac_probe(struct platform_device *pdev) free_irq: devm_free_irq(mdev->dev, irq, dev); - free_device: - mt76_free_device(&dev->mt76); + if (mtk_wed_device_active(&mdev->mmio.wed)) + mtk_wed_device_detach(&mdev->mmio.wed); + mt76_free_device(mdev); return ret; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c index efb9bb8231e2..0d76ae31b376 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c @@ -44,14 +44,14 @@ mt7915_tm_set_tx_power(struct mt7915_phy *phy) int ret; struct { u8 format_id; - u8 dbdc_idx; + u8 band_idx; s8 tx_power; u8 ant_idx; /* Only 0 is valid */ u8 center_chan; u8 rsv[3]; } __packed req = { .format_id = 0xf, - .dbdc_idx = phy != &dev->phy, + .band_idx = phy->mt76->band_idx, .center_chan = ieee80211_frequency_to_channel(freq), }; u8 *tx_power = NULL; @@ -77,7 +77,7 @@ mt7915_tm_set_freq_offset(struct mt7915_phy *phy, bool en, u32 val) struct mt7915_tm_cmd req = { .testmode_en = en, .param_idx = MCU_ATE_SET_FREQ_OFFSET, - .param.freq.band = phy != &dev->phy, + .param.freq.band = phy->mt76->band_idx, .param.freq.freq_offset = cpu_to_le32(val), }; @@ -111,7 +111,7 @@ mt7915_tm_set_trx(struct mt7915_phy *phy, int type, bool en) .param_idx = MCU_ATE_SET_TRX, .param.trx.type = type, .param.trx.enable = en, - .param.trx.band = phy != &dev->phy, + .param.trx.band = phy->mt76->band_idx, }; return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(ATE_CTRL), &req, @@ -126,7 +126,7 @@ mt7915_tm_clean_hwq(struct mt7915_phy *phy, u8 wcid) .testmode_en = 1, .param_idx = MCU_ATE_CLEAN_TXQUEUE, .param.clean.wcid = wcid, - .param.clean.band = phy != &dev->phy, + .param.clean.band = phy->mt76->band_idx, }; return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(ATE_CTRL), &req, @@ -144,7 +144,7 @@ mt7915_tm_set_slot_time(struct mt7915_phy *phy, u8 slot_time, u8 sifs) .param.slot.sifs = sifs, .param.slot.rifs = 2, .param.slot.eifs = cpu_to_le16(60), - .param.slot.band = phy != &dev->phy, + .param.slot.band = phy->mt76->band_idx, }; return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(ATE_CTRL), &req, @@ -198,6 +198,7 @@ mt7915_tm_set_ipg_params(struct mt7915_phy *phy, u32 ipg, u8 mode) u8 sig_ext = (mode == MT76_TM_TX_MODE_CCK) ? 0 : 6; u8 slot_time = 9, sifs = TM_DEFAULT_SIFS; u8 aifsn = TM_MIN_AIFSN; + u8 band = phy->mt76->band_idx; u32 i2t_time, tr2t_time, txv_time; u16 cw = 0; @@ -232,14 +233,14 @@ mt7915_tm_set_ipg_params(struct mt7915_phy *phy, u32 ipg, u8 mode) sifs = min_t(u32, ipg, TM_MAX_SIFS); } done: - txv_time = mt76_get_field(dev, MT_TMAC_ATCR(phy->band_idx), + txv_time = mt76_get_field(dev, MT_TMAC_ATCR(band), MT_TMAC_ATCR_TXV_TOUT); txv_time *= 50; /* normal clock time */ i2t_time = (slot_time * 1000 - txv_time - BBP_PROC_TIME) / 50; tr2t_time = (sifs * 1000 - txv_time - BBP_PROC_TIME) / 50; - mt76_set(dev, MT_TMAC_TRCR0(phy->band_idx), + mt76_set(dev, MT_TMAC_TRCR0(band), FIELD_PREP(MT_TMAC_TRCR0_TR2T_CHK, tr2t_time) | FIELD_PREP(MT_TMAC_TRCR0_I2T_CHK, i2t_time)); @@ -336,6 +337,7 @@ mt7915_tm_reg_backup_restore(struct mt7915_phy *phy) int n_regs = ARRAY_SIZE(reg_backup_list); struct mt7915_dev *dev = phy->dev; u32 *b = phy->test.reg_backup; + u8 band = phy->mt76->band_idx; int i; REG_BAND_IDX(reg_backup_list[0], AGG_PCR0, 0); @@ -358,7 +360,7 @@ mt7915_tm_reg_backup_restore(struct mt7915_phy *phy) if (phy->mt76->test.state == MT76_TM_STATE_OFF) { for (i = 0; i < n_regs; i++) - mt76_wr(dev, reg_backup_list[i].band[phy->band_idx], b[i]); + mt76_wr(dev, reg_backup_list[i].band[band], b[i]); return; } @@ -369,33 +371,33 @@ mt7915_tm_reg_backup_restore(struct mt7915_phy *phy) phy->test.reg_backup = b; for (i = 0; i < n_regs; i++) - b[i] = mt76_rr(dev, reg_backup_list[i].band[phy->band_idx]); + b[i] = mt76_rr(dev, reg_backup_list[i].band[band]); } - mt76_clear(dev, MT_AGG_PCR0(phy->band_idx, 0), MT_AGG_PCR0_MM_PROT | + mt76_clear(dev, MT_AGG_PCR0(band, 0), MT_AGG_PCR0_MM_PROT | MT_AGG_PCR0_GF_PROT | MT_AGG_PCR0_ERP_PROT | MT_AGG_PCR0_VHT_PROT | MT_AGG_PCR0_BW20_PROT | MT_AGG_PCR0_BW40_PROT | MT_AGG_PCR0_BW80_PROT); - mt76_set(dev, MT_AGG_PCR0(phy->band_idx, 0), MT_AGG_PCR0_PTA_WIN_DIS); + mt76_set(dev, MT_AGG_PCR0(band, 0), MT_AGG_PCR0_PTA_WIN_DIS); - mt76_wr(dev, MT_AGG_PCR0(phy->band_idx, 1), MT_AGG_PCR1_RTS0_NUM_THRES | + mt76_wr(dev, MT_AGG_PCR0(band, 1), MT_AGG_PCR1_RTS0_NUM_THRES | MT_AGG_PCR1_RTS0_LEN_THRES); - mt76_clear(dev, MT_AGG_MRCR(phy->band_idx), MT_AGG_MRCR_BAR_CNT_LIMIT | + mt76_clear(dev, MT_AGG_MRCR(band), MT_AGG_MRCR_BAR_CNT_LIMIT | MT_AGG_MRCR_LAST_RTS_CTS_RN | MT_AGG_MRCR_RTS_FAIL_LIMIT | MT_AGG_MRCR_TXCMD_RTS_FAIL_LIMIT); - mt76_rmw(dev, MT_AGG_MRCR(phy->band_idx), MT_AGG_MRCR_RTS_FAIL_LIMIT | + mt76_rmw(dev, MT_AGG_MRCR(band), MT_AGG_MRCR_RTS_FAIL_LIMIT | MT_AGG_MRCR_TXCMD_RTS_FAIL_LIMIT, FIELD_PREP(MT_AGG_MRCR_RTS_FAIL_LIMIT, 1) | FIELD_PREP(MT_AGG_MRCR_TXCMD_RTS_FAIL_LIMIT, 1)); - mt76_wr(dev, MT_TMAC_TFCR0(phy->band_idx), 0); - mt76_clear(dev, MT_TMAC_TCR0(phy->band_idx), MT_TMAC_TCR0_TBTT_STOP_CTRL); + mt76_wr(dev, MT_TMAC_TFCR0(band), 0); + mt76_clear(dev, MT_TMAC_TCR0(band), MT_TMAC_TCR0_TBTT_STOP_CTRL); /* config rx filter for testmode rx */ - mt76_wr(dev, MT_WF_RFCR(phy->band_idx), 0xcf70a); - mt76_wr(dev, MT_WF_RFCR1(phy->band_idx), 0); + mt76_wr(dev, MT_WF_RFCR(band), 0xcf70a); + mt76_wr(dev, MT_WF_RFCR1(band), 0); } static void @@ -432,8 +434,6 @@ mt7915_tm_update_channel(struct mt7915_phy *phy) static void mt7915_tm_set_tx_frames(struct mt7915_phy *phy, bool en) { - static const u8 spe_idx_map[] = {0, 0, 1, 0, 3, 2, 4, 0, - 9, 8, 6, 10, 16, 12, 18, 0}; struct mt76_testmode_data *td = &phy->mt76->test; struct mt7915_dev *dev = phy->dev; struct ieee80211_tx_info *info; @@ -447,15 +447,10 @@ mt7915_tm_set_tx_frames(struct mt7915_phy *phy, bool en) if (en) { mt7915_tm_update_channel(phy); - if (td->tx_spe_idx) { + if (td->tx_spe_idx) phy->test.spe_idx = td->tx_spe_idx; - } else { - u8 tx_ant = td->tx_antenna_mask; - - if (phy != &dev->phy) - tx_ant >>= dev->chainshift; - phy->test.spe_idx = spe_idx_map[tx_ant]; - } + else + phy->test.spe_idx = mt76_connac_spe_idx(td->tx_antenna_mask); } mt7915_tm_set_tam_arb(phy, en, @@ -495,7 +490,7 @@ mt7915_tm_set_rx_frames(struct mt7915_phy *phy, bool en) mt7915_tm_update_channel(phy); /* read-clear */ - mt76_rr(dev, MT_MIB_SDR3(phy != &dev->phy)); + mt76_rr(dev, MT_MIB_SDR3(phy->mt76->band_idx)); mt7915_tm_set_trx(phy, TM_MAC_RX_RXV, en); } } @@ -522,6 +517,7 @@ mt7915_tm_set_tx_cont(struct mt7915_phy *phy, bool en) struct mt76_testmode_data *td = &phy->mt76->test; u32 func_idx = en ? TX_CONT_START : TX_CONT_STOP; u8 rate_idx = td->tx_rate_idx, mode; + u8 band = phy->mt76->band_idx; u16 rateval; struct mt7915_tm_rf_test req = { .action = 1, @@ -533,7 +529,7 @@ mt7915_tm_set_tx_cont(struct mt7915_phy *phy, bool en) tx_cont->control_ch = chandef->chan->hw_value; tx_cont->center_ch = freq1; tx_cont->tx_ant = td->tx_antenna_mask; - tx_cont->band = phy != &dev->phy; + tx_cont->band = band; switch (chandef->width) { case NL80211_CHAN_WIDTH_40: @@ -565,7 +561,7 @@ mt7915_tm_set_tx_cont(struct mt7915_phy *phy, bool en) } if (!en) { - req.op.rf.param.func_data = cpu_to_le32(phy != &dev->phy); + req.op.rf.param.func_data = cpu_to_le32(band); goto out; } @@ -696,7 +692,9 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, { struct mt76_testmode_data *td = &mphy->test; struct mt7915_phy *phy = mphy->priv; - u32 changed = 0; + struct mt7915_dev *dev = phy->dev; + u32 chainmask = mphy->chainmask, changed = 0; + bool ext_phy = phy != &dev->phy; int i; BUILD_BUG_ON(NUM_TM_CHANGED >= 32); @@ -705,7 +703,8 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, td->state == MT76_TM_STATE_OFF) return 0; - if (td->tx_antenna_mask & ~mphy->chainmask) + chainmask = ext_phy ? chainmask >> dev->chainshift : chainmask; + if (td->tx_antenna_mask > chainmask) return -EINVAL; for (i = 0; i < ARRAY_SIZE(tm_change_map); i++) { @@ -771,11 +770,11 @@ mt7915_tm_dump_stats(struct mt76_phy *mphy, struct sk_buff *msg) nla_nest_end(msg, rx); - cnt = mt76_rr(dev, MT_MIB_SDR3(phy->band_idx)); + cnt = mt76_rr(dev, MT_MIB_SDR3(phy->mt76->band_idx)); fcs_err = is_mt7915(&dev->mt76) ? FIELD_GET(MT_MIB_SDR3_FCS_ERR_MASK, cnt) : FIELD_GET(MT_MIB_SDR3_FCS_ERR_MASK_MT7916, cnt); - q = phy->band_idx ? MT_RXQ_BAND1 : MT_RXQ_MAIN; + q = phy->mt76->band_idx ? MT_RXQ_BAND1 : MT_RXQ_MAIN; mphy->test.rx_stats.packets[q] += fcs_err; mphy->test.rx_stats.fcs_error[q] += fcs_err; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c index bce76417f95d..29d8883268f6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c @@ -85,7 +85,7 @@ mt7921_ampdu_stat_read_phy(struct mt7921_phy *phy, seq_puts(file, "\nCount: "); for (i = 0; i < ARRAY_SIZE(bound); i++) - seq_printf(file, "%8d | ", dev->mt76.aggr_stats[i]); + seq_printf(file, "%8d | ", phy->mt76->aggr_stats[i]); seq_puts(file, "\n"); seq_printf(file, "BA miss count: %d\n", phy->mib.ba_miss_cnt); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c index dcdb3cf04ac1..542dfd425129 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c @@ -2,6 +2,7 @@ /* Copyright (C) 2020 MediaTek Inc. */ #include <linux/etherdevice.h> +#include <linux/firmware.h> #include "mt7921.h" #include "mac.h" #include "mcu.h" @@ -25,6 +26,27 @@ static const struct ieee80211_iface_combination if_comb[] = { .max_interfaces = MT7921_MAX_INTERFACES, .num_different_channels = 1, .beacon_int_infra_match = true, + }, +}; + +static const struct ieee80211_iface_limit if_limits_chanctx[] = { + { + .max = 2, + .types = BIT(NL80211_IFTYPE_STATION), + }, + { + .max = 1, + .types = BIT(NL80211_IFTYPE_AP), + } +}; + +static const struct ieee80211_iface_combination if_comb_chanctx[] = { + { + .limits = if_limits_chanctx, + .n_limits = ARRAY_SIZE(if_limits_chanctx), + .max_interfaces = 2, + .num_different_channels = 2, + .beacon_int_infra_match = false, } }; @@ -37,6 +59,7 @@ mt7921_regd_notifier(struct wiphy *wiphy, memcpy(dev->mt76.alpha2, request->alpha2, sizeof(dev->mt76.alpha2)); dev->mt76.region = request->dfs_region; + dev->country_ie_env = request->country_ie_env; mt7921_mutex_acquire(dev); mt7921_mcu_set_clc(dev, request->alpha2, request->country_ie_env); @@ -65,12 +88,20 @@ mt7921_init_wiphy(struct ieee80211_hw *hw) hw->sta_data_size = sizeof(struct mt7921_sta); hw->vif_data_size = sizeof(struct mt7921_vif); - wiphy->iface_combinations = if_comb; + if (dev->fw_features & MT7921_FW_CAP_CNM) { + wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; + wiphy->iface_combinations = if_comb_chanctx; + wiphy->n_iface_combinations = ARRAY_SIZE(if_comb_chanctx); + } else { + wiphy->flags &= ~WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; + wiphy->iface_combinations = if_comb; + wiphy->n_iface_combinations = ARRAY_SIZE(if_comb); + } wiphy->flags &= ~(WIPHY_FLAG_IBSS_RSN | WIPHY_FLAG_4ADDR_AP | WIPHY_FLAG_4ADDR_STATION); wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) | BIT(NL80211_IFTYPE_AP); - wiphy->n_iface_combinations = ARRAY_SIZE(if_comb); + wiphy->max_remain_on_channel_duration = 5000; wiphy->max_scan_ie_len = MT76_CONNAC_SCAN_IE_LEN; wiphy->max_scan_ssids = 4; wiphy->max_sched_scan_plan_interval = @@ -129,6 +160,58 @@ mt7921_mac_init_band(struct mt7921_dev *dev, u8 band) mt76_clear(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN); } +u8 mt7921_check_offload_capability(struct device *dev, const char *fw_wm) +{ + struct mt7921_fw_features *features = NULL; + const struct mt76_connac2_fw_trailer *hdr; + struct mt7921_realease_info *rel_info; + const struct firmware *fw; + int ret, i, offset = 0; + const u8 *data, *end; + + ret = request_firmware(&fw, fw_wm, dev); + if (ret) + return ret; + + if (!fw || !fw->data || fw->size < sizeof(*hdr)) { + dev_err(dev, "Invalid firmware\n"); + return -EINVAL; + } + + data = fw->data; + hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); + + for (i = 0; i < hdr->n_region; i++) { + const struct mt76_connac2_fw_region *region; + + region = (const void *)((const u8 *)hdr - + (hdr->n_region - i) * sizeof(*region)); + offset += le32_to_cpu(region->len); + } + + data += offset + 16; + rel_info = (struct mt7921_realease_info *)data; + data += sizeof(*rel_info); + end = data + le16_to_cpu(rel_info->len); + + while (data < end) { + rel_info = (struct mt7921_realease_info *)data; + data += sizeof(*rel_info); + + if (rel_info->tag == MT7921_FW_TAG_FEATURE) { + features = (struct mt7921_fw_features *)data; + break; + } + + data += le16_to_cpu(rel_info->len) + rel_info->pad_len; + } + + release_firmware(fw); + + return features ? features->data : 0; +} +EXPORT_SYMBOL_GPL(mt7921_check_offload_capability); + int mt7921_mac_init(struct mt7921_dev *dev) { int i; @@ -278,6 +361,10 @@ int mt7921_register_device(struct mt7921_dev *dev) INIT_WORK(&dev->reset_work, mt7921_mac_reset_work); INIT_WORK(&dev->init_work, mt7921_init_work); + INIT_WORK(&dev->phy.roc_work, mt7921_roc_work); + timer_setup(&dev->phy.roc_timer, mt7921_roc_timer, 0); + init_waitqueue_head(&dev->phy.roc_wait); + dev->pm.idle_timeout = MT7921_PM_TIMEOUT; dev->pm.stats.last_wake_event = jiffies; dev->pm.stats.last_doze_event = jiffies; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index 650ab97ae052..82db3762be33 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -168,14 +168,6 @@ static void mt7921_get_status_freq_info(struct mt7921_dev *dev, struct mt76_phy *mphy, struct mt76_rx_status *status, u8 chfreq) { - if (!test_bit(MT76_HW_SCANNING, &mphy->state) && - !test_bit(MT76_HW_SCHED_SCANNING, &mphy->state) && - !test_bit(MT76_STATE_ROC, &mphy->state)) { - status->freq = mphy->chandef.chan->center_freq; - status->band = mphy->chandef.chan->band; - return; - } - if (chfreq > 180) { status->band = NL80211_BAND_6GHZ; chfreq = (chfreq - 181) * 4 + 1; @@ -396,6 +388,27 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) if (v0 & MT_PRXV_HT_AD_CODE) status->enc_flags |= RX_ENC_FLAG_LDPC; + ret = mt76_connac2_mac_fill_rx_rate(&dev->mt76, status, sband, + rxv, &mode); + if (ret < 0) + return ret; + + if (rxd1 & MT_RXD1_NORMAL_GROUP_5) { + rxd += 6; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + + rxv = rxd; + /* Monitor mode would use RCPI described in GROUP 5 + * instead. + */ + v1 = le32_to_cpu(rxv[0]); + + rxd += 12; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + } + status->chains = mphy->antenna_mask; status->chain_signal[0] = to_rssi(MT_PRXV_RCPI0, v1); status->chain_signal[1] = to_rssi(MT_PRXV_RCPI1, v1); @@ -410,17 +423,6 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) status->signal = max(status->signal, status->chain_signal[i]); } - - ret = mt76_connac2_mac_fill_rx_rate(&dev->mt76, status, sband, - rxv, &mode); - if (ret < 0) - return ret; - - if (rxd1 & MT_RXD1_NORMAL_GROUP_5) { - rxd += 18; - if ((u8 *)rxd - skb->data >= skb->len) - return -EINVAL; - } } amsdu_info = FIELD_GET(MT_RXD4_NORMAL_PAYLOAD_FORMAT, rxd4); @@ -682,7 +684,7 @@ bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len) EXPORT_SYMBOL_GPL(mt7921_rx_check); void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) + struct sk_buff *skb, u32 *info) { struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); __le32 *rxd = (__le32 *)skb->data; @@ -735,7 +737,7 @@ void mt7921_mac_reset_counters(struct mt7921_phy *phy) } dev->mt76.phy.survey_time = ktime_get_boottime(); - memset(&dev->mt76.aggr_stats[0], 0, sizeof(dev->mt76.aggr_stats) / 2); + memset(phy->mt76->aggr_stats, 0, sizeof(phy->mt76->aggr_stats)); /* reset airtime counters */ mt76_rr(dev, MT_MIB_SDR9(0)); @@ -856,7 +858,7 @@ mt7921_vif_connect_iter(void *priv, u8 *mac, if (vif->type == NL80211_IFTYPE_AP) { mt76_connac_mcu_uni_add_bss(dev->phy.mt76, vif, &mvif->sta.wcid, - true); + true, NULL); mt7921_mcu_sta_update(dev, NULL, vif, true, MT76_STA_INFO_STATE_NONE); mt7921_mcu_uni_add_beacon_offload(dev, hw, vif, true); @@ -974,16 +976,16 @@ void mt7921_mac_update_mib_stats(struct mt7921_phy *phy) mib->tx_amsdu_cnt += val; } - for (i = 0, aggr1 = aggr0 + 4; i < 4; i++) { + for (i = 0, aggr1 = aggr0 + 8; i < 4; i++) { u32 val2; val = mt76_rr(dev, MT_TX_AGG_CNT(0, i)); val2 = mt76_rr(dev, MT_TX_AGG_CNT2(0, i)); - dev->mt76.aggr_stats[aggr0++] += val & 0xffff; - dev->mt76.aggr_stats[aggr0++] += val >> 16; - dev->mt76.aggr_stats[aggr1++] += val2 & 0xffff; - dev->mt76.aggr_stats[aggr1++] += val2 >> 16; + phy->mt76->aggr_stats[aggr0++] += val & 0xffff; + phy->mt76->aggr_stats[aggr0++] += val >> 16; + phy->mt76->aggr_stats[aggr1++] += val2 & 0xffff; + phy->mt76->aggr_stats[aggr1++] += val2 >> 16; } } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 7e409ac7d9a8..76ac5069638f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -386,6 +386,116 @@ static void mt7921_remove_interface(struct ieee80211_hw *hw, mt76_packet_id_flush(&dev->mt76, &msta->wcid); } +static void mt7921_roc_iter(void *priv, u8 *mac, + struct ieee80211_vif *vif) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_phy *phy = priv; + + mt7921_mcu_abort_roc(phy, mvif, phy->roc_token_id); +} + +void mt7921_roc_work(struct work_struct *work) +{ + struct mt7921_phy *phy; + + phy = (struct mt7921_phy *)container_of(work, struct mt7921_phy, + roc_work); + + if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) + return; + + mt7921_mutex_acquire(phy->dev); + ieee80211_iterate_active_interfaces(phy->mt76->hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7921_roc_iter, phy); + mt7921_mutex_release(phy->dev); + ieee80211_remain_on_channel_expired(phy->mt76->hw); +} + +void mt7921_roc_timer(struct timer_list *timer) +{ + struct mt7921_phy *phy = from_timer(phy, timer, roc_timer); + + ieee80211_queue_work(phy->mt76->hw, &phy->roc_work); +} + +static int mt7921_abort_roc(struct mt7921_phy *phy, struct mt7921_vif *vif) +{ + int err; + + if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) + return 0; + + del_timer_sync(&phy->roc_timer); + cancel_work_sync(&phy->roc_work); + err = mt7921_mcu_abort_roc(phy, vif, phy->roc_token_id); + clear_bit(MT76_STATE_ROC, &phy->mt76->state); + + return err; +} + +static int mt7921_set_roc(struct mt7921_phy *phy, + struct mt7921_vif *vif, + struct ieee80211_channel *chan, + int duration, + enum mt7921_roc_req type) +{ + int err; + + if (test_and_set_bit(MT76_STATE_ROC, &phy->mt76->state)) + return -EBUSY; + + phy->roc_grant = false; + + err = mt7921_mcu_set_roc(phy, vif, chan, duration, type, + ++phy->roc_token_id); + if (err < 0) { + clear_bit(MT76_STATE_ROC, &phy->mt76->state); + goto out; + } + + if (!wait_event_timeout(phy->roc_wait, phy->roc_grant, HZ)) { + mt7921_mcu_abort_roc(phy, vif, phy->roc_token_id); + clear_bit(MT76_STATE_ROC, &phy->mt76->state); + err = -ETIMEDOUT; + } + +out: + return err; +} + +static int mt7921_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel *chan, + int duration, + enum ieee80211_roc_type type) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_phy *phy = mt7921_hw_phy(hw); + int err; + + mt7921_mutex_acquire(phy->dev); + err = mt7921_set_roc(phy, mvif, chan, duration, MT7921_ROC_REQ_ROC); + mt7921_mutex_release(phy->dev); + + return err; +} + +static int mt7921_cancel_remain_on_channel(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_phy *phy = mt7921_hw_phy(hw); + int err; + + mt7921_mutex_acquire(phy->dev); + err = mt7921_abort_roc(phy, mvif); + mt7921_mutex_release(phy->dev); + + return err; +} + static int mt7921_set_channel(struct mt7921_phy *phy) { struct mt7921_dev *dev = phy->dev; @@ -748,7 +858,7 @@ void mt7921_mac_sta_assoc(struct mt76_dev *mdev, struct ieee80211_vif *vif, if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) mt76_connac_mcu_uni_add_bss(&dev->mphy, vif, &mvif->sta.wcid, - true); + true, mvif->ctx); mt7921_mac_wtbl_update(dev, msta->wcid.idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); @@ -780,7 +890,8 @@ void mt7921_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, ewma_rssi_init(&mvif->rssi); if (!sta->tdls) mt76_connac_mcu_uni_add_bss(&dev->mphy, vif, - &mvif->sta.wcid, false); + &mvif->sta.wcid, false, + mvif->ctx); } spin_lock_bh(&dev->sta_poll_lock); @@ -1075,7 +1186,7 @@ void mt7921_get_et_stats(struct ieee80211_hw *hw, struct ieee80211_vif *vif, /* Tx ampdu stat */ for (i = 0; i < 15; i++) - data[ei++] = dev->mt76.aggr_stats[i]; + data[ei++] = phy->mt76->aggr_stats[i]; data[ei++] = phy->mib.ba_miss_cnt; @@ -1504,7 +1615,13 @@ static int mt7921_set_sar_specs(struct ieee80211_hw *hw, int err; mt7921_mutex_acquire(dev); + err = mt7921_mcu_set_clc(dev, dev->mt76.alpha2, + dev->country_ie_env); + if (err < 0) + goto out; + err = mt7921_set_tx_sar_pwr(hw, sar); +out: mt7921_mutex_release(dev); return err; @@ -1534,7 +1651,7 @@ mt7921_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mt7921_mutex_acquire(dev); err = mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, - true); + true, mvif->ctx); if (err) goto out; @@ -1565,12 +1682,109 @@ mt7921_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, if (err) goto out; - mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, false); + mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, false, + mvif->ctx); out: mt7921_mutex_release(dev); } +static int +mt7921_add_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) +{ + return 0; +} + +static void +mt7921_remove_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) +{ +} + +static void mt7921_ctx_iter(void *priv, u8 *mac, + struct ieee80211_vif *vif) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct ieee80211_chanctx_conf *ctx = priv; + + if (ctx != mvif->ctx) + return; + + mt76_connac_mcu_uni_set_chctx(mvif->phy->mt76, &mvif->mt76, ctx); +} + +static void +mt7921_change_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx, + u32 changed) +{ + struct mt7921_phy *phy = mt7921_hw_phy(hw); + + mt7921_mutex_acquire(phy->dev); + ieee80211_iterate_active_interfaces(phy->mt76->hw, + IEEE80211_IFACE_ITER_ACTIVE, + mt7921_ctx_iter, ctx); + mt7921_mutex_release(phy->dev); +} + +static int +mt7921_assign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_dev *dev = mt7921_hw_dev(hw); + + mutex_lock(&dev->mt76.mutex); + mvif->ctx = ctx; + mutex_unlock(&dev->mt76.mutex); + + return 0; +} + +static void +mt7921_unassign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_dev *dev = mt7921_hw_dev(hw); + + mutex_lock(&dev->mt76.mutex); + mvif->ctx = NULL; + mutex_unlock(&dev->mt76.mutex); +} + +static void mt7921_mgd_prepare_tx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_prep_tx_info *info) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_dev *dev = mt7921_hw_dev(hw); + u16 duration = info->duration ? info->duration : + jiffies_to_msecs(HZ); + + mt7921_mutex_acquire(dev); + mt7921_set_roc(mvif->phy, mvif, mvif->ctx->def.chan, duration, + MT7921_ROC_REQ_JOIN); + mt7921_mutex_release(dev); +} + +static void mt7921_mgd_complete_tx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_prep_tx_info *info) +{ + struct mt7921_vif *mvif = (struct mt7921_vif *)vif->drv_priv; + struct mt7921_dev *dev = mt7921_hw_dev(hw); + + mt7921_mutex_acquire(dev); + mt7921_abort_roc(mvif->phy, mvif); + mt7921_mutex_release(dev); +} + const struct ieee80211_ops mt7921_ops = { .tx = mt7921_tx, .start = mt7921_start, @@ -1621,6 +1835,15 @@ const struct ieee80211_ops mt7921_ops = { #endif /* CONFIG_PM */ .flush = mt7921_flush, .set_sar_specs = mt7921_set_sar_specs, + .remain_on_channel = mt7921_remain_on_channel, + .cancel_remain_on_channel = mt7921_cancel_remain_on_channel, + .add_chanctx = mt7921_add_chanctx, + .remove_chanctx = mt7921_remove_chanctx, + .change_chanctx = mt7921_change_chanctx, + .assign_vif_chanctx = mt7921_assign_vif_chanctx, + .unassign_vif_chanctx = mt7921_unassign_vif_chanctx, + .mgd_prepare_tx = mt7921_mgd_prepare_tx, + .mgd_complete_tx = mt7921_mgd_complete_tx, }; EXPORT_SYMBOL_GPL(mt7921_ops); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 67bf92969a7b..fb9c0f66cb27 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -155,6 +155,29 @@ void mt7921_mcu_set_suspend_iter(void *priv, u8 *mac, struct ieee80211_vif *vif) #endif /* CONFIG_PM */ static void +mt7921_mcu_uni_roc_event(struct mt7921_dev *dev, struct sk_buff *skb) +{ + struct mt7921_roc_grant_tlv *grant; + struct mt76_connac2_mcu_rxd *rxd; + int duration; + + rxd = (struct mt76_connac2_mcu_rxd *)skb->data; + grant = (struct mt7921_roc_grant_tlv *)(rxd->tlv + 4); + + /* should never happen */ + WARN_ON_ONCE((le16_to_cpu(grant->tag) != UNI_EVENT_ROC_GRANT)); + + if (grant->reqtype == MT7921_ROC_REQ_ROC) + ieee80211_ready_on_channel(dev->mt76.phy.hw); + + dev->phy.roc_grant = true; + wake_up(&dev->phy.roc_wait); + duration = le32_to_cpu(grant->max_interval); + mod_timer(&dev->phy.roc_timer, + round_jiffies_up(jiffies + msecs_to_jiffies(duration))); +} + +static void mt7921_mcu_scan_event(struct mt7921_dev *dev, struct sk_buff *skb) { struct mt76_phy *mphy = &dev->mt76.phy; @@ -200,20 +223,6 @@ mt7921_mcu_connection_loss_event(struct mt7921_dev *dev, struct sk_buff *skb) } static void -mt7921_mcu_bss_event(struct mt7921_dev *dev, struct sk_buff *skb) -{ - struct mt76_phy *mphy = &dev->mt76.phy; - struct mt76_connac_mcu_bss_event *event; - - skb_pull(skb, sizeof(struct mt76_connac2_mcu_rxd)); - event = (struct mt76_connac_mcu_bss_event *)skb->data; - if (event->is_absent) - ieee80211_stop_queues(mphy->hw); - else - ieee80211_wake_queues(mphy->hw); -} - -static void mt7921_mcu_debug_msg_event(struct mt7921_dev *dev, struct sk_buff *skb) { struct mt7921_debug_msg { @@ -279,9 +288,6 @@ mt7921_mcu_rx_unsolicited_event(struct mt7921_dev *dev, struct sk_buff *skb) case MCU_EVENT_SCAN_DONE: mt7921_mcu_scan_event(dev, skb); return; - case MCU_EVENT_BSS_ABSENCE: - mt7921_mcu_bss_event(dev, skb); - break; case MCU_EVENT_DBG_MSG: mt7921_mcu_debug_msg_event(dev, skb); break; @@ -302,6 +308,24 @@ mt7921_mcu_rx_unsolicited_event(struct mt7921_dev *dev, struct sk_buff *skb) dev_kfree_skb(skb); } +static void +mt7921_mcu_uni_rx_unsolicited_event(struct mt7921_dev *dev, + struct sk_buff *skb) +{ + struct mt76_connac2_mcu_rxd *rxd; + + rxd = (struct mt76_connac2_mcu_rxd *)skb->data; + + switch (rxd->eid) { + case MCU_UNI_EVENT_ROC: + mt7921_mcu_uni_roc_event(dev, skb); + break; + default: + break; + } + dev_kfree_skb(skb); +} + void mt7921_mcu_rx_event(struct mt7921_dev *dev, struct sk_buff *skb) { struct mt76_connac2_mcu_rxd *rxd; @@ -311,6 +335,11 @@ void mt7921_mcu_rx_event(struct mt7921_dev *dev, struct sk_buff *skb) rxd = (struct mt76_connac2_mcu_rxd *)skb->data; + if (rxd->option & MCU_UNI_CMD_UNSOLICITED_EVENT) { + mt7921_mcu_uni_rx_unsolicited_event(dev, skb); + return; + } + if (rxd->eid == 0x6) { mt76_mcu_rx_event(&dev->mt76, skb); return; @@ -319,7 +348,6 @@ void mt7921_mcu_rx_event(struct mt7921_dev *dev, struct sk_buff *skb) if (rxd->ext_eid == MCU_EXT_EVENT_RATE_REPORT || rxd->eid == MCU_EVENT_BSS_BEACON_LOSS || rxd->eid == MCU_EVENT_SCHED_SCAN_DONE || - rxd->eid == MCU_EVENT_BSS_ABSENCE || rxd->eid == MCU_EVENT_SCAN_DONE || rxd->eid == MCU_EVENT_TX_DONE || rxd->eid == MCU_EVENT_DBG_MSG || @@ -636,6 +664,103 @@ int mt7921_mcu_set_tx(struct mt7921_dev *dev, struct ieee80211_vif *vif) &req_mu, sizeof(req_mu), false); } +int mt7921_mcu_set_roc(struct mt7921_phy *phy, struct mt7921_vif *vif, + struct ieee80211_channel *chan, int duration, + enum mt7921_roc_req type, u8 token_id) +{ + int center_ch = ieee80211_frequency_to_channel(chan->center_freq); + struct mt7921_dev *dev = phy->dev; + struct { + struct { + u8 rsv[4]; + } __packed hdr; + struct roc_acquire_tlv { + __le16 tag; + __le16 len; + u8 bss_idx; + u8 tokenid; + u8 control_channel; + u8 sco; + u8 band; + u8 bw; + u8 center_chan; + u8 center_chan2; + u8 bw_from_ap; + u8 center_chan_from_ap; + u8 center_chan2_from_ap; + u8 reqtype; + __le32 maxinterval; + u8 dbdcband; + u8 rsv[3]; + } __packed roc; + } __packed req = { + .roc = { + .tag = cpu_to_le16(UNI_ROC_ACQUIRE), + .len = cpu_to_le16(sizeof(struct roc_acquire_tlv)), + .tokenid = token_id, + .reqtype = type, + .maxinterval = cpu_to_le32(duration), + .bss_idx = vif->mt76.idx, + .control_channel = chan->hw_value, + .bw = CMD_CBW_20MHZ, + .bw_from_ap = CMD_CBW_20MHZ, + .center_chan = center_ch, + .center_chan_from_ap = center_ch, + .dbdcband = 0xff, /* auto */ + }, + }; + + if (chan->hw_value < center_ch) + req.roc.sco = 1; /* SCA */ + else if (chan->hw_value > center_ch) + req.roc.sco = 3; /* SCB */ + + switch (chan->band) { + case NL80211_BAND_6GHZ: + req.roc.band = 3; + break; + case NL80211_BAND_5GHZ: + req.roc.band = 2; + break; + default: + req.roc.band = 1; + break; + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(ROC), + &req, sizeof(req), false); +} + +int mt7921_mcu_abort_roc(struct mt7921_phy *phy, struct mt7921_vif *vif, + u8 token_id) +{ + struct mt7921_dev *dev = phy->dev; + struct { + struct { + u8 rsv[4]; + } __packed hdr; + struct roc_abort_tlv { + __le16 tag; + __le16 len; + u8 bss_idx; + u8 tokenid; + u8 dbdcband; + u8 rsv[5]; + } __packed abort; + } __packed req = { + .abort = { + .tag = cpu_to_le16(UNI_ROC_ABORT), + .len = cpu_to_le16(sizeof(struct roc_abort_tlv)), + .tokenid = token_id, + .bss_idx = vif->mt76.idx, + .dbdcband = 0xff, /* auto*/ + }, + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(ROC), + &req, sizeof(req), false); +} + int mt7921_mcu_set_chan_info(struct mt7921_phy *phy, int cmd) { struct mt7921_dev *dev = phy->dev; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h index eaba114a9c7e..15d6b7fe1c6c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h @@ -32,6 +32,9 @@ #define MT7921_MCU_INIT_RETRY_COUNT 10 #define MT7921_WFSYS_INIT_RETRY_COUNT 2 +#define MT7921_FW_TAG_FEATURE 4 +#define MT7921_FW_CAP_CNM BIT(7) + #define MT7921_FIRMWARE_WM "mediatek/WIFI_RAM_CODE_MT7961_1.bin" #define MT7921_ROM_PATCH "mediatek/WIFI_MT7961_patch_mcu_1_2_hdr.bin" @@ -53,6 +56,55 @@ #define MT7921_SDIO_HDR_TX_BYTES GENMASK(15, 0) #define MT7921_SDIO_HDR_PKT_TYPE GENMASK(17, 16) +#define MCU_UNI_EVENT_ROC 0x27 + +enum { + UNI_ROC_ACQUIRE, + UNI_ROC_ABORT, + UNI_ROC_NUM +}; + +enum mt7921_roc_req { + MT7921_ROC_REQ_JOIN, + MT7921_ROC_REQ_ROC, + MT7921_ROC_REQ_NUM +}; + +enum { + UNI_EVENT_ROC_GRANT = 0, + UNI_EVENT_ROC_TAG_NUM +}; + +struct mt7921_realease_info { + __le16 len; + u8 pad_len; + u8 tag; +} __packed; + +struct mt7921_fw_features { + u8 segment; + u8 data; + u8 rsv[14]; +} __packed; + +struct mt7921_roc_grant_tlv { + __le16 tag; + __le16 len; + u8 bss_idx; + u8 tokenid; + u8 status; + u8 primarychannel; + u8 rfsco; + u8 rfband; + u8 channelwidth; + u8 centerfreqseg1; + u8 centerfreqseg2; + u8 reqtype; + u8 dbdcband; + u8 rsv[1]; + __le32 max_interval; +} __packed; + enum mt7921_sdio_pkt_type { MT7921_SDIO_TXD, MT7921_SDIO_DATA, @@ -119,6 +171,7 @@ struct mt7921_vif { struct ewma_rssi rssi; struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; + struct ieee80211_chanctx_conf *ctx; }; struct mib_stats { @@ -171,7 +224,7 @@ struct mt7921_clc { u8 type; u8 rsv[8]; u8 data[]; -}; +} __packed; struct mt7921_phy { struct mt76_phy *mt76; @@ -200,6 +253,12 @@ struct mt7921_phy { #endif struct mt7921_clc *clc[MT7921_CLC_MAX_NUM]; + + struct work_struct roc_work; + struct timer_list roc_timer; + wait_queue_head_t roc_wait; + u8 roc_token_id; + bool roc_grant; }; #define mt7921_init_reset(dev) ((dev)->hif_ops->init_reset(dev)) @@ -236,6 +295,7 @@ struct mt7921_dev { struct work_struct init_work; u8 fw_debug; + u8 fw_features; struct mt76_connac_pm pm; struct mt76_connac_coredump coredump; @@ -244,6 +304,8 @@ struct mt7921_dev { struct work_struct ipv6_ns_work; /* IPv6 addresses for WoWLAN */ struct sk_buff_head ipv6_ns_list; + + enum environment_cap country_ie_env; }; enum { @@ -408,7 +470,7 @@ void mt7921_tx_worker(struct mt76_worker *w); void mt7921_tx_token_put(struct mt7921_dev *dev); bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len); void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); + struct sk_buff *skb, u32 *info); void mt7921_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); void mt7921_stats_work(struct work_struct *work); void mt7921_set_stream_he_caps(struct mt7921_phy *phy); @@ -425,6 +487,8 @@ int mt7921_mcu_uni_rx_ba(struct mt7921_dev *dev, struct ieee80211_ampdu_params *params, bool enable); void mt7921_scan_work(struct work_struct *work); +void mt7921_roc_work(struct work_struct *work); +void mt7921_roc_timer(struct timer_list *timer); int mt7921_mcu_uni_bss_ps(struct mt7921_dev *dev, struct ieee80211_vif *vif); int mt7921_mcu_drv_pmctrl(struct mt7921_dev *dev); int mt7921_mcu_fw_pmctrl(struct mt7921_dev *dev); @@ -508,4 +572,10 @@ int mt7921_set_tx_sar_pwr(struct ieee80211_hw *hw, int mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2, enum environment_cap env_cap); +int mt7921_mcu_set_roc(struct mt7921_phy *phy, struct mt7921_vif *vif, + struct ieee80211_channel *chan, int duration, + enum mt7921_roc_req type, u8 token_id); +int mt7921_mcu_abort_roc(struct mt7921_phy *phy, struct mt7921_vif *vif, + u8 token_id); +u8 mt7921_check_offload_capability(struct device *dev, const char *fw_wm); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index 8a53d8f286db..cb72ded37256 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -13,10 +13,14 @@ #include "../trace.h" static const struct pci_device_id mt7921_pci_device_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7961) }, - { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7922) }, - { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x0608) }, - { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x0616) }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7961), + .driver_data = (kernel_ulong_t)MT7921_FIRMWARE_WM }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7922), + .driver_data = (kernel_ulong_t)MT7922_FIRMWARE_WM }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x0608), + .driver_data = (kernel_ulong_t)MT7921_FIRMWARE_WM }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x0616), + .driver_data = (kernel_ulong_t)MT7921_FIRMWARE_WM }, { }, }; @@ -228,7 +232,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev, static const struct mt76_driver_ops drv_ops = { /* txwi_size = txd size + txp size */ .txwi_size = MT_TXD_SIZE + sizeof(struct mt76_connac_hw_txp), - .drv_flags = MT_DRV_TXWI_NO_FREE | MT_DRV_HW_MGMT_TXQ, + .drv_flags = MT_DRV_TXWI_NO_FREE | MT_DRV_HW_MGMT_TXQ | + MT_DRV_AMSDU_OFFLOAD, .survey_flags = SURVEY_INFO_TIME_TX | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_BSS_RX, @@ -252,9 +257,11 @@ static int mt7921_pci_probe(struct pci_dev *pdev, .fw_own = mt7921e_mcu_fw_pmctrl, }; + struct ieee80211_ops *ops; struct mt76_bus_ops *bus_ops; struct mt7921_dev *dev; struct mt76_dev *mdev; + u8 features; int ret; ret = pcim_enable_device(pdev); @@ -278,8 +285,28 @@ static int mt7921_pci_probe(struct pci_dev *pdev, if (mt7921_disable_aspm) mt76_pci_disable_aspm(pdev); - mdev = mt76_alloc_device(&pdev->dev, sizeof(*dev), &mt7921_ops, - &drv_ops); + features = mt7921_check_offload_capability(&pdev->dev, (const char *) + id->driver_data); + ops = devm_kmemdup(&pdev->dev, &mt7921_ops, sizeof(mt7921_ops), + GFP_KERNEL); + if (!ops) { + ret = -ENOMEM; + goto err_free_pci_vec; + } + + if (!(features & MT7921_FW_CAP_CNM)) { + ops->remain_on_channel = NULL; + ops->cancel_remain_on_channel = NULL; + ops->add_chanctx = NULL; + ops->remove_chanctx = NULL; + ops->change_chanctx = NULL; + ops->assign_vif_chanctx = NULL; + ops->unassign_vif_chanctx = NULL; + ops->mgd_prepare_tx = NULL; + ops->mgd_complete_tx = NULL; + } + + mdev = mt76_alloc_device(&pdev->dev, sizeof(*dev), ops, &drv_ops); if (!mdev) { ret = -ENOMEM; goto err_free_pci_vec; @@ -288,8 +315,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, mdev); dev = container_of(mdev, struct mt7921_dev, mt76); + dev->fw_features = features; dev->hif_ops = &mt7921_pcie_ops; - mt76_mmio_init(&dev->mt76, pcim_iomap_table(pdev)[0]); tasklet_init(&dev->irq_tasklet, mt7921_irq_tasklet, (unsigned long)dev); @@ -480,6 +507,21 @@ failed: return err; } +static void mt7921_pci_shutdown(struct pci_dev *pdev) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); + struct mt76_connac_pm *pm = &dev->pm; + + cancel_delayed_work_sync(&pm->ps_work); + cancel_work_sync(&pm->wake_work); + + /* chip cleanup before reboot */ + mt7921_mcu_drv_pmctrl(dev); + mt7921_dma_cleanup(dev); + mt7921_wfsys_reset(dev); +} + static DEFINE_SIMPLE_DEV_PM_OPS(mt7921_pm_ops, mt7921_pci_suspend, mt7921_pci_resume); static struct pci_driver mt7921_pci_driver = { @@ -487,6 +529,7 @@ static struct pci_driver mt7921_pci_driver = { .id_table = mt7921_pci_device_table, .probe = mt7921_pci_probe, .remove = mt7921_pci_remove, + .shutdown = mt7921_pci_shutdown, .driver.pm = pm_sleep_ptr(&mt7921_pm_ops), }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c index 3b25a06fd946..8ce4252b8ae7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c @@ -17,7 +17,8 @@ #include "mcu.h" static const struct sdio_device_id mt7921s_table[] = { - { SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, 0x7901) }, + { SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, 0x7901), + .driver_data = (kernel_ulong_t)MT7921_FIRMWARE_WM }, { } /* Terminating entry */ }; @@ -89,6 +90,7 @@ static int mt7921s_probe(struct sdio_func *func, { static const struct mt76_driver_ops drv_ops = { .txwi_size = MT_SDIO_TXD_SIZE, + .drv_flags = MT_DRV_AMSDU_OFFLOAD, .survey_flags = SURVEY_INFO_TIME_TX | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_BSS_RX, @@ -121,18 +123,39 @@ static int mt7921s_probe(struct sdio_func *func, .fw_own = mt7921s_mcu_fw_pmctrl, }; + struct ieee80211_ops *ops; struct mt7921_dev *dev; struct mt76_dev *mdev; + u8 features; int ret; - mdev = mt76_alloc_device(&func->dev, sizeof(*dev), &mt7921_ops, - &drv_ops); + features = mt7921_check_offload_capability(&func->dev, (const char *) + id->driver_data); + + ops = devm_kmemdup(&func->dev, &mt7921_ops, sizeof(mt7921_ops), + GFP_KERNEL); + if (!ops) + return -ENOMEM; + + if (!(features & MT7921_FW_CAP_CNM)) { + ops->remain_on_channel = NULL; + ops->cancel_remain_on_channel = NULL; + ops->add_chanctx = NULL; + ops->remove_chanctx = NULL; + ops->change_chanctx = NULL; + ops->assign_vif_chanctx = NULL; + ops->unassign_vif_chanctx = NULL; + ops->mgd_prepare_tx = NULL; + ops->mgd_complete_tx = NULL; + } + + mdev = mt76_alloc_device(&func->dev, sizeof(*dev), ops, &drv_ops); if (!mdev) return -ENOMEM; dev = container_of(mdev, struct mt7921_dev, mt76); + dev->fw_features = features; dev->hif_ops = &mt7921_sdio_ops; - sdio_set_drvdata(func, dev); ret = mt76s_init(mdev, func, &mt7921s_ops); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c index 29c0ee330dbe..5321d20dcdcb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c @@ -13,7 +13,8 @@ #include "mac.h" static const struct usb_device_id mt7921u_device_table[] = { - { USB_DEVICE_AND_INTERFACE_INFO(0x0e8d, 0x7961, 0xff, 0xff, 0xff) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x0e8d, 0x7961, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, { }, }; @@ -170,7 +171,8 @@ static int mt7921u_probe(struct usb_interface *usb_intf, { static const struct mt76_driver_ops drv_ops = { .txwi_size = MT_SDIO_TXD_SIZE, - .drv_flags = MT_DRV_RX_DMA_HDR | MT_DRV_HW_MGMT_TXQ, + .drv_flags = MT_DRV_RX_DMA_HDR | MT_DRV_HW_MGMT_TXQ | + MT_DRV_AMSDU_OFFLOAD, .survey_flags = SURVEY_INFO_TIME_TX | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_BSS_RX, @@ -203,13 +205,28 @@ static int mt7921u_probe(struct usb_interface *usb_intf, struct ieee80211_hw *hw; struct mt7921_dev *dev; struct mt76_dev *mdev; + u8 features; int ret; + features = mt7921_check_offload_capability(&usb_intf->dev, (const char *) + id->driver_info); ops = devm_kmemdup(&usb_intf->dev, &mt7921_ops, sizeof(mt7921_ops), GFP_KERNEL); if (!ops) return -ENOMEM; + if (!(features & MT7921_FW_CAP_CNM)) { + ops->remain_on_channel = NULL; + ops->cancel_remain_on_channel = NULL; + ops->add_chanctx = NULL; + ops->remove_chanctx = NULL; + ops->change_chanctx = NULL; + ops->assign_vif_chanctx = NULL; + ops->unassign_vif_chanctx = NULL; + ops->mgd_prepare_tx = NULL; + ops->mgd_complete_tx = NULL; + } + ops->stop = mt7921u_stop; mdev = mt76_alloc_device(&usb_intf->dev, sizeof(*dev), ops, &drv_ops); @@ -217,6 +234,7 @@ static int mt7921u_probe(struct usb_interface *usb_intf, return -ENOMEM; dev = container_of(mdev, struct mt7921_dev, mt76); + dev->fw_features = features; dev->hif_ops = &hif_ops; udev = usb_get_dev(udev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig b/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig new file mode 100644 index 000000000000..5c5fc569e6d5 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: ISC +config MT7996E + tristate "MediaTek MT7996 (PCIe) support" + select MT76_CONNAC_LIB + depends on MAC80211 + depends on PCI + help + This adds support for MT7996-based wireless PCIe devices, + which support concurrent tri-band operation at 6GHz, 5GHz, + and 2.4GHz IEEE 802.11be 4x4:4SS 4096-QAM, 320MHz channels. + + To compile this driver as a module, choose M here. diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/Makefile b/drivers/net/wireless/mediatek/mt76/mt7996/Makefile new file mode 100644 index 000000000000..bcb9a3c53149 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: ISC + +obj-$(CONFIG_MT7996E) += mt7996e.o + +mt7996e-y := pci.o init.o dma.o eeprom.o main.o mcu.o mac.o \ + debugfs.o mmio.o diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c new file mode 100644 index 000000000000..2e4a8909b9e8 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/relay.h> +#include "mt7996.h" +#include "eeprom.h" +#include "mcu.h" +#include "mac.h" + +#define FW_BIN_LOG_MAGIC 0x44d9c99a + +/** global debugfs **/ + +struct hw_queue_map { + const char *name; + u8 index; + u8 pid; + u8 qid; +}; + +static int +mt7996_implicit_txbf_set(void *data, u64 val) +{ + struct mt7996_dev *dev = data; + + /* The existing connected stations shall reconnect to apply + * new implicit txbf configuration. + */ + dev->ibf = !!val; + + return mt7996_mcu_set_txbf(dev, BF_HW_EN_UPDATE); +} + +static int +mt7996_implicit_txbf_get(void *data, u64 *val) +{ + struct mt7996_dev *dev = data; + + *val = dev->ibf; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_implicit_txbf, mt7996_implicit_txbf_get, + mt7996_implicit_txbf_set, "%lld\n"); + +/* test knob of system error recovery */ +static ssize_t +mt7996_fw_ser_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct mt7996_phy *phy = file->private_data; + struct mt7996_dev *dev = phy->dev; + u8 band_idx = phy->mt76->band_idx; + char buf[16]; + int ret = 0; + u16 val; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + + if (count && buf[count - 1] == '\n') + buf[count - 1] = '\0'; + else + buf[count] = '\0'; + + if (kstrtou16(buf, 0, &val)) + return -EINVAL; + + switch (val) { + case SER_SET_RECOVER_L1: + case SER_SET_RECOVER_L2: + case SER_SET_RECOVER_L3_RX_ABORT: + case SER_SET_RECOVER_L3_TX_ABORT: + case SER_SET_RECOVER_L3_TX_DISABLE: + case SER_SET_RECOVER_L3_BF: + ret = mt7996_mcu_set_ser(dev, SER_ENABLE, BIT(val), band_idx); + if (ret) + return ret; + + ret = mt7996_mcu_set_ser(dev, SER_RECOVER, val, band_idx); + break; + default: + break; + } + + return ret ? ret : count; +} + +static const struct file_operations mt7996_fw_ser_ops = { + .write = mt7996_fw_ser_set, + /* TODO: ser read */ + .open = simple_open, + .llseek = default_llseek, +}; + +static int +mt7996_radar_trigger(void *data, u64 val) +{ + struct mt7996_dev *dev = data; + + if (val > MT_RX_SEL2) + return -EINVAL; + + return mt7996_mcu_rdd_cmd(dev, RDD_RADAR_EMULATE, + val, 0, 0); +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_radar_trigger, NULL, + mt7996_radar_trigger, "%lld\n"); + +static int +mt7996_rdd_monitor(struct seq_file *s, void *data) +{ + struct mt7996_dev *dev = dev_get_drvdata(s->private); + struct cfg80211_chan_def *chandef = &dev->rdd2_chandef; + const char *bw; + int ret = 0; + + mutex_lock(&dev->mt76.mutex); + + if (!cfg80211_chandef_valid(chandef)) { + ret = -EINVAL; + goto out; + } + + if (!dev->rdd2_phy) { + seq_puts(s, "not running\n"); + goto out; + } + + switch (chandef->width) { + case NL80211_CHAN_WIDTH_40: + bw = "40"; + break; + case NL80211_CHAN_WIDTH_80: + bw = "80"; + break; + case NL80211_CHAN_WIDTH_160: + bw = "160"; + break; + case NL80211_CHAN_WIDTH_80P80: + bw = "80P80"; + break; + default: + bw = "20"; + break; + } + + seq_printf(s, "channel %d (%d MHz) width %s MHz center1: %d MHz\n", + chandef->chan->hw_value, chandef->chan->center_freq, + bw, chandef->center_freq1); +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static int +mt7996_fw_debug_wm_set(void *data, u64 val) +{ + struct mt7996_dev *dev = data; + enum { + DEBUG_TXCMD = 62, + DEBUG_CMD_RPT_TX, + DEBUG_CMD_RPT_TRIG, + DEBUG_SPL, + DEBUG_RPT_RX, + DEBUG_RPT_RA = 68, + } debug; + bool tx, rx, en; + int ret; + + dev->fw_debug_wm = val ? MCU_FW_LOG_TO_HOST : 0; + + if (dev->fw_debug_bin) + val = MCU_FW_LOG_RELAY; + else + val = dev->fw_debug_wm; + + tx = dev->fw_debug_wm || (dev->fw_debug_bin & BIT(1)); + rx = dev->fw_debug_wm || (dev->fw_debug_bin & BIT(2)); + en = dev->fw_debug_wm || (dev->fw_debug_bin & BIT(0)); + + ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WM, val); + if (ret) + return ret; + + for (debug = DEBUG_TXCMD; debug <= DEBUG_RPT_RA; debug++) { + if (debug == 67) + continue; + + if (debug == DEBUG_RPT_RX) + val = en && rx; + else + val = en && tx; + + ret = mt7996_mcu_fw_dbg_ctrl(dev, debug, val); + if (ret) + return ret; + } + + return 0; +} + +static int +mt7996_fw_debug_wm_get(void *data, u64 *val) +{ + struct mt7996_dev *dev = data; + + *val = dev->fw_debug_wm; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_fw_debug_wm, mt7996_fw_debug_wm_get, + mt7996_fw_debug_wm_set, "%lld\n"); + +static int +mt7996_fw_debug_wa_set(void *data, u64 val) +{ + struct mt7996_dev *dev = data; + int ret; + + dev->fw_debug_wa = val ? MCU_FW_LOG_TO_HOST : 0; + + ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WA, dev->fw_debug_wa); + if (ret) + return ret; + + return mt7996_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(SET), MCU_WA_PARAM_PDMA_RX, + !!dev->fw_debug_wa, 0); +} + +static int +mt7996_fw_debug_wa_get(void *data, u64 *val) +{ + struct mt7996_dev *dev = data; + + *val = dev->fw_debug_wa; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_fw_debug_wa, mt7996_fw_debug_wa_get, + mt7996_fw_debug_wa_set, "%lld\n"); + +static struct dentry * +create_buf_file_cb(const char *filename, struct dentry *parent, umode_t mode, + struct rchan_buf *buf, int *is_global) +{ + struct dentry *f; + + f = debugfs_create_file("fwlog_data", mode, parent, buf, + &relay_file_operations); + if (IS_ERR(f)) + return NULL; + + *is_global = 1; + + return f; +} + +static int +remove_buf_file_cb(struct dentry *f) +{ + debugfs_remove(f); + + return 0; +} + +static int +mt7996_fw_debug_bin_set(void *data, u64 val) +{ + static struct rchan_callbacks relay_cb = { + .create_buf_file = create_buf_file_cb, + .remove_buf_file = remove_buf_file_cb, + }; + struct mt7996_dev *dev = data; + + if (!dev->relay_fwlog) + dev->relay_fwlog = relay_open("fwlog_data", dev->debugfs_dir, + 1500, 512, &relay_cb, NULL); + if (!dev->relay_fwlog) + return -ENOMEM; + + dev->fw_debug_bin = val; + + relay_reset(dev->relay_fwlog); + + return mt7996_fw_debug_wm_set(dev, dev->fw_debug_wm); +} + +static int +mt7996_fw_debug_bin_get(void *data, u64 *val) +{ + struct mt7996_dev *dev = data; + + *val = dev->fw_debug_bin; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_fw_debug_bin, mt7996_fw_debug_bin_get, + mt7996_fw_debug_bin_set, "%lld\n"); + +static int +mt7996_fw_util_wa_show(struct seq_file *file, void *data) +{ + struct mt7996_dev *dev = file->private; + + if (dev->fw_debug_wa) + return mt7996_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(QUERY), + MCU_WA_PARAM_CPU_UTIL, 0, 0); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_fw_util_wa); + +static void +mt7996_ampdu_stat_read_phy(struct mt7996_phy *phy, struct seq_file *file) +{ + struct mt7996_dev *dev = phy->dev; + int bound[15], range[8], i; + u8 band_idx = phy->mt76->band_idx; + + /* Tx ampdu stat */ + for (i = 0; i < ARRAY_SIZE(range); i++) + range[i] = mt76_rr(dev, MT_MIB_ARNG(band_idx, i)); + + for (i = 0; i < ARRAY_SIZE(bound); i++) + bound[i] = MT_MIB_ARNCR_RANGE(range[i / 2], i % 2) + 1; + + seq_printf(file, "\nPhy %s, Phy band %d\n", + wiphy_name(phy->mt76->hw->wiphy), band_idx); + + seq_printf(file, "Length: %8d | ", bound[0]); + for (i = 0; i < ARRAY_SIZE(bound) - 1; i++) + seq_printf(file, "%3d -%3d | ", + bound[i] + 1, bound[i + 1]); + + seq_puts(file, "\nCount: "); + for (i = 0; i < ARRAY_SIZE(bound); i++) + seq_printf(file, "%8d | ", phy->mt76->aggr_stats[i]); + seq_puts(file, "\n"); + + seq_printf(file, "BA miss count: %d\n", phy->mib.ba_miss_cnt); +} + +static void +mt7996_txbf_stat_read_phy(struct mt7996_phy *phy, struct seq_file *s) +{ + static const char * const bw[] = { + "BW20", "BW40", "BW80", "BW160" + }; + struct mib_stats *mib = &phy->mib; + + /* Tx Beamformer monitor */ + seq_puts(s, "\nTx Beamformer applied PPDU counts: "); + + seq_printf(s, "iBF: %d, eBF: %d\n", + mib->tx_bf_ibf_ppdu_cnt, + mib->tx_bf_ebf_ppdu_cnt); + + /* Tx Beamformer Rx feedback monitor */ + seq_puts(s, "Tx Beamformer Rx feedback statistics: "); + + seq_printf(s, "All: %d, HE: %d, VHT: %d, HT: %d, ", + mib->tx_bf_rx_fb_all_cnt, + mib->tx_bf_rx_fb_he_cnt, + mib->tx_bf_rx_fb_vht_cnt, + mib->tx_bf_rx_fb_ht_cnt); + + seq_printf(s, "%s, NC: %d, NR: %d\n", + bw[mib->tx_bf_rx_fb_bw], + mib->tx_bf_rx_fb_nc_cnt, + mib->tx_bf_rx_fb_nr_cnt); + + /* Tx Beamformee Rx NDPA & Tx feedback report */ + seq_printf(s, "Tx Beamformee successful feedback frames: %d\n", + mib->tx_bf_fb_cpl_cnt); + seq_printf(s, "Tx Beamformee feedback triggered counts: %d\n", + mib->tx_bf_fb_trig_cnt); + + /* Tx SU & MU counters */ + seq_printf(s, "Tx multi-user Beamforming counts: %d\n", + mib->tx_mu_bf_cnt); + seq_printf(s, "Tx multi-user MPDU counts: %d\n", mib->tx_mu_mpdu_cnt); + seq_printf(s, "Tx multi-user successful MPDU counts: %d\n", + mib->tx_mu_acked_mpdu_cnt); + seq_printf(s, "Tx single-user successful MPDU counts: %d\n", + mib->tx_su_acked_mpdu_cnt); + + seq_puts(s, "\n"); +} + +static int +mt7996_tx_stats_show(struct seq_file *file, void *data) +{ + struct mt7996_phy *phy = file->private; + struct mt7996_dev *dev = phy->dev; + struct mib_stats *mib = &phy->mib; + int i; + u32 attempts, success, per; + + mutex_lock(&dev->mt76.mutex); + + mt7996_mac_update_stats(phy); + mt7996_ampdu_stat_read_phy(phy, file); + + attempts = mib->tx_mpdu_attempts_cnt; + success = mib->tx_mpdu_success_cnt; + per = attempts ? 100 - success * 100 / attempts : 100; + seq_printf(file, "Tx attempts: %8u (MPDUs)\n", attempts); + seq_printf(file, "Tx success: %8u (MPDUs)\n", success); + seq_printf(file, "Tx PER: %u%%\n", per); + + mt7996_txbf_stat_read_phy(phy, file); + + /* Tx amsdu info */ + seq_puts(file, "Tx MSDU statistics:\n"); + for (i = 0; i < ARRAY_SIZE(mib->tx_amsdu); i++) { + seq_printf(file, "AMSDU pack count of %d MSDU in TXD: %8d ", + i + 1, mib->tx_amsdu[i]); + if (mib->tx_amsdu_cnt) + seq_printf(file, "(%3d%%)\n", + mib->tx_amsdu[i] * 100 / mib->tx_amsdu_cnt); + else + seq_puts(file, "\n"); + } + + mutex_unlock(&dev->mt76.mutex); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_tx_stats); + +static void +mt7996_hw_queue_read(struct seq_file *s, u32 size, + const struct hw_queue_map *map) +{ + struct mt7996_phy *phy = s->private; + struct mt7996_dev *dev = phy->dev; + u32 i, val; + + val = mt76_rr(dev, MT_FL_Q_EMPTY); + for (i = 0; i < size; i++) { + u32 ctrl, head, tail, queued; + + if (val & BIT(map[i].index)) + continue; + + ctrl = BIT(31) | (map[i].pid << 10) | (map[i].qid << 24); + mt76_wr(dev, MT_FL_Q0_CTRL, ctrl); + + head = mt76_get_field(dev, MT_FL_Q2_CTRL, + GENMASK(11, 0)); + tail = mt76_get_field(dev, MT_FL_Q2_CTRL, + GENMASK(27, 16)); + queued = mt76_get_field(dev, MT_FL_Q3_CTRL, + GENMASK(11, 0)); + + seq_printf(s, "\t%s: ", map[i].name); + seq_printf(s, "queued:0x%03x head:0x%03x tail:0x%03x\n", + queued, head, tail); + } +} + +static void +mt7996_sta_hw_queue_read(void *data, struct ieee80211_sta *sta) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = msta->vif->phy->dev; + struct seq_file *s = data; + u8 ac; + + for (ac = 0; ac < 4; ac++) { + u32 qlen, ctrl, val; + u32 idx = msta->wcid.idx >> 5; + u8 offs = msta->wcid.idx & GENMASK(4, 0); + + ctrl = BIT(31) | BIT(11) | (ac << 24); + val = mt76_rr(dev, MT_PLE_AC_QEMPTY(ac, idx)); + + if (val & BIT(offs)) + continue; + + mt76_wr(dev, MT_FL_Q0_CTRL, ctrl | msta->wcid.idx); + qlen = mt76_get_field(dev, MT_FL_Q3_CTRL, + GENMASK(11, 0)); + seq_printf(s, "\tSTA %pM wcid %d: AC%d%d queued:%d\n", + sta->addr, msta->wcid.idx, + msta->vif->mt76.wmm_idx, ac, qlen); + } +} + +static int +mt7996_hw_queues_show(struct seq_file *file, void *data) +{ + struct mt7996_phy *phy = file->private; + struct mt7996_dev *dev = phy->dev; + static const struct hw_queue_map ple_queue_map[] = { + { "CPU_Q0", 0, 1, MT_CTX0 }, + { "CPU_Q1", 1, 1, MT_CTX0 + 1 }, + { "CPU_Q2", 2, 1, MT_CTX0 + 2 }, + { "CPU_Q3", 3, 1, MT_CTX0 + 3 }, + { "ALTX_Q0", 8, 2, MT_LMAC_ALTX0 }, + { "BMC_Q0", 9, 2, MT_LMAC_BMC0 }, + { "BCN_Q0", 10, 2, MT_LMAC_BCN0 }, + { "PSMP_Q0", 11, 2, MT_LMAC_PSMP0 }, + { "ALTX_Q1", 12, 2, MT_LMAC_ALTX0 + 4 }, + { "BMC_Q1", 13, 2, MT_LMAC_BMC0 + 4 }, + { "BCN_Q1", 14, 2, MT_LMAC_BCN0 + 4 }, + { "PSMP_Q1", 15, 2, MT_LMAC_PSMP0 + 4 }, + }; + static const struct hw_queue_map pse_queue_map[] = { + { "CPU Q0", 0, 1, MT_CTX0 }, + { "CPU Q1", 1, 1, MT_CTX0 + 1 }, + { "CPU Q2", 2, 1, MT_CTX0 + 2 }, + { "CPU Q3", 3, 1, MT_CTX0 + 3 }, + { "HIF_Q0", 8, 0, MT_HIF0 }, + { "HIF_Q1", 9, 0, MT_HIF0 + 1 }, + { "HIF_Q2", 10, 0, MT_HIF0 + 2 }, + { "HIF_Q3", 11, 0, MT_HIF0 + 3 }, + { "HIF_Q4", 12, 0, MT_HIF0 + 4 }, + { "HIF_Q5", 13, 0, MT_HIF0 + 5 }, + { "LMAC_Q", 16, 2, 0 }, + { "MDP_TXQ", 17, 2, 1 }, + { "MDP_RXQ", 18, 2, 2 }, + { "SEC_TXQ", 19, 2, 3 }, + { "SEC_RXQ", 20, 2, 4 }, + }; + u32 val, head, tail; + + /* ple queue */ + val = mt76_rr(dev, MT_PLE_FREEPG_CNT); + head = mt76_get_field(dev, MT_PLE_FREEPG_HEAD_TAIL, GENMASK(11, 0)); + tail = mt76_get_field(dev, MT_PLE_FREEPG_HEAD_TAIL, GENMASK(27, 16)); + seq_puts(file, "PLE page info:\n"); + seq_printf(file, + "\tTotal free page: 0x%08x head: 0x%03x tail: 0x%03x\n", + val, head, tail); + + val = mt76_rr(dev, MT_PLE_PG_HIF_GROUP); + head = mt76_get_field(dev, MT_PLE_HIF_PG_INFO, GENMASK(11, 0)); + tail = mt76_get_field(dev, MT_PLE_HIF_PG_INFO, GENMASK(27, 16)); + seq_printf(file, "\tHIF free page: 0x%03x res: 0x%03x used: 0x%03x\n", + val, head, tail); + + seq_puts(file, "PLE non-empty queue info:\n"); + mt7996_hw_queue_read(file, ARRAY_SIZE(ple_queue_map), + &ple_queue_map[0]); + + /* iterate per-sta ple queue */ + ieee80211_iterate_stations_atomic(phy->mt76->hw, + mt7996_sta_hw_queue_read, file); + /* pse queue */ + seq_puts(file, "PSE non-empty queue info:\n"); + mt7996_hw_queue_read(file, ARRAY_SIZE(pse_queue_map), + &pse_queue_map[0]); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_hw_queues); + +static int +mt7996_xmit_queues_show(struct seq_file *file, void *data) +{ + struct mt7996_phy *phy = file->private; + struct mt7996_dev *dev = phy->dev; + struct { + struct mt76_queue *q; + char *queue; + } queue_map[] = { + { phy->mt76->q_tx[MT_TXQ_BE], " MAIN" }, + { dev->mt76.q_mcu[MT_MCUQ_WM], " MCUWM" }, + { dev->mt76.q_mcu[MT_MCUQ_WA], " MCUWA" }, + { dev->mt76.q_mcu[MT_MCUQ_FWDL], "MCUFWDL" }, + }; + int i; + + seq_puts(file, " queue | hw-queued | head | tail |\n"); + for (i = 0; i < ARRAY_SIZE(queue_map); i++) { + struct mt76_queue *q = queue_map[i].q; + + if (!q) + continue; + + seq_printf(file, " %s | %9d | %9d | %9d |\n", + queue_map[i].queue, q->queued, q->head, + q->tail); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_xmit_queues); + +static int +mt7996_twt_stats(struct seq_file *s, void *data) +{ + struct mt7996_dev *dev = dev_get_drvdata(s->private); + struct mt7996_twt_flow *iter; + + rcu_read_lock(); + + seq_puts(s, " wcid | id | flags | exp | mantissa"); + seq_puts(s, " | duration | tsf |\n"); + list_for_each_entry_rcu(iter, &dev->twt_list, list) + seq_printf(s, + "%9d | %8d | %5c%c%c%c | %8d | %8d | %8d | %14lld |\n", + iter->wcid, iter->id, + iter->sched ? 's' : 'u', + iter->protection ? 'p' : '-', + iter->trigger ? 't' : '-', + iter->flowtype ? '-' : 'a', + iter->exp, iter->mantissa, + iter->duration, iter->tsf); + + rcu_read_unlock(); + + return 0; +} + +/* The index of RF registers use the generic regidx, combined with two parts: + * WF selection [31:24] and offset [23:0]. + */ +static int +mt7996_rf_regval_get(void *data, u64 *val) +{ + struct mt7996_dev *dev = data; + u32 regval; + int ret; + + ret = mt7996_mcu_rf_regval(dev, dev->mt76.debugfs_reg, ®val, false); + if (ret) + return ret; + + *val = regval; + + return 0; +} + +static int +mt7996_rf_regval_set(void *data, u64 val) +{ + struct mt7996_dev *dev = data; + + return mt7996_mcu_rf_regval(dev, dev->mt76.debugfs_reg, (u32 *)&val, true); +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_rf_regval, mt7996_rf_regval_get, + mt7996_rf_regval_set, "0x%08llx\n"); + +int mt7996_init_debugfs(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + struct dentry *dir; + + dir = mt76_register_debugfs_fops(phy->mt76, NULL); + if (!dir) + return -ENOMEM; + debugfs_create_file("hw-queues", 0400, dir, phy, + &mt7996_hw_queues_fops); + debugfs_create_file("xmit-queues", 0400, dir, phy, + &mt7996_xmit_queues_fops); + debugfs_create_file("tx_stats", 0400, dir, phy, &mt7996_tx_stats_fops); + debugfs_create_file("fw_debug_wm", 0600, dir, dev, &fops_fw_debug_wm); + debugfs_create_file("fw_debug_wa", 0600, dir, dev, &fops_fw_debug_wa); + debugfs_create_file("fw_debug_bin", 0600, dir, dev, &fops_fw_debug_bin); + /* TODO: wm fw cpu utilization */ + debugfs_create_file("fw_util_wa", 0400, dir, dev, + &mt7996_fw_util_wa_fops); + debugfs_create_file("implicit_txbf", 0600, dir, dev, + &fops_implicit_txbf); + debugfs_create_devm_seqfile(dev->mt76.dev, "twt_stats", dir, + mt7996_twt_stats); + debugfs_create_file("fw_ser", 0600, dir, phy, &mt7996_fw_ser_ops); + debugfs_create_file("rf_regval", 0600, dir, dev, &fops_rf_regval); + + if (phy->mt76->cap.has_5ghz) { + debugfs_create_u32("dfs_hw_pattern", 0400, dir, + &dev->hw_pattern); + debugfs_create_file("radar_trigger", 0200, dir, dev, + &fops_radar_trigger); + debugfs_create_devm_seqfile(dev->mt76.dev, "rdd_monitor", dir, + mt7996_rdd_monitor); + } + + if (phy == &dev->phy) + dev->debugfs_dir = dir; + + return 0; +} + +static void +mt7996_debugfs_write_fwlog(struct mt7996_dev *dev, const void *hdr, int hdrlen, + const void *data, int len) +{ + static DEFINE_SPINLOCK(lock); + unsigned long flags; + void *dest; + + spin_lock_irqsave(&lock, flags); + dest = relay_reserve(dev->relay_fwlog, hdrlen + len + 4); + if (dest) { + *(u32 *)dest = hdrlen + len; + dest += 4; + + if (hdrlen) { + memcpy(dest, hdr, hdrlen); + dest += hdrlen; + } + + memcpy(dest, data, len); + relay_flush(dev->relay_fwlog); + } + spin_unlock_irqrestore(&lock, flags); +} + +void mt7996_debugfs_rx_fw_monitor(struct mt7996_dev *dev, const void *data, int len) +{ + struct { + __le32 magic; + u8 version; + u8 _rsv; + __le16 serial_id; + __le32 timestamp; + __le16 msg_type; + __le16 len; + } hdr = { + .version = 0x1, + .magic = cpu_to_le32(FW_BIN_LOG_MAGIC), + .msg_type = cpu_to_le16(PKT_TYPE_RX_FW_MONITOR), + }; + + if (!dev->relay_fwlog) + return; + + hdr.serial_id = cpu_to_le16(dev->fw_debug_seq++); + hdr.timestamp = cpu_to_le32(mt76_rr(dev, MT_LPON_FRCR(0))); + hdr.len = *(__le16 *)data; + mt7996_debugfs_write_fwlog(dev, &hdr, sizeof(hdr), data, len); +} + +bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len) +{ + if (get_unaligned_le32(data) != FW_BIN_LOG_MAGIC) + return false; + + if (dev->relay_fwlog) + mt7996_debugfs_write_fwlog(dev, NULL, 0, data, len); + + return true; +} + +#ifdef CONFIG_MAC80211_DEBUGFS +/** per-station debugfs **/ + +static ssize_t mt7996_sta_fixed_rate_set(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ +#define SHORT_PREAMBLE 0 +#define LONG_PREAMBLE 1 + struct ieee80211_sta *sta = file->private_data; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = msta->vif->phy->dev; + struct ra_rate phy = {}; + char buf[100]; + int ret; + u16 gi, ltf; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + + if (count && buf[count - 1] == '\n') + buf[count - 1] = '\0'; + else + buf[count] = '\0'; + + /* mode - cck: 0, ofdm: 1, ht: 2, gf: 3, vht: 4, he_su: 8, he_er: 9 + * bw - bw20: 0, bw40: 1, bw80: 2, bw160: 3 + * nss - vht: 1~4, he: 1~4, others: ignore + * mcs - cck: 0~4, ofdm: 0~7, ht: 0~32, vht: 0~9, he_su: 0~11, he_er: 0~2 + * gi - (ht/vht) lgi: 0, sgi: 1; (he) 0.8us: 0, 1.6us: 1, 3.2us: 2 + * preamble - short: 1, long: 0 + * ldpc - off: 0, on: 1 + * stbc - off: 0, on: 1 + * ltf - 1xltf: 0, 2xltf: 1, 4xltf: 2 + */ + if (sscanf(buf, "%hhu %hhu %hhu %hhu %hu %hhu %hhu %hhu %hhu %hu", + &phy.mode, &phy.bw, &phy.mcs, &phy.nss, &gi, + &phy.preamble, &phy.stbc, &phy.ldpc, &phy.spe, <f) != 10) { + dev_warn(dev->mt76.dev, + "format: Mode BW MCS NSS GI Preamble STBC LDPC SPE ltf\n"); + goto out; + } + + phy.wlan_idx = cpu_to_le16(msta->wcid.idx); + phy.gi = cpu_to_le16(gi); + phy.ltf = cpu_to_le16(ltf); + phy.ldpc = phy.ldpc ? 7 : 0; + phy.preamble = phy.preamble ? SHORT_PREAMBLE : LONG_PREAMBLE; + + ret = mt7996_mcu_set_fixed_rate_ctrl(dev, &phy, 0); + if (ret) + return -EFAULT; + +out: + return count; +} + +static const struct file_operations fops_fixed_rate = { + .write = mt7996_sta_fixed_rate_set, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static int +mt7996_queues_show(struct seq_file *s, void *data) +{ + struct ieee80211_sta *sta = s->private; + + mt7996_sta_hw_queue_read(s, sta); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_queues); + +void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, struct dentry *dir) +{ + debugfs_create_file("fixed_rate", 0600, dir, sta, &fops_fixed_rate); + debugfs_create_file("hw-queues", 0400, dir, sta, &mt7996_queues_fops); +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c new file mode 100644 index 000000000000..c09fe4274935 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include "mt7996.h" +#include "../dma.h" +#include "mac.h" + +static int mt7996_poll_tx(struct napi_struct *napi, int budget) +{ + struct mt7996_dev *dev; + + dev = container_of(napi, struct mt7996_dev, mt76.tx_napi); + + mt76_connac_tx_cleanup(&dev->mt76); + if (napi_complete_done(napi, 0)) + mt7996_irq_enable(dev, MT_INT_TX_DONE_MCU); + + return 0; +} + +static void mt7996_dma_config(struct mt7996_dev *dev) +{ +#define Q_CONFIG(q, wfdma, int, id) do { \ + if (wfdma) \ + dev->q_wfdma_mask |= (1 << (q)); \ + dev->q_int_mask[(q)] = int; \ + dev->q_id[(q)] = id; \ +} while (0) + +#define MCUQ_CONFIG(q, wfdma, int, id) Q_CONFIG(q, (wfdma), (int), (id)) +#define RXQ_CONFIG(q, wfdma, int, id) Q_CONFIG(__RXQ(q), (wfdma), (int), (id)) +#define TXQ_CONFIG(q, wfdma, int, id) Q_CONFIG(__TXQ(q), (wfdma), (int), (id)) + + /* rx queue */ + RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, MT7996_RXQ_MCU_WM); + RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, MT7996_RXQ_MCU_WA); + + /* band0/band1 */ + RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, MT7996_RXQ_BAND0); + RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN, MT7996_RXQ_MCU_WA_MAIN); + + /* band2 */ + RXQ_CONFIG(MT_RXQ_BAND2, WFDMA0, MT_INT_RX_DONE_BAND2, MT7996_RXQ_BAND2); + RXQ_CONFIG(MT_RXQ_BAND2_WA, WFDMA0, MT_INT_RX_DONE_WA_TRI, MT7996_RXQ_MCU_WA_TRI); + + /* data tx queue */ + TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7996_TXQ_BAND0); + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); + TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND2, MT7996_TXQ_BAND2); + + /* mcu tx queue */ + MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7996_TXQ_MCU_WM); + MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA, MT7996_TXQ_MCU_WA); + MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, MT7996_TXQ_FWDL); +} + +static void __mt7996_dma_prefetch(struct mt7996_dev *dev, u32 ofs) +{ +#define PREFETCH(_base, _depth) ((_base) << 16 | (_depth)) + /* prefetch SRAM wrapping boundary for tx/rx ring. */ + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(0x0, 0x2)); + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(0x20, 0x2)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(0) + ofs, PREFETCH(0x40, 0x4)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0x80, 0x4)); + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(0xc0, 0x2)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(2) + ofs, PREFETCH(0xe0, 0x4)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU) + ofs, PREFETCH(0x120, 0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU_WA) + ofs, PREFETCH(0x140, 0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN_WA) + ofs, PREFETCH(0x160, 0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND2_WA) + ofs, PREFETCH(0x180, 0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN) + ofs, PREFETCH(0x1a0, 0x10)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND2) + ofs, PREFETCH(0x2a0, 0x10)); + + mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT1 + ofs, WF_WFDMA0_GLO_CFG_EXT1_CALC_MODE); +} + +void mt7996_dma_prefetch(struct mt7996_dev *dev) +{ + __mt7996_dma_prefetch(dev, 0); + if (dev->hif2) + __mt7996_dma_prefetch(dev, MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0)); +} + +static void mt7996_dma_disable(struct mt7996_dev *dev, bool reset) +{ + u32 hif1_ofs = 0; + + if (dev->hif2) + hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + + if (reset) { + mt76_clear(dev, MT_WFDMA0_RST, + MT_WFDMA0_RST_DMASHDL_ALL_RST | + MT_WFDMA0_RST_LOGIC_RST); + + mt76_set(dev, MT_WFDMA0_RST, + MT_WFDMA0_RST_DMASHDL_ALL_RST | + MT_WFDMA0_RST_LOGIC_RST); + + if (dev->hif2) { + mt76_clear(dev, MT_WFDMA0_RST + hif1_ofs, + MT_WFDMA0_RST_DMASHDL_ALL_RST | + MT_WFDMA0_RST_LOGIC_RST); + + mt76_set(dev, MT_WFDMA0_RST + hif1_ofs, + MT_WFDMA0_RST_DMASHDL_ALL_RST | + MT_WFDMA0_RST_LOGIC_RST); + } + } + + /* disable */ + mt76_clear(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + + if (dev->hif2) { + mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + } +} + +static int mt7996_dma_enable(struct mt7996_dev *dev) +{ + u32 hif1_ofs = 0; + u32 irq_mask; + + if (dev->hif2) + hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + + /* reset dma idx */ + mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR, ~0); + if (dev->hif2) + mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR + hif1_ofs, ~0); + + /* configure delay interrupt off */ + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG0, 0); + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG1, 0); + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG2, 0); + + if (dev->hif2) { + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG0 + hif1_ofs, 0); + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG1 + hif1_ofs, 0); + mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG2 + hif1_ofs, 0); + } + + /* configure perfetch settings */ + mt7996_dma_prefetch(dev); + + /* hif wait WFDMA idle */ + mt76_set(dev, MT_WFDMA0_BUSY_ENA, + MT_WFDMA0_BUSY_ENA_TX_FIFO0 | + MT_WFDMA0_BUSY_ENA_TX_FIFO1 | + MT_WFDMA0_BUSY_ENA_RX_FIFO); + + if (dev->hif2) + mt76_set(dev, MT_WFDMA0_BUSY_ENA + hif1_ofs, + MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO0 | + MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO1 | + MT_WFDMA0_PCIE1_BUSY_ENA_RX_FIFO); + + mt76_poll(dev, MT_WFDMA_EXT_CSR_HIF_MISC, + MT_WFDMA_EXT_CSR_HIF_MISC_BUSY, 0, 1000); + + /* set WFDMA Tx/Rx */ + mt76_set(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + + /* GLO_CFG_EXT0 */ + mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT0, + WF_WFDMA0_GLO_CFG_EXT0_RX_WB_RXD | + WF_WFDMA0_GLO_CFG_EXT0_WED_MERGE_MODE); + + /* GLO_CFG_EXT1 */ + mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT1, + WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE); + + if (dev->hif2) { + mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + + /* GLO_CFG_EXT0 */ + mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT0 + hif1_ofs, + WF_WFDMA0_GLO_CFG_EXT0_RX_WB_RXD | + WF_WFDMA0_GLO_CFG_EXT0_WED_MERGE_MODE); + + /* GLO_CFG_EXT1 */ + mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT1 + hif1_ofs, + WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE); + + mt76_set(dev, MT_WFDMA_HOST_CONFIG, + MT_WFDMA_HOST_CONFIG_PDMA_BAND); + } + + if (dev->hif2) { + /* fix hardware limitation, pcie1's rx ring3 is not available + * so, redirect pcie0 rx ring3 interrupt to pcie1 + */ + mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL, + MT_WFDMA0_RX_INT_SEL_RING3); + + /* TODO: redirect rx ring6 interrupt to pcie0 for wed function */ + } + + /* enable interrupts for TX/RX rings */ + irq_mask = MT_INT_RX_DONE_MCU | + MT_INT_TX_DONE_MCU | + MT_INT_MCU_CMD; + + if (!dev->mphy.band_idx) + irq_mask |= MT_INT_BAND0_RX_DONE; + + if (dev->dbdc_support) + irq_mask |= MT_INT_BAND1_RX_DONE; + + if (dev->tbtc_support) + irq_mask |= MT_INT_BAND2_RX_DONE; + + mt7996_irq_enable(dev, irq_mask); + + return 0; +} + +int mt7996_dma_init(struct mt7996_dev *dev) +{ + u32 hif1_ofs = 0; + int ret; + + mt7996_dma_config(dev); + + mt76_dma_attach(&dev->mt76); + + if (dev->hif2) + hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + + mt7996_dma_disable(dev, true); + + /* init tx queue */ + ret = mt76_connac_init_tx_queues(dev->phy.mt76, + MT_TXQ_ID(dev->mphy.band_idx), + MT7996_TX_RING_SIZE, + MT_TXQ_RING_BASE(0), 0); + if (ret) + return ret; + + /* command to WM */ + ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WM, + MT_MCUQ_ID(MT_MCUQ_WM), + MT7996_TX_MCU_RING_SIZE, + MT_MCUQ_RING_BASE(MT_MCUQ_WM)); + if (ret) + return ret; + + /* command to WA */ + ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WA, + MT_MCUQ_ID(MT_MCUQ_WA), + MT7996_TX_MCU_RING_SIZE, + MT_MCUQ_RING_BASE(MT_MCUQ_WA)); + if (ret) + return ret; + + /* firmware download */ + ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_FWDL, + MT_MCUQ_ID(MT_MCUQ_FWDL), + MT7996_TX_FWDL_RING_SIZE, + MT_MCUQ_RING_BASE(MT_MCUQ_FWDL)); + if (ret) + return ret; + + /* event from WM */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU], + MT_RXQ_ID(MT_RXQ_MCU), + MT7996_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MCU)); + if (ret) + return ret; + + /* event from WA */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA], + MT_RXQ_ID(MT_RXQ_MCU_WA), + MT7996_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MCU_WA)); + if (ret) + return ret; + + /* rx data queue for band0 and band1 */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN], + MT_RXQ_ID(MT_RXQ_MAIN), + MT7996_RX_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MAIN)); + if (ret) + return ret; + + /* tx free notify event from WA for band0 */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA], + MT_RXQ_ID(MT_RXQ_MAIN_WA), + MT7996_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA)); + if (ret) + return ret; + + if (dev->tbtc_support || dev->mphy.band_idx == MT_BAND2) { + /* rx data queue for band2 */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND2], + MT_RXQ_ID(MT_RXQ_BAND2), + MT7996_RX_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_BAND2) + hif1_ofs); + if (ret) + return ret; + + /* tx free notify event from WA for band2 + * use pcie0's rx ring3, but, redirect pcie0 rx ring3 interrupt to pcie1 + */ + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND2_WA], + MT_RXQ_ID(MT_RXQ_BAND2_WA), + MT7996_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_BAND2_WA)); + if (ret) + return ret; + } + + ret = mt76_init_queues(dev, mt76_dma_rx_poll); + if (ret < 0) + return ret; + + netif_napi_add_tx(&dev->mt76.tx_napi_dev, &dev->mt76.tx_napi, + mt7996_poll_tx); + napi_enable(&dev->mt76.tx_napi); + + mt7996_dma_enable(dev); + + return 0; +} + +void mt7996_dma_cleanup(struct mt7996_dev *dev) +{ + mt7996_dma_disable(dev, true); + + mt76_dma_cleanup(&dev->mt76); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c new file mode 100644 index 000000000000..b9f62bedbc48 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/firmware.h> +#include "mt7996.h" +#include "eeprom.h" + +static int mt7996_check_eeprom(struct mt7996_dev *dev) +{ + u8 *eeprom = dev->mt76.eeprom.data; + u16 val = get_unaligned_le16(eeprom); + + switch (val) { + case 0x7990: + return 0; + default: + return -EINVAL; + } +} + +static char *mt7996_eeprom_name(struct mt7996_dev *dev) +{ + /* reserve for future variants */ + return MT7996_EEPROM_DEFAULT; +} + +static int +mt7996_eeprom_load_default(struct mt7996_dev *dev) +{ + u8 *eeprom = dev->mt76.eeprom.data; + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, mt7996_eeprom_name(dev), dev->mt76.dev); + if (ret) + return ret; + + if (!fw || !fw->data) { + dev_err(dev->mt76.dev, "Invalid default bin\n"); + ret = -EINVAL; + goto out; + } + + memcpy(eeprom, fw->data, MT7996_EEPROM_SIZE); + dev->flash_mode = true; + +out: + release_firmware(fw); + + return ret; +} + +static int mt7996_eeprom_load(struct mt7996_dev *dev) +{ + int ret; + + ret = mt76_eeprom_init(&dev->mt76, MT7996_EEPROM_SIZE); + if (ret < 0) + return ret; + + if (ret) { + dev->flash_mode = true; + } else { + u8 free_block_num; + u32 block_num, i; + + /* TODO: check free block event */ + mt7996_mcu_get_eeprom_free_block(dev, &free_block_num); + /* efuse info not enough */ + if (free_block_num >= 59) + return -EINVAL; + + /* read eeprom data from efuse */ + block_num = DIV_ROUND_UP(MT7996_EEPROM_SIZE, MT7996_EEPROM_BLOCK_SIZE); + for (i = 0; i < block_num; i++) + mt7996_mcu_get_eeprom(dev, i * MT7996_EEPROM_BLOCK_SIZE); + } + + return mt7996_check_eeprom(dev); +} + +static int mt7996_eeprom_parse_band_config(struct mt7996_phy *phy) +{ + u8 *eeprom = phy->dev->mt76.eeprom.data; + u32 val = eeprom[MT_EE_WIFI_CONF]; + int ret = 0; + + switch (phy->mt76->band_idx) { + case MT_BAND1: + val = FIELD_GET(MT_EE_WIFI_CONF1_BAND_SEL, val); + break; + case MT_BAND2: + val = eeprom[MT_EE_WIFI_CONF + 1]; + val = FIELD_GET(MT_EE_WIFI_CONF2_BAND_SEL, val); + break; + default: + val = FIELD_GET(MT_EE_WIFI_CONF0_BAND_SEL, val); + break; + } + + switch (val) { + case MT_EE_BAND_SEL_2GHZ: + phy->mt76->cap.has_2ghz = true; + break; + case MT_EE_BAND_SEL_5GHZ: + phy->mt76->cap.has_5ghz = true; + break; + case MT_EE_BAND_SEL_6GHZ: + phy->mt76->cap.has_6ghz = true; + break; + case MT_EE_BAND_SEL_5GHZ_6GHZ: + phy->mt76->cap.has_5ghz = true; + phy->mt76->cap.has_6ghz = true; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +int mt7996_eeprom_parse_hw_cap(struct mt7996_dev *dev, struct mt7996_phy *phy) +{ + u8 path, nss, band_idx = phy->mt76->band_idx; + u8 *eeprom = dev->mt76.eeprom.data; + struct mt76_phy *mphy = phy->mt76; + + switch (band_idx) { + case MT_BAND1: + path = FIELD_GET(MT_EE_WIFI_CONF2_TX_PATH_BAND1, + eeprom[MT_EE_WIFI_CONF + 2]); + nss = FIELD_GET(MT_EE_WIFI_CONF5_STREAM_NUM_BAND1, + eeprom[MT_EE_WIFI_CONF + 5]); + break; + case MT_BAND2: + path = FIELD_GET(MT_EE_WIFI_CONF2_TX_PATH_BAND2, + eeprom[MT_EE_WIFI_CONF + 2]); + nss = FIELD_GET(MT_EE_WIFI_CONF5_STREAM_NUM_BAND2, + eeprom[MT_EE_WIFI_CONF + 5]); + break; + default: + path = FIELD_GET(MT_EE_WIFI_CONF1_TX_PATH_BAND0, + eeprom[MT_EE_WIFI_CONF + 1]); + nss = FIELD_GET(MT_EE_WIFI_CONF4_STREAM_NUM_BAND0, + eeprom[MT_EE_WIFI_CONF + 4]); + break; + } + + if (!path || path > 4) + path = 4; + + nss = min_t(u8, min_t(u8, 4, nss), path); + + mphy->antenna_mask = BIT(nss) - 1; + mphy->chainmask = (BIT(path) - 1) << dev->chainshift[band_idx]; + dev->chainmask |= mphy->chainmask; + if (band_idx < MT_BAND2) + dev->chainshift[band_idx + 1] = dev->chainshift[band_idx] + + hweight16(mphy->chainmask); + + return mt7996_eeprom_parse_band_config(phy); +} + +int mt7996_eeprom_init(struct mt7996_dev *dev) +{ + int ret; + + ret = mt7996_eeprom_load(dev); + if (ret < 0) { + if (ret != -EINVAL) + return ret; + + dev_warn(dev->mt76.dev, "eeprom load fail, use default bin\n"); + ret = mt7996_eeprom_load_default(dev); + if (ret) + return ret; + } + + ret = mt7996_eeprom_parse_hw_cap(dev, &dev->phy); + if (ret < 0) + return ret; + + memcpy(dev->mphy.macaddr, dev->mt76.eeprom.data + MT_EE_MAC_ADDR, ETH_ALEN); + mt76_eeprom_override(&dev->mphy); + + return 0; +} + +int mt7996_eeprom_get_target_power(struct mt7996_dev *dev, + struct ieee80211_channel *chan) +{ + u8 *eeprom = dev->mt76.eeprom.data; + int target_power; + + if (chan->band == NL80211_BAND_5GHZ) + target_power = eeprom[MT_EE_TX0_POWER_5G + + mt7996_get_channel_group_5g(chan->hw_value)]; + else if (chan->band == NL80211_BAND_6GHZ) + target_power = eeprom[MT_EE_TX0_POWER_6G + + mt7996_get_channel_group_6g(chan->hw_value)]; + else + target_power = eeprom[MT_EE_TX0_POWER_2G]; + + return target_power; +} + +s8 mt7996_eeprom_get_power_delta(struct mt7996_dev *dev, int band) +{ + u8 *eeprom = dev->mt76.eeprom.data; + u32 val; + s8 delta; + + if (band == NL80211_BAND_5GHZ) + val = eeprom[MT_EE_RATE_DELTA_5G]; + else if (band == NL80211_BAND_6GHZ) + val = eeprom[MT_EE_RATE_DELTA_6G]; + else + val = eeprom[MT_EE_RATE_DELTA_2G]; + + if (!(val & MT_EE_RATE_DELTA_EN)) + return 0; + + delta = FIELD_GET(MT_EE_RATE_DELTA_MASK, val); + + return val & MT_EE_RATE_DELTA_SIGN ? delta : -delta; +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h new file mode 100644 index 000000000000..8da599e0abea --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: ISC */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef __MT7996_EEPROM_H +#define __MT7996_EEPROM_H + +#include "mt7996.h" + +enum mt7996_eeprom_field { + MT_EE_CHIP_ID = 0x000, + MT_EE_VERSION = 0x002, + MT_EE_MAC_ADDR = 0x004, + MT_EE_MAC_ADDR2 = 0x00a, + MT_EE_WIFI_CONF = 0x190, + MT_EE_MAC_ADDR3 = 0x2c0, + MT_EE_RATE_DELTA_2G = 0x1400, + MT_EE_RATE_DELTA_5G = 0x147d, + MT_EE_RATE_DELTA_6G = 0x154a, + MT_EE_TX0_POWER_2G = 0x1300, + MT_EE_TX0_POWER_5G = 0x1301, + MT_EE_TX0_POWER_6G = 0x1310, + + __MT_EE_MAX = 0x1dff, +}; + +#define MT_EE_WIFI_CONF0_TX_PATH GENMASK(2, 0) +#define MT_EE_WIFI_CONF0_BAND_SEL GENMASK(2, 0) +#define MT_EE_WIFI_CONF1_BAND_SEL GENMASK(5, 3) +#define MT_EE_WIFI_CONF2_BAND_SEL GENMASK(2, 0) + +#define MT_EE_WIFI_CONF1_TX_PATH_BAND0 GENMASK(5, 3) +#define MT_EE_WIFI_CONF2_TX_PATH_BAND1 GENMASK(5, 3) +#define MT_EE_WIFI_CONF2_TX_PATH_BAND2 GENMASK(2, 0) +#define MT_EE_WIFI_CONF4_STREAM_NUM_BAND0 GENMASK(5, 3) +#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND1 GENMASK(5, 3) +#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND2 GENMASK(2, 0) + +#define MT_EE_RATE_DELTA_MASK GENMASK(5, 0) +#define MT_EE_RATE_DELTA_SIGN BIT(6) +#define MT_EE_RATE_DELTA_EN BIT(7) + +enum mt7996_eeprom_band { + MT_EE_BAND_SEL_DEFAULT, + MT_EE_BAND_SEL_2GHZ, + MT_EE_BAND_SEL_5GHZ, + MT_EE_BAND_SEL_6GHZ, + MT_EE_BAND_SEL_5GHZ_6GHZ, +}; + +static inline int +mt7996_get_channel_group_5g(int channel) +{ + if (channel <= 64) + return 0; + if (channel <= 96) + return 1; + if (channel <= 128) + return 2; + if (channel <= 144) + return 3; + return 4; +} + +static inline int +mt7996_get_channel_group_6g(int channel) +{ + if (channel <= 29) + return 0; + + return DIV_ROUND_UP(channel - 29, 32); +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c new file mode 100644 index 000000000000..46b290526092 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/etherdevice.h> +#include <linux/thermal.h> +#include "mt7996.h" +#include "mac.h" +#include "mcu.h" +#include "eeprom.h" + +static const struct ieee80211_iface_limit if_limits[] = { + { + .max = 1, + .types = BIT(NL80211_IFTYPE_ADHOC) + }, { + .max = 16, + .types = BIT(NL80211_IFTYPE_AP) +#ifdef CONFIG_MAC80211_MESH + | BIT(NL80211_IFTYPE_MESH_POINT) +#endif + }, { + .max = MT7996_MAX_INTERFACES, + .types = BIT(NL80211_IFTYPE_STATION) + } +}; + +static const struct ieee80211_iface_combination if_comb[] = { + { + .limits = if_limits, + .n_limits = ARRAY_SIZE(if_limits), + .max_interfaces = MT7996_MAX_INTERFACES, + .num_different_channels = 1, + .beacon_int_infra_match = true, + .radar_detect_widths = BIT(NL80211_CHAN_WIDTH_20_NOHT) | + BIT(NL80211_CHAN_WIDTH_20) | + BIT(NL80211_CHAN_WIDTH_40) | + BIT(NL80211_CHAN_WIDTH_80) | + BIT(NL80211_CHAN_WIDTH_160) | + BIT(NL80211_CHAN_WIDTH_80P80), + } +}; + +static void mt7996_led_set_config(struct led_classdev *led_cdev, + u8 delay_on, u8 delay_off) +{ + struct mt7996_dev *dev; + struct mt76_dev *mt76; + u32 val; + + mt76 = container_of(led_cdev, struct mt76_dev, led_cdev); + dev = container_of(mt76, struct mt7996_dev, mt76); + + /* select TX blink mode, 2: only data frames */ + mt76_rmw_field(dev, MT_TMAC_TCR0(0), MT_TMAC_TCR0_TX_BLINK, 2); + + /* enable LED */ + mt76_wr(dev, MT_LED_EN(0), 1); + + /* set LED Tx blink on/off time */ + val = FIELD_PREP(MT_LED_TX_BLINK_ON_MASK, delay_on) | + FIELD_PREP(MT_LED_TX_BLINK_OFF_MASK, delay_off); + mt76_wr(dev, MT_LED_TX_BLINK(0), val); + + /* control LED */ + val = MT_LED_CTRL_BLINK_MODE | MT_LED_CTRL_KICK; + if (dev->mt76.led_al) + val |= MT_LED_CTRL_POLARITY; + + mt76_wr(dev, MT_LED_CTRL(0), val); + mt76_clear(dev, MT_LED_CTRL(0), MT_LED_CTRL_KICK); +} + +static int mt7996_led_set_blink(struct led_classdev *led_cdev, + unsigned long *delay_on, + unsigned long *delay_off) +{ + u16 delta_on = 0, delta_off = 0; + +#define HW_TICK 10 +#define TO_HW_TICK(_t) (((_t) > HW_TICK) ? ((_t) / HW_TICK) : HW_TICK) + + if (*delay_on) + delta_on = TO_HW_TICK(*delay_on); + if (*delay_off) + delta_off = TO_HW_TICK(*delay_off); + + mt7996_led_set_config(led_cdev, delta_on, delta_off); + + return 0; +} + +static void mt7996_led_set_brightness(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + if (!brightness) + mt7996_led_set_config(led_cdev, 0, 0xff); + else + mt7996_led_set_config(led_cdev, 0xff, 0); +} + +static void +mt7996_init_txpower(struct mt7996_dev *dev, + struct ieee80211_supported_band *sband) +{ + int i, nss = hweight8(dev->mphy.antenna_mask); + int nss_delta = mt76_tx_power_nss_delta(nss); + int pwr_delta = mt7996_eeprom_get_power_delta(dev, sband->band); + struct mt76_power_limits limits; + + for (i = 0; i < sband->n_channels; i++) { + struct ieee80211_channel *chan = &sband->channels[i]; + int target_power = mt7996_eeprom_get_target_power(dev, chan); + + target_power += pwr_delta; + target_power = mt76_get_rate_power_limits(&dev->mphy, chan, + &limits, + target_power); + target_power += nss_delta; + target_power = DIV_ROUND_UP(target_power, 2); + chan->max_power = min_t(int, chan->max_reg_power, + target_power); + chan->orig_mpwr = target_power; + } +} + +static void +mt7996_regd_notifier(struct wiphy *wiphy, + struct regulatory_request *request) +{ + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + + memcpy(dev->mt76.alpha2, request->alpha2, sizeof(dev->mt76.alpha2)); + dev->mt76.region = request->dfs_region; + + if (dev->mt76.region == NL80211_DFS_UNSET) + mt7996_mcu_rdd_background_enable(phy, NULL); + + mt7996_init_txpower(dev, &phy->mt76->sband_2g.sband); + mt7996_init_txpower(dev, &phy->mt76->sband_5g.sband); + mt7996_init_txpower(dev, &phy->mt76->sband_6g.sband); + + phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN; + mt7996_dfs_init_radar_detector(phy); +} + +static void +mt7996_init_wiphy(struct ieee80211_hw *hw) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt76_dev *mdev = &phy->dev->mt76; + struct wiphy *wiphy = hw->wiphy; + + hw->queues = 4; + hw->max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; + hw->max_tx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; + hw->netdev_features = NETIF_F_RXCSUM; + + hw->radiotap_timestamp.units_pos = + IEEE80211_RADIOTAP_TIMESTAMP_UNIT_US; + + phy->slottime = 9; + + hw->sta_data_size = sizeof(struct mt7996_sta); + hw->vif_data_size = sizeof(struct mt7996_vif); + + wiphy->iface_combinations = if_comb; + wiphy->n_iface_combinations = ARRAY_SIZE(if_comb); + wiphy->reg_notifier = mt7996_regd_notifier; + wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH; + + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BSS_COLOR); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_LEGACY); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_HT); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_VHT); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_HE); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_DISCOVERY); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT); + + if (!mdev->dev->of_node || + !of_property_read_bool(mdev->dev->of_node, + "mediatek,disable-radar-background")) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_RADAR_BACKGROUND); + + ieee80211_hw_set(hw, HAS_RATE_CONTROL); + ieee80211_hw_set(hw, SUPPORTS_TX_ENCAP_OFFLOAD); + ieee80211_hw_set(hw, SUPPORTS_RX_DECAP_OFFLOAD); + ieee80211_hw_set(hw, WANT_MONITOR_VIF); + + hw->max_tx_fragments = 4; + + if (phy->mt76->cap.has_2ghz) + phy->mt76->sband_2g.sband.ht_cap.cap |= + IEEE80211_HT_CAP_LDPC_CODING | + IEEE80211_HT_CAP_MAX_AMSDU; + + if (phy->mt76->cap.has_5ghz) { + phy->mt76->sband_5g.sband.ht_cap.cap |= + IEEE80211_HT_CAP_LDPC_CODING | + IEEE80211_HT_CAP_MAX_AMSDU; + + phy->mt76->sband_5g.sband.vht_cap.cap |= + IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 | + IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK | + IEEE80211_VHT_CAP_SHORT_GI_160 | + IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ; + } + + mt76_set_stream_caps(phy->mt76, true); + mt7996_set_stream_vht_txbf_caps(phy); + mt7996_set_stream_he_caps(phy); + + wiphy->available_antennas_rx = phy->mt76->antenna_mask; + wiphy->available_antennas_tx = phy->mt76->antenna_mask; +} + +static void +mt7996_mac_init_band(struct mt7996_dev *dev, u8 band) +{ + u32 mask, set; + + /* clear estimated value of EIFS for Rx duration & OBSS time */ + mt76_wr(dev, MT_WF_RMAC_RSVD0(band), MT_WF_RMAC_RSVD0_EIFS_CLR); + + /* clear backoff time for Rx duration */ + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME1(band), + MT_WF_RMAC_MIB_NONQOSD_BACKOFF); + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME3(band), + MT_WF_RMAC_MIB_QOS01_BACKOFF); + mt76_clear(dev, MT_WF_RMAC_MIB_AIRTIME4(band), + MT_WF_RMAC_MIB_QOS23_BACKOFF); + + /* clear backoff time and set software compensation for OBSS time */ + mask = MT_WF_RMAC_MIB_OBSS_BACKOFF | MT_WF_RMAC_MIB_ED_OFFSET; + set = FIELD_PREP(MT_WF_RMAC_MIB_OBSS_BACKOFF, 0) | + FIELD_PREP(MT_WF_RMAC_MIB_ED_OFFSET, 4); + mt76_rmw(dev, MT_WF_RMAC_MIB_AIRTIME0(band), mask, set); + + /* filter out non-resp frames and get instanstaeous signal reporting */ + mask = MT_WTBLOFF_RSCR_RCPI_MODE | MT_WTBLOFF_RSCR_RCPI_PARAM; + set = FIELD_PREP(MT_WTBLOFF_RSCR_RCPI_MODE, 0) | + FIELD_PREP(MT_WTBLOFF_RSCR_RCPI_PARAM, 0x3); + mt76_rmw(dev, MT_WTBLOFF_RSCR(band), mask, set); +} + +static void mt7996_mac_init(struct mt7996_dev *dev) +{ +#define HIF_TXD_V2_1 4 + int i; + + mt76_clear(dev, MT_MDP_DCR2, MT_MDP_DCR2_RX_TRANS_SHORT); + + for (i = 0; i < MT7996_WTBL_SIZE; i++) + mt7996_mac_wtbl_update(dev, i, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + + if (IS_ENABLED(CONFIG_MT76_LEDS)) { + i = dev->mt76.led_pin ? MT_LED_GPIO_MUX3 : MT_LED_GPIO_MUX2; + mt76_rmw_field(dev, i, MT_LED_GPIO_SEL_MASK, 4); + } + + /* txs report queue */ + mt76_rmw_field(dev, MT_DMA_TCRF1(0), MT_DMA_TCRF1_QIDX, 0); + mt76_rmw_field(dev, MT_DMA_TCRF1(1), MT_DMA_TCRF1_QIDX, 6); + mt76_rmw_field(dev, MT_DMA_TCRF1(2), MT_DMA_TCRF1_QIDX, 0); + + /* rro module init */ + mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, 2); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 3); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 1); + + mt7996_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(SET), + MCU_WA_PARAM_HW_PATH_HIF_VER, + HIF_TXD_V2_1, 0); + + for (i = MT_BAND0; i <= MT_BAND2; i++) + mt7996_mac_init_band(dev, i); +} + +static int mt7996_txbf_init(struct mt7996_dev *dev) +{ + int ret; + + if (dev->dbdc_support) { + ret = mt7996_mcu_set_txbf(dev, BF_MOD_EN_CTRL); + if (ret) + return ret; + } + + /* trigger sounding packets */ + ret = mt7996_mcu_set_txbf(dev, BF_SOUNDING_ON); + if (ret) + return ret; + + /* enable eBF */ + return mt7996_mcu_set_txbf(dev, BF_HW_EN_UPDATE); +} + +static int mt7996_register_phy(struct mt7996_dev *dev, struct mt7996_phy *phy, + enum mt76_band_id band) +{ + struct mt76_phy *mphy; + u32 mac_ofs, hif1_ofs = 0; + int ret; + + if (band != MT_BAND1 && band != MT_BAND2) + return 0; + + if ((band == MT_BAND1 && !dev->dbdc_support) || + (band == MT_BAND2 && !dev->tbtc_support)) + return 0; + + if (phy) + return 0; + + if (band == MT_BAND2 && dev->hif2) + hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + + mphy = mt76_alloc_phy(&dev->mt76, sizeof(*phy), &mt7996_ops, band); + if (!mphy) + return -ENOMEM; + + phy = mphy->priv; + phy->dev = dev; + phy->mt76 = mphy; + mphy->dev->phys[band] = mphy; + + INIT_DELAYED_WORK(&mphy->mac_work, mt7996_mac_work); + + ret = mt7996_eeprom_parse_hw_cap(dev, phy); + if (ret) + goto error; + + mac_ofs = band == MT_BAND2 ? MT_EE_MAC_ADDR3 : MT_EE_MAC_ADDR2; + memcpy(mphy->macaddr, dev->mt76.eeprom.data + mac_ofs, ETH_ALEN); + /* Make the extra PHY MAC address local without overlapping with + * the usual MAC address allocation scheme on multiple virtual interfaces + */ + if (!is_valid_ether_addr(mphy->macaddr)) { + memcpy(mphy->macaddr, dev->mt76.eeprom.data + MT_EE_MAC_ADDR, + ETH_ALEN); + mphy->macaddr[0] |= 2; + mphy->macaddr[0] ^= BIT(7); + if (band == MT_BAND2) + mphy->macaddr[0] ^= BIT(6); + } + mt76_eeprom_override(mphy); + + /* init wiphy according to mphy and phy */ + mt7996_init_wiphy(mphy->hw); + ret = mt76_connac_init_tx_queues(phy->mt76, + MT_TXQ_ID(band), + MT7996_TX_RING_SIZE, + MT_TXQ_RING_BASE(band) + hif1_ofs, 0); + if (ret) + goto error; + + ret = mt76_register_phy(mphy, true, mt76_rates, + ARRAY_SIZE(mt76_rates)); + if (ret) + goto error; + + ret = mt7996_init_debugfs(phy); + if (ret) + goto error; + + return 0; + +error: + mphy->dev->phys[band] = NULL; + ieee80211_free_hw(mphy->hw); + return ret; +} + +static void +mt7996_unregister_phy(struct mt7996_phy *phy, enum mt76_band_id band) +{ + struct mt76_phy *mphy; + + if (!phy) + return; + + mphy = phy->dev->mt76.phys[band]; + mt76_unregister_phy(mphy); + ieee80211_free_hw(mphy->hw); + phy->dev->mt76.phys[band] = NULL; +} + +static void mt7996_init_work(struct work_struct *work) +{ + struct mt7996_dev *dev = container_of(work, struct mt7996_dev, + init_work); + + mt7996_mcu_set_eeprom(dev); + mt7996_mac_init(dev); + mt7996_init_txpower(dev, &dev->mphy.sband_2g.sband); + mt7996_init_txpower(dev, &dev->mphy.sband_5g.sband); + mt7996_init_txpower(dev, &dev->mphy.sband_6g.sband); + mt7996_txbf_init(dev); +} + +void mt7996_wfsys_reset(struct mt7996_dev *dev) +{ + mt76_set(dev, MT_WF_SUBSYS_RST, 0x1); + msleep(20); + + mt76_clear(dev, MT_WF_SUBSYS_RST, 0x1); + msleep(20); +} + +static int mt7996_init_hardware(struct mt7996_dev *dev) +{ + int ret, idx; + + mt76_wr(dev, MT_INT_SOURCE_CSR, ~0); + + INIT_WORK(&dev->init_work, mt7996_init_work); + + dev->dbdc_support = true; + dev->tbtc_support = true; + + ret = mt7996_dma_init(dev); + if (ret) + return ret; + + set_bit(MT76_STATE_INITIALIZED, &dev->mphy.state); + + ret = mt7996_mcu_init(dev); + if (ret) + return ret; + + ret = mt7996_eeprom_init(dev); + if (ret < 0) + return ret; + + /* Beacon and mgmt frames should occupy wcid 0 */ + idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7996_WTBL_STA); + if (idx) + return -ENOSPC; + + dev->mt76.global_wcid.idx = idx; + dev->mt76.global_wcid.hw_key_idx = -1; + dev->mt76.global_wcid.tx_info |= MT_WCID_TX_INFO_SET; + rcu_assign_pointer(dev->mt76.wcid[idx], &dev->mt76.global_wcid); + + return 0; +} + +void mt7996_set_stream_vht_txbf_caps(struct mt7996_phy *phy) +{ + int sts; + u32 *cap; + + if (!phy->mt76->cap.has_5ghz) + return; + + sts = hweight16(phy->mt76->chainmask); + cap = &phy->mt76->sband_5g.sband.vht_cap.cap; + + *cap |= IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE | + IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE | + (3 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT); + + *cap &= ~(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK | + IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE | + IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE); + + if (sts < 2) + return; + + *cap |= IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE | + IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE | + FIELD_PREP(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK, sts - 1); +} + +static void +mt7996_set_stream_he_txbf_caps(struct mt7996_phy *phy, + struct ieee80211_sta_he_cap *he_cap, int vif) +{ + struct ieee80211_he_cap_elem *elem = &he_cap->he_cap_elem; + int sts = hweight16(phy->mt76->chainmask); + u8 c; + +#ifdef CONFIG_MAC80211_MESH + if (vif == NL80211_IFTYPE_MESH_POINT) + return; +#endif + + elem->phy_cap_info[3] &= ~IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; + elem->phy_cap_info[4] &= ~IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER; + + c = IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK | + IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK; + elem->phy_cap_info[5] &= ~c; + + c = IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | + IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB; + elem->phy_cap_info[6] &= ~c; + + elem->phy_cap_info[7] &= ~IEEE80211_HE_PHY_CAP7_MAX_NC_MASK; + + c = IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO; + elem->phy_cap_info[2] |= c; + + c = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4 | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; + elem->phy_cap_info[4] |= c; + + /* do not support NG16 due to spec D4.0 changes subcarrier idx */ + c = IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU | + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU; + + if (vif == NL80211_IFTYPE_STATION) + c |= IEEE80211_HE_PHY_CAP6_PARTIAL_BANDWIDTH_DL_MUMIMO; + + elem->phy_cap_info[6] |= c; + + if (sts < 2) + return; + + /* the maximum cap is 4 x 3, (Nr, Nc) = (3, 2) */ + elem->phy_cap_info[7] |= min_t(int, sts - 1, 2) << 3; + + if (vif != NL80211_IFTYPE_AP) + return; + + elem->phy_cap_info[3] |= IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; + elem->phy_cap_info[4] |= IEEE80211_HE_PHY_CAP4_MU_BEAMFORMER; + + c = FIELD_PREP(IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK, + sts - 1) | + FIELD_PREP(IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK, + sts - 1); + elem->phy_cap_info[5] |= c; + + c = IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | + IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB; + elem->phy_cap_info[6] |= c; + + c = IEEE80211_HE_PHY_CAP7_STBC_TX_ABOVE_80MHZ | + IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ; + elem->phy_cap_info[7] |= c; +} + +static void +mt7996_gen_ppe_thresh(u8 *he_ppet, int nss) +{ + u8 i, ppet_bits, ppet_size, ru_bit_mask = 0x7; /* HE80 */ + static const u8 ppet16_ppet8_ru3_ru0[] = {0x1c, 0xc7, 0x71}; + + he_ppet[0] = FIELD_PREP(IEEE80211_PPE_THRES_NSS_MASK, nss - 1) | + FIELD_PREP(IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK, + ru_bit_mask); + + ppet_bits = IEEE80211_PPE_THRES_INFO_PPET_SIZE * + nss * hweight8(ru_bit_mask) * 2; + ppet_size = DIV_ROUND_UP(ppet_bits, 8); + + for (i = 0; i < ppet_size - 1; i++) + he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3]; + + he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3] & + (0xff >> (8 - (ppet_bits - 1) % 8)); +} + +static int +mt7996_init_he_caps(struct mt7996_phy *phy, enum nl80211_band band, + struct ieee80211_sband_iftype_data *data) +{ + int i, idx = 0, nss = hweight8(phy->mt76->antenna_mask); + u16 mcs_map = 0; + + for (i = 0; i < 8; i++) { + if (i < nss) + mcs_map |= (IEEE80211_HE_MCS_SUPPORT_0_11 << (i * 2)); + else + mcs_map |= (IEEE80211_HE_MCS_NOT_SUPPORTED << (i * 2)); + } + + for (i = 0; i < NUM_NL80211_IFTYPES; i++) { + struct ieee80211_sta_he_cap *he_cap = &data[idx].he_cap; + struct ieee80211_he_cap_elem *he_cap_elem = + &he_cap->he_cap_elem; + struct ieee80211_he_mcs_nss_supp *he_mcs = + &he_cap->he_mcs_nss_supp; + + switch (i) { + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_AP: +#ifdef CONFIG_MAC80211_MESH + case NL80211_IFTYPE_MESH_POINT: +#endif + break; + default: + continue; + } + + data[idx].types_mask = BIT(i); + he_cap->has_he = true; + + he_cap_elem->mac_cap_info[0] = + IEEE80211_HE_MAC_CAP0_HTC_HE; + he_cap_elem->mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3; + he_cap_elem->mac_cap_info[4] = + IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU; + + if (band == NL80211_BAND_2GHZ) + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G; + else + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G; + + he_cap_elem->phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD; + he_cap_elem->phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ; + + switch (i) { + case NL80211_IFTYPE_AP: + he_cap_elem->mac_cap_info[0] |= + IEEE80211_HE_MAC_CAP0_TWT_RES; + he_cap_elem->mac_cap_info[2] |= + IEEE80211_HE_MAC_CAP2_BSR; + he_cap_elem->mac_cap_info[4] |= + IEEE80211_HE_MAC_CAP4_BQR; + he_cap_elem->mac_cap_info[5] |= + IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX; + he_cap_elem->phy_cap_info[3] |= + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK | + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_QPSK; + he_cap_elem->phy_cap_info[6] |= + IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE | + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT; + he_cap_elem->phy_cap_info[9] |= + IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU; + break; + case NL80211_IFTYPE_STATION: + he_cap_elem->mac_cap_info[1] |= + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; + + if (band == NL80211_BAND_2GHZ) + he_cap_elem->phy_cap_info[0] |= + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_2G; + else + he_cap_elem->phy_cap_info[0] |= + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_5G; + + he_cap_elem->phy_cap_info[1] |= + IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | + IEEE80211_HE_PHY_CAP1_HE_LTF_AND_GI_FOR_HE_PPDUS_0_8US; + he_cap_elem->phy_cap_info[3] |= + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK | + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_QPSK; + he_cap_elem->phy_cap_info[6] |= + IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB | + IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE | + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT; + he_cap_elem->phy_cap_info[7] |= + IEEE80211_HE_PHY_CAP7_POWER_BOOST_FACTOR_SUPP | + IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI; + he_cap_elem->phy_cap_info[8] |= + IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_484; + he_cap_elem->phy_cap_info[9] |= + IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM | + IEEE80211_HE_PHY_CAP9_NON_TRIGGERED_CQI_FEEDBACK | + IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB; + break; + } + + he_mcs->rx_mcs_80 = cpu_to_le16(mcs_map); + he_mcs->tx_mcs_80 = cpu_to_le16(mcs_map); + he_mcs->rx_mcs_160 = cpu_to_le16(mcs_map); + he_mcs->tx_mcs_160 = cpu_to_le16(mcs_map); + he_mcs->rx_mcs_80p80 = cpu_to_le16(mcs_map); + he_mcs->tx_mcs_80p80 = cpu_to_le16(mcs_map); + + mt7996_set_stream_he_txbf_caps(phy, he_cap, i); + + memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres)); + if (he_cap_elem->phy_cap_info[6] & + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) { + mt7996_gen_ppe_thresh(he_cap->ppe_thres, nss); + } else { + he_cap_elem->phy_cap_info[9] |= + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US; + } + + if (band == NL80211_BAND_6GHZ) { + u16 cap = IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS | + IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS; + + cap |= u16_encode_bits(IEEE80211_HT_MPDU_DENSITY_2, + IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START) | + u16_encode_bits(IEEE80211_VHT_MAX_AMPDU_1024K, + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | + u16_encode_bits(IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454, + IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN); + + data[idx].he_6ghz_capa.capa = cpu_to_le16(cap); + } + + idx++; + } + + return idx; +} + +void mt7996_set_stream_he_caps(struct mt7996_phy *phy) +{ + struct ieee80211_sband_iftype_data *data; + struct ieee80211_supported_band *band; + int n; + + if (phy->mt76->cap.has_2ghz) { + data = phy->iftype[NL80211_BAND_2GHZ]; + n = mt7996_init_he_caps(phy, NL80211_BAND_2GHZ, data); + + band = &phy->mt76->sband_2g.sband; + band->iftype_data = data; + band->n_iftype_data = n; + } + + if (phy->mt76->cap.has_5ghz) { + data = phy->iftype[NL80211_BAND_5GHZ]; + n = mt7996_init_he_caps(phy, NL80211_BAND_5GHZ, data); + + band = &phy->mt76->sband_5g.sband; + band->iftype_data = data; + band->n_iftype_data = n; + } + + if (phy->mt76->cap.has_6ghz) { + data = phy->iftype[NL80211_BAND_6GHZ]; + n = mt7996_init_he_caps(phy, NL80211_BAND_6GHZ, data); + + band = &phy->mt76->sband_6g.sband; + band->iftype_data = data; + band->n_iftype_data = n; + } +} + +int mt7996_register_device(struct mt7996_dev *dev) +{ + struct ieee80211_hw *hw = mt76_hw(dev); + int ret; + + dev->phy.dev = dev; + dev->phy.mt76 = &dev->mt76.phy; + dev->mt76.phy.priv = &dev->phy; + INIT_WORK(&dev->rc_work, mt7996_mac_sta_rc_work); + INIT_DELAYED_WORK(&dev->mphy.mac_work, mt7996_mac_work); + INIT_LIST_HEAD(&dev->sta_rc_list); + INIT_LIST_HEAD(&dev->sta_poll_list); + INIT_LIST_HEAD(&dev->twt_list); + spin_lock_init(&dev->sta_poll_lock); + + init_waitqueue_head(&dev->reset_wait); + INIT_WORK(&dev->reset_work, mt7996_mac_reset_work); + + ret = mt7996_init_hardware(dev); + if (ret) + return ret; + + mt7996_init_wiphy(hw); + + /* init led callbacks */ + if (IS_ENABLED(CONFIG_MT76_LEDS)) { + dev->mt76.led_cdev.brightness_set = mt7996_led_set_brightness; + dev->mt76.led_cdev.blink_set = mt7996_led_set_blink; + } + + ret = mt76_register_device(&dev->mt76, true, mt76_rates, + ARRAY_SIZE(mt76_rates)); + if (ret) + return ret; + + ieee80211_queue_work(mt76_hw(dev), &dev->init_work); + + ret = mt7996_register_phy(dev, mt7996_phy2(dev), MT_BAND1); + if (ret) + return ret; + + ret = mt7996_register_phy(dev, mt7996_phy3(dev), MT_BAND2); + if (ret) + return ret; + + return mt7996_init_debugfs(&dev->phy); +} + +void mt7996_unregister_device(struct mt7996_dev *dev) +{ + mt7996_unregister_phy(mt7996_phy3(dev), MT_BAND2); + mt7996_unregister_phy(mt7996_phy2(dev), MT_BAND1); + mt76_unregister_device(&dev->mt76); + mt7996_mcu_exit(dev); + mt7996_tx_token_put(dev); + mt7996_dma_cleanup(dev); + tasklet_disable(&dev->irq_tasklet); + + mt76_free_device(&dev->mt76); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c new file mode 100644 index 000000000000..0b3e28748e76 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -0,0 +1,2498 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/etherdevice.h> +#include <linux/timekeeping.h> +#include "mt7996.h" +#include "../dma.h" +#include "mac.h" +#include "mcu.h" + +#define to_rssi(field, rcpi) ((FIELD_GET(field, rcpi) - 220) / 2) + +#define HE_BITS(f) cpu_to_le16(IEEE80211_RADIOTAP_HE_##f) +#define HE_PREP(f, m, v) le16_encode_bits(le32_get_bits(v, MT_CRXV_HE_##m),\ + IEEE80211_RADIOTAP_HE_##f) + +static const struct mt7996_dfs_radar_spec etsi_radar_specs = { + .pulse_th = { 110, -10, -80, 40, 5200, 128, 5200 }, + .radar_pattern = { + [5] = { 1, 0, 6, 32, 28, 0, 990, 5010, 17, 1, 1 }, + [6] = { 1, 0, 9, 32, 28, 0, 615, 5010, 27, 1, 1 }, + [7] = { 1, 0, 15, 32, 28, 0, 240, 445, 27, 1, 1 }, + [8] = { 1, 0, 12, 32, 28, 0, 240, 510, 42, 1, 1 }, + [9] = { 1, 1, 0, 0, 0, 0, 2490, 3343, 14, 0, 0, 12, 32, 28, { }, 126 }, + [10] = { 1, 1, 0, 0, 0, 0, 2490, 3343, 14, 0, 0, 15, 32, 24, { }, 126 }, + [11] = { 1, 1, 0, 0, 0, 0, 823, 2510, 14, 0, 0, 18, 32, 28, { }, 54 }, + [12] = { 1, 1, 0, 0, 0, 0, 823, 2510, 14, 0, 0, 27, 32, 24, { }, 54 }, + }, +}; + +static const struct mt7996_dfs_radar_spec fcc_radar_specs = { + .pulse_th = { 110, -10, -80, 40, 5200, 128, 5200 }, + .radar_pattern = { + [0] = { 1, 0, 8, 32, 28, 0, 508, 3076, 13, 1, 1 }, + [1] = { 1, 0, 12, 32, 28, 0, 140, 240, 17, 1, 1 }, + [2] = { 1, 0, 8, 32, 28, 0, 190, 510, 22, 1, 1 }, + [3] = { 1, 0, 6, 32, 28, 0, 190, 510, 32, 1, 1 }, + [4] = { 1, 0, 9, 255, 28, 0, 323, 343, 13, 1, 32 }, + }, +}; + +static const struct mt7996_dfs_radar_spec jp_radar_specs = { + .pulse_th = { 110, -10, -80, 40, 5200, 128, 5200 }, + .radar_pattern = { + [0] = { 1, 0, 8, 32, 28, 0, 508, 3076, 13, 1, 1 }, + [1] = { 1, 0, 12, 32, 28, 0, 140, 240, 17, 1, 1 }, + [2] = { 1, 0, 8, 32, 28, 0, 190, 510, 22, 1, 1 }, + [3] = { 1, 0, 6, 32, 28, 0, 190, 510, 32, 1, 1 }, + [4] = { 1, 0, 9, 255, 28, 0, 323, 343, 13, 1, 32 }, + [13] = { 1, 0, 7, 32, 28, 0, 3836, 3856, 14, 1, 1 }, + [14] = { 1, 0, 6, 32, 28, 0, 615, 5010, 110, 1, 1 }, + [15] = { 1, 1, 0, 0, 0, 0, 15, 5010, 110, 0, 0, 12, 32, 28 }, + }, +}; + +static struct mt76_wcid *mt7996_rx_get_wcid(struct mt7996_dev *dev, + u16 idx, bool unicast) +{ + struct mt7996_sta *sta; + struct mt76_wcid *wcid; + + if (idx >= ARRAY_SIZE(dev->mt76.wcid)) + return NULL; + + wcid = rcu_dereference(dev->mt76.wcid[idx]); + if (unicast || !wcid) + return wcid; + + if (!wcid->sta) + return NULL; + + sta = container_of(wcid, struct mt7996_sta, wcid); + if (!sta->vif) + return NULL; + + return &sta->vif->sta.wcid; +} + +void mt7996_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps) +{ +} + +bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask) +{ + mt76_rmw(dev, MT_WTBL_UPDATE, MT_WTBL_UPDATE_WLAN_IDX, + FIELD_PREP(MT_WTBL_UPDATE_WLAN_IDX, idx) | mask); + + return mt76_poll(dev, MT_WTBL_UPDATE, MT_WTBL_UPDATE_BUSY, + 0, 5000); +} + +u32 mt7996_mac_wtbl_lmac_addr(struct mt7996_dev *dev, u16 wcid, u8 dw) +{ + mt76_wr(dev, MT_WTBLON_TOP_WDUCR, + FIELD_PREP(MT_WTBLON_TOP_WDUCR_GROUP, (wcid >> 7))); + + return MT_WTBL_LMAC_OFFS(wcid, dw); +} + +static void mt7996_mac_sta_poll(struct mt7996_dev *dev) +{ + static const u8 ac_to_tid[] = { + [IEEE80211_AC_BE] = 0, + [IEEE80211_AC_BK] = 1, + [IEEE80211_AC_VI] = 4, + [IEEE80211_AC_VO] = 6 + }; + struct ieee80211_sta *sta; + struct mt7996_sta *msta; + struct rate_info *rate; + u32 tx_time[IEEE80211_NUM_ACS], rx_time[IEEE80211_NUM_ACS]; + LIST_HEAD(sta_poll_list); + int i; + + spin_lock_bh(&dev->sta_poll_lock); + list_splice_init(&dev->sta_poll_list, &sta_poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + + rcu_read_lock(); + + while (true) { + bool clear = false; + u32 addr, val; + u16 idx; + s8 rssi[4]; + u8 bw; + + spin_lock_bh(&dev->sta_poll_lock); + if (list_empty(&sta_poll_list)) { + spin_unlock_bh(&dev->sta_poll_lock); + break; + } + msta = list_first_entry(&sta_poll_list, + struct mt7996_sta, poll_list); + list_del_init(&msta->poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + + idx = msta->wcid.idx; + + /* refresh peer's airtime reporting */ + addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 20); + + for (i = 0; i < IEEE80211_NUM_ACS; i++) { + u32 tx_last = msta->airtime_ac[i]; + u32 rx_last = msta->airtime_ac[i + 4]; + + msta->airtime_ac[i] = mt76_rr(dev, addr); + msta->airtime_ac[i + 4] = mt76_rr(dev, addr + 4); + + tx_time[i] = msta->airtime_ac[i] - tx_last; + rx_time[i] = msta->airtime_ac[i + 4] - rx_last; + + if ((tx_last | rx_last) & BIT(30)) + clear = true; + + addr += 8; + } + + if (clear) { + mt7996_mac_wtbl_update(dev, idx, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + memset(msta->airtime_ac, 0, sizeof(msta->airtime_ac)); + } + + if (!msta->wcid.sta) + continue; + + sta = container_of((void *)msta, struct ieee80211_sta, + drv_priv); + for (i = 0; i < IEEE80211_NUM_ACS; i++) { + u8 q = mt76_connac_lmac_mapping(i); + u32 tx_cur = tx_time[q]; + u32 rx_cur = rx_time[q]; + u8 tid = ac_to_tid[i]; + + if (!tx_cur && !rx_cur) + continue; + + ieee80211_sta_register_airtime(sta, tid, tx_cur, rx_cur); + } + + /* We don't support reading GI info from txs packets. + * For accurate tx status reporting and AQL improvement, + * we need to make sure that flags match so polling GI + * from per-sta counters directly. + */ + rate = &msta->wcid.rate; + + switch (rate->bw) { + case RATE_INFO_BW_160: + bw = IEEE80211_STA_RX_BW_160; + break; + case RATE_INFO_BW_80: + bw = IEEE80211_STA_RX_BW_80; + break; + case RATE_INFO_BW_40: + bw = IEEE80211_STA_RX_BW_40; + break; + default: + bw = IEEE80211_STA_RX_BW_20; + break; + } + + addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 6); + val = mt76_rr(dev, addr); + if (rate->flags & RATE_INFO_FLAGS_HE_MCS) { + u8 offs = 24 + 2 * bw; + + rate->he_gi = (val & (0x3 << offs)) >> offs; + } else if (rate->flags & + (RATE_INFO_FLAGS_VHT_MCS | RATE_INFO_FLAGS_MCS)) { + if (val & BIT(12 + bw)) + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; + else + rate->flags &= ~RATE_INFO_FLAGS_SHORT_GI; + } + + /* get signal strength of resp frames (CTS/BA/ACK) */ + addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 34); + val = mt76_rr(dev, addr); + + rssi[0] = to_rssi(GENMASK(7, 0), val); + rssi[1] = to_rssi(GENMASK(15, 8), val); + rssi[2] = to_rssi(GENMASK(23, 16), val); + rssi[3] = to_rssi(GENMASK(31, 14), val); + + msta->ack_signal = + mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); + + ewma_avg_signal_add(&msta->avg_ack_signal, -msta->ack_signal); + } + + rcu_read_unlock(); +} + +void mt7996_mac_enable_rtscts(struct mt7996_dev *dev, + struct ieee80211_vif *vif, bool enable) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + u32 addr; + + addr = mt7996_mac_wtbl_lmac_addr(dev, mvif->sta.wcid.idx, 5); + if (enable) + mt76_set(dev, addr, BIT(5)); + else + mt76_clear(dev, addr, BIT(5)); +} + +static void +mt7996_mac_decode_he_radiotap_ru(struct mt76_rx_status *status, + struct ieee80211_radiotap_he *he, + __le32 *rxv) +{ + u32 ru_h, ru_l; + u8 ru, offs = 0; + + ru_l = le32_get_bits(rxv[0], MT_PRXV_HE_RU_ALLOC_L); + ru_h = le32_get_bits(rxv[1], MT_PRXV_HE_RU_ALLOC_H); + ru = (u8)(ru_l | ru_h << 4); + + status->bw = RATE_INFO_BW_HE_RU; + + switch (ru) { + case 0 ... 36: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_26; + offs = ru; + break; + case 37 ... 52: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_52; + offs = ru - 37; + break; + case 53 ... 60: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_106; + offs = ru - 53; + break; + case 61 ... 64: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_242; + offs = ru - 61; + break; + case 65 ... 66: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_484; + offs = ru - 65; + break; + case 67: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_996; + break; + case 68: + status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_2x996; + break; + } + + he->data1 |= HE_BITS(DATA1_BW_RU_ALLOC_KNOWN); + he->data2 |= HE_BITS(DATA2_RU_OFFSET_KNOWN) | + le16_encode_bits(offs, + IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET); +} + +static void +mt7996_mac_decode_he_mu_radiotap(struct sk_buff *skb, __le32 *rxv) +{ + struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; + static const struct ieee80211_radiotap_he_mu mu_known = { + .flags1 = HE_BITS(MU_FLAGS1_SIG_B_MCS_KNOWN) | + HE_BITS(MU_FLAGS1_SIG_B_DCM_KNOWN) | + HE_BITS(MU_FLAGS1_CH1_RU_KNOWN) | + HE_BITS(MU_FLAGS1_SIG_B_SYMS_USERS_KNOWN), + .flags2 = HE_BITS(MU_FLAGS2_BW_FROM_SIG_A_BW_KNOWN), + }; + struct ieee80211_radiotap_he_mu *he_mu = NULL; + + status->flag |= RX_FLAG_RADIOTAP_HE_MU; + + he_mu = skb_push(skb, sizeof(mu_known)); + memcpy(he_mu, &mu_known, sizeof(mu_known)); + +#define MU_PREP(f, v) le16_encode_bits(v, IEEE80211_RADIOTAP_HE_MU_##f) + + he_mu->flags1 |= MU_PREP(FLAGS1_SIG_B_MCS, status->rate_idx); + if (status->he_dcm) + he_mu->flags1 |= MU_PREP(FLAGS1_SIG_B_DCM, status->he_dcm); + + he_mu->flags2 |= MU_PREP(FLAGS2_BW_FROM_SIG_A_BW, status->bw) | + MU_PREP(FLAGS2_SIG_B_SYMS_USERS, + le32_get_bits(rxv[2], MT_CRXV_HE_NUM_USER)); + + he_mu->ru_ch1[0] = le32_get_bits(rxv[3], MT_CRXV_HE_RU0); + + if (status->bw >= RATE_INFO_BW_40) { + he_mu->flags1 |= HE_BITS(MU_FLAGS1_CH2_RU_KNOWN); + he_mu->ru_ch2[0] = le32_get_bits(rxv[3], MT_CRXV_HE_RU1); + } + + if (status->bw >= RATE_INFO_BW_80) { + he_mu->ru_ch1[1] = le32_get_bits(rxv[3], MT_CRXV_HE_RU2); + he_mu->ru_ch2[1] = le32_get_bits(rxv[3], MT_CRXV_HE_RU3); + } +} + +static void +mt7996_mac_decode_he_radiotap(struct sk_buff *skb, __le32 *rxv, u8 mode) +{ + struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; + static const struct ieee80211_radiotap_he known = { + .data1 = HE_BITS(DATA1_DATA_MCS_KNOWN) | + HE_BITS(DATA1_DATA_DCM_KNOWN) | + HE_BITS(DATA1_STBC_KNOWN) | + HE_BITS(DATA1_CODING_KNOWN) | + HE_BITS(DATA1_LDPC_XSYMSEG_KNOWN) | + HE_BITS(DATA1_DOPPLER_KNOWN) | + HE_BITS(DATA1_SPTL_REUSE_KNOWN) | + HE_BITS(DATA1_BSS_COLOR_KNOWN), + .data2 = HE_BITS(DATA2_GI_KNOWN) | + HE_BITS(DATA2_TXBF_KNOWN) | + HE_BITS(DATA2_PE_DISAMBIG_KNOWN) | + HE_BITS(DATA2_TXOP_KNOWN), + }; + struct ieee80211_radiotap_he *he = NULL; + u32 ltf_size = le32_get_bits(rxv[2], MT_CRXV_HE_LTF_SIZE) + 1; + + status->flag |= RX_FLAG_RADIOTAP_HE; + + he = skb_push(skb, sizeof(known)); + memcpy(he, &known, sizeof(known)); + + he->data3 = HE_PREP(DATA3_BSS_COLOR, BSS_COLOR, rxv[14]) | + HE_PREP(DATA3_LDPC_XSYMSEG, LDPC_EXT_SYM, rxv[2]); + he->data4 = HE_PREP(DATA4_SU_MU_SPTL_REUSE, SR_MASK, rxv[11]); + he->data5 = HE_PREP(DATA5_PE_DISAMBIG, PE_DISAMBIG, rxv[2]) | + le16_encode_bits(ltf_size, + IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE); + if (le32_to_cpu(rxv[0]) & MT_PRXV_TXBF) + he->data5 |= HE_BITS(DATA5_TXBF); + he->data6 = HE_PREP(DATA6_TXOP, TXOP_DUR, rxv[14]) | + HE_PREP(DATA6_DOPPLER, DOPPLER, rxv[14]); + + switch (mode) { + case MT_PHY_TYPE_HE_SU: + he->data1 |= HE_BITS(DATA1_FORMAT_SU) | + HE_BITS(DATA1_UL_DL_KNOWN) | + HE_BITS(DATA1_BEAM_CHANGE_KNOWN) | + HE_BITS(DATA1_BW_RU_ALLOC_KNOWN); + + he->data3 |= HE_PREP(DATA3_BEAM_CHANGE, BEAM_CHNG, rxv[14]) | + HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]); + break; + case MT_PHY_TYPE_HE_EXT_SU: + he->data1 |= HE_BITS(DATA1_FORMAT_EXT_SU) | + HE_BITS(DATA1_UL_DL_KNOWN) | + HE_BITS(DATA1_BW_RU_ALLOC_KNOWN); + + he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]); + break; + case MT_PHY_TYPE_HE_MU: + he->data1 |= HE_BITS(DATA1_FORMAT_MU) | + HE_BITS(DATA1_UL_DL_KNOWN); + + he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]); + he->data4 |= HE_PREP(DATA4_MU_STA_ID, MU_AID, rxv[7]); + + mt7996_mac_decode_he_radiotap_ru(status, he, rxv); + mt7996_mac_decode_he_mu_radiotap(skb, rxv); + break; + case MT_PHY_TYPE_HE_TB: + he->data1 |= HE_BITS(DATA1_FORMAT_TRIG) | + HE_BITS(DATA1_SPTL_REUSE2_KNOWN) | + HE_BITS(DATA1_SPTL_REUSE3_KNOWN) | + HE_BITS(DATA1_SPTL_REUSE4_KNOWN); + + he->data4 |= HE_PREP(DATA4_TB_SPTL_REUSE1, SR_MASK, rxv[11]) | + HE_PREP(DATA4_TB_SPTL_REUSE2, SR1_MASK, rxv[11]) | + HE_PREP(DATA4_TB_SPTL_REUSE3, SR2_MASK, rxv[11]) | + HE_PREP(DATA4_TB_SPTL_REUSE4, SR3_MASK, rxv[11]); + + mt7996_mac_decode_he_radiotap_ru(status, he, rxv); + break; + default: + break; + } +} + +/* The HW does not translate the mac header to 802.3 for mesh point */ +static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap) +{ + struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; + struct ethhdr *eth_hdr = (struct ethhdr *)(skb->data + hdr_gap); + struct mt7996_sta *msta = (struct mt7996_sta *)status->wcid; + __le32 *rxd = (__le32 *)skb->data; + struct ieee80211_sta *sta; + struct ieee80211_vif *vif; + struct ieee80211_hdr hdr; + u16 frame_control; + + if (le32_get_bits(rxd[3], MT_RXD3_NORMAL_ADDR_TYPE) != + MT_RXD3_NORMAL_U2M) + return -EINVAL; + + if (!(le32_to_cpu(rxd[1]) & MT_RXD1_NORMAL_GROUP_4)) + return -EINVAL; + + if (!msta || !msta->vif) + return -EINVAL; + + sta = container_of((void *)msta, struct ieee80211_sta, drv_priv); + vif = container_of((void *)msta->vif, struct ieee80211_vif, drv_priv); + + /* store the info from RXD and ethhdr to avoid being overridden */ + frame_control = le32_get_bits(rxd[8], MT_RXD8_FRAME_CONTROL); + hdr.frame_control = cpu_to_le16(frame_control); + hdr.seq_ctrl = cpu_to_le16(le32_get_bits(rxd[10], MT_RXD10_SEQ_CTRL)); + hdr.duration_id = 0; + + ether_addr_copy(hdr.addr1, vif->addr); + ether_addr_copy(hdr.addr2, sta->addr); + switch (frame_control & (IEEE80211_FCTL_TODS | + IEEE80211_FCTL_FROMDS)) { + case 0: + ether_addr_copy(hdr.addr3, vif->bss_conf.bssid); + break; + case IEEE80211_FCTL_FROMDS: + ether_addr_copy(hdr.addr3, eth_hdr->h_source); + break; + case IEEE80211_FCTL_TODS: + ether_addr_copy(hdr.addr3, eth_hdr->h_dest); + break; + case IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS: + ether_addr_copy(hdr.addr3, eth_hdr->h_dest); + ether_addr_copy(hdr.addr4, eth_hdr->h_source); + break; + default: + break; + } + + skb_pull(skb, hdr_gap + sizeof(struct ethhdr) - 2); + if (eth_hdr->h_proto == cpu_to_be16(ETH_P_AARP) || + eth_hdr->h_proto == cpu_to_be16(ETH_P_IPX)) + ether_addr_copy(skb_push(skb, ETH_ALEN), bridge_tunnel_header); + else if (be16_to_cpu(eth_hdr->h_proto) >= ETH_P_802_3_MIN) + ether_addr_copy(skb_push(skb, ETH_ALEN), rfc1042_header); + else + skb_pull(skb, 2); + + if (ieee80211_has_order(hdr.frame_control)) + memcpy(skb_push(skb, IEEE80211_HT_CTL_LEN), &rxd[11], + IEEE80211_HT_CTL_LEN); + if (ieee80211_is_data_qos(hdr.frame_control)) { + __le16 qos_ctrl; + + qos_ctrl = cpu_to_le16(le32_get_bits(rxd[10], MT_RXD10_QOS_CTL)); + memcpy(skb_push(skb, IEEE80211_QOS_CTL_LEN), &qos_ctrl, + IEEE80211_QOS_CTL_LEN); + } + + if (ieee80211_has_a4(hdr.frame_control)) + memcpy(skb_push(skb, sizeof(hdr)), &hdr, sizeof(hdr)); + else + memcpy(skb_push(skb, sizeof(hdr) - 6), &hdr, sizeof(hdr) - 6); + + return 0; +} + +static int +mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, + struct mt76_rx_status *status, + struct ieee80211_supported_band *sband, + __le32 *rxv, u8 *mode) +{ + u32 v0, v2; + u8 stbc, gi, bw, dcm, nss; + int i, idx; + bool cck = false; + + v0 = le32_to_cpu(rxv[0]); + v2 = le32_to_cpu(rxv[2]); + + idx = FIELD_GET(MT_PRXV_TX_RATE, v0); + i = idx; + nss = FIELD_GET(MT_PRXV_NSTS, v0) + 1; + + stbc = FIELD_GET(MT_PRXV_HT_STBC, v2); + gi = FIELD_GET(MT_PRXV_HT_SHORT_GI, v2); + *mode = FIELD_GET(MT_PRXV_TX_MODE, v2); + dcm = FIELD_GET(MT_PRXV_DCM, v2); + bw = FIELD_GET(MT_PRXV_FRAME_MODE, v2); + + switch (*mode) { + case MT_PHY_TYPE_CCK: + cck = true; + fallthrough; + case MT_PHY_TYPE_OFDM: + i = mt76_get_rate(&dev->mt76, sband, i, cck); + break; + case MT_PHY_TYPE_HT_GF: + case MT_PHY_TYPE_HT: + status->encoding = RX_ENC_HT; + if (gi) + status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + if (i > 31) + return -EINVAL; + break; + case MT_PHY_TYPE_VHT: + status->nss = nss; + status->encoding = RX_ENC_VHT; + if (gi) + status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + if (i > 11) + return -EINVAL; + break; + case MT_PHY_TYPE_HE_MU: + case MT_PHY_TYPE_HE_SU: + case MT_PHY_TYPE_HE_EXT_SU: + case MT_PHY_TYPE_HE_TB: + status->nss = nss; + status->encoding = RX_ENC_HE; + i &= GENMASK(3, 0); + + if (gi <= NL80211_RATE_INFO_HE_GI_3_2) + status->he_gi = gi; + + status->he_dcm = dcm; + break; + default: + return -EINVAL; + } + status->rate_idx = i; + + switch (bw) { + case IEEE80211_STA_RX_BW_20: + break; + case IEEE80211_STA_RX_BW_40: + if (*mode & MT_PHY_TYPE_HE_EXT_SU && + (idx & MT_PRXV_TX_ER_SU_106T)) { + status->bw = RATE_INFO_BW_HE_RU; + status->he_ru = + NL80211_RATE_INFO_HE_RU_ALLOC_106; + } else { + status->bw = RATE_INFO_BW_40; + } + break; + case IEEE80211_STA_RX_BW_80: + status->bw = RATE_INFO_BW_80; + break; + case IEEE80211_STA_RX_BW_160: + status->bw = RATE_INFO_BW_160; + break; + default: + return -EINVAL; + } + + status->enc_flags |= RX_ENC_FLAG_STBC_MASK * stbc; + if (*mode < MT_PHY_TYPE_HE_SU && gi) + status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + + return 0; +} + +static int +mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; + struct mt76_phy *mphy = &dev->mt76.phy; + struct mt7996_phy *phy = &dev->phy; + struct ieee80211_supported_band *sband; + __le32 *rxd = (__le32 *)skb->data; + __le32 *rxv = NULL; + u32 rxd0 = le32_to_cpu(rxd[0]); + u32 rxd1 = le32_to_cpu(rxd[1]); + u32 rxd2 = le32_to_cpu(rxd[2]); + u32 rxd3 = le32_to_cpu(rxd[3]); + u32 rxd4 = le32_to_cpu(rxd[4]); + u32 csum_mask = MT_RXD0_NORMAL_IP_SUM | MT_RXD0_NORMAL_UDP_TCP_SUM; + u32 csum_status = *(u32 *)skb->cb; + bool unicast, insert_ccmp_hdr = false; + u8 remove_pad, amsdu_info, band_idx; + u8 mode = 0, qos_ctl = 0; + bool hdr_trans; + u16 hdr_gap; + u16 seq_ctrl = 0; + __le16 fc = 0; + int idx; + + memset(status, 0, sizeof(*status)); + + band_idx = FIELD_GET(MT_RXD1_NORMAL_BAND_IDX, rxd1); + mphy = dev->mt76.phys[band_idx]; + phy = mphy->priv; + status->phy_idx = mphy->band_idx; + + if (!test_bit(MT76_STATE_RUNNING, &mphy->state)) + return -EINVAL; + + if (rxd2 & MT_RXD2_NORMAL_AMSDU_ERR) + return -EINVAL; + + hdr_trans = rxd2 & MT_RXD2_NORMAL_HDR_TRANS; + if (hdr_trans && (rxd1 & MT_RXD1_NORMAL_CM)) + return -EINVAL; + + /* ICV error or CCMP/BIP/WPI MIC error */ + if (rxd1 & MT_RXD1_NORMAL_ICV_ERR) + status->flag |= RX_FLAG_ONLY_MONITOR; + + unicast = FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) == MT_RXD3_NORMAL_U2M; + idx = FIELD_GET(MT_RXD1_NORMAL_WLAN_IDX, rxd1); + status->wcid = mt7996_rx_get_wcid(dev, idx, unicast); + + if (status->wcid) { + struct mt7996_sta *msta; + + msta = container_of(status->wcid, struct mt7996_sta, wcid); + spin_lock_bh(&dev->sta_poll_lock); + if (list_empty(&msta->poll_list)) + list_add_tail(&msta->poll_list, &dev->sta_poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + } + + status->freq = mphy->chandef.chan->center_freq; + status->band = mphy->chandef.chan->band; + if (status->band == NL80211_BAND_5GHZ) + sband = &mphy->sband_5g.sband; + else if (status->band == NL80211_BAND_6GHZ) + sband = &mphy->sband_6g.sband; + else + sband = &mphy->sband_2g.sband; + + if (!sband->channels) + return -EINVAL; + + if ((rxd0 & csum_mask) == csum_mask && + !(csum_status & (BIT(0) | BIT(2) | BIT(3)))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + + if (rxd1 & MT_RXD3_NORMAL_FCS_ERR) + status->flag |= RX_FLAG_FAILED_FCS_CRC; + + if (rxd1 & MT_RXD1_NORMAL_TKIP_MIC_ERR) + status->flag |= RX_FLAG_MMIC_ERROR; + + if (FIELD_GET(MT_RXD2_NORMAL_SEC_MODE, rxd2) != 0 && + !(rxd1 & (MT_RXD1_NORMAL_CLM | MT_RXD1_NORMAL_CM))) { + status->flag |= RX_FLAG_DECRYPTED; + status->flag |= RX_FLAG_IV_STRIPPED; + status->flag |= RX_FLAG_MMIC_STRIPPED | RX_FLAG_MIC_STRIPPED; + } + + remove_pad = FIELD_GET(MT_RXD2_NORMAL_HDR_OFFSET, rxd2); + + if (rxd2 & MT_RXD2_NORMAL_MAX_LEN_ERROR) + return -EINVAL; + + rxd += 8; + if (rxd1 & MT_RXD1_NORMAL_GROUP_4) { + u32 v0 = le32_to_cpu(rxd[0]); + u32 v2 = le32_to_cpu(rxd[2]); + + fc = cpu_to_le16(FIELD_GET(MT_RXD8_FRAME_CONTROL, v0)); + qos_ctl = FIELD_GET(MT_RXD10_QOS_CTL, v2); + seq_ctrl = FIELD_GET(MT_RXD10_SEQ_CTRL, v2); + + rxd += 4; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + } + + if (rxd1 & MT_RXD1_NORMAL_GROUP_1) { + u8 *data = (u8 *)rxd; + + if (status->flag & RX_FLAG_DECRYPTED) { + switch (FIELD_GET(MT_RXD2_NORMAL_SEC_MODE, rxd2)) { + case MT_CIPHER_AES_CCMP: + case MT_CIPHER_CCMP_CCX: + case MT_CIPHER_CCMP_256: + insert_ccmp_hdr = + FIELD_GET(MT_RXD2_NORMAL_FRAG, rxd2); + fallthrough; + case MT_CIPHER_TKIP: + case MT_CIPHER_TKIP_NO_MIC: + case MT_CIPHER_GCMP: + case MT_CIPHER_GCMP_256: + status->iv[0] = data[5]; + status->iv[1] = data[4]; + status->iv[2] = data[3]; + status->iv[3] = data[2]; + status->iv[4] = data[1]; + status->iv[5] = data[0]; + break; + default: + break; + } + } + rxd += 4; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + } + + if (rxd1 & MT_RXD1_NORMAL_GROUP_2) { + status->timestamp = le32_to_cpu(rxd[0]); + status->flag |= RX_FLAG_MACTIME_START; + + if (!(rxd2 & MT_RXD2_NORMAL_NON_AMPDU)) { + status->flag |= RX_FLAG_AMPDU_DETAILS; + + /* all subframes of an A-MPDU have the same timestamp */ + if (phy->rx_ampdu_ts != status->timestamp) { + if (!++phy->ampdu_ref) + phy->ampdu_ref++; + } + phy->rx_ampdu_ts = status->timestamp; + + status->ampdu_ref = phy->ampdu_ref; + } + + rxd += 4; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + } + + /* RXD Group 3 - P-RXV */ + if (rxd1 & MT_RXD1_NORMAL_GROUP_3) { + u32 v3; + int ret; + + rxv = rxd; + rxd += 4; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + + v3 = le32_to_cpu(rxv[3]); + + status->chains = mphy->antenna_mask; + status->chain_signal[0] = to_rssi(MT_PRXV_RCPI0, v3); + status->chain_signal[1] = to_rssi(MT_PRXV_RCPI1, v3); + status->chain_signal[2] = to_rssi(MT_PRXV_RCPI2, v3); + status->chain_signal[3] = to_rssi(MT_PRXV_RCPI3, v3); + + /* RXD Group 5 - C-RXV */ + if (rxd1 & MT_RXD1_NORMAL_GROUP_5) { + rxd += 24; + if ((u8 *)rxd - skb->data >= skb->len) + return -EINVAL; + } + + ret = mt7996_mac_fill_rx_rate(dev, status, sband, rxv, &mode); + if (ret < 0) + return ret; + } + + amsdu_info = FIELD_GET(MT_RXD4_NORMAL_PAYLOAD_FORMAT, rxd4); + status->amsdu = !!amsdu_info; + if (status->amsdu) { + status->first_amsdu = amsdu_info == MT_RXD4_FIRST_AMSDU_FRAME; + status->last_amsdu = amsdu_info == MT_RXD4_LAST_AMSDU_FRAME; + } + + hdr_gap = (u8 *)rxd - skb->data + 2 * remove_pad; + if (hdr_trans && ieee80211_has_morefrags(fc)) { + if (mt7996_reverse_frag0_hdr_trans(skb, hdr_gap)) + return -EINVAL; + hdr_trans = false; + } else { + int pad_start = 0; + + skb_pull(skb, hdr_gap); + if (!hdr_trans && status->amsdu) { + pad_start = ieee80211_get_hdrlen_from_skb(skb); + } else if (hdr_trans && (rxd2 & MT_RXD2_NORMAL_HDR_TRANS_ERROR)) { + /* When header translation failure is indicated, + * the hardware will insert an extra 2-byte field + * containing the data length after the protocol + * type field. + */ + pad_start = 12; + if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q) + pad_start += 4; + else + pad_start = 0; + } + + if (pad_start) { + memmove(skb->data + 2, skb->data, pad_start); + skb_pull(skb, 2); + } + } + + if (!hdr_trans) { + struct ieee80211_hdr *hdr; + + if (insert_ccmp_hdr) { + u8 key_id = FIELD_GET(MT_RXD1_NORMAL_KEY_ID, rxd1); + + mt76_insert_ccmp_hdr(skb, key_id); + } + + hdr = mt76_skb_get_hdr(skb); + fc = hdr->frame_control; + if (ieee80211_is_data_qos(fc)) { + seq_ctrl = le16_to_cpu(hdr->seq_ctrl); + qos_ctl = *ieee80211_get_qos_ctl(hdr); + } + } else { + status->flag |= RX_FLAG_8023; + } + + if (rxv && mode >= MT_PHY_TYPE_HE_SU && !(status->flag & RX_FLAG_8023)) + mt7996_mac_decode_he_radiotap(skb, rxv, mode); + + if (!status->wcid || !ieee80211_is_data_qos(fc)) + return 0; + + status->aggr = unicast && + !ieee80211_is_qos_nullfunc(fc); + status->qos_ctl = qos_ctl; + status->seqno = IEEE80211_SEQ_TO_SN(seq_ctrl); + + return 0; +} + +static void +mt7996_mac_write_txwi_8023(struct mt7996_dev *dev, __le32 *txwi, + struct sk_buff *skb, struct mt76_wcid *wcid) +{ + u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; + u8 fc_type, fc_stype; + u16 ethertype; + bool wmm = false; + u32 val; + + if (wcid->sta) { + struct ieee80211_sta *sta; + + sta = container_of((void *)wcid, struct ieee80211_sta, drv_priv); + wmm = sta->wme; + } + + val = FIELD_PREP(MT_TXD1_HDR_FORMAT, MT_HDR_FORMAT_802_3) | + FIELD_PREP(MT_TXD1_TID, tid); + + ethertype = get_unaligned_be16(&skb->data[12]); + if (ethertype >= ETH_P_802_3_MIN) + val |= MT_TXD1_ETH_802_3; + + txwi[1] |= cpu_to_le32(val); + + fc_type = IEEE80211_FTYPE_DATA >> 2; + fc_stype = wmm ? IEEE80211_STYPE_QOS_DATA >> 4 : 0; + + val = FIELD_PREP(MT_TXD2_FRAME_TYPE, fc_type) | + FIELD_PREP(MT_TXD2_SUB_TYPE, fc_stype); + + txwi[2] |= cpu_to_le32(val); +} + +static void +mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, + struct sk_buff *skb, struct ieee80211_key_conf *key) +{ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; + struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data; + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + bool multicast = is_multicast_ether_addr(hdr->addr1); + u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; + __le16 fc = hdr->frame_control; + u8 fc_type, fc_stype; + u32 val; + + if (ieee80211_is_action(fc) && + mgmt->u.action.category == WLAN_CATEGORY_BACK && + mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) + tid = MT_TX_ADDBA; + else if (ieee80211_is_mgmt(hdr->frame_control)) + tid = MT_TX_NORMAL; + + val = FIELD_PREP(MT_TXD1_HDR_FORMAT, MT_HDR_FORMAT_802_11) | + FIELD_PREP(MT_TXD1_HDR_INFO, + ieee80211_get_hdrlen_from_skb(skb) / 2) | + FIELD_PREP(MT_TXD1_TID, tid); + + if (!ieee80211_is_data(fc) || multicast || + info->flags & IEEE80211_TX_CTL_USE_MINRATE) + val |= MT_TXD1_FIXED_RATE; + + if (key && multicast && ieee80211_is_robust_mgmt_frame(skb) && + key->cipher == WLAN_CIPHER_SUITE_AES_CMAC) { + val |= MT_TXD1_BIP; + txwi[3] &= ~cpu_to_le32(MT_TXD3_PROTECT_FRAME); + } + + txwi[1] |= cpu_to_le32(val); + + fc_type = (le16_to_cpu(fc) & IEEE80211_FCTL_FTYPE) >> 2; + fc_stype = (le16_to_cpu(fc) & IEEE80211_FCTL_STYPE) >> 4; + + val = FIELD_PREP(MT_TXD2_FRAME_TYPE, fc_type) | + FIELD_PREP(MT_TXD2_SUB_TYPE, fc_stype); + + txwi[2] |= cpu_to_le32(val); + + txwi[3] |= cpu_to_le32(FIELD_PREP(MT_TXD3_BCM, multicast)); + if (ieee80211_is_beacon(fc)) { + txwi[3] &= ~cpu_to_le32(MT_TXD3_SW_POWER_MGMT); + txwi[3] |= cpu_to_le32(MT_TXD3_REM_TX_COUNT); + } + + if (info->flags & IEEE80211_TX_CTL_INJECTED) { + u16 seqno = le16_to_cpu(hdr->seq_ctrl); + + if (ieee80211_is_back_req(hdr->frame_control)) { + struct ieee80211_bar *bar; + + bar = (struct ieee80211_bar *)skb->data; + seqno = le16_to_cpu(bar->start_seq_num); + } + + val = MT_TXD3_SN_VALID | + FIELD_PREP(MT_TXD3_SEQ, IEEE80211_SEQ_TO_SN(seqno)); + txwi[3] |= cpu_to_le32(val); + txwi[3] &= ~cpu_to_le32(MT_TXD3_HW_AMSDU); + } +} + +static u16 +mt7996_mac_tx_rate_val(struct mt76_phy *mphy, struct ieee80211_vif *vif, + bool beacon, bool mcast) +{ + u8 mode = 0, band = mphy->chandef.chan->band; + int rateidx = 0, mcast_rate; + + if (beacon) { + struct cfg80211_bitrate_mask *mask; + + mask = &vif->bss_conf.beacon_tx_rate; + if (hweight16(mask->control[band].he_mcs[0]) == 1) { + rateidx = ffs(mask->control[band].he_mcs[0]) - 1; + mode = MT_PHY_TYPE_HE_SU; + goto out; + } else if (hweight16(mask->control[band].vht_mcs[0]) == 1) { + rateidx = ffs(mask->control[band].vht_mcs[0]) - 1; + mode = MT_PHY_TYPE_VHT; + goto out; + } else if (hweight8(mask->control[band].ht_mcs[0]) == 1) { + rateidx = ffs(mask->control[band].ht_mcs[0]) - 1; + mode = MT_PHY_TYPE_HT; + goto out; + } else if (hweight32(mask->control[band].legacy) == 1) { + rateidx = ffs(mask->control[band].legacy) - 1; + goto legacy; + } + } + + mcast_rate = vif->bss_conf.mcast_rate[band]; + if (mcast && mcast_rate > 0) + rateidx = mcast_rate - 1; + else + rateidx = ffs(vif->bss_conf.basic_rates) - 1; + +legacy: + rateidx = mt76_calculate_default_rate(mphy, rateidx); + mode = rateidx >> 8; + rateidx &= GENMASK(7, 0); + +out: + return FIELD_PREP(MT_TX_RATE_IDX, rateidx) | + FIELD_PREP(MT_TX_RATE_MODE, mode); +} + +void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, + struct sk_buff *skb, struct mt76_wcid *wcid, int pid, + struct ieee80211_key_conf *key, u32 changed) +{ + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct ieee80211_vif *vif = info->control.vif; + struct mt76_phy *mphy = &dev->mphy; + u8 band_idx = (info->hw_queue & MT_TX_HW_QUEUE_PHY) >> 2; + u8 p_fmt, q_idx, omac_idx = 0, wmm_idx = 0; + bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP; + u16 tx_count = 15; + u32 val; + bool beacon = !!(changed & (BSS_CHANGED_BEACON | + BSS_CHANGED_BEACON_ENABLED)); + bool inband_disc = !!(changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP | + BSS_CHANGED_FILS_DISCOVERY)); + + if (vif) { + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + + omac_idx = mvif->mt76.omac_idx; + wmm_idx = mvif->mt76.wmm_idx; + band_idx = mvif->mt76.band_idx; + } + + mphy = mt76_dev_phy(&dev->mt76, band_idx); + + if (inband_disc) { + p_fmt = MT_TX_TYPE_FW; + q_idx = MT_LMAC_ALTX0; + } else if (beacon) { + p_fmt = MT_TX_TYPE_FW; + q_idx = MT_LMAC_BCN0; + } else if (skb_get_queue_mapping(skb) >= MT_TXQ_PSD) { + p_fmt = MT_TX_TYPE_CT; + q_idx = MT_LMAC_ALTX0; + } else { + p_fmt = MT_TX_TYPE_CT; + q_idx = wmm_idx * MT7996_MAX_WMM_SETS + + mt76_connac_lmac_mapping(skb_get_queue_mapping(skb)); + } + + val = FIELD_PREP(MT_TXD0_TX_BYTES, skb->len + MT_TXD_SIZE) | + FIELD_PREP(MT_TXD0_PKT_FMT, p_fmt) | + FIELD_PREP(MT_TXD0_Q_IDX, q_idx); + txwi[0] = cpu_to_le32(val); + + val = FIELD_PREP(MT_TXD1_WLAN_IDX, wcid->idx) | + FIELD_PREP(MT_TXD1_OWN_MAC, omac_idx); + + if (band_idx) + val |= FIELD_PREP(MT_TXD1_TGID, band_idx); + + txwi[1] = cpu_to_le32(val); + txwi[2] = 0; + + val = MT_TXD3_SW_POWER_MGMT | + FIELD_PREP(MT_TXD3_REM_TX_COUNT, tx_count); + if (key) + val |= MT_TXD3_PROTECT_FRAME; + if (info->flags & IEEE80211_TX_CTL_NO_ACK) + val |= MT_TXD3_NO_ACK; + if (wcid->amsdu) + val |= MT_TXD3_HW_AMSDU; + + txwi[3] = cpu_to_le32(val); + txwi[4] = 0; + + val = FIELD_PREP(MT_TXD5_PID, pid); + if (pid >= MT_PACKET_ID_FIRST) + val |= MT_TXD5_TX_STATUS_HOST; + txwi[5] = cpu_to_le32(val); + + val = MT_TXD6_DIS_MAT | MT_TXD6_DAS | + FIELD_PREP(MT_TXD6_MSDU_CNT, 1); + txwi[6] = cpu_to_le32(val); + txwi[7] = 0; + + if (is_8023) + mt7996_mac_write_txwi_8023(dev, txwi, skb, wcid); + else + mt7996_mac_write_txwi_80211(dev, txwi, skb, key); + + if (txwi[1] & cpu_to_le32(MT_TXD1_FIXED_RATE)) { + /* Fixed rata is available just for 802.11 txd */ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; + bool multicast = is_multicast_ether_addr(hdr->addr1); + u16 rate = mt7996_mac_tx_rate_val(mphy, vif, beacon, multicast); + + /* fix to bw 20 */ + val = MT_TXD6_FIXED_BW | + FIELD_PREP(MT_TXD6_BW, 0) | + FIELD_PREP(MT_TXD6_TX_RATE, rate); + + txwi[6] |= cpu_to_le32(val); + txwi[3] |= cpu_to_le32(MT_TXD3_BA_DISABLE); + } +} + +int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, + enum mt76_txq_id qid, struct mt76_wcid *wcid, + struct ieee80211_sta *sta, + struct mt76_tx_info *tx_info) +{ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx_info->skb->data; + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_info->skb); + struct ieee80211_key_conf *key = info->control.hw_key; + struct ieee80211_vif *vif = info->control.vif; + struct mt76_txwi_cache *t; + struct mt7996_txp *txp; + int id, i, pid, nbuf = tx_info->nbuf - 1; + bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP; + u8 *txwi = (u8 *)txwi_ptr; + + if (unlikely(tx_info->skb->len <= ETH_HLEN)) + return -EINVAL; + + if (!wcid) + wcid = &dev->mt76.global_wcid; + + if (sta) { + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + + if (time_after(jiffies, msta->jiffies + HZ / 4)) { + info->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS; + msta->jiffies = jiffies; + } + } + + t = (struct mt76_txwi_cache *)(txwi + mdev->drv->txwi_size); + t->skb = tx_info->skb; + + id = mt76_token_consume(mdev, &t); + if (id < 0) + return id; + + pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb); + memset(txwi_ptr, 0, MT_TXD_SIZE); + /* Transmit non qos data by 802.11 header and need to fill txd by host*/ + if (!is_8023 || pid >= MT_PACKET_ID_FIRST) + mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, pid, + key, 0); + + txp = (struct mt7996_txp *)(txwi + MT_TXD_SIZE); + for (i = 0; i < nbuf; i++) { + txp->buf[i] = cpu_to_le32(tx_info->buf[i + 1].addr); + txp->len[i] = cpu_to_le16(tx_info->buf[i + 1].len); + } + txp->nbuf = nbuf; + + txp->flags = cpu_to_le16(MT_CT_INFO_FROM_HOST); + + if (!is_8023 || pid >= MT_PACKET_ID_FIRST) + txp->flags |= cpu_to_le16(MT_CT_INFO_APPLY_TXD); + + if (!key) + txp->flags |= cpu_to_le16(MT_CT_INFO_NONE_CIPHER_FRAME); + + if (!is_8023 && ieee80211_is_mgmt(hdr->frame_control)) + txp->flags |= cpu_to_le16(MT_CT_INFO_MGMT_FRAME); + + if (vif) { + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + + txp->bss_idx = mvif->mt76.idx; + } + + txp->token = cpu_to_le16(id); + if (test_bit(MT_WCID_FLAG_4ADDR, &wcid->flags)) + txp->rept_wds_wcid = cpu_to_le16(wcid->idx); + else + txp->rept_wds_wcid = cpu_to_le16(0xfff); + tx_info->skb = DMA_DUMMY_DATA; + + /* pass partial skb header to fw */ + tx_info->buf[1].len = MT_CT_PARSE_LEN; + tx_info->buf[1].skip_unmap = true; + tx_info->nbuf = MT_CT_DMA_BUF_NUM; + + return 0; +} + +static void +mt7996_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi) +{ + struct mt7996_sta *msta; + u16 fc, tid; + u32 val; + + if (!sta || !(sta->deflink.ht_cap.ht_supported || sta->deflink.he_cap.has_he)) + return; + + tid = le32_get_bits(txwi[1], MT_TXD1_TID); + if (tid >= 6) /* skip VO queue */ + return; + + val = le32_to_cpu(txwi[2]); + fc = FIELD_GET(MT_TXD2_FRAME_TYPE, val) << 2 | + FIELD_GET(MT_TXD2_SUB_TYPE, val) << 4; + if (unlikely(fc != (IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA))) + return; + + msta = (struct mt7996_sta *)sta->drv_priv; + if (!test_and_set_bit(tid, &msta->ampdu_state)) + ieee80211_start_tx_ba_session(sta, tid, 0); +} + +static void +mt7996_txp_skb_unmap(struct mt76_dev *dev, struct mt76_txwi_cache *t) +{ + struct mt7996_txp *txp; + int i; + + txp = mt7996_txwi_to_txp(dev, t); + for (i = 0; i < txp->nbuf; i++) + dma_unmap_single(dev->dev, le32_to_cpu(txp->buf[i]), + le16_to_cpu(txp->len[i]), DMA_TO_DEVICE); +} + +static void +mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, + struct ieee80211_sta *sta, struct list_head *free_list) +{ + struct mt76_dev *mdev = &dev->mt76; + struct mt76_wcid *wcid; + __le32 *txwi; + u16 wcid_idx; + + mt7996_txp_skb_unmap(mdev, t); + if (!t->skb) + goto out; + + txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t); + if (sta) { + wcid = (struct mt76_wcid *)sta->drv_priv; + wcid_idx = wcid->idx; + + if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE))) + mt7996_tx_check_aggr(sta, txwi); + } else { + wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX); + } + + __mt76_tx_complete_skb(mdev, wcid_idx, t->skb, free_list); + +out: + t->skb = NULL; + mt76_put_txwi(mdev, t); +} + +static void +mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) +{ + __le32 *tx_free = (__le32 *)data, *cur_info; + struct mt76_dev *mdev = &dev->mt76; + struct mt76_phy *phy2 = mdev->phys[MT_BAND1]; + struct mt76_phy *phy3 = mdev->phys[MT_BAND2]; + struct mt76_txwi_cache *txwi; + struct ieee80211_sta *sta = NULL; + LIST_HEAD(free_list); + struct sk_buff *skb, *tmp; + void *end = data + len; + bool wake = false; + u16 total, count = 0; + + /* clean DMA queues and unmap buffers first */ + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false); + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_BE], false); + if (phy2) { + mt76_queue_tx_cleanup(dev, phy2->q_tx[MT_TXQ_PSD], false); + mt76_queue_tx_cleanup(dev, phy2->q_tx[MT_TXQ_BE], false); + } + if (phy3) { + mt76_queue_tx_cleanup(dev, phy3->q_tx[MT_TXQ_PSD], false); + mt76_queue_tx_cleanup(dev, phy3->q_tx[MT_TXQ_BE], false); + } + + if (WARN_ON_ONCE(le32_get_bits(tx_free[1], MT_TXFREE1_VER) < 4)) + return; + + total = le32_get_bits(tx_free[0], MT_TXFREE0_MSDU_CNT); + for (cur_info = &tx_free[2]; count < total; cur_info++) { + u32 msdu, info; + u8 i; + + if (WARN_ON_ONCE((void *)cur_info >= end)) + return; + /* 1'b1: new wcid pair. + * 1'b0: msdu_id with the same 'wcid pair' as above. + */ + info = le32_to_cpu(*cur_info); + if (info & MT_TXFREE_INFO_PAIR) { + struct mt7996_sta *msta; + struct mt76_wcid *wcid; + u16 idx; + + idx = FIELD_GET(MT_TXFREE_INFO_WLAN_ID, info); + wcid = rcu_dereference(dev->mt76.wcid[idx]); + sta = wcid_to_sta(wcid); + if (!sta) + continue; + + msta = container_of(wcid, struct mt7996_sta, wcid); + spin_lock_bh(&dev->sta_poll_lock); + if (list_empty(&msta->poll_list)) + list_add_tail(&msta->poll_list, &dev->sta_poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + continue; + } + + if (info & MT_TXFREE_INFO_HEADER) + continue; + + for (i = 0; i < 2; i++) { + msdu = (info >> (15 * i)) & MT_TXFREE_INFO_MSDU_ID; + if (msdu == MT_TXFREE_INFO_MSDU_ID) + continue; + + count++; + txwi = mt76_token_release(mdev, msdu, &wake); + if (!txwi) + continue; + + mt7996_txwi_free(dev, txwi, sta, &free_list); + } + } + + mt7996_mac_sta_poll(dev); + + if (wake) + mt76_set_tx_blocked(&dev->mt76, false); + + mt76_worker_schedule(&dev->mt76.tx_worker); + + list_for_each_entry_safe(skb, tmp, &free_list, list) { + skb_list_del_init(skb); + napi_consume_skb(skb, 1); + } +} + +static bool +mt7996_mac_add_txs_skb(struct mt7996_dev *dev, struct mt76_wcid *wcid, int pid, + __le32 *txs_data, struct mt76_sta_stats *stats) +{ + struct ieee80211_supported_band *sband; + struct mt76_dev *mdev = &dev->mt76; + struct mt76_phy *mphy; + struct ieee80211_tx_info *info; + struct sk_buff_head list; + struct rate_info rate = {}; + struct sk_buff *skb; + bool cck = false; + u32 txrate, txs, mode, stbc; + + mt76_tx_status_lock(mdev, &list); + skb = mt76_tx_status_skb_get(mdev, wcid, pid, &list); + if (!skb) + goto out_no_skb; + + txs = le32_to_cpu(txs_data[0]); + + info = IEEE80211_SKB_CB(skb); + if (!(txs & MT_TXS0_ACK_ERROR_MASK)) + info->flags |= IEEE80211_TX_STAT_ACK; + + info->status.ampdu_len = 1; + info->status.ampdu_ack_len = !!(info->flags & + IEEE80211_TX_STAT_ACK); + + info->status.rates[0].idx = -1; + + txrate = FIELD_GET(MT_TXS0_TX_RATE, txs); + + rate.mcs = FIELD_GET(MT_TX_RATE_IDX, txrate); + rate.nss = FIELD_GET(MT_TX_RATE_NSS, txrate) + 1; + stbc = le32_get_bits(txs_data[3], MT_TXS3_RATE_STBC); + + if (stbc && rate.nss > 1) + rate.nss >>= 1; + + if (rate.nss - 1 < ARRAY_SIZE(stats->tx_nss)) + stats->tx_nss[rate.nss - 1]++; + if (rate.mcs < ARRAY_SIZE(stats->tx_mcs)) + stats->tx_mcs[rate.mcs]++; + + mode = FIELD_GET(MT_TX_RATE_MODE, txrate); + switch (mode) { + case MT_PHY_TYPE_CCK: + cck = true; + fallthrough; + case MT_PHY_TYPE_OFDM: + mphy = mt76_dev_phy(mdev, wcid->phy_idx); + + if (mphy->chandef.chan->band == NL80211_BAND_5GHZ) + sband = &mphy->sband_5g.sband; + else if (mphy->chandef.chan->band == NL80211_BAND_6GHZ) + sband = &mphy->sband_6g.sband; + else + sband = &mphy->sband_2g.sband; + + rate.mcs = mt76_get_rate(mphy->dev, sband, rate.mcs, cck); + rate.legacy = sband->bitrates[rate.mcs].bitrate; + break; + case MT_PHY_TYPE_HT: + case MT_PHY_TYPE_HT_GF: + if (rate.mcs > 31) + goto out; + + rate.flags = RATE_INFO_FLAGS_MCS; + if (wcid->rate.flags & RATE_INFO_FLAGS_SHORT_GI) + rate.flags |= RATE_INFO_FLAGS_SHORT_GI; + break; + case MT_PHY_TYPE_VHT: + if (rate.mcs > 9) + goto out; + + rate.flags = RATE_INFO_FLAGS_VHT_MCS; + break; + case MT_PHY_TYPE_HE_SU: + case MT_PHY_TYPE_HE_EXT_SU: + case MT_PHY_TYPE_HE_TB: + case MT_PHY_TYPE_HE_MU: + if (rate.mcs > 11) + goto out; + + rate.he_gi = wcid->rate.he_gi; + rate.he_dcm = FIELD_GET(MT_TX_RATE_DCM, txrate); + rate.flags = RATE_INFO_FLAGS_HE_MCS; + break; + default: + goto out; + } + + stats->tx_mode[mode]++; + + switch (FIELD_GET(MT_TXS0_BW, txs)) { + case IEEE80211_STA_RX_BW_160: + rate.bw = RATE_INFO_BW_160; + stats->tx_bw[3]++; + break; + case IEEE80211_STA_RX_BW_80: + rate.bw = RATE_INFO_BW_80; + stats->tx_bw[2]++; + break; + case IEEE80211_STA_RX_BW_40: + rate.bw = RATE_INFO_BW_40; + stats->tx_bw[1]++; + break; + default: + rate.bw = RATE_INFO_BW_20; + stats->tx_bw[0]++; + break; + } + wcid->rate = rate; + +out: + mt76_tx_status_skb_done(mdev, skb, &list); + +out_no_skb: + mt76_tx_status_unlock(mdev, &list); + + return !!skb; +} + +static void mt7996_mac_add_txs(struct mt7996_dev *dev, void *data) +{ + struct mt7996_sta *msta = NULL; + struct mt76_wcid *wcid; + __le32 *txs_data = data; + u16 wcidx; + u8 pid; + + if (le32_get_bits(txs_data[0], MT_TXS0_TXS_FORMAT) > 1) + return; + + wcidx = le32_get_bits(txs_data[2], MT_TXS2_WCID); + pid = le32_get_bits(txs_data[3], MT_TXS3_PID); + + if (pid < MT_PACKET_ID_FIRST) + return; + + if (wcidx >= MT7996_WTBL_SIZE) + return; + + rcu_read_lock(); + + wcid = rcu_dereference(dev->mt76.wcid[wcidx]); + if (!wcid) + goto out; + + msta = container_of(wcid, struct mt7996_sta, wcid); + + mt7996_mac_add_txs_skb(dev, wcid, pid, txs_data, &msta->stats); + + if (!wcid->sta) + goto out; + + spin_lock_bh(&dev->sta_poll_lock); + if (list_empty(&msta->poll_list)) + list_add_tail(&msta->poll_list, &dev->sta_poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + +out: + rcu_read_unlock(); +} + +bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + __le32 *rxd = (__le32 *)data; + __le32 *end = (__le32 *)&rxd[len / 4]; + enum rx_pkt_type type; + + type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE); + if (type != PKT_TYPE_NORMAL) { + u32 sw_type = le32_get_bits(rxd[0], MT_RXD0_SW_PKT_TYPE_MASK); + + if (unlikely((sw_type & MT_RXD0_SW_PKT_TYPE_MAP) == + MT_RXD0_SW_PKT_TYPE_FRAME)) + return true; + } + + switch (type) { + case PKT_TYPE_TXRX_NOTIFY: + mt7996_mac_tx_free(dev, data, len); + return false; + case PKT_TYPE_TXS: + for (rxd += 4; rxd + 8 <= end; rxd += 8) + mt7996_mac_add_txs(dev, rxd); + return false; + case PKT_TYPE_RX_FW_MONITOR: + mt7996_debugfs_rx_fw_monitor(dev, data, len); + return false; + default: + return true; + } +} + +void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, + struct sk_buff *skb, u32 *info) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + __le32 *rxd = (__le32 *)skb->data; + __le32 *end = (__le32 *)&skb->data[skb->len]; + enum rx_pkt_type type; + + type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE); + if (type != PKT_TYPE_NORMAL) { + u32 sw_type = le32_get_bits(rxd[0], MT_RXD0_SW_PKT_TYPE_MASK); + + if (unlikely((sw_type & MT_RXD0_SW_PKT_TYPE_MAP) == + MT_RXD0_SW_PKT_TYPE_FRAME)) + type = PKT_TYPE_NORMAL; + } + + switch (type) { + case PKT_TYPE_TXRX_NOTIFY: + mt7996_mac_tx_free(dev, skb->data, skb->len); + napi_consume_skb(skb, 1); + break; + case PKT_TYPE_RX_EVENT: + mt7996_mcu_rx_event(dev, skb); + break; + case PKT_TYPE_TXS: + for (rxd += 4; rxd + 8 <= end; rxd += 8) + mt7996_mac_add_txs(dev, rxd); + dev_kfree_skb(skb); + break; + case PKT_TYPE_RX_FW_MONITOR: + mt7996_debugfs_rx_fw_monitor(dev, skb->data, skb->len); + dev_kfree_skb(skb); + break; + case PKT_TYPE_NORMAL: + if (!mt7996_mac_fill_rx(dev, skb)) { + mt76_rx(&dev->mt76, q, skb); + return; + } + fallthrough; + default: + dev_kfree_skb(skb); + break; + } +} + +void mt7996_tx_complete_skb(struct mt76_dev *mdev, struct mt76_queue_entry *e) +{ + if (!e->txwi) { + dev_kfree_skb_any(e->skb); + return; + } + + /* error path */ + if (e->skb == DMA_DUMMY_DATA) { + struct mt76_txwi_cache *t; + struct mt7996_txp *txp; + + txp = mt7996_txwi_to_txp(mdev, e->txwi); + t = mt76_token_put(mdev, le16_to_cpu(txp->token)); + e->skb = t ? t->skb : NULL; + } + + if (e->skb) + mt76_tx_complete_skb(mdev, e->wcid, e->skb); +} + +void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + u32 reg = MT_WF_PHYRX_BAND_RX_CTRL1(phy->mt76->band_idx); + + mt76_clear(dev, reg, MT_WF_PHYRX_BAND_RX_CTRL1_STSCNT_EN); + mt76_set(dev, reg, BIT(11) | BIT(9)); +} + +void mt7996_mac_reset_counters(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + u8 band_idx = phy->mt76->band_idx; + int i; + + for (i = 0; i < 16; i++) + mt76_rr(dev, MT_TX_AGG_CNT(band_idx, i)); + + phy->mt76->survey_time = ktime_get_boottime(); + + memset(phy->mt76->aggr_stats, 0, sizeof(phy->mt76->aggr_stats)); + + /* reset airtime counters */ + mt76_set(dev, MT_WF_RMAC_MIB_AIRTIME0(band_idx), + MT_WF_RMAC_MIB_RXTIME_CLR); + + mt7996_mcu_get_chan_mib_info(phy, true); +} + +void mt7996_mac_set_timing(struct mt7996_phy *phy) +{ + s16 coverage_class = phy->coverage_class; + struct mt7996_dev *dev = phy->dev; + struct mt7996_phy *phy2 = mt7996_phy2(dev); + struct mt7996_phy *phy3 = mt7996_phy3(dev); + u32 val, reg_offset; + u32 cck = FIELD_PREP(MT_TIMEOUT_VAL_PLCP, 231) | + FIELD_PREP(MT_TIMEOUT_VAL_CCA, 48); + u32 ofdm = FIELD_PREP(MT_TIMEOUT_VAL_PLCP, 60) | + FIELD_PREP(MT_TIMEOUT_VAL_CCA, 28); + u8 band_idx = phy->mt76->band_idx; + int offset; + bool a_band = !(phy->mt76->chandef.chan->band == NL80211_BAND_2GHZ); + + if (!test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) + return; + + if (phy2) + coverage_class = max_t(s16, dev->phy.coverage_class, + phy2->coverage_class); + + if (phy3) + coverage_class = max_t(s16, coverage_class, + phy3->coverage_class); + + mt76_set(dev, MT_ARB_SCR(band_idx), + MT_ARB_SCR_TX_DISABLE | MT_ARB_SCR_RX_DISABLE); + udelay(1); + + offset = 3 * coverage_class; + reg_offset = FIELD_PREP(MT_TIMEOUT_VAL_PLCP, offset) | + FIELD_PREP(MT_TIMEOUT_VAL_CCA, offset); + + mt76_wr(dev, MT_TMAC_CDTR(band_idx), cck + reg_offset); + mt76_wr(dev, MT_TMAC_ODTR(band_idx), ofdm + reg_offset); + mt76_wr(dev, MT_TMAC_ICR0(band_idx), + FIELD_PREP(MT_IFS_EIFS_OFDM, a_band ? 84 : 78) | + FIELD_PREP(MT_IFS_RIFS, 2) | + FIELD_PREP(MT_IFS_SIFS, 10) | + FIELD_PREP(MT_IFS_SLOT, phy->slottime)); + + if (!a_band) + mt76_wr(dev, MT_TMAC_ICR1(band_idx), + FIELD_PREP(MT_IFS_EIFS_CCK, 314)); + + if (phy->slottime < 20 || a_band) + val = MT7996_CFEND_RATE_DEFAULT; + else + val = MT7996_CFEND_RATE_11B; + + mt76_rmw_field(dev, MT_AGG_ACR0(band_idx), MT_AGG_ACR_CFEND_RATE, val); + mt76_clear(dev, MT_ARB_SCR(band_idx), + MT_ARB_SCR_TX_DISABLE | MT_ARB_SCR_RX_DISABLE); +} + +void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band) +{ + mt76_set(dev, MT_WF_PHYRX_CSD_BAND_RXTD12(band), + MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR_ONLY | + MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR); + + mt76_set(dev, MT_WF_PHYRX_BAND_RX_CTRL1(band), + FIELD_PREP(MT_WF_PHYRX_BAND_RX_CTRL1_IPI_EN, 0x5)); +} + +static u8 +mt7996_phy_get_nf(struct mt7996_phy *phy, u8 band_idx) +{ + static const u8 nf_power[] = { 92, 89, 86, 83, 80, 75, 70, 65, 60, 55, 52 }; + struct mt7996_dev *dev = phy->dev; + u32 val, sum = 0, n = 0; + int ant, i; + + for (ant = 0; ant < hweight8(phy->mt76->antenna_mask); ant++) { + u32 reg = MT_WF_PHYRX_CSD_IRPI(band_idx, ant); + + for (i = 0; i < ARRAY_SIZE(nf_power); i++, reg += 4) { + val = mt76_rr(dev, reg); + sum += val * nf_power[i]; + n += val; + } + } + + return n ? sum / n : 0; +} + +void mt7996_update_channel(struct mt76_phy *mphy) +{ + struct mt7996_phy *phy = (struct mt7996_phy *)mphy->priv; + struct mt76_channel_state *state = mphy->chan_state; + int nf; + + mt7996_mcu_get_chan_mib_info(phy, false); + + nf = mt7996_phy_get_nf(phy, mphy->band_idx); + if (!phy->noise) + phy->noise = nf << 4; + else if (nf) + phy->noise += nf - (phy->noise >> 4); + + state->noise = -(phy->noise >> 4); +} + +static bool +mt7996_wait_reset_state(struct mt7996_dev *dev, u32 state) +{ + bool ret; + + ret = wait_event_timeout(dev->reset_wait, + (READ_ONCE(dev->reset_state) & state), + MT7996_RESET_TIMEOUT); + + WARN(!ret, "Timeout waiting for MCU reset state %x\n", state); + return ret; +} + +static void +mt7996_update_vif_beacon(void *priv, u8 *mac, struct ieee80211_vif *vif) +{ + struct ieee80211_hw *hw = priv; + + switch (vif->type) { + case NL80211_IFTYPE_MESH_POINT: + case NL80211_IFTYPE_ADHOC: + case NL80211_IFTYPE_AP: + mt7996_mcu_add_beacon(hw, vif, vif->bss_conf.enable_beacon); + break; + default: + break; + } +} + +static void +mt7996_update_beacons(struct mt7996_dev *dev) +{ + struct mt76_phy *phy2, *phy3; + + ieee80211_iterate_active_interfaces(dev->mt76.hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_update_vif_beacon, dev->mt76.hw); + + phy2 = dev->mt76.phys[MT_BAND1]; + if (!phy2) + return; + + ieee80211_iterate_active_interfaces(phy2->hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_update_vif_beacon, phy2->hw); + + phy3 = dev->mt76.phys[MT_BAND2]; + if (!phy3) + return; + + ieee80211_iterate_active_interfaces(phy3->hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_update_vif_beacon, phy3->hw); +} + +static void +mt7996_dma_reset(struct mt7996_dev *dev) +{ + struct mt76_phy *phy2 = dev->mt76.phys[MT_BAND1]; + struct mt76_phy *phy3 = dev->mt76.phys[MT_BAND2]; + u32 hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + int i; + + mt76_clear(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN); + + if (dev->hif2) + mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN); + + usleep_range(1000, 2000); + + for (i = 0; i < __MT_TXQ_MAX; i++) { + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true); + if (phy2) + mt76_queue_tx_cleanup(dev, phy2->q_tx[i], true); + if (phy3) + mt76_queue_tx_cleanup(dev, phy3->q_tx[i], true); + } + + for (i = 0; i < __MT_MCUQ_MAX; i++) + mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true); + + mt76_for_each_q_rx(&dev->mt76, i) + mt76_queue_rx_reset(dev, i); + + mt76_tx_status_check(&dev->mt76, true); + + /* re-init prefetch settings after reset */ + mt7996_dma_prefetch(dev); + + mt76_set(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN); + + if (dev->hif2) + mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN); +} + +void mt7996_tx_token_put(struct mt7996_dev *dev) +{ + struct mt76_txwi_cache *txwi; + int id; + + spin_lock_bh(&dev->mt76.token_lock); + idr_for_each_entry(&dev->mt76.token, txwi, id) { + mt7996_txwi_free(dev, txwi, NULL, NULL); + dev->mt76.token_count--; + } + spin_unlock_bh(&dev->mt76.token_lock); + idr_destroy(&dev->mt76.token); +} + +/* system error recovery */ +void mt7996_mac_reset_work(struct work_struct *work) +{ + struct mt7996_phy *phy2, *phy3; + struct mt7996_dev *dev; + int i; + + dev = container_of(work, struct mt7996_dev, reset_work); + phy2 = mt7996_phy2(dev); + phy3 = mt7996_phy3(dev); + + if (!(READ_ONCE(dev->reset_state) & MT_MCU_CMD_STOP_DMA)) + return; + + ieee80211_stop_queues(mt76_hw(dev)); + if (phy2) + ieee80211_stop_queues(phy2->mt76->hw); + if (phy3) + ieee80211_stop_queues(phy3->mt76->hw); + + set_bit(MT76_RESET, &dev->mphy.state); + set_bit(MT76_MCU_RESET, &dev->mphy.state); + wake_up(&dev->mt76.mcu.wait); + cancel_delayed_work_sync(&dev->mphy.mac_work); + if (phy2) { + set_bit(MT76_RESET, &phy2->mt76->state); + cancel_delayed_work_sync(&phy2->mt76->mac_work); + } + if (phy3) { + set_bit(MT76_RESET, &phy3->mt76->state); + cancel_delayed_work_sync(&phy3->mt76->mac_work); + } + mt76_worker_disable(&dev->mt76.tx_worker); + mt76_for_each_q_rx(&dev->mt76, i) + napi_disable(&dev->mt76.napi[i]); + napi_disable(&dev->mt76.tx_napi); + + mutex_lock(&dev->mt76.mutex); + + mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); + + if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { + mt7996_dma_reset(dev); + + mt7996_tx_token_put(dev); + idr_init(&dev->mt76.token); + + mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_INIT); + mt7996_wait_reset_state(dev, MT_MCU_CMD_RECOVERY_DONE); + } + + clear_bit(MT76_MCU_RESET, &dev->mphy.state); + clear_bit(MT76_RESET, &dev->mphy.state); + if (phy2) + clear_bit(MT76_RESET, &phy2->mt76->state); + if (phy3) + clear_bit(MT76_RESET, &phy3->mt76->state); + + local_bh_disable(); + mt76_for_each_q_rx(&dev->mt76, i) { + napi_enable(&dev->mt76.napi[i]); + napi_schedule(&dev->mt76.napi[i]); + } + local_bh_enable(); + + tasklet_schedule(&dev->irq_tasklet); + + mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_RESET_DONE); + mt7996_wait_reset_state(dev, MT_MCU_CMD_NORMAL_STATE); + + mt76_worker_enable(&dev->mt76.tx_worker); + + local_bh_disable(); + napi_enable(&dev->mt76.tx_napi); + napi_schedule(&dev->mt76.tx_napi); + local_bh_enable(); + + ieee80211_wake_queues(mt76_hw(dev)); + if (phy2) + ieee80211_wake_queues(phy2->mt76->hw); + if (phy3) + ieee80211_wake_queues(phy3->mt76->hw); + + mutex_unlock(&dev->mt76.mutex); + + mt7996_update_beacons(dev); + + ieee80211_queue_delayed_work(mt76_hw(dev), &dev->mphy.mac_work, + MT7996_WATCHDOG_TIME); + if (phy2) + ieee80211_queue_delayed_work(phy2->mt76->hw, + &phy2->mt76->mac_work, + MT7996_WATCHDOG_TIME); + if (phy3) + ieee80211_queue_delayed_work(phy3->mt76->hw, + &phy3->mt76->mac_work, + MT7996_WATCHDOG_TIME); +} + +void mt7996_mac_update_stats(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + struct mib_stats *mib = &phy->mib; + u8 band_idx = phy->mt76->band_idx; + u32 cnt; + int i; + + cnt = mt76_rr(dev, MT_MIB_RSCR1(band_idx)); + mib->fcs_err_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR33(band_idx)); + mib->rx_fifo_full_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR31(band_idx)); + mib->rx_mpdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_SDR6(band_idx)); + mib->channel_idle_cnt += FIELD_GET(MT_MIB_SDR6_CHANNEL_IDL_CNT_MASK, cnt); + + cnt = mt76_rr(dev, MT_MIB_RVSR0(band_idx)); + mib->rx_vector_mismatch_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR35(band_idx)); + mib->rx_delimiter_fail_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR36(band_idx)); + mib->rx_len_mismatch_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR0(band_idx)); + mib->tx_ampdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR2(band_idx)); + mib->tx_stop_q_empty_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR3(band_idx)); + mib->tx_mpdu_attempts_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR4(band_idx)); + mib->tx_mpdu_success_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR27(band_idx)); + mib->rx_ampdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR28(band_idx)); + mib->rx_ampdu_bytes_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR29(band_idx)); + mib->rx_ampdu_valid_subframe_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RSCR30(band_idx)); + mib->rx_ampdu_valid_subframe_bytes_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_SDR27(band_idx)); + mib->tx_rwp_fail_cnt += FIELD_GET(MT_MIB_SDR27_TX_RWP_FAIL_CNT, cnt); + + cnt = mt76_rr(dev, MT_MIB_SDR28(band_idx)); + mib->tx_rwp_need_cnt += FIELD_GET(MT_MIB_SDR28_TX_RWP_NEED_CNT, cnt); + + cnt = mt76_rr(dev, MT_UMIB_RPDCR(band_idx)); + mib->rx_pfdrop_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_RVSR1(band_idx)); + mib->rx_vec_queue_overflow_drop_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR1(band_idx)); + mib->rx_ba_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR0(band_idx)); + mib->tx_bf_ebf_ppdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR1(band_idx)); + mib->tx_bf_ibf_ppdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR2(band_idx)); + mib->tx_mu_bf_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR5(band_idx)); + mib->tx_mu_mpdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR6(band_idx)); + mib->tx_mu_acked_mpdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_TSCR7(band_idx)); + mib->tx_su_acked_mpdu_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR3(band_idx)); + mib->tx_bf_rx_fb_ht_cnt += cnt; + mib->tx_bf_rx_fb_all_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR4(band_idx)); + mib->tx_bf_rx_fb_vht_cnt += cnt; + mib->tx_bf_rx_fb_all_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR5(band_idx)); + mib->tx_bf_rx_fb_he_cnt += cnt; + mib->tx_bf_rx_fb_all_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR6(band_idx)); + mib->tx_bf_rx_fb_eht_cnt += cnt; + mib->tx_bf_rx_fb_all_cnt += cnt; + + cnt = mt76_rr(dev, MT_ETBF_RX_FB_CONT(band_idx)); + mib->tx_bf_rx_fb_bw = FIELD_GET(MT_ETBF_RX_FB_BW, cnt); + mib->tx_bf_rx_fb_nc_cnt += FIELD_GET(MT_ETBF_RX_FB_NC, cnt); + mib->tx_bf_rx_fb_nr_cnt += FIELD_GET(MT_ETBF_RX_FB_NR, cnt); + + cnt = mt76_rr(dev, MT_MIB_BSCR7(band_idx)); + mib->tx_bf_fb_trig_cnt += cnt; + + cnt = mt76_rr(dev, MT_MIB_BSCR17(band_idx)); + mib->tx_bf_fb_cpl_cnt += cnt; + + for (i = 0; i < ARRAY_SIZE(mib->tx_amsdu); i++) { + cnt = mt76_rr(dev, MT_PLE_AMSDU_PACK_MSDU_CNT(i)); + mib->tx_amsdu[i] += cnt; + mib->tx_amsdu_cnt += cnt; + } + + /* rts count */ + cnt = mt76_rr(dev, MT_MIB_BTSCR5(band_idx)); + mib->rts_cnt += cnt; + + /* rts retry count */ + cnt = mt76_rr(dev, MT_MIB_BTSCR6(band_idx)); + mib->rts_retries_cnt += cnt; + + /* ba miss count */ + cnt = mt76_rr(dev, MT_MIB_BTSCR0(band_idx)); + mib->ba_miss_cnt += cnt; + + /* ack fail count */ + cnt = mt76_rr(dev, MT_MIB_BFTFCR(band_idx)); + mib->ack_fail_cnt += cnt; + + for (i = 0; i < 16; i++) { + cnt = mt76_rr(dev, MT_TX_AGG_CNT(band_idx, i)); + phy->mt76->aggr_stats[i] += cnt; + } +} + +void mt7996_mac_sta_rc_work(struct work_struct *work) +{ + struct mt7996_dev *dev = container_of(work, struct mt7996_dev, rc_work); + struct ieee80211_sta *sta; + struct ieee80211_vif *vif; + struct mt7996_sta *msta; + u32 changed; + LIST_HEAD(list); + + spin_lock_bh(&dev->sta_poll_lock); + list_splice_init(&dev->sta_rc_list, &list); + + while (!list_empty(&list)) { + msta = list_first_entry(&list, struct mt7996_sta, rc_list); + list_del_init(&msta->rc_list); + changed = msta->changed; + msta->changed = 0; + spin_unlock_bh(&dev->sta_poll_lock); + + sta = container_of((void *)msta, struct ieee80211_sta, drv_priv); + vif = container_of((void *)msta->vif, struct ieee80211_vif, drv_priv); + + if (changed & (IEEE80211_RC_SUPP_RATES_CHANGED | + IEEE80211_RC_NSS_CHANGED | + IEEE80211_RC_BW_CHANGED)) + mt7996_mcu_add_rate_ctrl(dev, vif, sta, true); + + /* TODO: smps change */ + + spin_lock_bh(&dev->sta_poll_lock); + } + + spin_unlock_bh(&dev->sta_poll_lock); +} + +void mt7996_mac_work(struct work_struct *work) +{ + struct mt7996_phy *phy; + struct mt76_phy *mphy; + + mphy = (struct mt76_phy *)container_of(work, struct mt76_phy, + mac_work.work); + phy = mphy->priv; + + mutex_lock(&mphy->dev->mutex); + + mt76_update_survey(mphy); + if (++mphy->mac_work_count == 5) { + mphy->mac_work_count = 0; + + mt7996_mac_update_stats(phy); + } + + mutex_unlock(&mphy->dev->mutex); + + mt76_tx_status_check(mphy->dev, false); + + ieee80211_queue_delayed_work(mphy->hw, &mphy->mac_work, + MT7996_WATCHDOG_TIME); +} + +static void mt7996_dfs_stop_radar_detector(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + + if (phy->rdd_state & BIT(0)) + mt7996_mcu_rdd_cmd(dev, RDD_STOP, 0, + MT_RX_SEL0, 0); + if (phy->rdd_state & BIT(1)) + mt7996_mcu_rdd_cmd(dev, RDD_STOP, 1, + MT_RX_SEL0, 0); +} + +static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int chain) +{ + int err, region; + + switch (dev->mt76.region) { + case NL80211_DFS_ETSI: + region = 0; + break; + case NL80211_DFS_JP: + region = 2; + break; + case NL80211_DFS_FCC: + default: + region = 1; + break; + } + + err = mt7996_mcu_rdd_cmd(dev, RDD_START, chain, + MT_RX_SEL0, region); + if (err < 0) + return err; + + return mt7996_mcu_rdd_cmd(dev, RDD_DET_MODE, chain, + MT_RX_SEL0, 1); +} + +static int mt7996_dfs_start_radar_detector(struct mt7996_phy *phy) +{ + struct cfg80211_chan_def *chandef = &phy->mt76->chandef; + struct mt7996_dev *dev = phy->dev; + u8 band_idx = phy->mt76->band_idx; + int err; + + /* start CAC */ + err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_START, band_idx, + MT_RX_SEL0, 0); + if (err < 0) + return err; + + err = mt7996_dfs_start_rdd(dev, band_idx); + if (err < 0) + return err; + + phy->rdd_state |= BIT(band_idx); + + if (chandef->width == NL80211_CHAN_WIDTH_160 || + chandef->width == NL80211_CHAN_WIDTH_80P80) { + err = mt7996_dfs_start_rdd(dev, 1); + if (err < 0) + return err; + + phy->rdd_state |= BIT(1); + } + + return 0; +} + +static int +mt7996_dfs_init_radar_specs(struct mt7996_phy *phy) +{ + const struct mt7996_dfs_radar_spec *radar_specs; + struct mt7996_dev *dev = phy->dev; + int err, i; + + switch (dev->mt76.region) { + case NL80211_DFS_FCC: + radar_specs = &fcc_radar_specs; + err = mt7996_mcu_set_fcc5_lpn(dev, 8); + if (err < 0) + return err; + break; + case NL80211_DFS_ETSI: + radar_specs = &etsi_radar_specs; + break; + case NL80211_DFS_JP: + radar_specs = &jp_radar_specs; + break; + default: + return -EINVAL; + } + + for (i = 0; i < ARRAY_SIZE(radar_specs->radar_pattern); i++) { + err = mt7996_mcu_set_radar_th(dev, i, + &radar_specs->radar_pattern[i]); + if (err < 0) + return err; + } + + return mt7996_mcu_set_pulse_th(dev, &radar_specs->pulse_th); +} + +int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + enum mt76_dfs_state dfs_state, prev_state; + int err; + + prev_state = phy->mt76->dfs_state; + dfs_state = mt76_phy_dfs_state(phy->mt76); + + if (prev_state == dfs_state) + return 0; + + if (prev_state == MT_DFS_STATE_UNKNOWN) + mt7996_dfs_stop_radar_detector(phy); + + if (dfs_state == MT_DFS_STATE_DISABLED) + goto stop; + + if (prev_state <= MT_DFS_STATE_DISABLED) { + err = mt7996_dfs_init_radar_specs(phy); + if (err < 0) + return err; + + err = mt7996_dfs_start_radar_detector(phy); + if (err < 0) + return err; + + phy->mt76->dfs_state = MT_DFS_STATE_CAC; + } + + if (dfs_state == MT_DFS_STATE_CAC) + return 0; + + err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_END, + phy->mt76->band_idx, MT_RX_SEL0, 0); + if (err < 0) { + phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN; + return err; + } + + phy->mt76->dfs_state = MT_DFS_STATE_ACTIVE; + return 0; + +stop: + err = mt7996_mcu_rdd_cmd(dev, RDD_NORMAL_START, + phy->mt76->band_idx, MT_RX_SEL0, 0); + if (err < 0) + return err; + + mt7996_dfs_stop_radar_detector(phy); + phy->mt76->dfs_state = MT_DFS_STATE_DISABLED; + + return 0; +} + +static int +mt7996_mac_twt_duration_align(int duration) +{ + return duration << 8; +} + +static u64 +mt7996_mac_twt_sched_list_add(struct mt7996_dev *dev, + struct mt7996_twt_flow *flow) +{ + struct mt7996_twt_flow *iter, *iter_next; + u32 duration = flow->duration << 8; + u64 start_tsf; + + iter = list_first_entry_or_null(&dev->twt_list, + struct mt7996_twt_flow, list); + if (!iter || !iter->sched || iter->start_tsf > duration) { + /* add flow as first entry in the list */ + list_add(&flow->list, &dev->twt_list); + return 0; + } + + list_for_each_entry_safe(iter, iter_next, &dev->twt_list, list) { + start_tsf = iter->start_tsf + + mt7996_mac_twt_duration_align(iter->duration); + if (list_is_last(&iter->list, &dev->twt_list)) + break; + + if (!iter_next->sched || + iter_next->start_tsf > start_tsf + duration) { + list_add(&flow->list, &iter->list); + goto out; + } + } + + /* add flow as last entry in the list */ + list_add_tail(&flow->list, &dev->twt_list); +out: + return start_tsf; +} + +static int mt7996_mac_check_twt_req(struct ieee80211_twt_setup *twt) +{ + struct ieee80211_twt_params *twt_agrt; + u64 interval, duration; + u16 mantissa; + u8 exp; + + /* only individual agreement supported */ + if (twt->control & IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST) + return -EOPNOTSUPP; + + /* only 256us unit supported */ + if (twt->control & IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT) + return -EOPNOTSUPP; + + twt_agrt = (struct ieee80211_twt_params *)twt->params; + + /* explicit agreement not supported */ + if (!(twt_agrt->req_type & cpu_to_le16(IEEE80211_TWT_REQTYPE_IMPLICIT))) + return -EOPNOTSUPP; + + exp = FIELD_GET(IEEE80211_TWT_REQTYPE_WAKE_INT_EXP, + le16_to_cpu(twt_agrt->req_type)); + mantissa = le16_to_cpu(twt_agrt->mantissa); + duration = twt_agrt->min_twt_dur << 8; + + interval = (u64)mantissa << exp; + if (interval < duration) + return -EOPNOTSUPP; + + return 0; +} + +void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw, + struct ieee80211_sta *sta, + struct ieee80211_twt_setup *twt) +{ + enum ieee80211_twt_setup_cmd setup_cmd = TWT_SETUP_CMD_REJECT; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct ieee80211_twt_params *twt_agrt = (void *)twt->params; + u16 req_type = le16_to_cpu(twt_agrt->req_type); + enum ieee80211_twt_setup_cmd sta_setup_cmd; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_twt_flow *flow; + int flowid, table_id; + u8 exp; + + if (mt7996_mac_check_twt_req(twt)) + goto out; + + mutex_lock(&dev->mt76.mutex); + + if (dev->twt.n_agrt == MT7996_MAX_TWT_AGRT) + goto unlock; + + if (hweight8(msta->twt.flowid_mask) == ARRAY_SIZE(msta->twt.flow)) + goto unlock; + + flowid = ffs(~msta->twt.flowid_mask) - 1; + le16p_replace_bits(&twt_agrt->req_type, flowid, + IEEE80211_TWT_REQTYPE_FLOWID); + + table_id = ffs(~dev->twt.table_mask) - 1; + exp = FIELD_GET(IEEE80211_TWT_REQTYPE_WAKE_INT_EXP, req_type); + sta_setup_cmd = FIELD_GET(IEEE80211_TWT_REQTYPE_SETUP_CMD, req_type); + + flow = &msta->twt.flow[flowid]; + memset(flow, 0, sizeof(*flow)); + INIT_LIST_HEAD(&flow->list); + flow->wcid = msta->wcid.idx; + flow->table_id = table_id; + flow->id = flowid; + flow->duration = twt_agrt->min_twt_dur; + flow->mantissa = twt_agrt->mantissa; + flow->exp = exp; + flow->protection = !!(req_type & IEEE80211_TWT_REQTYPE_PROTECTION); + flow->flowtype = !!(req_type & IEEE80211_TWT_REQTYPE_FLOWTYPE); + flow->trigger = !!(req_type & IEEE80211_TWT_REQTYPE_TRIGGER); + + if (sta_setup_cmd == TWT_SETUP_CMD_REQUEST || + sta_setup_cmd == TWT_SETUP_CMD_SUGGEST) { + u64 interval = (u64)le16_to_cpu(twt_agrt->mantissa) << exp; + u64 flow_tsf, curr_tsf; + u32 rem; + + flow->sched = true; + flow->start_tsf = mt7996_mac_twt_sched_list_add(dev, flow); + curr_tsf = __mt7996_get_tsf(hw, msta->vif); + div_u64_rem(curr_tsf - flow->start_tsf, interval, &rem); + flow_tsf = curr_tsf + interval - rem; + twt_agrt->twt = cpu_to_le64(flow_tsf); + } else { + list_add_tail(&flow->list, &dev->twt_list); + } + flow->tsf = le64_to_cpu(twt_agrt->twt); + + if (mt7996_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) + goto unlock; + + setup_cmd = TWT_SETUP_CMD_ACCEPT; + dev->twt.table_mask |= BIT(table_id); + msta->twt.flowid_mask |= BIT(flowid); + dev->twt.n_agrt++; + +unlock: + mutex_unlock(&dev->mt76.mutex); +out: + le16p_replace_bits(&twt_agrt->req_type, setup_cmd, + IEEE80211_TWT_REQTYPE_SETUP_CMD); + twt->control = (twt->control & IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT) | + (twt->control & IEEE80211_TWT_CONTROL_RX_DISABLED); +} + +void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev, + struct mt7996_sta *msta, + u8 flowid) +{ + struct mt7996_twt_flow *flow; + + lockdep_assert_held(&dev->mt76.mutex); + + if (flowid >= ARRAY_SIZE(msta->twt.flow)) + return; + + if (!(msta->twt.flowid_mask & BIT(flowid))) + return; + + flow = &msta->twt.flow[flowid]; + if (mt7996_mcu_twt_agrt_update(dev, msta->vif, flow, + MCU_TWT_AGRT_DELETE)) + return; + + list_del_init(&flow->list); + msta->twt.flowid_mask &= ~BIT(flowid); + dev->twt.table_mask &= ~BIT(flow->table_id); + dev->twt.n_agrt--; +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.h b/drivers/net/wireless/mediatek/mt76/mt7996/mac.h new file mode 100644 index 000000000000..9f68852012b9 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.h @@ -0,0 +1,398 @@ +/* SPDX-License-Identifier: ISC */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef __MT7996_MAC_H +#define __MT7996_MAC_H + +#define MT_CT_PARSE_LEN 72 +#define MT_CT_DMA_BUF_NUM 2 + +#define MT_RXD0_LENGTH GENMASK(15, 0) +#define MT_RXD0_PKT_TYPE GENMASK(31, 27) + +#define MT_RXD0_NORMAL_ETH_TYPE_OFS GENMASK(22, 16) +#define MT_RXD0_NORMAL_IP_SUM BIT(23) +#define MT_RXD0_NORMAL_UDP_TCP_SUM BIT(24) + +#define MT_RXD0_SW_PKT_TYPE_MASK GENMASK(31, 16) +#define MT_RXD0_SW_PKT_TYPE_MAP 0x380F +#define MT_RXD0_SW_PKT_TYPE_FRAME 0x3801 + +enum rx_pkt_type { + PKT_TYPE_TXS, + PKT_TYPE_TXRXV, + PKT_TYPE_NORMAL, + PKT_TYPE_RX_DUP_RFB, + PKT_TYPE_RX_TMR, + PKT_TYPE_RETRIEVE, + PKT_TYPE_TXRX_NOTIFY, + PKT_TYPE_RX_EVENT, + PKT_TYPE_RX_FW_MONITOR = 0x0c, +}; + +/* RXD DW1 */ +#define MT_RXD1_NORMAL_WLAN_IDX GENMASK(11, 0) +#define MT_RXD1_NORMAL_GROUP_1 BIT(16) +#define MT_RXD1_NORMAL_GROUP_2 BIT(17) +#define MT_RXD1_NORMAL_GROUP_3 BIT(18) +#define MT_RXD1_NORMAL_GROUP_4 BIT(19) +#define MT_RXD1_NORMAL_GROUP_5 BIT(20) +#define MT_RXD1_NORMAL_KEY_ID GENMASK(22, 21) +#define MT_RXD1_NORMAL_CM BIT(23) +#define MT_RXD1_NORMAL_CLM BIT(24) +#define MT_RXD1_NORMAL_ICV_ERR BIT(25) +#define MT_RXD1_NORMAL_TKIP_MIC_ERR BIT(26) +#define MT_RXD1_NORMAL_BAND_IDX GENMASK(28, 27) +#define MT_RXD1_NORMAL_SPP_EN BIT(29) +#define MT_RXD1_NORMAL_ADD_OM BIT(30) +#define MT_RXD1_NORMAL_SEC_DONE BIT(31) + +/* RXD DW2 */ +#define MT_RXD2_NORMAL_BSSID GENMASK(5, 0) +#define MT_RXD2_NORMAL_MAC_HDR_LEN GENMASK(12, 8) +#define MT_RXD2_NORMAL_HDR_TRANS BIT(7) +#define MT_RXD2_NORMAL_HDR_OFFSET GENMASK(15, 13) +#define MT_RXD2_NORMAL_SEC_MODE GENMASK(20, 16) +#define MT_RXD2_NORMAL_MU_BAR BIT(21) +#define MT_RXD2_NORMAL_SW_BIT BIT(22) +#define MT_RXD2_NORMAL_AMSDU_ERR BIT(23) +#define MT_RXD2_NORMAL_MAX_LEN_ERROR BIT(24) +#define MT_RXD2_NORMAL_HDR_TRANS_ERROR BIT(25) +#define MT_RXD2_NORMAL_INT_FRAME BIT(26) +#define MT_RXD2_NORMAL_FRAG BIT(27) +#define MT_RXD2_NORMAL_NULL_FRAME BIT(28) +#define MT_RXD2_NORMAL_NDATA BIT(29) +#define MT_RXD2_NORMAL_NON_AMPDU BIT(30) +#define MT_RXD2_NORMAL_BF_REPORT BIT(31) + +/* RXD DW3 */ +#define MT_RXD3_NORMAL_RXV_SEQ GENMASK(7, 0) +#define MT_RXD3_NORMAL_CH_FREQ GENMASK(15, 8) +#define MT_RXD3_NORMAL_ADDR_TYPE GENMASK(17, 16) +#define MT_RXD3_NORMAL_U2M BIT(0) +#define MT_RXD3_NORMAL_HTC_VLD BIT(18) +#define MT_RXD3_NORMAL_BEACON_MC BIT(20) +#define MT_RXD3_NORMAL_BEACON_UC BIT(21) +#define MT_RXD3_NORMAL_CO_ANT BIT(22) +#define MT_RXD3_NORMAL_FCS_ERR BIT(24) +#define MT_RXD3_NORMAL_VLAN2ETH BIT(31) + +/* RXD DW4 */ +#define MT_RXD4_NORMAL_PAYLOAD_FORMAT GENMASK(1, 0) +#define MT_RXD4_FIRST_AMSDU_FRAME GENMASK(1, 0) +#define MT_RXD4_MID_AMSDU_FRAME BIT(1) +#define MT_RXD4_LAST_AMSDU_FRAME BIT(0) + +#define MT_RXV_HDR_BAND_IDX BIT(24) + +/* RXD GROUP4 */ +#define MT_RXD8_FRAME_CONTROL GENMASK(15, 0) + +#define MT_RXD10_SEQ_CTRL GENMASK(15, 0) +#define MT_RXD10_QOS_CTL GENMASK(31, 16) + +#define MT_RXD11_HT_CONTROL GENMASK(31, 0) + +/* P-RXV */ +#define MT_PRXV_TX_RATE GENMASK(6, 0) +#define MT_PRXV_TX_DCM BIT(4) +#define MT_PRXV_TX_ER_SU_106T BIT(5) +#define MT_PRXV_NSTS GENMASK(10, 7) +#define MT_PRXV_TXBF BIT(11) +#define MT_PRXV_HT_AD_CODE BIT(12) +#define MT_PRXV_HE_RU_ALLOC_L GENMASK(31, 28) +#define MT_PRXV_HE_RU_ALLOC_H GENMASK(3, 0) +#define MT_PRXV_RCPI3 GENMASK(31, 24) +#define MT_PRXV_RCPI2 GENMASK(23, 16) +#define MT_PRXV_RCPI1 GENMASK(15, 8) +#define MT_PRXV_RCPI0 GENMASK(7, 0) +#define MT_PRXV_HT_SHORT_GI GENMASK(4, 3) +#define MT_PRXV_HT_STBC GENMASK(10, 9) +#define MT_PRXV_TX_MODE GENMASK(14, 11) +#define MT_PRXV_FRAME_MODE GENMASK(2, 0) +#define MT_PRXV_DCM BIT(5) +#define MT_PRXV_NUM_RX BIT(8, 6) + +/* C-RXV */ +#define MT_CRXV_HT_STBC GENMASK(1, 0) +#define MT_CRXV_TX_MODE GENMASK(7, 4) +#define MT_CRXV_FRAME_MODE GENMASK(10, 8) +#define MT_CRXV_HT_SHORT_GI GENMASK(14, 13) +#define MT_CRXV_HE_LTF_SIZE GENMASK(18, 17) +#define MT_CRXV_HE_LDPC_EXT_SYM BIT(20) +#define MT_CRXV_HE_PE_DISAMBIG BIT(23) +#define MT_CRXV_HE_NUM_USER GENMASK(30, 24) +#define MT_CRXV_HE_UPLINK BIT(31) +#define MT_CRXV_HE_RU0 GENMASK(7, 0) +#define MT_CRXV_HE_RU1 GENMASK(15, 8) +#define MT_CRXV_HE_RU2 GENMASK(23, 16) +#define MT_CRXV_HE_RU3 GENMASK(31, 24) + +#define MT_CRXV_HE_MU_AID GENMASK(30, 20) + +#define MT_CRXV_HE_SR_MASK GENMASK(11, 8) +#define MT_CRXV_HE_SR1_MASK GENMASK(16, 12) +#define MT_CRXV_HE_SR2_MASK GENMASK(20, 17) +#define MT_CRXV_HE_SR3_MASK GENMASK(24, 21) + +#define MT_CRXV_HE_BSS_COLOR GENMASK(5, 0) +#define MT_CRXV_HE_TXOP_DUR GENMASK(12, 6) +#define MT_CRXV_HE_BEAM_CHNG BIT(13) +#define MT_CRXV_HE_DOPPLER BIT(16) + +enum tx_header_format { + MT_HDR_FORMAT_802_3, + MT_HDR_FORMAT_CMD, + MT_HDR_FORMAT_802_11, + MT_HDR_FORMAT_802_11_EXT, +}; + +enum tx_pkt_type { + MT_TX_TYPE_CT, + MT_TX_TYPE_SF, + MT_TX_TYPE_CMD, + MT_TX_TYPE_FW, +}; + +enum tx_port_idx { + MT_TX_PORT_IDX_LMAC, + MT_TX_PORT_IDX_MCU +}; + +enum tx_mcu_port_q_idx { + MT_TX_MCU_PORT_RX_Q0 = 0x20, + MT_TX_MCU_PORT_RX_Q1, + MT_TX_MCU_PORT_RX_Q2, + MT_TX_MCU_PORT_RX_Q3, + MT_TX_MCU_PORT_RX_FWDL = 0x3e +}; + +enum tx_mgnt_type { + MT_TX_NORMAL, + MT_TX_TIMING, + MT_TX_ADDBA, +}; + +#define MT_CT_INFO_APPLY_TXD BIT(0) +#define MT_CT_INFO_COPY_HOST_TXD_ALL BIT(1) +#define MT_CT_INFO_MGMT_FRAME BIT(2) +#define MT_CT_INFO_NONE_CIPHER_FRAME BIT(3) +#define MT_CT_INFO_HSR2_TX BIT(4) +#define MT_CT_INFO_FROM_HOST BIT(7) + +#define MT_TXD_SIZE (8 * 4) + +#define MT_TXD0_Q_IDX GENMASK(31, 25) +#define MT_TXD0_PKT_FMT GENMASK(24, 23) +#define MT_TXD0_ETH_TYPE_OFFSET GENMASK(22, 16) +#define MT_TXD0_TX_BYTES GENMASK(15, 0) + +#define MT_TXD1_FIXED_RATE BIT(31) +#define MT_TXD1_OWN_MAC GENMASK(30, 25) +#define MT_TXD1_TID GENMASK(24, 21) +#define MT_TXD1_BIP BIT(24) +#define MT_TXD1_ETH_802_3 BIT(20) +#define MT_TXD1_HDR_INFO GENMASK(20, 16) +#define MT_TXD1_HDR_FORMAT GENMASK(15, 14) +#define MT_TXD1_TGID GENMASK(13, 12) +#define MT_TXD1_WLAN_IDX GENMASK(11, 0) + +#define MT_TXD2_POWER_OFFSET GENMASK(31, 26) +#define MT_TXD2_MAX_TX_TIME GENMASK(25, 16) +#define MT_TXD2_FRAG GENMASK(15, 14) +#define MT_TXD2_HTC_VLD BIT(13) +#define MT_TXD2_DURATION BIT(12) +#define MT_TXD2_HDR_PAD GENMASK(11, 10) +#define MT_TXD2_RTS BIT(9) +#define MT_TXD2_OWN_MAC_MAP BIT(8) +#define MT_TXD2_BF_TYPE GENMASK(6, 7) +#define MT_TXD2_FRAME_TYPE GENMASK(5, 4) +#define MT_TXD2_SUB_TYPE GENMASK(3, 0) + +#define MT_TXD3_SN_VALID BIT(31) +#define MT_TXD3_PN_VALID BIT(30) +#define MT_TXD3_SW_POWER_MGMT BIT(29) +#define MT_TXD3_BA_DISABLE BIT(28) +#define MT_TXD3_SEQ GENMASK(27, 16) +#define MT_TXD3_REM_TX_COUNT GENMASK(15, 11) +#define MT_TXD3_TX_COUNT GENMASK(10, 6) +#define MT_TXD3_HW_AMSDU BIT(5) +#define MT_TXD3_BCM BIT(4) +#define MT_TXD3_EEOSP BIT(3) +#define MT_TXD3_EMRD BIT(2) +#define MT_TXD3_PROTECT_FRAME BIT(1) +#define MT_TXD3_NO_ACK BIT(0) + +#define MT_TXD4_PN_LOW GENMASK(31, 0) + +#define MT_TXD5_PN_HIGH GENMASK(31, 16) +#define MT_TXD5_FL BIT(15) +#define MT_TXD5_BYPASS_TBB BIT(14) +#define MT_TXD5_BYPASS_RBB BIT(13) +#define MT_TXD5_BSS_COLOR_ZERO BIT(12) +#define MT_TXD5_TX_STATUS_HOST BIT(10) +#define MT_TXD5_TX_STATUS_MCU BIT(9) +#define MT_TXD5_TX_STATUS_FMT BIT(8) +#define MT_TXD5_PID GENMASK(7, 0) + +#define MT_TXD6_TX_SRC GENMASK(31, 30) +#define MT_TXD6_VTA BIT(28) +#define MT_TXD6_FIXED_BW BIT(25) +#define MT_TXD6_BW GENMASK(24, 22) +#define MT_TXD6_TX_RATE GENMASK(21, 16) +#define MT_TXD6_TIMESTAMP_OFS_EN BIT(15) +#define MT_TXD6_TIMESTAMP_OFS_IDX GENMASK(14, 10) +#define MT_TXD6_MSDU_CNT GENMASK(9, 4) +#define MT_TXD6_SPE_ID_IDX BIT(10) +#define MT_TXD6_ANT_ID GENMASK(7, 4) +#define MT_TXD6_DIS_MAT BIT(3) +#define MT_TXD6_DAS BIT(2) +#define MT_TXD6_AMSDU_CAP BIT(1) + +#define MT_TXD7_TXD_LEN GENMASK(31, 30) +#define MT_TXD7_IP_SUM BIT(29) +#define MT_TXD7_DROP_BY_SDO BIT(28) +#define MT_TXD7_MAC_TXD BIT(27) +#define MT_TXD7_CTXD BIT(26) +#define MT_TXD7_CTXD_CNT GENMASK(25, 22) +#define MT_TXD7_UDP_TCP_SUM BIT(15) +#define MT_TXD7_TX_TIME GENMASK(9, 0) + +#define MT_TX_RATE_STBC BIT(13) +#define MT_TX_RATE_NSS GENMASK(13, 10) +#define MT_TX_RATE_MODE GENMASK(9, 6) +#define MT_TX_RATE_SU_EXT_TONE BIT(5) +#define MT_TX_RATE_DCM BIT(4) +/* VHT/HE only use bits 0-3 */ +#define MT_TX_RATE_IDX GENMASK(5, 0) + +struct mt7996_txp { + __le16 flags; + __le16 token; + u8 bss_idx; + __le16 rept_wds_wcid; + u8 nbuf; +#define MT_TXP_MAX_BUF_NUM 6 + __le32 buf[MT_TXP_MAX_BUF_NUM]; + __le16 len[MT_TXP_MAX_BUF_NUM]; +} __packed __aligned(4); + +#define MT_TXFREE0_PKT_TYPE GENMASK(31, 27) +#define MT_TXFREE0_MSDU_CNT GENMASK(25, 16) +#define MT_TXFREE0_RX_BYTE GENMASK(15, 0) + +#define MT_TXFREE1_VER GENMASK(18, 16) + +#define MT_TXFREE_INFO_PAIR BIT(31) +#define MT_TXFREE_INFO_HEADER BIT(30) +#define MT_TXFREE_INFO_WLAN_ID GENMASK(23, 12) +#define MT_TXFREE_INFO_MSDU_ID GENMASK(14, 0) + +#define MT_TXS0_BW GENMASK(31, 29) +#define MT_TXS0_TID GENMASK(28, 26) +#define MT_TXS0_AMPDU BIT(25) +#define MT_TXS0_TXS_FORMAT GENMASK(24, 23) +#define MT_TXS0_BA_ERROR BIT(22) +#define MT_TXS0_PS_FLAG BIT(21) +#define MT_TXS0_TXOP_TIMEOUT BIT(20) +#define MT_TXS0_BIP_ERROR BIT(19) + +#define MT_TXS0_QUEUE_TIMEOUT BIT(18) +#define MT_TXS0_RTS_TIMEOUT BIT(17) +#define MT_TXS0_ACK_TIMEOUT BIT(16) +#define MT_TXS0_ACK_ERROR_MASK GENMASK(18, 16) + +#define MT_TXS0_TX_STATUS_HOST BIT(15) +#define MT_TXS0_TX_STATUS_MCU BIT(14) +#define MT_TXS0_TX_RATE GENMASK(13, 0) + +#define MT_TXS1_SEQNO GENMASK(31, 20) +#define MT_TXS1_RESP_RATE GENMASK(19, 16) +#define MT_TXS1_RXV_SEQNO GENMASK(15, 8) +#define MT_TXS1_TX_POWER_DBM GENMASK(7, 0) + +#define MT_TXS2_BF_STATUS GENMASK(31, 30) +#define MT_TXS2_BAND GENMASK(29, 28) +#define MT_TXS2_WCID GENMASK(27, 16) +#define MT_TXS2_TX_DELAY GENMASK(15, 0) + +#define MT_TXS3_PID GENMASK(31, 24) +#define MT_TXS3_RATE_STBC BIT(7) +#define MT_TXS3_FIXED_RATE BIT(6) +#define MT_TXS3_SRC GENMASK(5, 4) +#define MT_TXS3_SHARED_ANTENNA BIT(3) +#define MT_TXS3_LAST_TX_RATE GENMASK(2, 0) + +#define MT_TXS4_TIMESTAMP GENMASK(31, 0) + +#define MT_TXS5_F0_FINAL_MPDU BIT(31) +#define MT_TXS5_F0_QOS BIT(30) +#define MT_TXS5_F0_TX_COUNT GENMASK(29, 25) +#define MT_TXS5_F0_FRONT_TIME GENMASK(24, 0) +#define MT_TXS5_F1_MPDU_TX_COUNT GENMASK(31, 24) +#define MT_TXS5_F1_MPDU_TX_BYTES GENMASK(23, 0) + +#define MT_TXS6_F0_NOISE_3 GENMASK(31, 24) +#define MT_TXS6_F0_NOISE_2 GENMASK(23, 16) +#define MT_TXS6_F0_NOISE_1 GENMASK(15, 8) +#define MT_TXS6_F0_NOISE_0 GENMASK(7, 0) +#define MT_TXS6_F1_MPDU_FAIL_COUNT GENMASK(31, 24) +#define MT_TXS6_F1_MPDU_FAIL_BYTES GENMASK(23, 0) + +#define MT_TXS7_F0_RCPI_3 GENMASK(31, 24) +#define MT_TXS7_F0_RCPI_2 GENMASK(23, 16) +#define MT_TXS7_F0_RCPI_1 GENMASK(15, 8) +#define MT_TXS7_F0_RCPI_0 GENMASK(7, 0) +#define MT_TXS7_F1_MPDU_RETRY_COUNT GENMASK(31, 24) +#define MT_TXS7_F1_MPDU_RETRY_BYTES GENMASK(23, 0) + +struct mt7996_dfs_pulse { + u32 max_width; /* us */ + int max_pwr; /* dbm */ + int min_pwr; /* dbm */ + u32 min_stgr_pri; /* us */ + u32 max_stgr_pri; /* us */ + u32 min_cr_pri; /* us */ + u32 max_cr_pri; /* us */ +}; + +struct mt7996_dfs_pattern { + u8 enb; + u8 stgr; + u8 min_crpn; + u8 max_crpn; + u8 min_crpr; + u8 min_pw; + u32 min_pri; + u32 max_pri; + u8 max_pw; + u8 min_crbn; + u8 max_crbn; + u8 min_stgpn; + u8 max_stgpn; + u8 min_stgpr; + u8 rsv[2]; + u32 min_stgpr_diff; +} __packed; + +struct mt7996_dfs_radar_spec { + struct mt7996_dfs_pulse pulse_th; + struct mt7996_dfs_pattern radar_pattern[16]; +}; + +static inline struct mt7996_txp * +mt7996_txwi_to_txp(struct mt76_dev *dev, struct mt76_txwi_cache *t) +{ + u8 *txwi; + + if (!t) + return NULL; + + txwi = mt76_get_txwi_ptr(dev, t); + + return (struct mt7996_txp *)(txwi + MT_TXD_SIZE); +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c new file mode 100644 index 000000000000..4421cd54311b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -0,0 +1,1334 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include "mt7996.h" +#include "mcu.h" + +static bool mt7996_dev_running(struct mt7996_dev *dev) +{ + struct mt7996_phy *phy; + + if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) + return true; + + phy = mt7996_phy2(dev); + if (phy && test_bit(MT76_STATE_RUNNING, &phy->mt76->state)) + return true; + + phy = mt7996_phy3(dev); + + return phy && test_bit(MT76_STATE_RUNNING, &phy->mt76->state); +} + +static int mt7996_start(struct ieee80211_hw *hw) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + bool running; + int ret; + + flush_work(&dev->init_work); + + mutex_lock(&dev->mt76.mutex); + + running = mt7996_dev_running(dev); + if (!running) { + ret = mt7996_mcu_set_hdr_trans(dev, true); + if (ret) + goto out; + } + + mt7996_mac_enable_nf(dev, phy->mt76->band_idx); + + ret = mt7996_mcu_set_rts_thresh(phy, 0x92b); + if (ret) + goto out; + + ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_RX_PATH); + if (ret) + goto out; + + set_bit(MT76_STATE_RUNNING, &phy->mt76->state); + + ieee80211_iterate_interfaces(dev->mt76.hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_mcu_set_pm, dev->mt76.hw); + + ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, + MT7996_WATCHDOG_TIME); + + if (!running) + mt7996_mac_reset_counters(phy); + +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static void mt7996_stop(struct ieee80211_hw *hw) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + + cancel_delayed_work_sync(&phy->mt76->mac_work); + + mutex_lock(&dev->mt76.mutex); + + clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); + + ieee80211_iterate_interfaces(dev->mt76.hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_mcu_set_pm, dev->mt76.hw); + + mutex_unlock(&dev->mt76.mutex); +} + +static inline int get_free_idx(u32 mask, u8 start, u8 end) +{ + return ffs(~mask & GENMASK(end, start)); +} + +static int get_omac_idx(enum nl80211_iftype type, u64 mask) +{ + int i; + + switch (type) { + case NL80211_IFTYPE_MESH_POINT: + case NL80211_IFTYPE_ADHOC: + case NL80211_IFTYPE_STATION: + /* prefer hw bssid slot 1-3 */ + i = get_free_idx(mask, HW_BSSID_1, HW_BSSID_3); + if (i) + return i - 1; + + if (type != NL80211_IFTYPE_STATION) + break; + + i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX); + if (i) + return i - 1; + + if (~mask & BIT(HW_BSSID_0)) + return HW_BSSID_0; + + break; + case NL80211_IFTYPE_MONITOR: + case NL80211_IFTYPE_AP: + /* ap uses hw bssid 0 and ext bssid */ + if (~mask & BIT(HW_BSSID_0)) + return HW_BSSID_0; + + i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX); + if (i) + return i - 1; + + break; + default: + WARN_ON(1); + break; + } + + return -1; +} + +static void mt7996_init_bitrate_mask(struct ieee80211_vif *vif) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + int i; + + for (i = 0; i < ARRAY_SIZE(mvif->bitrate_mask.control); i++) { + mvif->bitrate_mask.control[i].gi = NL80211_TXRATE_DEFAULT_GI; + mvif->bitrate_mask.control[i].he_gi = 0xff; + mvif->bitrate_mask.control[i].he_ltf = 0xff; + mvif->bitrate_mask.control[i].legacy = GENMASK(31, 0); + memset(mvif->bitrate_mask.control[i].ht_mcs, 0xff, + sizeof(mvif->bitrate_mask.control[i].ht_mcs)); + memset(mvif->bitrate_mask.control[i].vht_mcs, 0xff, + sizeof(mvif->bitrate_mask.control[i].vht_mcs)); + memset(mvif->bitrate_mask.control[i].he_mcs, 0xff, + sizeof(mvif->bitrate_mask.control[i].he_mcs)); + } +} + +static int mt7996_add_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt76_txq *mtxq; + u8 band_idx = phy->mt76->band_idx; + int idx, ret = 0; + + mutex_lock(&dev->mt76.mutex); + + if (vif->type == NL80211_IFTYPE_MONITOR && + is_zero_ether_addr(vif->addr)) + phy->monitor_vif = vif; + + mvif->mt76.idx = __ffs64(~dev->mt76.vif_mask); + if (mvif->mt76.idx >= (MT7996_MAX_INTERFACES << dev->dbdc_support)) { + ret = -ENOSPC; + goto out; + } + + idx = get_omac_idx(vif->type, phy->omac_mask); + if (idx < 0) { + ret = -ENOSPC; + goto out; + } + mvif->mt76.omac_idx = idx; + mvif->phy = phy; + mvif->mt76.band_idx = band_idx; + mvif->mt76.wmm_idx = band_idx; + + ret = mt7996_mcu_add_dev_info(phy, vif, true); + if (ret) + goto out; + + ret = mt7996_mcu_set_radio_en(phy, true); + if (ret) + goto out; + + dev->mt76.vif_mask |= BIT_ULL(mvif->mt76.idx); + phy->omac_mask |= BIT_ULL(mvif->mt76.omac_idx); + + idx = MT7996_WTBL_RESERVED - mvif->mt76.idx; + + INIT_LIST_HEAD(&mvif->sta.rc_list); + INIT_LIST_HEAD(&mvif->sta.poll_list); + mvif->sta.wcid.idx = idx; + mvif->sta.wcid.phy_idx = band_idx; + mvif->sta.wcid.hw_key_idx = -1; + mvif->sta.wcid.tx_info |= MT_WCID_TX_INFO_SET; + mt76_packet_id_init(&mvif->sta.wcid); + + mt7996_mac_wtbl_update(dev, idx, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + + if (vif->txq) { + mtxq = (struct mt76_txq *)vif->txq->drv_priv; + mtxq->wcid = idx; + } + + if (vif->type != NL80211_IFTYPE_AP && + (!mvif->mt76.omac_idx || mvif->mt76.omac_idx > 3)) + vif->offload_flags = 0; + vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR; + + mt7996_init_bitrate_mask(vif); + memset(&mvif->cap, -1, sizeof(mvif->cap)); + + mt7996_mcu_add_bss_info(phy, vif, true); + mt7996_mcu_add_sta(dev, vif, NULL, true); + rcu_assign_pointer(dev->mt76.wcid[idx], &mvif->sta.wcid); + +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static void mt7996_remove_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta = &mvif->sta; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + int idx = msta->wcid.idx; + + mt7996_mcu_add_bss_info(phy, vif, false); + mt7996_mcu_add_sta(dev, vif, NULL, false); + + if (vif == phy->monitor_vif) + phy->monitor_vif = NULL; + + mt7996_mcu_add_dev_info(phy, vif, false); + mt7996_mcu_set_radio_en(phy, false); + + rcu_assign_pointer(dev->mt76.wcid[idx], NULL); + + mutex_lock(&dev->mt76.mutex); + dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); + phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mutex_unlock(&dev->mt76.mutex); + + spin_lock_bh(&dev->sta_poll_lock); + if (!list_empty(&msta->poll_list)) + list_del_init(&msta->poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + + mt76_packet_id_flush(&dev->mt76, &msta->wcid); +} + +int mt7996_set_channel(struct mt7996_phy *phy) +{ + struct mt7996_dev *dev = phy->dev; + int ret; + + cancel_delayed_work_sync(&phy->mt76->mac_work); + + mutex_lock(&dev->mt76.mutex); + set_bit(MT76_RESET, &phy->mt76->state); + + mt76_set_channel(phy->mt76); + + ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_SWITCH); + if (ret) + goto out; + + mt7996_mac_set_timing(phy); + ret = mt7996_dfs_init_radar_detector(phy); + mt7996_mac_cca_stats_reset(phy); + + mt7996_mac_reset_counters(phy); + phy->noise = 0; + +out: + clear_bit(MT76_RESET, &phy->mt76->state); + mutex_unlock(&dev->mt76.mutex); + + mt76_txq_schedule_all(phy->mt76); + + ieee80211_queue_delayed_work(phy->mt76->hw, + &phy->mt76->mac_work, + MT7996_WATCHDOG_TIME); + + return ret; +} + +static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, + struct ieee80211_vif *vif, struct ieee80211_sta *sta, + struct ieee80211_key_conf *key) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta = sta ? (struct mt7996_sta *)sta->drv_priv : + &mvif->sta; + struct mt76_wcid *wcid = &msta->wcid; + u8 *wcid_keyidx = &wcid->hw_key_idx; + int idx = key->keyidx; + int err = 0; + + /* The hardware does not support per-STA RX GTK, fallback + * to software mode for these. + */ + if ((vif->type == NL80211_IFTYPE_ADHOC || + vif->type == NL80211_IFTYPE_MESH_POINT) && + (key->cipher == WLAN_CIPHER_SUITE_TKIP || + key->cipher == WLAN_CIPHER_SUITE_CCMP) && + !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) + return -EOPNOTSUPP; + + /* fall back to sw encryption for unsupported ciphers */ + switch (key->cipher) { + case WLAN_CIPHER_SUITE_AES_CMAC: + wcid_keyidx = &wcid->hw_key_idx2; + key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE; + break; + case WLAN_CIPHER_SUITE_TKIP: + case WLAN_CIPHER_SUITE_CCMP: + case WLAN_CIPHER_SUITE_CCMP_256: + case WLAN_CIPHER_SUITE_GCMP: + case WLAN_CIPHER_SUITE_GCMP_256: + case WLAN_CIPHER_SUITE_SMS4: + break; + case WLAN_CIPHER_SUITE_WEP40: + case WLAN_CIPHER_SUITE_WEP104: + default: + return -EOPNOTSUPP; + } + + mutex_lock(&dev->mt76.mutex); + + if (cmd == SET_KEY && !sta && !mvif->mt76.cipher) { + mvif->mt76.cipher = mt76_connac_mcu_get_cipher(key->cipher); + mt7996_mcu_add_bss_info(phy, vif, true); + } + + if (cmd == SET_KEY) + *wcid_keyidx = idx; + else if (idx == *wcid_keyidx) + *wcid_keyidx = -1; + else + goto out; + + mt76_wcid_key_setup(&dev->mt76, wcid, + cmd == SET_KEY ? key : NULL); + + err = mt7996_mcu_add_key(&dev->mt76, vif, &msta->bip, + key, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), + &msta->wcid, cmd); +out: + mutex_unlock(&dev->mt76.mutex); + + return err; +} + +static int mt7996_config(struct ieee80211_hw *hw, u32 changed) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + int ret; + + if (changed & IEEE80211_CONF_CHANGE_CHANNEL) { + ieee80211_stop_queues(hw); + ret = mt7996_set_channel(phy); + if (ret) + return ret; + ieee80211_wake_queues(hw); + } + + mutex_lock(&dev->mt76.mutex); + + if (changed & IEEE80211_CONF_CHANGE_MONITOR) { + bool enabled = !!(hw->conf.flags & IEEE80211_CONF_MONITOR); + + if (!enabled) + phy->rxfilter |= MT_WF_RFCR_DROP_OTHER_UC; + else + phy->rxfilter &= ~MT_WF_RFCR_DROP_OTHER_UC; + + mt76_rmw_field(dev, MT_DMA_DCR0(phy->mt76->band_idx), + MT_DMA_DCR0_RXD_G5_EN, enabled); + mt76_wr(dev, MT_WF_RFCR(phy->mt76->band_idx), phy->rxfilter); + } + + mutex_unlock(&dev->mt76.mutex); + + return 0; +} + +static int +mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + unsigned int link_id, u16 queue, + const struct ieee80211_tx_queue_params *params) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + + /* no need to update right away, we'll get BSS_CHANGED_QOS */ + queue = mt76_connac_lmac_mapping(queue); + mvif->queue_params[queue] = *params; + + return 0; +} + +static void mt7996_configure_filter(struct ieee80211_hw *hw, + unsigned int changed_flags, + unsigned int *total_flags, + u64 multicast) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + u32 ctl_flags = MT_WF_RFCR1_DROP_ACK | + MT_WF_RFCR1_DROP_BF_POLL | + MT_WF_RFCR1_DROP_BA | + MT_WF_RFCR1_DROP_CFEND | + MT_WF_RFCR1_DROP_CFACK; + u32 flags = 0; + +#define MT76_FILTER(_flag, _hw) do { \ + flags |= *total_flags & FIF_##_flag; \ + phy->rxfilter &= ~(_hw); \ + phy->rxfilter |= !(flags & FIF_##_flag) * (_hw); \ + } while (0) + + mutex_lock(&dev->mt76.mutex); + + phy->rxfilter &= ~(MT_WF_RFCR_DROP_OTHER_BSS | + MT_WF_RFCR_DROP_OTHER_BEACON | + MT_WF_RFCR_DROP_FRAME_REPORT | + MT_WF_RFCR_DROP_PROBEREQ | + MT_WF_RFCR_DROP_MCAST_FILTERED | + MT_WF_RFCR_DROP_MCAST | + MT_WF_RFCR_DROP_BCAST | + MT_WF_RFCR_DROP_DUPLICATE | + MT_WF_RFCR_DROP_A2_BSSID | + MT_WF_RFCR_DROP_UNWANTED_CTL | + MT_WF_RFCR_DROP_STBC_MULTI); + + MT76_FILTER(OTHER_BSS, MT_WF_RFCR_DROP_OTHER_TIM | + MT_WF_RFCR_DROP_A3_MAC | + MT_WF_RFCR_DROP_A3_BSSID); + + MT76_FILTER(FCSFAIL, MT_WF_RFCR_DROP_FCSFAIL); + + MT76_FILTER(CONTROL, MT_WF_RFCR_DROP_CTS | + MT_WF_RFCR_DROP_RTS | + MT_WF_RFCR_DROP_CTL_RSV | + MT_WF_RFCR_DROP_NDPA); + + *total_flags = flags; + mt76_wr(dev, MT_WF_RFCR(phy->mt76->band_idx), phy->rxfilter); + + if (*total_flags & FIF_CONTROL) + mt76_clear(dev, MT_WF_RFCR1(phy->mt76->band_idx), ctl_flags); + else + mt76_set(dev, MT_WF_RFCR1(phy->mt76->band_idx), ctl_flags); + + mutex_unlock(&dev->mt76.mutex); +} + +static void +mt7996_update_bss_color(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct cfg80211_he_bss_color *bss_color) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + + switch (vif->type) { + case NL80211_IFTYPE_AP: { + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + + if (mvif->mt76.omac_idx > HW_BSSID_MAX) + return; + fallthrough; + } + case NL80211_IFTYPE_STATION: + mt7996_mcu_update_bss_color(dev, vif, bss_color); + break; + default: + break; + } +} + +static void mt7996_bss_info_changed(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *info, + u64 changed) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = mt7996_hw_dev(hw); + + mutex_lock(&dev->mt76.mutex); + + /* station mode uses BSSID to map the wlan entry to a peer, + * and then peer references bss_info_rfch to set bandwidth cap. + */ + if (changed & BSS_CHANGED_BSSID && + vif->type == NL80211_IFTYPE_STATION) { + bool join = !is_zero_ether_addr(info->bssid); + + mt7996_mcu_add_bss_info(phy, vif, join); + mt7996_mcu_add_sta(dev, vif, NULL, join); + } + + if (changed & BSS_CHANGED_ASSOC) + mt7996_mcu_add_bss_info(phy, vif, vif->cfg.assoc); + + if (changed & BSS_CHANGED_ERP_CTS_PROT) + mt7996_mac_enable_rtscts(dev, vif, info->use_cts_prot); + + if (changed & BSS_CHANGED_ERP_SLOT) { + int slottime = info->use_short_slot ? 9 : 20; + + if (slottime != phy->slottime) { + phy->slottime = slottime; + mt7996_mac_set_timing(phy); + } + } + + if (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon) { + mt7996_mcu_add_bss_info(phy, vif, true); + mt7996_mcu_add_sta(dev, vif, NULL, true); + } + + /* ensure that enable txcmd_mode after bss_info */ + if (changed & (BSS_CHANGED_QOS | BSS_CHANGED_BEACON_ENABLED)) + mt7996_mcu_set_tx(dev, vif); + + if (changed & BSS_CHANGED_HE_OBSS_PD) + mt7996_mcu_add_obss_spr(phy, vif, &info->he_obss_pd); + + if (changed & BSS_CHANGED_HE_BSS_COLOR) + mt7996_update_bss_color(hw, vif, &info->he_bss_color); + + if (changed & (BSS_CHANGED_BEACON | + BSS_CHANGED_BEACON_ENABLED)) + mt7996_mcu_add_beacon(hw, vif, info->enable_beacon); + + if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP || + changed & BSS_CHANGED_FILS_DISCOVERY) + mt7996_mcu_beacon_inband_discov(dev, vif, changed); + + mutex_unlock(&dev->mt76.mutex); +} + +static void +mt7996_channel_switch_beacon(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct cfg80211_chan_def *chandef) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + + mutex_lock(&dev->mt76.mutex); + mt7996_mcu_add_beacon(hw, vif, true); + mutex_unlock(&dev->mt76.mutex); +} + +int mt7996_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + u8 band_idx = mvif->phy->mt76->band_idx; + int ret, idx; + + idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7996_WTBL_STA); + if (idx < 0) + return -ENOSPC; + + INIT_LIST_HEAD(&msta->rc_list); + INIT_LIST_HEAD(&msta->poll_list); + msta->vif = mvif; + msta->wcid.sta = 1; + msta->wcid.idx = idx; + msta->wcid.phy_idx = band_idx; + msta->wcid.tx_info |= MT_WCID_TX_INFO_SET; + msta->jiffies = jiffies; + + ewma_avg_signal_init(&msta->avg_ack_signal); + + mt7996_mac_wtbl_update(dev, idx, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + + ret = mt7996_mcu_add_sta(dev, vif, sta, true); + if (ret) + return ret; + + return mt7996_mcu_add_rate_ctrl(dev, vif, sta, false); +} + +void mt7996_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + int i; + + mt7996_mcu_add_sta(dev, vif, sta, false); + + mt7996_mac_wtbl_update(dev, msta->wcid.idx, + MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + + for (i = 0; i < ARRAY_SIZE(msta->twt.flow); i++) + mt7996_mac_twt_teardown_flow(dev, msta, i); + + spin_lock_bh(&dev->sta_poll_lock); + if (!list_empty(&msta->poll_list)) + list_del_init(&msta->poll_list); + if (!list_empty(&msta->rc_list)) + list_del_init(&msta->rc_list); + spin_unlock_bh(&dev->sta_poll_lock); +} + +static void mt7996_tx(struct ieee80211_hw *hw, + struct ieee80211_tx_control *control, + struct sk_buff *skb) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt76_phy *mphy = hw->priv; + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct ieee80211_vif *vif = info->control.vif; + struct mt76_wcid *wcid = &dev->mt76.global_wcid; + + if (control->sta) { + struct mt7996_sta *sta; + + sta = (struct mt7996_sta *)control->sta->drv_priv; + wcid = &sta->wcid; + } + + if (vif && !control->sta) { + struct mt7996_vif *mvif; + + mvif = (struct mt7996_vif *)vif->drv_priv; + wcid = &mvif->sta.wcid; + } + + mt76_tx(mphy, control->sta, wcid, skb); +} + +static int mt7996_set_rts_threshold(struct ieee80211_hw *hw, u32 val) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + int ret; + + mutex_lock(&phy->dev->mt76.mutex); + ret = mt7996_mcu_set_rts_thresh(phy, val); + mutex_unlock(&phy->dev->mt76.mutex); + + return ret; +} + +static int +mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_ampdu_params *params) +{ + enum ieee80211_ampdu_mlme_action action = params->action; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct ieee80211_sta *sta = params->sta; + struct ieee80211_txq *txq = sta->txq[params->tid]; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + u16 tid = params->tid; + u16 ssn = params->ssn; + struct mt76_txq *mtxq; + int ret = 0; + + if (!txq) + return -EINVAL; + + mtxq = (struct mt76_txq *)txq->drv_priv; + + mutex_lock(&dev->mt76.mutex); + switch (action) { + case IEEE80211_AMPDU_RX_START: + mt76_rx_aggr_start(&dev->mt76, &msta->wcid, tid, ssn, + params->buf_size); + ret = mt7996_mcu_add_rx_ba(dev, params, true); + break; + case IEEE80211_AMPDU_RX_STOP: + mt76_rx_aggr_stop(&dev->mt76, &msta->wcid, tid); + ret = mt7996_mcu_add_rx_ba(dev, params, false); + break; + case IEEE80211_AMPDU_TX_OPERATIONAL: + mtxq->aggr = true; + mtxq->send_bar = false; + ret = mt7996_mcu_add_tx_ba(dev, params, true); + break; + case IEEE80211_AMPDU_TX_STOP_FLUSH: + case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT: + mtxq->aggr = false; + clear_bit(tid, &msta->ampdu_state); + ret = mt7996_mcu_add_tx_ba(dev, params, false); + break; + case IEEE80211_AMPDU_TX_START: + set_bit(tid, &msta->ampdu_state); + ret = IEEE80211_AMPDU_TX_START_IMMEDIATE; + break; + case IEEE80211_AMPDU_TX_STOP_CONT: + mtxq->aggr = false; + clear_bit(tid, &msta->ampdu_state); + ret = mt7996_mcu_add_tx_ba(dev, params, false); + ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid); + break; + } + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static int +mt7996_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + return mt76_sta_state(hw, vif, sta, IEEE80211_STA_NOTEXIST, + IEEE80211_STA_NONE); +} + +static int +mt7996_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + return mt76_sta_state(hw, vif, sta, IEEE80211_STA_NONE, + IEEE80211_STA_NOTEXIST); +} + +static int +mt7996_get_stats(struct ieee80211_hw *hw, + struct ieee80211_low_level_stats *stats) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mib_stats *mib = &phy->mib; + + mutex_lock(&dev->mt76.mutex); + + stats->dot11RTSSuccessCount = mib->rts_cnt; + stats->dot11RTSFailureCount = mib->rts_retries_cnt; + stats->dot11FCSErrorCount = mib->fcs_err_cnt; + stats->dot11ACKFailureCount = mib->ack_fail_cnt; + + mutex_unlock(&dev->mt76.mutex); + + return 0; +} + +u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif *mvif) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + union { + u64 t64; + u32 t32[2]; + } tsf; + u16 n; + + lockdep_assert_held(&dev->mt76.mutex); + + n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 + : mvif->mt76.omac_idx; + /* TSF software read */ + mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE, + MT_LPON_TCR_SW_READ); + tsf.t32[0] = mt76_rr(dev, MT_LPON_UTTR0(phy->mt76->band_idx)); + tsf.t32[1] = mt76_rr(dev, MT_LPON_UTTR1(phy->mt76->band_idx)); + + return tsf.t64; +} + +static u64 +mt7996_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + u64 ret; + + mutex_lock(&dev->mt76.mutex); + ret = __mt7996_get_tsf(hw, mvif); + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static void +mt7996_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + u64 timestamp) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + union { + u64 t64; + u32 t32[2]; + } tsf = { .t64 = timestamp, }; + u16 n; + + mutex_lock(&dev->mt76.mutex); + + n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 + : mvif->mt76.omac_idx; + mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]); + mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]); + /* TSF software overwrite */ + mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE, + MT_LPON_TCR_SW_WRITE); + + mutex_unlock(&dev->mt76.mutex); +} + +static void +mt7996_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + s64 timestamp) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + union { + u64 t64; + u32 t32[2]; + } tsf = { .t64 = timestamp, }; + u16 n; + + mutex_lock(&dev->mt76.mutex); + + n = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 + : mvif->mt76.omac_idx; + mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]); + mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]); + /* TSF software adjust*/ + mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE, + MT_LPON_TCR_SW_ADJUST); + + mutex_unlock(&dev->mt76.mutex); +} + +static void +mt7996_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = phy->dev; + + mutex_lock(&dev->mt76.mutex); + phy->coverage_class = max_t(s16, coverage_class, 0); + mt7996_mac_set_timing(phy); + mutex_unlock(&dev->mt76.mutex); +} + +static int +mt7996_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + int max_nss = hweight8(hw->wiphy->available_antennas_tx); + u8 band_idx = phy->mt76->band_idx, shift = dev->chainshift[band_idx]; + + if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) + return -EINVAL; + + if ((BIT(hweight8(tx_ant)) - 1) != tx_ant) + tx_ant = BIT(ffs(tx_ant) - 1) - 1; + + mutex_lock(&dev->mt76.mutex); + + phy->mt76->antenna_mask = tx_ant; + + /* restore to the origin chainmask which might have auxiliary path */ + if (hweight8(tx_ant) == max_nss) + phy->mt76->chainmask = (dev->chainmask >> shift) << shift; + else + phy->mt76->chainmask = tx_ant << shift; + + mt76_set_stream_caps(phy->mt76, true); + mt7996_set_stream_vht_txbf_caps(phy); + mt7996_set_stream_he_caps(phy); + + mutex_unlock(&dev->mt76.mutex); + + return 0; +} + +static void mt7996_sta_statistics(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct station_info *sinfo) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct rate_info *txrate = &msta->wcid.rate; + + if (!txrate->legacy && !txrate->flags) + return; + + if (txrate->legacy) { + sinfo->txrate.legacy = txrate->legacy; + } else { + sinfo->txrate.mcs = txrate->mcs; + sinfo->txrate.nss = txrate->nss; + sinfo->txrate.bw = txrate->bw; + sinfo->txrate.he_gi = txrate->he_gi; + sinfo->txrate.he_dcm = txrate->he_dcm; + sinfo->txrate.he_ru_alloc = txrate->he_ru_alloc; + } + sinfo->txrate.flags = txrate->flags; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); + + sinfo->ack_signal = (s8)msta->ack_signal; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL); + + sinfo->avg_ack_signal = -(s8)ewma_avg_signal_read(&msta->avg_ack_signal); + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG); +} + +static void mt7996_sta_rc_work(void *data, struct ieee80211_sta *sta) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = msta->vif->phy->dev; + u32 *changed = data; + + spin_lock_bh(&dev->sta_poll_lock); + msta->changed |= *changed; + if (list_empty(&msta->rc_list)) + list_add_tail(&msta->rc_list, &dev->sta_rc_list); + spin_unlock_bh(&dev->sta_poll_lock); +} + +static void mt7996_sta_rc_update(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + u32 changed) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = phy->dev; + + mt7996_sta_rc_work(&changed, sta); + ieee80211_queue_work(hw, &dev->rc_work); +} + +static int +mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + const struct cfg80211_bitrate_mask *mask) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = phy->dev; + u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED; + + mvif->bitrate_mask = *mask; + + /* if multiple rates across different preambles are given we can + * reconfigure this info with all peers using sta_rec command with + * the below exception cases. + * - single rate : if a rate is passed along with different preambles, + * we select the highest one as fixed rate. i.e VHT MCS for VHT peers. + * - multiple rates: if it's not in range format i.e 0-{7,8,9} for VHT + * then multiple MCS setting (MCS 4,5,6) is not supported. + */ + ieee80211_iterate_stations_atomic(hw, mt7996_sta_rc_work, &changed); + ieee80211_queue_work(hw, &dev->rc_work); + + return 0; +} + +static void mt7996_sta_set_4addr(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + bool enabled) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + + if (enabled) + set_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags); + else + clear_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags); + + mt7996_mcu_wtbl_update_hdr_trans(dev, vif, sta); +} + +static void mt7996_sta_set_decap_offload(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + bool enabled) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + + if (enabled) + set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); + else + clear_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); + + mt7996_mcu_wtbl_update_hdr_trans(dev, vif, sta); +} + +static const char mt7996_gstrings_stats[][ETH_GSTRING_LEN] = { + "tx_ampdu_cnt", + "tx_stop_q_empty_cnt", + "tx_mpdu_attempts", + "tx_mpdu_success", + "tx_rwp_fail_cnt", + "tx_rwp_need_cnt", + "tx_pkt_ebf_cnt", + "tx_pkt_ibf_cnt", + "tx_ampdu_len:0-1", + "tx_ampdu_len:2-10", + "tx_ampdu_len:11-19", + "tx_ampdu_len:20-28", + "tx_ampdu_len:29-37", + "tx_ampdu_len:38-46", + "tx_ampdu_len:47-55", + "tx_ampdu_len:56-79", + "tx_ampdu_len:80-103", + "tx_ampdu_len:104-127", + "tx_ampdu_len:128-151", + "tx_ampdu_len:152-175", + "tx_ampdu_len:176-199", + "tx_ampdu_len:200-223", + "tx_ampdu_len:224-247", + "ba_miss_count", + "tx_beamformer_ppdu_iBF", + "tx_beamformer_ppdu_eBF", + "tx_beamformer_rx_feedback_all", + "tx_beamformer_rx_feedback_he", + "tx_beamformer_rx_feedback_vht", + "tx_beamformer_rx_feedback_ht", + "tx_beamformer_rx_feedback_bw", /* zero based idx: 20, 40, 80, 160 */ + "tx_beamformer_rx_feedback_nc", + "tx_beamformer_rx_feedback_nr", + "tx_beamformee_ok_feedback_pkts", + "tx_beamformee_feedback_trig", + "tx_mu_beamforming", + "tx_mu_mpdu", + "tx_mu_successful_mpdu", + "tx_su_successful_mpdu", + "tx_msdu_pack_1", + "tx_msdu_pack_2", + "tx_msdu_pack_3", + "tx_msdu_pack_4", + "tx_msdu_pack_5", + "tx_msdu_pack_6", + "tx_msdu_pack_7", + "tx_msdu_pack_8", + + /* rx counters */ + "rx_fifo_full_cnt", + "rx_mpdu_cnt", + "channel_idle_cnt", + "rx_vector_mismatch_cnt", + "rx_delimiter_fail_cnt", + "rx_len_mismatch_cnt", + "rx_ampdu_cnt", + "rx_ampdu_bytes_cnt", + "rx_ampdu_valid_subframe_cnt", + "rx_ampdu_valid_subframe_b_cnt", + "rx_pfdrop_cnt", + "rx_vec_queue_overflow_drop_cnt", + "rx_ba_cnt", + + /* per vif counters */ + "v_tx_mode_cck", + "v_tx_mode_ofdm", + "v_tx_mode_ht", + "v_tx_mode_ht_gf", + "v_tx_mode_vht", + "v_tx_mode_he_su", + "v_tx_mode_he_ext_su", + "v_tx_mode_he_tb", + "v_tx_mode_he_mu", + "v_tx_bw_20", + "v_tx_bw_40", + "v_tx_bw_80", + "v_tx_bw_160", + "v_tx_mcs_0", + "v_tx_mcs_1", + "v_tx_mcs_2", + "v_tx_mcs_3", + "v_tx_mcs_4", + "v_tx_mcs_5", + "v_tx_mcs_6", + "v_tx_mcs_7", + "v_tx_mcs_8", + "v_tx_mcs_9", + "v_tx_mcs_10", + "v_tx_mcs_11", +}; + +#define MT7996_SSTATS_LEN ARRAY_SIZE(mt7996_gstrings_stats) + +/* Ethtool related API */ +static +void mt7996_get_et_strings(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + u32 sset, u8 *data) +{ + if (sset == ETH_SS_STATS) + memcpy(data, *mt7996_gstrings_stats, + sizeof(mt7996_gstrings_stats)); +} + +static +int mt7996_get_et_sset_count(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, int sset) +{ + if (sset == ETH_SS_STATS) + return MT7996_SSTATS_LEN; + + return 0; +} + +static void mt7996_ethtool_worker(void *wi_data, struct ieee80211_sta *sta) +{ + struct mt76_ethtool_worker_info *wi = wi_data; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + + if (msta->vif->mt76.idx != wi->idx) + return; + + mt76_ethtool_worker(wi, &msta->stats); +} + +static +void mt7996_get_et_stats(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ethtool_stats *stats, u64 *data) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt76_ethtool_worker_info wi = { + .data = data, + .idx = mvif->mt76.idx, + }; + struct mib_stats *mib = &phy->mib; + /* See mt7996_ampdu_stat_read_phy, etc */ + int i, ei = 0; + + mutex_lock(&dev->mt76.mutex); + + mt7996_mac_update_stats(phy); + + data[ei++] = mib->tx_ampdu_cnt; + data[ei++] = mib->tx_stop_q_empty_cnt; + data[ei++] = mib->tx_mpdu_attempts_cnt; + data[ei++] = mib->tx_mpdu_success_cnt; + data[ei++] = mib->tx_rwp_fail_cnt; + data[ei++] = mib->tx_rwp_need_cnt; + data[ei++] = mib->tx_bf_ebf_ppdu_cnt; + data[ei++] = mib->tx_bf_ibf_ppdu_cnt; + + /* Tx ampdu stat */ + for (i = 0; i < 15 /*ARRAY_SIZE(bound)*/; i++) + data[ei++] = phy->mt76->aggr_stats[i]; + data[ei++] = phy->mib.ba_miss_cnt; + + /* Tx Beamformer monitor */ + data[ei++] = mib->tx_bf_ibf_ppdu_cnt; + data[ei++] = mib->tx_bf_ebf_ppdu_cnt; + + /* Tx Beamformer Rx feedback monitor */ + data[ei++] = mib->tx_bf_rx_fb_all_cnt; + data[ei++] = mib->tx_bf_rx_fb_he_cnt; + data[ei++] = mib->tx_bf_rx_fb_vht_cnt; + data[ei++] = mib->tx_bf_rx_fb_ht_cnt; + + data[ei++] = mib->tx_bf_rx_fb_bw; + data[ei++] = mib->tx_bf_rx_fb_nc_cnt; + data[ei++] = mib->tx_bf_rx_fb_nr_cnt; + + /* Tx Beamformee Rx NDPA & Tx feedback report */ + data[ei++] = mib->tx_bf_fb_cpl_cnt; + data[ei++] = mib->tx_bf_fb_trig_cnt; + + /* Tx SU & MU counters */ + data[ei++] = mib->tx_mu_bf_cnt; + data[ei++] = mib->tx_mu_mpdu_cnt; + data[ei++] = mib->tx_mu_acked_mpdu_cnt; + data[ei++] = mib->tx_su_acked_mpdu_cnt; + + /* Tx amsdu info (pack-count histogram) */ + for (i = 0; i < ARRAY_SIZE(mib->tx_amsdu); i++) + data[ei++] = mib->tx_amsdu[i]; + + /* rx counters */ + data[ei++] = mib->rx_fifo_full_cnt; + data[ei++] = mib->rx_mpdu_cnt; + data[ei++] = mib->channel_idle_cnt; + data[ei++] = mib->rx_vector_mismatch_cnt; + data[ei++] = mib->rx_delimiter_fail_cnt; + data[ei++] = mib->rx_len_mismatch_cnt; + data[ei++] = mib->rx_ampdu_cnt; + data[ei++] = mib->rx_ampdu_bytes_cnt; + data[ei++] = mib->rx_ampdu_valid_subframe_cnt; + data[ei++] = mib->rx_ampdu_valid_subframe_bytes_cnt; + data[ei++] = mib->rx_pfdrop_cnt; + data[ei++] = mib->rx_vec_queue_overflow_drop_cnt; + data[ei++] = mib->rx_ba_cnt; + + /* Add values for all stations owned by this vif */ + wi.initial_stat_idx = ei; + ieee80211_iterate_stations_atomic(hw, mt7996_ethtool_worker, &wi); + + mutex_unlock(&dev->mt76.mutex); + + if (wi.sta_count == 0) + return; + + ei += wi.worker_stat_count; + if (ei != MT7996_SSTATS_LEN) + dev_err(dev->mt76.dev, "ei: %d MT7996_SSTATS_LEN: %d", + ei, (int)MT7996_SSTATS_LEN); +} + +static void +mt7996_twt_teardown_request(struct ieee80211_hw *hw, + struct ieee80211_sta *sta, + u8 flowid) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + + mutex_lock(&dev->mt76.mutex); + mt7996_mac_twt_teardown_flow(dev, msta, flowid); + mutex_unlock(&dev->mt76.mutex); +} + +static int +mt7996_set_radar_background(struct ieee80211_hw *hw, + struct cfg80211_chan_def *chandef) +{ + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_dev *dev = phy->dev; + int ret = -EINVAL; + bool running; + + mutex_lock(&dev->mt76.mutex); + + if (dev->mt76.region == NL80211_DFS_UNSET) + goto out; + + if (dev->rdd2_phy && dev->rdd2_phy != phy) { + /* rdd2 is already locked */ + ret = -EBUSY; + goto out; + } + + /* rdd2 already configured on a radar channel */ + running = dev->rdd2_phy && + cfg80211_chandef_valid(&dev->rdd2_chandef) && + !!(dev->rdd2_chandef.chan->flags & IEEE80211_CHAN_RADAR); + + if (!chandef || running || + !(chandef->chan->flags & IEEE80211_CHAN_RADAR)) { + ret = mt7996_mcu_rdd_background_enable(phy, NULL); + if (ret) + goto out; + + if (!running) + goto update_phy; + } + + ret = mt7996_mcu_rdd_background_enable(phy, chandef); + if (ret) + goto out; + +update_phy: + dev->rdd2_phy = chandef ? phy : NULL; + if (chandef) + dev->rdd2_chandef = *chandef; +out: + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +const struct ieee80211_ops mt7996_ops = { + .tx = mt7996_tx, + .start = mt7996_start, + .stop = mt7996_stop, + .add_interface = mt7996_add_interface, + .remove_interface = mt7996_remove_interface, + .config = mt7996_config, + .conf_tx = mt7996_conf_tx, + .configure_filter = mt7996_configure_filter, + .bss_info_changed = mt7996_bss_info_changed, + .sta_add = mt7996_sta_add, + .sta_remove = mt7996_sta_remove, + .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, + .sta_rc_update = mt7996_sta_rc_update, + .set_key = mt7996_set_key, + .ampdu_action = mt7996_ampdu_action, + .set_rts_threshold = mt7996_set_rts_threshold, + .wake_tx_queue = mt76_wake_tx_queue, + .sw_scan_start = mt76_sw_scan, + .sw_scan_complete = mt76_sw_scan_complete, + .release_buffered_frames = mt76_release_buffered_frames, + .get_txpower = mt76_get_txpower, + .channel_switch_beacon = mt7996_channel_switch_beacon, + .get_stats = mt7996_get_stats, + .get_et_sset_count = mt7996_get_et_sset_count, + .get_et_stats = mt7996_get_et_stats, + .get_et_strings = mt7996_get_et_strings, + .get_tsf = mt7996_get_tsf, + .set_tsf = mt7996_set_tsf, + .offset_tsf = mt7996_offset_tsf, + .get_survey = mt76_get_survey, + .get_antenna = mt76_get_antenna, + .set_antenna = mt7996_set_antenna, + .set_bitrate_mask = mt7996_set_bitrate_mask, + .set_coverage_class = mt7996_set_coverage_class, + .sta_statistics = mt7996_sta_statistics, + .sta_set_4addr = mt7996_sta_set_4addr, + .sta_set_decap_offload = mt7996_sta_set_decap_offload, + .add_twt_setup = mt7996_mac_add_twt_setup, + .twt_teardown_request = mt7996_twt_teardown_request, +#ifdef CONFIG_MAC80211_DEBUGFS + .sta_add_debugfs = mt7996_sta_add_debugfs, +#endif + .set_radar_background = mt7996_set_radar_background, +}; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c new file mode 100644 index 000000000000..04e1d10bbd21 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -0,0 +1,3607 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/firmware.h> +#include <linux/fs.h> +#include "mt7996.h" +#include "mcu.h" +#include "mac.h" +#include "eeprom.h" + +struct mt7996_patch_hdr { + char build_date[16]; + char platform[4]; + __be32 hw_sw_ver; + __be32 patch_ver; + __be16 checksum; + u16 reserved; + struct { + __be32 patch_ver; + __be32 subsys; + __be32 feature; + __be32 n_region; + __be32 crc; + u32 reserved[11]; + } desc; +} __packed; + +struct mt7996_patch_sec { + __be32 type; + __be32 offs; + __be32 size; + union { + __be32 spec[13]; + struct { + __be32 addr; + __be32 len; + __be32 sec_key_idx; + __be32 align_len; + u32 reserved[9]; + } info; + }; +} __packed; + +struct mt7996_fw_trailer { + u8 chip_id; + u8 eco_code; + u8 n_region; + u8 format_ver; + u8 format_flag; + u8 reserved[2]; + char fw_ver[10]; + char build_date[15]; + u32 crc; +} __packed; + +struct mt7996_fw_region { + __le32 decomp_crc; + __le32 decomp_len; + __le32 decomp_blk_sz; + u8 reserved[4]; + __le32 addr; + __le32 len; + u8 feature_set; + u8 reserved1[15]; +} __packed; + +#define MCU_PATCH_ADDRESS 0x200000 + +#define HE_PHY(p, c) u8_get_bits(c, IEEE80211_HE_PHY_##p) +#define HE_MAC(m, c) u8_get_bits(c, IEEE80211_HE_MAC_##m) + +static bool sr_scene_detect = true; +module_param(sr_scene_detect, bool, 0644); +MODULE_PARM_DESC(sr_scene_detect, "Enable firmware scene detection algorithm"); + +static u8 +mt7996_mcu_get_sta_nss(u16 mcs_map) +{ + u8 nss; + + for (nss = 8; nss > 0; nss--) { + u8 nss_mcs = (mcs_map >> (2 * (nss - 1))) & 3; + + if (nss_mcs != IEEE80211_VHT_MCS_NOT_SUPPORTED) + break; + } + + return nss - 1; +} + +static void +mt7996_mcu_set_sta_he_mcs(struct ieee80211_sta *sta, __le16 *he_mcs, + u16 mcs_map) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + enum nl80211_band band = msta->vif->phy->mt76->chandef.chan->band; + const u16 *mask = msta->vif->bitrate_mask.control[band].he_mcs; + int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss; + + for (nss = 0; nss < max_nss; nss++) { + int mcs; + + switch ((mcs_map >> (2 * nss)) & 0x3) { + case IEEE80211_HE_MCS_SUPPORT_0_11: + mcs = GENMASK(11, 0); + break; + case IEEE80211_HE_MCS_SUPPORT_0_9: + mcs = GENMASK(9, 0); + break; + case IEEE80211_HE_MCS_SUPPORT_0_7: + mcs = GENMASK(7, 0); + break; + default: + mcs = 0; + } + + mcs = mcs ? fls(mcs & mask[nss]) - 1 : -1; + + switch (mcs) { + case 0 ... 7: + mcs = IEEE80211_HE_MCS_SUPPORT_0_7; + break; + case 8 ... 9: + mcs = IEEE80211_HE_MCS_SUPPORT_0_9; + break; + case 10 ... 11: + mcs = IEEE80211_HE_MCS_SUPPORT_0_11; + break; + default: + mcs = IEEE80211_HE_MCS_NOT_SUPPORTED; + break; + } + mcs_map &= ~(0x3 << (nss * 2)); + mcs_map |= mcs << (nss * 2); + } + + *he_mcs = cpu_to_le16(mcs_map); +} + +static void +mt7996_mcu_set_sta_vht_mcs(struct ieee80211_sta *sta, __le16 *vht_mcs, + const u16 *mask) +{ + u16 mcs, mcs_map = le16_to_cpu(sta->deflink.vht_cap.vht_mcs.rx_mcs_map); + int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss; + + for (nss = 0; nss < max_nss; nss++, mcs_map >>= 2) { + switch (mcs_map & 0x3) { + case IEEE80211_VHT_MCS_SUPPORT_0_9: + mcs = GENMASK(9, 0); + break; + case IEEE80211_VHT_MCS_SUPPORT_0_8: + mcs = GENMASK(8, 0); + break; + case IEEE80211_VHT_MCS_SUPPORT_0_7: + mcs = GENMASK(7, 0); + break; + default: + mcs = 0; + } + + vht_mcs[nss] = cpu_to_le16(mcs & mask[nss]); + } +} + +static void +mt7996_mcu_set_sta_ht_mcs(struct ieee80211_sta *sta, u8 *ht_mcs, + const u8 *mask) +{ + int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss; + + for (nss = 0; nss < max_nss; nss++) + ht_mcs[nss] = sta->deflink.ht_cap.mcs.rx_mask[nss] & mask[nss]; +} + +static int +mt7996_mcu_parse_response(struct mt76_dev *mdev, int cmd, + struct sk_buff *skb, int seq) +{ + struct mt7996_mcu_rxd *rxd; + struct mt7996_mcu_uni_event *event; + int mcu_cmd = FIELD_GET(__MCU_CMD_FIELD_ID, cmd); + int ret = 0; + + if (!skb) { + dev_err(mdev->dev, "Message %08x (seq %d) timeout\n", + cmd, seq); + return -ETIMEDOUT; + } + + rxd = (struct mt7996_mcu_rxd *)skb->data; + if (seq != rxd->seq) + return -EAGAIN; + + if (cmd == MCU_CMD(PATCH_SEM_CONTROL)) { + skb_pull(skb, sizeof(*rxd) - 4); + ret = *skb->data; + } else if ((rxd->option & MCU_UNI_CMD_EVENT) && + rxd->eid == MCU_UNI_EVENT_RESULT) { + skb_pull(skb, sizeof(*rxd)); + event = (struct mt7996_mcu_uni_event *)skb->data; + ret = le32_to_cpu(event->status); + /* skip invalid event */ + if (mcu_cmd != event->cid) + ret = -EAGAIN; + } else { + skb_pull(skb, sizeof(struct mt7996_mcu_rxd)); + } + + return ret; +} + +static int +mt7996_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, + int cmd, int *wait_seq) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + int txd_len, mcu_cmd = FIELD_GET(__MCU_CMD_FIELD_ID, cmd); + struct mt76_connac2_mcu_uni_txd *uni_txd; + struct mt76_connac2_mcu_txd *mcu_txd; + enum mt76_mcuq_id qid; + __le32 *txd; + u32 val; + u8 seq; + + mdev->mcu.timeout = 20 * HZ; + + seq = ++dev->mt76.mcu.msg_seq & 0xf; + if (!seq) + seq = ++dev->mt76.mcu.msg_seq & 0xf; + + if (cmd == MCU_CMD(FW_SCATTER)) { + qid = MT_MCUQ_FWDL; + goto exit; + } + + txd_len = cmd & __MCU_CMD_FIELD_UNI ? sizeof(*uni_txd) : sizeof(*mcu_txd); + txd = (__le32 *)skb_push(skb, txd_len); + if (test_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state)) + qid = MT_MCUQ_WA; + else + qid = MT_MCUQ_WM; + + val = FIELD_PREP(MT_TXD0_TX_BYTES, skb->len) | + FIELD_PREP(MT_TXD0_PKT_FMT, MT_TX_TYPE_CMD) | + FIELD_PREP(MT_TXD0_Q_IDX, MT_TX_MCU_PORT_RX_Q0); + txd[0] = cpu_to_le32(val); + + val = FIELD_PREP(MT_TXD1_HDR_FORMAT, MT_HDR_FORMAT_CMD); + txd[1] = cpu_to_le32(val); + + if (cmd & __MCU_CMD_FIELD_UNI) { + uni_txd = (struct mt76_connac2_mcu_uni_txd *)txd; + uni_txd->len = cpu_to_le16(skb->len - sizeof(uni_txd->txd)); + uni_txd->cid = cpu_to_le16(mcu_cmd); + uni_txd->s2d_index = MCU_S2D_H2CN; + uni_txd->pkt_type = MCU_PKT_ID; + uni_txd->seq = seq; + + if (cmd & __MCU_CMD_FIELD_QUERY) + uni_txd->option = MCU_CMD_UNI_QUERY_ACK; + else + uni_txd->option = MCU_CMD_UNI_EXT_ACK; + + if ((cmd & __MCU_CMD_FIELD_WA) && (cmd & __MCU_CMD_FIELD_WM)) + uni_txd->s2d_index = MCU_S2D_H2CN; + else if (cmd & __MCU_CMD_FIELD_WA) + uni_txd->s2d_index = MCU_S2D_H2C; + else if (cmd & __MCU_CMD_FIELD_WM) + uni_txd->s2d_index = MCU_S2D_H2N; + + goto exit; + } + + mcu_txd = (struct mt76_connac2_mcu_txd *)txd; + mcu_txd->len = cpu_to_le16(skb->len - sizeof(mcu_txd->txd)); + mcu_txd->pq_id = cpu_to_le16(MCU_PQ_ID(MT_TX_PORT_IDX_MCU, + MT_TX_MCU_PORT_RX_Q0)); + mcu_txd->pkt_type = MCU_PKT_ID; + mcu_txd->seq = seq; + + mcu_txd->cid = FIELD_GET(__MCU_CMD_FIELD_ID, cmd); + mcu_txd->set_query = MCU_Q_NA; + mcu_txd->ext_cid = FIELD_GET(__MCU_CMD_FIELD_EXT_ID, cmd); + if (mcu_txd->ext_cid) { + mcu_txd->ext_cid_ack = 1; + + if (cmd & __MCU_CMD_FIELD_QUERY) + mcu_txd->set_query = MCU_Q_QUERY; + else + mcu_txd->set_query = MCU_Q_SET; + } + + if (cmd & __MCU_CMD_FIELD_WA) + mcu_txd->s2d_index = MCU_S2D_H2C; + else + mcu_txd->s2d_index = MCU_S2D_H2N; + +exit: + if (wait_seq) + *wait_seq = seq; + + return mt76_tx_queue_skb_raw(dev, mdev->q_mcu[qid], skb, 0); +} + +int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3) +{ + struct { + __le32 args[3]; + } req = { + .args = { + cpu_to_le32(a1), + cpu_to_le32(a2), + cpu_to_le32(a3), + }, + }; + + return mt76_mcu_send_msg(&dev->mt76, cmd, &req, sizeof(req), false); +} + +static void +mt7996_mcu_csa_finish(void *priv, u8 *mac, struct ieee80211_vif *vif) +{ + if (vif->bss_conf.csa_active) + ieee80211_csa_finish(vif); +} + +static void +mt7996_mcu_rx_radar_detected(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt76_phy *mphy = &dev->mt76.phy; + struct mt7996_mcu_rdd_report *r; + + r = (struct mt7996_mcu_rdd_report *)skb->data; + + mphy = dev->mt76.phys[r->band_idx]; + if (!mphy) + return; + + if (r->band_idx == MT_RX_SEL2) + cfg80211_background_radar_event(mphy->hw->wiphy, + &dev->rdd2_chandef, + GFP_ATOMIC); + else + ieee80211_radar_detected(mphy->hw); + dev->hw_pattern++; +} + +static void +mt7996_mcu_rx_log_message(struct mt7996_dev *dev, struct sk_buff *skb) +{ +#define UNI_EVENT_FW_LOG_FORMAT 0 + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + const char *data = (char *)&rxd[1] + 4, *type; + struct tlv *tlv = (struct tlv *)data; + int len; + + if (!(rxd->option & MCU_UNI_CMD_EVENT)) { + len = skb->len - sizeof(*rxd); + data = (char *)&rxd[1]; + goto out; + } + + if (le16_to_cpu(tlv->tag) != UNI_EVENT_FW_LOG_FORMAT) + return; + + data += sizeof(*tlv) + 4; + len = le16_to_cpu(tlv->len) - sizeof(*tlv) - 4; + +out: + switch (rxd->s2d_index) { + case 0: + if (mt7996_debugfs_rx_log(dev, data, len)) + return; + + type = "WM"; + break; + case 2: + type = "WA"; + break; + default: + type = "unknown"; + break; + } + + wiphy_info(mt76_hw(dev)->wiphy, "%s: %.*s", type, len, data); +} + +static void +mt7996_mcu_cca_finish(void *priv, u8 *mac, struct ieee80211_vif *vif) +{ + if (!vif->bss_conf.color_change_active) + return; + + ieee80211_color_change_finish(vif); +} + +static void +mt7996_mcu_ie_countdown(struct mt7996_dev *dev, struct sk_buff *skb) +{ +#define UNI_EVENT_IE_COUNTDOWN_CSA 0 +#define UNI_EVENT_IE_COUNTDOWN_BCC 1 + struct header { + u8 band; + u8 rsv[3]; + }; + struct mt76_phy *mphy = &dev->mt76.phy; + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + const char *data = (char *)&rxd[1], *tail; + struct header *hdr = (struct header *)data; + struct tlv *tlv = (struct tlv *)(data + 4); + + if (hdr->band && dev->mt76.phys[hdr->band]) + mphy = dev->mt76.phys[hdr->band]; + + tail = skb->data + le16_to_cpu(rxd->len); + while (data + sizeof(struct tlv) < tail && le16_to_cpu(tlv->len)) { + switch (le16_to_cpu(tlv->tag)) { + case UNI_EVENT_IE_COUNTDOWN_CSA: + ieee80211_iterate_active_interfaces_atomic(mphy->hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_mcu_csa_finish, mphy->hw); + break; + case UNI_EVENT_IE_COUNTDOWN_BCC: + ieee80211_iterate_active_interfaces_atomic(mphy->hw, + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7996_mcu_cca_finish, mphy->hw); + break; + } + + data += le16_to_cpu(tlv->len); + tlv = (struct tlv *)data; + } +} + +static void +mt7996_mcu_rx_ext_event(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + + switch (rxd->ext_eid) { + case MCU_EXT_EVENT_FW_LOG_2_HOST: + mt7996_mcu_rx_log_message(dev, skb); + break; + default: + break; + } +} + +static void +mt7996_mcu_rx_unsolicited_event(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + + switch (rxd->eid) { + case MCU_EVENT_EXT: + mt7996_mcu_rx_ext_event(dev, skb); + break; + default: + break; + } + dev_kfree_skb(skb); +} + +static void +mt7996_mcu_uni_rx_unsolicited_event(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + + switch (rxd->eid) { + case MCU_UNI_EVENT_FW_LOG_2_HOST: + mt7996_mcu_rx_log_message(dev, skb); + break; + case MCU_UNI_EVENT_IE_COUNTDOWN: + mt7996_mcu_ie_countdown(dev, skb); + break; + case MCU_UNI_EVENT_RDD_REPORT: + mt7996_mcu_rx_radar_detected(dev, skb); + break; + default: + break; + } + dev_kfree_skb(skb); +} + +void mt7996_mcu_rx_event(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt7996_mcu_rxd *rxd = (struct mt7996_mcu_rxd *)skb->data; + + if (rxd->option & MCU_UNI_CMD_UNSOLICITED_EVENT) { + mt7996_mcu_uni_rx_unsolicited_event(dev, skb); + return; + } + + /* WA still uses legacy event*/ + if (rxd->ext_eid == MCU_EXT_EVENT_FW_LOG_2_HOST || + !rxd->seq) + mt7996_mcu_rx_unsolicited_event(dev, skb); + else + mt76_mcu_rx_event(&dev->mt76, skb); +} + +static struct tlv * +mt7996_mcu_add_uni_tlv(struct sk_buff *skb, u16 tag, u16 len) +{ + struct tlv *ptlv, tlv = { + .tag = cpu_to_le16(tag), + .len = cpu_to_le16(len), + }; + + ptlv = skb_put(skb, len); + memcpy(ptlv, &tlv, sizeof(tlv)); + + return ptlv; +} + +static void +mt7996_mcu_bss_rfch_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, + struct mt7996_phy *phy) +{ + static const u8 rlm_ch_band[] = { + [NL80211_BAND_2GHZ] = 1, + [NL80211_BAND_5GHZ] = 2, + [NL80211_BAND_6GHZ] = 3, + }; + struct cfg80211_chan_def *chandef = &phy->mt76->chandef; + struct bss_rlm_tlv *ch; + struct tlv *tlv; + int freq1 = chandef->center_freq1; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_RLM, sizeof(*ch)); + + ch = (struct bss_rlm_tlv *)tlv; + ch->control_channel = chandef->chan->hw_value; + ch->center_chan = ieee80211_frequency_to_channel(freq1); + ch->bw = mt76_connac_chan_bw(chandef); + ch->tx_streams = hweight8(phy->mt76->antenna_mask); + ch->rx_streams = hweight8(phy->mt76->antenna_mask); + ch->band = rlm_ch_band[chandef->chan->band]; + + if (chandef->width == NL80211_CHAN_WIDTH_80P80) { + int freq2 = chandef->center_freq2; + + ch->center_chan2 = ieee80211_frequency_to_channel(freq2); + } +} + +static void +mt7996_mcu_bss_ra_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, + struct mt7996_phy *phy) +{ + struct bss_ra_tlv *ra; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_RA, sizeof(*ra)); + + ra = (struct bss_ra_tlv *)tlv; + ra->short_preamble = true; +} + +static void +mt7996_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, + struct mt7996_phy *phy) +{ +#define DEFAULT_HE_PE_DURATION 4 +#define DEFAULT_HE_DURATION_RTS_THRES 1023 + const struct ieee80211_sta_he_cap *cap; + struct bss_info_uni_he *he; + struct tlv *tlv; + + cap = mt76_connac_get_he_phy_cap(phy->mt76, vif); + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_HE_BASIC, sizeof(*he)); + + he = (struct bss_info_uni_he *)tlv; + he->he_pe_duration = vif->bss_conf.htc_trig_based_pkt_ext; + if (!he->he_pe_duration) + he->he_pe_duration = DEFAULT_HE_PE_DURATION; + + he->he_rts_thres = cpu_to_le16(vif->bss_conf.frame_time_rts_th); + if (!he->he_rts_thres) + he->he_rts_thres = cpu_to_le16(DEFAULT_HE_DURATION_RTS_THRES); + + he->max_nss_mcs[CMD_HE_MCS_BW80] = cap->he_mcs_nss_supp.tx_mcs_80; + he->max_nss_mcs[CMD_HE_MCS_BW160] = cap->he_mcs_nss_supp.tx_mcs_160; + he->max_nss_mcs[CMD_HE_MCS_BW8080] = cap->he_mcs_nss_supp.tx_mcs_80p80; +} + +static void +mt7996_mcu_bss_bmc_tlv(struct sk_buff *skb, struct mt7996_phy *phy) +{ + struct bss_rate_tlv *bmc; + struct cfg80211_chan_def *chandef = &phy->mt76->chandef; + enum nl80211_band band = chandef->chan->band; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_RATE, sizeof(*bmc)); + + bmc = (struct bss_rate_tlv *)tlv; + if (band == NL80211_BAND_2GHZ) { + bmc->short_preamble = true; + } else { + bmc->bc_trans = cpu_to_le16(0x8080); + bmc->mc_trans = cpu_to_le16(0x8080); + bmc->bc_fixed_rate = 1; + bmc->mc_fixed_rate = 1; + bmc->short_preamble = 1; + } +} + +static void +mt7996_mcu_bss_txcmd_tlv(struct sk_buff *skb, bool en) +{ + struct bss_txcmd_tlv *txcmd; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_TXCMD, sizeof(*txcmd)); + + txcmd = (struct bss_txcmd_tlv *)tlv; + txcmd->txcmd_mode = en; +} + +static void +mt7996_mcu_bss_mld_tlv(struct sk_buff *skb, struct ieee80211_vif *vif) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct bss_mld_tlv *mld; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_MLD, sizeof(*mld)); + + mld = (struct bss_mld_tlv *)tlv; + mld->group_mld_id = 0xff; + mld->own_mld_id = mvif->mt76.idx; + mld->remap_idx = 0xff; +} + +static void +mt7996_mcu_bss_sec_tlv(struct sk_buff *skb, struct ieee80211_vif *vif) +{ + struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; + struct bss_sec_tlv *sec; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_SEC, sizeof(*sec)); + + sec = (struct bss_sec_tlv *)tlv; + sec->cipher = mvif->cipher; +} + +static int +mt7996_mcu_muar_config(struct mt7996_phy *phy, struct ieee80211_vif *vif, + bool bssid, bool enable) +{ +#define UNI_MUAR_ENTRY 2 + struct mt7996_dev *dev = phy->dev; + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + u32 idx = mvif->mt76.omac_idx - REPEATER_BSSID_START; + const u8 *addr = vif->addr; + + struct { + struct { + u8 band; + u8 __rsv[3]; + } hdr; + + __le16 tag; + __le16 len; + + bool smesh; + u8 bssid; + u8 index; + u8 entry_add; + u8 addr[ETH_ALEN]; + u8 __rsv[2]; + } __packed req = { + .hdr.band = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_MUAR_ENTRY), + .len = cpu_to_le16(sizeof(req) - sizeof(req.hdr)), + .smesh = false, + .index = idx * 2 + bssid, + .entry_add = true, + }; + + if (bssid) + addr = vif->bss_conf.bssid; + + if (enable) + memcpy(req.addr, addr, ETH_ALEN); + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(REPT_MUAR), &req, + sizeof(req), true); +} + +static int +mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct mt76_phy *phy, u16 wlan_idx, + bool enable) +{ + struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; + struct cfg80211_chan_def *chandef = &phy->chandef; + struct mt76_connac_bss_basic_tlv *bss; + u32 type = CONNECTION_INFRA_AP; + struct tlv *tlv; + int idx; + + switch (vif->type) { + case NL80211_IFTYPE_MESH_POINT: + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_MONITOR: + break; + case NL80211_IFTYPE_STATION: + if (enable) { + rcu_read_lock(); + if (!sta) + sta = ieee80211_find_sta(vif, + vif->bss_conf.bssid); + /* TODO: enable BSS_INFO_UAPSD & BSS_INFO_PM */ + if (sta) { + struct mt76_wcid *wcid; + + wcid = (struct mt76_wcid *)sta->drv_priv; + wlan_idx = wcid->idx; + } + rcu_read_unlock(); + } + type = CONNECTION_INFRA_STA; + break; + case NL80211_IFTYPE_ADHOC: + type = CONNECTION_IBSS_ADHOC; + break; + default: + WARN_ON(1); + break; + } + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_BASIC, sizeof(*bss)); + + bss = (struct mt76_connac_bss_basic_tlv *)tlv; + bss->bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int); + bss->dtim_period = vif->bss_conf.dtim_period; + bss->bmc_tx_wlan_idx = cpu_to_le16(wlan_idx); + bss->sta_idx = cpu_to_le16(wlan_idx); + bss->conn_type = cpu_to_le32(type); + bss->omac_idx = mvif->omac_idx; + bss->band_idx = mvif->band_idx; + bss->wmm_idx = mvif->wmm_idx; + bss->conn_state = !enable; + bss->active = enable; + + idx = mvif->omac_idx > EXT_BSSID_START ? HW_BSSID_0 : mvif->omac_idx; + bss->hw_bss_idx = idx; + + if (vif->type == NL80211_IFTYPE_MONITOR) { + memcpy(bss->bssid, phy->macaddr, ETH_ALEN); + return 0; + } + + memcpy(bss->bssid, vif->bss_conf.bssid, ETH_ALEN); + bss->bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int); + bss->dtim_period = vif->bss_conf.dtim_period; + bss->phymode = mt76_connac_get_phy_mode(phy, vif, + chandef->chan->band, NULL); + + if (chandef->chan->band == NL80211_BAND_6GHZ) + bss->phymode_ext |= PHY_MODE_AX_6G; + + return 0; +} + +static struct sk_buff * +__mt7996_mcu_alloc_bss_req(struct mt76_dev *dev, struct mt76_vif *mvif, int len) +{ + struct bss_req_hdr hdr = { + .bss_idx = mvif->idx, + }; + struct sk_buff *skb; + + skb = mt76_mcu_msg_alloc(dev, NULL, len); + if (!skb) + return ERR_PTR(-ENOMEM); + + skb_put_data(skb, &hdr, sizeof(hdr)); + + return skb; +} + +int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, + struct ieee80211_vif *vif, int enable) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = phy->dev; + struct sk_buff *skb; + + if (mvif->mt76.omac_idx >= REPEATER_BSSID_START) { + mt7996_mcu_muar_config(phy, vif, false, enable); + mt7996_mcu_muar_config(phy, vif, true, enable); + } + + skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, + MT7996_BSS_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + /* bss_basic must be first */ + mt7996_mcu_bss_basic_tlv(skb, vif, NULL, phy->mt76, + mvif->sta.wcid.idx, enable); + mt7996_mcu_bss_sec_tlv(skb, vif); + + if (vif->type == NL80211_IFTYPE_MONITOR) + goto out; + + if (enable) { + mt7996_mcu_bss_rfch_tlv(skb, vif, phy); + mt7996_mcu_bss_bmc_tlv(skb, phy); + mt7996_mcu_bss_ra_tlv(skb, vif, phy); + mt7996_mcu_bss_txcmd_tlv(skb, true); + + if (vif->bss_conf.he_support) + mt7996_mcu_bss_he_tlv(skb, vif, phy); + + /* this tag is necessary no matter if the vif is MLD */ + mt7996_mcu_bss_mld_tlv(skb, vif); + } +out: + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + +static int +mt7996_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, + struct ieee80211_ampdu_params *params, + bool enable, bool tx) +{ + struct mt76_wcid *wcid = (struct mt76_wcid *)params->sta->drv_priv; + struct sta_rec_ba_uni *ba; + struct sk_buff *skb; + struct tlv *tlv; + + skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BA, sizeof(*ba)); + + ba = (struct sta_rec_ba_uni *)tlv; + ba->ba_type = tx ? MT_BA_TYPE_ORIGINATOR : MT_BA_TYPE_RECIPIENT; + ba->winsize = cpu_to_le16(params->buf_size); + ba->ssn = cpu_to_le16(params->ssn); + ba->ba_en = enable << params->tid; + ba->amsdu = params->amsdu; + ba->tid = params->tid; + + return mt76_mcu_skb_send_msg(dev, skb, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); +} + +/** starec & wtbl **/ +int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, + struct ieee80211_ampdu_params *params, + bool enable) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)params->sta->drv_priv; + struct mt7996_vif *mvif = msta->vif; + + if (enable && !params->amsdu) + msta->wcid.amsdu = false; + + return mt7996_mcu_sta_ba(&dev->mt76, &mvif->mt76, params, + enable, true); +} + +int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, + struct ieee80211_ampdu_params *params, + bool enable) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)params->sta->drv_priv; + struct mt7996_vif *mvif = msta->vif; + + return mt7996_mcu_sta_ba(&dev->mt76, &mvif->mt76, params, + enable, false); +} + +static void +mt7996_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) +{ + struct ieee80211_he_cap_elem *elem = &sta->deflink.he_cap.he_cap_elem; + struct ieee80211_he_mcs_nss_supp mcs_map; + struct sta_rec_he_v2 *he; + struct tlv *tlv; + int i = 0; + + if (!sta->deflink.he_cap.has_he) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HE_V2, sizeof(*he)); + + he = (struct sta_rec_he_v2 *)tlv; + for (i = 0; i < 11; i++) { + if (i < 6) + he->he_mac_cap[i] = cpu_to_le16(elem->mac_cap_info[i]); + he->he_phy_cap[i] = cpu_to_le16(elem->phy_cap_info[i]); + } + + mcs_map = sta->deflink.he_cap.he_mcs_nss_supp; + switch (sta->deflink.bandwidth) { + case IEEE80211_STA_RX_BW_160: + if (elem->phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G) + mt7996_mcu_set_sta_he_mcs(sta, + &he->max_nss_mcs[CMD_HE_MCS_BW8080], + le16_to_cpu(mcs_map.rx_mcs_80p80)); + + mt7996_mcu_set_sta_he_mcs(sta, + &he->max_nss_mcs[CMD_HE_MCS_BW160], + le16_to_cpu(mcs_map.rx_mcs_160)); + fallthrough; + default: + mt7996_mcu_set_sta_he_mcs(sta, + &he->max_nss_mcs[CMD_HE_MCS_BW80], + le16_to_cpu(mcs_map.rx_mcs_80)); + break; + } + + he->pkt_ext = 2; +} + +static void +mt7996_mcu_sta_he_6g_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) +{ + struct sta_rec_he_6g_capa *he_6g; + struct tlv *tlv; + + if (!sta->deflink.he_6ghz_capa.capa) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HE_6G, sizeof(*he_6g)); + + he_6g = (struct sta_rec_he_6g_capa *)tlv; + he_6g->capa = sta->deflink.he_6ghz_capa.capa; +} + +static void +mt7996_mcu_sta_ht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) +{ + struct sta_rec_ht *ht; + struct tlv *tlv; + + if (!sta->deflink.ht_cap.ht_supported) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HT, sizeof(*ht)); + + ht = (struct sta_rec_ht *)tlv; + ht->ht_cap = cpu_to_le16(sta->deflink.ht_cap.cap); +} + +static void +mt7996_mcu_sta_vht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) +{ + struct sta_rec_vht *vht; + struct tlv *tlv; + + /* For 6G band, this tlv is necessary to let hw work normally */ + if (!sta->deflink.he_6ghz_capa.capa && !sta->deflink.vht_cap.vht_supported) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_VHT, sizeof(*vht)); + + vht = (struct sta_rec_vht *)tlv; + vht->vht_cap = cpu_to_le32(sta->deflink.vht_cap.cap); + vht->vht_rx_mcs_map = sta->deflink.vht_cap.vht_mcs.rx_mcs_map; + vht->vht_tx_mcs_map = sta->deflink.vht_cap.vht_mcs.tx_mcs_map; +} + +static void +mt7996_mcu_sta_amsdu_tlv(struct mt7996_dev *dev, struct sk_buff *skb, + struct ieee80211_vif *vif, struct ieee80211_sta *sta) +{ + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct sta_rec_amsdu *amsdu; + struct tlv *tlv; + + if (vif->type != NL80211_IFTYPE_STATION && + vif->type != NL80211_IFTYPE_AP) + return; + + if (!sta->deflink.agg.max_amsdu_len) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu)); + amsdu = (struct sta_rec_amsdu *)tlv; + amsdu->max_amsdu_num = 8; + amsdu->amsdu_en = true; + msta->wcid.amsdu = true; + + switch (sta->deflink.agg.max_amsdu_len) { + case IEEE80211_MAX_MPDU_LEN_VHT_11454: + amsdu->max_mpdu_size = + IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454; + return; + case IEEE80211_MAX_MPDU_LEN_HT_7935: + case IEEE80211_MAX_MPDU_LEN_VHT_7991: + amsdu->max_mpdu_size = IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991; + return; + default: + amsdu->max_mpdu_size = IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895; + return; + } +} + +static inline bool +mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, bool bfee) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + int tx_ant = hweight8(phy->mt76->antenna_mask) - 1; + + if (vif->type != NL80211_IFTYPE_STATION && + vif->type != NL80211_IFTYPE_AP) + return false; + + if (!bfee && tx_ant < 2) + return false; + + if (sta->deflink.he_cap.has_he) { + struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem; + + if (bfee) + return mvif->cap.he_su_ebfee && + HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]); + else + return mvif->cap.he_su_ebfer && + HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]); + } + + if (sta->deflink.vht_cap.vht_supported) { + u32 cap = sta->deflink.vht_cap.cap; + + if (bfee) + return mvif->cap.vht_su_ebfee && + (cap & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE); + else + return mvif->cap.vht_su_ebfer && + (cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE); + } + + return false; +} + +static void +mt7996_mcu_sta_sounding_rate(struct sta_rec_bf *bf) +{ + bf->sounding_phy = MT_PHY_TYPE_OFDM; + bf->ndp_rate = 0; /* mcs0 */ + bf->ndpa_rate = MT7996_CFEND_RATE_DEFAULT; /* ofdm 24m */ + bf->rept_poll_rate = MT7996_CFEND_RATE_DEFAULT; /* ofdm 24m */ +} + +static void +mt7996_mcu_sta_bfer_ht(struct ieee80211_sta *sta, struct mt7996_phy *phy, + struct sta_rec_bf *bf) +{ + struct ieee80211_mcs_info *mcs = &sta->deflink.ht_cap.mcs; + u8 n = 0; + + bf->tx_mode = MT_PHY_TYPE_HT; + + if ((mcs->tx_params & IEEE80211_HT_MCS_TX_RX_DIFF) && + (mcs->tx_params & IEEE80211_HT_MCS_TX_DEFINED)) + n = FIELD_GET(IEEE80211_HT_MCS_TX_MAX_STREAMS_MASK, + mcs->tx_params); + else if (mcs->rx_mask[3]) + n = 3; + else if (mcs->rx_mask[2]) + n = 2; + else if (mcs->rx_mask[1]) + n = 1; + + bf->nrow = hweight8(phy->mt76->antenna_mask) - 1; + bf->ncol = min_t(u8, bf->nrow, n); + bf->ibf_ncol = n; +} + +static void +mt7996_mcu_sta_bfer_vht(struct ieee80211_sta *sta, struct mt7996_phy *phy, + struct sta_rec_bf *bf, bool explicit) +{ + struct ieee80211_sta_vht_cap *pc = &sta->deflink.vht_cap; + struct ieee80211_sta_vht_cap *vc = &phy->mt76->sband_5g.sband.vht_cap; + u16 mcs_map = le16_to_cpu(pc->vht_mcs.rx_mcs_map); + u8 nss_mcs = mt7996_mcu_get_sta_nss(mcs_map); + u8 tx_ant = hweight8(phy->mt76->antenna_mask) - 1; + + bf->tx_mode = MT_PHY_TYPE_VHT; + + if (explicit) { + u8 sts, snd_dim; + + mt7996_mcu_sta_sounding_rate(bf); + + sts = FIELD_GET(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK, + pc->cap); + snd_dim = FIELD_GET(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK, + vc->cap); + bf->nrow = min_t(u8, min_t(u8, snd_dim, sts), tx_ant); + bf->ncol = min_t(u8, nss_mcs, bf->nrow); + bf->ibf_ncol = bf->ncol; + + if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_160) + bf->nrow = 1; + } else { + bf->nrow = tx_ant; + bf->ncol = min_t(u8, nss_mcs, bf->nrow); + bf->ibf_ncol = nss_mcs; + + if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_160) + bf->ibf_nrow = 1; + } +} + +static void +mt7996_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif, + struct mt7996_phy *phy, struct sta_rec_bf *bf) +{ + struct ieee80211_sta_he_cap *pc = &sta->deflink.he_cap; + struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem; + const struct ieee80211_sta_he_cap *vc = + mt76_connac_get_he_phy_cap(phy->mt76, vif); + const struct ieee80211_he_cap_elem *ve = &vc->he_cap_elem; + u16 mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80); + u8 nss_mcs = mt7996_mcu_get_sta_nss(mcs_map); + u8 snd_dim, sts; + + bf->tx_mode = MT_PHY_TYPE_HE_SU; + + mt7996_mcu_sta_sounding_rate(bf); + + bf->trigger_su = HE_PHY(CAP6_TRIG_SU_BEAMFORMING_FB, + pe->phy_cap_info[6]); + bf->trigger_mu = HE_PHY(CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB, + pe->phy_cap_info[6]); + snd_dim = HE_PHY(CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK, + ve->phy_cap_info[5]); + sts = HE_PHY(CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK, + pe->phy_cap_info[4]); + bf->nrow = min_t(u8, snd_dim, sts); + bf->ncol = min_t(u8, nss_mcs, bf->nrow); + bf->ibf_ncol = bf->ncol; + + if (sta->deflink.bandwidth != IEEE80211_STA_RX_BW_160) + return; + + /* go over for 160MHz and 80p80 */ + if (pe->phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G) { + mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_160); + nss_mcs = mt7996_mcu_get_sta_nss(mcs_map); + + bf->ncol_gt_bw80 = nss_mcs; + } + + if (pe->phy_cap_info[0] & + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G) { + mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80p80); + nss_mcs = mt7996_mcu_get_sta_nss(mcs_map); + + if (bf->ncol_gt_bw80) + bf->ncol_gt_bw80 = min_t(u8, bf->ncol_gt_bw80, nss_mcs); + else + bf->ncol_gt_bw80 = nss_mcs; + } + + snd_dim = HE_PHY(CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK, + ve->phy_cap_info[5]); + sts = HE_PHY(CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_MASK, + pe->phy_cap_info[4]); + + bf->nrow_gt_bw80 = min_t(int, snd_dim, sts); +} + +static void +mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb, + struct ieee80211_vif *vif, struct ieee80211_sta *sta) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_phy *phy = mvif->phy; + int tx_ant = hweight8(phy->mt76->antenna_mask) - 1; + struct sta_rec_bf *bf; + struct tlv *tlv; + const u8 matrix[4][4] = { + {0, 0, 0, 0}, + {1, 1, 0, 0}, /* 2x1, 2x2, 2x3, 2x4 */ + {2, 4, 4, 0}, /* 3x1, 3x2, 3x3, 3x4 */ + {3, 5, 6, 0} /* 4x1, 4x2, 4x3, 4x4 */ + }; + bool ebf; + + if (!(sta->deflink.ht_cap.ht_supported || sta->deflink.he_cap.has_he)) + return; + + ebf = mt7996_is_ebf_supported(phy, vif, sta, false); + if (!ebf && !dev->ibf) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BF, sizeof(*bf)); + bf = (struct sta_rec_bf *)tlv; + + /* he: eBF only, in accordance with spec + * vht: support eBF and iBF + * ht: iBF only, since mac80211 lacks of eBF support + */ + if (sta->deflink.he_cap.has_he && ebf) + mt7996_mcu_sta_bfer_he(sta, vif, phy, bf); + else if (sta->deflink.vht_cap.vht_supported) + mt7996_mcu_sta_bfer_vht(sta, phy, bf, ebf); + else if (sta->deflink.ht_cap.ht_supported) + mt7996_mcu_sta_bfer_ht(sta, phy, bf); + else + return; + + bf->bf_cap = ebf ? ebf : dev->ibf << 1; + bf->bw = sta->deflink.bandwidth; + bf->ibf_dbw = sta->deflink.bandwidth; + bf->ibf_nrow = tx_ant; + + if (!ebf && sta->deflink.bandwidth <= IEEE80211_STA_RX_BW_40 && !bf->ncol) + bf->ibf_timeout = 0x48; + else + bf->ibf_timeout = 0x18; + + if (ebf && bf->nrow != tx_ant) + bf->mem_20m = matrix[tx_ant][bf->ncol]; + else + bf->mem_20m = matrix[bf->nrow][bf->ncol]; + + switch (sta->deflink.bandwidth) { + case IEEE80211_STA_RX_BW_160: + case IEEE80211_STA_RX_BW_80: + bf->mem_total = bf->mem_20m * 2; + break; + case IEEE80211_STA_RX_BW_40: + bf->mem_total = bf->mem_20m; + break; + case IEEE80211_STA_RX_BW_20: + default: + break; + } +} + +static void +mt7996_mcu_sta_bfee_tlv(struct mt7996_dev *dev, struct sk_buff *skb, + struct ieee80211_vif *vif, struct ieee80211_sta *sta) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_phy *phy = mvif->phy; + int tx_ant = hweight8(phy->mt76->antenna_mask) - 1; + struct sta_rec_bfee *bfee; + struct tlv *tlv; + u8 nrow = 0; + + if (!(sta->deflink.vht_cap.vht_supported || sta->deflink.he_cap.has_he)) + return; + + if (!mt7996_is_ebf_supported(phy, vif, sta, true)) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BFEE, sizeof(*bfee)); + bfee = (struct sta_rec_bfee *)tlv; + + if (sta->deflink.he_cap.has_he) { + struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem; + + nrow = HE_PHY(CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK, + pe->phy_cap_info[5]); + } else if (sta->deflink.vht_cap.vht_supported) { + struct ieee80211_sta_vht_cap *pc = &sta->deflink.vht_cap; + + nrow = FIELD_GET(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK, + pc->cap); + } + + /* reply with identity matrix to avoid 2x2 BF negative gain */ + bfee->fb_identity_matrix = (nrow == 1 && tx_ant == 2); +} + +static void +mt7996_mcu_sta_phy_tlv(struct mt7996_dev *dev, struct sk_buff *skb, + struct ieee80211_vif *vif, struct ieee80211_sta *sta) +{ + struct sta_rec_phy *phy; + struct tlv *tlv; + u8 af = 0, mm = 0; + + if (!sta->deflink.ht_cap.ht_supported && !sta->deflink.he_6ghz_capa.capa) + return; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_PHY, sizeof(*phy)); + + phy = (struct sta_rec_phy *)tlv; + if (sta->deflink.ht_cap.ht_supported) { + af = sta->deflink.ht_cap.ampdu_factor; + mm = sta->deflink.ht_cap.ampdu_density; + } + + if (sta->deflink.vht_cap.vht_supported) { + u8 vht_af = FIELD_GET(IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK, + sta->deflink.vht_cap.cap); + + af = max_t(u8, af, vht_af); + } + + if (sta->deflink.he_6ghz_capa.capa) { + af = le16_get_bits(sta->deflink.he_6ghz_capa.capa, + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP); + mm = le16_get_bits(sta->deflink.he_6ghz_capa.capa, + IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START); + } + + phy->ampdu = FIELD_PREP(IEEE80211_HT_AMPDU_PARM_FACTOR, af) | + FIELD_PREP(IEEE80211_HT_AMPDU_PARM_DENSITY, mm); + phy->max_ampdu_len = af; +} + +static void +mt7996_mcu_sta_hdrt_tlv(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct sta_rec_hdrt *hdrt; + struct tlv *tlv; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HDRT, sizeof(*hdrt)); + + hdrt = (struct sta_rec_hdrt *)tlv; + hdrt->hdrt_mode = 1; +} + +static void +mt7996_mcu_sta_hdr_trans_tlv(struct mt7996_dev *dev, struct sk_buff *skb, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct sta_rec_hdr_trans *hdr_trans; + struct mt76_wcid *wcid; + struct tlv *tlv; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HDR_TRANS, sizeof(*hdr_trans)); + hdr_trans = (struct sta_rec_hdr_trans *)tlv; + hdr_trans->dis_rx_hdr_tran = true; + + if (vif->type == NL80211_IFTYPE_STATION) + hdr_trans->to_ds = true; + else + hdr_trans->from_ds = true; + + wcid = (struct mt76_wcid *)sta->drv_priv; + if (!wcid) + return; + + hdr_trans->dis_rx_hdr_tran = !test_bit(MT_WCID_FLAG_HDR_TRANS, &wcid->flags); + if (test_bit(MT_WCID_FLAG_4ADDR, &wcid->flags)) { + hdr_trans->to_ds = true; + hdr_trans->from_ds = true; + } +} + +static enum mcu_mmps_mode +mt7996_mcu_get_mmps_mode(enum ieee80211_smps_mode smps) +{ + switch (smps) { + case IEEE80211_SMPS_OFF: + return MCU_MMPS_DISABLE; + case IEEE80211_SMPS_STATIC: + return MCU_MMPS_STATIC; + case IEEE80211_SMPS_DYNAMIC: + return MCU_MMPS_DYNAMIC; + default: + return MCU_MMPS_DISABLE; + } +} + +int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev, + void *data, u16 version) +{ + struct ra_fixed_rate *req; + struct uni_header hdr; + struct sk_buff *skb; + struct tlv *tlv; + int len; + + len = sizeof(hdr) + sizeof(*req); + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &hdr, sizeof(hdr)); + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_RA_FIXED_RATE, sizeof(*req)); + req = (struct ra_fixed_rate *)tlv; + req->version = cpu_to_le16(version); + memcpy(&req->rate, data, sizeof(req->rate)); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(RA), true); +} + +static void +mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev, + struct ieee80211_vif *vif, struct ieee80211_sta *sta) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt76_phy *mphy = mvif->phy->mt76; + struct cfg80211_chan_def *chandef = &mphy->chandef; + struct cfg80211_bitrate_mask *mask = &mvif->bitrate_mask; + enum nl80211_band band = chandef->chan->band; + struct sta_rec_ra *ra; + struct tlv *tlv; + u32 supp_rate = sta->deflink.supp_rates[band]; + u32 cap = sta->wme ? STA_CAP_WMM : 0; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA, sizeof(*ra)); + ra = (struct sta_rec_ra *)tlv; + + ra->valid = true; + ra->auto_rate = true; + ra->phy_mode = mt76_connac_get_phy_mode(mphy, vif, band, sta); + ra->channel = chandef->chan->hw_value; + ra->bw = sta->deflink.bandwidth; + ra->phy.bw = sta->deflink.bandwidth; + ra->mmps_mode = mt7996_mcu_get_mmps_mode(sta->deflink.smps_mode); + + if (supp_rate) { + supp_rate &= mask->control[band].legacy; + ra->rate_len = hweight32(supp_rate); + + if (band == NL80211_BAND_2GHZ) { + ra->supp_mode = MODE_CCK; + ra->supp_cck_rate = supp_rate & GENMASK(3, 0); + + if (ra->rate_len > 4) { + ra->supp_mode |= MODE_OFDM; + ra->supp_ofdm_rate = supp_rate >> 4; + } + } else { + ra->supp_mode = MODE_OFDM; + ra->supp_ofdm_rate = supp_rate; + } + } + + if (sta->deflink.ht_cap.ht_supported) { + ra->supp_mode |= MODE_HT; + ra->af = sta->deflink.ht_cap.ampdu_factor; + ra->ht_gf = !!(sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_GRN_FLD); + + cap |= STA_CAP_HT; + if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SGI_20) + cap |= STA_CAP_SGI_20; + if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SGI_40) + cap |= STA_CAP_SGI_40; + if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_TX_STBC) + cap |= STA_CAP_TX_STBC; + if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_RX_STBC) + cap |= STA_CAP_RX_STBC; + if (mvif->cap.ht_ldpc && + (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_LDPC_CODING)) + cap |= STA_CAP_LDPC; + + mt7996_mcu_set_sta_ht_mcs(sta, ra->ht_mcs, + mask->control[band].ht_mcs); + ra->supp_ht_mcs = *(__le32 *)ra->ht_mcs; + } + + if (sta->deflink.vht_cap.vht_supported) { + u8 af; + + ra->supp_mode |= MODE_VHT; + af = FIELD_GET(IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK, + sta->deflink.vht_cap.cap); + ra->af = max_t(u8, ra->af, af); + + cap |= STA_CAP_VHT; + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80) + cap |= STA_CAP_VHT_SGI_80; + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_160) + cap |= STA_CAP_VHT_SGI_160; + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC) + cap |= STA_CAP_VHT_TX_STBC; + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1) + cap |= STA_CAP_VHT_RX_STBC; + if (mvif->cap.vht_ldpc && + (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)) + cap |= STA_CAP_VHT_LDPC; + + mt7996_mcu_set_sta_vht_mcs(sta, ra->supp_vht_mcs, + mask->control[band].vht_mcs); + } + + if (sta->deflink.he_cap.has_he) { + ra->supp_mode |= MODE_HE; + cap |= STA_CAP_HE; + + if (sta->deflink.he_6ghz_capa.capa) + ra->af = le16_get_bits(sta->deflink.he_6ghz_capa.capa, + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP); + } + ra->sta_cap = cpu_to_le32(cap); +} + +int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, bool changed) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct sk_buff *skb; + + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->mt76, + &msta->wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + /* firmware rc algorithm refers to sta_rec_he for HE control. + * once dev->rc_work changes the settings driver should also + * update sta_rec_he here. + */ + if (changed) + mt7996_mcu_sta_he_tlv(skb, sta); + + /* sta_rec_ra accommodates BW, NSS and only MCS range format + * i.e 0-{7,8,9} for VHT. + */ + mt7996_mcu_sta_rate_ctrl_tlv(skb, dev, vif, sta); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); +} + +static int +mt7996_mcu_add_group(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ +#define MT_STA_BSS_GROUP 1 + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta; + struct { + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + __le16 wlan_idx; + u8 __rsv2[2]; + __le32 action; + __le32 val; + u8 __rsv3[8]; + } __packed req = { + .tag = cpu_to_le16(UNI_VOW_DRR_CTRL), + .len = cpu_to_le16(sizeof(req) - 4), + .action = cpu_to_le32(MT_STA_BSS_GROUP), + .val = cpu_to_le32(mvif->mt76.idx % 16), + }; + + msta = sta ? (struct mt7996_sta *)sta->drv_priv : &mvif->sta; + req.wlan_idx = cpu_to_le16(msta->wcid.idx); + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(VOW), &req, + sizeof(req), true); +} + +int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, bool enable) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta; + struct sk_buff *skb; + int ret; + + msta = sta ? (struct mt7996_sta *)sta->drv_priv : &mvif->sta; + + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->mt76, + &msta->wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + /* starec basic */ + mt76_connac_mcu_sta_basic_tlv(skb, vif, sta, enable, + !rcu_access_pointer(dev->mt76.wcid[msta->wcid.idx])); + if (!enable) + goto out; + + /* tag order is in accordance with firmware dependency. */ + if (sta) { + /* starec phy */ + mt7996_mcu_sta_phy_tlv(dev, skb, vif, sta); + /* starec hdrt mode */ + mt7996_mcu_sta_hdrt_tlv(dev, skb); + /* starec bfer */ + mt7996_mcu_sta_bfer_tlv(dev, skb, vif, sta); + /* starec ht */ + mt7996_mcu_sta_ht_tlv(skb, sta); + /* starec vht */ + mt7996_mcu_sta_vht_tlv(skb, sta); + /* starec uapsd */ + mt76_connac_mcu_sta_uapsd(skb, vif, sta); + /* starec amsdu */ + mt7996_mcu_sta_amsdu_tlv(dev, skb, vif, sta); + /* starec he */ + mt7996_mcu_sta_he_tlv(skb, sta); + /* starec he 6g*/ + mt7996_mcu_sta_he_6g_tlv(skb, sta); + /* TODO: starec muru */ + /* starec bfee */ + mt7996_mcu_sta_bfee_tlv(dev, skb, vif, sta); + /* starec hdr trans */ + mt7996_mcu_sta_hdr_trans_tlv(dev, skb, vif, sta); + } + + ret = mt7996_mcu_add_group(dev, vif, sta); + if (ret) { + dev_kfree_skb(skb); + return ret; + } +out: + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); +} + +static int +mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, + struct mt76_connac_sta_key_conf *sta_key_conf, + struct sk_buff *skb, + struct ieee80211_key_conf *key, + enum set_key_cmd cmd) +{ + struct sta_rec_sec_uni *sec; + struct tlv *tlv; + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_KEY_V2, sizeof(*sec)); + sec = (struct sta_rec_sec_uni *)tlv; + sec->add = cmd; + + if (cmd == SET_KEY) { + struct sec_key_uni *sec_key; + u8 cipher; + + cipher = mt76_connac_mcu_get_cipher(key->cipher); + if (cipher == MCU_CIPHER_NONE) + return -EOPNOTSUPP; + + sec_key = &sec->key[0]; + sec_key->cipher_len = sizeof(*sec_key); + + if (cipher == MCU_CIPHER_BIP_CMAC_128) { + sec_key->wlan_idx = cpu_to_le16(wcid->idx); + sec_key->cipher_id = MCU_CIPHER_AES_CCMP; + sec_key->key_id = sta_key_conf->keyidx; + sec_key->key_len = 16; + memcpy(sec_key->key, sta_key_conf->key, 16); + + sec_key = &sec->key[1]; + sec_key->wlan_idx = cpu_to_le16(wcid->idx); + sec_key->cipher_id = MCU_CIPHER_BIP_CMAC_128; + sec_key->cipher_len = sizeof(*sec_key); + sec_key->key_len = 16; + memcpy(sec_key->key, key->key, 16); + sec->n_cipher = 2; + } else { + sec_key->wlan_idx = cpu_to_le16(wcid->idx); + sec_key->cipher_id = cipher; + sec_key->key_id = key->keyidx; + sec_key->key_len = key->keylen; + memcpy(sec_key->key, key->key, key->keylen); + + if (cipher == MCU_CIPHER_TKIP) { + /* Rx/Tx MIC keys are swapped */ + memcpy(sec_key->key + 16, key->key + 24, 8); + memcpy(sec_key->key + 24, key->key + 16, 8); + } + + /* store key_conf for BIP batch update */ + if (cipher == MCU_CIPHER_AES_CCMP) { + memcpy(sta_key_conf->key, key->key, key->keylen); + sta_key_conf->keyidx = key->keyidx; + } + + sec->n_cipher = 1; + } + } else { + sec->n_cipher = 0; + } + + return 0; +} + +int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, + struct mt76_connac_sta_key_conf *sta_key_conf, + struct ieee80211_key_conf *key, int mcu_cmd, + struct mt76_wcid *wcid, enum set_key_cmd cmd) +{ + struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; + struct sk_buff *skb; + int ret; + + skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + ret = mt7996_mcu_sta_key_tlv(wcid, sta_key_conf, skb, key, cmd); + if (ret) + return ret; + + return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); +} + +int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, + struct ieee80211_vif *vif, bool enable) +{ + struct mt7996_dev *dev = phy->dev; + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct { + struct req_hdr { + u8 omac_idx; + u8 band_idx; + u8 __rsv[2]; + } __packed hdr; + struct req_tlv { + __le16 tag; + __le16 len; + u8 active; + u8 __rsv; + u8 omac_addr[ETH_ALEN]; + } __packed tlv; + } data = { + .hdr = { + .omac_idx = mvif->mt76.omac_idx, + .band_idx = mvif->mt76.band_idx, + }, + .tlv = { + .tag = cpu_to_le16(DEV_INFO_ACTIVE), + .len = cpu_to_le16(sizeof(struct req_tlv)), + .active = enable, + }, + }; + + if (mvif->mt76.omac_idx >= REPEATER_BSSID_START) + return mt7996_mcu_muar_config(phy, vif, false, enable); + + memcpy(data.tlv.omac_addr, vif->addr, ETH_ALEN); + return mt76_mcu_send_msg(&dev->mt76, MCU_WMWA_UNI_CMD(DEV_INFO_UPDATE), + &data, sizeof(data), true); +} + +static void +mt7996_mcu_beacon_cntdwn(struct ieee80211_vif *vif, struct sk_buff *rskb, + struct sk_buff *skb, + struct ieee80211_mutable_offsets *offs) +{ + struct bss_bcn_cntdwn_tlv *info; + struct tlv *tlv; + u16 tag; + + if (!offs->cntdwn_counter_offs[0]) + return; + + tag = vif->bss_conf.csa_active ? UNI_BSS_INFO_BCN_CSA : UNI_BSS_INFO_BCN_BCC; + + tlv = mt7996_mcu_add_uni_tlv(rskb, tag, sizeof(*info)); + + info = (struct bss_bcn_cntdwn_tlv *)tlv; + info->cnt = skb->data[offs->cntdwn_counter_offs[0]]; +} + +static void +mt7996_mcu_beacon_cont(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct sk_buff *rskb, struct sk_buff *skb, + struct bss_bcn_content_tlv *bcn, + struct ieee80211_mutable_offsets *offs) +{ + struct mt76_wcid *wcid = &dev->mt76.global_wcid; + u8 *buf; + + bcn->pkt_len = cpu_to_le16(MT_TXD_SIZE + skb->len); + bcn->tim_ie_pos = cpu_to_le16(offs->tim_offset); + + if (offs->cntdwn_counter_offs[0]) { + u16 offset = offs->cntdwn_counter_offs[0]; + + if (vif->bss_conf.csa_active) + bcn->csa_ie_pos = cpu_to_le16(offset - 4); + if (vif->bss_conf.color_change_active) + bcn->bcc_ie_pos = cpu_to_le16(offset - 3); + } + + buf = (u8 *)bcn + sizeof(*bcn) - MAX_BEACON_SIZE; + mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, 0, NULL, + BSS_CHANGED_BEACON); + memcpy(buf + MT_TXD_SIZE, skb->data, skb->len); +} + +static void +mt7996_mcu_beacon_check_caps(struct mt7996_phy *phy, struct ieee80211_vif *vif, + struct sk_buff *skb) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_vif_cap *vc = &mvif->cap; + const struct ieee80211_he_cap_elem *he; + const struct ieee80211_vht_cap *vht; + const struct ieee80211_ht_cap *ht; + struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data; + const u8 *ie; + u32 len, bc; + + /* Check missing configuration options to allow AP mode in mac80211 + * to remain in sync with hostapd settings, and get a subset of + * beacon and hardware capabilities. + */ + if (WARN_ON_ONCE(skb->len <= (mgmt->u.beacon.variable - skb->data))) + return; + + memset(vc, 0, sizeof(*vc)); + + len = skb->len - (mgmt->u.beacon.variable - skb->data); + + ie = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, mgmt->u.beacon.variable, + len); + if (ie && ie[1] >= sizeof(*ht)) { + ht = (void *)(ie + 2); + vc->ht_ldpc |= !!(le16_to_cpu(ht->cap_info) & + IEEE80211_HT_CAP_LDPC_CODING); + } + + ie = cfg80211_find_ie(WLAN_EID_VHT_CAPABILITY, mgmt->u.beacon.variable, + len); + if (ie && ie[1] >= sizeof(*vht)) { + u32 pc = phy->mt76->sband_5g.sband.vht_cap.cap; + + vht = (void *)(ie + 2); + bc = le32_to_cpu(vht->vht_cap_info); + + vc->vht_ldpc |= !!(bc & IEEE80211_VHT_CAP_RXLDPC); + vc->vht_su_ebfer = + (bc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE) && + (pc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE); + vc->vht_su_ebfee = + (bc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE) && + (pc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE); + vc->vht_mu_ebfer = + (bc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE) && + (pc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE); + vc->vht_mu_ebfee = + (bc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE) && + (pc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE); + } + + ie = cfg80211_find_ext_ie(WLAN_EID_EXT_HE_CAPABILITY, + mgmt->u.beacon.variable, len); + if (ie && ie[1] >= sizeof(*he) + 1) { + const struct ieee80211_sta_he_cap *pc = + mt76_connac_get_he_phy_cap(phy->mt76, vif); + const struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem; + + he = (void *)(ie + 3); + + vc->he_ldpc = + HE_PHY(CAP1_LDPC_CODING_IN_PAYLOAD, pe->phy_cap_info[1]); + vc->he_su_ebfer = + HE_PHY(CAP3_SU_BEAMFORMER, he->phy_cap_info[3]) && + HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]); + vc->he_su_ebfee = + HE_PHY(CAP4_SU_BEAMFORMEE, he->phy_cap_info[4]) && + HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]); + vc->he_mu_ebfer = + HE_PHY(CAP4_MU_BEAMFORMER, he->phy_cap_info[4]) && + HE_PHY(CAP4_MU_BEAMFORMER, pe->phy_cap_info[4]); + } +} + +int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, int en) +{ + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct ieee80211_mutable_offsets offs; + struct ieee80211_tx_info *info; + struct sk_buff *skb, *rskb; + struct tlv *tlv; + struct bss_bcn_content_tlv *bcn; + + rskb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, + MT7996_BEACON_UPDATE_SIZE); + if (IS_ERR(rskb)) + return PTR_ERR(rskb); + + tlv = mt7996_mcu_add_uni_tlv(rskb, + UNI_BSS_INFO_BCN_CONTENT, sizeof(*bcn)); + bcn = (struct bss_bcn_content_tlv *)tlv; + bcn->enable = en; + + if (!en) + goto out; + + skb = ieee80211_beacon_get_template(hw, vif, &offs, 0); + if (!skb) + return -EINVAL; + + if (skb->len > MAX_BEACON_SIZE - MT_TXD_SIZE) { + dev_err(dev->mt76.dev, "Bcn size limit exceed\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + + info = IEEE80211_SKB_CB(skb); + info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->mt76->band_idx); + + mt7996_mcu_beacon_check_caps(phy, vif, skb); + + mt7996_mcu_beacon_cont(dev, vif, rskb, skb, bcn, &offs); + /* TODO: subtag - 11v MBSSID */ + mt7996_mcu_beacon_cntdwn(vif, rskb, skb, &offs); + dev_kfree_skb(skb); +out: + return mt76_mcu_skb_send_msg(&phy->dev->mt76, rskb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + +int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev, + struct ieee80211_vif *vif, u32 changed) +{ +#define OFFLOAD_TX_MODE_SU BIT(0) +#define OFFLOAD_TX_MODE_MU BIT(1) + struct ieee80211_hw *hw = mt76_hw(dev); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct cfg80211_chan_def *chandef = &mvif->phy->mt76->chandef; + enum nl80211_band band = chandef->chan->band; + struct mt76_wcid *wcid = &dev->mt76.global_wcid; + struct bss_inband_discovery_tlv *discov; + struct ieee80211_tx_info *info; + struct sk_buff *rskb, *skb = NULL; + struct tlv *tlv; + u8 *buf, interval; + + rskb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, + MT7996_INBAND_FRAME_SIZE); + if (IS_ERR(rskb)) + return PTR_ERR(rskb); + + if (changed & BSS_CHANGED_FILS_DISCOVERY && + vif->bss_conf.fils_discovery.max_interval) { + interval = vif->bss_conf.fils_discovery.max_interval; + skb = ieee80211_get_fils_discovery_tmpl(hw, vif); + } else if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP && + vif->bss_conf.unsol_bcast_probe_resp_interval) { + interval = vif->bss_conf.unsol_bcast_probe_resp_interval; + skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif); + } + + if (!skb) + return -EINVAL; + + if (skb->len > MAX_INBAND_FRAME_SIZE - MT_TXD_SIZE) { + dev_err(dev->mt76.dev, "inband discovery size limit exceed\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + + info = IEEE80211_SKB_CB(skb); + info->control.vif = vif; + info->band = band; + info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->mt76->band_idx); + + tlv = mt7996_mcu_add_uni_tlv(rskb, UNI_BSS_INFO_OFFLOAD, sizeof(*discov)); + + discov = (struct bss_inband_discovery_tlv *)tlv; + discov->tx_mode = OFFLOAD_TX_MODE_SU; + /* 0: UNSOL PROBE RESP, 1: FILS DISCOV */ + discov->tx_type = !!(changed & BSS_CHANGED_FILS_DISCOVERY); + discov->tx_interval = interval; + discov->prob_rsp_len = cpu_to_le16(MT_TXD_SIZE + skb->len); + discov->enable = true; + discov->wcid = cpu_to_le16(MT7996_WTBL_RESERVED); + + buf = (u8 *)tlv + sizeof(*discov) - MAX_INBAND_FRAME_SIZE; + + mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, 0, NULL, + changed); + + memcpy(buf + MT_TXD_SIZE, skb->data, skb->len); + + dev_kfree_skb(skb); + + return mt76_mcu_skb_send_msg(&dev->mt76, rskb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + +static int mt7996_driver_own(struct mt7996_dev *dev, u8 band) +{ + mt76_wr(dev, MT_TOP_LPCR_HOST_BAND(band), MT_TOP_LPCR_HOST_DRV_OWN); + if (!mt76_poll_msec(dev, MT_TOP_LPCR_HOST_BAND(band), + MT_TOP_LPCR_HOST_FW_OWN_STAT, 0, 500)) { + dev_err(dev->mt76.dev, "Timeout for driver own\n"); + return -EIO; + } + + /* clear irq when the driver own success */ + mt76_wr(dev, MT_TOP_LPCR_HOST_BAND_IRQ_STAT(band), + MT_TOP_LPCR_HOST_BAND_STAT); + + return 0; +} + +static u32 mt7996_patch_sec_mode(u32 key_info) +{ + u32 sec = u32_get_bits(key_info, MT7996_PATCH_SEC), key = 0; + + if (key_info == GENMASK(31, 0) || sec == MT7996_SEC_MODE_PLAIN) + return 0; + + if (sec == MT7996_SEC_MODE_AES) + key = u32_get_bits(key_info, MT7996_PATCH_AES_KEY); + else + key = u32_get_bits(key_info, MT7996_PATCH_SCRAMBLE_KEY); + + return MT7996_SEC_ENCRYPT | MT7996_SEC_IV | + u32_encode_bits(key, MT7996_SEC_KEY_IDX); +} + +static int mt7996_load_patch(struct mt7996_dev *dev) +{ + const struct mt7996_patch_hdr *hdr; + const struct firmware *fw = NULL; + int i, ret, sem; + + sem = mt76_connac_mcu_patch_sem_ctrl(&dev->mt76, 1); + switch (sem) { + case PATCH_IS_DL: + return 0; + case PATCH_NOT_DL_SEM_SUCCESS: + break; + default: + dev_err(dev->mt76.dev, "Failed to get patch semaphore\n"); + return -EAGAIN; + } + + ret = request_firmware(&fw, MT7996_ROM_PATCH, dev->mt76.dev); + if (ret) + goto out; + + if (!fw || !fw->data || fw->size < sizeof(*hdr)) { + dev_err(dev->mt76.dev, "Invalid firmware\n"); + ret = -EINVAL; + goto out; + } + + hdr = (const struct mt7996_patch_hdr *)(fw->data); + + dev_info(dev->mt76.dev, "HW/SW Version: 0x%x, Build Time: %.16s\n", + be32_to_cpu(hdr->hw_sw_ver), hdr->build_date); + + for (i = 0; i < be32_to_cpu(hdr->desc.n_region); i++) { + struct mt7996_patch_sec *sec; + const u8 *dl; + u32 len, addr, sec_key_idx, mode = DL_MODE_NEED_RSP; + + sec = (struct mt7996_patch_sec *)(fw->data + sizeof(*hdr) + + i * sizeof(*sec)); + if ((be32_to_cpu(sec->type) & PATCH_SEC_TYPE_MASK) != + PATCH_SEC_TYPE_INFO) { + ret = -EINVAL; + goto out; + } + + addr = be32_to_cpu(sec->info.addr); + len = be32_to_cpu(sec->info.len); + sec_key_idx = be32_to_cpu(sec->info.sec_key_idx); + dl = fw->data + be32_to_cpu(sec->offs); + + mode |= mt7996_patch_sec_mode(sec_key_idx); + + ret = mt76_connac_mcu_init_download(&dev->mt76, addr, len, + mode); + if (ret) { + dev_err(dev->mt76.dev, "Download request failed\n"); + goto out; + } + + ret = __mt76_mcu_send_firmware(&dev->mt76, MCU_CMD(FW_SCATTER), + dl, len, 4096); + if (ret) { + dev_err(dev->mt76.dev, "Failed to send patch\n"); + goto out; + } + } + + ret = mt76_connac_mcu_start_patch(&dev->mt76); + if (ret) + dev_err(dev->mt76.dev, "Failed to start patch\n"); + +out: + sem = mt76_connac_mcu_patch_sem_ctrl(&dev->mt76, 0); + switch (sem) { + case PATCH_REL_SEM_SUCCESS: + break; + default: + ret = -EAGAIN; + dev_err(dev->mt76.dev, "Failed to release patch semaphore\n"); + break; + } + release_firmware(fw); + + return ret; +} + +static int +mt7996_mcu_send_ram_firmware(struct mt7996_dev *dev, + const struct mt7996_fw_trailer *hdr, + const u8 *data, bool is_wa) +{ + int i, offset = 0; + u32 override = 0, option = 0; + + for (i = 0; i < hdr->n_region; i++) { + const struct mt7996_fw_region *region; + int err; + u32 len, addr, mode; + + region = (const struct mt7996_fw_region *)((const u8 *)hdr - + (hdr->n_region - i) * sizeof(*region)); + mode = mt76_connac_mcu_gen_dl_mode(&dev->mt76, + region->feature_set, is_wa); + len = le32_to_cpu(region->len); + addr = le32_to_cpu(region->addr); + + if (region->feature_set & FW_FEATURE_OVERRIDE_ADDR) + override = addr; + + err = mt76_connac_mcu_init_download(&dev->mt76, addr, len, + mode); + if (err) { + dev_err(dev->mt76.dev, "Download request failed\n"); + return err; + } + + err = __mt76_mcu_send_firmware(&dev->mt76, MCU_CMD(FW_SCATTER), + data + offset, len, 4096); + if (err) { + dev_err(dev->mt76.dev, "Failed to send firmware.\n"); + return err; + } + + offset += len; + } + + if (override) + option |= FW_START_OVERRIDE; + + if (is_wa) + option |= FW_START_WORKING_PDA_CR4; + + return mt76_connac_mcu_start_firmware(&dev->mt76, override, option); +} + +static int mt7996_load_ram(struct mt7996_dev *dev) +{ + const struct mt7996_fw_trailer *hdr; + const struct firmware *fw; + int ret; + + ret = request_firmware(&fw, MT7996_FIRMWARE_WM, dev->mt76.dev); + if (ret) + return ret; + + if (!fw || !fw->data || fw->size < sizeof(*hdr)) { + dev_err(dev->mt76.dev, "Invalid firmware\n"); + ret = -EINVAL; + goto out; + } + + hdr = (const struct mt7996_fw_trailer *)(fw->data + fw->size - sizeof(*hdr)); + + dev_info(dev->mt76.dev, "WM Firmware Version: %.10s, Build Time: %.15s\n", + hdr->fw_ver, hdr->build_date); + + ret = mt7996_mcu_send_ram_firmware(dev, hdr, fw->data, false); + if (ret) { + dev_err(dev->mt76.dev, "Failed to start WM firmware\n"); + goto out; + } + + release_firmware(fw); + + ret = request_firmware(&fw, MT7996_FIRMWARE_WA, dev->mt76.dev); + if (ret) + return ret; + + if (!fw || !fw->data || fw->size < sizeof(*hdr)) { + dev_err(dev->mt76.dev, "Invalid firmware\n"); + ret = -EINVAL; + goto out; + } + + hdr = (const struct mt7996_fw_trailer *)(fw->data + fw->size - sizeof(*hdr)); + + dev_info(dev->mt76.dev, "WA Firmware Version: %.10s, Build Time: %.15s\n", + hdr->fw_ver, hdr->build_date); + + ret = mt7996_mcu_send_ram_firmware(dev, hdr, fw->data, true); + if (ret) { + dev_err(dev->mt76.dev, "Failed to start WA firmware\n"); + goto out; + } + + snprintf(dev->mt76.hw->wiphy->fw_version, + sizeof(dev->mt76.hw->wiphy->fw_version), + "%.10s-%.15s", hdr->fw_ver, hdr->build_date); + +out: + release_firmware(fw); + + return ret; +} + +static int +mt7996_firmware_state(struct mt7996_dev *dev, bool wa) +{ + u32 state = FIELD_PREP(MT_TOP_MISC_FW_STATE, + wa ? FW_STATE_RDY : FW_STATE_FW_DOWNLOAD); + + if (!mt76_poll_msec(dev, MT_TOP_MISC, MT_TOP_MISC_FW_STATE, + state, 1000)) { + dev_err(dev->mt76.dev, "Timeout for initializing firmware\n"); + return -EIO; + } + return 0; +} + +static int mt7996_load_firmware(struct mt7996_dev *dev) +{ + int ret; + + /* make sure fw is download state */ + if (mt7996_firmware_state(dev, false)) { + /* restart firmware once */ + __mt76_mcu_restart(&dev->mt76); + ret = mt7996_firmware_state(dev, false); + if (ret) { + dev_err(dev->mt76.dev, + "Firmware is not ready for download\n"); + return ret; + } + } + + ret = mt7996_load_patch(dev); + if (ret) + return ret; + + ret = mt7996_load_ram(dev); + if (ret) + return ret; + + ret = mt7996_firmware_state(dev, true); + if (ret) + return ret; + + mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_FWDL], false); + + dev_dbg(dev->mt76.dev, "Firmware init done\n"); + + return 0; +} + +int mt7996_mcu_fw_log_2_host(struct mt7996_dev *dev, u8 type, u8 ctrl) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + u8 ctrl; + u8 interval; + u8 _rsv2[2]; + } __packed data = { + .tag = cpu_to_le16(UNI_WSYS_CONFIG_FW_LOG_CTRL), + .len = cpu_to_le16(sizeof(data) - 4), + .ctrl = ctrl, + }; + + if (type == MCU_FW_LOG_WA) + return mt76_mcu_send_msg(&dev->mt76, MCU_WA_UNI_CMD(WSYS_CONFIG), + &data, sizeof(data), true); + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(WSYS_CONFIG), &data, + sizeof(data), true); +} + +int mt7996_mcu_fw_dbg_ctrl(struct mt7996_dev *dev, u32 module, u8 level) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + __le32 module_idx; + u8 level; + u8 _rsv2[3]; + } data = { + .tag = cpu_to_le16(UNI_WSYS_CONFIG_FW_DBG_CTRL), + .len = cpu_to_le16(sizeof(data) - 4), + .module_idx = cpu_to_le32(module), + .level = level, + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(WSYS_CONFIG), &data, + sizeof(data), false); +} + +static int mt7996_mcu_set_mwds(struct mt7996_dev *dev, bool enabled) +{ + struct { + u8 enable; + u8 _rsv[3]; + } __packed req = { + .enable = enabled + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WA_EXT_CMD(MWDS_SUPPORT), &req, + sizeof(req), false); +} + +static void mt7996_add_rx_airtime_tlv(struct sk_buff *skb, u8 band_idx) +{ + struct vow_rx_airtime *req; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_VOW_RX_AT_AIRTIME_CLR_EN, sizeof(*req)); + req = (struct vow_rx_airtime *)tlv; + req->enable = true; + req->band = band_idx; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_VOW_RX_AT_AIRTIME_EN, sizeof(*req)); + req = (struct vow_rx_airtime *)tlv; + req->enable = true; + req->band = band_idx; +} + +static int +mt7996_mcu_init_rx_airtime(struct mt7996_dev *dev) +{ + struct uni_header hdr = {}; + struct sk_buff *skb; + int len, num; + + num = 2 + 2 * (dev->dbdc_support + dev->tbtc_support); + len = sizeof(hdr) + num * sizeof(struct vow_rx_airtime); + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &hdr, sizeof(hdr)); + + mt7996_add_rx_airtime_tlv(skb, dev->mt76.phy.band_idx); + + if (dev->dbdc_support) + mt7996_add_rx_airtime_tlv(skb, MT_BAND1); + + if (dev->tbtc_support) + mt7996_add_rx_airtime_tlv(skb, MT_BAND2); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(VOW), true); +} + +static int +mt7996_mcu_restart(struct mt76_dev *dev) +{ + struct { + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + u8 power_mode; + u8 __rsv2[3]; + } __packed req = { + .tag = cpu_to_le16(UNI_POWER_OFF), + .len = cpu_to_le16(sizeof(req) - 4), + .power_mode = 1, + }; + + return mt76_mcu_send_msg(dev, MCU_WM_UNI_CMD(POWER_CREL), &req, + sizeof(req), false); +} + +int mt7996_mcu_init(struct mt7996_dev *dev) +{ + static const struct mt76_mcu_ops mt7996_mcu_ops = { + .headroom = sizeof(struct mt76_connac2_mcu_txd), /* reuse */ + .mcu_skb_send_msg = mt7996_mcu_send_message, + .mcu_parse_response = mt7996_mcu_parse_response, + .mcu_restart = mt7996_mcu_restart, + }; + int ret; + + dev->mt76.mcu_ops = &mt7996_mcu_ops; + + /* force firmware operation mode into normal state, + * which should be set before firmware download stage. + */ + mt76_wr(dev, MT_SWDEF_MODE, MT_SWDEF_NORMAL_MODE); + + ret = mt7996_driver_own(dev, 0); + if (ret) + return ret; + /* set driver own for band1 when two hif exist */ + if (dev->hif2) { + ret = mt7996_driver_own(dev, 1); + if (ret) + return ret; + } + + ret = mt7996_load_firmware(dev); + if (ret) + return ret; + + set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); + ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WM, 0); + if (ret) + return ret; + + ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WA, 0); + if (ret) + return ret; + + ret = mt7996_mcu_set_mwds(dev, 1); + if (ret) + return ret; + + ret = mt7996_mcu_init_rx_airtime(dev); + if (ret) + return ret; + + return mt7996_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(SET), + MCU_WA_PARAM_RED, 0, 0); +} + +void mt7996_mcu_exit(struct mt7996_dev *dev) +{ + __mt76_mcu_restart(&dev->mt76); + if (mt7996_firmware_state(dev, false)) { + dev_err(dev->mt76.dev, "Failed to exit mcu\n"); + return; + } + + mt76_wr(dev, MT_TOP_LPCR_HOST_BAND(0), MT_TOP_LPCR_HOST_FW_OWN); + if (dev->hif2) + mt76_wr(dev, MT_TOP_LPCR_HOST_BAND(1), + MT_TOP_LPCR_HOST_FW_OWN); + skb_queue_purge(&dev->mt76.mcu.res_q); +} + +int mt7996_mcu_set_hdr_trans(struct mt7996_dev *dev, bool hdr_trans) +{ + struct { + u8 __rsv[4]; + } __packed hdr; + struct hdr_trans_blacklist *req_blacklist; + struct hdr_trans_en *req_en; + struct sk_buff *skb; + struct tlv *tlv; + int len = MT7996_HDR_TRANS_MAX_SIZE + sizeof(hdr); + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &hdr, sizeof(hdr)); + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_HDR_TRANS_EN, sizeof(*req_en)); + req_en = (struct hdr_trans_en *)tlv; + req_en->enable = hdr_trans; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_HDR_TRANS_VLAN, + sizeof(struct hdr_trans_vlan)); + + if (hdr_trans) { + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_HDR_TRANS_BLACKLIST, + sizeof(*req_blacklist)); + req_blacklist = (struct hdr_trans_blacklist *)tlv; + req_blacklist->enable = 1; + req_blacklist->type = cpu_to_le16(ETH_P_PAE); + } + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(RX_HDR_TRANS), true); +} + +int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif) +{ +#define MCU_EDCA_AC_PARAM 0 +#define WMM_AIFS_SET BIT(0) +#define WMM_CW_MIN_SET BIT(1) +#define WMM_CW_MAX_SET BIT(2) +#define WMM_TXOP_SET BIT(3) +#define WMM_PARAM_SET (WMM_AIFS_SET | WMM_CW_MIN_SET | \ + WMM_CW_MAX_SET | WMM_TXOP_SET) + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct { + u8 bss_idx; + u8 __rsv[3]; + } __packed hdr = { + .bss_idx = mvif->mt76.idx, + }; + struct sk_buff *skb; + int len = sizeof(hdr) + IEEE80211_NUM_ACS * sizeof(struct edca); + int ac; + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &hdr, sizeof(hdr)); + + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { + struct ieee80211_tx_queue_params *q = &mvif->queue_params[ac]; + struct edca *e; + struct tlv *tlv; + + tlv = mt7996_mcu_add_uni_tlv(skb, MCU_EDCA_AC_PARAM, sizeof(*e)); + + e = (struct edca *)tlv; + e->set = WMM_PARAM_SET; + e->queue = ac + mvif->mt76.wmm_idx * MT7996_MAX_WMM_SETS; + e->aifs = q->aifs; + e->txop = cpu_to_le16(q->txop); + + if (q->cw_min) + e->cw_min = fls(q->cw_min); + else + e->cw_min = 5; + + if (q->cw_max) + e->cw_max = fls(q->cw_max); + else + e->cw_max = 10; + } + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(EDCA_UPDATE), true); +} + +int mt7996_mcu_set_fcc5_lpn(struct mt7996_dev *dev, int val) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + __le32 ctrl; + __le16 min_lpn; + u8 rsv[2]; + } __packed req = { + .tag = cpu_to_le16(UNI_RDD_CTRL_SET_TH), + .len = cpu_to_le16(sizeof(req) - 4), + + .ctrl = cpu_to_le32(0x1), + .min_lpn = cpu_to_le16(val), + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RDD_CTRL), + &req, sizeof(req), true); +} + +int mt7996_mcu_set_pulse_th(struct mt7996_dev *dev, + const struct mt7996_dfs_pulse *pulse) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + __le32 ctrl; + + __le32 max_width; /* us */ + __le32 max_pwr; /* dbm */ + __le32 min_pwr; /* dbm */ + __le32 min_stgr_pri; /* us */ + __le32 max_stgr_pri; /* us */ + __le32 min_cr_pri; /* us */ + __le32 max_cr_pri; /* us */ + } __packed req = { + .tag = cpu_to_le16(UNI_RDD_CTRL_SET_TH), + .len = cpu_to_le16(sizeof(req) - 4), + + .ctrl = cpu_to_le32(0x3), + +#define __req_field(field) .field = cpu_to_le32(pulse->field) + __req_field(max_width), + __req_field(max_pwr), + __req_field(min_pwr), + __req_field(min_stgr_pri), + __req_field(max_stgr_pri), + __req_field(min_cr_pri), + __req_field(max_cr_pri), +#undef __req_field + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RDD_CTRL), + &req, sizeof(req), true); +} + +int mt7996_mcu_set_radar_th(struct mt7996_dev *dev, int index, + const struct mt7996_dfs_pattern *pattern) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + __le32 ctrl; + __le16 radar_type; + + u8 enb; + u8 stgr; + u8 min_crpn; + u8 max_crpn; + u8 min_crpr; + u8 min_pw; + __le32 min_pri; + __le32 max_pri; + u8 max_pw; + u8 min_crbn; + u8 max_crbn; + u8 min_stgpn; + u8 max_stgpn; + u8 min_stgpr; + u8 rsv[2]; + __le32 min_stgpr_diff; + } __packed req = { + .tag = cpu_to_le16(UNI_RDD_CTRL_SET_TH), + .len = cpu_to_le16(sizeof(req) - 4), + + .ctrl = cpu_to_le32(0x2), + .radar_type = cpu_to_le16(index), + +#define __req_field_u8(field) .field = pattern->field +#define __req_field_u32(field) .field = cpu_to_le32(pattern->field) + __req_field_u8(enb), + __req_field_u8(stgr), + __req_field_u8(min_crpn), + __req_field_u8(max_crpn), + __req_field_u8(min_crpr), + __req_field_u8(min_pw), + __req_field_u32(min_pri), + __req_field_u32(max_pri), + __req_field_u8(max_pw), + __req_field_u8(min_crbn), + __req_field_u8(max_crbn), + __req_field_u8(min_stgpn), + __req_field_u8(max_stgpn), + __req_field_u8(min_stgpr), + __req_field_u32(min_stgpr_diff), +#undef __req_field_u8 +#undef __req_field_u32 + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RDD_CTRL), + &req, sizeof(req), true); +} + +static int +mt7996_mcu_background_chain_ctrl(struct mt7996_phy *phy, + struct cfg80211_chan_def *chandef, + int cmd) +{ + struct mt7996_dev *dev = phy->dev; + struct mt76_phy *mphy = phy->mt76; + struct ieee80211_channel *chan = mphy->chandef.chan; + int freq = mphy->chandef.center_freq1; + struct mt7996_mcu_background_chain_ctrl req = { + .tag = cpu_to_le16(0), + .len = cpu_to_le16(sizeof(req) - 4), + .monitor_scan_type = 2, /* simple rx */ + }; + + if (!chandef && cmd != CH_SWITCH_BACKGROUND_SCAN_STOP) + return -EINVAL; + + if (!cfg80211_chandef_valid(&mphy->chandef)) + return -EINVAL; + + switch (cmd) { + case CH_SWITCH_BACKGROUND_SCAN_START: { + req.chan = chan->hw_value; + req.central_chan = ieee80211_frequency_to_channel(freq); + req.bw = mt76_connac_chan_bw(&mphy->chandef); + req.monitor_chan = chandef->chan->hw_value; + req.monitor_central_chan = + ieee80211_frequency_to_channel(chandef->center_freq1); + req.monitor_bw = mt76_connac_chan_bw(chandef); + req.band_idx = phy->mt76->band_idx; + req.scan_mode = 1; + break; + } + case CH_SWITCH_BACKGROUND_SCAN_RUNNING: + req.monitor_chan = chandef->chan->hw_value; + req.monitor_central_chan = + ieee80211_frequency_to_channel(chandef->center_freq1); + req.band_idx = phy->mt76->band_idx; + req.scan_mode = 2; + break; + case CH_SWITCH_BACKGROUND_SCAN_STOP: + req.chan = chan->hw_value; + req.central_chan = ieee80211_frequency_to_channel(freq); + req.bw = mt76_connac_chan_bw(&mphy->chandef); + req.tx_stream = hweight8(mphy->antenna_mask); + req.rx_stream = mphy->antenna_mask; + break; + default: + return -EINVAL; + } + req.band = chandef ? chandef->chan->band == NL80211_BAND_5GHZ : 1; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(OFFCH_SCAN_CTRL), + &req, sizeof(req), false); +} + +int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, + struct cfg80211_chan_def *chandef) +{ + struct mt7996_dev *dev = phy->dev; + int err, region; + + if (!chandef) { /* disable offchain */ + err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, MT_RX_SEL2, + 0, 0); + if (err) + return err; + + return mt7996_mcu_background_chain_ctrl(phy, NULL, + CH_SWITCH_BACKGROUND_SCAN_STOP); + } + + err = mt7996_mcu_background_chain_ctrl(phy, chandef, + CH_SWITCH_BACKGROUND_SCAN_START); + if (err) + return err; + + switch (dev->mt76.region) { + case NL80211_DFS_ETSI: + region = 0; + break; + case NL80211_DFS_JP: + region = 2; + break; + case NL80211_DFS_FCC: + default: + region = 1; + break; + } + + return mt7996_mcu_rdd_cmd(dev, RDD_START, MT_RX_SEL2, + 0, region); +} + +int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag) +{ + static const u8 ch_band[] = { + [NL80211_BAND_2GHZ] = 0, + [NL80211_BAND_5GHZ] = 1, + [NL80211_BAND_6GHZ] = 2, + }; + struct mt7996_dev *dev = phy->dev; + struct cfg80211_chan_def *chandef = &phy->mt76->chandef; + int freq1 = chandef->center_freq1; + u8 band_idx = phy->mt76->band_idx; + struct { + /* fixed field */ + u8 __rsv[4]; + + __le16 tag; + __le16 len; + u8 control_ch; + u8 center_ch; + u8 bw; + u8 tx_path_num; + u8 rx_path; /* mask or num */ + u8 switch_reason; + u8 band_idx; + u8 center_ch2; /* for 80+80 only */ + __le16 cac_case; + u8 channel_band; + u8 rsv0; + __le32 outband_freq; + u8 txpower_drop; + u8 ap_bw; + u8 ap_center_ch; + u8 rsv1[53]; + } __packed req = { + .tag = cpu_to_le16(tag), + .len = cpu_to_le16(sizeof(req) - 4), + .control_ch = chandef->chan->hw_value, + .center_ch = ieee80211_frequency_to_channel(freq1), + .bw = mt76_connac_chan_bw(chandef), + .tx_path_num = hweight16(phy->mt76->chainmask), + .rx_path = phy->mt76->chainmask >> dev->chainshift[band_idx], + .band_idx = band_idx, + .channel_band = ch_band[chandef->chan->band], + }; + + if (tag == UNI_CHANNEL_RX_PATH || + dev->mt76.hw->conf.flags & IEEE80211_CONF_MONITOR) + req.switch_reason = CH_SWITCH_NORMAL; + else if (phy->mt76->hw->conf.flags & IEEE80211_CONF_OFFCHANNEL) + req.switch_reason = CH_SWITCH_SCAN_BYPASS_DPD; + else if (!cfg80211_reg_can_beacon(phy->mt76->hw->wiphy, chandef, + NL80211_IFTYPE_AP)) + req.switch_reason = CH_SWITCH_DFS; + else + req.switch_reason = CH_SWITCH_NORMAL; + + if (tag == UNI_CHANNEL_SWITCH) + req.rx_path = hweight8(req.rx_path); + + if (chandef->width == NL80211_CHAN_WIDTH_80P80) { + int freq2 = chandef->center_freq2; + + req.center_ch2 = ieee80211_frequency_to_channel(freq2); + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_WMWA_UNI_CMD(CHANNEL_SWITCH), + &req, sizeof(req), true); +} + +static int mt7996_mcu_set_eeprom_flash(struct mt7996_dev *dev) +{ +#define MAX_PAGE_IDX_MASK GENMASK(7, 5) +#define PAGE_IDX_MASK GENMASK(4, 2) +#define PER_PAGE_SIZE 0x400 + struct mt7996_mcu_eeprom req = { + .tag = cpu_to_le16(UNI_EFUSE_BUFFER_MODE), + .buffer_mode = EE_MODE_BUFFER + }; + u16 eeprom_size = MT7996_EEPROM_SIZE; + u8 total = DIV_ROUND_UP(eeprom_size, PER_PAGE_SIZE); + u8 *eep = (u8 *)dev->mt76.eeprom.data; + int eep_len, i; + + for (i = 0; i < total; i++, eep += eep_len) { + struct sk_buff *skb; + int ret, msg_len; + + if (i == total - 1 && !!(eeprom_size % PER_PAGE_SIZE)) + eep_len = eeprom_size % PER_PAGE_SIZE; + else + eep_len = PER_PAGE_SIZE; + + msg_len = sizeof(req) + eep_len; + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, msg_len); + if (!skb) + return -ENOMEM; + + req.len = cpu_to_le16(msg_len - 4); + req.format = FIELD_PREP(MAX_PAGE_IDX_MASK, total - 1) | + FIELD_PREP(PAGE_IDX_MASK, i) | EE_FORMAT_WHOLE; + req.buf_len = cpu_to_le16(eep_len); + + skb_put_data(skb, &req, sizeof(req)); + skb_put_data(skb, eep, eep_len); + + ret = mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(EFUSE_CTRL), true); + if (ret) + return ret; + } + + return 0; +} + +int mt7996_mcu_set_eeprom(struct mt7996_dev *dev) +{ + struct mt7996_mcu_eeprom req = { + .tag = cpu_to_le16(UNI_EFUSE_BUFFER_MODE), + .len = cpu_to_le16(sizeof(req) - 4), + .buffer_mode = EE_MODE_EFUSE, + .format = EE_FORMAT_WHOLE + }; + + if (dev->flash_mode) + return mt7996_mcu_set_eeprom_flash(dev); + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(EFUSE_CTRL), + &req, sizeof(req), true); +} + +int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + __le32 addr; + __le32 valid; + u8 data[16]; + } __packed req = { + .tag = cpu_to_le16(UNI_EFUSE_ACCESS), + .len = cpu_to_le16(sizeof(req) - 4), + .addr = cpu_to_le32(round_down(offset, + MT7996_EEPROM_BLOCK_SIZE)), + }; + struct sk_buff *skb; + bool valid; + int ret; + + ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_WM_UNI_CMD_QUERY(EFUSE_CTRL), &req, + sizeof(req), true, &skb); + if (ret) + return ret; + + valid = le32_to_cpu(*(__le32 *)(skb->data + 16)); + if (valid) { + u32 addr = le32_to_cpu(*(__le32 *)(skb->data + 12)); + u8 *buf = (u8 *)dev->mt76.eeprom.data + addr; + + skb_pull(skb, 64); + memcpy(buf, skb->data, MT7996_EEPROM_BLOCK_SIZE); + } + + dev_kfree_skb(skb); + + return 0; +} + +int mt7996_mcu_get_eeprom_free_block(struct mt7996_dev *dev, u8 *block_num) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + u8 num; + u8 version; + u8 die_idx; + u8 _rsv2; + } __packed req = { + .tag = cpu_to_le16(UNI_EFUSE_FREE_BLOCK), + .len = cpu_to_le16(sizeof(req) - 4), + .version = 2, + }; + struct sk_buff *skb; + int ret; + + ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_WM_UNI_CMD_QUERY(EFUSE_CTRL), &req, + sizeof(req), true, &skb); + if (ret) + return ret; + + *block_num = *(u8 *)(skb->data + 8); + dev_kfree_skb(skb); + + return 0; +} + +int mt7996_mcu_get_chan_mib_info(struct mt7996_phy *phy, bool chan_switch) +{ + struct { + struct { + u8 band; + u8 __rsv[3]; + } hdr; + struct { + __le16 tag; + __le16 len; + __le32 offs; + } data[4]; + } __packed req = { + .hdr.band = phy->mt76->band_idx, + }; + /* strict order */ + static const u32 offs[] = { + UNI_MIB_TX_TIME, + UNI_MIB_RX_TIME, + UNI_MIB_OBSS_AIRTIME, + UNI_MIB_NON_WIFI_TIME, + }; + struct mt76_channel_state *state = phy->mt76->chan_state; + struct mt76_channel_state *state_ts = &phy->state_ts; + struct mt7996_dev *dev = phy->dev; + struct mt7996_mcu_mib *res; + struct sk_buff *skb; + int i, ret; + + for (i = 0; i < 4; i++) { + req.data[i].tag = cpu_to_le16(UNI_CMD_MIB_DATA); + req.data[i].len = cpu_to_le16(sizeof(req.data[i])); + req.data[i].offs = cpu_to_le32(offs[i]); + } + + ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_WM_UNI_CMD_QUERY(GET_MIB_INFO), + &req, sizeof(req), true, &skb); + if (ret) + return ret; + + skb_pull(skb, sizeof(req.hdr)); + + res = (struct mt7996_mcu_mib *)(skb->data); + + if (chan_switch) + goto out; + +#define __res_u64(s) le64_to_cpu(res[s].data) + state->cc_tx += __res_u64(1) - state_ts->cc_tx; + state->cc_bss_rx += __res_u64(2) - state_ts->cc_bss_rx; + state->cc_rx += __res_u64(2) + __res_u64(3) - state_ts->cc_rx; + state->cc_busy += __res_u64(0) + __res_u64(1) + __res_u64(2) + __res_u64(3) - + state_ts->cc_busy; + +out: + state_ts->cc_tx = __res_u64(1); + state_ts->cc_bss_rx = __res_u64(2); + state_ts->cc_rx = __res_u64(2) + __res_u64(3); + state_ts->cc_busy = __res_u64(0) + __res_u64(1) + __res_u64(2) + __res_u64(3); +#undef __res_u64 + + dev_kfree_skb(skb); + + return 0; +} + +int mt7996_mcu_set_ser(struct mt7996_dev *dev, u8 action, u8 val, u8 band) +{ + struct { + u8 rsv[4]; + + __le16 tag; + __le16 len; + + union { + struct { + __le32 mask; + } __packed set; + + struct { + u8 method; + u8 band; + u8 rsv2[2]; + } __packed trigger; + }; + } __packed req = { + .tag = cpu_to_le16(action), + .len = cpu_to_le16(sizeof(req) - 4), + }; + + switch (action) { + case UNI_CMD_SER_SET: + req.set.mask = cpu_to_le32(val); + break; + case UNI_CMD_SER_TRIGGER: + req.trigger.method = val; + req.trigger.band = band; + break; + default: + return -EINVAL; + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(SER), + &req, sizeof(req), false); +} + +int mt7996_mcu_set_txbf(struct mt7996_dev *dev, u8 action) +{ +#define MT7996_BF_MAX_SIZE sizeof(union bf_tag_tlv) +#define BF_PROCESSING 4 + struct uni_header hdr; + struct sk_buff *skb; + struct tlv *tlv; + int len = sizeof(hdr) + MT7996_BF_MAX_SIZE; + + memset(&hdr, 0, sizeof(hdr)); + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &hdr, sizeof(hdr)); + + switch (action) { + case BF_SOUNDING_ON: { + struct bf_sounding_on *req_snd_on; + + tlv = mt7996_mcu_add_uni_tlv(skb, action, sizeof(*req_snd_on)); + req_snd_on = (struct bf_sounding_on *)tlv; + req_snd_on->snd_mode = BF_PROCESSING; + break; + } + case BF_HW_EN_UPDATE: { + struct bf_hw_en_status_update *req_hw_en; + + tlv = mt7996_mcu_add_uni_tlv(skb, action, sizeof(*req_hw_en)); + req_hw_en = (struct bf_hw_en_status_update *)tlv; + req_hw_en->ebf = true; + req_hw_en->ibf = dev->ibf; + break; + } + case BF_MOD_EN_CTRL: { + struct bf_mod_en_ctrl *req_mod_en; + + tlv = mt7996_mcu_add_uni_tlv(skb, action, sizeof(*req_mod_en)); + req_mod_en = (struct bf_mod_en_ctrl *)tlv; + req_mod_en->bf_num = 2; + req_mod_en->bf_bitmap = GENMASK(0, 0); + break; + } + default: + return -EINVAL; + } + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_WM_UNI_CMD(BF), true); +} + +static int +mt7996_mcu_enable_obss_spr(struct mt7996_phy *phy, u16 action, u8 val) +{ + struct mt7996_dev *dev = phy->dev; + struct { + u8 band_idx; + u8 __rsv[3]; + + __le16 tag; + __le16 len; + + __le32 val; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(action), + .len = cpu_to_le16(sizeof(req) - 4), + .val = cpu_to_le32(val), + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(SR), + &req, sizeof(req), true); +} + +static int +mt7996_mcu_set_obss_spr_pd(struct mt7996_phy *phy, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct mt7996_dev *dev = phy->dev; + u8 max_th = 82, non_srg_max_th = 62; + struct { + u8 band_idx; + u8 __rsv[3]; + + __le16 tag; + __le16 len; + + u8 pd_th_non_srg; + u8 pd_th_srg; + u8 period_offs; + u8 rcpi_src; + __le16 obss_pd_min; + __le16 obss_pd_min_srg; + u8 resp_txpwr_mode; + u8 txpwr_restrict_mode; + u8 txpwr_ref; + u8 __rsv2[3]; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_CMD_SR_SET_PARAM), + .len = cpu_to_le16(sizeof(req) - 4), + .obss_pd_min = cpu_to_le16(max_th), + .obss_pd_min_srg = cpu_to_le16(max_th), + .txpwr_restrict_mode = 2, + .txpwr_ref = 21 + }; + int ret; + + /* disable firmware dynamical PD asjustment */ + ret = mt7996_mcu_enable_obss_spr(phy, UNI_CMD_SR_ENABLE_DPD, false); + if (ret) + return ret; + + if (he_obss_pd->sr_ctrl & + IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED) + req.pd_th_non_srg = max_th; + else if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT) + req.pd_th_non_srg = max_th - he_obss_pd->non_srg_max_offset; + else + req.pd_th_non_srg = non_srg_max_th; + + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT) + req.pd_th_srg = max_th - he_obss_pd->max_offset; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(SR), + &req, sizeof(req), true); +} + +static int +mt7996_mcu_set_obss_spr_siga(struct mt7996_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_dev *dev = phy->dev; + u8 omac = mvif->mt76.omac_idx; + struct { + u8 band_idx; + u8 __rsv[3]; + + __le16 tag; + __le16 len; + + u8 omac; + u8 __rsv2[3]; + u8 flag[20]; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_CMD_SR_SET_SIGA), + .len = cpu_to_le16(sizeof(req) - 4), + .omac = omac > HW_BSSID_MAX ? omac - 12 : omac, + }; + int ret; + + if (he_obss_pd->sr_ctrl & IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED) + req.flag[req.omac] = 0xf; + else + return 0; + + /* switch to normal AP mode */ + ret = mt7996_mcu_enable_obss_spr(phy, UNI_CMD_SR_ENABLE_MODE, 0); + if (ret) + return ret; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(SR), + &req, sizeof(req), true); +} + +static int +mt7996_mcu_set_obss_spr_bitmap(struct mt7996_phy *phy, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + struct mt7996_dev *dev = phy->dev; + struct { + u8 band_idx; + u8 __rsv[3]; + + __le16 tag; + __le16 len; + + __le32 color_l[2]; + __le32 color_h[2]; + __le32 bssid_l[2]; + __le32 bssid_h[2]; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_CMD_SR_SET_SRG_BITMAP), + .len = cpu_to_le16(sizeof(req) - 4), + }; + u32 bitmap; + + memcpy(&bitmap, he_obss_pd->bss_color_bitmap, sizeof(bitmap)); + req.color_l[req.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->bss_color_bitmap + 4, sizeof(bitmap)); + req.color_h[req.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->partial_bssid_bitmap, sizeof(bitmap)); + req.bssid_l[req.band_idx] = cpu_to_le32(bitmap); + + memcpy(&bitmap, he_obss_pd->partial_bssid_bitmap + 4, sizeof(bitmap)); + req.bssid_h[req.band_idx] = cpu_to_le32(bitmap); + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(SR), &req, + sizeof(req), true); +} + +int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd) +{ + int ret; + + /* enable firmware scene detection algorithms */ + ret = mt7996_mcu_enable_obss_spr(phy, UNI_CMD_SR_ENABLE_SD, + sr_scene_detect); + if (ret) + return ret; + + /* firmware dynamically adjusts PD threshold so skip manual control */ + if (sr_scene_detect && !he_obss_pd->enable) + return 0; + + /* enable spatial reuse */ + ret = mt7996_mcu_enable_obss_spr(phy, UNI_CMD_SR_ENABLE, + he_obss_pd->enable); + if (ret) + return ret; + + if (sr_scene_detect || !he_obss_pd->enable) + return 0; + + ret = mt7996_mcu_enable_obss_spr(phy, UNI_CMD_SR_ENABLE_TX, true); + if (ret) + return ret; + + /* set SRG/non-SRG OBSS PD threshold */ + ret = mt7996_mcu_set_obss_spr_pd(phy, he_obss_pd); + if (ret) + return ret; + + /* Set SR prohibit */ + ret = mt7996_mcu_set_obss_spr_siga(phy, vif, he_obss_pd); + if (ret) + return ret; + + /* set SRG BSS color/BSSID bitmap */ + return mt7996_mcu_set_obss_spr_bitmap(phy, he_obss_pd); +} + +int mt7996_mcu_update_bss_color(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct cfg80211_he_bss_color *he_bss_color) +{ + int len = sizeof(struct bss_req_hdr) + sizeof(struct bss_color_tlv); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct bss_color_tlv *bss_color; + struct sk_buff *skb; + struct tlv *tlv; + + skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, len); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_BSS_COLOR, + sizeof(*bss_color)); + bss_color = (struct bss_color_tlv *)tlv; + bss_color->enable = he_bss_color->enabled; + bss_color->color = he_bss_color->color; + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + +#define TWT_AGRT_TRIGGER BIT(0) +#define TWT_AGRT_ANNOUNCE BIT(1) +#define TWT_AGRT_PROTECT BIT(2) + +int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev, + struct mt7996_vif *mvif, + struct mt7996_twt_flow *flow, + int cmd) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + u8 tbl_idx; + u8 cmd; + u8 own_mac_idx; + u8 flowid; /* 0xff for group id */ + __le16 peer_id; /* specify the peer_id (msb=0) + * or group_id (msb=1) + */ + u8 duration; /* 256 us */ + u8 bss_idx; + __le64 start_tsf; + __le16 mantissa; + u8 exponent; + u8 is_ap; + u8 agrt_params; + u8 __rsv2[135]; + } __packed req = { + .tag = cpu_to_le16(UNI_CMD_TWT_ARGT_UPDATE), + .len = cpu_to_le16(sizeof(req) - 4), + .tbl_idx = flow->table_id, + .cmd = cmd, + .own_mac_idx = mvif->mt76.omac_idx, + .flowid = flow->id, + .peer_id = cpu_to_le16(flow->wcid), + .duration = flow->duration, + .bss_idx = mvif->mt76.idx, + .start_tsf = cpu_to_le64(flow->tsf), + .mantissa = flow->mantissa, + .exponent = flow->exp, + .is_ap = true, + }; + + if (flow->protection) + req.agrt_params |= TWT_AGRT_PROTECT; + if (!flow->flowtype) + req.agrt_params |= TWT_AGRT_ANNOUNCE; + if (flow->trigger) + req.agrt_params |= TWT_AGRT_TRIGGER; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(TWT), + &req, sizeof(req), true); +} + +void mt7996_mcu_set_pm(void *priv, u8 *mac, struct ieee80211_vif *vif) +{ +#define EXIT_PM_STATE 0 +#define ENTER_PM_STATE 1 + struct ieee80211_hw *hw = priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct bss_power_save *ps; + struct sk_buff *skb; + struct tlv *tlv; + bool running = test_bit(MT76_STATE_RUNNING, &phy->mt76->state); + + skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, + MT7996_BSS_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return; + + tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_PS, sizeof(*ps)); + ps = (struct bss_power_save *)tlv; + ps->profile = running ? EXIT_PM_STATE : ENTER_PM_STATE; + + mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + +int mt7996_mcu_set_rts_thresh(struct mt7996_phy *phy, u32 val) +{ + struct { + u8 band_idx; + u8 _rsv[3]; + + __le16 tag; + __le16 len; + __le32 len_thresh; + __le32 pkt_thresh; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_BAND_CONFIG_RTS_THRESHOLD), + .len = cpu_to_le16(sizeof(req) - 4), + .len_thresh = cpu_to_le32(val), + .pkt_thresh = cpu_to_le32(0x2), + }; + + return mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(BAND_CONFIG), + &req, sizeof(req), true); +} + +int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable) +{ + struct { + u8 band_idx; + u8 _rsv[3]; + + __le16 tag; + __le16 len; + u8 enable; + u8 _rsv2[3]; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_BAND_CONFIG_RADIO_ENABLE), + .len = cpu_to_le16(sizeof(req) - 4), + .enable = enable, + }; + + return mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(BAND_CONFIG), + &req, sizeof(req), true); +} + +int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index, + u8 rx_sel, u8 val) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + u8 ctrl; + u8 rdd_idx; + u8 rdd_rx_sel; + u8 val; + u8 rsv[4]; + } __packed req = { + .tag = cpu_to_le16(UNI_RDD_CTRL_PARM), + .len = cpu_to_le16(sizeof(req) - 4), + .ctrl = cmd, + .rdd_idx = index, + .rdd_rx_sel = rx_sel, + .val = val, + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RDD_CTRL), + &req, sizeof(req), true); +} + +int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta; + struct sk_buff *skb; + + msta = sta ? (struct mt7996_sta *)sta->drv_priv : &mvif->sta; + + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->mt76, + &msta->wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + /* starec hdr trans */ + mt7996_mcu_sta_hdr_trans_tlv(dev, skb, vif, sta); + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); +} + +int mt7996_mcu_rf_regval(struct mt7996_dev *dev, u32 regidx, u32 *val, bool set) +{ + struct { + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + __le16 idx; + u8 __rsv2[2]; + __le32 ofs; + __le32 data; + } __packed *res, req = { + .tag = cpu_to_le16(UNI_CMD_ACCESS_RF_REG_BASIC), + .len = cpu_to_le16(sizeof(req) - 4), + + .idx = cpu_to_le16(u32_get_bits(regidx, GENMASK(31, 24))), + .ofs = cpu_to_le32(u32_get_bits(regidx, GENMASK(23, 0))), + .data = set ? cpu_to_le32(*val) : 0, + }; + struct sk_buff *skb; + int ret; + + if (set) + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(REG_ACCESS), + &req, sizeof(req), true); + + ret = mt76_mcu_send_and_get_msg(&dev->mt76, + MCU_WM_UNI_CMD_QUERY(REG_ACCESS), + &req, sizeof(req), true, &skb); + if (ret) + return ret; + + res = (void *)skb->data; + *val = le32_to_cpu(res->data); + dev_kfree_skb(skb); + + return 0; +} + +int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val) +{ + struct { + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + + union { + struct { + u8 type; + u8 __rsv2[3]; + } __packed platform_type; + struct { + u8 type; + u8 dest; + u8 __rsv2[2]; + } __packed bypass_mode; + struct { + u8 path; + u8 __rsv2[3]; + } __packed txfree_path; + }; + } __packed req = { + .tag = cpu_to_le16(tag), + .len = cpu_to_le16(sizeof(req) - 4), + }; + + switch (tag) { + case UNI_RRO_SET_PLATFORM_TYPE: + req.platform_type.type = val; + break; + case UNI_RRO_SET_BYPASS_MODE: + req.bypass_mode.type = val; + break; + case UNI_RRO_SET_TXFREE_PATH: + req.txfree_path.path = val; + break; + default: + return -EINVAL; + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RRO), &req, + sizeof(req), true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h new file mode 100644 index 000000000000..6084b2337598 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -0,0 +1,669 @@ +/* SPDX-License-Identifier: ISC */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef __MT7996_MCU_H +#define __MT7996_MCU_H + +#include "../mt76_connac_mcu.h" + +struct mt7996_mcu_rxd { + __le32 rxd[8]; + + __le16 len; + __le16 pkt_type_id; + + u8 eid; + u8 seq; + u8 option; + u8 __rsv; + + u8 ext_eid; + u8 __rsv1[2]; + u8 s2d_index; +}; + +struct mt7996_mcu_uni_event { + u8 cid; + u8 __rsv[3]; + __le32 status; /* 0: success, others: fail */ +} __packed; + +struct mt7996_mcu_csa_notify { + struct mt7996_mcu_rxd rxd; + + u8 omac_idx; + u8 csa_count; + u8 band_idx; + u8 rsv; +} __packed; + +struct mt7996_mcu_rdd_report { + struct mt7996_mcu_rxd rxd; + + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + + u8 band_idx; + u8 long_detected; + u8 constant_prf_detected; + u8 staggered_prf_detected; + u8 radar_type_idx; + u8 periodic_pulse_num; + u8 long_pulse_num; + u8 hw_pulse_num; + + u8 out_lpn; + u8 out_spn; + u8 out_crpn; + u8 out_crpw; + u8 out_crbn; + u8 out_stgpn; + u8 out_stgpw; + + u8 __rsv2; + + __le32 out_pri_const; + __le32 out_pri_stg[3]; + __le32 out_pri_stg_dmin; + + struct { + __le32 start; + __le16 pulse_width; + __le16 pulse_power; + u8 mdrdy_flag; + u8 rsv[3]; + } long_pulse[32]; + + struct { + __le32 start; + __le16 pulse_width; + __le16 pulse_power; + u8 mdrdy_flag; + u8 rsv[3]; + } periodic_pulse[32]; + + struct { + __le32 start; + __le16 pulse_width; + __le16 pulse_power; + u8 sc_pass; + u8 sw_reset; + u8 mdrdy_flag; + u8 tx_active; + } hw_pulse[32]; +} __packed; + +struct mt7996_mcu_background_chain_ctrl { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + u8 chan; /* primary channel */ + u8 central_chan; /* central channel */ + u8 bw; + u8 tx_stream; + u8 rx_stream; + + u8 monitor_chan; /* monitor channel */ + u8 monitor_central_chan;/* monitor central channel */ + u8 monitor_bw; + u8 monitor_tx_stream; + u8 monitor_rx_stream; + + u8 scan_mode; /* 0: ScanStop + * 1: ScanStart + * 2: ScanRunning + */ + u8 band_idx; /* DBDC */ + u8 monitor_scan_type; + u8 band; /* 0: 2.4GHz, 1: 5GHz */ + u8 rsv[2]; +} __packed; + +struct mt7996_mcu_eeprom { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + u8 buffer_mode; + u8 format; + __le16 buf_len; +} __packed; + +struct mt7996_mcu_phy_rx_info { + u8 category; + u8 rate; + u8 mode; + u8 nsts; + u8 gi; + u8 coding; + u8 stbc; + u8 bw; +}; + +struct mt7996_mcu_mib { + __le16 tag; + __le16 len; + __le32 offs; + __le64 data; +} __packed; + +enum mt7996_chan_mib_offs { + UNI_MIB_OBSS_AIRTIME = 26, + UNI_MIB_NON_WIFI_TIME = 27, + UNI_MIB_TX_TIME = 28, + UNI_MIB_RX_TIME = 29 +}; + +struct edca { + __le16 tag; + __le16 len; + + u8 queue; + u8 set; + u8 cw_min; + u8 cw_max; + __le16 txop; + u8 aifs; + u8 __rsv; +}; + +#define MCU_PQ_ID(p, q) (((p) << 15) | ((q) << 10)) +#define MCU_PKT_ID 0xa0 + +enum { + MCU_FW_LOG_WM, + MCU_FW_LOG_WA, + MCU_FW_LOG_TO_HOST, + MCU_FW_LOG_RELAY = 16 +}; + +enum { + MCU_TWT_AGRT_ADD, + MCU_TWT_AGRT_MODIFY, + MCU_TWT_AGRT_DELETE, + MCU_TWT_AGRT_TEARDOWN, + MCU_TWT_AGRT_GET_TSF, +}; + +enum { + MCU_WA_PARAM_CMD_QUERY, + MCU_WA_PARAM_CMD_SET, + MCU_WA_PARAM_CMD_CAPABILITY, + MCU_WA_PARAM_CMD_DEBUG, +}; + +enum { + MCU_WA_PARAM_PDMA_RX = 0x04, + MCU_WA_PARAM_CPU_UTIL = 0x0b, + MCU_WA_PARAM_RED = 0x0e, + MCU_WA_PARAM_HW_PATH_HIF_VER = 0x2f, +}; + +enum mcu_mmps_mode { + MCU_MMPS_STATIC, + MCU_MMPS_DYNAMIC, + MCU_MMPS_RSV, + MCU_MMPS_DISABLE, +}; + +struct bss_rate_tlv { + __le16 tag; + __le16 len; + u8 __rsv1[4]; + __le16 bc_trans; + __le16 mc_trans; + u8 short_preamble; + u8 bc_fixed_rate; + u8 mc_fixed_rate; + u8 __rsv2[1]; +} __packed; + +struct bss_ra_tlv { + __le16 tag; + __le16 len; + u8 short_preamble; + u8 force_sgi; + u8 force_gf; + u8 ht_mode; + u8 se_off; + u8 antenna_idx; + __le16 max_phyrate; + u8 force_tx_streams; + u8 __rsv[3]; +} __packed; + +struct bss_rlm_tlv { + __le16 tag; + __le16 len; + u8 control_channel; + u8 center_chan; + u8 center_chan2; + u8 bw; + u8 tx_streams; + u8 rx_streams; + u8 ht_op_info; + u8 sco; + u8 band; + u8 __rsv[3]; +} __packed; + +struct bss_color_tlv { + __le16 tag; + __le16 len; + u8 enable; + u8 color; + u8 rsv[2]; +} __packed; + +struct bss_inband_discovery_tlv { + __le16 tag; + __le16 len; + u8 tx_type; + u8 tx_mode; + u8 tx_interval; + u8 enable; + __le16 wcid; + __le16 prob_rsp_len; +#define MAX_INBAND_FRAME_SIZE 512 + u8 pkt[MAX_INBAND_FRAME_SIZE]; +} __packed; + +struct bss_bcn_content_tlv { + __le16 tag; + __le16 len; + __le16 tim_ie_pos; + __le16 csa_ie_pos; + __le16 bcc_ie_pos; + u8 enable; + u8 type; + __le16 pkt_len; +#define MAX_BEACON_SIZE 512 + u8 pkt[MAX_BEACON_SIZE]; +} __packed; + +struct bss_bcn_cntdwn_tlv { + __le16 tag; + __le16 len; + u8 cnt; + u8 rsv[3]; +} __packed; + +struct bss_bcn_mbss_tlv { + __le16 tag; + __le16 len; + __le32 bitmap; +#define MAX_BEACON_NUM 32 + __le16 offset[MAX_BEACON_NUM]; +} __packed __aligned(4); + +struct bss_txcmd_tlv { + __le16 tag; + __le16 len; + u8 txcmd_mode; + u8 __rsv[3]; +} __packed; + +struct bss_sec_tlv { + __le16 tag; + __le16 len; + u8 __rsv1[2]; + u8 cipher; + u8 __rsv2[1]; +} __packed; + +struct bss_power_save { + __le16 tag; + __le16 len; + u8 profile; + u8 _rsv[3]; +} __packed; + +struct bss_mld_tlv { + __le16 tag; + __le16 len; + u8 group_mld_id; + u8 own_mld_id; + u8 mac_addr[ETH_ALEN]; + u8 remap_idx; + u8 __rsv[3]; +} __packed; + +struct sta_rec_ba_uni { + __le16 tag; + __le16 len; + u8 tid; + u8 ba_type; + u8 amsdu; + u8 ba_en; + __le16 ssn; + __le16 winsize; + u8 ba_rdd_rro; + u8 __rsv[3]; +} __packed; + +struct sec_key_uni { + __le16 wlan_idx; + u8 mgmt_prot; + u8 cipher_id; + u8 cipher_len; + u8 key_id; + u8 key_len; + u8 need_resp; + u8 key[32]; +} __packed; + +struct sta_rec_sec_uni { + __le16 tag; + __le16 len; + u8 add; + u8 n_cipher; + u8 rsv[2]; + + struct sec_key_uni key[2]; +} __packed; + +struct sta_rec_hdrt { + __le16 tag; + __le16 len; + u8 hdrt_mode; + u8 rsv[3]; +} __packed; + +struct sta_rec_hdr_trans { + __le16 tag; + __le16 len; + u8 from_ds; + u8 to_ds; + u8 dis_rx_hdr_tran; + u8 rsv; +} __packed; + +struct hdr_trans_en { + __le16 tag; + __le16 len; + u8 enable; + u8 check_bssid; + u8 mode; + u8 __rsv; +} __packed; + +struct hdr_trans_vlan { + __le16 tag; + __le16 len; + u8 insert_vlan; + u8 remove_vlan; + u8 tid; + u8 __rsv; +} __packed; + +struct hdr_trans_blacklist { + __le16 tag; + __le16 len; + u8 idx; + u8 enable; + __le16 type; +} __packed; + +struct uni_header { + u8 __rsv[4]; +} __packed; + +struct vow_rx_airtime { + __le16 tag; + __le16 len; + + u8 enable; + u8 band; + u8 __rsv[2]; +} __packed; + +struct bf_sounding_on { + __le16 tag; + __le16 len; + + u8 snd_mode; + u8 sta_num; + u8 __rsv[2]; + __le16 wlan_id[4]; + __le32 snd_period; +} __packed; + +struct bf_hw_en_status_update { + __le16 tag; + __le16 len; + + bool ebf; + bool ibf; + u8 __rsv[2]; +} __packed; + +struct bf_mod_en_ctrl { + __le16 tag; + __le16 len; + + u8 bf_num; + u8 bf_bitmap; + u8 bf_sel[8]; + u8 __rsv[2]; +} __packed; + +union bf_tag_tlv { + struct bf_sounding_on bf_snd; + struct bf_hw_en_status_update bf_hw_en; + struct bf_mod_en_ctrl bf_mod_en; +}; + +struct ra_rate { + __le16 wlan_idx; + u8 mode; + u8 stbc; + __le16 gi; + u8 bw; + u8 ldpc; + u8 mcs; + u8 nss; + __le16 ltf; + u8 spe; + u8 preamble; + u8 __rsv[2]; +} __packed; + +struct ra_fixed_rate { + __le16 tag; + __le16 len; + + __le16 version; + struct ra_rate rate; +} __packed; + +enum { + UNI_RA_FIXED_RATE = 0xf, +}; + +#define MT7996_HDR_TRANS_MAX_SIZE (sizeof(struct hdr_trans_en) + \ + sizeof(struct hdr_trans_vlan) + \ + sizeof(struct hdr_trans_blacklist)) + +enum { + UNI_HDR_TRANS_EN, + UNI_HDR_TRANS_VLAN, + UNI_HDR_TRANS_BLACKLIST, +}; + +enum { + RATE_PARAM_FIXED = 3, + RATE_PARAM_MMPS_UPDATE = 5, + RATE_PARAM_FIXED_HE_LTF = 7, + RATE_PARAM_FIXED_MCS, + RATE_PARAM_FIXED_GI = 11, + RATE_PARAM_AUTO = 20, +}; + +enum { + BF_SOUNDING_ON = 1, + BF_HW_EN_UPDATE = 17, + BF_MOD_EN_CTRL = 20, +}; + +enum { + CMD_BAND_NONE, + CMD_BAND_24G, + CMD_BAND_5G, + CMD_BAND_6G, +}; + +struct bss_req_hdr { + u8 bss_idx; + u8 __rsv[3]; +} __packed; + +enum { + UNI_CHANNEL_SWITCH, + UNI_CHANNEL_RX_PATH, +}; + +#define MT7996_BSS_UPDATE_MAX_SIZE (sizeof(struct bss_req_hdr) + \ + sizeof(struct mt76_connac_bss_basic_tlv) + \ + sizeof(struct bss_rlm_tlv) + \ + sizeof(struct bss_ra_tlv) + \ + sizeof(struct bss_info_uni_he) + \ + sizeof(struct bss_rate_tlv) + \ + sizeof(struct bss_txcmd_tlv) + \ + sizeof(struct bss_power_save) + \ + sizeof(struct bss_sec_tlv) + \ + sizeof(struct bss_mld_tlv)) + +#define MT7996_STA_UPDATE_MAX_SIZE (sizeof(struct sta_req_hdr) + \ + sizeof(struct sta_rec_basic) + \ + sizeof(struct sta_rec_bf) + \ + sizeof(struct sta_rec_ht) + \ + sizeof(struct sta_rec_he_v2) + \ + sizeof(struct sta_rec_ba_uni) + \ + sizeof(struct sta_rec_vht) + \ + sizeof(struct sta_rec_uapsd) + \ + sizeof(struct sta_rec_amsdu) + \ + sizeof(struct sta_rec_bfee) + \ + sizeof(struct sta_rec_phy) + \ + sizeof(struct sta_rec_ra) + \ + sizeof(struct sta_rec_sec) + \ + sizeof(struct sta_rec_ra_fixed) + \ + sizeof(struct sta_rec_he_6g_capa) + \ + sizeof(struct sta_rec_hdrt) + \ + sizeof(struct sta_rec_hdr_trans) + \ + sizeof(struct tlv)) + +#define MT7996_BEACON_UPDATE_SIZE (sizeof(struct bss_req_hdr) + \ + sizeof(struct bss_bcn_content_tlv) + \ + sizeof(struct bss_bcn_cntdwn_tlv) + \ + sizeof(struct bss_bcn_mbss_tlv)) + +#define MT7996_INBAND_FRAME_SIZE (sizeof(struct bss_req_hdr) + \ + sizeof(struct bss_inband_discovery_tlv)) + +enum { + UNI_BAND_CONFIG_RADIO_ENABLE, + UNI_BAND_CONFIG_RTS_THRESHOLD = 0x08, +}; + +enum { + UNI_WSYS_CONFIG_FW_LOG_CTRL, + UNI_WSYS_CONFIG_FW_DBG_CTRL, +}; + +enum { + UNI_RDD_CTRL_PARM, + UNI_RDD_CTRL_SET_TH = 0x3, +}; + +enum { + UNI_EFUSE_ACCESS = 1, + UNI_EFUSE_BUFFER_MODE, + UNI_EFUSE_FREE_BLOCK, + UNI_EFUSE_BUFFER_RD, +}; + +enum { + UNI_VOW_DRR_CTRL, + UNI_VOW_RX_AT_AIRTIME_EN = 0x0b, + UNI_VOW_RX_AT_AIRTIME_CLR_EN = 0x0e, +}; + +enum { + UNI_CMD_MIB_DATA, +}; + +enum { + UNI_POWER_OFF, +}; + +enum { + UNI_CMD_TWT_ARGT_UPDATE = 0x0, + UNI_CMD_TWT_MGMT_OFFLOAD, +}; + +enum { + UNI_RRO_DEL_ENTRY = 0x1, + UNI_RRO_SET_PLATFORM_TYPE, + UNI_RRO_GET_BA_SESSION_TABLE, + UNI_RRO_SET_BYPASS_MODE, + UNI_RRO_SET_TXFREE_PATH, +}; + +enum{ + UNI_CMD_SR_ENABLE = 0x1, + UNI_CMD_SR_ENABLE_SD, + UNI_CMD_SR_ENABLE_MODE, + UNI_CMD_SR_ENABLE_DPD = 0x12, + UNI_CMD_SR_ENABLE_TX, + UNI_CMD_SR_SET_SRG_BITMAP = 0x80, + UNI_CMD_SR_SET_PARAM = 0xc1, + UNI_CMD_SR_SET_SIGA = 0xd0, +}; + +enum { + UNI_CMD_ACCESS_REG_BASIC = 0x0, + UNI_CMD_ACCESS_RF_REG_BASIC, +}; + +enum { + UNI_CMD_SER_QUERY = 0x0, + UNI_CMD_SER_SET = 0x2, + UNI_CMD_SER_TRIGGER = 0x3, +}; + +enum { + SER_QUERY, + /* recovery */ + SER_SET_RECOVER_L1, + SER_SET_RECOVER_L2, + SER_SET_RECOVER_L3_RX_ABORT, + SER_SET_RECOVER_L3_TX_ABORT, + SER_SET_RECOVER_L3_TX_DISABLE, + SER_SET_RECOVER_L3_BF, + /* action */ + SER_ENABLE = 2, + SER_RECOVER +}; + +enum { + MT7996_SEC_MODE_PLAIN, + MT7996_SEC_MODE_AES, + MT7996_SEC_MODE_SCRAMBLE, + MT7996_SEC_MODE_MAX, +}; + +#define MT7996_PATCH_SEC GENMASK(31, 24) +#define MT7996_PATCH_SCRAMBLE_KEY GENMASK(15, 8) +#define MT7996_PATCH_AES_KEY GENMASK(7, 0) + +#define MT7996_SEC_ENCRYPT BIT(0) +#define MT7996_SEC_KEY_IDX GENMASK(2, 1) +#define MT7996_SEC_IV BIT(3) + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c new file mode 100644 index 000000000000..521769eb6b0e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mt7996.h" +#include "mac.h" +#include "../trace.h" + +static const struct __base mt7996_reg_base[] = { + [WF_AGG_BASE] = { { 0x820e2000, 0x820f2000, 0x830e2000 } }, + [WF_ARB_BASE] = { { 0x820e3000, 0x820f3000, 0x830e3000 } }, + [WF_TMAC_BASE] = { { 0x820e4000, 0x820f4000, 0x830e4000 } }, + [WF_RMAC_BASE] = { { 0x820e5000, 0x820f5000, 0x830e5000 } }, + [WF_DMA_BASE] = { { 0x820e7000, 0x820f7000, 0x830e7000 } }, + [WF_WTBLOFF_BASE] = { { 0x820e9000, 0x820f9000, 0x830e9000 } }, + [WF_ETBF_BASE] = { { 0x820ea000, 0x820fa000, 0x830ea000 } }, + [WF_LPON_BASE] = { { 0x820eb000, 0x820fb000, 0x830eb000 } }, + [WF_MIB_BASE] = { { 0x820ed000, 0x820fd000, 0x830ed000 } }, +}; + +static const struct __map mt7996_reg_map[] = { + { 0x54000000, 0x02000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ + { 0x55000000, 0x03000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ + { 0x56000000, 0x04000, 0x1000 }, /* WFDMA reserved */ + { 0x57000000, 0x05000, 0x1000 }, /* WFDMA MCU wrap CR */ + { 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */ + { 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */ + { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */ + { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */ + { 0x820cc000, 0x0e000, 0x1000 }, /* WF_UMAC_TOP (PP) */ + { 0x74030000, 0x10000, 0x1000 }, /* PCIe MAC */ + { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ + { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ + { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ + { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ + { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ + { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ + { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */ + { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ + { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */ + { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ + { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ + { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ + { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ + { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ + { 0x820ca000, 0x26000, 0x2000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */ + { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */ + { 0x40000000, 0x70000, 0x10000 }, /* WF_UMAC_SYSRAM */ + { 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */ + { 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure register) */ + { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ + { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ + { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ + { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ + { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ + { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ + { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ + { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ + { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ + { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ + { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ + { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ + { 0x820cc000, 0xa5000, 0x2000 }, /* WF_LMAC_TOP BN1 (WF_MUCOP) */ + { 0x820c4000, 0xa8000, 0x4000 }, /* WF_LMAC_TOP BN1 (WF_MUCOP) */ + { 0x820b0000, 0xae000, 0x1000 }, /* [APB2] WFSYS_ON */ + { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */ + { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */ + { 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, wfdma */ + { 0x7c060000, 0xe0000, 0x10000 }, /* CONN_INFRA, conn_host_csr_top */ + { 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */ + { 0x0, 0x0, 0x0 }, /* imply end of search */ +}; + +static u32 mt7996_reg_map_l1(struct mt7996_dev *dev, u32 addr) +{ + u32 offset = FIELD_GET(MT_HIF_REMAP_L1_OFFSET, addr); + u32 base = FIELD_GET(MT_HIF_REMAP_L1_BASE, addr); + + dev->reg_l1_backup = dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L1); + dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L1, + MT_HIF_REMAP_L1_MASK, + FIELD_PREP(MT_HIF_REMAP_L1_MASK, base)); + /* use read to push write */ + dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L1); + + return MT_HIF_REMAP_BASE_L1 + offset; +} + +static u32 mt7996_reg_map_l2(struct mt7996_dev *dev, u32 addr) +{ + u32 offset = FIELD_GET(MT_HIF_REMAP_L2_OFFSET, addr); + u32 base = FIELD_GET(MT_HIF_REMAP_L2_BASE, addr); + + dev->reg_l2_backup = dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L2); + dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L2, + MT_HIF_REMAP_L2_MASK, + FIELD_PREP(MT_HIF_REMAP_L2_MASK, base)); + /* use read to push write */ + dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L2); + + return MT_HIF_REMAP_BASE_L2 + offset; +} + +static void mt7996_reg_remap_restore(struct mt7996_dev *dev) +{ + /* remap to ori status */ + if (unlikely(dev->reg_l1_backup)) { + dev->bus_ops->wr(&dev->mt76, MT_HIF_REMAP_L1, dev->reg_l1_backup); + dev->reg_l1_backup = 0; + } + + if (dev->reg_l2_backup) { + dev->bus_ops->wr(&dev->mt76, MT_HIF_REMAP_L2, dev->reg_l2_backup); + dev->reg_l2_backup = 0; + } +} + +static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr) +{ + int i; + + mt7996_reg_remap_restore(dev); + + if (addr < 0x100000) + return addr; + + for (i = 0; i < dev->reg.map_size; i++) { + u32 ofs; + + if (addr < dev->reg.map[i].phys) + continue; + + ofs = addr - dev->reg.map[i].phys; + if (ofs > dev->reg.map[i].size) + continue; + + return dev->reg.map[i].mapped + ofs; + } + + if ((addr >= MT_INFRA_BASE && addr < MT_WFSYS0_PHY_START) || + (addr >= MT_WFSYS0_PHY_START && addr < MT_WFSYS1_PHY_START) || + (addr >= MT_WFSYS1_PHY_START && addr <= MT_WFSYS1_PHY_END)) + return mt7996_reg_map_l1(dev, addr); + + if (dev_is_pci(dev->mt76.dev) && + ((addr >= MT_CBTOP1_PHY_START && addr <= MT_CBTOP1_PHY_END) || + (addr >= MT_CBTOP2_PHY_START && addr <= MT_CBTOP2_PHY_END))) + return mt7996_reg_map_l1(dev, addr); + + /* CONN_INFRA: covert to phyiscal addr and use layer 1 remap */ + if (addr >= MT_INFRA_MCU_START && addr <= MT_INFRA_MCU_END) { + addr = addr - MT_INFRA_MCU_START + MT_INFRA_BASE; + return mt7996_reg_map_l1(dev, addr); + } + + return mt7996_reg_map_l2(dev, addr); +} + +static u32 mt7996_rr(struct mt76_dev *mdev, u32 offset) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + + return dev->bus_ops->rr(mdev, __mt7996_reg_addr(dev, offset)); +} + +static void mt7996_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + + dev->bus_ops->wr(mdev, __mt7996_reg_addr(dev, offset), val); +} + +static u32 mt7996_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + + return dev->bus_ops->rmw(mdev, __mt7996_reg_addr(dev, offset), mask, val); +} + +static int mt7996_mmio_init(struct mt76_dev *mdev, + void __iomem *mem_base, + u32 device_id) +{ + struct mt76_bus_ops *bus_ops; + struct mt7996_dev *dev; + + dev = container_of(mdev, struct mt7996_dev, mt76); + mt76_mmio_init(&dev->mt76, mem_base); + + switch (device_id) { + case 0x7990: + dev->reg.base = mt7996_reg_base; + dev->reg.map = mt7996_reg_map; + dev->reg.map_size = ARRAY_SIZE(mt7996_reg_map); + break; + default: + return -EINVAL; + } + + dev->bus_ops = dev->mt76.bus; + bus_ops = devm_kmemdup(dev->mt76.dev, dev->bus_ops, sizeof(*bus_ops), + GFP_KERNEL); + if (!bus_ops) + return -ENOMEM; + + bus_ops->rr = mt7996_rr; + bus_ops->wr = mt7996_wr; + bus_ops->rmw = mt7996_rmw; + dev->mt76.bus = bus_ops; + + mdev->rev = (device_id << 16) | (mt76_rr(dev, MT_HW_REV) & 0xff); + + dev_dbg(mdev->dev, "ASIC revision: %04x\n", mdev->rev); + + return 0; +} + +void mt7996_dual_hif_set_irq_mask(struct mt7996_dev *dev, bool write_reg, + u32 clear, u32 set) +{ + struct mt76_dev *mdev = &dev->mt76; + unsigned long flags; + + spin_lock_irqsave(&mdev->mmio.irq_lock, flags); + + mdev->mmio.irqmask &= ~clear; + mdev->mmio.irqmask |= set; + + if (write_reg) { + mt76_wr(dev, MT_INT_MASK_CSR, mdev->mmio.irqmask); + mt76_wr(dev, MT_INT1_MASK_CSR, mdev->mmio.irqmask); + } + + spin_unlock_irqrestore(&mdev->mmio.irq_lock, flags); +} + +static void mt7996_rx_poll_complete(struct mt76_dev *mdev, + enum mt76_rxq_id q) +{ + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + + mt7996_irq_enable(dev, MT_INT_RX(q)); +} + +/* TODO: support 2/4/6/8 MSI-X vectors */ +static void mt7996_irq_tasklet(struct tasklet_struct *t) +{ + struct mt7996_dev *dev = from_tasklet(dev, t, irq_tasklet); + u32 i, intr, mask, intr1; + + mt76_wr(dev, MT_INT_MASK_CSR, 0); + if (dev->hif2) + mt76_wr(dev, MT_INT1_MASK_CSR, 0); + + intr = mt76_rr(dev, MT_INT_SOURCE_CSR); + intr &= dev->mt76.mmio.irqmask; + mt76_wr(dev, MT_INT_SOURCE_CSR, intr); + + if (dev->hif2) { + intr1 = mt76_rr(dev, MT_INT1_SOURCE_CSR); + intr1 &= dev->mt76.mmio.irqmask; + mt76_wr(dev, MT_INT1_SOURCE_CSR, intr1); + + intr |= intr1; + } + + trace_dev_irq(&dev->mt76, intr, dev->mt76.mmio.irqmask); + + mask = intr & MT_INT_RX_DONE_ALL; + if (intr & MT_INT_TX_DONE_MCU) + mask |= MT_INT_TX_DONE_MCU; + mt7996_irq_disable(dev, mask); + + if (intr & MT_INT_TX_DONE_MCU) + napi_schedule(&dev->mt76.tx_napi); + + for (i = 0; i < __MT_RXQ_MAX; i++) { + if ((intr & MT_INT_RX(i))) + napi_schedule(&dev->mt76.napi[i]); + } + + if (intr & MT_INT_MCU_CMD) { + u32 val = mt76_rr(dev, MT_MCU_CMD); + + mt76_wr(dev, MT_MCU_CMD, val); + if (val & MT_MCU_CMD_ERROR_MASK) { + dev->reset_state = val; + ieee80211_queue_work(mt76_hw(dev), &dev->reset_work); + wake_up(&dev->reset_wait); + } + } +} + +irqreturn_t mt7996_irq_handler(int irq, void *dev_instance) +{ + struct mt7996_dev *dev = dev_instance; + + mt76_wr(dev, MT_INT_MASK_CSR, 0); + if (dev->hif2) + mt76_wr(dev, MT_INT1_MASK_CSR, 0); + + if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state)) + return IRQ_NONE; + + tasklet_schedule(&dev->irq_tasklet); + + return IRQ_HANDLED; +} + +struct mt7996_dev *mt7996_mmio_probe(struct device *pdev, + void __iomem *mem_base, u32 device_id) +{ + static const struct mt76_driver_ops drv_ops = { + /* txwi_size = txd size + txp size */ + .txwi_size = MT_TXD_SIZE + sizeof(struct mt7996_txp), + .drv_flags = MT_DRV_TXWI_NO_FREE | + MT_DRV_HW_MGMT_TXQ, + .survey_flags = SURVEY_INFO_TIME_TX | + SURVEY_INFO_TIME_RX | + SURVEY_INFO_TIME_BSS_RX, + .token_size = MT7996_TOKEN_SIZE, + .tx_prepare_skb = mt7996_tx_prepare_skb, + .tx_complete_skb = mt7996_tx_complete_skb, + .rx_skb = mt7996_queue_rx_skb, + .rx_check = mt7996_rx_check, + .rx_poll_complete = mt7996_rx_poll_complete, + .sta_ps = mt7996_sta_ps, + .sta_add = mt7996_mac_sta_add, + .sta_remove = mt7996_mac_sta_remove, + .update_survey = mt7996_update_channel, + }; + struct mt7996_dev *dev; + struct mt76_dev *mdev; + int ret; + + mdev = mt76_alloc_device(pdev, sizeof(*dev), &mt7996_ops, &drv_ops); + if (!mdev) + return ERR_PTR(-ENOMEM); + + dev = container_of(mdev, struct mt7996_dev, mt76); + + ret = mt7996_mmio_init(mdev, mem_base, device_id); + if (ret) + goto error; + + tasklet_setup(&dev->irq_tasklet, mt7996_irq_tasklet); + + mt76_wr(dev, MT_INT_MASK_CSR, 0); + + return dev; + +error: + mt76_free_device(&dev->mt76); + + return ERR_PTR(ret); +} + +static int __init mt7996_init(void) +{ + int ret; + + ret = pci_register_driver(&mt7996_hif_driver); + if (ret) + return ret; + + ret = pci_register_driver(&mt7996_pci_driver); + if (ret) + pci_unregister_driver(&mt7996_hif_driver); + + return ret; +} + +static void __exit mt7996_exit(void) +{ + pci_unregister_driver(&mt7996_pci_driver); + pci_unregister_driver(&mt7996_hif_driver); +} + +module_init(mt7996_init); +module_exit(mt7996_exit); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h new file mode 100644 index 000000000000..725344791b4c --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -0,0 +1,523 @@ +/* SPDX-License-Identifier: ISC */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef __MT7996_H +#define __MT7996_H + +#include <linux/interrupt.h> +#include <linux/ktime.h> +#include "../mt76_connac.h" +#include "regs.h" + +#define MT7996_MAX_INTERFACES 19 +#define MT7996_MAX_WMM_SETS 4 +#define MT7996_WTBL_SIZE 544 +#define MT7996_WTBL_RESERVED (MT7996_WTBL_SIZE - 1) +#define MT7996_WTBL_STA (MT7996_WTBL_RESERVED - \ + MT7996_MAX_INTERFACES) + +#define MT7996_WATCHDOG_TIME (HZ / 10) +#define MT7996_RESET_TIMEOUT (30 * HZ) + +#define MT7996_TX_RING_SIZE 2048 +#define MT7996_TX_MCU_RING_SIZE 256 +#define MT7996_TX_FWDL_RING_SIZE 128 + +#define MT7996_RX_RING_SIZE 1536 +#define MT7996_RX_MCU_RING_SIZE 512 + +#define MT7996_FIRMWARE_WA "mediatek/mt7996/mt7996_wa.bin" +#define MT7996_FIRMWARE_WM "mediatek/mt7996/mt7996_wm.bin" +#define MT7996_ROM_PATCH "mediatek/mt7996/mt7996_rom_patch.bin" + +#define MT7996_EEPROM_DEFAULT "mediatek/mt7996/mt7996_eeprom.bin" +#define MT7996_EEPROM_SIZE 7680 +#define MT7996_EEPROM_BLOCK_SIZE 16 +#define MT7996_TOKEN_SIZE 8192 + +#define MT7996_CFEND_RATE_DEFAULT 0x49 /* OFDM 24M */ +#define MT7996_CFEND_RATE_11B 0x03 /* 11B LP, 11M */ + +#define MT7996_MAX_TWT_AGRT 16 +#define MT7996_MAX_STA_TWT_AGRT 8 +#define MT7996_MAX_QUEUE (__MT_RXQ_MAX + __MT_MCUQ_MAX + 3) + +struct mt7996_vif; +struct mt7996_sta; +struct mt7996_dfs_pulse; +struct mt7996_dfs_pattern; + +enum mt7996_txq_id { + MT7996_TXQ_FWDL = 16, + MT7996_TXQ_MCU_WM, + MT7996_TXQ_BAND0, + MT7996_TXQ_BAND1, + MT7996_TXQ_MCU_WA, + MT7996_TXQ_BAND2, +}; + +enum mt7996_rxq_id { + MT7996_RXQ_MCU_WM = 0, + MT7996_RXQ_MCU_WA, + MT7996_RXQ_MCU_WA_MAIN = 2, + MT7996_RXQ_MCU_WA_EXT = 2,/* unused */ + MT7996_RXQ_MCU_WA_TRI = 3, + MT7996_RXQ_BAND0 = 4, + MT7996_RXQ_BAND1 = 4,/* unused */ + MT7996_RXQ_BAND2 = 5, +}; + +struct mt7996_twt_flow { + struct list_head list; + u64 start_tsf; + u64 tsf; + u32 duration; + u16 wcid; + __le16 mantissa; + u8 exp; + u8 table_id; + u8 id; + u8 protection:1; + u8 flowtype:1; + u8 trigger:1; + u8 sched:1; +}; + +DECLARE_EWMA(avg_signal, 10, 8) + +struct mt7996_sta { + struct mt76_wcid wcid; /* must be first */ + + struct mt7996_vif *vif; + + struct list_head poll_list; + struct list_head rc_list; + u32 airtime_ac[8]; + + int ack_signal; + struct ewma_avg_signal avg_ack_signal; + + unsigned long changed; + unsigned long jiffies; + unsigned long ampdu_state; + + struct mt76_sta_stats stats; + + struct mt76_connac_sta_key_conf bip; + + struct { + u8 flowid_mask; + struct mt7996_twt_flow flow[MT7996_MAX_STA_TWT_AGRT]; + } twt; +}; + +struct mt7996_vif_cap { + bool ht_ldpc:1; + bool vht_ldpc:1; + bool he_ldpc:1; + bool vht_su_ebfer:1; + bool vht_su_ebfee:1; + bool vht_mu_ebfer:1; + bool vht_mu_ebfee:1; + bool he_su_ebfer:1; + bool he_su_ebfee:1; + bool he_mu_ebfer:1; +}; + +struct mt7996_vif { + struct mt76_vif mt76; /* must be first */ + + struct mt7996_vif_cap cap; + struct mt7996_sta sta; + struct mt7996_phy *phy; + + struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS]; + struct cfg80211_bitrate_mask bitrate_mask; +}; + +/* per-phy stats. */ +struct mib_stats { + u32 ack_fail_cnt; + u32 fcs_err_cnt; + u32 rts_cnt; + u32 rts_retries_cnt; + u32 ba_miss_cnt; + u32 tx_mu_bf_cnt; + u32 tx_mu_mpdu_cnt; + u32 tx_mu_acked_mpdu_cnt; + u32 tx_su_acked_mpdu_cnt; + u32 tx_bf_ibf_ppdu_cnt; + u32 tx_bf_ebf_ppdu_cnt; + + u32 tx_bf_rx_fb_all_cnt; + u32 tx_bf_rx_fb_eht_cnt; + u32 tx_bf_rx_fb_he_cnt; + u32 tx_bf_rx_fb_vht_cnt; + u32 tx_bf_rx_fb_ht_cnt; + + u32 tx_bf_rx_fb_bw; /* value of last sample, not cumulative */ + u32 tx_bf_rx_fb_nc_cnt; + u32 tx_bf_rx_fb_nr_cnt; + u32 tx_bf_fb_cpl_cnt; + u32 tx_bf_fb_trig_cnt; + + u32 tx_ampdu_cnt; + u32 tx_stop_q_empty_cnt; + u32 tx_mpdu_attempts_cnt; + u32 tx_mpdu_success_cnt; + /* BF counter is PPDU-based, so remove MPDU-based BF counter */ + + u32 tx_rwp_fail_cnt; + u32 tx_rwp_need_cnt; + + /* rx stats */ + u32 rx_fifo_full_cnt; + u32 channel_idle_cnt; + u32 rx_vector_mismatch_cnt; + u32 rx_delimiter_fail_cnt; + u32 rx_len_mismatch_cnt; + u32 rx_mpdu_cnt; + u32 rx_ampdu_cnt; + u32 rx_ampdu_bytes_cnt; + u32 rx_ampdu_valid_subframe_cnt; + u32 rx_ampdu_valid_subframe_bytes_cnt; + u32 rx_pfdrop_cnt; + u32 rx_vec_queue_overflow_drop_cnt; + u32 rx_ba_cnt; + + u32 tx_amsdu[8]; + u32 tx_amsdu_cnt; +}; + +struct mt7996_hif { + struct list_head list; + + struct device *dev; + void __iomem *regs; + int irq; +}; + +struct mt7996_phy { + struct mt76_phy *mt76; + struct mt7996_dev *dev; + + struct ieee80211_sband_iftype_data iftype[NUM_NL80211_BANDS][NUM_NL80211_IFTYPES]; + + struct ieee80211_vif *monitor_vif; + + u32 rxfilter; + u64 omac_mask; + + u16 noise; + + s16 coverage_class; + u8 slottime; + + u8 rdd_state; + + u32 rx_ampdu_ts; + u32 ampdu_ref; + + struct mib_stats mib; + struct mt76_channel_state state_ts; +}; + +struct mt7996_dev { + union { /* must be first */ + struct mt76_dev mt76; + struct mt76_phy mphy; + }; + + struct mt7996_hif *hif2; + struct mt7996_reg_desc reg; + u8 q_id[MT7996_MAX_QUEUE]; + u32 q_int_mask[MT7996_MAX_QUEUE]; + u32 q_wfdma_mask; + + const struct mt76_bus_ops *bus_ops; + struct tasklet_struct irq_tasklet; + struct mt7996_phy phy; + + /* monitor rx chain configured channel */ + struct cfg80211_chan_def rdd2_chandef; + struct mt7996_phy *rdd2_phy; + + u16 chainmask; + u8 chainshift[__MT_MAX_BAND]; + u32 hif_idx; + + struct work_struct init_work; + struct work_struct rc_work; + struct work_struct reset_work; + wait_queue_head_t reset_wait; + u32 reset_state; + + struct list_head sta_rc_list; + struct list_head sta_poll_list; + struct list_head twt_list; + spinlock_t sta_poll_lock; + + u32 hw_pattern; + + bool dbdc_support:1; + bool tbtc_support:1; + bool flash_mode:1; + + bool ibf; + u8 fw_debug_wm; + u8 fw_debug_wa; + u8 fw_debug_bin; + u16 fw_debug_seq; + + struct dentry *debugfs_dir; + struct rchan *relay_fwlog; + + struct { + u8 table_mask; + u8 n_agrt; + } twt; + + u32 reg_l1_backup; + u32 reg_l2_backup; +}; + +enum { + WFDMA0 = 0x0, + WFDMA1, + WFDMA_EXT, + __MT_WFDMA_MAX, +}; + +enum { + MT_CTX0, + MT_HIF0 = 0x0, + + MT_LMAC_AC00 = 0x0, + MT_LMAC_AC01, + MT_LMAC_AC02, + MT_LMAC_AC03, + MT_LMAC_ALTX0 = 0x10, + MT_LMAC_BMC0, + MT_LMAC_BCN0, + MT_LMAC_PSMP0, +}; + +enum { + MT_RX_SEL0, + MT_RX_SEL1, + MT_RX_SEL2, /* monitor chain */ +}; + +enum mt7996_rdd_cmd { + RDD_STOP, + RDD_START, + RDD_DET_MODE, + RDD_RADAR_EMULATE, + RDD_START_TXQ = 20, + RDD_CAC_START = 50, + RDD_CAC_END, + RDD_NORMAL_START, + RDD_DISABLE_DFS_CAL, + RDD_PULSE_DBG, + RDD_READ_PULSE, + RDD_RESUME_BF, + RDD_IRQ_OFF, +}; + +static inline struct mt7996_phy * +mt7996_hw_phy(struct ieee80211_hw *hw) +{ + struct mt76_phy *phy = hw->priv; + + return phy->priv; +} + +static inline struct mt7996_dev * +mt7996_hw_dev(struct ieee80211_hw *hw) +{ + struct mt76_phy *phy = hw->priv; + + return container_of(phy->dev, struct mt7996_dev, mt76); +} + +static inline struct mt7996_phy * +__mt7996_phy(struct mt7996_dev *dev, enum mt76_band_id band) +{ + struct mt76_phy *phy = dev->mt76.phys[band]; + + if (!phy) + return NULL; + + return phy->priv; +} + +static inline struct mt7996_phy * +mt7996_phy2(struct mt7996_dev *dev) +{ + return __mt7996_phy(dev, MT_BAND1); +} + +static inline struct mt7996_phy * +mt7996_phy3(struct mt7996_dev *dev) +{ + return __mt7996_phy(dev, MT_BAND2); +} + +extern const struct ieee80211_ops mt7996_ops; +extern struct pci_driver mt7996_pci_driver; +extern struct pci_driver mt7996_hif_driver; + +struct mt7996_dev *mt7996_mmio_probe(struct device *pdev, + void __iomem *mem_base, u32 device_id); +void mt7996_wfsys_reset(struct mt7996_dev *dev); +irqreturn_t mt7996_irq_handler(int irq, void *dev_instance); +u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif *mvif); +int mt7996_register_device(struct mt7996_dev *dev); +void mt7996_unregister_device(struct mt7996_dev *dev); +int mt7996_eeprom_init(struct mt7996_dev *dev); +int mt7996_eeprom_parse_hw_cap(struct mt7996_dev *dev, struct mt7996_phy *phy); +int mt7996_eeprom_get_target_power(struct mt7996_dev *dev, + struct ieee80211_channel *chan); +s8 mt7996_eeprom_get_power_delta(struct mt7996_dev *dev, int band); +int mt7996_dma_init(struct mt7996_dev *dev); +void mt7996_dma_prefetch(struct mt7996_dev *dev); +void mt7996_dma_cleanup(struct mt7996_dev *dev); +int mt7996_mcu_init(struct mt7996_dev *dev); +int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev, + struct mt7996_vif *mvif, + struct mt7996_twt_flow *flow, + int cmd); +int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, + struct ieee80211_vif *vif, bool enable); +int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, + struct ieee80211_vif *vif, int enable); +int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, bool enable); +int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, + struct ieee80211_ampdu_params *params, + bool add); +int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, + struct ieee80211_ampdu_params *params, + bool add); +int mt7996_mcu_update_bss_color(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct cfg80211_he_bss_color *he_bss_color); +int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + int enable); +int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev, + struct ieee80211_vif *vif, u32 changed); +int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy, struct ieee80211_vif *vif, + struct ieee80211_he_obss_pd *he_obss_pd); +int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, bool changed); +int mt7996_set_channel(struct mt7996_phy *phy); +int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag); +int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif); +int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev, + void *data, u16 version); +int mt7996_mcu_set_eeprom(struct mt7996_dev *dev); +int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset); +int mt7996_mcu_get_eeprom_free_block(struct mt7996_dev *dev, u8 *block_num); +int mt7996_mcu_set_ser(struct mt7996_dev *dev, u8 action, u8 set, u8 band); +int mt7996_mcu_set_txbf(struct mt7996_dev *dev, u8 action); +int mt7996_mcu_set_fcc5_lpn(struct mt7996_dev *dev, int val); +int mt7996_mcu_set_pulse_th(struct mt7996_dev *dev, + const struct mt7996_dfs_pulse *pulse); +int mt7996_mcu_set_radar_th(struct mt7996_dev *dev, int index, + const struct mt7996_dfs_pattern *pattern); +int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable); +void mt7996_mcu_set_pm(void *priv, u8 *mac, struct ieee80211_vif *vif); +int mt7996_mcu_set_rts_thresh(struct mt7996_phy *phy, u32 val); +int mt7996_mcu_get_chan_mib_info(struct mt7996_phy *phy, bool chan_switch); +int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index, + u8 rx_sel, u8 val); +int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, + struct cfg80211_chan_def *chandef); +int mt7996_mcu_rf_regval(struct mt7996_dev *dev, u32 regidx, u32 *val, bool set); +int mt7996_mcu_set_hdr_trans(struct mt7996_dev *dev, bool hdr_trans); +int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val); +int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3); +int mt7996_mcu_fw_log_2_host(struct mt7996_dev *dev, u8 type, u8 ctrl); +int mt7996_mcu_fw_dbg_ctrl(struct mt7996_dev *dev, u32 module, u8 level); +void mt7996_mcu_rx_event(struct mt7996_dev *dev, struct sk_buff *skb); +void mt7996_mcu_exit(struct mt7996_dev *dev); + +void mt7996_dual_hif_set_irq_mask(struct mt7996_dev *dev, bool write_reg, + u32 clear, u32 set); + +static inline void mt7996_irq_enable(struct mt7996_dev *dev, u32 mask) +{ + if (dev->hif2) + mt7996_dual_hif_set_irq_mask(dev, false, 0, mask); + else + mt76_set_irq_mask(&dev->mt76, 0, 0, mask); + + tasklet_schedule(&dev->irq_tasklet); +} + +static inline void mt7996_irq_disable(struct mt7996_dev *dev, u32 mask) +{ + if (dev->hif2) + mt7996_dual_hif_set_irq_mask(dev, true, mask, 0); + else + mt76_set_irq_mask(&dev->mt76, MT_INT_MASK_CSR, mask, 0); +} + +u32 mt7996_mac_wtbl_lmac_addr(struct mt7996_dev *dev, u16 wcid, u8 dw); +bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask); +void mt7996_mac_reset_counters(struct mt7996_phy *phy); +void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy); +void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); +void mt7996_mac_enable_rtscts(struct mt7996_dev *dev, + struct ieee80211_vif *vif, bool enable); +void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, + struct sk_buff *skb, struct mt76_wcid *wcid, int pid, + struct ieee80211_key_conf *key, u32 changed); +void mt7996_mac_set_timing(struct mt7996_phy *phy); +int mt7996_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta); +void mt7996_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta); +void mt7996_mac_work(struct work_struct *work); +void mt7996_mac_reset_work(struct work_struct *work); +void mt7996_mac_sta_rc_work(struct work_struct *work); +void mt7996_mac_update_stats(struct mt7996_phy *phy); +void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev, + struct mt7996_sta *msta, + u8 flowid); +void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw, + struct ieee80211_sta *sta, + struct ieee80211_twt_setup *twt); +int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, + enum mt76_txq_id qid, struct mt76_wcid *wcid, + struct ieee80211_sta *sta, + struct mt76_tx_info *tx_info); +void mt7996_tx_complete_skb(struct mt76_dev *mdev, struct mt76_queue_entry *e); +void mt7996_tx_token_put(struct mt7996_dev *dev); +void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, + struct sk_buff *skb, u32 *info); +bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len); +void mt7996_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); +void mt7996_stats_work(struct work_struct *work); +int mt76_dfs_start_rdd(struct mt7996_dev *dev, bool force); +int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy); +void mt7996_set_stream_he_caps(struct mt7996_phy *phy); +void mt7996_set_stream_vht_txbf_caps(struct mt7996_phy *phy); +void mt7996_update_channel(struct mt76_phy *mphy); +int mt7996_init_debugfs(struct mt7996_phy *phy); +void mt7996_debugfs_rx_fw_monitor(struct mt7996_dev *dev, const void *data, int len); +bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len); +int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, + struct mt76_connac_sta_key_conf *sta_key_conf, + struct ieee80211_key_conf *key, int mcu_cmd, + struct mt76_wcid *wcid, enum set_key_cmd cmd); +int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta); +#ifdef CONFIG_MAC80211_DEBUGFS +void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, struct dentry *dir); +#endif + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c new file mode 100644 index 000000000000..64aee3fb5445 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mt7996.h" +#include "mac.h" +#include "../trace.h" + +static LIST_HEAD(hif_list); +static DEFINE_SPINLOCK(hif_lock); +static u32 hif_idx; + +static const struct pci_device_id mt7996_pci_device_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7990) }, + { }, +}; + +static const struct pci_device_id mt7996_hif_device_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7991) }, + { }, +}; + +static struct mt7996_hif *mt7996_pci_get_hif2(u32 idx) +{ + struct mt7996_hif *hif; + u32 val; + + spin_lock_bh(&hif_lock); + + list_for_each_entry(hif, &hif_list, list) { + val = readl(hif->regs + MT_PCIE_RECOG_ID); + val &= MT_PCIE_RECOG_ID_MASK; + if (val != idx) + continue; + + get_device(hif->dev); + goto out; + } + hif = NULL; + +out: + spin_unlock_bh(&hif_lock); + + return hif; +} + +static void mt7996_put_hif2(struct mt7996_hif *hif) +{ + if (!hif) + return; + + put_device(hif->dev); +} + +static struct mt7996_hif *mt7996_pci_init_hif2(struct pci_dev *pdev) +{ + hif_idx++; + if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7991, NULL)) + return NULL; + + writel(hif_idx | MT_PCIE_RECOG_ID_SEM, + pcim_iomap_table(pdev)[0] + MT_PCIE_RECOG_ID); + + return mt7996_pci_get_hif2(hif_idx); +} + +static int mt7996_pci_hif2_probe(struct pci_dev *pdev) +{ + struct mt7996_hif *hif; + + hif = devm_kzalloc(&pdev->dev, sizeof(*hif), GFP_KERNEL); + if (!hif) + return -ENOMEM; + + hif->dev = &pdev->dev; + hif->regs = pcim_iomap_table(pdev)[0]; + hif->irq = pdev->irq; + spin_lock_bh(&hif_lock); + list_add(&hif->list, &hif_list); + spin_unlock_bh(&hif_lock); + pci_set_drvdata(pdev, hif); + + return 0; +} + +static int mt7996_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct pci_dev *hif2_dev; + struct mt7996_dev *dev; + struct mt76_dev *mdev; + struct mt7996_hif *hif2; + int irq, ret; + + ret = pcim_enable_device(pdev); + if (ret) + return ret; + + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret) + return ret; + + pci_set_master(pdev); + + ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + mt76_pci_disable_aspm(pdev); + + if (id->device == 0x7991) + return mt7996_pci_hif2_probe(pdev); + + dev = mt7996_mmio_probe(&pdev->dev, pcim_iomap_table(pdev)[0], + id->device); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + mdev = &dev->mt76; + mt7996_wfsys_reset(dev); + hif2 = mt7996_pci_init_hif2(pdev); + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + goto free_device; + + irq = pdev->irq; + ret = devm_request_irq(mdev->dev, irq, mt7996_irq_handler, + IRQF_SHARED, KBUILD_MODNAME, dev); + if (ret) + goto free_irq_vector; + + mt76_wr(dev, MT_INT_MASK_CSR, 0); + /* master switch of PCIe tnterrupt enable */ + mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); + + if (hif2) { + hif2_dev = container_of(hif2->dev, struct pci_dev, dev); + dev->hif2 = hif2; + + ret = pci_alloc_irq_vectors(hif2_dev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + goto free_hif2; + + dev->hif2->irq = hif2_dev->irq; + ret = devm_request_irq(mdev->dev, dev->hif2->irq, + mt7996_irq_handler, IRQF_SHARED, + KBUILD_MODNAME "-hif", dev); + if (ret) + goto free_hif2_irq_vector; + + mt76_wr(dev, MT_INT1_MASK_CSR, 0); + /* master switch of PCIe tnterrupt enable */ + mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0xff); + } + + ret = mt7996_register_device(dev); + if (ret) + goto free_hif2_irq; + + return 0; + +free_hif2_irq: + if (dev->hif2) + devm_free_irq(mdev->dev, dev->hif2->irq, dev); +free_hif2_irq_vector: + if (dev->hif2) + pci_free_irq_vectors(hif2_dev); +free_hif2: + if (dev->hif2) + put_device(dev->hif2->dev); + devm_free_irq(mdev->dev, irq, dev); +free_irq_vector: + pci_free_irq_vectors(pdev); +free_device: + mt76_free_device(&dev->mt76); + + return ret; +} + +static void mt7996_hif_remove(struct pci_dev *pdev) +{ + struct mt7996_hif *hif = pci_get_drvdata(pdev); + + list_del(&hif->list); +} + +static void mt7996_pci_remove(struct pci_dev *pdev) +{ + struct mt76_dev *mdev; + struct mt7996_dev *dev; + + mdev = pci_get_drvdata(pdev); + dev = container_of(mdev, struct mt7996_dev, mt76); + mt7996_put_hif2(dev->hif2); + mt7996_unregister_device(dev); +} + +struct pci_driver mt7996_hif_driver = { + .name = KBUILD_MODNAME "_hif", + .id_table = mt7996_hif_device_table, + .probe = mt7996_pci_probe, + .remove = mt7996_hif_remove, +}; + +struct pci_driver mt7996_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = mt7996_pci_device_table, + .probe = mt7996_pci_probe, + .remove = mt7996_pci_remove, +}; + +MODULE_DEVICE_TABLE(pci, mt7996_pci_device_table); +MODULE_DEVICE_TABLE(pci, mt7996_hif_device_table); +MODULE_FIRMWARE(MT7996_FIRMWARE_WA); +MODULE_FIRMWARE(MT7996_FIRMWARE_WM); +MODULE_FIRMWARE(MT7996_ROM_PATCH); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h new file mode 100644 index 000000000000..794f61b93a46 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h @@ -0,0 +1,542 @@ +/* SPDX-License-Identifier: ISC */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef __MT7996_REGS_H +#define __MT7996_REGS_H + +struct __map { + u32 phys; + u32 mapped; + u32 size; +}; + +struct __base { + u32 band_base[__MT_MAX_BAND]; +}; + +/* used to differentiate between generations */ +struct mt7996_reg_desc { + const struct __base *base; + const struct __map *map; + u32 map_size; +}; + +enum base_rev { + WF_AGG_BASE, + WF_ARB_BASE, + WF_TMAC_BASE, + WF_RMAC_BASE, + WF_DMA_BASE, + WF_WTBLOFF_BASE, + WF_ETBF_BASE, + WF_LPON_BASE, + WF_MIB_BASE, + __MT_REG_BASE_MAX, +}; + +#define __BASE(_id, _band) (dev->reg.base[(_id)].band_base[(_band)]) + +#define MT_MCU_INT_EVENT 0x2108 +#define MT_MCU_INT_EVENT_DMA_STOPPED BIT(0) +#define MT_MCU_INT_EVENT_DMA_INIT BIT(1) +#define MT_MCU_INT_EVENT_RESET_DONE BIT(3) + +/* PLE */ +#define MT_PLE_BASE 0x820c0000 +#define MT_PLE(ofs) (MT_PLE_BASE + (ofs)) + +#define MT_FL_Q_EMPTY MT_PLE(0x360) +#define MT_FL_Q0_CTRL MT_PLE(0x3e0) +#define MT_FL_Q2_CTRL MT_PLE(0x3e8) +#define MT_FL_Q3_CTRL MT_PLE(0x3ec) + +#define MT_PLE_FREEPG_CNT MT_PLE(0x380) +#define MT_PLE_FREEPG_HEAD_TAIL MT_PLE(0x384) +#define MT_PLE_PG_HIF_GROUP MT_PLE(0x00c) +#define MT_PLE_HIF_PG_INFO MT_PLE(0x388) + +#define MT_PLE_AC_QEMPTY(ac, n) MT_PLE(0x600 + 0x80 * (ac) + ((n) << 2)) +#define MT_PLE_AMSDU_PACK_MSDU_CNT(n) MT_PLE(0x10e0 + ((n) << 2)) + +/* WF MDP TOP */ +#define MT_MDP_BASE 0x820cc000 +#define MT_MDP(ofs) (MT_MDP_BASE + (ofs)) + +#define MT_MDP_DCR2 MT_MDP(0x8e8) +#define MT_MDP_DCR2_RX_TRANS_SHORT BIT(2) + +/* TMAC: band 0(0x820e4000), band 1(0x820f4000), band 2(0x830e4000) */ +#define MT_WF_TMAC_BASE(_band) __BASE(WF_TMAC_BASE, (_band)) +#define MT_WF_TMAC(_band, ofs) (MT_WF_TMAC_BASE(_band) + (ofs)) + +#define MT_TMAC_TCR0(_band) MT_WF_TMAC(_band, 0) +#define MT_TMAC_TCR0_TX_BLINK GENMASK(7, 6) + +#define MT_TMAC_CDTR(_band) MT_WF_TMAC(_band, 0x0c8) +#define MT_TMAC_ODTR(_band) MT_WF_TMAC(_band, 0x0cc) +#define MT_TIMEOUT_VAL_PLCP GENMASK(15, 0) +#define MT_TIMEOUT_VAL_CCA GENMASK(31, 16) + +#define MT_TMAC_ICR0(_band) MT_WF_TMAC(_band, 0x014) +#define MT_IFS_EIFS_OFDM GENMASK(8, 0) +#define MT_IFS_RIFS GENMASK(14, 10) +#define MT_IFS_SIFS GENMASK(22, 16) +#define MT_IFS_SLOT GENMASK(30, 24) + +#define MT_TMAC_ICR1(_band) MT_WF_TMAC(_band, 0x018) +#define MT_IFS_EIFS_CCK GENMASK(8, 0) + +/* WF DMA TOP: band 0(0x820e7000), band 1(0x820f7000), band 2(0x830e7000) */ +#define MT_WF_DMA_BASE(_band) __BASE(WF_DMA_BASE, (_band)) +#define MT_WF_DMA(_band, ofs) (MT_WF_DMA_BASE(_band) + (ofs)) + +#define MT_DMA_DCR0(_band) MT_WF_DMA(_band, 0x000) +#define MT_DMA_DCR0_RXD_G5_EN BIT(23) + +#define MT_DMA_TCRF1(_band) MT_WF_DMA(_band, 0x054) +#define MT_DMA_TCRF1_QIDX GENMASK(15, 13) + +/* WTBLOFF TOP: band 0(0x820e9000), band 1(0x820f9000), band 2(0x830e9000) */ +#define MT_WTBLOFF_BASE(_band) __BASE(WF_WTBLOFF_BASE, (_band)) +#define MT_WTBLOFF(_band, ofs) (MT_WTBLOFF_BASE(_band) + (ofs)) + +#define MT_WTBLOFF_RSCR(_band) MT_WTBLOFF(_band, 0x008) +#define MT_WTBLOFF_RSCR_RCPI_MODE GENMASK(31, 30) +#define MT_WTBLOFF_RSCR_RCPI_PARAM GENMASK(25, 24) + +/* ETBF: band 0(0x820ea000), band 1(0x820fa000), band 2(0x830ea000) */ +#define MT_WF_ETBF_BASE(_band) __BASE(WF_ETBF_BASE, (_band)) +#define MT_WF_ETBF(_band, ofs) (MT_WF_ETBF_BASE(_band) + (ofs)) + +#define MT_ETBF_RX_FB_CONT(_band) MT_WF_ETBF(_band, 0x100) +#define MT_ETBF_RX_FB_BW GENMASK(10, 8) +#define MT_ETBF_RX_FB_NC GENMASK(7, 4) +#define MT_ETBF_RX_FB_NR GENMASK(3, 0) + +/* LPON: band 0(0x820eb000), band 1(0x820fb000), band 2(0x830eb000) */ +#define MT_WF_LPON_BASE(_band) __BASE(WF_LPON_BASE, (_band)) +#define MT_WF_LPON(_band, ofs) (MT_WF_LPON_BASE(_band) + (ofs)) + +#define MT_LPON_UTTR0(_band) MT_WF_LPON(_band, 0x360) +#define MT_LPON_UTTR1(_band) MT_WF_LPON(_band, 0x364) +#define MT_LPON_FRCR(_band) MT_WF_LPON(_band, 0x37c) + +#define MT_LPON_TCR(_band, n) MT_WF_LPON(_band, 0x0a8 + (((n) * 4) << 4)) +#define MT_LPON_TCR_SW_MODE GENMASK(1, 0) +#define MT_LPON_TCR_SW_WRITE BIT(0) +#define MT_LPON_TCR_SW_ADJUST BIT(1) +#define MT_LPON_TCR_SW_READ GENMASK(1, 0) + +/* MIB: band 0(0x820ed000), band 1(0x820fd000), band 2(0x830ed000)*/ +/* These counters are (mostly?) clear-on-read. So, some should not + * be read at all in case firmware is already reading them. These + * are commented with 'DNR' below. The DNR stats will be read by querying + * the firmware API for the appropriate message. For counters the driver + * does read, the driver should accumulate the counters. + */ +#define MT_WF_MIB_BASE(_band) __BASE(WF_MIB_BASE, (_band)) +#define MT_WF_MIB(_band, ofs) (MT_WF_MIB_BASE(_band) + (ofs)) + +#define MT_MIB_BSCR0(_band) MT_WF_MIB(_band, 0x9cc) +#define MT_MIB_BSCR1(_band) MT_WF_MIB(_band, 0x9d0) +#define MT_MIB_BSCR2(_band) MT_WF_MIB(_band, 0x9d4) +#define MT_MIB_BSCR3(_band) MT_WF_MIB(_band, 0x9d8) +#define MT_MIB_BSCR4(_band) MT_WF_MIB(_band, 0x9dc) +#define MT_MIB_BSCR5(_band) MT_WF_MIB(_band, 0x9e0) +#define MT_MIB_BSCR6(_band) MT_WF_MIB(_band, 0x9e4) +#define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, 0x9e8) +#define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, 0xa10) + +#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, 0x6c4) +#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, 0x6c8) +#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, 0x6d0) + +#define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, 0x7ac) +/* rx mpdu counter, full 32 bits */ +#define MT_MIB_RSCR31(_band) MT_WF_MIB(_band, 0x964) +#define MT_MIB_RSCR33(_band) MT_WF_MIB(_band, 0x96c) + +#define MT_MIB_SDR6(_band) MT_WF_MIB(_band, 0x020) +#define MT_MIB_SDR6_CHANNEL_IDL_CNT_MASK GENMASK(15, 0) + +#define MT_MIB_RVSR0(_band) MT_WF_MIB(_band, 0x720) + +#define MT_MIB_RSCR35(_band) MT_WF_MIB(_band, 0x974) +#define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, 0x978) + +/* tx ampdu cnt, full 32 bits */ +#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, 0x6b0) +#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, 0x6b8) + +/* counts all mpdus in ampdu, regardless of success */ +#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, 0x6bc) + +/* counts all successfully tx'd mpdus in ampdu */ +#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, 0x6c0) + +/* rx ampdu count, 32-bit */ +#define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, 0x954) + +/* rx ampdu bytes count, 32-bit */ +#define MT_MIB_RSCR28(_band) MT_WF_MIB(_band, 0x958) + +/* rx ampdu valid subframe count */ +#define MT_MIB_RSCR29(_band) MT_WF_MIB(_band, 0x95c) + +/* rx ampdu valid subframe bytes count, 32bits */ +#define MT_MIB_RSCR30(_band) MT_WF_MIB(_band, 0x960) + +/* remaining windows protected stats */ +#define MT_MIB_SDR27(_band) MT_WF_MIB(_band, 0x080) +#define MT_MIB_SDR27_TX_RWP_FAIL_CNT GENMASK(15, 0) + +#define MT_MIB_SDR28(_band) MT_WF_MIB(_band, 0x084) +#define MT_MIB_SDR28_TX_RWP_NEED_CNT GENMASK(15, 0) + +#define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, 0x724) + +/* rx blockack count, 32 bits */ +#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, 0x6b4) + +#define MT_MIB_BTSCR0(_band) MT_WF_MIB(_band, 0x5e0) +#define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, 0x788) +#define MT_MIB_BTSCR6(_band) MT_WF_MIB(_band, 0x798) + +#define MT_MIB_BFTFCR(_band) MT_WF_MIB(_band, 0x5d0) + +#define MT_TX_AGG_CNT(_band, n) MT_WF_MIB(_band, 0xa28 + ((n) << 2)) +#define MT_MIB_ARNG(_band, n) MT_WF_MIB(_band, 0x0b0 + ((n) << 2)) +#define MT_MIB_ARNCR_RANGE(val, n) (((val) >> ((n) << 4)) & GENMASK(9, 0)) + +/* UMIB */ +#define MT_WF_UMIB_BASE 0x820cd000 +#define MT_WF_UMIB(ofs) (MT_WF_UMIB_BASE + (ofs)) + +#define MT_UMIB_RPDCR(_band) (MT_WF_UMIB(0x594) + (_band) * 0x164) + +/* WTBLON TOP */ +#define MT_WTBLON_TOP_BASE 0x820d4000 +#define MT_WTBLON_TOP(ofs) (MT_WTBLON_TOP_BASE + (ofs)) +#define MT_WTBLON_TOP_WDUCR MT_WTBLON_TOP(0x370) +#define MT_WTBLON_TOP_WDUCR_GROUP GENMASK(4, 0) + +#define MT_WTBL_UPDATE MT_WTBLON_TOP(0x380) +#define MT_WTBL_UPDATE_WLAN_IDX GENMASK(11, 0) +#define MT_WTBL_UPDATE_ADM_COUNT_CLEAR BIT(14) +#define MT_WTBL_UPDATE_BUSY BIT(31) + +/* WTBL */ +#define MT_WTBL_BASE 0x820d8000 +#define MT_WTBL_LMAC_ID GENMASK(14, 8) +#define MT_WTBL_LMAC_DW GENMASK(7, 2) +#define MT_WTBL_LMAC_OFFS(_id, _dw) (MT_WTBL_BASE | \ + FIELD_PREP(MT_WTBL_LMAC_ID, _id) | \ + FIELD_PREP(MT_WTBL_LMAC_DW, _dw)) + +/* AGG: band 0(0x820e2000), band 1(0x820f2000), band 2(0x830e2000) */ +#define MT_WF_AGG_BASE(_band) __BASE(WF_AGG_BASE, (_band)) +#define MT_WF_AGG(_band, ofs) (MT_WF_AGG_BASE(_band) + (ofs)) + +#define MT_AGG_ACR0(_band) MT_WF_AGG(_band, 0x054) +#define MT_AGG_ACR_CFEND_RATE GENMASK(13, 0) + +/* ARB: band 0(0x820e3000), band 1(0x820f3000), band 2(0x830e3000) */ +#define MT_WF_ARB_BASE(_band) __BASE(WF_ARB_BASE, (_band)) +#define MT_WF_ARB(_band, ofs) (MT_WF_ARB_BASE(_band) + (ofs)) + +#define MT_ARB_SCR(_band) MT_WF_ARB(_band, 0x000) +#define MT_ARB_SCR_TX_DISABLE BIT(8) +#define MT_ARB_SCR_RX_DISABLE BIT(9) + +/* RMAC: band 0(0x820e5000), band 1(0x820f5000), band 2(0x830e5000), */ +#define MT_WF_RMAC_BASE(_band) __BASE(WF_RMAC_BASE, (_band)) +#define MT_WF_RMAC(_band, ofs) (MT_WF_RMAC_BASE(_band) + (ofs)) + +#define MT_WF_RFCR(_band) MT_WF_RMAC(_band, 0x000) +#define MT_WF_RFCR_DROP_STBC_MULTI BIT(0) +#define MT_WF_RFCR_DROP_FCSFAIL BIT(1) +#define MT_WF_RFCR_DROP_PROBEREQ BIT(4) +#define MT_WF_RFCR_DROP_MCAST BIT(5) +#define MT_WF_RFCR_DROP_BCAST BIT(6) +#define MT_WF_RFCR_DROP_MCAST_FILTERED BIT(7) +#define MT_WF_RFCR_DROP_A3_MAC BIT(8) +#define MT_WF_RFCR_DROP_A3_BSSID BIT(9) +#define MT_WF_RFCR_DROP_A2_BSSID BIT(10) +#define MT_WF_RFCR_DROP_OTHER_BEACON BIT(11) +#define MT_WF_RFCR_DROP_FRAME_REPORT BIT(12) +#define MT_WF_RFCR_DROP_CTL_RSV BIT(13) +#define MT_WF_RFCR_DROP_CTS BIT(14) +#define MT_WF_RFCR_DROP_RTS BIT(15) +#define MT_WF_RFCR_DROP_DUPLICATE BIT(16) +#define MT_WF_RFCR_DROP_OTHER_BSS BIT(17) +#define MT_WF_RFCR_DROP_OTHER_UC BIT(18) +#define MT_WF_RFCR_DROP_OTHER_TIM BIT(19) +#define MT_WF_RFCR_DROP_NDPA BIT(20) +#define MT_WF_RFCR_DROP_UNWANTED_CTL BIT(21) + +#define MT_WF_RFCR1(_band) MT_WF_RMAC(_band, 0x004) +#define MT_WF_RFCR1_DROP_ACK BIT(4) +#define MT_WF_RFCR1_DROP_BF_POLL BIT(5) +#define MT_WF_RFCR1_DROP_BA BIT(6) +#define MT_WF_RFCR1_DROP_CFEND BIT(7) +#define MT_WF_RFCR1_DROP_CFACK BIT(8) + +#define MT_WF_RMAC_MIB_AIRTIME0(_band) MT_WF_RMAC(_band, 0x0380) +#define MT_WF_RMAC_MIB_RXTIME_CLR BIT(31) +#define MT_WF_RMAC_MIB_ED_OFFSET GENMASK(20, 16) +#define MT_WF_RMAC_MIB_OBSS_BACKOFF GENMASK(15, 0) + +#define MT_WF_RMAC_MIB_AIRTIME1(_band) MT_WF_RMAC(_band, 0x0384) +#define MT_WF_RMAC_MIB_NONQOSD_BACKOFF GENMASK(31, 16) + +#define MT_WF_RMAC_MIB_AIRTIME3(_band) MT_WF_RMAC(_band, 0x038c) +#define MT_WF_RMAC_MIB_QOS01_BACKOFF GENMASK(31, 0) + +#define MT_WF_RMAC_MIB_AIRTIME4(_band) MT_WF_RMAC(_band, 0x0390) +#define MT_WF_RMAC_MIB_QOS23_BACKOFF GENMASK(31, 0) + +#define MT_WF_RMAC_RSVD0(_band) MT_WF_RMAC(_band, 0x03e0) +#define MT_WF_RMAC_RSVD0_EIFS_CLR BIT(21) + +/* WFDMA0 */ +#define MT_WFDMA0_BASE 0xd4000 +#define MT_WFDMA0(ofs) (MT_WFDMA0_BASE + (ofs)) + +#define MT_WFDMA0_RST MT_WFDMA0(0x100) +#define MT_WFDMA0_RST_LOGIC_RST BIT(4) +#define MT_WFDMA0_RST_DMASHDL_ALL_RST BIT(5) + +#define MT_WFDMA0_BUSY_ENA MT_WFDMA0(0x13c) +#define MT_WFDMA0_BUSY_ENA_TX_FIFO0 BIT(0) +#define MT_WFDMA0_BUSY_ENA_TX_FIFO1 BIT(1) +#define MT_WFDMA0_BUSY_ENA_RX_FIFO BIT(2) + +#define MT_WFDMA0_RX_INT_PCIE_SEL MT_WFDMA0(0x154) +#define MT_WFDMA0_RX_INT_SEL_RING3 BIT(3) + +#define MT_WFDMA0_GLO_CFG MT_WFDMA0(0x208) +#define MT_WFDMA0_GLO_CFG_TX_DMA_EN BIT(0) +#define MT_WFDMA0_GLO_CFG_RX_DMA_EN BIT(2) +#define MT_WFDMA0_GLO_CFG_OMIT_TX_INFO BIT(28) +#define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO BIT(27) +#define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2 BIT(21) + +#define WF_WFDMA0_GLO_CFG_EXT0 MT_WFDMA0(0x2b0) +#define WF_WFDMA0_GLO_CFG_EXT0_RX_WB_RXD BIT(18) +#define WF_WFDMA0_GLO_CFG_EXT0_WED_MERGE_MODE BIT(14) + +#define WF_WFDMA0_GLO_CFG_EXT1 MT_WFDMA0(0x2b4) +#define WF_WFDMA0_GLO_CFG_EXT1_CALC_MODE BIT(31) +#define WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE BIT(28) + +#define MT_WFDMA0_RST_DTX_PTR MT_WFDMA0(0x20c) +#define MT_WFDMA0_PRI_DLY_INT_CFG0 MT_WFDMA0(0x2f0) +#define MT_WFDMA0_PRI_DLY_INT_CFG1 MT_WFDMA0(0x2f4) +#define MT_WFDMA0_PRI_DLY_INT_CFG2 MT_WFDMA0(0x2f8) + +/* WFDMA1 */ +#define MT_WFDMA1_BASE 0xd5000 + +/* WFDMA CSR */ +#define MT_WFDMA_EXT_CSR_BASE 0xd7000 +#define MT_WFDMA_EXT_CSR(ofs) (MT_WFDMA_EXT_CSR_BASE + (ofs)) + +#define MT_WFDMA_HOST_CONFIG MT_WFDMA_EXT_CSR(0x30) +#define MT_WFDMA_HOST_CONFIG_PDMA_BAND BIT(0) + +#define MT_WFDMA_EXT_CSR_HIF_MISC MT_WFDMA_EXT_CSR(0x44) +#define MT_WFDMA_EXT_CSR_HIF_MISC_BUSY BIT(0) + +#define MT_PCIE_RECOG_ID 0xd7090 +#define MT_PCIE_RECOG_ID_MASK GENMASK(30, 0) +#define MT_PCIE_RECOG_ID_SEM BIT(31) + +/* WFDMA0 PCIE1 */ +#define MT_WFDMA0_PCIE1_BASE 0xd8000 +#define MT_WFDMA0_PCIE1(ofs) (MT_WFDMA0_PCIE1_BASE + (ofs)) + +#define MT_WFDMA0_PCIE1_BUSY_ENA MT_WFDMA0_PCIE1(0x13c) +#define MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO0 BIT(0) +#define MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO1 BIT(1) +#define MT_WFDMA0_PCIE1_BUSY_ENA_RX_FIFO BIT(2) + +/* WFDMA COMMON */ +#define __RXQ(q) ((q) + __MT_MCUQ_MAX) +#define __TXQ(q) (__RXQ(q) + __MT_RXQ_MAX) + +#define MT_Q_ID(q) (dev->q_id[(q)]) +#define MT_Q_BASE(q) ((dev->q_wfdma_mask >> (q)) & 0x1 ? \ + MT_WFDMA1_BASE : MT_WFDMA0_BASE) + +#define MT_MCUQ_ID(q) MT_Q_ID(q) +#define MT_TXQ_ID(q) MT_Q_ID(__TXQ(q)) +#define MT_RXQ_ID(q) MT_Q_ID(__RXQ(q)) + +#define MT_MCUQ_RING_BASE(q) (MT_Q_BASE(q) + 0x300) +#define MT_TXQ_RING_BASE(q) (MT_Q_BASE(__TXQ(q)) + 0x300) +#define MT_RXQ_RING_BASE(q) (MT_Q_BASE(__RXQ(q)) + 0x500) + +#define MT_MCUQ_EXT_CTRL(q) (MT_Q_BASE(q) + 0x600 + \ + MT_MCUQ_ID(q) * 0x4) +#define MT_RXQ_BAND1_CTRL(q) (MT_Q_BASE(__RXQ(q)) + 0x680 + \ + MT_RXQ_ID(q) * 0x4) +#define MT_TXQ_EXT_CTRL(q) (MT_Q_BASE(__TXQ(q)) + 0x600 + \ + MT_TXQ_ID(q) * 0x4) + +#define MT_INT_SOURCE_CSR MT_WFDMA0(0x200) +#define MT_INT_MASK_CSR MT_WFDMA0(0x204) + +#define MT_INT1_SOURCE_CSR MT_WFDMA0_PCIE1(0x200) +#define MT_INT1_MASK_CSR MT_WFDMA0_PCIE1(0x204) + +#define MT_INT_RX_DONE_BAND0 BIT(12) +#define MT_INT_RX_DONE_BAND1 BIT(12) +#define MT_INT_RX_DONE_BAND2 BIT(13) +#define MT_INT_RX_DONE_WM BIT(0) +#define MT_INT_RX_DONE_WA BIT(1) +#define MT_INT_RX_DONE_WA_MAIN BIT(2) +#define MT_INT_RX_DONE_WA_EXT BIT(2) +#define MT_INT_RX_DONE_WA_TRI BIT(3) +#define MT_INT_RX_TXFREE_MAIN BIT(17) +#define MT_INT_RX_TXFREE_TRI BIT(15) +#define MT_INT_MCU_CMD BIT(29) + +#define MT_INT_RX(q) (dev->q_int_mask[__RXQ(q)]) +#define MT_INT_TX_MCU(q) (dev->q_int_mask[(q)]) + +#define MT_INT_RX_DONE_MCU (MT_INT_RX(MT_RXQ_MCU) | \ + MT_INT_RX(MT_RXQ_MCU_WA)) + +#define MT_INT_BAND0_RX_DONE (MT_INT_RX(MT_RXQ_MAIN) | \ + MT_INT_RX(MT_RXQ_MAIN_WA)) + +#define MT_INT_BAND1_RX_DONE (MT_INT_RX(MT_RXQ_BAND1) | \ + MT_INT_RX(MT_RXQ_BAND1_WA) | \ + MT_INT_RX(MT_RXQ_MAIN_WA)) + +#define MT_INT_BAND2_RX_DONE (MT_INT_RX(MT_RXQ_BAND2) | \ + MT_INT_RX(MT_RXQ_BAND2_WA) | \ + MT_INT_RX(MT_RXQ_MAIN_WA)) + +#define MT_INT_RX_DONE_ALL (MT_INT_RX_DONE_MCU | \ + MT_INT_BAND0_RX_DONE | \ + MT_INT_BAND1_RX_DONE | \ + MT_INT_BAND2_RX_DONE) + +#define MT_INT_TX_DONE_FWDL BIT(26) +#define MT_INT_TX_DONE_MCU_WM BIT(27) +#define MT_INT_TX_DONE_MCU_WA BIT(22) +#define MT_INT_TX_DONE_BAND0 BIT(30) +#define MT_INT_TX_DONE_BAND1 BIT(31) +#define MT_INT_TX_DONE_BAND2 BIT(15) + +#define MT_INT_TX_DONE_MCU (MT_INT_TX_MCU(MT_MCUQ_WA) | \ + MT_INT_TX_MCU(MT_MCUQ_WM) | \ + MT_INT_TX_MCU(MT_MCUQ_FWDL)) + +#define MT_MCU_CMD MT_WFDMA0(0x1f0) +#define MT_MCU_CMD_STOP_DMA BIT(2) +#define MT_MCU_CMD_RESET_DONE BIT(3) +#define MT_MCU_CMD_RECOVERY_DONE BIT(4) +#define MT_MCU_CMD_NORMAL_STATE BIT(5) +#define MT_MCU_CMD_ERROR_MASK GENMASK(5, 1) + +/* l1/l2 remap */ +#define MT_HIF_REMAP_L1 0x155024 +#define MT_HIF_REMAP_L1_MASK GENMASK(31, 16) +#define MT_HIF_REMAP_L1_OFFSET GENMASK(15, 0) +#define MT_HIF_REMAP_L1_BASE GENMASK(31, 16) +#define MT_HIF_REMAP_BASE_L1 0x130000 + +#define MT_HIF_REMAP_L2 0x1b4 +#define MT_HIF_REMAP_L2_MASK GENMASK(19, 0) +#define MT_HIF_REMAP_L2_OFFSET GENMASK(11, 0) +#define MT_HIF_REMAP_L2_BASE GENMASK(31, 12) +#define MT_HIF_REMAP_BASE_L2 0x1000 + +#define MT_INFRA_BASE 0x18000000 +#define MT_WFSYS0_PHY_START 0x18400000 +#define MT_WFSYS1_PHY_START 0x18800000 +#define MT_WFSYS1_PHY_END 0x18bfffff +#define MT_CBTOP1_PHY_START 0x70000000 +#define MT_CBTOP1_PHY_END 0x77ffffff +#define MT_CBTOP2_PHY_START 0xf0000000 +#define MT_CBTOP2_PHY_END 0xffffffff +#define MT_INFRA_MCU_START 0x7c000000 +#define MT_INFRA_MCU_END 0x7c3fffff + +/* FW MODE SYNC */ +#define MT_SWDEF_MODE 0x9143c +#define MT_SWDEF_NORMAL_MODE 0 + +/* LED */ +#define MT_LED_TOP_BASE 0x18013000 +#define MT_LED_PHYS(_n) (MT_LED_TOP_BASE + (_n)) + +#define MT_LED_CTRL(_n) MT_LED_PHYS(0x00 + ((_n) * 4)) +#define MT_LED_CTRL_KICK BIT(7) +#define MT_LED_CTRL_BLINK_MODE BIT(2) +#define MT_LED_CTRL_POLARITY BIT(1) + +#define MT_LED_TX_BLINK(_n) MT_LED_PHYS(0x10 + ((_n) * 4)) +#define MT_LED_TX_BLINK_ON_MASK GENMASK(7, 0) +#define MT_LED_TX_BLINK_OFF_MASK GENMASK(15, 8) + +#define MT_LED_EN(_n) MT_LED_PHYS(0x40 + ((_n) * 4)) + +#define MT_LED_GPIO_MUX2 0x70005058 /* GPIO 18 */ +#define MT_LED_GPIO_MUX3 0x7000505C /* GPIO 26 */ +#define MT_LED_GPIO_SEL_MASK GENMASK(11, 8) + +/* MT TOP */ +#define MT_TOP_BASE 0xe0000 +#define MT_TOP(ofs) (MT_TOP_BASE + (ofs)) + +#define MT_TOP_LPCR_HOST_BAND(_band) MT_TOP(0x10 + ((_band) * 0x10)) +#define MT_TOP_LPCR_HOST_FW_OWN BIT(0) +#define MT_TOP_LPCR_HOST_DRV_OWN BIT(1) +#define MT_TOP_LPCR_HOST_FW_OWN_STAT BIT(2) + +#define MT_TOP_LPCR_HOST_BAND_IRQ_STAT(_band) MT_TOP(0x14 + ((_band) * 0x10)) +#define MT_TOP_LPCR_HOST_BAND_STAT BIT(0) + +#define MT_TOP_MISC MT_TOP(0xf0) +#define MT_TOP_MISC_FW_STATE GENMASK(2, 0) + +#define MT_HW_REV 0x70010204 +#define MT_WF_SUBSYS_RST 0x70002600 + +/* PCIE MAC */ +#define MT_PCIE_MAC_BASE 0x74030000 +#define MT_PCIE_MAC(ofs) (MT_PCIE_MAC_BASE + (ofs)) +#define MT_PCIE_MAC_INT_ENABLE MT_PCIE_MAC(0x188) + +#define MT_PCIE1_MAC_BASE 0x74090000 +#define MT_PCIE1_MAC(ofs) (MT_PCIE1_MAC_BASE + (ofs)) + +#define MT_PCIE1_MAC_INT_ENABLE MT_PCIE1_MAC(0x188) + +/* PHYRX CTRL */ +#define MT_WF_PHYRX_BAND_BASE 0x83080000 +#define MT_WF_PHYRX_BAND(_band, ofs) (MT_WF_PHYRX_BAND_BASE + \ + ((_band) << 20) + (ofs)) + +#define MT_WF_PHYRX_BAND_RX_CTRL1(_band) MT_WF_PHYRX_BAND(_band, 0x2004) +#define MT_WF_PHYRX_BAND_RX_CTRL1_IPI_EN GENMASK(2, 0) +#define MT_WF_PHYRX_BAND_RX_CTRL1_STSCNT_EN GENMASK(11, 9) + +/* PHYRX CSD */ +#define MT_WF_PHYRX_CSD_BASE 0x83000000 +#define MT_WF_PHYRX_CSD(_band, _wf, ofs) (MT_WF_PHYRX_CSD_BASE + \ + ((_band) << 20) + \ + ((_wf) << 16) + (ofs)) +#define MT_WF_PHYRX_CSD_IRPI(_band, _wf) MT_WF_PHYRX_CSD(_band, _wf, 0x1000) + +/* PHYRX CSD BAND */ +#define MT_WF_PHYRX_CSD_BAND_RXTD12(_band) MT_WF_PHYRX_BAND(_band, 0x8230) +#define MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR_ONLY BIT(18) +#define MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR BIT(29) + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 0ec308f99af5..228bc7d45011 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -395,7 +395,7 @@ mt76s_process_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) if (!e || !e->skb) break; - dev->drv->rx_skb(dev, MT_RXQ_MAIN, e->skb); + dev->drv->rx_skb(dev, MT_RXQ_MAIN, e->skb, NULL); e->skb = NULL; nframes++; } diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index 6c054850363f..24568b98ed9d 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -756,6 +756,23 @@ int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) } EXPORT_SYMBOL_GPL(mt76_token_consume); +int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, + struct mt76_txwi_cache *t, dma_addr_t phys) +{ + int token; + + spin_lock_bh(&dev->rx_token_lock); + token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size, + GFP_ATOMIC); + spin_unlock_bh(&dev->rx_token_lock); + + t->ptr = ptr; + t->dma_addr = phys; + + return token; +} +EXPORT_SYMBOL_GPL(mt76_rx_token_consume); + struct mt76_txwi_cache * mt76_token_release(struct mt76_dev *dev, int token, bool *wake) { @@ -784,3 +801,16 @@ mt76_token_release(struct mt76_dev *dev, int token, bool *wake) return txwi; } EXPORT_SYMBOL_GPL(mt76_token_release); + +struct mt76_txwi_cache * +mt76_rx_token_release(struct mt76_dev *dev, int token) +{ + struct mt76_txwi_cache *t; + + spin_lock_bh(&dev->rx_token_lock); + t = idr_remove(&dev->rx_token, token); + spin_unlock_bh(&dev->rx_token_lock); + + return t; +} +EXPORT_SYMBOL_GPL(mt76_rx_token_release); diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c index 4c4033bb1bb3..3e281715fcd4 100644 --- a/drivers/net/wireless/mediatek/mt76/usb.c +++ b/drivers/net/wireless/mediatek/mt76/usb.c @@ -547,7 +547,7 @@ mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb, len -= data_len; nsgs++; } - dev->drv->rx_skb(dev, MT_RXQ_MAIN, skb); + dev->drv->rx_skb(dev, MT_RXQ_MAIN, skb, NULL); return nsgs; } @@ -766,6 +766,9 @@ static void mt76u_status_worker(struct mt76_worker *w) struct mt76_queue *q; int i; + if (!test_bit(MT76_STATE_RUNNING, &dev->phy.state)) + return; + for (i = 0; i < IEEE80211_NUM_ACS; i++) { q = dev->phy.q_tx[i]; if (!q) @@ -785,11 +788,11 @@ static void mt76u_status_worker(struct mt76_worker *w) wake_up(&dev->tx_wait); mt76_worker_schedule(&dev->tx_worker); - - if (dev->drv->tx_status_data && - !test_and_set_bit(MT76_READING_STATS, &dev->phy.state)) - queue_work(dev->wq, &dev->usb.stat_work); } + + if (dev->drv->tx_status_data && + !test_and_set_bit(MT76_READING_STATS, &dev->phy.state)) + queue_work(dev->wq, &dev->usb.stat_work); } static void mt76u_tx_status_data(struct work_struct *work) diff --git a/drivers/net/wireless/mediatek/mt76/util.h b/drivers/net/wireless/mediatek/mt76/util.h index 49c52d781f40..260965dde94c 100644 --- a/drivers/net/wireless/mediatek/mt76/util.h +++ b/drivers/net/wireless/mediatek/mt76/util.h @@ -29,12 +29,6 @@ enum { int mt76_wcid_alloc(u32 *mask, int size); -static inline bool -mt76_wcid_mask_test(u32 *mask, int idx) -{ - return mask[idx / 32] & BIT(idx % 32); -} - static inline void mt76_wcid_mask_set(u32 *mask, int idx) { diff --git a/drivers/net/wireless/purelifi/plfxlc/usb.c b/drivers/net/wireless/purelifi/plfxlc/usb.c index 39e54b3787d6..76d0a778636a 100644 --- a/drivers/net/wireless/purelifi/plfxlc/usb.c +++ b/drivers/net/wireless/purelifi/plfxlc/usb.c @@ -247,6 +247,7 @@ error: for (i = 0; i < RX_URBS_COUNT; i++) free_rx_urb(urbs[i]); } + kfree(urbs); return r; } diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index e8fcd531c437..28f136064297 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -4645,7 +4645,6 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, if (sta->deflink.ht_cap.cap & (IEEE80211_HT_CAP_SGI_40 | IEEE80211_HT_CAP_SGI_20)) sgi = 1; - rcu_read_unlock(); highest_rate = fls(ramask) - 1; if (highest_rate < DESC_RATE_MCS0) { @@ -4670,6 +4669,7 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, else rarpt->txrate.bw = RATE_INFO_BW_20; } + rcu_read_unlock(); bit_rate = cfg80211_calculate_bitrate(&rarpt->txrate); rarpt->bit_rate = bit_rate; rarpt->desc_rate = highest_rate; diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c index 0b5f903c0f36..5e760c884f89 100644 --- a/drivers/net/wireless/realtek/rtw88/fw.c +++ b/drivers/net/wireless/realtek/rtw88/fw.c @@ -311,10 +311,10 @@ EXPORT_SYMBOL(rtw_fw_c2h_cmd_isr); static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, u8 *h2c) { + struct rtw_h2c_cmd *h2c_cmd = (struct rtw_h2c_cmd *)h2c; u8 box; u8 box_state; u32 box_reg, box_ex_reg; - int idx; int ret; rtw_dbg(rtwdev, RTW_DBG_FW, @@ -356,10 +356,8 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, goto out; } - for (idx = 0; idx < 4; idx++) - rtw_write8(rtwdev, box_reg + idx, h2c[idx]); - for (idx = 0; idx < 4; idx++) - rtw_write8(rtwdev, box_ex_reg + idx, h2c[idx + 4]); + rtw_write32(rtwdev, box_ex_reg, le32_to_cpu(h2c_cmd->msg_ext)); + rtw_write32(rtwdev, box_reg, le32_to_cpu(h2c_cmd->msg)); if (++rtwdev->h2c.last_box_num >= 4) rtwdev->h2c.last_box_num = 0; @@ -824,6 +822,16 @@ void rtw_fw_set_nlo_info(struct rtw_dev *rtwdev, bool enable) rtw_fw_send_h2c_command(rtwdev, h2c_pkt); } +void rtw_fw_set_recover_bt_device(struct rtw_dev *rtwdev) +{ + u8 h2c_pkt[H2C_PKT_SIZE] = {0}; + + SET_H2C_CMD_ID_CLASS(h2c_pkt, H2C_CMD_RECOVER_BT_DEV); + SET_RECOVER_BT_DEV_EN(h2c_pkt, 1); + + rtw_fw_send_h2c_command(rtwdev, h2c_pkt); +} + void rtw_fw_set_pg_info(struct rtw_dev *rtwdev) { struct rtw_lps_conf *conf = &rtwdev->lps_conf; diff --git a/drivers/net/wireless/realtek/rtw88/fw.h b/drivers/net/wireless/realtek/rtw88/fw.h index a5a965803a3c..0a386e6d6e0d 100644 --- a/drivers/net/wireless/realtek/rtw88/fw.h +++ b/drivers/net/wireless/realtek/rtw88/fw.h @@ -81,6 +81,11 @@ struct rtw_c2h_adaptivity { u8 option; } __packed; +struct rtw_h2c_cmd { + __le32 msg; + __le32 msg_ext; +} __packed; + enum rtw_rsvd_packet_type { RSVD_BEACON, RSVD_DUMMY, @@ -550,6 +555,8 @@ static inline void rtw_h2c_pkt_set_header(u8 *h2c_pkt, u8 sub_id) #define H2C_CMD_AOAC_GLOBAL_INFO 0x82 #define H2C_CMD_NLO_INFO 0x8C +#define H2C_CMD_RECOVER_BT_DEV 0xD1 + #define SET_H2C_CMD_ID_CLASS(h2c_pkt, value) \ le32p_replace_bits((__le32 *)(h2c_pkt) + 0x00, value, GENMASK(7, 0)) @@ -749,6 +756,9 @@ static inline void rtw_h2c_pkt_set_header(u8 *h2c_pkt, u8 sub_id) #define SET_NLO_LOC_NLO_INFO(h2c_pkt, value) \ le32p_replace_bits((__le32 *)(h2c_pkt) + 0x00, value, GENMASK(23, 16)) +#define SET_RECOVER_BT_DEV_EN(h2c_pkt, value) \ + le32p_replace_bits((__le32 *)(h2c_pkt) + 0x00, value, BIT(8)) + #define GET_FW_DUMP_LEN(_header) \ le32_get_bits(*((__le32 *)(_header) + 0x00), GENMASK(15, 0)) #define GET_FW_DUMP_SEQ(_header) \ @@ -838,6 +848,7 @@ void rtw_fw_set_aoac_global_info_cmd(struct rtw_dev *rtwdev, u8 group_key_enc); void rtw_fw_set_nlo_info(struct rtw_dev *rtwdev, bool enable); +void rtw_fw_set_recover_bt_device(struct rtw_dev *rtwdev); void rtw_fw_update_pkt_probe_req(struct rtw_dev *rtwdev, struct cfg80211_ssid *ssid); void rtw_fw_channel_switch(struct rtw_dev *rtwdev, bool enable); diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c index 52076e89d59a..c7e64f7036ac 100644 --- a/drivers/net/wireless/realtek/rtw88/mac.c +++ b/drivers/net/wireless/realtek/rtw88/mac.c @@ -906,7 +906,8 @@ out: return ret; } -int rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw) +static +int _rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw) { if (rtw_chip_wcpu_11n(rtwdev)) return __rtw_download_firmware_legacy(rtwdev, fw); @@ -914,6 +915,21 @@ int rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw) return __rtw_download_firmware(rtwdev, fw); } +int rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw) +{ + int ret; + + ret = _rtw_download_firmware(rtwdev, fw); + if (ret) + return ret; + + if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE && + rtwdev->chip->id == RTW_CHIP_TYPE_8821C) + rtw_fw_set_recover_bt_device(rtwdev); + + return 0; +} + static u32 get_priority_queues(struct rtw_dev *rtwdev, u32 queues) { const struct rtw_rqpn *rqpn = rtwdev->fifo.rqpn; diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c index a4f61c2f6512..90596806bc93 100644 --- a/drivers/net/wireless/realtek/rtw89/chan.c +++ b/drivers/net/wireless/realtek/rtw89/chan.c @@ -4,6 +4,7 @@ #include "chan.h" #include "debug.h" +#include "util.h" static enum rtw89_subband rtw89_get_subband_type(enum rtw89_band band, u8 center_chan) @@ -108,8 +109,8 @@ bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev, const struct rtw89_chan *new) { struct rtw89_hal *hal = &rtwdev->hal; - struct rtw89_chan *chan = &hal->chan[idx]; - struct rtw89_chan_rcd *rcd = &hal->chan_rcd[idx]; + struct rtw89_chan *chan = &hal->sub[idx].chan; + struct rtw89_chan_rcd *rcd = &hal->sub[idx].rcd; bool band_changed; rcd->prev_primary_channel = chan->primary_channel; @@ -127,7 +128,7 @@ static void __rtw89_config_entity_chandef(struct rtw89_dev *rtwdev, { struct rtw89_hal *hal = &rtwdev->hal; - hal->chandef[idx] = *chandef; + hal->sub[idx].chandef = *chandef; if (from_stack) set_bit(idx, hal->entity_map); @@ -195,6 +196,7 @@ int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev, rtw89_config_entity_chandef(rtwdev, idx, &ctx->def); rtw89_set_channel(rtwdev); cfg->idx = idx; + hal->sub[idx].cfg = cfg; return 0; } @@ -203,8 +205,34 @@ void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev, { struct rtw89_hal *hal = &rtwdev->hal; struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv; + struct rtw89_vif *rtwvif; + u8 drop, roll; - clear_bit(cfg->idx, hal->entity_map); + drop = cfg->idx; + if (drop != RTW89_SUB_ENTITY_0) + goto out; + + roll = find_next_bit(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY, drop + 1); + + /* Follow rtw89_config_default_chandef() when rtw89_entity_recalc(). */ + if (roll == NUM_OF_RTW89_SUB_ENTITY) + goto out; + + /* RTW89_SUB_ENTITY_0 is going to release, and another exists. + * Make another roll down to RTW89_SUB_ENTITY_0 to replace. + */ + hal->sub[roll].cfg->idx = RTW89_SUB_ENTITY_0; + hal->sub[RTW89_SUB_ENTITY_0] = hal->sub[roll]; + + rtw89_for_each_rtwvif(rtwdev, rtwvif) { + if (rtwvif->sub_entity_idx == roll) + rtwvif->sub_entity_idx = RTW89_SUB_ENTITY_0; + } + + drop = roll; + +out: + clear_bit(drop, hal->entity_map); rtw89_set_channel(rtwdev); } @@ -225,6 +253,9 @@ int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, struct ieee80211_chanctx_conf *ctx) { + struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv; + + rtwvif->sub_entity_idx = cfg->idx; return 0; } @@ -232,4 +263,5 @@ void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, struct ieee80211_chanctx_conf *ctx) { + rtwvif->sub_entity_idx = RTW89_SUB_ENTITY_0; } diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index f30aadc41f2b..5ab95250755d 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -1196,7 +1196,11 @@ static void rtw89_core_parse_phy_status_ie01(struct rtw89_dev *rtwdev, u8 *addr, if (phy_ppdu->rate < RTW89_HW_RATE_OFDM6) return; /* sign conversion for S(12,2) */ - cfo = sign_extend32(RTW89_GET_PHY_STS_IE01_CFO(addr), 11); + if (rtwdev->chip->cfo_src_fd) + cfo = sign_extend32(RTW89_GET_PHY_STS_IE01_FD_CFO(addr), 11); + else + cfo = sign_extend32(RTW89_GET_PHY_STS_IE01_PREMB_CFO(addr), 11); + rtw89_phy_cfo_parse(rtwdev, cfo, phy_ppdu); } @@ -1401,6 +1405,9 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; const u8 *bssid = iter_data->bssid; + if (!vif->bss_conf.bssid) + return; + if (ieee80211_is_trigger(hdr->frame_control)) { rtw89_stats_trigger_frame(rtwdev, vif, skb); return; @@ -1473,6 +1480,27 @@ static void rtw89_core_hw_to_sband_rate(struct ieee80211_rx_status *rx_status) rx_status->rate_idx -= 4; } +static void rtw89_core_update_radiotap(struct rtw89_dev *rtwdev, + struct sk_buff *skb, + struct ieee80211_rx_status *rx_status) +{ + static const struct ieee80211_radiotap_he known_he = { + .data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN | + IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN), + .data2 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_GI_KNOWN), + }; + struct ieee80211_radiotap_he *he; + + if (!(rtwdev->hw->conf.flags & IEEE80211_CONF_MONITOR)) + return; + + if (rx_status->encoding == RX_ENC_HE) { + rx_status->flag |= RX_FLAG_RADIOTAP_HE; + he = skb_push(skb, sizeof(*he)); + *he = known_he; + } +} + static void rtw89_core_rx_to_mac80211(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct rtw89_rx_desc_info *desc_info, @@ -1487,6 +1515,7 @@ static void rtw89_core_rx_to_mac80211(struct rtw89_dev *rtwdev, rtw89_core_hw_to_sband_rate(rx_status); rtw89_core_rx_stats(rtwdev, phy_ppdu, desc_info, skb_ppdu); + rtw89_core_update_radiotap(rtwdev, skb_ppdu, rx_status); /* In low power mode, it does RX in thread context. */ local_bh_disable(); ieee80211_rx_napi(rtwdev->hw, NULL, skb_ppdu, napi); @@ -2233,6 +2262,7 @@ static void rtw89_track_work(struct work_struct *work) rtw89_phy_ra_update(rtwdev); rtw89_phy_cfo_track(rtwdev); rtw89_phy_tx_path_div_track(rtwdev); + rtw89_phy_ul_tb_ctrl_track(rtwdev); if (rtwdev->lps_enabled && !rtwdev->btc.lps) rtw89_enter_lps_track(rtwdev); @@ -2381,6 +2411,8 @@ void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc) rtwvif->self_role = RTW89_SELF_ROLE_CLIENT; rtwvif->addr_cam.sec_ent_mode = RTW89_ADDR_CAM_SEC_NORMAL; break; + case NL80211_IFTYPE_MONITOR: + break; default: WARN_ON(1); break; @@ -2556,6 +2588,7 @@ int rtw89_core_sta_assoc(struct rtw89_dev *rtwdev, rtw89_btc_ntfy_role_info(rtwdev, rtwvif, rtwsta, BTC_ROLE_MSTS_STA_CONN_END); rtw89_core_get_no_ul_ofdma_htc(rtwdev, &rtwsta->htc_template); + rtw89_phy_ul_tb_assoc(rtwdev, rtwvif); } return ret; @@ -2941,6 +2974,41 @@ void rtw89_core_update_beacon_work(struct work_struct *work) mutex_unlock(&rtwdev->mutex); } +int rtw89_wait_for_cond(struct rtw89_wait_info *wait, unsigned int cond) +{ + struct completion *cmpl = &wait->completion; + unsigned long timeout; + unsigned int cur; + + cur = atomic_cmpxchg(&wait->cond, RTW89_WAIT_COND_IDLE, cond); + if (cur != RTW89_WAIT_COND_IDLE) + return -EBUSY; + + timeout = wait_for_completion_timeout(cmpl, RTW89_WAIT_FOR_COND_TIMEOUT); + if (timeout == 0) { + atomic_set(&wait->cond, RTW89_WAIT_COND_IDLE); + return -ETIMEDOUT; + } + + if (wait->data.err) + return -EFAULT; + + return 0; +} + +void rtw89_complete_cond(struct rtw89_wait_info *wait, unsigned int cond, + const struct rtw89_completion_data *data) +{ + unsigned int cur; + + cur = atomic_cmpxchg(&wait->cond, cond, RTW89_WAIT_COND_IDLE); + if (cur != cond) + return; + + wait->data = *data; + complete(&wait->completion); +} + int rtw89_core_start(struct rtw89_dev *rtwdev) { int ret; @@ -3062,6 +3130,8 @@ int rtw89_core_init(struct rtw89_dev *rtwdev) mutex_init(&rtwdev->rf_mutex); rtwdev->total_sta_assoc = 0; + rtw89_init_wait(&rtwdev->mcc.wait); + INIT_WORK(&rtwdev->c2h_work, rtw89_fw_c2h_work); INIT_WORK(&rtwdev->ips_work, rtw89_ips_work); skb_queue_head_init(&rtwdev->c2h_queue); @@ -3260,6 +3330,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev) ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS); ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); + ieee80211_hw_set(hw, WANT_MONITOR_VIF); hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) | BIT(NL80211_IFTYPE_AP) | diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index b60de6662548..2badb96d2ae3 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -35,6 +35,7 @@ extern const struct ieee80211_ops rtw89_ops; #define RSSI_FACTOR 1 #define RTW89_RSSI_RAW_TO_DBM(rssi) ((s8)((rssi) >> RSSI_FACTOR) - MAX_RSSI) #define RTW89_TX_DIV_RSSI_RAW_TH (2 << RSSI_FACTOR) +#define RTW89_RADIOTAP_ROOM ALIGN(sizeof(struct ieee80211_radiotap_he), 64) #define RTW89_HTC_MASK_VARIANT GENMASK(1, 0) #define RTW89_HTC_VARIANT_HE 3 @@ -2239,6 +2240,8 @@ struct rtw89_phy_rate_pattern { struct rtw89_vif { struct list_head list; struct rtw89_dev *rtwdev; + enum rtw89_sub_entity_idx sub_entity_idx; + u8 mac_id; u8 port; u8 mac_addr[ETH_ALEN]; @@ -2261,6 +2264,8 @@ struct rtw89_vif { bool wowlan_magic; bool is_hesta; bool last_a_ctrl; + bool dyn_tb_bedge_en; + u8 def_tri_idx; struct work_struct update_beacon_work; struct rtw89_addr_cam_entry addr_cam; struct rtw89_bssid_cam_entry bssid_cam; @@ -2646,6 +2651,11 @@ struct rtw89_dig_regs { struct rtw89_reg_def p1_s20_pagcugc_en; }; +struct rtw89_phy_ul_tb_info { + bool dyn_tb_tri_en; + u8 def_if_bandedge; +}; + struct rtw89_chip_info { enum rtw89_core_chip_id chip_id; const struct rtw89_chip_ops *ops; @@ -2663,6 +2673,7 @@ struct rtw89_chip_info { u8 support_chanctx_num; u8 support_bands; bool support_bw160; + bool support_ul_tb_ctrl; bool hw_sec_hdr; u8 rf_path_num; u8 tx_nss; @@ -2755,6 +2766,7 @@ struct rtw89_chip_info { u32 c2h_ctrl_reg; const u32 *c2h_regs; const struct rtw89_page_regs *page_regs; + bool cfo_src_fd; const struct rtw89_reg_def *dcfo_comp; u8 dcfo_comp_sft; const struct rtw89_imr_info *imr_info; @@ -2802,6 +2814,28 @@ struct rtw89_mac_info { u8 cpwm_seq_num; }; +#define RTW89_COMPLETION_BUF_SIZE 24 +#define RTW89_WAIT_COND_IDLE UINT_MAX + +struct rtw89_completion_data { + bool err; + u8 buf[RTW89_COMPLETION_BUF_SIZE]; +}; + +struct rtw89_wait_info { + atomic_t cond; + struct completion completion; + struct rtw89_completion_data data; +}; + +#define RTW89_WAIT_FOR_COND_TIMEOUT msecs_to_jiffies(100) + +static inline void rtw89_init_wait(struct rtw89_wait_info *wait) +{ + init_completion(&wait->completion); + atomic_set(&wait->cond, RTW89_WAIT_COND_IDLE); +} + enum rtw89_fw_type { RTW89_FW_NORMAL = 1, RTW89_FW_WOWLAN = 3, @@ -2921,6 +2955,13 @@ enum rtw89_entity_mode { RTW89_ENTITY_MODE_SCC, }; +struct rtw89_sub_entity { + struct cfg80211_chan_def chandef; + struct rtw89_chan chan; + struct rtw89_chan_rcd rcd; + struct rtw89_chanctx_cfg *cfg; +}; + struct rtw89_hal { u32 rx_fltr; u8 cv; @@ -2934,13 +2975,10 @@ struct rtw89_hal { bool support_igi; DECLARE_BITMAP(entity_map, NUM_OF_RTW89_SUB_ENTITY); - struct cfg80211_chan_def chandef[NUM_OF_RTW89_SUB_ENTITY]; + struct rtw89_sub_entity sub[NUM_OF_RTW89_SUB_ENTITY]; bool entity_active; enum rtw89_entity_mode entity_mode; - - struct rtw89_chan chan[NUM_OF_RTW89_SUB_ENTITY]; - struct rtw89_chan_rcd chan_rcd[NUM_OF_RTW89_SUB_ENTITY]; }; #define RTW89_MAX_MAC_ID_NUM 128 @@ -2959,6 +2997,7 @@ enum rtw89_flags { RTW89_FLAG_CRASH_SIMULATING, RTW89_FLAG_WOWLAN, RTW89_FLAG_FORBIDDEN_TRACK_WROK, + RTW89_FLAG_CHANGING_INTERFACE, NUM_OF_RTW89_FLAGS, }; @@ -3021,7 +3060,7 @@ struct rtw89_dack_info { #define RTW89_IQK_CHS_NR 2 #define RTW89_IQK_PATH_NR 4 -struct rtw89_mcc_info { +struct rtw89_rfk_mcc_info { u8 ch[RTW89_IQK_CHS_NR]; u8 band[RTW89_IQK_CHS_NR]; u8 table_idx; @@ -3514,6 +3553,10 @@ struct rtw89_wow_param { struct list_head pkt_list; }; +struct rtw89_mcc_info { + struct rtw89_wait_info wait; +}; + struct rtw89_dev { struct ieee80211_hw *hw; struct device *dev; @@ -3524,6 +3567,7 @@ struct rtw89_dev { const struct rtw89_chip_info *chip; const struct rtw89_pci_info *pci_info; struct rtw89_hal hal; + struct rtw89_mcc_info mcc; struct rtw89_mac_info mac; struct rtw89_fw_info fw; struct rtw89_hci_info hci; @@ -3567,7 +3611,7 @@ struct rtw89_dev { struct rtw89_dack_info dack; struct rtw89_iqk_info iqk; struct rtw89_dpk_info dpk; - struct rtw89_mcc_info mcc; + struct rtw89_rfk_mcc_info rfk_mcc; struct rtw89_lck_info lck; struct rtw89_rx_dck_info rx_dck; bool is_tssi_mode[RF_PATH_MAX]; @@ -3584,6 +3628,7 @@ struct rtw89_dev { struct rtw89_phy_ch_info ch_info; struct rtw89_phy_bb_gain_info bb_gain; struct rtw89_phy_efuse_gain efuse_gain; + struct rtw89_phy_ul_tb_info ul_tb_info; struct delayed_work track_work; struct delayed_work coex_act1_work; @@ -4099,7 +4144,7 @@ const struct cfg80211_chan_def *rtw89_chandef_get(struct rtw89_dev *rtwdev, { struct rtw89_hal *hal = &rtwdev->hal; - return &hal->chandef[idx]; + return &hal->sub[idx].chandef; } static inline @@ -4108,7 +4153,7 @@ const struct rtw89_chan *rtw89_chan_get(struct rtw89_dev *rtwdev, { struct rtw89_hal *hal = &rtwdev->hal; - return &hal->chan[idx]; + return &hal->sub[idx].chan; } static inline @@ -4117,7 +4162,7 @@ const struct rtw89_chan_rcd *rtw89_chan_rcd_get(struct rtw89_dev *rtwdev, { struct rtw89_hal *hal = &rtwdev->hal; - return &hal->chan_rcd[idx]; + return &hal->sub[idx].rcd; } static inline void rtw89_chip_fem_setup(struct rtw89_dev *rtwdev) @@ -4372,6 +4417,23 @@ static inline struct rtw89_fw_suit *rtw89_fw_suit_get(struct rtw89_dev *rtwdev, return &fw_info->normal; } +static inline struct sk_buff *rtw89_alloc_skb_for_rx(struct rtw89_dev *rtwdev, + unsigned int length) +{ + struct sk_buff *skb; + + if (rtwdev->hw->conf.flags & IEEE80211_CONF_MONITOR) { + skb = dev_alloc_skb(length + RTW89_RADIOTAP_ROOM); + if (!skb) + return NULL; + + skb_reserve(skb, RTW89_RADIOTAP_ROOM); + return skb; + } + + return dev_alloc_skb(length); +} + int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel); int rtw89_h2c_tx(struct rtw89_dev *rtwdev, @@ -4440,6 +4502,9 @@ int rtw89_regd_init(struct rtw89_dev *rtwdev, void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request); void rtw89_traffic_stats_init(struct rtw89_dev *rtwdev, struct rtw89_traffic_stats *stats); +int rtw89_wait_for_cond(struct rtw89_wait_info *wait, unsigned int cond); +void rtw89_complete_cond(struct rtw89_wait_info *wait, unsigned int cond, + const struct rtw89_completion_data *data); int rtw89_core_start(struct rtw89_dev *rtwdev); void rtw89_core_stop(struct rtw89_dev *rtwdev); void rtw89_core_update_beacon_work(struct work_struct *work); diff --git a/drivers/net/wireless/realtek/rtw89/debug.h b/drivers/net/wireless/realtek/rtw89/debug.h index e7971583acbc..d1de5e600836 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.h +++ b/drivers/net/wireless/realtek/rtw89/debug.h @@ -27,6 +27,7 @@ enum rtw89_debug_mask { RTW89_DBG_SAR = BIT(16), RTW89_DBG_STATE = BIT(17), RTW89_DBG_WOW = BIT(18), + RTW89_DBG_UL_TB = BIT(19), RTW89_DBG_UNEXP = BIT(31), }; diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 10c5ed21da02..7e682709232d 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -11,6 +11,9 @@ #include "phy.h" #include "reg.h" +static void rtw89_fw_c2h_cmd_handle(struct rtw89_dev *rtwdev, + struct sk_buff *skb); + static struct sk_buff *rtw89_fw_h2c_alloc_skb(struct rtw89_dev *rtwdev, u32 len, bool header) { @@ -2263,7 +2266,7 @@ fail: int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); - struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; + struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc; struct rtw89_fw_h2c_rf_get_mccch *mccch; struct sk_buff *skb; int ret; @@ -2276,10 +2279,10 @@ int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev) skb_put(skb, sizeof(*mccch)); mccch = (struct rtw89_fw_h2c_rf_get_mccch *)skb->data; - mccch->ch_0 = cpu_to_le32(mcc_info->ch[0]); - mccch->ch_1 = cpu_to_le32(mcc_info->ch[1]); - mccch->band_0 = cpu_to_le32(mcc_info->band[0]); - mccch->band_1 = cpu_to_le32(mcc_info->band[1]); + mccch->ch_0 = cpu_to_le32(rfk_mcc->ch[0]); + mccch->ch_1 = cpu_to_le32(rfk_mcc->ch[1]); + mccch->band_0 = cpu_to_le32(rfk_mcc->band[0]); + mccch->band_1 = cpu_to_le32(rfk_mcc->band[1]); mccch->current_channel = cpu_to_le32(chan->channel); mccch->current_band_type = cpu_to_le32(chan->band_type); @@ -2382,8 +2385,43 @@ void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev) mutex_unlock(&rtwdev->mutex); } +static void rtw89_fw_c2h_parse_attr(struct sk_buff *c2h) +{ + struct rtw89_fw_c2h_attr *attr = RTW89_SKB_C2H_CB(c2h); + + attr->category = RTW89_GET_C2H_CATEGORY(c2h->data); + attr->class = RTW89_GET_C2H_CLASS(c2h->data); + attr->func = RTW89_GET_C2H_FUNC(c2h->data); + attr->len = RTW89_GET_C2H_LEN(c2h->data); +} + +static bool rtw89_fw_c2h_chk_atomic(struct rtw89_dev *rtwdev, + struct sk_buff *c2h) +{ + struct rtw89_fw_c2h_attr *attr = RTW89_SKB_C2H_CB(c2h); + u8 category = attr->category; + u8 class = attr->class; + u8 func = attr->func; + + switch (category) { + default: + return false; + case RTW89_C2H_CAT_MAC: + return rtw89_mac_c2h_chk_atomic(rtwdev, class, func); + } +} + void rtw89_fw_c2h_irqsafe(struct rtw89_dev *rtwdev, struct sk_buff *c2h) { + rtw89_fw_c2h_parse_attr(c2h); + if (!rtw89_fw_c2h_chk_atomic(rtwdev, c2h)) + goto enqueue; + + rtw89_fw_c2h_cmd_handle(rtwdev, c2h); + dev_kfree_skb_any(c2h); + return; + +enqueue: skb_queue_tail(&rtwdev->c2h_queue, c2h); ieee80211_queue_work(rtwdev->hw, &rtwdev->c2h_work); } @@ -2391,10 +2429,11 @@ void rtw89_fw_c2h_irqsafe(struct rtw89_dev *rtwdev, struct sk_buff *c2h) static void rtw89_fw_c2h_cmd_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb) { - u8 category = RTW89_GET_C2H_CATEGORY(skb->data); - u8 class = RTW89_GET_C2H_CLASS(skb->data); - u8 func = RTW89_GET_C2H_FUNC(skb->data); - u16 len = RTW89_GET_C2H_LEN(skb->data); + struct rtw89_fw_c2h_attr *attr = RTW89_SKB_C2H_CB(skb); + u8 category = attr->category; + u8 class = attr->class; + u8 func = attr->func; + u16 len = attr->len; bool dump = true; if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags)) @@ -3224,3 +3263,332 @@ fail: return ret; } + +static int rtw89_h2c_tx_and_wait(struct rtw89_dev *rtwdev, struct sk_buff *skb, + struct rtw89_wait_info *wait, unsigned int cond) +{ + int ret; + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + dev_kfree_skb_any(skb); + return -EBUSY; + } + + return rtw89_wait_for_cond(wait, cond); +} + +#define H2C_ADD_MCC_LEN 16 +int rtw89_fw_h2c_add_mcc(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_add_req *p) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_ADD_MCC_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for add mcc\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_ADD_MCC_LEN); + RTW89_SET_FWCMD_ADD_MCC_MACID(skb->data, p->macid); + RTW89_SET_FWCMD_ADD_MCC_CENTRAL_CH_SEG0(skb->data, p->central_ch_seg0); + RTW89_SET_FWCMD_ADD_MCC_CENTRAL_CH_SEG1(skb->data, p->central_ch_seg1); + RTW89_SET_FWCMD_ADD_MCC_PRIMARY_CH(skb->data, p->primary_ch); + RTW89_SET_FWCMD_ADD_MCC_BANDWIDTH(skb->data, p->bandwidth); + RTW89_SET_FWCMD_ADD_MCC_GROUP(skb->data, p->group); + RTW89_SET_FWCMD_ADD_MCC_C2H_RPT(skb->data, p->c2h_rpt); + RTW89_SET_FWCMD_ADD_MCC_DIS_TX_NULL(skb->data, p->dis_tx_null); + RTW89_SET_FWCMD_ADD_MCC_DIS_SW_RETRY(skb->data, p->dis_sw_retry); + RTW89_SET_FWCMD_ADD_MCC_IN_CURR_CH(skb->data, p->in_curr_ch); + RTW89_SET_FWCMD_ADD_MCC_SW_RETRY_COUNT(skb->data, p->sw_retry_count); + RTW89_SET_FWCMD_ADD_MCC_TX_NULL_EARLY(skb->data, p->tx_null_early); + RTW89_SET_FWCMD_ADD_MCC_BTC_IN_2G(skb->data, p->btc_in_2g); + RTW89_SET_FWCMD_ADD_MCC_PTA_EN(skb->data, p->pta_en); + RTW89_SET_FWCMD_ADD_MCC_RFK_BY_PASS(skb->data, p->rfk_by_pass); + RTW89_SET_FWCMD_ADD_MCC_CH_BAND_TYPE(skb->data, p->ch_band_type); + RTW89_SET_FWCMD_ADD_MCC_DURATION(skb->data, p->duration); + RTW89_SET_FWCMD_ADD_MCC_COURTESY_EN(skb->data, p->courtesy_en); + RTW89_SET_FWCMD_ADD_MCC_COURTESY_NUM(skb->data, p->courtesy_num); + RTW89_SET_FWCMD_ADD_MCC_COURTESY_TARGET(skb->data, p->courtesy_target); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_ADD_MCC, 0, 0, + H2C_ADD_MCC_LEN); + + cond = RTW89_MCC_WAIT_COND(p->group, H2C_FUNC_ADD_MCC); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_START_MCC_LEN 12 +int rtw89_fw_h2c_start_mcc(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_start_req *p) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_START_MCC_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for start mcc\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_START_MCC_LEN); + RTW89_SET_FWCMD_START_MCC_GROUP(skb->data, p->group); + RTW89_SET_FWCMD_START_MCC_BTC_IN_GROUP(skb->data, p->btc_in_group); + RTW89_SET_FWCMD_START_MCC_OLD_GROUP_ACTION(skb->data, p->old_group_action); + RTW89_SET_FWCMD_START_MCC_OLD_GROUP(skb->data, p->old_group); + RTW89_SET_FWCMD_START_MCC_NOTIFY_CNT(skb->data, p->notify_cnt); + RTW89_SET_FWCMD_START_MCC_NOTIFY_RXDBG_EN(skb->data, p->notify_rxdbg_en); + RTW89_SET_FWCMD_START_MCC_MACID(skb->data, p->macid); + RTW89_SET_FWCMD_START_MCC_TSF_LOW(skb->data, p->tsf_low); + RTW89_SET_FWCMD_START_MCC_TSF_HIGH(skb->data, p->tsf_high); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_START_MCC, 0, 0, + H2C_START_MCC_LEN); + + cond = RTW89_MCC_WAIT_COND(p->group, H2C_FUNC_START_MCC); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_STOP_MCC_LEN 4 +int rtw89_fw_h2c_stop_mcc(struct rtw89_dev *rtwdev, u8 group, u8 macid, + bool prev_groups) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_STOP_MCC_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for stop mcc\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_STOP_MCC_LEN); + RTW89_SET_FWCMD_STOP_MCC_MACID(skb->data, macid); + RTW89_SET_FWCMD_STOP_MCC_GROUP(skb->data, group); + RTW89_SET_FWCMD_STOP_MCC_PREV_GROUPS(skb->data, prev_groups); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_STOP_MCC, 0, 0, + H2C_STOP_MCC_LEN); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_STOP_MCC); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_DEL_MCC_GROUP_LEN 4 +int rtw89_fw_h2c_del_mcc_group(struct rtw89_dev *rtwdev, u8 group, + bool prev_groups) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_DEL_MCC_GROUP_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for del mcc group\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_DEL_MCC_GROUP_LEN); + RTW89_SET_FWCMD_DEL_MCC_GROUP_GROUP(skb->data, group); + RTW89_SET_FWCMD_DEL_MCC_GROUP_PREV_GROUPS(skb->data, prev_groups); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_DEL_MCC_GROUP, 0, 0, + H2C_DEL_MCC_GROUP_LEN); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_DEL_MCC_GROUP); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_RESET_MCC_GROUP_LEN 4 +int rtw89_fw_h2c_reset_mcc_group(struct rtw89_dev *rtwdev, u8 group) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_RESET_MCC_GROUP_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for reset mcc group\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_RESET_MCC_GROUP_LEN); + RTW89_SET_FWCMD_RESET_MCC_GROUP_GROUP(skb->data, group); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_RESET_MCC_GROUP, 0, 0, + H2C_RESET_MCC_GROUP_LEN); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_RESET_MCC_GROUP); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_MCC_REQ_TSF_LEN 4 +int rtw89_fw_h2c_mcc_req_tsf(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_tsf_req *req, + struct rtw89_mac_mcc_tsf_rpt *rpt) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct rtw89_mac_mcc_tsf_rpt *tmp; + struct sk_buff *skb; + unsigned int cond; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_MCC_REQ_TSF_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for mcc req tsf\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_MCC_REQ_TSF_LEN); + RTW89_SET_FWCMD_MCC_REQ_TSF_GROUP(skb->data, req->group); + RTW89_SET_FWCMD_MCC_REQ_TSF_MACID_X(skb->data, req->macid_x); + RTW89_SET_FWCMD_MCC_REQ_TSF_MACID_Y(skb->data, req->macid_y); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_MCC_REQ_TSF, 0, 0, + H2C_MCC_REQ_TSF_LEN); + + cond = RTW89_MCC_WAIT_COND(req->group, H2C_FUNC_MCC_REQ_TSF); + ret = rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); + if (ret) + return ret; + + tmp = (struct rtw89_mac_mcc_tsf_rpt *)wait->data.buf; + *rpt = *tmp; + + return 0; +} + +#define H2C_MCC_MACID_BITMAP_DSC_LEN 4 +int rtw89_fw_h2c_mcc_macid_bitamp(struct rtw89_dev *rtwdev, u8 group, u8 macid, + u8 *bitmap) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + u8 map_len; + u8 h2c_len; + + BUILD_BUG_ON(RTW89_MAX_MAC_ID_NUM % 8); + map_len = RTW89_MAX_MAC_ID_NUM / 8; + h2c_len = H2C_MCC_MACID_BITMAP_DSC_LEN + map_len; + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, h2c_len); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for mcc macid bitmap\n"); + return -ENOMEM; + } + + skb_put(skb, h2c_len); + RTW89_SET_FWCMD_MCC_MACID_BITMAP_GROUP(skb->data, group); + RTW89_SET_FWCMD_MCC_MACID_BITMAP_MACID(skb->data, macid); + RTW89_SET_FWCMD_MCC_MACID_BITMAP_BITMAP_LENGTH(skb->data, map_len); + RTW89_SET_FWCMD_MCC_MACID_BITMAP_BITMAP(skb->data, bitmap, map_len); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_MCC_MACID_BITMAP, 0, 0, + h2c_len); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_MCC_MACID_BITMAP); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_MCC_SYNC_LEN 4 +int rtw89_fw_h2c_mcc_sync(struct rtw89_dev *rtwdev, u8 group, u8 source, + u8 target, u8 offset) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_MCC_SYNC_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for mcc sync\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_MCC_SYNC_LEN); + RTW89_SET_FWCMD_MCC_SYNC_GROUP(skb->data, group); + RTW89_SET_FWCMD_MCC_SYNC_MACID_SOURCE(skb->data, source); + RTW89_SET_FWCMD_MCC_SYNC_MACID_TARGET(skb->data, target); + RTW89_SET_FWCMD_MCC_SYNC_SYNC_OFFSET(skb->data, offset); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_MCC_SYNC, 0, 0, + H2C_MCC_SYNC_LEN); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_MCC_SYNC); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} + +#define H2C_MCC_SET_DURATION_LEN 20 +int rtw89_fw_h2c_mcc_set_duration(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_duration *p) +{ + struct rtw89_wait_info *wait = &rtwdev->mcc.wait; + struct sk_buff *skb; + unsigned int cond; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_MCC_SET_DURATION_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for mcc set duration\n"); + return -ENOMEM; + } + + skb_put(skb, H2C_MCC_SET_DURATION_LEN); + RTW89_SET_FWCMD_MCC_SET_DURATION_GROUP(skb->data, p->group); + RTW89_SET_FWCMD_MCC_SET_DURATION_BTC_IN_GROUP(skb->data, p->btc_in_group); + RTW89_SET_FWCMD_MCC_SET_DURATION_START_MACID(skb->data, p->start_macid); + RTW89_SET_FWCMD_MCC_SET_DURATION_MACID_X(skb->data, p->macid_x); + RTW89_SET_FWCMD_MCC_SET_DURATION_MACID_Y(skb->data, p->macid_y); + RTW89_SET_FWCMD_MCC_SET_DURATION_START_TSF_LOW(skb->data, + p->start_tsf_low); + RTW89_SET_FWCMD_MCC_SET_DURATION_START_TSF_HIGH(skb->data, + p->start_tsf_high); + RTW89_SET_FWCMD_MCC_SET_DURATION_DURATION_X(skb->data, p->duration_x); + RTW89_SET_FWCMD_MCC_SET_DURATION_DURATION_Y(skb->data, p->duration_y); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MCC, + H2C_FUNC_MCC_SET_DURATION, 0, 0, + H2C_MCC_SET_DURATION_LEN); + + cond = RTW89_MCC_WAIT_COND(p->group, H2C_FUNC_MCC_SET_DURATION); + return rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond); +} diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 509a3eac5ffe..46d57414f24e 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -2767,6 +2767,355 @@ static inline void RTW89_SET_FWCMD_TSF32_TOGL_EARLY(void *cmd, u32 val) le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 16)); } +enum rtw89_fw_mcc_c2h_rpt_cfg { + RTW89_FW_MCC_C2H_RPT_OFF = 0, + RTW89_FW_MCC_C2H_RPT_FAIL_ONLY = 1, + RTW89_FW_MCC_C2H_RPT_ALL = 2, +}; + +struct rtw89_fw_mcc_add_req { + u8 macid; + u8 central_ch_seg0; + u8 central_ch_seg1; + u8 primary_ch; + enum rtw89_bandwidth bandwidth: 4; + u32 group: 2; + u32 c2h_rpt: 2; + u32 dis_tx_null: 1; + u32 dis_sw_retry: 1; + u32 in_curr_ch: 1; + u32 sw_retry_count: 3; + u32 tx_null_early: 4; + u32 btc_in_2g: 1; + u32 pta_en: 1; + u32 rfk_by_pass: 1; + u32 ch_band_type: 2; + u32 rsvd0: 9; + u32 duration; + u8 courtesy_en; + u8 courtesy_num; + u8 courtesy_target; + u8 rsvd1; +}; + +static inline void RTW89_SET_FWCMD_ADD_MCC_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_CENTRAL_CH_SEG0(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_CENTRAL_CH_SEG1(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_PRIMARY_CH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_BANDWIDTH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(3, 0)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(5, 4)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_C2H_RPT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(7, 6)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_DIS_TX_NULL(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(8)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_DIS_SW_RETRY(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(9)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_IN_CURR_CH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(10)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_SW_RETRY_COUNT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(13, 11)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_TX_NULL_EARLY(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(17, 14)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_BTC_IN_2G(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(18)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_PTA_EN(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(19)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_RFK_BY_PASS(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, BIT(20)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_CH_BAND_TYPE(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(22, 21)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_DURATION(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 2, val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_COURTESY_EN(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 3, val, BIT(0)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_COURTESY_NUM(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 3, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_ADD_MCC_COURTESY_TARGET(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 3, val, GENMASK(23, 16)); +} + +struct rtw89_fw_mcc_start_req { + u32 group: 2; + u32 btc_in_group: 1; + u32 old_group_action: 2; + u32 old_group: 2; + u32 rsvd0: 9; + u32 notify_cnt: 3; + u32 rsvd1: 2; + u32 notify_rxdbg_en: 1; + u32 rsvd2: 2; + u32 macid: 8; + u32 tsf_low; + u32 tsf_high; +}; + +static inline void RTW89_SET_FWCMD_START_MCC_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_BTC_IN_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(2)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_OLD_GROUP_ACTION(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(4, 3)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_OLD_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(6, 5)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_NOTIFY_CNT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(18, 16)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_NOTIFY_RXDBG_EN(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(21)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_TSF_LOW(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_START_MCC_TSF_HIGH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 2, val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_STOP_MCC_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_STOP_MCC_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(9, 8)); +} + +static inline void RTW89_SET_FWCMD_STOP_MCC_PREV_GROUPS(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(10)); +} + +static inline void RTW89_SET_FWCMD_DEL_MCC_GROUP_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static inline void RTW89_SET_FWCMD_DEL_MCC_GROUP_PREV_GROUPS(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(2)); +} + +static inline void RTW89_SET_FWCMD_RESET_MCC_GROUP_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +struct rtw89_fw_mcc_tsf_req { + u8 group: 2; + u8 rsvd0: 6; + u8 macid_x; + u8 macid_y; + u8 rsvd1; +}; + +static inline void RTW89_SET_FWCMD_MCC_REQ_TSF_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static inline void RTW89_SET_FWCMD_MCC_REQ_TSF_MACID_X(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_MCC_REQ_TSF_MACID_Y(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_MCC_MACID_BITMAP_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static inline void RTW89_SET_FWCMD_MCC_MACID_BITMAP_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_MCC_MACID_BITMAP_BITMAP_LENGTH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_MCC_MACID_BITMAP_BITMAP(void *cmd, + u8 *bitmap, u8 len) +{ + memcpy((__le32 *)cmd + 1, bitmap, len); +} + +static inline void RTW89_SET_FWCMD_MCC_SYNC_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static inline void RTW89_SET_FWCMD_MCC_SYNC_MACID_SOURCE(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_MCC_SYNC_MACID_TARGET(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_MCC_SYNC_SYNC_OFFSET(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24)); +} + +struct rtw89_fw_mcc_duration { + u32 group: 2; + u32 btc_in_group: 1; + u32 rsvd0: 5; + u32 start_macid: 8; + u32 macid_x: 8; + u32 macid_y: 8; + u32 start_tsf_low; + u32 start_tsf_high; + u32 duration_x; + u32 duration_y; +}; + +static inline void RTW89_SET_FWCMD_MCC_SET_DURATION_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(1, 0)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_BTC_IN_GROUP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(2)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_START_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_MCC_SET_DURATION_MACID_X(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_MCC_SET_DURATION_MACID_Y(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_START_TSF_LOW(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(31, 0)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_START_TSF_HIGH(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 2, val, GENMASK(31, 0)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_DURATION_X(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 3, val, GENMASK(31, 0)); +} + +static +inline void RTW89_SET_FWCMD_MCC_SET_DURATION_DURATION_Y(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 4, val, GENMASK(31, 0)); +} + #define RTW89_C2H_HEADER_LEN 8 #define RTW89_GET_C2H_CATEGORY(c2h) \ @@ -2778,6 +3127,20 @@ static inline void RTW89_SET_FWCMD_TSF32_TOGL_EARLY(void *cmd, u32 val) #define RTW89_GET_C2H_LEN(c2h) \ le32_get_bits(*((const __le32 *)(c2h) + 1), GENMASK(13, 0)) +struct rtw89_fw_c2h_attr { + u8 category; + u8 class; + u8 func; + u16 len; +}; + +static inline struct rtw89_fw_c2h_attr *RTW89_SKB_C2H_CB(struct sk_buff *skb) +{ + static_assert(sizeof(skb->cb) >= sizeof(struct rtw89_fw_c2h_attr)); + + return (struct rtw89_fw_c2h_attr *)skb->cb; +} + #define RTW89_GET_C2H_LOG_SRT_PRT(c2h) (char *)((__le32 *)(c2h) + 2) #define RTW89_GET_C2H_LOG_LEN(len) ((len) - RTW89_C2H_HEADER_LEN) @@ -2845,6 +3208,55 @@ static inline void RTW89_SET_FWCMD_TSF32_TOGL_EARLY(void *cmd, u32 val) #define RTW89_GET_MAC_C2H_SCANOFLD_BAND(c2h) \ le32_get_bits(*((const __le32 *)(c2h) + 5), GENMASK(25, 24)) +#define RTW89_GET_MAC_C2H_MCC_RCV_ACK_GROUP(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(1, 0)) +#define RTW89_GET_MAC_C2H_MCC_RCV_ACK_H2C_FUNC(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(15, 8)) + +#define RTW89_GET_MAC_C2H_MCC_REQ_ACK_GROUP(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(1, 0)) +#define RTW89_GET_MAC_C2H_MCC_REQ_ACK_H2C_RETURN(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(7, 2)) +#define RTW89_GET_MAC_C2H_MCC_REQ_ACK_H2C_FUNC(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(15, 8)) + +struct rtw89_mac_mcc_tsf_rpt { + u32 macid_x; + u32 macid_y; + u32 tsf_x_low; + u32 tsf_x_high; + u32 tsf_y_low; + u32 tsf_y_high; +}; + +static_assert(sizeof(struct rtw89_mac_mcc_tsf_rpt) <= RTW89_COMPLETION_BUF_SIZE); + +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_MACID_X(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(7, 0)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_MACID_Y(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(15, 8)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_GROUP(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(17, 16)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_LOW_X(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 1), GENMASK(31, 0)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_HIGH_X(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(31, 0)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_LOW_Y(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 3), GENMASK(31, 0)) +#define RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_HIGH_Y(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 4), GENMASK(31, 0)) + +#define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_STATUS(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(5, 0)) +#define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_GROUP(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(7, 6)) +#define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_MACID(c2h) \ + le32_get_bits(*((const __le32 *)(c2h)), GENMASK(15, 8)) +#define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_LOW(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 1), GENMASK(31, 0)) +#define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_HIGH(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(31, 0)) + #define RTW89_FW_HDR_SIZE 32 #define RTW89_FW_SECTION_HDR_SIZE 16 @@ -2966,6 +3378,25 @@ struct rtw89_fw_h2c_rf_reg_info { #define H2C_CL_BA_CAM 0xc #define H2C_FUNC_MAC_BA_CAM 0x0 +/* CLASS 14 - MCC */ +#define H2C_CL_MCC 0xe +enum rtw89_mcc_h2c_func { + H2C_FUNC_ADD_MCC = 0x0, + H2C_FUNC_START_MCC = 0x1, + H2C_FUNC_STOP_MCC = 0x2, + H2C_FUNC_DEL_MCC_GROUP = 0x3, + H2C_FUNC_RESET_MCC_GROUP = 0x4, + H2C_FUNC_MCC_REQ_TSF = 0x5, + H2C_FUNC_MCC_MACID_BITMAP = 0x6, + H2C_FUNC_MCC_SYNC = 0x7, + H2C_FUNC_MCC_SET_DURATION = 0x8, + + NUM_OF_RTW89_MCC_H2C_FUNC, +}; + +#define RTW89_MCC_WAIT_COND(group, func) \ + ((group) * NUM_OF_RTW89_MCC_H2C_FUNC + (func)) + #define H2C_CAT_OUTSRC 0x2 #define H2C_CL_OUTSRC_RA 0x1 @@ -3103,9 +3534,27 @@ int rtw89_fw_h2c_wow_global(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, bool enable); int rtw89_fw_h2c_wow_wakeup_ctrl(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, bool enable); - int rtw89_fw_wow_cam_update(struct rtw89_dev *rtwdev, struct rtw89_wow_cam_info *cam_info); +int rtw89_fw_h2c_add_mcc(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_add_req *p); +int rtw89_fw_h2c_start_mcc(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_start_req *p); +int rtw89_fw_h2c_stop_mcc(struct rtw89_dev *rtwdev, u8 group, u8 macid, + bool prev_groups); +int rtw89_fw_h2c_del_mcc_group(struct rtw89_dev *rtwdev, u8 group, + bool prev_groups); +int rtw89_fw_h2c_reset_mcc_group(struct rtw89_dev *rtwdev, u8 group); +int rtw89_fw_h2c_mcc_req_tsf(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_tsf_req *req, + struct rtw89_mac_mcc_tsf_rpt *rpt); +int rtw89_fw_h2c_mcc_macid_bitamp(struct rtw89_dev *rtwdev, u8 group, u8 macid, + u8 *bitmap); +int rtw89_fw_h2c_mcc_sync(struct rtw89_dev *rtwdev, u8 group, u8 source, + u8 target, u8 offset); +int rtw89_fw_h2c_mcc_set_duration(struct rtw89_dev *rtwdev, + const struct rtw89_fw_mcc_duration *p); + static inline void rtw89_fw_h2c_init_ba_cam(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index ecd603a88134..d80050c2e9b3 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -3600,6 +3600,13 @@ int rtw89_mac_set_macid_pause(struct rtw89_dev *rtwdev, u8 macid, bool pause) u8 grp = macid >> 5; int ret; + /* If this is called by change_interface() in the case of P2P, it could + * be power-off, so ignore this operation. + */ + if (test_bit(RTW89_FLAG_CHANGING_INTERFACE, rtwdev->flags) && + !test_bit(RTW89_FLAG_POWERON, rtwdev->flags)) + return 0; + ret = rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_0, RTW89_CMAC_SEL); if (ret) return ret; @@ -4180,6 +4187,164 @@ rtw89_mac_c2h_tsf32_toggle_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, { } +static void +rtw89_mac_c2h_mcc_rcv_ack(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + u8 group = RTW89_GET_MAC_C2H_MCC_RCV_ACK_GROUP(c2h->data); + u8 func = RTW89_GET_MAC_C2H_MCC_RCV_ACK_H2C_FUNC(c2h->data); + + switch (func) { + case H2C_FUNC_ADD_MCC: + case H2C_FUNC_START_MCC: + case H2C_FUNC_STOP_MCC: + case H2C_FUNC_DEL_MCC_GROUP: + case H2C_FUNC_RESET_MCC_GROUP: + case H2C_FUNC_MCC_REQ_TSF: + case H2C_FUNC_MCC_MACID_BITMAP: + case H2C_FUNC_MCC_SYNC: + case H2C_FUNC_MCC_SET_DURATION: + break; + default: + rtw89_debug(rtwdev, RTW89_DBG_FW, + "invalid MCC C2H RCV ACK: func %d\n", func); + return; + } + + rtw89_debug(rtwdev, RTW89_DBG_FW, + "MCC C2H RCV ACK: group %d, func %d\n", group, func); +} + +static void +rtw89_mac_c2h_mcc_req_ack(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + u8 group = RTW89_GET_MAC_C2H_MCC_REQ_ACK_GROUP(c2h->data); + u8 func = RTW89_GET_MAC_C2H_MCC_REQ_ACK_H2C_FUNC(c2h->data); + u8 retcode = RTW89_GET_MAC_C2H_MCC_REQ_ACK_H2C_RETURN(c2h->data); + struct rtw89_completion_data data = {}; + unsigned int cond; + bool next = false; + + switch (func) { + case H2C_FUNC_MCC_REQ_TSF: + next = true; + break; + case H2C_FUNC_MCC_MACID_BITMAP: + case H2C_FUNC_MCC_SYNC: + case H2C_FUNC_MCC_SET_DURATION: + break; + case H2C_FUNC_ADD_MCC: + case H2C_FUNC_START_MCC: + case H2C_FUNC_STOP_MCC: + case H2C_FUNC_DEL_MCC_GROUP: + case H2C_FUNC_RESET_MCC_GROUP: + default: + rtw89_debug(rtwdev, RTW89_DBG_FW, + "invalid MCC C2H REQ ACK: func %d\n", func); + return; + } + + rtw89_debug(rtwdev, RTW89_DBG_FW, + "MCC C2H REQ ACK: group %d, func %d, return code %d\n", + group, func, retcode); + + if (!retcode && next) + return; + + data.err = !!retcode; + cond = RTW89_MCC_WAIT_COND(group, func); + rtw89_complete_cond(&rtwdev->mcc.wait, cond, &data); +} + +static void +rtw89_mac_c2h_mcc_tsf_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + u8 group = RTW89_GET_MAC_C2H_MCC_TSF_RPT_GROUP(c2h->data); + struct rtw89_completion_data data = {}; + struct rtw89_mac_mcc_tsf_rpt *rpt; + unsigned int cond; + + rpt = (struct rtw89_mac_mcc_tsf_rpt *)data.buf; + rpt->macid_x = RTW89_GET_MAC_C2H_MCC_TSF_RPT_MACID_X(c2h->data); + rpt->macid_y = RTW89_GET_MAC_C2H_MCC_TSF_RPT_MACID_Y(c2h->data); + rpt->tsf_x_low = RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_LOW_X(c2h->data); + rpt->tsf_x_high = RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_HIGH_X(c2h->data); + rpt->tsf_y_low = RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_LOW_Y(c2h->data); + rpt->tsf_y_high = RTW89_GET_MAC_C2H_MCC_TSF_RPT_TSF_HIGH_Y(c2h->data); + + cond = RTW89_MCC_WAIT_COND(group, H2C_FUNC_MCC_REQ_TSF); + rtw89_complete_cond(&rtwdev->mcc.wait, cond, &data); +} + +static void +rtw89_mac_c2h_mcc_status_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + u8 group = RTW89_GET_MAC_C2H_MCC_STATUS_RPT_GROUP(c2h->data); + u8 macid = RTW89_GET_MAC_C2H_MCC_STATUS_RPT_MACID(c2h->data); + u8 status = RTW89_GET_MAC_C2H_MCC_STATUS_RPT_STATUS(c2h->data); + u32 tsf_low = RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_LOW(c2h->data); + u32 tsf_high = RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_HIGH(c2h->data); + struct rtw89_completion_data data = {}; + unsigned int cond; + bool rsp = true; + bool err; + u8 func; + + switch (status) { + case RTW89_MAC_MCC_ADD_ROLE_OK: + case RTW89_MAC_MCC_ADD_ROLE_FAIL: + func = H2C_FUNC_ADD_MCC; + err = status == RTW89_MAC_MCC_ADD_ROLE_FAIL; + break; + case RTW89_MAC_MCC_START_GROUP_OK: + case RTW89_MAC_MCC_START_GROUP_FAIL: + func = H2C_FUNC_START_MCC; + err = status == RTW89_MAC_MCC_START_GROUP_FAIL; + break; + case RTW89_MAC_MCC_STOP_GROUP_OK: + case RTW89_MAC_MCC_STOP_GROUP_FAIL: + func = H2C_FUNC_STOP_MCC; + err = status == RTW89_MAC_MCC_STOP_GROUP_FAIL; + break; + case RTW89_MAC_MCC_DEL_GROUP_OK: + case RTW89_MAC_MCC_DEL_GROUP_FAIL: + func = H2C_FUNC_DEL_MCC_GROUP; + err = status == RTW89_MAC_MCC_DEL_GROUP_FAIL; + break; + case RTW89_MAC_MCC_RESET_GROUP_OK: + case RTW89_MAC_MCC_RESET_GROUP_FAIL: + func = H2C_FUNC_RESET_MCC_GROUP; + err = status == RTW89_MAC_MCC_RESET_GROUP_FAIL; + break; + case RTW89_MAC_MCC_SWITCH_CH_OK: + case RTW89_MAC_MCC_SWITCH_CH_FAIL: + case RTW89_MAC_MCC_TXNULL0_OK: + case RTW89_MAC_MCC_TXNULL0_FAIL: + case RTW89_MAC_MCC_TXNULL1_OK: + case RTW89_MAC_MCC_TXNULL1_FAIL: + case RTW89_MAC_MCC_SWITCH_EARLY: + case RTW89_MAC_MCC_TBTT: + case RTW89_MAC_MCC_DURATION_START: + case RTW89_MAC_MCC_DURATION_END: + rsp = false; + break; + default: + rtw89_debug(rtwdev, RTW89_DBG_FW, + "invalid MCC C2H STS RPT: status %d\n", status); + return; + } + + rtw89_debug(rtwdev, RTW89_DBG_FW, + "MCC C2H STS RPT: group %d, macid %d, status %d, tsf {%d, %d}\n", + group, macid, status, tsf_low, tsf_high); + + if (!rsp) + return; + + data.err = err; + cond = RTW89_MCC_WAIT_COND(group, func); + rtw89_complete_cond(&rtwdev->mcc.wait, cond, &data); +} + static void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) = { @@ -4201,6 +4366,25 @@ void (* const rtw89_mac_c2h_info_handler[])(struct rtw89_dev *rtwdev, [RTW89_MAC_C2H_FUNC_BCN_CNT] = rtw89_mac_c2h_bcn_cnt, }; +static +void (* const rtw89_mac_c2h_mcc_handler[])(struct rtw89_dev *rtwdev, + struct sk_buff *c2h, u32 len) = { + [RTW89_MAC_C2H_FUNC_MCC_RCV_ACK] = rtw89_mac_c2h_mcc_rcv_ack, + [RTW89_MAC_C2H_FUNC_MCC_REQ_ACK] = rtw89_mac_c2h_mcc_req_ack, + [RTW89_MAC_C2H_FUNC_MCC_TSF_RPT] = rtw89_mac_c2h_mcc_tsf_rpt, + [RTW89_MAC_C2H_FUNC_MCC_STATUS_RPT] = rtw89_mac_c2h_mcc_status_rpt, +}; + +bool rtw89_mac_c2h_chk_atomic(struct rtw89_dev *rtwdev, u8 class, u8 func) +{ + switch (class) { + default: + return false; + case RTW89_MAC_C2H_CLASS_MCC: + return true; + } +} + void rtw89_mac_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32 len, u8 class, u8 func) { @@ -4216,6 +4400,10 @@ void rtw89_mac_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, if (func < RTW89_MAC_C2H_FUNC_OFLD_MAX) handler = rtw89_mac_c2h_ofld_handler[func]; break; + case RTW89_MAC_C2H_CLASS_MCC: + if (func < NUM_OF_RTW89_MAC_C2H_FUNC_MCC) + handler = rtw89_mac_c2h_mcc_handler[func]; + break; case RTW89_MAC_C2H_CLASS_FWDBG: return; default: diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 045e8ec61a41..adb0c86a98d3 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -368,6 +368,15 @@ enum rtw89_mac_c2h_info_func { RTW89_MAC_C2H_FUNC_INFO_MAX, }; +enum rtw89_mac_c2h_mcc_func { + RTW89_MAC_C2H_FUNC_MCC_RCV_ACK = 0, + RTW89_MAC_C2H_FUNC_MCC_REQ_ACK = 1, + RTW89_MAC_C2H_FUNC_MCC_TSF_RPT = 2, + RTW89_MAC_C2H_FUNC_MCC_STATUS_RPT = 3, + + NUM_OF_RTW89_MAC_C2H_FUNC_MCC, +}; + enum rtw89_mac_c2h_class { RTW89_MAC_C2H_CLASS_INFO, RTW89_MAC_C2H_CLASS_OFLD, @@ -378,6 +387,31 @@ enum rtw89_mac_c2h_class { RTW89_MAC_C2H_CLASS_MAX, }; +enum rtw89_mac_mcc_status { + RTW89_MAC_MCC_ADD_ROLE_OK = 0, + RTW89_MAC_MCC_START_GROUP_OK = 1, + RTW89_MAC_MCC_STOP_GROUP_OK = 2, + RTW89_MAC_MCC_DEL_GROUP_OK = 3, + RTW89_MAC_MCC_RESET_GROUP_OK = 4, + RTW89_MAC_MCC_SWITCH_CH_OK = 5, + RTW89_MAC_MCC_TXNULL0_OK = 6, + RTW89_MAC_MCC_TXNULL1_OK = 7, + + RTW89_MAC_MCC_SWITCH_EARLY = 10, + RTW89_MAC_MCC_TBTT = 11, + RTW89_MAC_MCC_DURATION_START = 12, + RTW89_MAC_MCC_DURATION_END = 13, + + RTW89_MAC_MCC_ADD_ROLE_FAIL = 20, + RTW89_MAC_MCC_START_GROUP_FAIL = 21, + RTW89_MAC_MCC_STOP_GROUP_FAIL = 22, + RTW89_MAC_MCC_DEL_GROUP_FAIL = 23, + RTW89_MAC_MCC_RESET_GROUP_FAIL = 24, + RTW89_MAC_MCC_SWITCH_CH_FAIL = 25, + RTW89_MAC_MCC_TXNULL0_FAIL = 26, + RTW89_MAC_MCC_TXNULL1_FAIL = 27, +}; + struct rtw89_mac_ax_coex { #define RTW89_MAC_AX_COEX_RTK_MODE 0 #define RTW89_MAC_AX_COEX_CSR_MODE 1 @@ -894,6 +928,7 @@ static inline int rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev) u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev); int rtw89_mac_set_err_status(struct rtw89_dev *rtwdev, u32 err); +bool rtw89_mac_c2h_chk_atomic(struct rtw89_dev *rtwdev, u8 class, u8 func); void rtw89_mac_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32 len, u8 class, u8 func); int rtw89_mac_setup_phycap(struct rtw89_dev *rtwdev); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 6e79bf899901..1a99267d710d 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -131,6 +131,7 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw, rtwvif->bcn_hit_cond = 0; rtwvif->mac_idx = RTW89_MAC_0; rtwvif->phy_idx = RTW89_PHY_0; + rtwvif->sub_entity_idx = RTW89_SUB_ENTITY_0; rtwvif->hit_rule = 0; ether_addr_copy(rtwvif->mac_addr, vif->addr); @@ -174,6 +175,9 @@ static int rtw89_ops_change_interface(struct ieee80211_hw *hw, enum nl80211_iftype type, bool p2p) { struct rtw89_dev *rtwdev = hw->priv; + int ret; + + set_bit(RTW89_FLAG_CHANGING_INTERFACE, rtwdev->flags); rtw89_debug(rtwdev, RTW89_DBG_STATE, "change vif %pM (%d)->(%d), p2p (%d)->(%d)\n", vif->addr, vif->type, type, vif->p2p, p2p); @@ -183,7 +187,13 @@ static int rtw89_ops_change_interface(struct ieee80211_hw *hw, vif->type = type; vif->p2p = p2p; - return rtw89_ops_add_interface(hw, vif); + ret = rtw89_ops_add_interface(hw, vif); + if (ret) + rtw89_warn(rtwdev, "failed to change interface %d\n", ret); + + clear_bit(RTW89_FLAG_CHANGING_INTERFACE, rtwdev->flags); + + return ret; } static void rtw89_ops_configure_filter(struct ieee80211_hw *hw, diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 7aa0af18cdd5..1c4500ba777c 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -267,7 +267,7 @@ static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, rtw89_core_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size); - new = dev_alloc_skb(desc_info->pkt_size); + new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size); if (!new) goto err_sync_device; diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index dd4685698983..017710c580c7 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -2,6 +2,7 @@ /* Copyright(c) 2019-2020 Realtek Corporation */ +#include "coex.h" #include "debug.h" #include "fw.h" #include "mac.h" @@ -9,7 +10,7 @@ #include "ps.h" #include "reg.h" #include "sar.h" -#include "coex.h" +#include "util.h" static u16 get_max_amsdu_len(struct rtw89_dev *rtwdev, const struct rtw89_ra_report *report) @@ -2794,6 +2795,129 @@ void rtw89_phy_cfo_parse(struct rtw89_dev *rtwdev, s16 cfo_val, cfo->packet_count++; } +void rtw89_phy_ul_tb_assoc(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + struct rtw89_phy_ul_tb_info *ul_tb_info = &rtwdev->ul_tb_info; + + if (!chip->support_ul_tb_ctrl) + return; + + rtwvif->def_tri_idx = + rtw89_phy_read32_mask(rtwdev, R_DCFO_OPT, B_TXSHAPE_TRIANGULAR_CFG); + + if (chip->chip_id == RTL8852B && rtwdev->hal.cv > CHIP_CBV) + rtwvif->dyn_tb_bedge_en = false; + else if (chan->band_type >= RTW89_BAND_5G && + chan->band_width >= RTW89_CHANNEL_WIDTH_40) + rtwvif->dyn_tb_bedge_en = true; + else + rtwvif->dyn_tb_bedge_en = false; + + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] def_if_bandedge=%d, def_tri_idx=%d\n", + ul_tb_info->def_if_bandedge, rtwvif->def_tri_idx); + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] dyn_tb_begde_en=%d, dyn_tb_tri_en=%d\n", + rtwvif->dyn_tb_bedge_en, ul_tb_info->dyn_tb_tri_en); +} + +struct rtw89_phy_ul_tb_check_data { + bool valid; + bool high_tf_client; + bool low_tf_client; + bool dyn_tb_bedge_en; + u8 def_tri_idx; +}; + +static +void rtw89_phy_ul_tb_ctrl_check(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, + struct rtw89_phy_ul_tb_check_data *ul_tb_data) +{ + struct rtw89_traffic_stats *stats = &rtwdev->stats; + struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); + + if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION) + return; + + if (!vif->cfg.assoc) + return; + + if (stats->rx_tf_periodic > UL_TB_TF_CNT_L2H_TH) + ul_tb_data->high_tf_client = true; + else if (stats->rx_tf_periodic < UL_TB_TF_CNT_H2L_TH) + ul_tb_data->low_tf_client = true; + + ul_tb_data->valid = true; + ul_tb_data->def_tri_idx = rtwvif->def_tri_idx; + ul_tb_data->dyn_tb_bedge_en = rtwvif->dyn_tb_bedge_en; +} + +void rtw89_phy_ul_tb_ctrl_track(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_phy_ul_tb_info *ul_tb_info = &rtwdev->ul_tb_info; + struct rtw89_phy_ul_tb_check_data ul_tb_data = {}; + struct rtw89_vif *rtwvif; + + if (!chip->support_ul_tb_ctrl) + return; + + if (rtwdev->total_sta_assoc != 1) + return; + + rtw89_for_each_rtwvif(rtwdev, rtwvif) + rtw89_phy_ul_tb_ctrl_check(rtwdev, rtwvif, &ul_tb_data); + + if (!ul_tb_data.valid) + return; + + if (ul_tb_data.dyn_tb_bedge_en) { + if (ul_tb_data.high_tf_client) { + rtw89_phy_write32_mask(rtwdev, R_BANDEDGE, B_BANDEDGE_EN, 0); + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] Turn off if_bandedge\n"); + } else if (ul_tb_data.low_tf_client) { + rtw89_phy_write32_mask(rtwdev, R_BANDEDGE, B_BANDEDGE_EN, + ul_tb_info->def_if_bandedge); + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] Set to default if_bandedge = %d\n", + ul_tb_info->def_if_bandedge); + } + } + + if (ul_tb_info->dyn_tb_tri_en) { + if (ul_tb_data.high_tf_client) { + rtw89_phy_write32_mask(rtwdev, R_DCFO_OPT, + B_TXSHAPE_TRIANGULAR_CFG, 0); + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] Turn off Tx triangle\n"); + } else if (ul_tb_data.low_tf_client) { + rtw89_phy_write32_mask(rtwdev, R_DCFO_OPT, + B_TXSHAPE_TRIANGULAR_CFG, + ul_tb_data.def_tri_idx); + rtw89_debug(rtwdev, RTW89_DBG_UL_TB, + "[ULTB] Set to default tx_shap_idx = %d\n", + ul_tb_data.def_tri_idx); + } + } +} + +static void rtw89_phy_ul_tb_info_init(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_phy_ul_tb_info *ul_tb_info = &rtwdev->ul_tb_info; + + if (!chip->support_ul_tb_ctrl) + return; + + ul_tb_info->dyn_tb_tri_en = true; + ul_tb_info->def_if_bandedge = + rtw89_phy_read32_mask(rtwdev, R_BANDEDGE, B_BANDEDGE_EN); +} + static void rtw89_phy_stat_thermal_update(struct rtw89_dev *rtwdev) { struct rtw89_phy_stat *phystat = &rtwdev->phystat; @@ -3340,7 +3464,7 @@ void rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev) static bool rtw89_physts_ie_page_valid(enum rtw89_phy_status_bitmap *ie_page) { - if (*ie_page > RTW89_PHYSTS_BITMAP_NUM || + if (*ie_page >= RTW89_PHYSTS_BITMAP_NUM || *ie_page == RTW89_RSVD_9) return false; else if (*ie_page > RTW89_RSVD_9) @@ -3980,6 +4104,7 @@ void rtw89_phy_dm_init(struct rtw89_dev *rtwdev) rtw89_physts_parsing_init(rtwdev); rtw89_phy_dig_init(rtwdev); rtw89_phy_cfo_init(rtwdev); + rtw89_phy_ul_tb_info_init(rtwdev); rtw89_phy_init_rf_nctl(rtwdev); rtw89_chip_rfk_init(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw89/phy.h b/drivers/net/wireless/realtek/rtw89/phy.h index dac69a02e868..21233f094644 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.h +++ b/drivers/net/wireless/realtek/rtw89/phy.h @@ -64,6 +64,9 @@ #define MAX_CFO_TOLERANCE 30 #define CFO_TF_CNT_TH 300 +#define UL_TB_TF_CNT_L2H_TH 100 +#define UL_TB_TF_CNT_H2L_TH 70 + #define CCX_MAX_PERIOD 2097 #define CCX_MAX_PERIOD_UNIT 32 #define MS_TO_4US_RATIO 250 @@ -550,5 +553,7 @@ void rtw89_phy_set_bss_color(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx, enum rtw89_tssi_bandedge_cfg bandedge_cfg); +void rtw89_phy_ul_tb_assoc(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); +void rtw89_phy_ul_tb_ctrl_track(struct rtw89_dev *rtwdev); #endif diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h index dbe06542f443..f2634062f377 100644 --- a/drivers/net/wireless/realtek/rtw89/reg.h +++ b/drivers/net/wireless/realtek/rtw89/reg.h @@ -4236,6 +4236,8 @@ #define R_P1_NBIIDX 0x4770 #define B_P1_NBIIDX_VAL GENMASK(11, 0) #define B_P1_NBIIDX_NOTCH_EN BIT(12) +#define R_PKT_CTRL 0x47D4 +#define B_PKT_POP_EN BIT(8) #define R_SEG0R_PD 0x481C #define R_SEG0R_PD_V1 0x4860 #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK_V1 BIT(30) diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 4cea5fb4327d..eff6519cf019 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -2082,6 +2082,7 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .support_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ), .support_bw160 = false, + .support_ul_tb_ctrl = false, .hw_sec_hdr = false, .rf_path_num = 2, .tx_nss = 2, @@ -2143,6 +2144,7 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .c2h_ctrl_reg = R_AX_C2HREG_CTRL, .c2h_regs = rtw8852a_c2h_regs, .page_regs = &rtw8852a_page_regs, + .cfo_src_fd = false, .dcfo_comp = &rtw8852a_dcfo_comp, .dcfo_comp_sft = 3, .imr_info = &rtw8852a_imr_info, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c index 0df044b1c392..b635ac1d1ca2 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852b.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c @@ -318,27 +318,27 @@ static const struct rtw89_dig_regs rtw8852b_dig_regs = { }; static const struct rtw89_btc_rf_trx_para rtw89_btc_8852b_rf_ul[] = { - {15, 0, 0, 7}, /* 0 -> original */ - {15, 2, 0, 7}, /* 1 -> for BT-connected ACI issue && BTG co-rx */ - {15, 0, 0, 7}, /* 2 ->reserved for shared-antenna */ - {15, 0, 0, 7}, /* 3- >reserved for shared-antenna */ - {15, 0, 0, 7}, /* 4 ->reserved for shared-antenna */ - {15, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */ + {255, 0, 0, 7}, /* 0 -> original */ + {255, 2, 0, 7}, /* 1 -> for BT-connected ACI issue && BTG co-rx */ + {255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */ + {255, 0, 0, 7}, /* 3- >reserved for shared-antenna */ + {255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */ + {255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */ {6, 1, 0, 7}, {13, 1, 0, 7}, {13, 1, 0, 7} }; static const struct rtw89_btc_rf_trx_para rtw89_btc_8852b_rf_dl[] = { - {15, 0, 0, 7}, /* 0 -> original */ - {15, 2, 0, 7}, /* 1 -> reserved for shared-antenna */ - {15, 0, 0, 7}, /* 2 ->reserved for shared-antenna */ - {15, 0, 0, 7}, /* 3- >reserved for shared-antenna */ - {15, 0, 0, 7}, /* 4 ->reserved for shared-antenna */ - {15, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */ - {15, 1, 0, 7}, - {15, 1, 0, 7}, - {15, 1, 0, 7} + {255, 0, 0, 7}, /* 0 -> original */ + {255, 2, 0, 7}, /* 1 -> reserved for shared-antenna */ + {255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */ + {255, 0, 0, 7}, /* 3- >reserved for shared-antenna */ + {255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */ + {255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */ + {255, 1, 0, 7}, + {255, 1, 0, 7}, + {255, 1, 0, 7} }; static const struct rtw89_btc_fbtc_mreg rtw89_btc_8852b_mon_reg[] = { @@ -1411,6 +1411,12 @@ static void rtw8852b_bb_sethw(struct rtw89_dev *rtwdev) rtw89_phy_read32_mask(rtwdev, R_P1_RPL1, B_P0_RPL1_BIAS_MASK); } +static void rtw8852b_bb_set_pop(struct rtw89_dev *rtwdev) +{ + if (rtwdev->hw->conf.flags & IEEE80211_CONF_MONITOR) + rtw89_phy_write32_clr(rtwdev, R_PKT_CTRL, B_PKT_POP_EN); +} + static void rtw8852b_set_channel_bb(struct rtw89_dev *rtwdev, const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { @@ -1441,6 +1447,7 @@ static void rtw8852b_set_channel_bb(struct rtw89_dev *rtwdev, const struct rtw89 rtw89_phy_write32_mask(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, chan->primary_channel); rtw8852b_5m_mask(rtwdev, chan, phy_idx); + rtw8852b_bb_set_pop(rtwdev); rtw8852b_bb_reset_all(rtwdev, phy_idx); } @@ -2452,6 +2459,7 @@ const struct rtw89_chip_info rtw8852b_chip_info = { .support_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ), .support_bw160 = false, + .support_ul_tb_ctrl = true, .hw_sec_hdr = false, .rf_path_num = 2, .tx_nss = 2, @@ -2512,6 +2520,7 @@ const struct rtw89_chip_info rtw8852b_chip_info = { .c2h_ctrl_reg = R_AX_C2HREG_CTRL, .c2h_regs = rtw8852b_c2h_regs, .page_regs = &rtw8852b_page_regs, + .cfo_src_fd = true, .dcfo_comp = &rtw8852b_dcfo_comp, .dcfo_comp_sft = 3, .imr_info = &rtw8852b_imr_info, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c index 6619ba730719..a87482cc25f5 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c @@ -1832,11 +1832,11 @@ static void rtw8852c_set_channel_help(struct rtw89_dev *rtwdev, bool enter, static void rtw8852c_rfk_init(struct rtw89_dev *rtwdev) { - struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; + struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc; rtwdev->is_tssi_mode[RF_PATH_A] = false; rtwdev->is_tssi_mode[RF_PATH_B] = false; - memset(mcc_info, 0, sizeof(*mcc_info)); + memset(rfk_mcc, 0, sizeof(*rfk_mcc)); rtw8852c_lck_init(rtwdev); rtw8852c_rck(rtwdev); @@ -2891,6 +2891,7 @@ const struct rtw89_chip_info rtw8852c_chip_info = { BIT(NL80211_BAND_5GHZ) | BIT(NL80211_BAND_6GHZ), .support_bw160 = true, + .support_ul_tb_ctrl = false, .hw_sec_hdr = true, .rf_path_num = 2, .tx_nss = 2, @@ -2953,6 +2954,7 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .c2h_ctrl_reg = R_AX_C2HREG_CTRL_V1, .c2h_regs = rtw8852c_c2h_regs, .page_regs = &rtw8852c_page_regs, + .cfo_src_fd = false, .dcfo_comp = &rtw8852c_dcfo_comp, .dcfo_comp_sft = 5, .imr_info = &rtw8852c_imr_info, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c index b0672b906e7b..60cd676fe22c 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c @@ -1030,9 +1030,9 @@ static bool _iqk_nbtxk(struct rtw89_dev *rtwdev, static bool _lok_finetune_check(struct rtw89_dev *rtwdev, u8 path) { - struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; + struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc; struct rtw89_iqk_info *iqk_info = &rtwdev->iqk; - u8 idx = mcc_info->table_idx; + u8 idx = rfk_mcc->table_idx; bool is_fail1, is_fail2; u32 val; u32 core_i; @@ -1375,10 +1375,10 @@ static void _iqk_afebb_restore(struct rtw89_dev *rtwdev, static void _iqk_preset(struct rtw89_dev *rtwdev, u8 path) { - struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; + struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc; u8 idx = 0; - idx = mcc_info->table_idx; + idx = rfk_mcc->table_idx; rtw89_phy_write32_mask(rtwdev, R_COEF_SEL + (path << 8), B_COEF_SEL_IQC, idx); rtw89_phy_write32_mask(rtwdev, R_CFIR_LUT + (path << 8), B_CFIR_LUT_G3, idx); rtw89_write_rf(rtwdev, path, RR_RSV1, RR_RSV1_RST, 0x0); @@ -3824,20 +3824,20 @@ void rtw8852c_set_channel_rf(struct rtw89_dev *rtwdev, void rtw8852c_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); - struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; - u8 idx = mcc_info->table_idx; + struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc; + u8 idx = rfk_mcc->table_idx; int i; for (i = 0; i < RTW89_IQK_CHS_NR; i++) { - if (mcc_info->ch[idx] == 0) + if (rfk_mcc->ch[idx] == 0) break; if (++idx >= RTW89_IQK_CHS_NR) idx = 0; } - mcc_info->table_idx = idx; - mcc_info->ch[idx] = chan->channel; - mcc_info->band[idx] = chan->band_type; + rfk_mcc->table_idx = idx; + rfk_mcc->ch[idx] = chan->channel; + rfk_mcc->band[idx] = chan->band_type; } void rtw8852c_rck(struct rtw89_dev *rtwdev) diff --git a/drivers/net/wireless/realtek/rtw89/txrx.h b/drivers/net/wireless/realtek/rtw89/txrx.h index b889e7bf34c0..9d4c6b6fa125 100644 --- a/drivers/net/wireless/realtek/rtw89/txrx.h +++ b/drivers/net/wireless/realtek/rtw89/txrx.h @@ -298,7 +298,9 @@ le32_get_bits(*((const __le32 *)ie), GENMASK(11, 5)) #define RTW89_GET_PHY_STS_IE01_CH_IDX(ie) \ le32_get_bits(*((const __le32 *)ie), GENMASK(23, 16)) -#define RTW89_GET_PHY_STS_IE01_CFO(ie) \ +#define RTW89_GET_PHY_STS_IE01_FD_CFO(ie) \ + le32_get_bits(*((const __le32 *)(ie) + 1), GENMASK(19, 8)) +#define RTW89_GET_PHY_STS_IE01_PREMB_CFO(ie) \ le32_get_bits(*((const __le32 *)(ie) + 1), GENMASK(31, 20)) enum rtw89_tx_channel { diff --git a/drivers/net/wwan/iosm/iosm_ipc_mux.c b/drivers/net/wwan/iosm/iosm_ipc_mux.c index 9c7a9a2a1f25..fc928b298a98 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_mux.c +++ b/drivers/net/wwan/iosm/iosm_ipc_mux.c @@ -332,6 +332,7 @@ struct iosm_mux *ipc_mux_init(struct ipc_mux_config *mux_cfg, if (!ipc_mux->ul_adb.pp_qlt[i]) { for (j = i - 1; j >= 0; j--) kfree(ipc_mux->ul_adb.pp_qlt[j]); + kfree(ipc_mux); return NULL; } } diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 1545cbee77a4..3dbfc8a6924e 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -386,7 +386,7 @@ int xenvif_dealloc_kthread(void *data); irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data); bool xenvif_have_rx_work(struct xenvif_queue *queue, bool test_kthread); -void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); +bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); void xenvif_carrier_on(struct xenvif *vif); diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 650fa180220f..f3f2c07423a6 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -254,14 +254,16 @@ xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev) if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) skb_clear_hash(skb); - xenvif_rx_queue_tail(queue, skb); + if (!xenvif_rx_queue_tail(queue, skb)) + goto drop; + xenvif_kick_thread(queue); return NETDEV_TX_OK; drop: vif->dev->stats.tx_dropped++; - dev_kfree_skb(skb); + dev_kfree_skb_any(skb); return NETDEV_TX_OK; } diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 3d2081bbbc86..bf627af723bf 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -332,10 +332,13 @@ static int xenvif_count_requests(struct xenvif_queue *queue, struct xenvif_tx_cb { - u16 pending_idx; + u16 copy_pending_idx[XEN_NETBK_LEGACY_SLOTS_MAX + 1]; + u8 copy_count; }; #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb) +#define copy_pending_idx(skb, i) (XENVIF_TX_CB(skb)->copy_pending_idx[i]) +#define copy_count(skb) (XENVIF_TX_CB(skb)->copy_count) static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue, u16 pending_idx, @@ -370,31 +373,93 @@ static inline struct sk_buff *xenvif_alloc_skb(unsigned int size) return skb; } -static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *queue, - struct sk_buff *skb, - struct xen_netif_tx_request *txp, - struct gnttab_map_grant_ref *gop, - unsigned int frag_overflow, - struct sk_buff *nskb) +static void xenvif_get_requests(struct xenvif_queue *queue, + struct sk_buff *skb, + struct xen_netif_tx_request *first, + struct xen_netif_tx_request *txfrags, + unsigned *copy_ops, + unsigned *map_ops, + unsigned int frag_overflow, + struct sk_buff *nskb, + unsigned int extra_count, + unsigned int data_len) { struct skb_shared_info *shinfo = skb_shinfo(skb); skb_frag_t *frags = shinfo->frags; - u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx; - int start; + u16 pending_idx; pending_ring_idx_t index; unsigned int nr_slots; + struct gnttab_copy *cop = queue->tx_copy_ops + *copy_ops; + struct gnttab_map_grant_ref *gop = queue->tx_map_ops + *map_ops; + struct xen_netif_tx_request *txp = first; + + nr_slots = shinfo->nr_frags + 1; + + copy_count(skb) = 0; + + /* Create copy ops for exactly data_len bytes into the skb head. */ + __skb_put(skb, data_len); + while (data_len > 0) { + int amount = data_len > txp->size ? txp->size : data_len; + + cop->source.u.ref = txp->gref; + cop->source.domid = queue->vif->domid; + cop->source.offset = txp->offset; + + cop->dest.domid = DOMID_SELF; + cop->dest.offset = (offset_in_page(skb->data + + skb_headlen(skb) - + data_len)) & ~XEN_PAGE_MASK; + cop->dest.u.gmfn = virt_to_gfn(skb->data + skb_headlen(skb) + - data_len); + + cop->len = amount; + cop->flags = GNTCOPY_source_gref; - nr_slots = shinfo->nr_frags; + index = pending_index(queue->pending_cons); + pending_idx = queue->pending_ring[index]; + callback_param(queue, pending_idx).ctx = NULL; + copy_pending_idx(skb, copy_count(skb)) = pending_idx; + copy_count(skb)++; + + cop++; + data_len -= amount; - /* Skip first skb fragment if it is on same page as header fragment. */ - start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); + if (amount == txp->size) { + /* The copy op covered the full tx_request */ + + memcpy(&queue->pending_tx_info[pending_idx].req, + txp, sizeof(*txp)); + queue->pending_tx_info[pending_idx].extra_count = + (txp == first) ? extra_count : 0; + + if (txp == first) + txp = txfrags; + else + txp++; + queue->pending_cons++; + nr_slots--; + } else { + /* The copy op partially covered the tx_request. + * The remainder will be mapped. + */ + txp->offset += amount; + txp->size -= amount; + } + } - for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots; - shinfo->nr_frags++, txp++, gop++) { + for (shinfo->nr_frags = 0; shinfo->nr_frags < nr_slots; + shinfo->nr_frags++, gop++) { index = pending_index(queue->pending_cons++); pending_idx = queue->pending_ring[index]; - xenvif_tx_create_map_op(queue, pending_idx, txp, 0, gop); + xenvif_tx_create_map_op(queue, pending_idx, txp, + txp == first ? extra_count : 0, gop); frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); + + if (txp == first) + txp = txfrags; + else + txp++; } if (frag_overflow) { @@ -415,7 +480,8 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *que skb_shinfo(skb)->frag_list = nskb; } - return gop; + (*copy_ops) = cop - queue->tx_copy_ops; + (*map_ops) = gop - queue->tx_map_ops; } static inline void xenvif_grant_handle_set(struct xenvif_queue *queue, @@ -451,7 +517,7 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, struct gnttab_copy **gopp_copy) { struct gnttab_map_grant_ref *gop_map = *gopp_map; - u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx; + u16 pending_idx; /* This always points to the shinfo of the skb being checked, which * could be either the first or the one on the frag_list */ @@ -462,24 +528,37 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, struct skb_shared_info *first_shinfo = NULL; int nr_frags = shinfo->nr_frags; const bool sharedslot = nr_frags && - frag_get_pending_idx(&shinfo->frags[0]) == pending_idx; - int i, err; + frag_get_pending_idx(&shinfo->frags[0]) == + copy_pending_idx(skb, copy_count(skb) - 1); + int i, err = 0; - /* Check status of header. */ - err = (*gopp_copy)->status; - if (unlikely(err)) { - if (net_ratelimit()) - netdev_dbg(queue->vif->dev, - "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", - (*gopp_copy)->status, - pending_idx, - (*gopp_copy)->source.u.ref); - /* The first frag might still have this slot mapped */ - if (!sharedslot) - xenvif_idx_release(queue, pending_idx, - XEN_NETIF_RSP_ERROR); + for (i = 0; i < copy_count(skb); i++) { + int newerr; + + /* Check status of header. */ + pending_idx = copy_pending_idx(skb, i); + + newerr = (*gopp_copy)->status; + if (likely(!newerr)) { + /* The first frag might still have this slot mapped */ + if (i < copy_count(skb) - 1 || !sharedslot) + xenvif_idx_release(queue, pending_idx, + XEN_NETIF_RSP_OKAY); + } else { + err = newerr; + if (net_ratelimit()) + netdev_dbg(queue->vif->dev, + "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", + (*gopp_copy)->status, + pending_idx, + (*gopp_copy)->source.u.ref); + /* The first frag might still have this slot mapped */ + if (i < copy_count(skb) - 1 || !sharedslot) + xenvif_idx_release(queue, pending_idx, + XEN_NETIF_RSP_ERROR); + } + (*gopp_copy)++; } - (*gopp_copy)++; check_frags: for (i = 0; i < nr_frags; i++, gop_map++) { @@ -526,14 +605,6 @@ check_frags: if (err) continue; - /* First error: if the header haven't shared a slot with the - * first frag, release it as well. - */ - if (!sharedslot) - xenvif_idx_release(queue, - XENVIF_TX_CB(skb)->pending_idx, - XEN_NETIF_RSP_OKAY); - /* Invalidate preceding fragments of this skb. */ for (j = 0; j < i; j++) { pending_idx = frag_get_pending_idx(&shinfo->frags[j]); @@ -803,7 +874,6 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, unsigned *copy_ops, unsigned *map_ops) { - struct gnttab_map_grant_ref *gop = queue->tx_map_ops; struct sk_buff *skb, *nskb; int ret; unsigned int frag_overflow; @@ -885,8 +955,12 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, continue; } + data_len = (txreq.size > XEN_NETBACK_TX_COPY_LEN) ? + XEN_NETBACK_TX_COPY_LEN : txreq.size; + ret = xenvif_count_requests(queue, &txreq, extra_count, txfrags, work_to_do); + if (unlikely(ret < 0)) break; @@ -912,9 +986,8 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, index = pending_index(queue->pending_cons); pending_idx = queue->pending_ring[index]; - data_len = (txreq.size > XEN_NETBACK_TX_COPY_LEN && - ret < XEN_NETBK_LEGACY_SLOTS_MAX) ? - XEN_NETBACK_TX_COPY_LEN : txreq.size; + if (ret >= XEN_NETBK_LEGACY_SLOTS_MAX - 1 && data_len < txreq.size) + data_len = txreq.size; skb = xenvif_alloc_skb(data_len); if (unlikely(skb == NULL)) { @@ -925,8 +998,6 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, } skb_shinfo(skb)->nr_frags = ret; - if (data_len < txreq.size) - skb_shinfo(skb)->nr_frags++; /* At this point shinfo->nr_frags is in fact the number of * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX. */ @@ -988,54 +1059,19 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, type); } - XENVIF_TX_CB(skb)->pending_idx = pending_idx; - - __skb_put(skb, data_len); - queue->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref; - queue->tx_copy_ops[*copy_ops].source.domid = queue->vif->domid; - queue->tx_copy_ops[*copy_ops].source.offset = txreq.offset; - - queue->tx_copy_ops[*copy_ops].dest.u.gmfn = - virt_to_gfn(skb->data); - queue->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF; - queue->tx_copy_ops[*copy_ops].dest.offset = - offset_in_page(skb->data) & ~XEN_PAGE_MASK; - - queue->tx_copy_ops[*copy_ops].len = data_len; - queue->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref; - - (*copy_ops)++; - - if (data_len < txreq.size) { - frag_set_pending_idx(&skb_shinfo(skb)->frags[0], - pending_idx); - xenvif_tx_create_map_op(queue, pending_idx, &txreq, - extra_count, gop); - gop++; - } else { - frag_set_pending_idx(&skb_shinfo(skb)->frags[0], - INVALID_PENDING_IDX); - memcpy(&queue->pending_tx_info[pending_idx].req, - &txreq, sizeof(txreq)); - queue->pending_tx_info[pending_idx].extra_count = - extra_count; - } - - queue->pending_cons++; - - gop = xenvif_get_requests(queue, skb, txfrags, gop, - frag_overflow, nskb); + xenvif_get_requests(queue, skb, &txreq, txfrags, copy_ops, + map_ops, frag_overflow, nskb, extra_count, + data_len); __skb_queue_tail(&queue->tx_queue, skb); queue->tx.req_cons = idx; - if (((gop-queue->tx_map_ops) >= ARRAY_SIZE(queue->tx_map_ops)) || + if ((*map_ops >= ARRAY_SIZE(queue->tx_map_ops)) || (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops))) break; } - (*map_ops) = gop - queue->tx_map_ops; return; } @@ -1114,9 +1150,8 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) { struct xen_netif_tx_request *txp; u16 pending_idx; - unsigned data_len; - pending_idx = XENVIF_TX_CB(skb)->pending_idx; + pending_idx = copy_pending_idx(skb, 0); txp = &queue->pending_tx_info[pending_idx].req; /* Check the remap error code. */ @@ -1135,18 +1170,6 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) continue; } - data_len = skb->len; - callback_param(queue, pending_idx).ctx = NULL; - if (data_len < txp->size) { - /* Append the packet payload as a fragment. */ - txp->offset += data_len; - txp->size -= data_len; - } else { - /* Schedule a response immediately. */ - xenvif_idx_release(queue, pending_idx, - XEN_NETIF_RSP_OKAY); - } - if (txp->flags & XEN_NETTXF_csum_blank) skb->ip_summed = CHECKSUM_PARTIAL; else if (txp->flags & XEN_NETTXF_data_validated) @@ -1332,7 +1355,7 @@ static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) /* Called after netfront has transmitted */ int xenvif_tx_action(struct xenvif_queue *queue, int budget) { - unsigned nr_mops, nr_cops = 0; + unsigned nr_mops = 0, nr_cops = 0; int work_done, ret; if (unlikely(!tx_work_todo(queue))) diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c index 932762177110..0ba754ebc5ba 100644 --- a/drivers/net/xen-netback/rx.c +++ b/drivers/net/xen-netback/rx.c @@ -82,9 +82,10 @@ static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue) return false; } -void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb) +bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb) { unsigned long flags; + bool ret = true; spin_lock_irqsave(&queue->rx_queue.lock, flags); @@ -92,8 +93,7 @@ void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb) struct net_device *dev = queue->vif->dev; netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); - kfree_skb(skb); - queue->vif->dev->stats.rx_dropped++; + ret = false; } else { if (skb_queue_empty(&queue->rx_queue)) xenvif_update_needed_slots(queue, skb); @@ -104,6 +104,8 @@ void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb) } spin_unlock_irqrestore(&queue->rx_queue.lock, flags); + + return ret; } static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue) diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index ef4e53bf5604..14aec417fa06 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1862,6 +1862,12 @@ static int netfront_resume(struct xenbus_device *dev) netif_tx_unlock_bh(info->netdev); xennet_disconnect_backend(info); + + rtnl_lock(); + if (info->queues) + xennet_destroy_queues(info); + rtnl_unlock(); + return 0; } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index da55ce45ac70..69e333922bea 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4304,7 +4304,7 @@ static void nvme_ns_remove(struct nvme_ns *ns) mutex_unlock(&ns->ctrl->subsys->lock); /* guarantee not available in head->list */ - synchronize_rcu(); + synchronize_srcu(&ns->head->srcu); if (!nvme_ns_head_multipath(ns->head)) nvme_cdev_del(&ns->cdev, &ns->cdev_device); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 93e2138a8b42..7e025b8948cb 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -174,11 +174,14 @@ void nvme_mpath_revalidate_paths(struct nvme_ns *ns) struct nvme_ns_head *head = ns->head; sector_t capacity = get_capacity(head->disk); int node; + int srcu_idx; + srcu_idx = srcu_read_lock(&head->srcu); list_for_each_entry_rcu(ns, &head->list, siblings) { if (capacity != get_capacity(ns->disk)) clear_bit(NVME_NS_READY, &ns->flags); } + srcu_read_unlock(&head->srcu, srcu_idx); for_each_node(node) rcu_assign_pointer(head->current_path[node], NULL); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index f4335519399d..488ad7dabeb8 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -797,6 +797,8 @@ static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev, cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); if (bv->bv_len > first_prp_len) cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); + else + cmnd->dptr.prp2 = 0; return BLK_STS_OK; } diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index 52ecd66ce357..047a8374b4fd 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -436,9 +436,14 @@ static void __intel_gpio_set_direction(void __iomem *padcfg0, bool input) writel(value, padcfg0); } +static int __intel_gpio_get_gpio_mode(u32 value) +{ + return (value & PADCFG0_PMODE_MASK) >> PADCFG0_PMODE_SHIFT; +} + static int intel_gpio_get_gpio_mode(void __iomem *padcfg0) { - return (readl(padcfg0) & PADCFG0_PMODE_MASK) >> PADCFG0_PMODE_SHIFT; + return __intel_gpio_get_gpio_mode(readl(padcfg0)); } static void intel_gpio_set_gpio_mode(void __iomem *padcfg0) @@ -1674,6 +1679,7 @@ EXPORT_SYMBOL_GPL(intel_pinctrl_get_soc_data); static bool intel_pinctrl_should_save(struct intel_pinctrl *pctrl, unsigned int pin) { const struct pin_desc *pd = pin_desc_get(pctrl->pctldev, pin); + u32 value; if (!pd || !intel_pad_usable(pctrl, pin)) return false; @@ -1688,6 +1694,25 @@ static bool intel_pinctrl_should_save(struct intel_pinctrl *pctrl, unsigned int gpiochip_line_is_irq(&pctrl->chip, intel_pin_to_gpio(pctrl, pin))) return true; + /* + * The firmware on some systems may configure GPIO pins to be + * an interrupt source in so called "direct IRQ" mode. In such + * cases the GPIO controller driver has no idea if those pins + * are being used or not. At the same time, there is a known bug + * in the firmwares that don't restore the pin settings correctly + * after suspend, i.e. by an unknown reason the Rx value becomes + * inverted. + * + * Hence, let's save and restore the pins that are configured + * as GPIOs in the input mode with GPIROUTIOXAPIC bit set. + * + * See https://bugzilla.kernel.org/show_bug.cgi?id=214749. + */ + value = readl(intel_get_padcfg(pctrl, pin, PADCFG0)); + if ((value & PADCFG0_GPIROUTIOXAPIC) && (value & PADCFG0_GPIOTXDIS) && + (__intel_gpio_get_gpio_mode(value) == PADCFG0_PMODE_GPIO)) + return true; + return false; } diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 65d312967619..27f0a54e12bf 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -303,12 +303,15 @@ static struct irq_chip mtk_eint_irq_chip = { static unsigned int mtk_eint_hw_init(struct mtk_eint *eint) { - void __iomem *reg = eint->base + eint->regs->dom_en; + void __iomem *dom_en = eint->base + eint->regs->dom_en; + void __iomem *mask_set = eint->base + eint->regs->mask_set; unsigned int i; for (i = 0; i < eint->hw->ap_num; i += 32) { - writel(0xffffffff, reg); - reg += 4; + writel(0xffffffff, dom_en); + writel(0xffffffff, mask_set); + dom_en += 4; + mask_set += 4; } return 0; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 67bec7ea0f8b..414ee6bb8ac9 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -727,7 +727,7 @@ static int pcs_allocate_pin_table(struct pcs_device *pcs) mux_bytes = pcs->width / BITS_PER_BYTE; - if (pcs->bits_per_mux) { + if (pcs->bits_per_mux && pcs->fmask) { pcs->bits_per_pin = fls(pcs->fmask); nr_pins = (pcs->size * BITS_PER_BYTE) / pcs->bits_per_pin; } else { diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c index ef4ae977b8e0..439d282aafd1 100644 --- a/drivers/platform/x86/amd/pmc.c +++ b/drivers/platform/x86/amd/pmc.c @@ -739,8 +739,14 @@ static void amd_pmc_s2idle_prepare(void) static void amd_pmc_s2idle_check(void) { struct amd_pmc_dev *pdev = &pmc; + struct smu_metrics table; int rc; + /* CZN: Ensure that future s0i3 entry attempts at least 10ms passed */ + if (pdev->cpu_id == AMD_CPU_ID_CZN && !get_metrics_table(pdev, &table) && + table.s0i3_last_entry_status) + usleep_range(10000, 20000); + /* Dump the IdleMask before we add to the STB */ amd_pmc_idlemask_read(pdev, pdev->dev, NULL); diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index 154d58cbd9ce..4bbaccd543ad 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -1647,10 +1647,6 @@ ptp_ocp_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, char buf[32]; int err; - err = devlink_info_driver_name_put(req, KBUILD_MODNAME); - if (err) - return err; - fw_image = bp->fw_loader ? "loader" : "fw"; sprintf(buf, "%d.%d", bp->fw_tag, bp->fw_version); err = devlink_info_version_running_put(req, fw_image, buf); diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index 9dc935886e9f..c6ded3fdd715 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -758,7 +758,6 @@ static void qeth_l2_br2dev_worker(struct work_struct *work) struct list_head *iter; int err = 0; - kfree(br2dev_event_work); QETH_CARD_TEXT_(card, 4, "b2dw%04lx", event); QETH_CARD_TEXT_(card, 4, "ma%012llx", ether_addr_to_u64(addr)); @@ -815,6 +814,7 @@ unlock: dev_put(brdev); dev_put(lsyncdev); dev_put(dstdev); + kfree(br2dev_event_work); } static int qeth_l2_br2dev_queue_work(struct net_device *brdev, diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c index 4d59d927ae3e..127fac1af676 100644 --- a/drivers/usb/core/message.c +++ b/drivers/usb/core/message.c @@ -1037,6 +1037,7 @@ char *usb_cache_string(struct usb_device *udev, int index) } return smallbuf; } +EXPORT_SYMBOL_GPL(usb_cache_string); /* * usb_get_device_descriptor - (re)reads the device descriptor (usbcore) diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index 82538daac8b8..0eac7d4285d1 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -47,7 +47,6 @@ extern int usb_get_device_descriptor(struct usb_device *dev, extern int usb_set_isoch_delay(struct usb_device *dev); extern int usb_get_bos_descriptor(struct usb_device *dev); extern void usb_release_bos_descriptor(struct usb_device *dev); -extern char *usb_cache_string(struct usb_device *udev, int index); extern int usb_set_configuration(struct usb_device *dev, int configuration); extern int usb_choose_configuration(struct usb_device *udev); extern int usb_generic_driver_probe(struct usb_device *udev); diff --git a/fs/afs/server.c b/fs/afs/server.c index 4981baf97835..b5237206eac3 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -406,7 +406,7 @@ void afs_put_server(struct afs_net *net, struct afs_server *server, if (!server) return; - a = atomic_inc_return(&server->active); + a = atomic_read(&server->active); zero = __refcount_dec_and_test(&server->ref, &r); trace_afs_server(debug_id, r - 1, a, reason); if (unlikely(zero)) diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 451d8a077e12..bce2492186d0 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -605,6 +605,14 @@ again: set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); queue = true; } + /* + * We could race with cookie_lru which may set LRU_DISCARD bit + * but has yet to run the cookie state machine. If this happens + * and another thread tries to use the cookie, clear LRU_DISCARD + * so we don't end up withdrawing the cookie while in use. + */ + if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear); break; case FSCACHE_COOKIE_STATE_FAILED: diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index 3b55e239705f..9930fa901039 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -111,6 +111,13 @@ static void nilfs_dat_commit_free(struct inode *dat, kunmap_atomic(kaddr); nilfs_dat_commit_entry(dat, req); + + if (unlikely(req->pr_desc_bh == NULL || req->pr_bitmap_bh == NULL)) { + nilfs_error(dat->i_sb, + "state inconsistency probably due to duplicate use of vblocknr = %llu", + (unsigned long long)req->pr_entry_nr); + return; + } nilfs_palloc_commit_free_entry(dat, req); } diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 492dce43236e..cab7cfebf40b 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,12 +222,16 @@ extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #define tlb_needs_table_invalidate() (true) #endif +void tlb_remove_table_sync_one(void); + #else #ifdef tlb_needs_table_invalidate #error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE #endif +static inline void tlb_remove_table_sync_one(void) { } + #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 528bd44b59e2..2b7d077de7ef 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -68,6 +68,7 @@ struct css_task_iter { struct list_head iters_node; /* css_set->task_iters */ }; +extern struct file_system_type cgroup_fs_type; extern struct cgroup_root cgrp_dfl_root; extern struct css_set init_css_set; diff --git a/include/linux/gfp.h b/include/linux/gfp.h index ef4aea3b356e..65a78773dcca 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -210,6 +210,20 @@ alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct p return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array); } +static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) +{ + gfp_t warn_gfp = gfp_mask & (__GFP_THISNODE|__GFP_NOWARN); + + if (warn_gfp != (__GFP_THISNODE|__GFP_NOWARN)) + return; + + if (node_online(this_node)) + return; + + pr_warn("%pGg allocation from offline node %d\n", &gfp_mask, this_node); + dump_stack(); +} + /* * Allocate pages, preferring the node given as nid. The node must be valid and * online. For more general interface, see alloc_pages_node(). @@ -218,7 +232,7 @@ static inline struct page * __alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order) { VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - VM_WARN_ON((gfp_mask & __GFP_THISNODE) && !node_online(nid)); + warn_if_node_offline(nid, gfp_mask); return __alloc_pages(gfp_mask, order, nid, NULL); } @@ -227,7 +241,7 @@ static inline struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid) { VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - VM_WARN_ON((gfp & __GFP_THISNODE) && !node_online(nid)); + warn_if_node_offline(nid, gfp); return __folio_alloc(gfp, order, nid, NULL); } diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 6252f02f38b7..80d6308dea06 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -338,6 +338,17 @@ struct ieee80211_qos_hdr { __le16 qos_ctrl; } __packed __aligned(2); +struct ieee80211_qos_hdr_4addr { + __le16 frame_control; + __le16 duration_id; + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; + u8 addr3[ETH_ALEN]; + __le16 seq_ctrl; + u8 addr4[ETH_ALEN]; + __le16 qos_ctrl; +} __packed __aligned(2); + struct ieee80211_trigger { __le16 frame_control; __le16 duration; @@ -4060,16 +4071,21 @@ struct ieee80211_he_6ghz_capa { * @hdr: the frame * * The qos ctrl bytes come after the frame_control, duration, seq_num - * and 3 or 4 addresses of length ETH_ALEN. - * 3 addr: 2 + 2 + 2 + 3*6 = 24 - * 4 addr: 2 + 2 + 2 + 4*6 = 30 + * and 3 or 4 addresses of length ETH_ALEN. Checks frame_control to choose + * between struct ieee80211_qos_hdr_4addr and struct ieee80211_qos_hdr. */ static inline u8 *ieee80211_get_qos_ctl(struct ieee80211_hdr *hdr) { - if (ieee80211_has_a4(hdr->frame_control)) - return (u8 *)hdr + 30; + union { + struct ieee80211_qos_hdr addr3; + struct ieee80211_qos_hdr_4addr addr4; + } *qos; + + qos = (void *)hdr; + if (ieee80211_has_a4(qos->addr3.frame_control)) + return (u8 *)&qos->addr4.qos_ctrl; else - return (u8 *)hdr + 24; + return (u8 *)&qos->addr3.qos_ctrl; } /** diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 570831ca9951..4e968ebadce6 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -224,9 +224,10 @@ extern bool arch_jump_label_transform_queue(struct jump_entry *entry, enum jump_label_type type); extern void arch_jump_label_transform_apply(void); extern int jump_label_text_reserved(void *start, void *end); -extern void static_key_slow_inc(struct static_key *key); +extern bool static_key_slow_inc(struct static_key *key); +extern bool static_key_fast_inc_not_disabled(struct static_key *key); extern void static_key_slow_dec(struct static_key *key); -extern void static_key_slow_inc_cpuslocked(struct static_key *key); +extern bool static_key_slow_inc_cpuslocked(struct static_key *key); extern void static_key_slow_dec_cpuslocked(struct static_key *key); extern int static_key_count(struct static_key *key); extern void static_key_enable(struct static_key *key); @@ -278,11 +279,23 @@ static __always_inline bool static_key_true(struct static_key *key) return false; } -static inline void static_key_slow_inc(struct static_key *key) +static inline bool static_key_fast_inc_not_disabled(struct static_key *key) { + int v; + STATIC_KEY_CHECK_USE(key); - atomic_inc(&key->enabled); + /* + * Prevent key->enabled getting negative to follow the same semantics + * as for CONFIG_JUMP_LABEL=y, see kernel/jump_label.c comment. + */ + v = atomic_read(&key->enabled); + do { + if (v < 0 || (v + 1) < 0) + return false; + } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1))); + return true; } +#define static_key_slow_inc(key) static_key_fast_inc_not_disabled(key) static inline void static_key_slow_dec(struct static_key *key) { diff --git a/include/linux/license.h b/include/linux/license.h index ad937f57f2cb..7cce390f120b 100644 --- a/include/linux/license.h +++ b/include/linux/license.h @@ -2,8 +2,6 @@ #ifndef __LICENSE_H #define __LICENSE_H -#include <linux/string.h> - static inline int license_is_gpl_compatible(const char *license) { return (strcmp(license, "GPL") == 0 diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index eb3fac30488b..5fe5d198b57a 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -290,10 +290,9 @@ enum { MLX5_UMR_INLINE = (1 << 7), }; -#define MLX5_UMR_KLM_ALIGNMENT 4 -#define MLX5_UMR_MTT_ALIGNMENT 0x40 -#define MLX5_UMR_MTT_MASK (MLX5_UMR_MTT_ALIGNMENT - 1) -#define MLX5_UMR_MTT_MIN_CHUNK_SIZE MLX5_UMR_MTT_ALIGNMENT +#define MLX5_UMR_FLEX_ALIGNMENT 0x40 +#define MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_mtt)) +#define MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_klm)) #define MLX5_USER_INDEX_LEN (MLX5_FLD_SZ_BYTES(qpc, user_index) * 8) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 06cbad166225..d476255c9a3f 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -606,8 +606,6 @@ struct mlx5_priv { struct list_head pgdir_list; /* end: alloc staff */ - struct list_head ctx_list; - spinlock_t ctx_lock; struct mlx5_adev **adev; int adev_idx; int sw_vhca_id; diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index c7a91981cd5a..ba6958b49a8e 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -50,6 +50,7 @@ enum mlx5_flow_destination_type { MLX5_FLOW_DESTINATION_TYPE_PORT, MLX5_FLOW_DESTINATION_TYPE_COUNTER, MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM, + MLX5_FLOW_DESTINATION_TYPE_RANGE, }; enum { @@ -143,6 +144,10 @@ enum { MLX5_FLOW_DEST_VPORT_REFORMAT_ID = BIT(1), }; +enum mlx5_flow_dest_range_field { + MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN = 0, +}; + struct mlx5_flow_destination { enum mlx5_flow_destination_type type; union { @@ -156,6 +161,13 @@ struct mlx5_flow_destination { struct mlx5_pkt_reformat *pkt_reformat; u8 flags; } vport; + struct { + struct mlx5_flow_table *hit_ft; + struct mlx5_flow_table *miss_ft; + enum mlx5_flow_dest_range_field field; + u32 min; + u32 max; + } range; u32 sampler_id; }; }; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 5a4e914e2a6f..152d2d7f8743 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -68,6 +68,7 @@ enum { MLX5_SET_HCA_CAP_OP_MOD_ODP = 0x2, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC = 0x3, MLX5_SET_HCA_CAP_OP_MOD_ROCE = 0x4, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2 = 0x20, MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION = 0x25, }; @@ -445,7 +446,10 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 max_modify_header_actions[0x8]; u8 max_ft_level[0x8]; - u8 reserved_at_40[0x6]; + u8 reformat_add_esp_trasport[0x1]; + u8 reserved_at_41[0x2]; + u8 reformat_del_esp_trasport[0x1]; + u8 reserved_at_44[0x2]; u8 execute_aso[0x1]; u8 reserved_at_47[0x19]; @@ -638,8 +642,10 @@ struct mlx5_ifc_fte_match_set_misc2_bits { u8 reserved_at_1a0[0x8]; u8 macsec_syndrome[0x8]; + u8 ipsec_syndrome[0x8]; + u8 reserved_at_1b8[0x8]; - u8 reserved_at_1b0[0x50]; + u8 reserved_at_1c0[0x40]; }; struct mlx5_ifc_fte_match_set_misc3_bits { @@ -1875,7 +1881,10 @@ struct mlx5_ifc_cmd_hca_cap_bits { }; struct mlx5_ifc_cmd_hca_cap_2_bits { - u8 reserved_at_0[0xa0]; + u8 reserved_at_0[0x80]; + + u8 migratable[0x1]; + u8 reserved_at_81[0x1f]; u8 max_reformat_insert_size[0x8]; u8 max_reformat_insert_offset[0x8]; @@ -6104,6 +6113,38 @@ struct mlx5_ifc_match_definer_format_32_bits { u8 inner_dmac_15_0[0x10]; }; +enum { + MLX5_IFC_DEFINER_FORMAT_ID_SELECT = 61, +}; + +#define MLX5_IFC_DEFINER_FORMAT_OFFSET_UNUSED 0x0 +#define MLX5_IFC_DEFINER_FORMAT_OFFSET_OUTER_ETH_PKT_LEN 0x48 +#define MLX5_IFC_DEFINER_DW_SELECTORS_NUM 9 +#define MLX5_IFC_DEFINER_BYTE_SELECTORS_NUM 8 + +struct mlx5_ifc_match_definer_match_mask_bits { + u8 reserved_at_1c0[5][0x20]; + u8 match_dw_8[0x20]; + u8 match_dw_7[0x20]; + u8 match_dw_6[0x20]; + u8 match_dw_5[0x20]; + u8 match_dw_4[0x20]; + u8 match_dw_3[0x20]; + u8 match_dw_2[0x20]; + u8 match_dw_1[0x20]; + u8 match_dw_0[0x20]; + + u8 match_byte_7[0x8]; + u8 match_byte_6[0x8]; + u8 match_byte_5[0x8]; + u8 match_byte_4[0x8]; + + u8 match_byte_3[0x8]; + u8 match_byte_2[0x8]; + u8 match_byte_1[0x8]; + u8 match_byte_0[0x8]; +}; + struct mlx5_ifc_match_definer_bits { u8 modify_field_select[0x40]; @@ -6112,9 +6153,41 @@ struct mlx5_ifc_match_definer_bits { u8 reserved_at_80[0x10]; u8 format_id[0x10]; - u8 reserved_at_a0[0x160]; + u8 reserved_at_a0[0x60]; + + u8 format_select_dw3[0x8]; + u8 format_select_dw2[0x8]; + u8 format_select_dw1[0x8]; + u8 format_select_dw0[0x8]; + + u8 format_select_dw7[0x8]; + u8 format_select_dw6[0x8]; + u8 format_select_dw5[0x8]; + u8 format_select_dw4[0x8]; + + u8 reserved_at_100[0x18]; + u8 format_select_dw8[0x8]; + + u8 reserved_at_120[0x20]; + + u8 format_select_byte3[0x8]; + u8 format_select_byte2[0x8]; + u8 format_select_byte1[0x8]; + u8 format_select_byte0[0x8]; + + u8 format_select_byte7[0x8]; + u8 format_select_byte6[0x8]; + u8 format_select_byte5[0x8]; + u8 format_select_byte4[0x8]; + + u8 reserved_at_180[0x40]; - u8 match_mask[16][0x20]; + union { + struct { + u8 match_mask[16][0x20]; + }; + struct mlx5_ifc_match_definer_match_mask_bits match_mask_format; + }; }; struct mlx5_ifc_general_obj_in_cmd_hdr_bits { @@ -6384,6 +6457,9 @@ enum mlx5_reformat_ctx_type { MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2, MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3, MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5, + MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb, MLX5_REFORMAT_TYPE_INSERT_HDR = 0xf, MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10, MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11, @@ -11563,6 +11639,41 @@ enum { MLX5_IPSEC_OBJECT_ICV_LEN_16B, }; +enum { + MLX5_IPSEC_ASO_REG_C_0_1 = 0x0, + MLX5_IPSEC_ASO_REG_C_2_3 = 0x1, + MLX5_IPSEC_ASO_REG_C_4_5 = 0x2, + MLX5_IPSEC_ASO_REG_C_6_7 = 0x3, +}; + +enum { + MLX5_IPSEC_ASO_MODE = 0x0, + MLX5_IPSEC_ASO_REPLAY_PROTECTION = 0x1, + MLX5_IPSEC_ASO_INC_SN = 0x2, +}; + +struct mlx5_ifc_ipsec_aso_bits { + u8 valid[0x1]; + u8 reserved_at_201[0x1]; + u8 mode[0x2]; + u8 window_sz[0x2]; + u8 soft_lft_arm[0x1]; + u8 hard_lft_arm[0x1]; + u8 remove_flow_enable[0x1]; + u8 esn_event_arm[0x1]; + u8 reserved_at_20a[0x16]; + + u8 remove_flow_pkt_cnt[0x20]; + + u8 remove_flow_soft_lft[0x20]; + + u8 reserved_at_260[0x80]; + + u8 mode_parameter[0x20]; + + u8 replay_protection_window[0x100]; +}; + struct mlx5_ifc_ipsec_obj_bits { u8 modify_field_select[0x40]; u8 full_offload[0x1]; @@ -11584,7 +11695,11 @@ struct mlx5_ifc_ipsec_obj_bits { u8 implicit_iv[0x40]; - u8 reserved_at_100[0x700]; + u8 reserved_at_100[0x8]; + u8 ipsec_aso_access_pd[0x18]; + u8 reserved_at_120[0xe0]; + + struct mlx5_ifc_ipsec_aso_bits ipsec_aso; }; struct mlx5_ifc_create_ipsec_obj_in_bits { diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index aad53cb72f17..7f31432f44c2 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -132,4 +132,6 @@ int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev, int mlx5_nic_vport_unaffiliate_multiport(struct mlx5_core_dev *port_mdev); u64 mlx5_query_nic_system_image_guid(struct mlx5_core_dev *mdev); +int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out, + u16 opmod); #endif /* __MLX5_VPORT_H__ */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 8bbcccbc5565..974ccca609d2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1852,6 +1852,25 @@ static void __maybe_unused show_free_areas(unsigned int flags, nodemask_t *nodem __show_free_areas(flags, nodemask, MAX_NR_ZONES - 1); } +/* + * Parameter block passed down to zap_pte_range in exceptional cases. + */ +struct zap_details { + struct folio *single_folio; /* Locked folio to be unmapped */ + bool even_cows; /* Zap COWed private pages too? */ + zap_flags_t zap_flags; /* Extra flags for zapping */ +}; + +/* + * Whether to drop the pte markers, for example, the uffd-wp information for + * file-backed memory. This should only be specified when we will completely + * drop the page in the mm, either by truncation or unmapping of the vma. By + * default, the flag is not set. + */ +#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) +/* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ +#define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) + #ifdef CONFIG_MMU extern bool can_do_mlock(void); #else @@ -1869,6 +1888,8 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, unsigned long size); void zap_page_range(struct vm_area_struct *vma, unsigned long address, unsigned long size); +void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, + unsigned long size, struct zap_details *details); void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, struct vm_area_struct *start_vma, unsigned long start, unsigned long end); @@ -3467,12 +3488,4 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, } #endif -/* - * Whether to drop the pte markers, for example, the uffd-wp information for - * file-backed memory. This should only be specified when we will completely - * drop the page in the mm, either by truncation or unmapping of the vma. By - * default, the flag is not set. - */ -#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) - #endif /* _LINUX_MM_H */ diff --git a/include/linux/mmc/mmc.h b/include/linux/mmc/mmc.h index 9c50bc40f8ff..6f7993803ee7 100644 --- a/include/linux/mmc/mmc.h +++ b/include/linux/mmc/mmc.h @@ -451,7 +451,7 @@ static inline bool mmc_ready_for_data(u32 status) #define MMC_SECURE_TRIM1_ARG 0x80000001 #define MMC_SECURE_TRIM2_ARG 0x80008000 #define MMC_SECURE_ARGS 0x80000000 -#define MMC_TRIM_ARGS 0x00008001 +#define MMC_TRIM_OR_DISCARD_ARGS 0x00008003 #define mmc_driver_type_mask(n) (1 << (n)) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5aa35c58c342..2287cb8eb9e4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -78,6 +78,7 @@ struct xdp_buff; void synchronize_net(void); void netdev_set_default_ethtool_ops(struct net_device *dev, const struct ethtool_ops *ops); +void netdev_sw_irq_coalesce_default_on(struct net_device *dev); /* Backlog congestion levels */ #define NET_RX_SUCCESS 0 /* keep 'em coming, baby */ @@ -1040,6 +1041,10 @@ struct xfrmdev_ops { bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); + int (*xdo_dev_policy_add) (struct xfrm_policy *x); + void (*xdo_dev_policy_delete) (struct xfrm_policy *x); + void (*xdo_dev_policy_free) (struct xfrm_policy *x); }; #endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index a108b60a6962..5f0d7d0b9471 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -165,6 +165,13 @@ static inline pte_t *virt_to_kpte(unsigned long vaddr) return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); } +#ifndef pmd_young +static inline int pmd_young(pmd_t pmd) +{ + return 0; +} +#endif + #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, @@ -260,6 +267,17 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif +#ifndef arch_has_hw_nonleaf_pmd_young +/* + * Return whether the accessed bit in non-leaf PMD entries is supported on the + * local CPU. + */ +static inline bool arch_has_hw_nonleaf_pmd_young(void) +{ + return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG); +} +#endif + #ifndef arch_has_hw_pte_young /* * Return whether the accessed bit is supported on the local CPU. diff --git a/include/linux/phy.h b/include/linux/phy.h index 9a3752c0c444..71eeb4e3b1fd 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -529,6 +529,8 @@ struct macsec_ops; * * @mdio: MDIO bus this PHY is on * @drv: Pointer to the driver for this PHY instance + * @devlink: Create a link between phy dev and mac dev, if the external phy + * used by current mac interface is managed by another mac interface. * @phy_id: UID for this device found during discovery * @c45_ids: 802.3-c45 Device Identifiers if is_c45. * @is_c45: Set to true if this PHY uses clause 45 addressing. @@ -618,6 +620,8 @@ struct phy_device { /* And management functions */ struct phy_driver *drv; + struct device_link *devlink; + u32 phy_id; struct phy_c45_device_ids c45_ids; diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 68dab3e08aad..5b5357c0bd8c 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -323,29 +323,36 @@ static inline struct rhash_lock_head __rcu **rht_bucket_insert( * When we write to a bucket without unlocking, we use rht_assign_locked(). */ -static inline void rht_lock(struct bucket_table *tbl, - struct rhash_lock_head __rcu **bkt) +static inline unsigned long rht_lock(struct bucket_table *tbl, + struct rhash_lock_head __rcu **bkt) { - local_bh_disable(); + unsigned long flags; + + local_irq_save(flags); bit_spin_lock(0, (unsigned long *)bkt); lock_map_acquire(&tbl->dep_map); + return flags; } -static inline void rht_lock_nested(struct bucket_table *tbl, - struct rhash_lock_head __rcu **bucket, - unsigned int subclass) +static inline unsigned long rht_lock_nested(struct bucket_table *tbl, + struct rhash_lock_head __rcu **bucket, + unsigned int subclass) { - local_bh_disable(); + unsigned long flags; + + local_irq_save(flags); bit_spin_lock(0, (unsigned long *)bucket); lock_acquire_exclusive(&tbl->dep_map, subclass, 0, NULL, _THIS_IP_); + return flags; } static inline void rht_unlock(struct bucket_table *tbl, - struct rhash_lock_head __rcu **bkt) + struct rhash_lock_head __rcu **bkt, + unsigned long flags) { lock_map_release(&tbl->dep_map); bit_spin_unlock(0, (unsigned long *)bkt); - local_bh_enable(); + local_irq_restore(flags); } static inline struct rhash_head *__rht_ptr( @@ -393,7 +400,8 @@ static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, static inline void rht_assign_unlock(struct bucket_table *tbl, struct rhash_lock_head __rcu **bkt, - struct rhash_head *obj) + struct rhash_head *obj, + unsigned long flags) { if (rht_is_a_nulls(obj)) obj = NULL; @@ -401,7 +409,7 @@ static inline void rht_assign_unlock(struct bucket_table *tbl, rcu_assign_pointer(*bkt, (void *)obj); preempt_enable(); __release(bitlock); - local_bh_enable(); + local_irq_restore(flags); } /** @@ -706,6 +714,7 @@ static inline void *__rhashtable_insert_fast( struct rhash_head __rcu **pprev; struct bucket_table *tbl; struct rhash_head *head; + unsigned long flags; unsigned int hash; int elasticity; void *data; @@ -720,11 +729,11 @@ static inline void *__rhashtable_insert_fast( if (!bkt) goto out; pprev = NULL; - rht_lock(tbl, bkt); + flags = rht_lock(tbl, bkt); if (unlikely(rcu_access_pointer(tbl->future_tbl))) { slow_path: - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); rcu_read_unlock(); return rhashtable_insert_slow(ht, key, obj); } @@ -756,9 +765,9 @@ slow_path: RCU_INIT_POINTER(list->rhead.next, head); if (pprev) { rcu_assign_pointer(*pprev, obj); - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); } else - rht_assign_unlock(tbl, bkt, obj); + rht_assign_unlock(tbl, bkt, obj, flags); data = NULL; goto out; } @@ -785,7 +794,7 @@ slow_path: } atomic_inc(&ht->nelems); - rht_assign_unlock(tbl, bkt, obj); + rht_assign_unlock(tbl, bkt, obj, flags); if (rht_grow_above_75(ht, tbl)) schedule_work(&ht->run_work); @@ -797,7 +806,7 @@ out: return data; out_unlock: - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); goto out; } @@ -991,6 +1000,7 @@ static inline int __rhashtable_remove_fast_one( struct rhash_lock_head __rcu **bkt; struct rhash_head __rcu **pprev; struct rhash_head *he; + unsigned long flags; unsigned int hash; int err = -ENOENT; @@ -999,7 +1009,7 @@ static inline int __rhashtable_remove_fast_one( if (!bkt) return -ENOENT; pprev = NULL; - rht_lock(tbl, bkt); + flags = rht_lock(tbl, bkt); rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) { struct rhlist_head *list; @@ -1043,14 +1053,14 @@ static inline int __rhashtable_remove_fast_one( if (pprev) { rcu_assign_pointer(*pprev, obj); - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); } else { - rht_assign_unlock(tbl, bkt, obj); + rht_assign_unlock(tbl, bkt, obj, flags); } goto unlocked; } - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); unlocked: if (err > 0) { atomic_dec(&ht->nelems); @@ -1143,6 +1153,7 @@ static inline int __rhashtable_replace_fast( struct rhash_lock_head __rcu **bkt; struct rhash_head __rcu **pprev; struct rhash_head *he; + unsigned long flags; unsigned int hash; int err = -ENOENT; @@ -1158,7 +1169,7 @@ static inline int __rhashtable_replace_fast( return -ENOENT; pprev = NULL; - rht_lock(tbl, bkt); + flags = rht_lock(tbl, bkt); rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) { if (he != obj_old) { @@ -1169,15 +1180,15 @@ static inline int __rhashtable_replace_fast( rcu_assign_pointer(obj_new->next, obj_old->next); if (pprev) { rcu_assign_pointer(*pprev, obj_new); - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); } else { - rht_assign_unlock(tbl, bkt, obj_new); + rht_assign_unlock(tbl, bkt, obj_new, flags); } err = 0; goto unlocked; } - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); unlocked: return err; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 4e464a27adaf..4c8492401a10 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1255,6 +1255,7 @@ struct sk_buff *build_skb_around(struct sk_buff *skb, void skb_attempt_defer_free(struct sk_buff *skb); struct sk_buff *napi_build_skb(void *data, unsigned int frag_size); +struct sk_buff *slab_build_skb(void *data); /** * alloc_skb - allocate a network buffer diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h index beb190449704..a0746d4aec20 100644 --- a/include/linux/soc/mediatek/mtk_wed.h +++ b/include/linux/soc/mediatek/mtk_wed.h @@ -160,7 +160,7 @@ struct mtk_wed_ops { int (*tx_ring_setup)(struct mtk_wed_device *dev, int ring, void __iomem *regs, bool reset); int (*rx_ring_setup)(struct mtk_wed_device *dev, int ring, - void __iomem *regs); + void __iomem *regs, bool reset); int (*txfree_ring_setup)(struct mtk_wed_device *dev, void __iomem *regs); int (*msg_update)(struct mtk_wed_device *dev, int cmd_id, @@ -228,8 +228,8 @@ mtk_wed_get_rx_capa(struct mtk_wed_device *dev) (_dev)->ops->irq_get(_dev, _mask) #define mtk_wed_device_irq_set_mask(_dev, _mask) \ (_dev)->ops->irq_set_mask(_dev, _mask) -#define mtk_wed_device_rx_ring_setup(_dev, _ring, _regs) \ - (_dev)->ops->rx_ring_setup(_dev, _ring, _regs) +#define mtk_wed_device_rx_ring_setup(_dev, _ring, _regs, _reset) \ + (_dev)->ops->rx_ring_setup(_dev, _ring, _regs, _reset) #define mtk_wed_device_ppe_check(_dev, _skb, _reason, _hash) \ (_dev)->ops->ppe_check(_dev, _skb, _reason, _hash) #define mtk_wed_device_update_msg(_dev, _id, _msg, _len) \ @@ -249,7 +249,7 @@ static inline bool mtk_wed_device_active(struct mtk_wed_device *dev) #define mtk_wed_device_reg_write(_dev, _reg, _val) do {} while (0) #define mtk_wed_device_irq_get(_dev, _mask) 0 #define mtk_wed_device_irq_set_mask(_dev, _mask) do {} while (0) -#define mtk_wed_device_rx_ring_setup(_dev, _ring, _regs) -ENODEV +#define mtk_wed_device_rx_ring_setup(_dev, _ring, _regs, _reset) -ENODEV #define mtk_wed_device_ppe_check(_dev, _skb, _reason, _hash) do {} while (0) #define mtk_wed_device_update_msg(_dev, _id, _msg, _len) -ENODEV #define mtk_wed_device_stop(_dev) do {} while (0) diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index fb2e88614f5d..83ca2e8eb6b5 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -271,5 +271,6 @@ struct plat_stmmacenet_data { int msi_tx_base_vec; bool use_phy_wol; bool sph_disable; + bool serdes_up_after_phy_linkup; }; #endif diff --git a/include/linux/usb.h b/include/linux/usb.h index 9ff1ad4dfad1..d2d2f41052c0 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -1829,6 +1829,7 @@ static inline int usb_get_ptm_status(struct usb_device *dev, void *data) extern int usb_string(struct usb_device *dev, int index, char *buf, size_t size); +extern char *usb_cache_string(struct usb_device *udev, int index); /* wrappers that also update important state inside usbcore */ extern int usb_clear_halt(struct usb_device *dev, int pipe); diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index a960de68ac69..bdf8de2cdd93 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -15,6 +15,7 @@ static inline bool virtio_net_hdr_match_proto(__be16 protocol, __u8 gso_type) case VIRTIO_NET_HDR_GSO_TCPV6: return protocol == cpu_to_be16(ETH_P_IPV6); case VIRTIO_NET_HDR_GSO_UDP: + case VIRTIO_NET_HDR_GSO_UDP_L4: return protocol == cpu_to_be16(ETH_P_IP) || protocol == cpu_to_be16(ETH_P_IPV6); default: @@ -31,6 +32,7 @@ static inline int virtio_net_hdr_set_proto(struct sk_buff *skb, switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: case VIRTIO_NET_HDR_GSO_UDP: + case VIRTIO_NET_HDR_GSO_UDP_L4: skb->protocol = cpu_to_be16(ETH_P_IP); break; case VIRTIO_NET_HDR_GSO_TCPV6: @@ -69,6 +71,11 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, ip_proto = IPPROTO_UDP; thlen = sizeof(struct udphdr); break; + case VIRTIO_NET_HDR_GSO_UDP_L4: + gso_type = SKB_GSO_UDP_L4; + ip_proto = IPPROTO_UDP; + thlen = sizeof(struct udphdr); + break; default: return -EINVAL; } @@ -182,6 +189,8 @@ static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; else if (sinfo->gso_type & SKB_GSO_TCPV6) hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; + else if (sinfo->gso_type & SKB_GSO_UDP_L4) + hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP_L4; else return -EINVAL; if (sinfo->gso_type & SKB_GSO_TCP_ECN) diff --git a/include/net/act_api.h b/include/net/act_api.h index c94ea1a306e0..2a6f443f0ef6 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -101,11 +101,6 @@ static inline enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats) return hw_stats; } -#ifdef CONFIG_NET_CLS_ACT - -#define ACT_P_CREATED 1 -#define ACT_P_DELETED 1 - typedef void (*tc_action_priv_destructor)(void *priv); struct tc_action_ops { @@ -140,6 +135,11 @@ struct tc_action_ops { struct netlink_ext_ack *extack); }; +#ifdef CONFIG_NET_CLS_ACT + +#define ACT_P_CREATED 1 +#define ACT_P_DELETED 1 + struct tc_action_net { struct tcf_idrinfo *idrinfo; const struct tc_action_ops *ops; diff --git a/include/net/af_rxrpc.h b/include/net/af_rxrpc.h index b69ca695935c..d5a5ae926380 100644 --- a/include/net/af_rxrpc.h +++ b/include/net/af_rxrpc.h @@ -66,10 +66,10 @@ int rxrpc_kernel_charge_accept(struct socket *, rxrpc_notify_rx_t, void rxrpc_kernel_set_tx_length(struct socket *, struct rxrpc_call *, s64); bool rxrpc_kernel_check_life(const struct socket *, const struct rxrpc_call *); u32 rxrpc_kernel_get_epoch(struct socket *, struct rxrpc_call *); -bool rxrpc_kernel_call_is_complete(struct rxrpc_call *); void rxrpc_kernel_set_max_life(struct socket *, struct rxrpc_call *, unsigned long); int rxrpc_sock_set_min_security_level(struct sock *sk, unsigned int val); +int rxrpc_sock_set_security_keyring(struct sock *, struct key *); #endif /* _NET_RXRPC_H */ diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index e004ba04a9ae..684f1cd28730 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -228,6 +228,17 @@ enum { */ HCI_QUIRK_VALID_LE_STATES, + /* When this quirk is set, then erroneous data reporting + * is ignored. This is mainly due to the fact that the HCI + * Read Default Erroneous Data Reporting command is advertised, + * but not supported; these controllers often reply with unknown + * command and tend to lock up randomly. Needing a hard reset. + * + * This quirk can be set before hci_register_dev is called or + * during the hdev->setup vendor callback. + */ + HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, + /* * When this quirk is set, then the hci_suspend_notifier is not * registered. This is intended for devices which drop completely @@ -1424,7 +1435,6 @@ struct hci_std_codecs_v2 { } __packed; struct hci_vnd_codec_v2 { - __u8 id; __le16 cid; __le16 vid; __u8 transport; diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 11a370e64143..03d4f4deadae 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -4742,7 +4742,7 @@ struct ieee80211_iface_limit { * * struct ieee80211_iface_limit limits1[] = { * { .max = 1, .types = BIT(NL80211_IFTYPE_STATION), }, - * { .max = 1, .types = BIT(NL80211_IFTYPE_AP}, }, + * { .max = 1, .types = BIT(NL80211_IFTYPE_AP), }, * }; * struct ieee80211_iface_combination combination1 = { * .limits = limits1, diff --git a/include/net/cfg802154.h b/include/net/cfg802154.h index e1481f9cf049..d09c393d229f 100644 --- a/include/net/cfg802154.h +++ b/include/net/cfg802154.h @@ -260,6 +260,24 @@ struct ieee802154_addr { }; }; +/** + * struct ieee802154_coord_desc - Coordinator descriptor + * @addr: PAN ID and coordinator address + * @page: page this coordinator is using + * @channel: channel this coordinator is using + * @superframe_spec: SuperFrame specification as received + * @link_quality: link quality indicator at which the beacon was received + * @gts_permit: the coordinator accepts GTS requests + */ +struct ieee802154_coord_desc { + struct ieee802154_addr addr; + u8 page; + u8 channel; + u16 superframe_spec; + u8 link_quality; + bool gts_permit; +}; + struct ieee802154_llsec_key_id { u8 mode; u8 id; diff --git a/include/net/devlink.h b/include/net/devlink.h index 074a79b8933f..6a2e4f21779f 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -621,6 +621,8 @@ enum devlink_param_generic_id { #define DEVLINK_INFO_VERSION_GENERIC_FW_ROCE "fw.roce" /* Firmware bundle identifier */ #define DEVLINK_INFO_VERSION_GENERIC_FW_BUNDLE_ID "fw.bundle_id" +/* Bootloader */ +#define DEVLINK_INFO_VERSION_GENERIC_FW_BOOTLOADER "fw.bootloader" /** * struct devlink_flash_update_params - Flash Update parameters @@ -650,6 +652,10 @@ struct devlink_info_req; * the data variable must be updated to point to the snapshot data. * The function will be called while the devlink instance lock is * held. + * @read: callback to directly read a portion of the region. On success, + * the data pointer will be updated with the contents of the + * requested portion of the region. The function will be called + * while the devlink instance lock is held. * @priv: Pointer to driver private data for the region operation */ struct devlink_region_ops { @@ -659,6 +665,10 @@ struct devlink_region_ops { const struct devlink_region_ops *ops, struct netlink_ext_ack *extack, u8 **data); + int (*read)(struct devlink *devlink, + const struct devlink_region_ops *ops, + struct netlink_ext_ack *extack, + u64 offset, u32 size, u8 *data); void *priv; }; @@ -670,6 +680,10 @@ struct devlink_region_ops { * the data variable must be updated to point to the snapshot data. * The function will be called while the devlink instance lock is * held. + * @read: callback to directly read a portion of the region. On success, + * the data pointer will be updated with the contents of the + * requested portion of the region. The function will be called + * while the devlink instance lock is held. * @priv: Pointer to driver private data for the region operation */ struct devlink_port_region_ops { @@ -679,6 +693,10 @@ struct devlink_port_region_ops { const struct devlink_port_region_ops *ops, struct netlink_ext_ack *extack, u8 **data); + int (*read)(struct devlink_port *port, + const struct devlink_port_region_ops *ops, + struct netlink_ext_ack *extack, + u64 offset, u32 size, u8 *data); void *priv; }; @@ -1436,6 +1454,45 @@ struct devlink_ops { const u8 *hw_addr, int hw_addr_len, struct netlink_ext_ack *extack); /** + * @port_fn_roce_get: Port function's roce get function. + * + * Query RoCE state of a function managed by the devlink port. + * Return -EOPNOTSUPP if port function RoCE handling is not supported. + */ + int (*port_fn_roce_get)(struct devlink_port *devlink_port, + bool *is_enable, + struct netlink_ext_ack *extack); + /** + * @port_fn_roce_set: Port function's roce set function. + * + * Enable/Disable the RoCE state of a function managed by the devlink + * port. + * Return -EOPNOTSUPP if port function RoCE handling is not supported. + */ + int (*port_fn_roce_set)(struct devlink_port *devlink_port, + bool enable, struct netlink_ext_ack *extack); + /** + * @port_fn_migratable_get: Port function's migratable get function. + * + * Query migratable state of a function managed by the devlink port. + * Return -EOPNOTSUPP if port function migratable handling is not + * supported. + */ + int (*port_fn_migratable_get)(struct devlink_port *devlink_port, + bool *is_enable, + struct netlink_ext_ack *extack); + /** + * @port_fn_migratable_set: Port function's migratable set function. + * + * Enable/Disable migratable state of a function managed by the devlink + * port. + * Return -EOPNOTSUPP if port function migratable handling is not + * supported. + */ + int (*port_fn_migratable_set)(struct devlink_port *devlink_port, + bool enable, + struct netlink_ext_ack *extack); + /** * port_new() - Add a new port function of a specified flavor * @devlink: Devlink instance * @attrs: attributes of the new port @@ -1746,8 +1803,6 @@ int devlink_region_snapshot_create(struct devlink_region *region, u8 *data, u32 snapshot_id); int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn); -int devlink_info_driver_name_put(struct devlink_info_req *req, - const char *name); int devlink_info_board_serial_number_put(struct devlink_info_req *req, const char *bsn); diff --git a/include/net/fq_impl.h b/include/net/fq_impl.h index 524b510f1c68..9467e33dfb36 100644 --- a/include/net/fq_impl.h +++ b/include/net/fq_impl.h @@ -200,6 +200,7 @@ static void fq_tin_enqueue(struct fq *fq, fq_skb_free_t free_func) { struct fq_flow *flow; + struct sk_buff *next; bool oom; lockdep_assert_held(&fq->lock); @@ -214,11 +215,15 @@ static void fq_tin_enqueue(struct fq *fq, } flow->tin = tin; - flow->backlog += skb->len; - tin->backlog_bytes += skb->len; - tin->backlog_packets++; - fq->memory_usage += skb->truesize; - fq->backlog++; + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + flow->backlog += skb->len; + tin->backlog_bytes += skb->len; + tin->backlog_packets++; + fq->memory_usage += skb->truesize; + fq->backlog++; + __skb_queue_tail(&flow->queue, skb); + } if (list_empty(&flow->flowchain)) { flow->deficit = fq->quantum; @@ -226,7 +231,6 @@ static void fq_tin_enqueue(struct fq *fq, &tin->new_flows); } - __skb_queue_tail(&flow->queue, skb); oom = (fq->memory_usage > fq->memory_limit); while (fq->backlog > fq->limit || oom) { flow = fq_find_fattest_flow(fq); diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 721c450a9ccd..689da327ce2e 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -1807,6 +1807,10 @@ struct ieee80211_vif_cfg { * @addr: address of this interface * @p2p: indicates whether this AP or STA interface is a p2p * interface, i.e. a GO or p2p-sta respectively + * @netdev_features: tx netdev features supported by the hardware for this + * vif. mac80211 initializes this to hw->netdev_features, and the driver + * can mask out specific tx features. mac80211 will handle software fixup + * for masked offloads (GSO, CSUM) * @driver_flags: flags/capabilities the driver has for this interface, * these need to be set (or cleared) when the interface is added * or, if supported by the driver, the interface type is changed @@ -1848,6 +1852,7 @@ struct ieee80211_vif { struct ieee80211_txq *txq; + netdev_features_t netdev_features; u32 driver_flags; u32 offload_flags; diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 28d0687bf7da..d80c78506f19 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -522,7 +522,14 @@ enum { #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0) -#define GDMA_DRV_CAP_FLAGS1 GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT +/* Advertise to the NIC firmware: the NAPI work_done variable race is fixed, + * so the driver is able to reliably support features like busy_poll. + */ +#define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2) + +#define GDMA_DRV_CAP_FLAGS1 \ + (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \ + GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX) #define GDMA_DRV_CAP_FLAGS2 0 diff --git a/include/net/netfilter/nf_nat.h b/include/net/netfilter/nf_nat.h index e9eb01e99d2f..9877f064548a 100644 --- a/include/net/netfilter/nf_nat.h +++ b/include/net/netfilter/nf_nat.h @@ -104,6 +104,10 @@ unsigned int nf_nat_inet_fn(void *priv, struct sk_buff *skb, const struct nf_hook_state *state); +int nf_ct_nat(struct sk_buff *skb, struct nf_conn *ct, + enum ip_conntrack_info ctinfo, int *action, + const struct nf_nat_range2 *range, bool commit); + static inline int nf_nat_initialized(const struct nf_conn *ct, enum nf_nat_manip_type manip) { diff --git a/include/net/nl802154.h b/include/net/nl802154.h index f5850b569c52..b79a89d5207c 100644 --- a/include/net/nl802154.h +++ b/include/net/nl802154.h @@ -72,6 +72,8 @@ enum nl802154_commands { NL802154_CMD_NEW_SEC_LEVEL, NL802154_CMD_DEL_SEC_LEVEL, + NL802154_CMD_SCAN_EVENT, + /* add new commands above here */ /* used to define NL802154_CMD_MAX below */ @@ -131,6 +133,8 @@ enum nl802154_attrs { NL802154_ATTR_PID, NL802154_ATTR_NETNS_FD, + NL802154_ATTR_COORDINATOR, + /* add attributes here, update the policy in nl802154.c */ #ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL @@ -217,6 +221,45 @@ enum nl802154_wpan_phy_capability_attr { }; /** + * enum nl802154_coord - Netlink attributes for a coord + * + * @__NL802154_COORD_INVALID: invalid + * @NL802154_COORD_PANID: PANID of the coordinator (2 bytes) + * @NL802154_COORD_ADDR: coordinator address, (8 bytes or 2 bytes) + * @NL802154_COORD_CHANNEL: channel number, related to @NL802154_COORD_PAGE (u8) + * @NL802154_COORD_PAGE: channel page, related to @NL802154_COORD_CHANNEL (u8) + * @NL802154_COORD_PREAMBLE_CODE: Preamble code used when the beacon was received, + * this is PHY dependent and optional (u8) + * @NL802154_COORD_MEAN_PRF: Mean PRF used when the beacon was received, + * this is PHY dependent and optional (u8) + * @NL802154_COORD_SUPERFRAME_SPEC: superframe specification of the PAN (u16) + * @NL802154_COORD_LINK_QUALITY: signal quality of beacon in unspecified units, + * scaled to 0..255 (u8) + * @NL802154_COORD_GTS_PERMIT: set to true if GTS is permitted on this PAN + * @NL802154_COORD_PAYLOAD_DATA: binary data containing the raw data from the + * frame payload, (only if beacon or probe response had data) + * @NL802154_COORD_PAD: attribute used for padding for 64-bit alignment + * @NL802154_COORD_MAX: highest coordinator attribute + */ +enum nl802154_coord { + __NL802154_COORD_INVALID, + NL802154_COORD_PANID, + NL802154_COORD_ADDR, + NL802154_COORD_CHANNEL, + NL802154_COORD_PAGE, + NL802154_COORD_PREAMBLE_CODE, + NL802154_COORD_MEAN_PRF, + NL802154_COORD_SUPERFRAME_SPEC, + NL802154_COORD_LINK_QUALITY, + NL802154_COORD_GTS_PERMIT, + NL802154_COORD_PAYLOAD_DATA, + NL802154_COORD_PAD, + + /* keep last */ + NL802154_COORD_MAX, +}; + +/** * enum nl802154_cca_modes - cca modes * * @__NL802154_CCA_INVALID: cca mode number 0 is reserved diff --git a/include/net/ping.h b/include/net/ping.h index e4ff3911cbf5..9233ad3de0ad 100644 --- a/include/net/ping.h +++ b/include/net/ping.h @@ -16,9 +16,6 @@ #define PING_HTABLE_SIZE 64 #define PING_HTABLE_MASK (PING_HTABLE_SIZE-1) -#define ping_portaddr_for_each_entry(__sk, node, list) \ - hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node) - /* * gid_t is either uint or ushort. We want to pass it to * proc_dointvec_minmax(), so it must not be larger than MAX_INT diff --git a/include/net/sctp/stream_sched.h b/include/net/sctp/stream_sched.h index 65058faea4db..fa00dc20a0d7 100644 --- a/include/net/sctp/stream_sched.h +++ b/include/net/sctp/stream_sched.h @@ -28,8 +28,6 @@ struct sctp_sched_ops { int (*init_sid)(struct sctp_stream *stream, __u16 sid, gfp_t gfp); /* free a stream */ void (*free_sid)(struct sctp_stream *stream, __u16 sid); - /* Frees the entire thing */ - void (*free)(struct sctp_stream *stream); /* Enqueue a chunk */ void (*enqueue)(struct sctp_outq *q, struct sctp_datamsg *msg); diff --git a/include/net/sock.h b/include/net/sock.h index 6d207e7c4ad0..ecea3dcc2217 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -503,10 +503,10 @@ struct sock { #if BITS_PER_LONG==32 seqlock_t sk_stamp_seq; #endif - u16 sk_tsflags; - u8 sk_shutdown; atomic_t sk_tskey; atomic_t sk_zckey; + u32 sk_tsflags; + u8 sk_shutdown; u8 sk_clockid; u8 sk_txtime_deadline_mode : 1, @@ -1899,7 +1899,7 @@ static inline void sock_replace_proto(struct sock *sk, struct proto *proto) struct sockcm_cookie { u64 transmit_time; u32 mark; - u16 tsflags; + u32 tsflags; }; static inline void sockcm_init(struct sockcm_cookie *sockc, diff --git a/include/net/tc_wrapper.h b/include/net/tc_wrapper.h new file mode 100644 index 000000000000..ceed2fc089ff --- /dev/null +++ b/include/net/tc_wrapper.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NET_TC_WRAPPER_H +#define __NET_TC_WRAPPER_H + +#include <net/pkt_cls.h> + +#if IS_ENABLED(CONFIG_RETPOLINE) + +#include <linux/cpufeature.h> +#include <linux/static_key.h> +#include <linux/indirect_call_wrapper.h> + +#define TC_INDIRECT_SCOPE + +extern struct static_key_false tc_skip_wrapper; + +/* TC Actions */ +#ifdef CONFIG_NET_CLS_ACT + +#define TC_INDIRECT_ACTION_DECLARE(fname) \ + INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ + const struct tc_action *a, \ + struct tcf_result *res)) + +TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act); +TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act); +TC_INDIRECT_ACTION_DECLARE(tcf_csum_act); +TC_INDIRECT_ACTION_DECLARE(tcf_ct_act); +TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act); +TC_INDIRECT_ACTION_DECLARE(tcf_gact_act); +TC_INDIRECT_ACTION_DECLARE(tcf_gate_act); +TC_INDIRECT_ACTION_DECLARE(tcf_ife_act); +TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act); +TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act); +TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act); +TC_INDIRECT_ACTION_DECLARE(tcf_nat_act); +TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act); +TC_INDIRECT_ACTION_DECLARE(tcf_police_act); +TC_INDIRECT_ACTION_DECLARE(tcf_sample_act); +TC_INDIRECT_ACTION_DECLARE(tcf_simp_act); +TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act); +TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act); +TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act); +TC_INDIRECT_ACTION_DECLARE(tunnel_key_act); + +static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) +{ + if (static_branch_likely(&tc_skip_wrapper)) + goto skip; + +#if IS_BUILTIN(CONFIG_NET_ACT_GACT) + if (a->ops->act == tcf_gact_act) + return tcf_gact_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED) + if (a->ops->act == tcf_mirred_act) + return tcf_mirred_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT) + if (a->ops->act == tcf_pedit_act) + return tcf_pedit_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) + if (a->ops->act == tcf_skbedit_act) + return tcf_skbedit_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) + if (a->ops->act == tcf_skbmod_act) + return tcf_skbmod_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_POLICE) + if (a->ops->act == tcf_police_act) + return tcf_police_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_BPF) + if (a->ops->act == tcf_bpf_act) + return tcf_bpf_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) + if (a->ops->act == tcf_connmark_act) + return tcf_connmark_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_CSUM) + if (a->ops->act == tcf_csum_act) + return tcf_csum_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_CT) + if (a->ops->act == tcf_ct_act) + return tcf_ct_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO) + if (a->ops->act == tcf_ctinfo_act) + return tcf_ctinfo_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_GATE) + if (a->ops->act == tcf_gate_act) + return tcf_gate_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_MPLS) + if (a->ops->act == tcf_mpls_act) + return tcf_mpls_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_NAT) + if (a->ops->act == tcf_nat_act) + return tcf_nat_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) + if (a->ops->act == tunnel_key_act) + return tunnel_key_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_VLAN) + if (a->ops->act == tcf_vlan_act) + return tcf_vlan_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_IFE) + if (a->ops->act == tcf_ife_act) + return tcf_ife_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_IPT) + if (a->ops->act == tcf_ipt_act) + return tcf_ipt_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_SIMP) + if (a->ops->act == tcf_simp_act) + return tcf_simp_act(skb, a, res); +#endif +#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE) + if (a->ops->act == tcf_sample_act) + return tcf_sample_act(skb, a, res); +#endif + +skip: + return a->ops->act(skb, a, res); +} + +#endif /* CONFIG_NET_CLS_ACT */ + +/* TC Filters */ +#ifdef CONFIG_NET_CLS + +#define TC_INDIRECT_FILTER_DECLARE(fname) \ + INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ + const struct tcf_proto *tp, \ + struct tcf_result *res)) + +TC_INDIRECT_FILTER_DECLARE(basic_classify); +TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify); +TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify); +TC_INDIRECT_FILTER_DECLARE(fl_classify); +TC_INDIRECT_FILTER_DECLARE(flow_classify); +TC_INDIRECT_FILTER_DECLARE(fw_classify); +TC_INDIRECT_FILTER_DECLARE(mall_classify); +TC_INDIRECT_FILTER_DECLARE(route4_classify); +TC_INDIRECT_FILTER_DECLARE(rsvp_classify); +TC_INDIRECT_FILTER_DECLARE(rsvp6_classify); +TC_INDIRECT_FILTER_DECLARE(tcindex_classify); +TC_INDIRECT_FILTER_DECLARE(u32_classify); + +static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, + struct tcf_result *res) +{ + if (static_branch_likely(&tc_skip_wrapper)) + goto skip; + +#if IS_BUILTIN(CONFIG_NET_CLS_BPF) + if (tp->classify == cls_bpf_classify) + return cls_bpf_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_U32) + if (tp->classify == u32_classify) + return u32_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER) + if (tp->classify == fl_classify) + return fl_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_FW) + if (tp->classify == fw_classify) + return fw_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) + if (tp->classify == mall_classify) + return mall_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_BASIC) + if (tp->classify == basic_classify) + return basic_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP) + if (tp->classify == cls_cgroup_classify) + return cls_cgroup_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_FLOW) + if (tp->classify == flow_classify) + return flow_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4) + if (tp->classify == route4_classify) + return route4_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_RSVP) + if (tp->classify == rsvp_classify) + return rsvp_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_RSVP6) + if (tp->classify == rsvp6_classify) + return rsvp6_classify(skb, tp, res); +#endif +#if IS_BUILTIN(CONFIG_NET_CLS_TCINDEX) + if (tp->classify == tcindex_classify) + return tcindex_classify(skb, tp, res); +#endif + +skip: + return tp->classify(skb, tp, res); +} + +static inline void tc_wrapper_init(void) +{ +#ifdef CONFIG_X86 + if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE)) + static_branch_enable(&tc_skip_wrapper); +#endif +} + +#endif /* CONFIG_NET_CLS */ + +#else + +#define TC_INDIRECT_SCOPE static + +static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) +{ + return a->ops->act(skb, a, res); +} + +static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, + struct tcf_result *res) +{ + return tp->classify(skb, tp, res); +} + +static inline void tc_wrapper_init(void) +{ +} + +#endif + +#endif /* __NET_TC_WRAPPER_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index b87e7381bddf..db9f828e9d1e 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1675,7 +1675,11 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb); int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags, - const u8 *newkey, u8 newkeylen, gfp_t gfp); + const u8 *newkey, u8 newkeylen); +int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, + struct tcp_md5sig_key *key); + int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags); struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, @@ -1683,7 +1687,7 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, #ifdef CONFIG_TCP_MD5SIG #include <linux/jump_label.h> -extern struct static_key_false tcp_md5_needed; +extern struct static_key_false_deferred tcp_md5_needed; struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, const union tcp_md5_addr *addr, int family); @@ -1691,7 +1695,7 @@ static inline struct tcp_md5sig_key * tcp_md5_do_lookup(const struct sock *sk, int l3index, const union tcp_md5_addr *addr, int family) { - if (!static_branch_unlikely(&tcp_md5_needed)) + if (!static_branch_unlikely(&tcp_md5_needed.key)) return NULL; return __tcp_md5_do_lookup(sk, l3index, addr, family); } diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 3707e6b34e67..3e1f70e8e424 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -129,6 +129,13 @@ struct xfrm_state_walk { enum { XFRM_DEV_OFFLOAD_IN = 1, XFRM_DEV_OFFLOAD_OUT, + XFRM_DEV_OFFLOAD_FWD, +}; + +enum { + XFRM_DEV_OFFLOAD_UNSPECIFIED, + XFRM_DEV_OFFLOAD_CRYPTO, + XFRM_DEV_OFFLOAD_PACKET, }; struct xfrm_dev_offload { @@ -137,6 +144,7 @@ struct xfrm_dev_offload { struct net_device *real_dev; unsigned long offload_handle; u8 dir : 2; + u8 type : 2; }; struct xfrm_mode { @@ -534,6 +542,8 @@ struct xfrm_policy { struct xfrm_tmpl xfrm_vec[XFRM_MAX_DEPTH]; struct hlist_node bydst_inexact_list; struct rcu_head rcu; + + struct xfrm_dev_offload xdo; }; static inline struct net *xp_net(const struct xfrm_policy *xp) @@ -1093,6 +1103,29 @@ xfrm_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, un } #ifdef CONFIG_XFRM +static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb) +{ + struct sec_path *sp = skb_sec_path(skb); + + return sp->xvec[sp->len - 1]; +} +#endif + +static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) +{ +#ifdef CONFIG_XFRM + struct sec_path *sp = skb_sec_path(skb); + + if (!sp || !sp->olen || sp->len != sp->olen) + return NULL; + + return &sp->ovec[sp->olen - 1]; +#else + return NULL; +#endif +} + +#ifdef CONFIG_XFRM int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb, unsigned short family); @@ -1123,10 +1156,19 @@ static inline int __xfrm_policy_check2(struct sock *sk, int dir, { struct net *net = dev_net(skb->dev); int ndir = dir | (reverse ? XFRM_POLICY_MASK + 1 : 0); + struct xfrm_offload *xo = xfrm_offload(skb); + struct xfrm_state *x; if (sk && sk->sk_policy[XFRM_POLICY_IN]) return __xfrm_policy_check(sk, ndir, skb, family); + if (xo) { + x = xfrm_input_state(skb); + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + return (xo->flags & CRYPTO_DONE) && + (xo->status & CRYPTO_SUCCESS); + } + return __xfrm_check_nopolicy(net, skb, dir) || __xfrm_check_dev_nopolicy(skb, dir, family) || __xfrm_policy_check(sk, ndir, skb, family); @@ -1529,6 +1571,23 @@ struct xfrm_state *xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, unsigned short family); int xfrm_state_check_expire(struct xfrm_state *x); +#ifdef CONFIG_XFRM_OFFLOAD +static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) +{ + struct xfrm_dev_offload *xdo = &x->xso; + struct net_device *dev = xdo->dev; + + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + return; + + if (dev && dev->xfrmdev_ops && + dev->xfrmdev_ops->xdo_dev_state_update_curlft) + dev->xfrmdev_ops->xdo_dev_state_update_curlft(x); + +} +#else +static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) {} +#endif void xfrm_state_insert(struct xfrm_state *x); int xfrm_state_add(struct xfrm_state *x); int xfrm_state_update(struct xfrm_state *x); @@ -1578,6 +1637,8 @@ struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); int xfrm_state_delete(struct xfrm_state *x); int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync); int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid); +int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, + bool task_valid); void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si); void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si); u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq); @@ -1860,29 +1921,6 @@ static inline void xfrm_states_delete(struct xfrm_state **states, int n) } #endif -#ifdef CONFIG_XFRM -static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb) -{ - struct sec_path *sp = skb_sec_path(skb); - - return sp->xvec[sp->len - 1]; -} -#endif - -static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) -{ -#ifdef CONFIG_XFRM - struct sec_path *sp = skb_sec_path(skb); - - if (!sp || !sp->olen || sp->len != sp->olen) - return NULL; - - return &sp->ovec[sp->olen - 1]; -#else - return NULL; -#endif -} - void __init xfrm_dev_init(void); #ifdef CONFIG_XFRM_OFFLOAD @@ -1892,6 +1930,9 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack); +int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack); bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x); static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x) @@ -1940,6 +1981,28 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) netdev_put(dev, &xso->dev_tracker); } } + +static inline void xfrm_dev_policy_delete(struct xfrm_policy *x) +{ + struct xfrm_dev_offload *xdo = &x->xdo; + struct net_device *dev = xdo->dev; + + if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_policy_delete) + dev->xfrmdev_ops->xdo_dev_policy_delete(x); +} + +static inline void xfrm_dev_policy_free(struct xfrm_policy *x) +{ + struct xfrm_dev_offload *xdo = &x->xdo; + struct net_device *dev = xdo->dev; + + if (dev && dev->xfrmdev_ops) { + if (dev->xfrmdev_ops->xdo_dev_policy_free) + dev->xfrmdev_ops->xdo_dev_policy_free(x); + xdo->dev = NULL; + netdev_put(dev, &xdo->dev_tracker); + } +} #else static inline void xfrm_dev_resume(struct sk_buff *skb) { @@ -1967,6 +2030,21 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) { } +static inline int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack) +{ + return 0; +} + +static inline void xfrm_dev_policy_delete(struct xfrm_policy *x) +{ +} + +static inline void xfrm_dev_policy_free(struct xfrm_policy *x) +{ +} + static inline bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) { return false; diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index c078c48a8e6d..a6190aa1b406 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -66,6 +66,7 @@ enum fscache_cookie_trace { fscache_cookie_put_work, fscache_cookie_see_active, fscache_cookie_see_lru_discard, + fscache_cookie_see_lru_discard_clear, fscache_cookie_see_lru_do_one, fscache_cookie_see_relinquish, fscache_cookie_see_withdraw, @@ -149,6 +150,7 @@ enum fscache_access_trace { EM(fscache_cookie_put_work, "PQ work ") \ EM(fscache_cookie_see_active, "- activ") \ EM(fscache_cookie_see_lru_discard, "- x-lru") \ + EM(fscache_cookie_see_lru_discard_clear,"- lrudc") \ EM(fscache_cookie_see_lru_do_one, "- lrudo") \ EM(fscache_cookie_see_relinquish, "- x-rlq") \ EM(fscache_cookie_see_withdraw, "- x-wth") \ diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h index b9886d1df825..049b52e7aa6a 100644 --- a/include/trace/events/rxrpc.h +++ b/include/trace/events/rxrpc.h @@ -16,44 +16,121 @@ /* * Declare tracing information enums and their string mappings for display. */ +#define rxrpc_call_poke_traces \ + EM(rxrpc_call_poke_error, "Error") \ + EM(rxrpc_call_poke_idle, "Idle") \ + EM(rxrpc_call_poke_start, "Start") \ + EM(rxrpc_call_poke_timer, "Timer") \ + E_(rxrpc_call_poke_timer_now, "Timer-now") + #define rxrpc_skb_traces \ - EM(rxrpc_skb_ack, "ACK") \ - EM(rxrpc_skb_cleaned, "CLN") \ - EM(rxrpc_skb_cloned_jumbo, "CLJ") \ - EM(rxrpc_skb_freed, "FRE") \ - EM(rxrpc_skb_got, "GOT") \ - EM(rxrpc_skb_lost, "*L*") \ - EM(rxrpc_skb_new, "NEW") \ - EM(rxrpc_skb_purged, "PUR") \ - EM(rxrpc_skb_received, "RCV") \ - EM(rxrpc_skb_rotated, "ROT") \ - EM(rxrpc_skb_seen, "SEE") \ - EM(rxrpc_skb_unshared, "UNS") \ - E_(rxrpc_skb_unshared_nomem, "US0") + EM(rxrpc_skb_eaten_by_unshare, "ETN unshare ") \ + EM(rxrpc_skb_eaten_by_unshare_nomem, "ETN unshar-nm") \ + EM(rxrpc_skb_get_conn_work, "GET conn-work") \ + EM(rxrpc_skb_get_local_work, "GET locl-work") \ + EM(rxrpc_skb_get_reject_work, "GET rej-work ") \ + EM(rxrpc_skb_get_to_recvmsg, "GET to-recv ") \ + EM(rxrpc_skb_get_to_recvmsg_oos, "GET to-recv-o") \ + EM(rxrpc_skb_new_encap_rcv, "NEW encap-rcv") \ + EM(rxrpc_skb_new_error_report, "NEW error-rpt") \ + EM(rxrpc_skb_new_jumbo_subpacket, "NEW jumbo-sub") \ + EM(rxrpc_skb_new_unshared, "NEW unshared ") \ + EM(rxrpc_skb_put_conn_work, "PUT conn-work") \ + EM(rxrpc_skb_put_error_report, "PUT error-rep") \ + EM(rxrpc_skb_put_input, "PUT input ") \ + EM(rxrpc_skb_put_jumbo_subpacket, "PUT jumbo-sub") \ + EM(rxrpc_skb_put_purge, "PUT purge ") \ + EM(rxrpc_skb_put_rotate, "PUT rotate ") \ + EM(rxrpc_skb_put_unknown, "PUT unknown ") \ + EM(rxrpc_skb_see_conn_work, "SEE conn-work") \ + EM(rxrpc_skb_see_recvmsg, "SEE recvmsg ") \ + EM(rxrpc_skb_see_reject, "SEE reject ") \ + EM(rxrpc_skb_see_rotate, "SEE rotate ") \ + E_(rxrpc_skb_see_version, "SEE version ") #define rxrpc_local_traces \ - EM(rxrpc_local_got, "GOT") \ - EM(rxrpc_local_new, "NEW") \ - EM(rxrpc_local_processing, "PRO") \ - EM(rxrpc_local_put, "PUT") \ - EM(rxrpc_local_queued, "QUE") \ - E_(rxrpc_local_tx_ack, "TAK") + EM(rxrpc_local_free, "FREE ") \ + EM(rxrpc_local_get_call, "GET call ") \ + EM(rxrpc_local_get_client_conn, "GET conn-cln") \ + EM(rxrpc_local_get_for_use, "GET for-use ") \ + EM(rxrpc_local_get_peer, "GET peer ") \ + EM(rxrpc_local_get_prealloc_conn, "GET conn-pre") \ + EM(rxrpc_local_new, "NEW ") \ + EM(rxrpc_local_put_bind, "PUT bind ") \ + EM(rxrpc_local_put_call, "PUT call ") \ + EM(rxrpc_local_put_for_use, "PUT for-use ") \ + EM(rxrpc_local_put_kill_conn, "PUT conn-kil") \ + EM(rxrpc_local_put_peer, "PUT peer ") \ + EM(rxrpc_local_put_prealloc_conn, "PUT conn-pre") \ + EM(rxrpc_local_put_release_sock, "PUT rel-sock") \ + EM(rxrpc_local_stop, "STOP ") \ + EM(rxrpc_local_stopped, "STOPPED ") \ + EM(rxrpc_local_unuse_bind, "UNU bind ") \ + EM(rxrpc_local_unuse_conn_work, "UNU conn-wrk") \ + EM(rxrpc_local_unuse_peer_keepalive, "UNU peer-kpa") \ + EM(rxrpc_local_unuse_release_sock, "UNU rel-sock") \ + EM(rxrpc_local_use_conn_work, "USE conn-wrk") \ + EM(rxrpc_local_use_lookup, "USE lookup ") \ + E_(rxrpc_local_use_peer_keepalive, "USE peer-kpa") #define rxrpc_peer_traces \ - EM(rxrpc_peer_got, "GOT") \ - EM(rxrpc_peer_new, "NEW") \ - EM(rxrpc_peer_processing, "PRO") \ - E_(rxrpc_peer_put, "PUT") + EM(rxrpc_peer_free, "FREE ") \ + EM(rxrpc_peer_get_accept, "GET accept ") \ + EM(rxrpc_peer_get_activate_call, "GET act-call") \ + EM(rxrpc_peer_get_bundle, "GET bundle ") \ + EM(rxrpc_peer_get_client_conn, "GET cln-conn") \ + EM(rxrpc_peer_get_input, "GET input ") \ + EM(rxrpc_peer_get_input_error, "GET inpt-err") \ + EM(rxrpc_peer_get_keepalive, "GET keepaliv") \ + EM(rxrpc_peer_get_lookup_client, "GET look-cln") \ + EM(rxrpc_peer_get_service_conn, "GET srv-conn") \ + EM(rxrpc_peer_new_client, "NEW client ") \ + EM(rxrpc_peer_new_prealloc, "NEW prealloc") \ + EM(rxrpc_peer_put_bundle, "PUT bundle ") \ + EM(rxrpc_peer_put_call, "PUT call ") \ + EM(rxrpc_peer_put_conn, "PUT conn ") \ + EM(rxrpc_peer_put_discard_tmp, "PUT disc-tmp") \ + EM(rxrpc_peer_put_input, "PUT input ") \ + EM(rxrpc_peer_put_input_error, "PUT inpt-err") \ + E_(rxrpc_peer_put_keepalive, "PUT keepaliv") + +#define rxrpc_bundle_traces \ + EM(rxrpc_bundle_free, "FREE ") \ + EM(rxrpc_bundle_get_client_call, "GET clt-call") \ + EM(rxrpc_bundle_get_client_conn, "GET clt-conn") \ + EM(rxrpc_bundle_get_service_conn, "GET svc-conn") \ + EM(rxrpc_bundle_put_conn, "PUT conn ") \ + EM(rxrpc_bundle_put_discard, "PUT discard ") \ + E_(rxrpc_bundle_new, "NEW ") #define rxrpc_conn_traces \ - EM(rxrpc_conn_got, "GOT") \ - EM(rxrpc_conn_new_client, "NWc") \ - EM(rxrpc_conn_new_service, "NWs") \ - EM(rxrpc_conn_put_client, "PTc") \ - EM(rxrpc_conn_put_service, "PTs") \ - EM(rxrpc_conn_queued, "QUE") \ - EM(rxrpc_conn_reap_service, "RPs") \ - E_(rxrpc_conn_seen, "SEE") + EM(rxrpc_conn_free, "FREE ") \ + EM(rxrpc_conn_get_activate_call, "GET act-call") \ + EM(rxrpc_conn_get_call_input, "GET inp-call") \ + EM(rxrpc_conn_get_conn_input, "GET inp-conn") \ + EM(rxrpc_conn_get_idle, "GET idle ") \ + EM(rxrpc_conn_get_poke, "GET poke ") \ + EM(rxrpc_conn_get_service_conn, "GET svc-conn") \ + EM(rxrpc_conn_new_client, "NEW client ") \ + EM(rxrpc_conn_new_service, "NEW service ") \ + EM(rxrpc_conn_put_call, "PUT call ") \ + EM(rxrpc_conn_put_call_input, "PUT inp-call") \ + EM(rxrpc_conn_put_conn_input, "PUT inp-conn") \ + EM(rxrpc_conn_put_discard, "PUT discard ") \ + EM(rxrpc_conn_put_discard_idle, "PUT disc-idl") \ + EM(rxrpc_conn_put_local_dead, "PUT loc-dead") \ + EM(rxrpc_conn_put_noreuse, "PUT noreuse ") \ + EM(rxrpc_conn_put_poke, "PUT poke ") \ + EM(rxrpc_conn_put_service_reaped, "PUT svc-reap") \ + EM(rxrpc_conn_put_unbundle, "PUT unbundle") \ + EM(rxrpc_conn_put_unidle, "PUT unidle ") \ + EM(rxrpc_conn_queue_challenge, "QUE chall ") \ + EM(rxrpc_conn_queue_retry_work, "QUE retry-wk") \ + EM(rxrpc_conn_queue_rx_work, "QUE rx-work ") \ + EM(rxrpc_conn_queue_timer, "QUE timer ") \ + EM(rxrpc_conn_see_new_service_conn, "SEE new-svc ") \ + EM(rxrpc_conn_see_reap_service, "SEE reap-svc") \ + E_(rxrpc_conn_see_work, "SEE work ") #define rxrpc_client_traces \ EM(rxrpc_client_activate_chans, "Activa") \ @@ -71,26 +148,36 @@ E_(rxrpc_client_to_idle, "->Idle") #define rxrpc_call_traces \ - EM(rxrpc_call_connected, "CON") \ - EM(rxrpc_call_error, "*E*") \ - EM(rxrpc_call_got, "GOT") \ - EM(rxrpc_call_got_kernel, "Gke") \ - EM(rxrpc_call_got_timer, "GTM") \ - EM(rxrpc_call_got_tx, "Gtx") \ - EM(rxrpc_call_got_userid, "Gus") \ - EM(rxrpc_call_new_client, "NWc") \ - EM(rxrpc_call_new_service, "NWs") \ - EM(rxrpc_call_put, "PUT") \ - EM(rxrpc_call_put_kernel, "Pke") \ - EM(rxrpc_call_put_noqueue, "PnQ") \ - EM(rxrpc_call_put_notimer, "PnT") \ - EM(rxrpc_call_put_timer, "PTM") \ - EM(rxrpc_call_put_tx, "Ptx") \ - EM(rxrpc_call_put_userid, "Pus") \ - EM(rxrpc_call_queued, "QUE") \ - EM(rxrpc_call_queued_ref, "QUR") \ - EM(rxrpc_call_release, "RLS") \ - E_(rxrpc_call_seen, "SEE") + EM(rxrpc_call_get_input, "GET input ") \ + EM(rxrpc_call_get_kernel_service, "GET krnl-srv") \ + EM(rxrpc_call_get_notify_socket, "GET notify ") \ + EM(rxrpc_call_get_poke, "GET poke ") \ + EM(rxrpc_call_get_recvmsg, "GET recvmsg ") \ + EM(rxrpc_call_get_release_sock, "GET rel-sock") \ + EM(rxrpc_call_get_sendmsg, "GET sendmsg ") \ + EM(rxrpc_call_get_userid, "GET user-id ") \ + EM(rxrpc_call_new_client, "NEW client ") \ + EM(rxrpc_call_new_prealloc_service, "NEW prealloc") \ + EM(rxrpc_call_put_discard_prealloc, "PUT disc-pre") \ + EM(rxrpc_call_put_discard_error, "PUT disc-err") \ + EM(rxrpc_call_put_input, "PUT input ") \ + EM(rxrpc_call_put_kernel, "PUT kernel ") \ + EM(rxrpc_call_put_poke, "PUT poke ") \ + EM(rxrpc_call_put_recvmsg, "PUT recvmsg ") \ + EM(rxrpc_call_put_release_sock, "PUT rls-sock") \ + EM(rxrpc_call_put_release_sock_tba, "PUT rls-sk-a") \ + EM(rxrpc_call_put_sendmsg, "PUT sendmsg ") \ + EM(rxrpc_call_put_unnotify, "PUT unnotify") \ + EM(rxrpc_call_put_userid_exists, "PUT u-exists") \ + EM(rxrpc_call_see_accept, "SEE accept ") \ + EM(rxrpc_call_see_activate_client, "SEE act-clnt") \ + EM(rxrpc_call_see_connect_failed, "SEE con-fail") \ + EM(rxrpc_call_see_connected, "SEE connect ") \ + EM(rxrpc_call_see_distribute_error, "SEE dist-err") \ + EM(rxrpc_call_see_input, "SEE input ") \ + EM(rxrpc_call_see_release, "SEE release ") \ + EM(rxrpc_call_see_userid_exists, "SEE u-exists") \ + E_(rxrpc_call_see_zap, "SEE zap ") #define rxrpc_txqueue_traces \ EM(rxrpc_txqueue_await_reply, "AWR") \ @@ -179,6 +266,7 @@ EM(rxrpc_propose_ack_respond_to_ping, "Rsp2Png") \ EM(rxrpc_propose_ack_retry_tx, "RetryTx") \ EM(rxrpc_propose_ack_rotate_rx, "RxAck ") \ + EM(rxrpc_propose_ack_rx_idle, "RxIdle ") \ E_(rxrpc_propose_ack_terminal_ack, "ClTerm ") #define rxrpc_congest_modes \ @@ -273,6 +361,7 @@ EM(rxrpc_txbuf_put_rotated, "PUT ROTATED") \ EM(rxrpc_txbuf_put_send_aborted, "PUT SEND-X ") \ EM(rxrpc_txbuf_put_trans, "PUT TRANS ") \ + EM(rxrpc_txbuf_see_out_of_step, "OUT-OF-STEP") \ EM(rxrpc_txbuf_see_send_more, "SEE SEND+ ") \ E_(rxrpc_txbuf_see_unacked, "SEE UNACKED") @@ -287,6 +376,8 @@ #define EM(a, b) a, #define E_(a, b) a +enum rxrpc_bundle_trace { rxrpc_bundle_traces } __mode(byte); +enum rxrpc_call_poke_trace { rxrpc_call_poke_traces } __mode(byte); enum rxrpc_call_trace { rxrpc_call_traces } __mode(byte); enum rxrpc_client_trace { rxrpc_client_traces } __mode(byte); enum rxrpc_congest_change { rxrpc_congest_changes } __mode(byte); @@ -316,6 +407,8 @@ enum rxrpc_txqueue_trace { rxrpc_txqueue_traces } __mode(byte); #define EM(a, b) TRACE_DEFINE_ENUM(a); #define E_(a, b) TRACE_DEFINE_ENUM(a); +rxrpc_bundle_traces; +rxrpc_call_poke_traces; rxrpc_call_traces; rxrpc_client_traces; rxrpc_congest_changes; @@ -345,83 +438,98 @@ rxrpc_txqueue_traces; TRACE_EVENT(rxrpc_local, TP_PROTO(unsigned int local_debug_id, enum rxrpc_local_trace op, - int usage, const void *where), + int ref, int usage), - TP_ARGS(local_debug_id, op, usage, where), + TP_ARGS(local_debug_id, op, ref, usage), TP_STRUCT__entry( __field(unsigned int, local ) __field(int, op ) + __field(int, ref ) __field(int, usage ) - __field(const void *, where ) ), TP_fast_assign( __entry->local = local_debug_id; __entry->op = op; + __entry->ref = ref; __entry->usage = usage; - __entry->where = where; ), - TP_printk("L=%08x %s u=%d sp=%pSR", + TP_printk("L=%08x %s r=%d u=%d", __entry->local, __print_symbolic(__entry->op, rxrpc_local_traces), - __entry->usage, - __entry->where) + __entry->ref, + __entry->usage) ); TRACE_EVENT(rxrpc_peer, - TP_PROTO(unsigned int peer_debug_id, enum rxrpc_peer_trace op, - int usage, const void *where), + TP_PROTO(unsigned int peer_debug_id, int ref, enum rxrpc_peer_trace why), - TP_ARGS(peer_debug_id, op, usage, where), + TP_ARGS(peer_debug_id, ref, why), TP_STRUCT__entry( __field(unsigned int, peer ) - __field(int, op ) - __field(int, usage ) - __field(const void *, where ) + __field(int, ref ) + __field(int, why ) ), TP_fast_assign( __entry->peer = peer_debug_id; - __entry->op = op; - __entry->usage = usage; - __entry->where = where; + __entry->ref = ref; + __entry->why = why; ), - TP_printk("P=%08x %s u=%d sp=%pSR", + TP_printk("P=%08x %s r=%d", __entry->peer, - __print_symbolic(__entry->op, rxrpc_peer_traces), - __entry->usage, - __entry->where) + __print_symbolic(__entry->why, rxrpc_peer_traces), + __entry->ref) + ); + +TRACE_EVENT(rxrpc_bundle, + TP_PROTO(unsigned int bundle_debug_id, int ref, enum rxrpc_bundle_trace why), + + TP_ARGS(bundle_debug_id, ref, why), + + TP_STRUCT__entry( + __field(unsigned int, bundle ) + __field(int, ref ) + __field(int, why ) + ), + + TP_fast_assign( + __entry->bundle = bundle_debug_id; + __entry->ref = ref; + __entry->why = why; + ), + + TP_printk("CB=%08x %s r=%d", + __entry->bundle, + __print_symbolic(__entry->why, rxrpc_bundle_traces), + __entry->ref) ); TRACE_EVENT(rxrpc_conn, - TP_PROTO(unsigned int conn_debug_id, enum rxrpc_conn_trace op, - int usage, const void *where), + TP_PROTO(unsigned int conn_debug_id, int ref, enum rxrpc_conn_trace why), - TP_ARGS(conn_debug_id, op, usage, where), + TP_ARGS(conn_debug_id, ref, why), TP_STRUCT__entry( __field(unsigned int, conn ) - __field(int, op ) - __field(int, usage ) - __field(const void *, where ) + __field(int, ref ) + __field(int, why ) ), TP_fast_assign( __entry->conn = conn_debug_id; - __entry->op = op; - __entry->usage = usage; - __entry->where = where; + __entry->ref = ref; + __entry->why = why; ), - TP_printk("C=%08x %s u=%d sp=%pSR", + TP_printk("C=%08x %s r=%d", __entry->conn, - __print_symbolic(__entry->op, rxrpc_conn_traces), - __entry->usage, - __entry->where) + __print_symbolic(__entry->why, rxrpc_conn_traces), + __entry->ref) ); TRACE_EVENT(rxrpc_client, @@ -455,63 +563,57 @@ TRACE_EVENT(rxrpc_client, ); TRACE_EVENT(rxrpc_call, - TP_PROTO(unsigned int call_debug_id, enum rxrpc_call_trace op, - int usage, const void *where, const void *aux), + TP_PROTO(unsigned int call_debug_id, int ref, unsigned long aux, + enum rxrpc_call_trace why), - TP_ARGS(call_debug_id, op, usage, where, aux), + TP_ARGS(call_debug_id, ref, aux, why), TP_STRUCT__entry( __field(unsigned int, call ) - __field(int, op ) - __field(int, usage ) - __field(const void *, where ) - __field(const void *, aux ) + __field(int, ref ) + __field(int, why ) + __field(unsigned long, aux ) ), TP_fast_assign( __entry->call = call_debug_id; - __entry->op = op; - __entry->usage = usage; - __entry->where = where; + __entry->ref = ref; + __entry->why = why; __entry->aux = aux; ), - TP_printk("c=%08x %s u=%d sp=%pSR a=%p", + TP_printk("c=%08x %s r=%d a=%lx", __entry->call, - __print_symbolic(__entry->op, rxrpc_call_traces), - __entry->usage, - __entry->where, + __print_symbolic(__entry->why, rxrpc_call_traces), + __entry->ref, __entry->aux) ); TRACE_EVENT(rxrpc_skb, - TP_PROTO(struct sk_buff *skb, enum rxrpc_skb_trace op, - int usage, int mod_count, const void *where), + TP_PROTO(struct sk_buff *skb, int usage, int mod_count, + enum rxrpc_skb_trace why), - TP_ARGS(skb, op, usage, mod_count, where), + TP_ARGS(skb, usage, mod_count, why), TP_STRUCT__entry( __field(struct sk_buff *, skb ) - __field(enum rxrpc_skb_trace, op ) __field(int, usage ) __field(int, mod_count ) - __field(const void *, where ) + __field(enum rxrpc_skb_trace, why ) ), TP_fast_assign( __entry->skb = skb; - __entry->op = op; __entry->usage = usage; __entry->mod_count = mod_count; - __entry->where = where; + __entry->why = why; ), - TP_printk("s=%p Rx %s u=%d m=%d p=%pSR", + TP_printk("s=%p Rx %s u=%d m=%d", __entry->skb, - __print_symbolic(__entry->op, rxrpc_skb_traces), + __print_symbolic(__entry->why, rxrpc_skb_traces), __entry->usage, - __entry->mod_count, - __entry->where) + __entry->mod_count) ); TRACE_EVENT(rxrpc_rx_packet, @@ -623,6 +725,7 @@ TRACE_EVENT(rxrpc_txqueue, __field(rxrpc_seq_t, acks_hard_ack ) __field(rxrpc_seq_t, tx_bottom ) __field(rxrpc_seq_t, tx_top ) + __field(rxrpc_seq_t, tx_prepared ) __field(int, tx_winsize ) ), @@ -632,16 +735,18 @@ TRACE_EVENT(rxrpc_txqueue, __entry->acks_hard_ack = call->acks_hard_ack; __entry->tx_bottom = call->tx_bottom; __entry->tx_top = call->tx_top; + __entry->tx_prepared = call->tx_prepared; __entry->tx_winsize = call->tx_winsize; ), - TP_printk("c=%08x %s f=%08x h=%08x n=%u/%u/%u", + TP_printk("c=%08x %s f=%08x h=%08x n=%u/%u/%u/%u", __entry->call, __print_symbolic(__entry->why, rxrpc_txqueue_traces), __entry->tx_bottom, __entry->acks_hard_ack, __entry->tx_top - __entry->tx_bottom, __entry->tx_top - __entry->acks_hard_ack, + __entry->tx_prepared - __entry->tx_bottom, __entry->tx_winsize) ); @@ -733,6 +838,66 @@ TRACE_EVENT(rxrpc_rx_abort, __entry->abort_code) ); +TRACE_EVENT(rxrpc_rx_challenge, + TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial, + u32 version, u32 nonce, u32 min_level), + + TP_ARGS(conn, serial, version, nonce, min_level), + + TP_STRUCT__entry( + __field(unsigned int, conn ) + __field(rxrpc_serial_t, serial ) + __field(u32, version ) + __field(u32, nonce ) + __field(u32, min_level ) + ), + + TP_fast_assign( + __entry->conn = conn->debug_id; + __entry->serial = serial; + __entry->version = version; + __entry->nonce = nonce; + __entry->min_level = min_level; + ), + + TP_printk("C=%08x CHALLENGE %08x v=%x n=%x ml=%x", + __entry->conn, + __entry->serial, + __entry->version, + __entry->nonce, + __entry->min_level) + ); + +TRACE_EVENT(rxrpc_rx_response, + TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial, + u32 version, u32 kvno, u32 ticket_len), + + TP_ARGS(conn, serial, version, kvno, ticket_len), + + TP_STRUCT__entry( + __field(unsigned int, conn ) + __field(rxrpc_serial_t, serial ) + __field(u32, version ) + __field(u32, kvno ) + __field(u32, ticket_len ) + ), + + TP_fast_assign( + __entry->conn = conn->debug_id; + __entry->serial = serial; + __entry->version = version; + __entry->kvno = kvno; + __entry->ticket_len = ticket_len; + ), + + TP_printk("C=%08x RESPONSE %08x v=%x kvno=%x tl=%x", + __entry->conn, + __entry->serial, + __entry->version, + __entry->kvno, + __entry->ticket_len) + ); + TRACE_EVENT(rxrpc_rx_rwind_change, TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t serial, u32 rwind, bool wake), @@ -1278,6 +1443,44 @@ TRACE_EVENT(rxrpc_congest, __entry->sum.retrans_timeo ? " rTxTo" : "") ); +TRACE_EVENT(rxrpc_reset_cwnd, + TP_PROTO(struct rxrpc_call *call, ktime_t now), + + TP_ARGS(call, now), + + TP_STRUCT__entry( + __field(unsigned int, call ) + __field(enum rxrpc_congest_mode, mode ) + __field(unsigned short, cwnd ) + __field(unsigned short, extra ) + __field(rxrpc_seq_t, hard_ack ) + __field(rxrpc_seq_t, prepared ) + __field(ktime_t, since_last_tx ) + __field(bool, has_data ) + ), + + TP_fast_assign( + __entry->call = call->debug_id; + __entry->mode = call->cong_mode; + __entry->cwnd = call->cong_cwnd; + __entry->extra = call->cong_extra; + __entry->hard_ack = call->acks_hard_ack; + __entry->prepared = call->tx_prepared - call->tx_bottom; + __entry->since_last_tx = ktime_sub(now, call->tx_last_sent); + __entry->has_data = !list_empty(&call->tx_sendmsg); + ), + + TP_printk("c=%08x q=%08x %s cw=%u+%u pr=%u tm=%llu d=%u", + __entry->call, + __entry->hard_ack, + __print_symbolic(__entry->mode, rxrpc_congest_modes), + __entry->cwnd, + __entry->extra, + __entry->prepared, + ktime_to_ns(__entry->since_last_tx), + __entry->has_data) + ); + TRACE_EVENT(rxrpc_disconnect_call, TP_PROTO(struct rxrpc_call *call), @@ -1352,6 +1555,7 @@ TRACE_EVENT(rxrpc_connect_call, __field(unsigned long, user_call_ID ) __field(u32, cid ) __field(u32, call_id ) + __field_struct(struct sockaddr_rxrpc, srx ) ), TP_fast_assign( @@ -1359,33 +1563,42 @@ TRACE_EVENT(rxrpc_connect_call, __entry->user_call_ID = call->user_call_ID; __entry->cid = call->cid; __entry->call_id = call->call_id; + __entry->srx = call->dest_srx; ), - TP_printk("c=%08x u=%p %08x:%08x", + TP_printk("c=%08x u=%p %08x:%08x dst=%pISp", __entry->call, (void *)__entry->user_call_ID, __entry->cid, - __entry->call_id) + __entry->call_id, + &__entry->srx.transport) ); TRACE_EVENT(rxrpc_resend, - TP_PROTO(struct rxrpc_call *call), + TP_PROTO(struct rxrpc_call *call, struct sk_buff *ack), - TP_ARGS(call), + TP_ARGS(call, ack), TP_STRUCT__entry( __field(unsigned int, call ) __field(rxrpc_seq_t, seq ) + __field(rxrpc_seq_t, transmitted ) + __field(rxrpc_serial_t, ack_serial ) ), TP_fast_assign( + struct rxrpc_skb_priv *sp = ack ? rxrpc_skb(ack) : NULL; __entry->call = call->debug_id; __entry->seq = call->acks_hard_ack; + __entry->transmitted = call->tx_transmitted; + __entry->ack_serial = sp ? sp->hdr.serial : 0; ), - TP_printk("c=%08x q=%x", + TP_printk("c=%08x r=%x q=%x tq=%x", __entry->call, - __entry->seq) + __entry->ack_serial, + __entry->seq, + __entry->transmitted) ); TRACE_EVENT(rxrpc_rx_icmp, @@ -1586,6 +1799,47 @@ TRACE_EVENT(rxrpc_txbuf, __entry->ref) ); +TRACE_EVENT(rxrpc_poke_call, + TP_PROTO(struct rxrpc_call *call, bool busy, + enum rxrpc_call_poke_trace what), + + TP_ARGS(call, busy, what), + + TP_STRUCT__entry( + __field(unsigned int, call_debug_id ) + __field(bool, busy ) + __field(enum rxrpc_call_poke_trace, what ) + ), + + TP_fast_assign( + __entry->call_debug_id = call->debug_id; + __entry->busy = busy; + __entry->what = what; + ), + + TP_printk("c=%08x %s%s", + __entry->call_debug_id, + __print_symbolic(__entry->what, rxrpc_call_poke_traces), + __entry->busy ? "!" : "") + ); + +TRACE_EVENT(rxrpc_call_poked, + TP_PROTO(struct rxrpc_call *call), + + TP_ARGS(call), + + TP_STRUCT__entry( + __field(unsigned int, call_debug_id ) + ), + + TP_fast_assign( + __entry->call_debug_id = call->debug_id; + ), + + TP_printk("c=%08x", + __entry->call_debug_id) + ); + #undef EM #undef E_ #endif /* _TRACE_RXRPC_H */ diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index 498d0d5d0957..3782d4219ac9 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -610,6 +610,8 @@ enum devlink_attr { DEVLINK_ATTR_RATE_TX_PRIORITY, /* u32 */ DEVLINK_ATTR_RATE_TX_WEIGHT, /* u32 */ + DEVLINK_ATTR_REGION_DIRECT, /* flag */ + /* add new attributes above here, update the policy in devlink.c */ __DEVLINK_ATTR_MAX, @@ -656,11 +658,24 @@ enum devlink_resource_unit { DEVLINK_RESOURCE_UNIT_ENTRY, }; +enum devlink_port_fn_attr_cap { + DEVLINK_PORT_FN_ATTR_CAP_ROCE_BIT, + DEVLINK_PORT_FN_ATTR_CAP_MIGRATABLE_BIT, + + /* Add new caps above */ + __DEVLINK_PORT_FN_ATTR_CAPS_MAX, +}; + +#define DEVLINK_PORT_FN_CAP_ROCE _BITUL(DEVLINK_PORT_FN_ATTR_CAP_ROCE_BIT) +#define DEVLINK_PORT_FN_CAP_MIGRATABLE \ + _BITUL(DEVLINK_PORT_FN_ATTR_CAP_MIGRATABLE_BIT) + enum devlink_port_function_attr { DEVLINK_PORT_FUNCTION_ATTR_UNSPEC, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, /* binary */ DEVLINK_PORT_FN_ATTR_STATE, /* u8 */ DEVLINK_PORT_FN_ATTR_OPSTATE, /* u8 */ + DEVLINK_PORT_FN_ATTR_CAPS, /* bitfield32 */ __DEVLINK_PORT_FUNCTION_ATTR_MAX, DEVLINK_PORT_FUNCTION_ATTR_MAX = __DEVLINK_PORT_FUNCTION_ATTR_MAX - 1 diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index aaf7c6963d61..5799a9db034e 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -51,6 +51,7 @@ enum { ETHTOOL_MSG_MODULE_SET, ETHTOOL_MSG_PSE_GET, ETHTOOL_MSG_PSE_SET, + ETHTOOL_MSG_RSS_GET, /* add new constants above here */ __ETHTOOL_MSG_USER_CNT, @@ -97,6 +98,7 @@ enum { ETHTOOL_MSG_MODULE_GET_REPLY, ETHTOOL_MSG_MODULE_NTF, ETHTOOL_MSG_PSE_GET_REPLY, + ETHTOOL_MSG_RSS_GET_REPLY, /* add new constants above here */ __ETHTOOL_MSG_KERNEL_CNT, @@ -880,6 +882,18 @@ enum { ETHTOOL_A_PSE_MAX = (__ETHTOOL_A_PSE_CNT - 1) }; +enum { + ETHTOOL_A_RSS_UNSPEC, + ETHTOOL_A_RSS_HEADER, + ETHTOOL_A_RSS_CONTEXT, /* u32 */ + ETHTOOL_A_RSS_HFUNC, /* u32 */ + ETHTOOL_A_RSS_INDIR, /* binary */ + ETHTOOL_A_RSS_HKEY, /* binary */ + + __ETHTOOL_A_RSS_CNT, + ETHTOOL_A_RSS_MAX = (__ETHTOOL_A_RSS_CNT - 1), +}; + /* generic netlink info */ #define ETHTOOL_GENL_NAME "ethtool" #define ETHTOOL_GENL_VERSION 1 diff --git a/include/uapi/linux/if_tun.h b/include/uapi/linux/if_tun.h index b6d7b868f290..287cdc81c939 100644 --- a/include/uapi/linux/if_tun.h +++ b/include/uapi/linux/if_tun.h @@ -90,6 +90,8 @@ #define TUN_F_TSO6 0x04 /* I can handle TSO for IPv6 packets */ #define TUN_F_TSO_ECN 0x08 /* I can handle TSO with ECN bits. */ #define TUN_F_UFO 0x10 /* I can handle UFO packets */ +#define TUN_F_USO4 0x20 /* I can handle USO for IPv4 packets */ +#define TUN_F_USO6 0x40 /* I can handle USO for IPv6 packets */ /* Protocol info prepended to the packets (when IFF_NO_PI is not set) */ #define TUN_PKT_STRIP 0x0001 diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h index dfe19bf13f4c..32af2d278cb4 100644 --- a/include/uapi/linux/mptcp.h +++ b/include/uapi/linux/mptcp.h @@ -160,6 +160,12 @@ struct mptcp_info { * daddr4 | daddr6, sport, dport, backup, if_idx * [, error] * The priority of a subflow has changed. 'error' should not be set. + * + * MPTCP_EVENT_LISTENER_CREATED: family, sport, saddr4 | saddr6 + * A new PM listener is created. + * + * MPTCP_EVENT_LISTENER_CLOSED: family, sport, saddr4 | saddr6 + * A PM listener is closed. */ enum mptcp_event_type { MPTCP_EVENT_UNSPEC = 0, @@ -174,6 +180,9 @@ enum mptcp_event_type { MPTCP_EVENT_SUB_CLOSED = 11, MPTCP_EVENT_SUB_PRIORITY = 13, + + MPTCP_EVENT_LISTENER_CREATED = 15, + MPTCP_EVENT_LISTENER_CLOSED = 16, }; enum mptcp_event_attr { diff --git a/include/uapi/linux/net_tstamp.h b/include/uapi/linux/net_tstamp.h index 55501e5e7ac8..a2c66b3d7f0f 100644 --- a/include/uapi/linux/net_tstamp.h +++ b/include/uapi/linux/net_tstamp.h @@ -31,8 +31,9 @@ enum { SOF_TIMESTAMPING_OPT_PKTINFO = (1<<13), SOF_TIMESTAMPING_OPT_TX_SWHW = (1<<14), SOF_TIMESTAMPING_BIND_PHC = (1 << 15), + SOF_TIMESTAMPING_OPT_ID_TCP = (1 << 16), - SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_BIND_PHC, + SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_OPT_ID_TCP, SOF_TIMESTAMPING_MASK = (SOF_TIMESTAMPING_LAST - 1) | SOF_TIMESTAMPING_LAST }; diff --git a/include/uapi/linux/openvswitch.h b/include/uapi/linux/openvswitch.h index 94066f87e9ee..c5d62ee82567 100644 --- a/include/uapi/linux/openvswitch.h +++ b/include/uapi/linux/openvswitch.h @@ -277,11 +277,25 @@ enum ovs_vport_attr { OVS_VPORT_ATTR_PAD, OVS_VPORT_ATTR_IFINDEX, OVS_VPORT_ATTR_NETNSID, + OVS_VPORT_ATTR_UPCALL_STATS, __OVS_VPORT_ATTR_MAX }; #define OVS_VPORT_ATTR_MAX (__OVS_VPORT_ATTR_MAX - 1) +/** + * enum ovs_vport_upcall_attr - attributes for %OVS_VPORT_UPCALL* commands + * @OVS_VPORT_UPCALL_SUCCESS: 64-bit upcall success packets. + * @OVS_VPORT_UPCALL_FAIL: 64-bit upcall fail packets. + */ +enum ovs_vport_upcall_attr { + OVS_VPORT_UPCALL_ATTR_SUCCESS, + OVS_VPORT_UPCALL_ATTR_FAIL, + __OVS_VPORT_UPCALL_ATTR_MAX +}; + +#define OVS_VPORT_UPCALL_ATTR_MAX (__OVS_VPORT_UPCALL_ATTR_MAX - 1) + enum { OVS_VXLAN_EXT_UNSPEC, OVS_VXLAN_EXT_GBP, /* Flag or __u32 */ diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index 6cb842ea8979..b4062bed186a 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -57,6 +57,9 @@ * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ #define VIRTIO_NET_F_NOTF_COAL 53 /* Device supports notifications coalescing */ +#define VIRTIO_NET_F_GUEST_USO4 54 /* Guest can handle USOv4 in. */ +#define VIRTIO_NET_F_GUEST_USO6 55 /* Guest can handle USOv6 in. */ +#define VIRTIO_NET_F_HOST_USO 56 /* Host can handle USO in. */ #define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */ #define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */ #define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ @@ -130,6 +133,7 @@ struct virtio_net_hdr_v1 { #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ #define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ +#define VIRTIO_NET_HDR_GSO_UDP_L4 5 /* GSO frame, IPv4& IPv6 UDP (USO) */ #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ __u8 gso_type; __virtio16 hdr_len; /* Ethernet + IP + tcp/udp hdrs */ diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h index 4f84ea7ee14c..23543c33fee8 100644 --- a/include/uapi/linux/xfrm.h +++ b/include/uapi/linux/xfrm.h @@ -519,6 +519,12 @@ struct xfrm_user_offload { */ #define XFRM_OFFLOAD_IPV6 1 #define XFRM_OFFLOAD_INBOUND 2 +/* Two bits above are relevant for state path only, while + * offload is used for both policy and state flows. + * + * In policy offload mode, they are free and can be safely reused. + */ +#define XFRM_OFFLOAD_PACKET 4 struct xfrm_userpolicy_default { #define XFRM_USERPOLICY_UNSPEC 0 diff --git a/ipc/sem.c b/ipc/sem.c index c8496f98b139..00f88aa01ac5 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -2179,14 +2179,15 @@ long __do_semtimedop(int semid, struct sembuf *sops, * scenarios where we were awakened externally, during the * window between wake_q_add() and wake_up_q(). */ + rcu_read_lock(); error = READ_ONCE(queue.status); if (error != -EINTR) { /* see SEM_BARRIER_2 for purpose/pairing */ smp_acquire__after_ctrl_dep(); + rcu_read_unlock(); goto out; } - rcu_read_lock(); locknum = sem_lock(sma, sops, nsops); if (!ipc_valid_object(&sma->sem_perm)) diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index fd4020835ec6..367b0a42ada9 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -167,7 +167,6 @@ struct cgroup_mgctx { extern spinlock_t css_set_lock; extern struct cgroup_subsys *cgroup_subsys[]; extern struct list_head cgroup_roots; -extern struct file_system_type cgroup_fs_type; /* iterate across the hierarchies */ #define for_each_root(root) \ diff --git a/kernel/events/core.c b/kernel/events/core.c index 7091bbf88ee7..7f04f995c975 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2291,6 +2291,7 @@ event_sched_out(struct perf_event *event, !event->pending_work) { event->pending_work = 1; dec = false; + WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount)); task_work_add(current, &event->pending_task, TWA_RESUME); } if (dec) @@ -2336,6 +2337,7 @@ group_sched_out(struct perf_event *group_event, #define DETACH_GROUP 0x01UL #define DETACH_CHILD 0x02UL +#define DETACH_DEAD 0x04UL /* * Cross CPU call to remove a performance event @@ -2356,12 +2358,20 @@ __perf_remove_from_context(struct perf_event *event, update_cgrp_time_from_cpuctx(cpuctx, false); } + /* + * Ensure event_sched_out() switches to OFF, at the very least + * this avoids raising perf_pending_task() at this time. + */ + if (flags & DETACH_DEAD) + event->pending_disable = 1; event_sched_out(event, cpuctx, ctx); if (flags & DETACH_GROUP) perf_group_detach(event); if (flags & DETACH_CHILD) perf_child_detach(event); list_del_event(event, ctx); + if (flags & DETACH_DEAD) + event->state = PERF_EVENT_STATE_DEAD; if (!ctx->nr_events && ctx->is_active) { if (ctx == &cpuctx->ctx) @@ -5121,9 +5131,7 @@ int perf_event_release_kernel(struct perf_event *event) ctx = perf_event_ctx_lock(event); WARN_ON_ONCE(ctx->parent_ctx); - perf_remove_from_context(event, DETACH_GROUP); - raw_spin_lock_irq(&ctx->lock); /* * Mark this event as STATE_DEAD, there is no external reference to it * anymore. @@ -5135,8 +5143,7 @@ int perf_event_release_kernel(struct perf_event *event) * Thus this guarantees that we will in fact observe and kill _ALL_ * child events. */ - event->state = PERF_EVENT_STATE_DEAD; - raw_spin_unlock_irq(&ctx->lock); + perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD); perf_event_ctx_unlock(event, ctx); @@ -6577,6 +6584,8 @@ static void perf_pending_task(struct callback_head *head) if (rctx >= 0) perf_swevent_put_recursion_context(rctx); preempt_enable_notrace(); + + put_event(event); } #ifdef CONFIG_GUEST_PERF_EVENTS diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 714ac4c3b556..d9c822bbffb8 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -113,11 +113,40 @@ int static_key_count(struct static_key *key) } EXPORT_SYMBOL_GPL(static_key_count); -void static_key_slow_inc_cpuslocked(struct static_key *key) +/* + * static_key_fast_inc_not_disabled - adds a user for a static key + * @key: static key that must be already enabled + * + * The caller must make sure that the static key can't get disabled while + * in this function. It doesn't patch jump labels, only adds a user to + * an already enabled static key. + * + * Returns true if the increment was done. Unlike refcount_t the ref counter + * is not saturated, but will fail to increment on overflow. + */ +bool static_key_fast_inc_not_disabled(struct static_key *key) { - int v, v1; + int v; STATIC_KEY_CHECK_USE(key); + /* + * Negative key->enabled has a special meaning: it sends + * static_key_slow_inc() down the slow path, and it is non-zero + * so it counts as "enabled" in jump_label_update(). Note that + * atomic_inc_unless_negative() checks >= 0, so roll our own. + */ + v = atomic_read(&key->enabled); + do { + if (v <= 0 || (v + 1) < 0) + return false; + } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1))); + + return true; +} +EXPORT_SYMBOL_GPL(static_key_fast_inc_not_disabled); + +bool static_key_slow_inc_cpuslocked(struct static_key *key) +{ lockdep_assert_cpus_held(); /* @@ -126,17 +155,9 @@ void static_key_slow_inc_cpuslocked(struct static_key *key) * jump_label_update() process. At the same time, however, * the jump_label_update() call below wants to see * static_key_enabled(&key) for jumps to be updated properly. - * - * So give a special meaning to negative key->enabled: it sends - * static_key_slow_inc() down the slow path, and it is non-zero - * so it counts as "enabled" in jump_label_update(). Note that - * atomic_inc_unless_negative() checks >= 0, so roll our own. */ - for (v = atomic_read(&key->enabled); v > 0; v = v1) { - v1 = atomic_cmpxchg(&key->enabled, v, v + 1); - if (likely(v1 == v)) - return; - } + if (static_key_fast_inc_not_disabled(key)) + return true; jump_label_lock(); if (atomic_read(&key->enabled) == 0) { @@ -148,16 +169,23 @@ void static_key_slow_inc_cpuslocked(struct static_key *key) */ atomic_set_release(&key->enabled, 1); } else { - atomic_inc(&key->enabled); + if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key))) { + jump_label_unlock(); + return false; + } } jump_label_unlock(); + return true; } -void static_key_slow_inc(struct static_key *key) +bool static_key_slow_inc(struct static_key *key) { + bool ret; + cpus_read_lock(); - static_key_slow_inc_cpuslocked(key); + ret = static_key_slow_inc_cpuslocked(key); cpus_read_unlock(); + return ret; } EXPORT_SYMBOL_GPL(static_key_slow_inc); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 188c305aeb8b..c6d9dec11b74 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -267,13 +267,14 @@ int proc_dostring(struct ctl_table *table, int write, ppos); } -static size_t proc_skip_spaces(char **buf) +static void proc_skip_spaces(char **buf, size_t *size) { - size_t ret; - char *tmp = skip_spaces(*buf); - ret = tmp - *buf; - *buf = tmp; - return ret; + while (*size) { + if (!isspace(**buf)) + break; + (*size)--; + (*buf)++; + } } static void proc_skip_char(char **buf, size_t *size, const char v) @@ -342,13 +343,12 @@ static int proc_get_long(char **buf, size_t *size, unsigned long *val, bool *neg, const char *perm_tr, unsigned perm_tr_len, char *tr) { - int len; char *p, tmp[TMPBUFLEN]; + ssize_t len = *size; - if (!*size) + if (len <= 0) return -EINVAL; - len = *size; if (len > TMPBUFLEN - 1) len = TMPBUFLEN - 1; @@ -521,7 +521,7 @@ static int __do_proc_dointvec(void *tbl_data, struct ctl_table *table, bool neg; if (write) { - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); if (!left) break; @@ -548,7 +548,7 @@ static int __do_proc_dointvec(void *tbl_data, struct ctl_table *table, if (!write && !first && left && !err) proc_put_char(&buffer, &left, '\n'); if (write && !err && left) - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); if (write && first) return err ? : -EINVAL; *lenp -= left; @@ -590,7 +590,7 @@ static int do_proc_douintvec_w(unsigned int *tbl_data, if (left > PAGE_SIZE - 1) left = PAGE_SIZE - 1; - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); if (!left) { err = -EINVAL; goto out_free; @@ -610,7 +610,7 @@ static int do_proc_douintvec_w(unsigned int *tbl_data, } if (!err && left) - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); out_free: if (err) @@ -1075,7 +1075,7 @@ static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, if (write) { bool neg; - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); if (!left) break; @@ -1104,7 +1104,7 @@ static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, if (!write && !first && left && !err) proc_put_char(&buffer, &left, '\n'); if (write && !err) - left -= proc_skip_spaces(&p); + proc_skip_spaces(&p, &left); if (write && first) return err ? : -EINVAL; *lenp -= left; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a1005415f0f4..3638b3424be5 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -399,6 +399,7 @@ config FRAME_WARN default 2048 if GCC_PLUGIN_LATENT_ENTROPY default 2048 if PARISC default 1536 if (!64BIT && XTENSA) + default 1280 if KASAN && !64BIT default 1024 if !64BIT default 2048 if 64BIT help @@ -1874,8 +1875,14 @@ config NETDEV_NOTIFIER_ERROR_INJECT If unsure, say N. config FUNCTION_ERROR_INJECTION - def_bool y + bool "Fault-injections of functions" depends on HAVE_FUNCTION_ERROR_INJECTION && KPROBES + help + Add fault injections into various functions that are annotated with + ALLOW_ERROR_INJECTION() in the kernel. BPF may also modify the return + value of theses functions. This is useful to test error paths of code. + + If unsure, say N config FAULT_INJECTION bool "Fault-injection framework" diff --git a/lib/rhashtable.c b/lib/rhashtable.c index e12bbfb240b8..6ae2ba8e06a2 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -231,6 +231,7 @@ static int rhashtable_rehash_one(struct rhashtable *ht, struct rhash_head *head, *next, *entry; struct rhash_head __rcu **pprev = NULL; unsigned int new_hash; + unsigned long flags; if (new_tbl->nest) goto out; @@ -253,13 +254,14 @@ static int rhashtable_rehash_one(struct rhashtable *ht, new_hash = head_hashfn(ht, new_tbl, entry); - rht_lock_nested(new_tbl, &new_tbl->buckets[new_hash], SINGLE_DEPTH_NESTING); + flags = rht_lock_nested(new_tbl, &new_tbl->buckets[new_hash], + SINGLE_DEPTH_NESTING); head = rht_ptr(new_tbl->buckets + new_hash, new_tbl, new_hash); RCU_INIT_POINTER(entry->next, head); - rht_assign_unlock(new_tbl, &new_tbl->buckets[new_hash], entry); + rht_assign_unlock(new_tbl, &new_tbl->buckets[new_hash], entry, flags); if (pprev) rcu_assign_pointer(*pprev, next); @@ -276,18 +278,19 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash); + unsigned long flags; int err; if (!bkt) return 0; - rht_lock(old_tbl, bkt); + flags = rht_lock(old_tbl, bkt); while (!(err = rhashtable_rehash_one(ht, bkt, old_hash))) ; if (err == -ENOENT) err = 0; - rht_unlock(old_tbl, bkt); + rht_unlock(old_tbl, bkt, flags); return err; } @@ -590,6 +593,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, struct bucket_table *new_tbl; struct bucket_table *tbl; struct rhash_lock_head __rcu **bkt; + unsigned long flags; unsigned int hash; void *data; @@ -607,7 +611,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); data = ERR_PTR(-EAGAIN); } else { - rht_lock(tbl, bkt); + flags = rht_lock(tbl, bkt); data = rhashtable_lookup_one(ht, bkt, tbl, hash, key, obj); new_tbl = rhashtable_insert_one(ht, bkt, tbl, @@ -615,7 +619,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, if (PTR_ERR(new_tbl) != -EEXIST) data = ERR_CAST(new_tbl); - rht_unlock(tbl, bkt); + rht_unlock(tbl, bkt, flags); } } while (!IS_ERR_OR_NULL(new_tbl)); diff --git a/mm/compaction.c b/mm/compaction.c index c51f7f545afe..1f6da31dd9a5 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -985,28 +985,28 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, } /* + * Be careful not to clear PageLRU until after we're + * sure the page is not being freed elsewhere -- the + * page release code relies on it. + */ + if (unlikely(!get_page_unless_zero(page))) + goto isolate_fail; + + /* * Migration will fail if an anonymous page is pinned in memory, * so avoid taking lru_lock and isolating it unnecessarily in an * admittedly racy check. */ mapping = page_mapping(page); - if (!mapping && page_count(page) > page_mapcount(page)) - goto isolate_fail; + if (!mapping && (page_count(page) - 1) > total_mapcount(page)) + goto isolate_fail_put; /* * Only allow to migrate anonymous pages in GFP_NOFS context * because those do not depend on fs locks. */ if (!(cc->gfp_mask & __GFP_FS) && mapping) - goto isolate_fail; - - /* - * Be careful not to clear PageLRU until after we're - * sure the page is not being freed elsewhere -- the - * page release code relies on it. - */ - if (unlikely(!get_page_unless_zero(page))) - goto isolate_fail; + goto isolate_fail_put; /* Only take pages on LRU: a check now makes later tests safe */ if (!PageLRU(page)) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 5ce403378c20..07e5f1bdf025 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2283,12 +2283,54 @@ static struct damos *damon_sysfs_mk_scheme( &wmarks); } +static void damon_sysfs_update_scheme(struct damos *scheme, + struct damon_sysfs_scheme *sysfs_scheme) +{ + struct damon_sysfs_access_pattern *access_pattern = + sysfs_scheme->access_pattern; + struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas; + struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights; + struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks; + + scheme->pattern.min_sz_region = access_pattern->sz->min; + scheme->pattern.max_sz_region = access_pattern->sz->max; + scheme->pattern.min_nr_accesses = access_pattern->nr_accesses->min; + scheme->pattern.max_nr_accesses = access_pattern->nr_accesses->max; + scheme->pattern.min_age_region = access_pattern->age->min; + scheme->pattern.max_age_region = access_pattern->age->max; + + scheme->action = sysfs_scheme->action; + + scheme->quota.ms = sysfs_quotas->ms; + scheme->quota.sz = sysfs_quotas->sz; + scheme->quota.reset_interval = sysfs_quotas->reset_interval_ms; + scheme->quota.weight_sz = sysfs_weights->sz; + scheme->quota.weight_nr_accesses = sysfs_weights->nr_accesses; + scheme->quota.weight_age = sysfs_weights->age; + + scheme->wmarks.metric = sysfs_wmarks->metric; + scheme->wmarks.interval = sysfs_wmarks->interval_us; + scheme->wmarks.high = sysfs_wmarks->high; + scheme->wmarks.mid = sysfs_wmarks->mid; + scheme->wmarks.low = sysfs_wmarks->low; +} + static int damon_sysfs_set_schemes(struct damon_ctx *ctx, struct damon_sysfs_schemes *sysfs_schemes) { - int i; + struct damos *scheme, *next; + int i = 0; + + damon_for_each_scheme_safe(scheme, next, ctx) { + if (i < sysfs_schemes->nr) + damon_sysfs_update_scheme(scheme, + sysfs_schemes->schemes_arr[i]); + else + damon_destroy_scheme(scheme); + i++; + } - for (i = 0; i < sysfs_schemes->nr; i++) { + for (; i < sysfs_schemes->nr; i++) { struct damos *scheme, *next; scheme = damon_sysfs_mk_scheme(sysfs_schemes->schemes_arr[i]); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f1385c3b6c96..e36ca75311a5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5206,17 +5206,22 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, __unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags); - /* - * Unlock and free the vma lock before releasing i_mmap_rwsem. When - * the vma_lock is freed, this makes the vma ineligible for pmd - * sharing. And, i_mmap_rwsem is required to set up pmd sharing. - * This is important as page tables for this unmapped range will - * be asynchrously deleted. If the page tables are shared, there - * will be issues when accessed by someone else. - */ - __hugetlb_vma_unlock_write_free(vma); - - i_mmap_unlock_write(vma->vm_file->f_mapping); + if (zap_flags & ZAP_FLAG_UNMAP) { /* final unmap */ + /* + * Unlock and free the vma lock before releasing i_mmap_rwsem. + * When the vma_lock is freed, this makes the vma ineligible + * for pmd sharing. And, i_mmap_rwsem is required to set up + * pmd sharing. This is important as page tables for this + * unmapped range will be asynchrously deleted. If the page + * tables are shared, there will be issues when accessed by + * someone else. + */ + __hugetlb_vma_unlock_write_free(vma); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } else { + i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); + } } void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index a8d5ef2a77d2..3703a56571c1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1051,6 +1051,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, _pmd = pmdp_collapse_flush(vma, address, pmd); spin_unlock(pmd_ptl); mmu_notifier_invalidate_range_end(&range); + tlb_remove_table_sync_one(); spin_lock(pte_ptl); result = __collapse_huge_page_isolate(vma, address, pte, cc, @@ -1379,16 +1380,43 @@ static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr, return SCAN_SUCCEED; } +/* + * A note about locking: + * Trying to take the page table spinlocks would be useless here because those + * are only used to synchronize: + * + * - modifying terminal entries (ones that point to a data page, not to another + * page table) + * - installing *new* non-terminal entries + * + * Instead, we need roughly the same kind of protection as free_pgtables() or + * mm_take_all_locks() (but only for a single VMA): + * The mmap lock together with this VMA's rmap locks covers all paths towards + * the page table entries we're messing with here, except for hardware page + * table walks and lockless_pages_from_mm(). + */ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - spinlock_t *ptl; pmd_t pmd; + struct mmu_notifier_range range; mmap_assert_write_locked(mm); - ptl = pmd_lock(vma->vm_mm, pmdp); + if (vma->vm_file) + lockdep_assert_held_write(&vma->vm_file->f_mapping->i_mmap_rwsem); + /* + * All anon_vmas attached to the VMA have the same root and are + * therefore locked by the same lock. + */ + if (vma->anon_vma) + lockdep_assert_held_write(&vma->anon_vma->root->rwsem); + + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, addr, + addr + HPAGE_PMD_SIZE); + mmu_notifier_invalidate_range_start(&range); pmd = pmdp_collapse_flush(vma, addr, pmdp); - spin_unlock(ptl); + tlb_remove_table_sync_one(); + mmu_notifier_invalidate_range_end(&range); mm_dec_nr_ptes(mm); page_table_check_pte_clear_range(mm, addr, pmd); pte_free(mm, pmd_pgtable(pmd)); @@ -1439,6 +1467,14 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) return SCAN_VMA_CHECK; + /* + * Symmetry with retract_page_tables(): Exclude MAP_PRIVATE mappings + * that got written to. Without this, we'd have to also lock the + * anon_vma if one exists. + */ + if (vma->anon_vma) + return SCAN_VMA_CHECK; + /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ if (userfaultfd_wp(vma)) return SCAN_PTE_UFFD_WP; @@ -1472,6 +1508,20 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, goto drop_hpage; } + /* + * We need to lock the mapping so that from here on, only GUP-fast and + * hardware page walks can access the parts of the page tables that + * we're operating on. + * See collapse_and_free_pmd(). + */ + i_mmap_lock_write(vma->vm_file->f_mapping); + + /* + * This spinlock should be unnecessary: Nobody else should be accessing + * the page tables under spinlock protection here, only + * lockless_pages_from_mm() and the hardware page walker can access page + * tables while all the high-level locks are held in write mode. + */ start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); result = SCAN_FAIL; @@ -1526,6 +1576,8 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, /* step 4: remove pte entries */ collapse_and_free_pmd(mm, vma, haddr, pmd); + i_mmap_unlock_write(vma->vm_file->f_mapping); + maybe_install_pmd: /* step 5: install pmd entry */ result = install_pmd @@ -1539,6 +1591,7 @@ drop_hpage: abort: pte_unmap_unlock(start_pte, ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); goto drop_hpage; } @@ -1595,7 +1648,8 @@ static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff, * An alternative would be drop the check, but check that page * table is clear before calling pmdp_collapse_flush() under * ptl. It has higher chance to recover THP for the VMA, but - * has higher cost too. + * has higher cost too. It would also probably require locking + * the anon_vma. */ if (vma->anon_vma) { result = SCAN_PAGE_ANON; diff --git a/mm/madvise.c b/mm/madvise.c index c7105ec6d08c..b913ba6efc10 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -772,8 +772,8 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, * Application no longer needs these pages. If the pages are dirty, * it's OK to just throw them away. The app will be more careful about * data it wants to keep. Be sure to free swap resources too. The - * zap_page_range call sets things up for shrink_active_list to actually free - * these pages later if no one else has touched them in the meantime, + * zap_page_range_single call sets things up for shrink_active_list to actually + * free these pages later if no one else has touched them in the meantime, * although we could add these pages to a global reuse list for * shrink_active_list to pick up before reclaiming other pages. * @@ -790,7 +790,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, static long madvise_dontneed_single_vma(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - zap_page_range(vma, start, end - start); + zap_page_range_single(vma, start, end - start, NULL); return 0; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a1a35c12635e..266a1ab05434 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4832,6 +4832,7 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, unsigned int efd, cfd; struct fd efile; struct fd cfile; + struct dentry *cdentry; const char *name; char *endp; int ret; @@ -4886,6 +4887,16 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, goto out_put_cfile; /* + * The control file must be a regular cgroup1 file. As a regular cgroup + * file can't be renamed, it's safe to access its name afterwards. + */ + cdentry = cfile.file->f_path.dentry; + if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) { + ret = -EINVAL; + goto out_put_cfile; + } + + /* * Determine the event callbacks and set them in @event. This used * to be done via struct cftype but cgroup core no longer knows * about these events. The following is crude but the whole thing @@ -4893,7 +4904,7 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, * * DO NOT ADD NEW FILES. */ - name = cfile.file->f_path.dentry->d_name.name; + name = cdentry->d_name.name; if (!strcmp(name, "memory.usage_in_bytes")) { event->register_event = mem_cgroup_usage_register_event; @@ -4917,7 +4928,7 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, * automatically removed on cgroup destruction but the removal is * asynchronous, so take an extra ref on @css. */ - cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, + cfile_css = css_tryget_online_from_dir(cdentry->d_parent, &memory_cgrp_subsys); ret = -EINVAL; if (IS_ERR(cfile_css)) diff --git a/mm/memory.c b/mm/memory.c index 8a6d5c823f91..8c8420934d60 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1341,15 +1341,6 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) return ret; } -/* - * Parameter block passed down to zap_pte_range in exceptional cases. - */ -struct zap_details { - struct folio *single_folio; /* Locked folio to be unmapped */ - bool even_cows; /* Zap COWed private pages too? */ - zap_flags_t zap_flags; /* Extra flags for zapping */ -}; - /* Whether we should zap all COWed (private) pages too */ static inline bool should_zap_cows(struct zap_details *details) { @@ -1720,7 +1711,7 @@ void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, { struct mmu_notifier_range range; struct zap_details details = { - .zap_flags = ZAP_FLAG_DROP_MARKER, + .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP, /* Careful - we need to zap private pages too! */ .even_cows = true, }; @@ -1774,19 +1765,27 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, * * The range must fit into one VMA. */ -static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, +void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, unsigned long size, struct zap_details *details) { + const unsigned long end = address + size; struct mmu_notifier_range range; struct mmu_gather tlb; lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, - address, address + size); + address, end); + if (is_vm_hugetlb_page(vma)) + adjust_range_if_pmd_sharing_possible(vma, &range.start, + &range.end); tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); - unmap_single_vma(&tlb, vma, address, range.end, details); + /* + * unmap 'address-end' not 'range.start-range.end' as range + * could have been expanded for hugetlb pmd sharing. + */ + unmap_single_vma(&tlb, vma, address, end, details); mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb); } diff --git a/mm/mmap.c b/mm/mmap.c index 74a84eb33b90..a5eb2f175da0 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1779,9 +1779,6 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, */ pgoff = 0; get_area = shmem_get_unmapped_area; - } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { - /* Ensures that larger anonymous mappings are THP aligned. */ - get_area = thp_get_unmapped_area; } addr = get_area(file, addr, len, pgoff, flags); diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index add4244e5790..3a2c3f8cad2f 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -153,7 +153,7 @@ static void tlb_remove_table_smp_sync(void *arg) /* Simply deliver the interrupt */ } -static void tlb_remove_table_sync_one(void) +void tlb_remove_table_sync_one(void) { /* * This isn't an RCU grace period and hence the page-tables cannot be @@ -177,8 +177,6 @@ static void tlb_remove_table_free(struct mmu_table_batch *batch) #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */ -static void tlb_remove_table_sync_one(void) { } - static void tlb_remove_table_free(struct mmu_table_batch *batch) { __tlb_remove_table_free(batch); diff --git a/mm/vmscan.c b/mm/vmscan.c index 026199c047e0..8fcc5fa768c0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3987,7 +3987,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area goto next; if (!pmd_trans_huge(pmd[i])) { - if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) pmdp_test_and_clear_young(vma, addr, pmd + i); goto next; @@ -4085,14 +4085,14 @@ restart: #endif walk->mm_stats[MM_NONLEAF_TOTAL]++; -#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG - if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { + if (arch_has_hw_nonleaf_pmd_young() && + get_cap(LRU_GEN_NONLEAF_YOUNG)) { if (!pmd_young(val)) continue; walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); } -#endif + if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) continue; @@ -5392,7 +5392,7 @@ static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, c if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) caps |= BIT(LRU_GEN_MM_WALK); - if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) caps |= BIT(LRU_GEN_NONLEAF_YOUNG); return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index 215af9b3b589..c57d643afb10 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -972,6 +972,7 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, hci_dev_lock(hdev); hcon = hci_conn_hash_lookup_le(hdev, addr, *addr_type); hci_dev_unlock(hdev); + hci_dev_put(hdev); if (!hcon) return -ENOENT; diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index dc65974f5adb..1c3c7ff5c3c6 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -737,7 +737,7 @@ static int __init bt_init(void) err = bt_sysfs_init(); if (err < 0) - return err; + goto cleanup_led; err = sock_register(&bt_sock_family_ops); if (err) @@ -773,6 +773,8 @@ unregister_socket: sock_unregister(PF_BLUETOOTH); cleanup_sysfs: bt_sysfs_cleanup(); +cleanup_led: + bt_leds_cleanup(); return err; } diff --git a/net/bluetooth/hci_codec.c b/net/bluetooth/hci_codec.c index 38201532f58e..3cc135bb1d30 100644 --- a/net/bluetooth/hci_codec.c +++ b/net/bluetooth/hci_codec.c @@ -72,9 +72,8 @@ static void hci_read_codec_capabilities(struct hci_dev *hdev, __u8 transport, continue; } - skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_CODEC_CAPS, - sizeof(*cmd), cmd, - HCI_CMD_TIMEOUT); + skb = __hci_cmd_sync_sk(hdev, HCI_OP_READ_LOCAL_CODEC_CAPS, + sizeof(*cmd), cmd, 0, HCI_CMD_TIMEOUT, NULL); if (IS_ERR(skb)) { bt_dev_err(hdev, "Failed to read codec capabilities (%ld)", PTR_ERR(skb)); @@ -127,8 +126,8 @@ void hci_read_supported_codecs(struct hci_dev *hdev) struct hci_op_read_local_codec_caps caps; __u8 i; - skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_CODECS, 0, NULL, - HCI_CMD_TIMEOUT); + skb = __hci_cmd_sync_sk(hdev, HCI_OP_READ_LOCAL_CODECS, 0, NULL, + 0, HCI_CMD_TIMEOUT, NULL); if (IS_ERR(skb)) { bt_dev_err(hdev, "Failed to read local supported codecs (%ld)", @@ -158,7 +157,8 @@ void hci_read_supported_codecs(struct hci_dev *hdev) for (i = 0; i < std_codecs->num; i++) { caps.id = std_codecs->codec[i]; caps.direction = 0x00; - hci_read_codec_capabilities(hdev, LOCAL_CODEC_ACL_MASK, &caps); + hci_read_codec_capabilities(hdev, + LOCAL_CODEC_ACL_MASK | LOCAL_CODEC_SCO_MASK, &caps); } skb_pull(skb, flex_array_size(std_codecs, codec, std_codecs->num) @@ -178,7 +178,8 @@ void hci_read_supported_codecs(struct hci_dev *hdev) caps.cid = vnd_codecs->codec[i].cid; caps.vid = vnd_codecs->codec[i].vid; caps.direction = 0x00; - hci_read_codec_capabilities(hdev, LOCAL_CODEC_ACL_MASK, &caps); + hci_read_codec_capabilities(hdev, + LOCAL_CODEC_ACL_MASK | LOCAL_CODEC_SCO_MASK, &caps); } error: @@ -194,8 +195,8 @@ void hci_read_supported_codecs_v2(struct hci_dev *hdev) struct hci_op_read_local_codec_caps caps; __u8 i; - skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_CODECS_V2, 0, NULL, - HCI_CMD_TIMEOUT); + skb = __hci_cmd_sync_sk(hdev, HCI_OP_READ_LOCAL_CODECS_V2, 0, NULL, + 0, HCI_CMD_TIMEOUT, NULL); if (IS_ERR(skb)) { bt_dev_err(hdev, "Failed to read local supported codecs (%ld)", diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 0540555b3704..d97fac4f7130 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -2764,7 +2764,8 @@ int hci_register_suspend_notifier(struct hci_dev *hdev) { int ret = 0; - if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { + if (!hdev->suspend_notifier.notifier_call && + !test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { hdev->suspend_notifier.notifier_call = hci_suspend_notifier; ret = register_pm_notifier(&hdev->suspend_notifier); } @@ -2776,8 +2777,11 @@ int hci_unregister_suspend_notifier(struct hci_dev *hdev) { int ret = 0; - if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) + if (hdev->suspend_notifier.notifier_call) { ret = unregister_pm_notifier(&hdev->suspend_notifier); + if (!ret) + hdev->suspend_notifier.notifier_call = NULL; + } return ret; } diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index 5a0296a4352e..f7e006a36382 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -269,7 +269,7 @@ void hci_req_add_ev(struct hci_request *req, u16 opcode, u32 plen, void hci_req_add(struct hci_request *req, u16 opcode, u32 plen, const void *param) { - bt_dev_err(req->hdev, "HCI_REQ-0x%4.4x", opcode); + bt_dev_dbg(req->hdev, "HCI_REQ-0x%4.4x", opcode); hci_req_add_ev(req, opcode, plen, param, 0); } diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 76c3107c9f91..1fc693122a47 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -12,6 +12,7 @@ #include <net/bluetooth/mgmt.h> #include "hci_request.h" +#include "hci_codec.h" #include "hci_debugfs.h" #include "smp.h" #include "eir.h" @@ -3780,7 +3781,8 @@ static int hci_read_page_scan_activity_sync(struct hci_dev *hdev) static int hci_read_def_err_data_reporting_sync(struct hci_dev *hdev) { if (!(hdev->commands[18] & 0x04) || - !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING)) + !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING) || + test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) return 0; return __hci_cmd_sync_status(hdev, HCI_OP_READ_DEF_ERR_DATA_REPORTING, @@ -4238,11 +4240,12 @@ static int hci_set_event_mask_page_2_sync(struct hci_dev *hdev) /* Read local codec list if the HCI command is supported */ static int hci_read_local_codecs_sync(struct hci_dev *hdev) { - if (!(hdev->commands[29] & 0x20)) - return 0; + if (hdev->commands[45] & 0x04) + hci_read_supported_codecs_v2(hdev); + else if (hdev->commands[29] & 0x20) + hci_read_supported_codecs(hdev); - return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_CODECS, 0, NULL, - HCI_CMD_TIMEOUT); + return 0; } /* Read local pairing options if the HCI command is supported */ @@ -4298,7 +4301,8 @@ static int hci_set_err_data_report_sync(struct hci_dev *hdev) bool enabled = hci_dev_test_flag(hdev, HCI_WIDEBAND_SPEECH_ENABLED); if (!(hdev->commands[18] & 0x08) || - !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING)) + !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING) || + test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) return 0; if (enabled == hdev->err_data_reporting) @@ -4457,6 +4461,9 @@ static const struct { HCI_QUIRK_BROKEN(STORED_LINK_KEY, "HCI Delete Stored Link Key command is advertised, " "but not supported."), + HCI_QUIRK_BROKEN(ERR_DATA_REPORTING, + "HCI Read Default Erroneous Data Reporting command is " + "advertised, but not supported."), HCI_QUIRK_BROKEN(READ_TRANSMIT_POWER, "HCI Read Transmit Power Level command is advertised, " "but not supported."), diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index f825857db6d0..26db929b97c4 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -879,6 +879,7 @@ static int iso_listen_bis(struct sock *sk) iso_pi(sk)->bc_sid); hci_dev_unlock(hdev); + hci_dev_put(hdev); return err; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 9c24947aa41e..9fdede5fe71c 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -4453,7 +4453,8 @@ static inline int l2cap_config_req(struct l2cap_conn *conn, chan->ident = cmd->ident; l2cap_send_cmd(conn, cmd->ident, L2CAP_CONF_RSP, len, rsp); - chan->num_conf_rsp++; + if (chan->num_conf_rsp < L2CAP_CONF_MAX_CONF_RSP) + chan->num_conf_rsp++; /* Reset config buffer. */ chan->conf_len = 0; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 444736e707a4..2723623429ac 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1135,7 +1135,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, } sock_init_data(NULL, sk); - skb = build_skb(data, 0); + skb = slab_build_skb(data); if (!skb) { kfree(data); kfree(ctx); diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c index 321be94c445a..ae7d93c08880 100644 --- a/net/bridge/br_mdb.c +++ b/net/bridge/br_mdb.c @@ -754,73 +754,6 @@ static const struct nla_policy br_mdbe_attrs_pol[MDBE_ATTR_MAX + 1] = { sizeof(struct in6_addr)), }; -static int br_mdb_parse(struct sk_buff *skb, struct nlmsghdr *nlh, - struct net_device **pdev, struct br_mdb_entry **pentry, - struct nlattr **mdb_attrs, struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(skb->sk); - struct br_mdb_entry *entry; - struct br_port_msg *bpm; - struct nlattr *tb[MDBA_SET_ENTRY_MAX+1]; - struct net_device *dev; - int err; - - err = nlmsg_parse_deprecated(nlh, sizeof(*bpm), tb, - MDBA_SET_ENTRY_MAX, NULL, NULL); - if (err < 0) - return err; - - bpm = nlmsg_data(nlh); - if (bpm->ifindex == 0) { - NL_SET_ERR_MSG_MOD(extack, "Invalid bridge ifindex"); - return -EINVAL; - } - - dev = __dev_get_by_index(net, bpm->ifindex); - if (dev == NULL) { - NL_SET_ERR_MSG_MOD(extack, "Bridge device doesn't exist"); - return -ENODEV; - } - - if (!netif_is_bridge_master(dev)) { - NL_SET_ERR_MSG_MOD(extack, "Device is not a bridge"); - return -EOPNOTSUPP; - } - - *pdev = dev; - - if (!tb[MDBA_SET_ENTRY]) { - NL_SET_ERR_MSG_MOD(extack, "Missing MDBA_SET_ENTRY attribute"); - return -EINVAL; - } - if (nla_len(tb[MDBA_SET_ENTRY]) != sizeof(struct br_mdb_entry)) { - NL_SET_ERR_MSG_MOD(extack, "Invalid MDBA_SET_ENTRY attribute length"); - return -EINVAL; - } - - entry = nla_data(tb[MDBA_SET_ENTRY]); - if (!is_valid_mdb_entry(entry, extack)) - return -EINVAL; - *pentry = entry; - - if (tb[MDBA_SET_ENTRY_ATTRS]) { - err = nla_parse_nested(mdb_attrs, MDBE_ATTR_MAX, - tb[MDBA_SET_ENTRY_ATTRS], - br_mdbe_attrs_pol, extack); - if (err) - return err; - if (mdb_attrs[MDBE_ATTR_SOURCE] && - !is_valid_mdb_source(mdb_attrs[MDBE_ATTR_SOURCE], - entry->addr.proto, extack)) - return -EINVAL; - } else { - memset(mdb_attrs, 0, - sizeof(struct nlattr *) * (MDBE_ATTR_MAX + 1)); - } - - return 0; -} - static struct net_bridge_mcast * __br_mdb_choose_context(struct net_bridge *br, const struct br_mdb_entry *entry, @@ -853,44 +786,26 @@ out: return brmctx; } -static int br_mdb_add_group(struct net_bridge *br, struct net_bridge_port *port, - struct br_mdb_entry *entry, - struct nlattr **mdb_attrs, +static int br_mdb_add_group(const struct br_mdb_config *cfg, struct netlink_ext_ack *extack) { struct net_bridge_mdb_entry *mp, *star_mp; struct net_bridge_port_group __rcu **pp; + struct br_mdb_entry *entry = cfg->entry; + struct net_bridge_port *port = cfg->p; + struct net_bridge *br = cfg->br; struct net_bridge_port_group *p; struct net_bridge_mcast *brmctx; - struct br_ip group, star_group; + struct br_ip group = cfg->group; unsigned long now = jiffies; unsigned char flags = 0; + struct br_ip star_group; u8 filter_mode; - __mdb_entry_to_br_ip(entry, &group, mdb_attrs); - brmctx = __br_mdb_choose_context(br, entry, extack); if (!brmctx) return -EINVAL; - /* host join errors which can happen before creating the group */ - if (!port && !br_group_is_l2(&group)) { - /* don't allow any flags for host-joined IP groups */ - if (entry->state) { - NL_SET_ERR_MSG_MOD(extack, "Flags are not allowed for host groups"); - return -EINVAL; - } - if (!br_multicast_is_star_g(&group)) { - NL_SET_ERR_MSG_MOD(extack, "Groups with sources cannot be manually host joined"); - return -EINVAL; - } - } - - if (br_group_is_l2(&group) && entry->state != MDB_PERMANENT) { - NL_SET_ERR_MSG_MOD(extack, "Only permanent L2 entries allowed"); - return -EINVAL; - } - mp = br_multicast_new_group(br, &group); if (IS_ERR(mp)) return PTR_ERR(mp); @@ -959,107 +874,197 @@ static int br_mdb_add_group(struct net_bridge *br, struct net_bridge_port *port, return 0; } -static int __br_mdb_add(struct net *net, struct net_bridge *br, - struct net_bridge_port *p, - struct br_mdb_entry *entry, - struct nlattr **mdb_attrs, +static int __br_mdb_add(const struct br_mdb_config *cfg, struct netlink_ext_ack *extack) { int ret; - spin_lock_bh(&br->multicast_lock); - ret = br_mdb_add_group(br, p, entry, mdb_attrs, extack); - spin_unlock_bh(&br->multicast_lock); + spin_lock_bh(&cfg->br->multicast_lock); + ret = br_mdb_add_group(cfg, extack); + spin_unlock_bh(&cfg->br->multicast_lock); return ret; } -static int br_mdb_add(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) +static int br_mdb_config_attrs_init(struct nlattr *set_attrs, + struct br_mdb_config *cfg, + struct netlink_ext_ack *extack) { struct nlattr *mdb_attrs[MDBE_ATTR_MAX + 1]; - struct net *net = sock_net(skb->sk); - struct net_bridge_vlan_group *vg; - struct net_bridge_port *p = NULL; - struct net_device *dev, *pdev; - struct br_mdb_entry *entry; - struct net_bridge_vlan *v; - struct net_bridge *br; int err; - err = br_mdb_parse(skb, nlh, &dev, &entry, mdb_attrs, extack); - if (err < 0) + err = nla_parse_nested(mdb_attrs, MDBE_ATTR_MAX, set_attrs, + br_mdbe_attrs_pol, extack); + if (err) return err; - br = netdev_priv(dev); + if (mdb_attrs[MDBE_ATTR_SOURCE] && + !is_valid_mdb_source(mdb_attrs[MDBE_ATTR_SOURCE], + cfg->entry->addr.proto, extack)) + return -EINVAL; + + __mdb_entry_to_br_ip(cfg->entry, &cfg->group, mdb_attrs); - if (!netif_running(br->dev)) { + return 0; +} + +static int br_mdb_config_init(struct net *net, const struct nlmsghdr *nlh, + struct br_mdb_config *cfg, + struct netlink_ext_ack *extack) +{ + struct nlattr *tb[MDBA_SET_ENTRY_MAX + 1]; + struct br_port_msg *bpm; + struct net_device *dev; + int err; + + err = nlmsg_parse_deprecated(nlh, sizeof(*bpm), tb, + MDBA_SET_ENTRY_MAX, NULL, extack); + if (err) + return err; + + memset(cfg, 0, sizeof(*cfg)); + + bpm = nlmsg_data(nlh); + if (!bpm->ifindex) { + NL_SET_ERR_MSG_MOD(extack, "Invalid bridge ifindex"); + return -EINVAL; + } + + dev = __dev_get_by_index(net, bpm->ifindex); + if (!dev) { + NL_SET_ERR_MSG_MOD(extack, "Bridge device doesn't exist"); + return -ENODEV; + } + + if (!netif_is_bridge_master(dev)) { + NL_SET_ERR_MSG_MOD(extack, "Device is not a bridge"); + return -EOPNOTSUPP; + } + + cfg->br = netdev_priv(dev); + + if (!netif_running(cfg->br->dev)) { NL_SET_ERR_MSG_MOD(extack, "Bridge device is not running"); return -EINVAL; } - if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) { + if (!br_opt_get(cfg->br, BROPT_MULTICAST_ENABLED)) { NL_SET_ERR_MSG_MOD(extack, "Bridge's multicast processing is disabled"); return -EINVAL; } - if (entry->ifindex != br->dev->ifindex) { - pdev = __dev_get_by_index(net, entry->ifindex); + if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_SET_ENTRY)) { + NL_SET_ERR_MSG_MOD(extack, "Missing MDBA_SET_ENTRY attribute"); + return -EINVAL; + } + if (nla_len(tb[MDBA_SET_ENTRY]) != sizeof(struct br_mdb_entry)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid MDBA_SET_ENTRY attribute length"); + return -EINVAL; + } + + cfg->entry = nla_data(tb[MDBA_SET_ENTRY]); + if (!is_valid_mdb_entry(cfg->entry, extack)) + return -EINVAL; + + if (cfg->entry->ifindex != cfg->br->dev->ifindex) { + struct net_device *pdev; + + pdev = __dev_get_by_index(net, cfg->entry->ifindex); if (!pdev) { NL_SET_ERR_MSG_MOD(extack, "Port net device doesn't exist"); return -ENODEV; } - p = br_port_get_rtnl(pdev); - if (!p) { + cfg->p = br_port_get_rtnl(pdev); + if (!cfg->p) { NL_SET_ERR_MSG_MOD(extack, "Net device is not a bridge port"); return -EINVAL; } - if (p->br != br) { + if (cfg->p->br != cfg->br) { NL_SET_ERR_MSG_MOD(extack, "Port belongs to a different bridge device"); return -EINVAL; } - if (p->state == BR_STATE_DISABLED && entry->state != MDB_PERMANENT) { + } + + if (tb[MDBA_SET_ENTRY_ATTRS]) + return br_mdb_config_attrs_init(tb[MDBA_SET_ENTRY_ATTRS], cfg, + extack); + else + __mdb_entry_to_br_ip(cfg->entry, &cfg->group, NULL); + + return 0; +} + +static int br_mdb_add(struct sk_buff *skb, struct nlmsghdr *nlh, + struct netlink_ext_ack *extack) +{ + struct net *net = sock_net(skb->sk); + struct net_bridge_vlan_group *vg; + struct net_bridge_vlan *v; + struct br_mdb_config cfg; + int err; + + err = br_mdb_config_init(net, nlh, &cfg, extack); + if (err) + return err; + + /* host join errors which can happen before creating the group */ + if (!cfg.p && !br_group_is_l2(&cfg.group)) { + /* don't allow any flags for host-joined IP groups */ + if (cfg.entry->state) { + NL_SET_ERR_MSG_MOD(extack, "Flags are not allowed for host groups"); + return -EINVAL; + } + if (!br_multicast_is_star_g(&cfg.group)) { + NL_SET_ERR_MSG_MOD(extack, "Groups with sources cannot be manually host joined"); + return -EINVAL; + } + } + + if (br_group_is_l2(&cfg.group) && cfg.entry->state != MDB_PERMANENT) { + NL_SET_ERR_MSG_MOD(extack, "Only permanent L2 entries allowed"); + return -EINVAL; + } + + if (cfg.p) { + if (cfg.p->state == BR_STATE_DISABLED && cfg.entry->state != MDB_PERMANENT) { NL_SET_ERR_MSG_MOD(extack, "Port is in disabled state and entry is not permanent"); return -EINVAL; } - vg = nbp_vlan_group(p); + vg = nbp_vlan_group(cfg.p); } else { - vg = br_vlan_group(br); + vg = br_vlan_group(cfg.br); } /* If vlan filtering is enabled and VLAN is not specified * install mdb entry on all vlans configured on the port. */ - if (br_vlan_enabled(br->dev) && vg && entry->vid == 0) { + if (br_vlan_enabled(cfg.br->dev) && vg && cfg.entry->vid == 0) { list_for_each_entry(v, &vg->vlan_list, vlist) { - entry->vid = v->vid; - err = __br_mdb_add(net, br, p, entry, mdb_attrs, extack); + cfg.entry->vid = v->vid; + cfg.group.vid = v->vid; + err = __br_mdb_add(&cfg, extack); if (err) break; } } else { - err = __br_mdb_add(net, br, p, entry, mdb_attrs, extack); + err = __br_mdb_add(&cfg, extack); } return err; } -static int __br_mdb_del(struct net_bridge *br, struct br_mdb_entry *entry, - struct nlattr **mdb_attrs) +static int __br_mdb_del(const struct br_mdb_config *cfg) { + struct br_mdb_entry *entry = cfg->entry; + struct net_bridge *br = cfg->br; struct net_bridge_mdb_entry *mp; struct net_bridge_port_group *p; struct net_bridge_port_group __rcu **pp; - struct br_ip ip; + struct br_ip ip = cfg->group; int err = -EINVAL; - if (!netif_running(br->dev) || !br_opt_get(br, BROPT_MULTICAST_ENABLED)) - return -EINVAL; - - __mdb_entry_to_br_ip(entry, &ip, mdb_attrs); - spin_lock_bh(&br->multicast_lock); mp = br_mdb_ip_get(br, &ip); if (!mp) @@ -1094,51 +1099,32 @@ unlock: static int br_mdb_del(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { - struct nlattr *mdb_attrs[MDBE_ATTR_MAX + 1]; struct net *net = sock_net(skb->sk); struct net_bridge_vlan_group *vg; - struct net_bridge_port *p = NULL; - struct net_device *dev, *pdev; - struct br_mdb_entry *entry; struct net_bridge_vlan *v; - struct net_bridge *br; + struct br_mdb_config cfg; int err; - err = br_mdb_parse(skb, nlh, &dev, &entry, mdb_attrs, extack); - if (err < 0) + err = br_mdb_config_init(net, nlh, &cfg, extack); + if (err) return err; - br = netdev_priv(dev); - - if (entry->ifindex != br->dev->ifindex) { - pdev = __dev_get_by_index(net, entry->ifindex); - if (!pdev) - return -ENODEV; - - p = br_port_get_rtnl(pdev); - if (!p) { - NL_SET_ERR_MSG_MOD(extack, "Net device is not a bridge port"); - return -EINVAL; - } - if (p->br != br) { - NL_SET_ERR_MSG_MOD(extack, "Port belongs to a different bridge device"); - return -EINVAL; - } - vg = nbp_vlan_group(p); - } else { - vg = br_vlan_group(br); - } + if (cfg.p) + vg = nbp_vlan_group(cfg.p); + else + vg = br_vlan_group(cfg.br); /* If vlan filtering is enabled and VLAN is not specified * delete mdb entry on all vlans configured on the port. */ - if (br_vlan_enabled(br->dev) && vg && entry->vid == 0) { + if (br_vlan_enabled(cfg.br->dev) && vg && cfg.entry->vid == 0) { list_for_each_entry(v, &vg->vlan_list, vlist) { - entry->vid = v->vid; - err = __br_mdb_del(br, entry, mdb_attrs); + cfg.entry->vid = v->vid; + cfg.group.vid = v->vid; + err = __br_mdb_del(&cfg); } } else { - err = __br_mdb_del(br, entry, mdb_attrs); + err = __br_mdb_del(&cfg); } return err; diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 5e988f0ed2c0..db4c3900ae95 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -1273,7 +1273,7 @@ br_multicast_new_group_src(struct net_bridge_port_group *pg, struct br_ip *src_i struct net_bridge_port_group *br_multicast_new_port_group( struct net_bridge_port *port, - struct br_ip *group, + const struct br_ip *group, struct net_bridge_port_group __rcu *next, unsigned char flags, const unsigned char *src, diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index 4c4fda930068..3997e16c15fc 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -92,6 +92,13 @@ struct bridge_mcast_stats { struct br_mcast_stats mstats; struct u64_stats_sync syncp; }; + +struct br_mdb_config { + struct net_bridge *br; + struct net_bridge_port *p; + struct br_mdb_entry *entry; + struct br_ip group; +}; #endif /* net_bridge_mcast_port must be always defined due to forwarding stubs */ @@ -934,7 +941,8 @@ br_mdb_ip_get(struct net_bridge *br, struct br_ip *dst); struct net_bridge_mdb_entry * br_multicast_new_group(struct net_bridge *br, struct br_ip *group); struct net_bridge_port_group * -br_multicast_new_port_group(struct net_bridge_port *port, struct br_ip *group, +br_multicast_new_port_group(struct net_bridge_port *port, + const struct br_ip *group, struct net_bridge_port_group __rcu *next, unsigned char flags, const unsigned char *src, u8 filter_mode, u8 rt_protocol); diff --git a/net/can/af_can.c b/net/can/af_can.c index 27dcdcc0b808..7343fd487dbe 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -446,7 +446,6 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, struct hlist_head *rcv_list; struct can_dev_rcv_lists *dev_rcv_lists; struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats; - int err = 0; /* insert new receiver (dev,canid,mask) -> (func,data) */ @@ -481,7 +480,7 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, rcv_lists_stats->rcv_entries); spin_unlock_bh(&net->can.rcvlists_lock); - return err; + return 0; } EXPORT_SYMBOL(can_rx_register); @@ -677,7 +676,7 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev) static int can_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { - if (unlikely(dev->type != ARPHRD_CAN || (!can_is_can_skb(skb)))) { + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || !can_is_can_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d\n", dev->type, skb->len); @@ -692,7 +691,7 @@ static int can_rcv(struct sk_buff *skb, struct net_device *dev, static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { - if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canfd_skb(skb)))) { + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || !can_is_canfd_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d\n", dev->type, skb->len); @@ -707,7 +706,7 @@ static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, static int canxl_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { - if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canxl_skb(skb)))) { + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || !can_is_canxl_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN XL skbuff: dev type %d, len %d\n", dev->type, skb->len); diff --git a/net/can/raw.c b/net/can/raw.c index 3eb7d3e2b541..81071cdb0301 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -857,6 +857,7 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) skb->dev = dev; skb->priority = sk->sk_priority; + skb->mark = sk->sk_mark; skb->tstamp = sockc.transmit_time; skb_setup_tx_timestamp(skb, sockc.tsflags); diff --git a/net/core/dev.c b/net/core/dev.c index 7627c475d991..b76fb37b381e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10517,6 +10517,22 @@ void netdev_set_default_ethtool_ops(struct net_device *dev, } EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); +/** + * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default + * @dev: netdev to enable the IRQ coalescing on + * + * Sets a conservative default for SW IRQ coalescing. Users can use + * sysfs attributes to override the default values. + */ +void netdev_sw_irq_coalesce_default_on(struct net_device *dev) +{ + WARN_ON(dev->reg_state == NETREG_REGISTERED); + + dev->gro_flush_timeout = 20000; + dev->napi_defer_hard_irqs = 1; +} +EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on); + void netdev_freemem(struct net_device *dev) { char *addr = (char *)dev - dev->padded; diff --git a/net/core/devlink.c b/net/core/devlink.c index 0e10a8a68c5e..6004bd0ccee4 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -41,7 +41,7 @@ struct devlink_dev_stats { struct devlink { u32 index; - struct list_head port_list; + struct xarray ports; struct list_head rate_list; struct list_head sb_list; struct list_head dpipe_table_list; @@ -195,11 +195,16 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); +#define DEVLINK_PORT_FN_CAPS_VALID_MASK \ + (_BITUL(__DEVLINK_PORT_FN_ATTR_CAPS_MAX) - 1) + static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = { [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY }, [DEVLINK_PORT_FN_ATTR_STATE] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE, DEVLINK_PORT_FN_STATE_ACTIVE), + [DEVLINK_PORT_FN_ATTR_CAPS] = + NLA_POLICY_BITFIELD32(DEVLINK_PORT_FN_CAPS_VALID_MASK), }; static const struct nla_policy devlink_selftest_nl_policy[DEVLINK_ATTR_SELFTEST_ID_MAX + 1] = { @@ -382,19 +387,7 @@ static struct devlink *devlink_get_from_attrs(struct net *net, static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink, unsigned int port_index) { - struct devlink_port *devlink_port; - - list_for_each_entry(devlink_port, &devlink->port_list, list) { - if (devlink_port->index == port_index) - return devlink_port; - } - return NULL; -} - -static bool devlink_port_index_exists(struct devlink *devlink, - unsigned int port_index) -{ - return devlink_port_get_by_index(devlink, port_index); + return xa_load(&devlink->ports, port_index); } static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink, @@ -692,6 +685,87 @@ devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb, return 0; } +static void devlink_port_fn_cap_fill(struct nla_bitfield32 *caps, + u32 cap, bool is_enable) +{ + caps->selector |= cap; + if (is_enable) + caps->value |= cap; +} + +static int devlink_port_fn_roce_fill(const struct devlink_ops *ops, + struct devlink_port *devlink_port, + struct nla_bitfield32 *caps, + struct netlink_ext_ack *extack) +{ + bool is_enable; + int err; + + if (!ops->port_fn_roce_get) + return 0; + + err = ops->port_fn_roce_get(devlink_port, &is_enable, extack); + if (err) { + if (err == -EOPNOTSUPP) + return 0; + return err; + } + + devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_ROCE, is_enable); + return 0; +} + +static int devlink_port_fn_migratable_fill(const struct devlink_ops *ops, + struct devlink_port *devlink_port, + struct nla_bitfield32 *caps, + struct netlink_ext_ack *extack) +{ + bool is_enable; + int err; + + if (!ops->port_fn_migratable_get || + devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF) + return 0; + + err = ops->port_fn_migratable_get(devlink_port, &is_enable, extack); + if (err) { + if (err == -EOPNOTSUPP) + return 0; + return err; + } + + devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_MIGRATABLE, is_enable); + return 0; +} + +static int devlink_port_fn_caps_fill(const struct devlink_ops *ops, + struct devlink_port *devlink_port, + struct sk_buff *msg, + struct netlink_ext_ack *extack, + bool *msg_updated) +{ + struct nla_bitfield32 caps = {}; + int err; + + err = devlink_port_fn_roce_fill(ops, devlink_port, &caps, extack); + if (err) + return err; + + err = devlink_port_fn_migratable_fill(ops, devlink_port, &caps, extack); + if (err) + return err; + + if (!caps.selector) + return 0; + err = nla_put_bitfield32(msg, DEVLINK_PORT_FN_ATTR_CAPS, caps.value, + caps.selector); + if (err) + return err; + + *msg_updated = true; + return 0; +} + static int devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb, struct genl_info *info, @@ -1276,6 +1350,51 @@ static int devlink_port_fn_state_fill(const struct devlink_ops *ops, } static int +devlink_port_fn_mig_set(struct devlink_port *devlink_port, bool enable, + struct netlink_ext_ack *extack) +{ + const struct devlink_ops *ops = devlink_port->devlink->ops; + + return ops->port_fn_migratable_set(devlink_port, enable, extack); +} + +static int +devlink_port_fn_roce_set(struct devlink_port *devlink_port, bool enable, + struct netlink_ext_ack *extack) +{ + const struct devlink_ops *ops = devlink_port->devlink->ops; + + return ops->port_fn_roce_set(devlink_port, enable, extack); +} + +static int devlink_port_fn_caps_set(struct devlink_port *devlink_port, + const struct nlattr *attr, + struct netlink_ext_ack *extack) +{ + struct nla_bitfield32 caps; + u32 caps_value; + int err; + + caps = nla_get_bitfield32(attr); + caps_value = caps.value & caps.selector; + if (caps.selector & DEVLINK_PORT_FN_CAP_ROCE) { + err = devlink_port_fn_roce_set(devlink_port, + caps_value & DEVLINK_PORT_FN_CAP_ROCE, + extack); + if (err) + return err; + } + if (caps.selector & DEVLINK_PORT_FN_CAP_MIGRATABLE) { + err = devlink_port_fn_mig_set(devlink_port, caps_value & + DEVLINK_PORT_FN_CAP_MIGRATABLE, + extack); + if (err) + return err; + } + return 0; +} + +static int devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port, struct netlink_ext_ack *extack) { @@ -1293,6 +1412,10 @@ devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *por &msg_updated); if (err) goto out; + err = devlink_port_fn_caps_fill(ops, port, msg, extack, + &msg_updated); + if (err) + goto out; err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated); out: if (err || !msg_updated) @@ -1565,14 +1688,14 @@ static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg, { struct devlink *devlink; struct devlink_port *devlink_port; + unsigned long index, port_index; int start = cb->args[0]; - unsigned long index; int idx = 0; int err; devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) { devl_lock(devlink); - list_for_each_entry(devlink_port, &devlink->port_list, list) { + xa_for_each(&devlink->ports, port_index, devlink_port) { if (idx < start) { idx++; continue; @@ -1644,11 +1767,6 @@ static int devlink_port_function_hw_addr_set(struct devlink_port *port, } } - if (!ops->port_function_hw_addr_set) { - NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes"); - return -EOPNOTSUPP; - } - return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len, extack); } @@ -1662,12 +1780,52 @@ static int devlink_port_fn_state_set(struct devlink_port *port, state = nla_get_u8(attr); ops = port->devlink->ops; - if (!ops->port_fn_state_set) { - NL_SET_ERR_MSG_MOD(extack, - "Function does not support state setting"); + return ops->port_fn_state_set(port, state, extack); +} + +static int devlink_port_function_validate(struct devlink_port *devlink_port, + struct nlattr **tb, + struct netlink_ext_ack *extack) +{ + const struct devlink_ops *ops = devlink_port->devlink->ops; + struct nlattr *attr; + + if (tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] && + !ops->port_function_hw_addr_set) { + NL_SET_ERR_MSG_ATTR(extack, tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR], + "Port doesn't support function attributes"); return -EOPNOTSUPP; } - return ops->port_fn_state_set(port, state, extack); + if (tb[DEVLINK_PORT_FN_ATTR_STATE] && !ops->port_fn_state_set) { + NL_SET_ERR_MSG_ATTR(extack, tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR], + "Function does not support state setting"); + return -EOPNOTSUPP; + } + attr = tb[DEVLINK_PORT_FN_ATTR_CAPS]; + if (attr) { + struct nla_bitfield32 caps; + + caps = nla_get_bitfield32(attr); + if (caps.selector & DEVLINK_PORT_FN_CAP_ROCE && + !ops->port_fn_roce_set) { + NL_SET_ERR_MSG_ATTR(extack, attr, + "Port doesn't support RoCE function attribute"); + return -EOPNOTSUPP; + } + if (caps.selector & DEVLINK_PORT_FN_CAP_MIGRATABLE) { + if (!ops->port_fn_migratable_set) { + NL_SET_ERR_MSG_ATTR(extack, attr, + "Port doesn't support migratable function attribute"); + return -EOPNOTSUPP; + } + if (devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF) { + NL_SET_ERR_MSG_ATTR(extack, attr, + "migratable function attribute supported for VFs only"); + return -EOPNOTSUPP; + } + } + } + return 0; } static int devlink_port_function_set(struct devlink_port *port, @@ -1684,12 +1842,24 @@ static int devlink_port_function_set(struct devlink_port *port, return err; } + err = devlink_port_function_validate(port, tb, extack); + if (err) + return err; + attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR]; if (attr) { err = devlink_port_function_hw_addr_set(port, attr, extack); if (err) return err; } + + attr = tb[DEVLINK_PORT_FN_ATTR_CAPS]; + if (attr) { + err = devlink_port_fn_caps_set(port, attr, extack); + if (err) + return err; + } + /* Keep this as the last function attribute set, so that when * multiple port function attributes are set along with state, * Those can be applied first before activating the state. @@ -2886,10 +3056,11 @@ static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx, { struct devlink_port *devlink_port; u16 pool_count = devlink_sb_pool_count(devlink_sb); + unsigned long port_index; u16 pool_index; int err; - list_for_each_entry(devlink_port, &devlink->port_list, list) { + xa_for_each(&devlink->ports, port_index, devlink_port) { for (pool_index = 0; pool_index < pool_count; pool_index++) { if (*p_idx < start) { (*p_idx)++; @@ -3107,10 +3278,11 @@ static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg, u32 portid, u32 seq) { struct devlink_port *devlink_port; + unsigned long port_index; u16 tc_index; int err; - list_for_each_entry(devlink_port, &devlink->port_list, list) { + xa_for_each(&devlink->ports, port_index, devlink_port) { for (tc_index = 0; tc_index < devlink_sb->ingress_tc_count; tc_index++) { if (*p_idx < start) { @@ -4269,9 +4441,10 @@ static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb, nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id, DEVLINK_ATTR_PAD)) goto nla_put_failure; - if (resource->size != resource->size_new) - nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW, - resource->size_new, DEVLINK_ATTR_PAD); + if (resource->size != resource->size_new && + nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW, + resource->size_new, DEVLINK_ATTR_PAD)) + goto nla_put_failure; if (devlink_resource_occ_put(resource, skb)) goto nla_put_failure; if (devlink_resource_size_params_put(resource, skb)) @@ -6207,6 +6380,7 @@ static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg, { struct devlink_region *region; struct devlink_port *port; + unsigned long port_index; int err = 0; devl_lock(devlink); @@ -6225,7 +6399,7 @@ static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg, (*idx)++; } - list_for_each_entry(port, &devlink->port_list, list) { + xa_for_each(&devlink->ports, port_index, port) { err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx, start); if (err) @@ -6431,7 +6605,6 @@ unlock: } static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg, - struct devlink *devlink, u8 *chunk, u32 chunk_size, u64 addr) { @@ -6461,39 +6634,37 @@ nla_put_failure: #define DEVLINK_REGION_READ_CHUNK_SIZE 256 -static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb, - struct devlink *devlink, - struct devlink_region *region, - struct nlattr **attrs, - u64 start_offset, - u64 end_offset, - u64 *new_offset) +typedef int devlink_chunk_fill_t(void *cb_priv, u8 *chunk, u32 chunk_size, + u64 curr_offset, + struct netlink_ext_ack *extack); + +static int +devlink_nl_region_read_fill(struct sk_buff *skb, devlink_chunk_fill_t *cb, + void *cb_priv, u64 start_offset, u64 end_offset, + u64 *new_offset, struct netlink_ext_ack *extack) { - struct devlink_snapshot *snapshot; u64 curr_offset = start_offset; - u32 snapshot_id; int err = 0; + u8 *data; - *new_offset = start_offset; + /* Allocate and re-use a single buffer */ + data = kmalloc(DEVLINK_REGION_READ_CHUNK_SIZE, GFP_KERNEL); + if (!data) + return -ENOMEM; - snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]); - snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id); - if (!snapshot) - return -EINVAL; + *new_offset = start_offset; while (curr_offset < end_offset) { u32 data_size; - u8 *data; - if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE) - data_size = end_offset - curr_offset; - else - data_size = DEVLINK_REGION_READ_CHUNK_SIZE; + data_size = min_t(u32, end_offset - curr_offset, + DEVLINK_REGION_READ_CHUNK_SIZE); + + err = cb(cb_priv, data, data_size, curr_offset, extack); + if (err) + break; - data = &snapshot->data[curr_offset]; - err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink, - data, data_size, - curr_offset); + err = devlink_nl_cmd_region_read_chunk_fill(skb, data, data_size, curr_offset); if (err) break; @@ -6501,21 +6672,57 @@ static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb, } *new_offset = curr_offset; + kfree(data); + return err; } +static int +devlink_region_snapshot_fill(void *cb_priv, u8 *chunk, u32 chunk_size, + u64 curr_offset, + struct netlink_ext_ack __always_unused *extack) +{ + struct devlink_snapshot *snapshot = cb_priv; + + memcpy(chunk, &snapshot->data[curr_offset], chunk_size); + + return 0; +} + +static int +devlink_region_port_direct_fill(void *cb_priv, u8 *chunk, u32 chunk_size, + u64 curr_offset, struct netlink_ext_ack *extack) +{ + struct devlink_region *region = cb_priv; + + return region->port_ops->read(region->port, region->port_ops, extack, + curr_offset, chunk_size, chunk); +} + +static int +devlink_region_direct_fill(void *cb_priv, u8 *chunk, u32 chunk_size, + u64 curr_offset, struct netlink_ext_ack *extack) +{ + struct devlink_region *region = cb_priv; + + return region->ops->read(region->devlink, region->ops, extack, + curr_offset, chunk_size, chunk); +} + static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { const struct genl_dumpit_info *info = genl_dumpit_info(cb); + struct nlattr *chunks_attr, *region_attr, *snapshot_attr; u64 ret_offset, start_offset, end_offset = U64_MAX; struct nlattr **attrs = info->attrs; struct devlink_port *port = NULL; + devlink_chunk_fill_t *region_cb; struct devlink_region *region; - struct nlattr *chunks_attr; const char *region_name; struct devlink *devlink; unsigned int index; + void *region_cb_priv; void *hdr; int err; @@ -6527,8 +6734,8 @@ static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb, devl_lock(devlink); - if (!attrs[DEVLINK_ATTR_REGION_NAME] || - !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) { + if (!attrs[DEVLINK_ATTR_REGION_NAME]) { + NL_SET_ERR_MSG(cb->extack, "No region name provided"); err = -EINVAL; goto out_unlock; } @@ -6543,7 +6750,8 @@ static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb, } } - region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]); + region_attr = attrs[DEVLINK_ATTR_REGION_NAME]; + region_name = nla_data(region_attr); if (port) region = devlink_port_region_get_by_name(port, region_name); @@ -6551,10 +6759,51 @@ static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb, region = devlink_region_get_by_name(devlink, region_name); if (!region) { + NL_SET_ERR_MSG_ATTR(cb->extack, region_attr, "Requested region does not exist"); err = -EINVAL; goto out_unlock; } + snapshot_attr = attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]; + if (!snapshot_attr) { + if (!nla_get_flag(attrs[DEVLINK_ATTR_REGION_DIRECT])) { + NL_SET_ERR_MSG(cb->extack, "No snapshot id provided"); + err = -EINVAL; + goto out_unlock; + } + + if (!region->ops->read) { + NL_SET_ERR_MSG(cb->extack, "Requested region does not support direct read"); + err = -EOPNOTSUPP; + goto out_unlock; + } + + if (port) + region_cb = &devlink_region_port_direct_fill; + else + region_cb = &devlink_region_direct_fill; + region_cb_priv = region; + } else { + struct devlink_snapshot *snapshot; + u32 snapshot_id; + + if (nla_get_flag(attrs[DEVLINK_ATTR_REGION_DIRECT])) { + NL_SET_ERR_MSG_ATTR(cb->extack, snapshot_attr, "Direct region read does not use snapshot"); + err = -EINVAL; + goto out_unlock; + } + + snapshot_id = nla_get_u32(snapshot_attr); + snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id); + if (!snapshot) { + NL_SET_ERR_MSG_ATTR(cb->extack, snapshot_attr, "Requested snapshot does not exist"); + err = -EINVAL; + goto out_unlock; + } + region_cb = &devlink_region_snapshot_fill; + region_cb_priv = snapshot; + } + if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] && attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) { if (!start_offset) @@ -6603,10 +6852,9 @@ static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb, goto nla_put_failure; } - err = devlink_nl_region_read_snapshot_fill(skb, devlink, - region, attrs, - start_offset, - end_offset, &ret_offset); + err = devlink_nl_region_read_fill(skb, region_cb, region_cb_priv, + start_offset, end_offset, &ret_offset, + cb->extack); if (err && err != -EMSGSIZE) goto nla_put_failure; @@ -6633,14 +6881,6 @@ out_unlock: return err; } -int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name) -{ - if (!req->msg) - return 0; - return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name); -} -EXPORT_SYMBOL_GPL(devlink_info_driver_name_put); - int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn) { if (!req->msg) @@ -6749,11 +6989,25 @@ int devlink_info_version_running_put_ext(struct devlink_info_req *req, } EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext); +static int devlink_nl_driver_info_get(struct device_driver *drv, + struct devlink_info_req *req) +{ + if (!drv) + return 0; + + if (drv->name[0]) + return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, + drv->name); + + return 0; +} + static int devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink, enum devlink_command cmd, u32 portid, u32 seq, int flags, struct netlink_ext_ack *extack) { + struct device *dev = devlink_to_dev(devlink); struct devlink_info_req req = {}; void *hdr; int err; @@ -6767,7 +7021,13 @@ devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink, goto err_cancel_msg; req.msg = msg; - err = devlink->ops->info_get(devlink, &req, extack); + if (devlink->ops->info_get) { + err = devlink->ops->info_get(devlink, &req, extack); + if (err) + goto err_cancel_msg; + } + + err = devlink_nl_driver_info_get(dev->driver, &req); if (err) goto err_cancel_msg; @@ -6786,9 +7046,6 @@ static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb, struct sk_buff *msg; int err; - if (!devlink->ops->info_get) - return -EOPNOTSUPP; - msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!msg) return -ENOMEM; @@ -6814,7 +7071,7 @@ static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg, int err = 0; devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) { - if (idx < start || !devlink->ops->info_get) + if (idx < start) goto inc; devl_lock(devlink); @@ -7974,10 +8231,10 @@ devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) { struct devlink_health_reporter *reporter; + unsigned long index, port_index; struct devlink_port *port; struct devlink *devlink; int start = cb->args[0]; - unsigned long index; int idx = 0; int err; @@ -8006,7 +8263,7 @@ devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg, devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) { devl_lock(devlink); - list_for_each_entry(port, &devlink->port_list, list) { + xa_for_each(&devlink->ports, port_index, port) { mutex_lock(&port->reporters_lock); list_for_each_entry(reporter, &port->reporter_list, list) { if (idx < start) { @@ -9251,6 +9508,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_SELFTESTS] = { .type = NLA_NESTED }, [DEVLINK_ATTR_RATE_TX_PRIORITY] = { .type = NLA_U32 }, [DEVLINK_ATTR_RATE_TX_WEIGHT] = { .type = NLA_U32 }, + [DEVLINK_ATTR_REGION_DIRECT] = { .type = NLA_FLAG }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -9724,9 +9982,9 @@ struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, devlink->dev = dev; devlink->ops = ops; + xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); write_pnet(&devlink->_net, net); - INIT_LIST_HEAD(&devlink->port_list); INIT_LIST_HEAD(&devlink->rate_list); INIT_LIST_HEAD(&devlink->linecard_list); INIT_LIST_HEAD(&devlink->sb_list); @@ -9778,12 +10036,13 @@ static void devlink_notify_register(struct devlink *devlink) struct devlink_linecard *linecard; struct devlink_rate *rate_node; struct devlink_region *region; + unsigned long port_index; devlink_notify(devlink, DEVLINK_CMD_NEW); list_for_each_entry(linecard, &devlink->linecard_list, list) devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); - list_for_each_entry(devlink_port, &devlink->port_list, list) + xa_for_each(&devlink->ports, port_index, devlink_port) devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW); list_for_each_entry(policer_item, &devlink->trap_policer_list, list) @@ -9817,6 +10076,7 @@ static void devlink_notify_unregister(struct devlink *devlink) struct devlink_port *devlink_port; struct devlink_rate *rate_node; struct devlink_region *region; + unsigned long port_index; list_for_each_entry_reverse(param_item, &devlink->param_list, list) devlink_param_notify(devlink, 0, param_item, @@ -9839,7 +10099,7 @@ static void devlink_notify_unregister(struct devlink *devlink) devlink_trap_policer_notify(devlink, policer_item, DEVLINK_CMD_TRAP_POLICER_DEL); - list_for_each_entry_reverse(devlink_port, &devlink->port_list, list) + xa_for_each(&devlink->ports, port_index, devlink_port) devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL); devlink_notify(devlink, DEVLINK_CMD_DEL); } @@ -9903,9 +10163,10 @@ void devlink_free(struct devlink *devlink) WARN_ON(!list_empty(&devlink->sb_list)); WARN_ON(!list_empty(&devlink->rate_list)); WARN_ON(!list_empty(&devlink->linecard_list)); - WARN_ON(!list_empty(&devlink->port_list)); + WARN_ON(!xa_empty(&devlink->ports)); xa_destroy(&devlink->snapshot_ids); + xa_destroy(&devlink->ports); WARN_ON_ONCE(unregister_netdevice_notifier_net(devlink_net(devlink), &devlink->netdevice_nb)); @@ -10004,10 +10265,9 @@ int devl_port_register(struct devlink *devlink, struct devlink_port *devlink_port, unsigned int port_index) { - devl_assert_locked(devlink); + int err; - if (devlink_port_index_exists(devlink, port_index)) - return -EEXIST; + devl_assert_locked(devlink); ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); @@ -10017,7 +10277,11 @@ int devl_port_register(struct devlink *devlink, spin_lock_init(&devlink_port->type_lock); INIT_LIST_HEAD(&devlink_port->reporter_list); mutex_init(&devlink_port->reporters_lock); - list_add_tail(&devlink_port->list, &devlink->port_list); + err = xa_insert(&devlink->ports, port_index, devlink_port, GFP_KERNEL); + if (err) { + mutex_destroy(&devlink_port->reporters_lock); + return err; + } INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn); devlink_port_type_warn_schedule(devlink_port); @@ -10066,7 +10330,7 @@ void devl_port_unregister(struct devlink_port *devlink_port) devlink_port_type_warn_cancel(devlink_port); devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL); - list_del(&devlink_port->list); + xa_erase(&devlink_port->devlink->ports, devlink_port->index); WARN_ON(!list_empty(&devlink_port->reporter_list)); mutex_destroy(&devlink_port->reporters_lock); devlink_port->registered = false; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 4bf95e36ed16..3cbba7099c0f 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -270,12 +270,10 @@ static struct sk_buff *napi_skb_cache_get(void) return skb; } -/* Caller must provide SKB that is memset cleared */ -static void __build_skb_around(struct sk_buff *skb, void *data, - unsigned int frag_size) +static inline void __finalize_skb_around(struct sk_buff *skb, void *data, + unsigned int size) { struct skb_shared_info *shinfo; - unsigned int size = frag_size ? : ksize(data); size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); @@ -297,15 +295,71 @@ static void __build_skb_around(struct sk_buff *skb, void *data, skb_set_kcov_handle(skb, kcov_common_handle()); } +static inline void *__slab_build_skb(struct sk_buff *skb, void *data, + unsigned int *size) +{ + void *resized; + + /* Must find the allocation size (and grow it to match). */ + *size = ksize(data); + /* krealloc() will immediately return "data" when + * "ksize(data)" is requested: it is the existing upper + * bounds. As a result, GFP_ATOMIC will be ignored. Note + * that this "new" pointer needs to be passed back to the + * caller for use so the __alloc_size hinting will be + * tracked correctly. + */ + resized = krealloc(data, *size, GFP_ATOMIC); + WARN_ON_ONCE(resized != data); + return resized; +} + +/* build_skb() variant which can operate on slab buffers. + * Note that this should be used sparingly as slab buffers + * cannot be combined efficiently by GRO! + */ +struct sk_buff *slab_build_skb(void *data) +{ + struct sk_buff *skb; + unsigned int size; + + skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); + if (unlikely(!skb)) + return NULL; + + memset(skb, 0, offsetof(struct sk_buff, tail)); + data = __slab_build_skb(skb, data, &size); + __finalize_skb_around(skb, data, size); + + return skb; +} +EXPORT_SYMBOL(slab_build_skb); + +/* Caller must provide SKB that is memset cleared */ +static void __build_skb_around(struct sk_buff *skb, void *data, + unsigned int frag_size) +{ + unsigned int size = frag_size; + + /* frag_size == 0 is considered deprecated now. Callers + * using slab buffer should use slab_build_skb() instead. + */ + if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) + data = __slab_build_skb(skb, data, &size); + + __finalize_skb_around(skb, data, size); +} + /** * __build_skb - build a network buffer * @data: data buffer provided by caller - * @frag_size: size of data, or 0 if head was kmalloced + * @frag_size: size of data (must not be 0) * * Allocate a new &sk_buff. Caller provides space holding head and - * skb_shared_info. @data must have been allocated by kmalloc() only if - * @frag_size is 0, otherwise data should come from the page allocator - * or vmalloc() + * skb_shared_info. @data must have been allocated from the page + * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() + * allocation is deprecated, and callers should use slab_build_skb() + * instead.) * The return is the new skb buffer. * On a failure the return is %NULL, and @data is not freed. * Notes : diff --git a/net/core/sock.c b/net/core/sock.c index 4571914a4aa8..b0ab841e0aed 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -901,13 +901,20 @@ int sock_set_timestamping(struct sock *sk, int optname, if (val & ~SOF_TIMESTAMPING_MASK) return -EINVAL; + if (val & SOF_TIMESTAMPING_OPT_ID_TCP && + !(val & SOF_TIMESTAMPING_OPT_ID)) + return -EINVAL; + if (val & SOF_TIMESTAMPING_OPT_ID && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { if (sk_is_tcp(sk)) { if ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) return -EINVAL; - atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); + if (val & SOF_TIMESTAMPING_OPT_ID_TCP) + atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); + else + atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); } else { atomic_set(&sk->sk_tskey, 0); } diff --git a/net/dsa/tag_hellcreek.c b/net/dsa/tag_hellcreek.c index 71884296fc70..03a1fb9c87a9 100644 --- a/net/dsa/tag_hellcreek.c +++ b/net/dsa/tag_hellcreek.c @@ -51,7 +51,8 @@ static struct sk_buff *hellcreek_rcv(struct sk_buff *skb, return NULL; } - pskb_trim_rcsum(skb, skb->len - HELLCREEK_TAG_LEN); + if (pskb_trim_rcsum(skb, skb->len - HELLCREEK_TAG_LEN)) + return NULL; dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_ksz.c b/net/dsa/tag_ksz.c index 0f6ae143afc9..080e5c369f5b 100644 --- a/net/dsa/tag_ksz.c +++ b/net/dsa/tag_ksz.c @@ -27,7 +27,8 @@ static struct sk_buff *ksz_common_rcv(struct sk_buff *skb, if (!skb->dev) return NULL; - pskb_trim_rcsum(skb, skb->len - len); + if (pskb_trim_rcsum(skb, skb->len - len)) + return NULL; dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index f14f51b41491..1c2ceba4771b 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -670,7 +670,8 @@ static struct sk_buff *sja1110_rcv_inband_control_extension(struct sk_buff *skb, * padding and trailer we need to account for the fact that * skb->data points to skb_mac_header(skb) + ETH_HLEN. */ - pskb_trim_rcsum(skb, start_of_padding - ETH_HLEN); + if (pskb_trim_rcsum(skb, start_of_padding - ETH_HLEN)) + return NULL; /* Trap-to-host frame, no timestamp trailer */ } else { *source_port = SJA1110_RX_HEADER_SRC_PORT(rx_header); diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index e02daa74e833..2edc8b796a4e 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -398,7 +398,7 @@ EXPORT_SYMBOL(alloc_etherdev_mqs); ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len) { - return scnprintf(buf, PAGE_SIZE, "%*phC\n", len, addr); + return sysfs_emit(buf, "%*phC\n", len, addr); } EXPORT_SYMBOL(sysfs_format_mac); diff --git a/net/ethtool/Makefile b/net/ethtool/Makefile index 72ab0944262a..228f13df2e18 100644 --- a/net/ethtool/Makefile +++ b/net/ethtool/Makefile @@ -4,7 +4,7 @@ obj-y += ioctl.o common.o obj-$(CONFIG_ETHTOOL_NETLINK) += ethtool_nl.o -ethtool_nl-y := netlink.o bitset.o strset.o linkinfo.o linkmodes.o \ +ethtool_nl-y := netlink.o bitset.o strset.o linkinfo.o linkmodes.o rss.o \ linkstate.o debug.o wol.o features.o privflags.o rings.o \ channels.o coalesce.o pause.o eee.o tsinfo.o cabletest.o \ tunnels.o fec.o eeprom.o stats.o phc_vclocks.o module.o \ diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 21cfe8557205..6f399afc2ff2 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -417,6 +417,7 @@ const char sof_timestamping_names[][ETH_GSTRING_LEN] = { [const_ilog2(SOF_TIMESTAMPING_OPT_PKTINFO)] = "option-pktinfo", [const_ilog2(SOF_TIMESTAMPING_OPT_TX_SWHW)] = "option-tx-swhw", [const_ilog2(SOF_TIMESTAMPING_BIND_PHC)] = "bind-phc", + [const_ilog2(SOF_TIMESTAMPING_OPT_ID_TCP)] = "option-id-tcp", }; static_assert(ARRAY_SIZE(sof_timestamping_names) == __SOF_TIMESTAMPING_CNT); diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 1a4c11356c96..aee98be6237f 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -287,6 +287,7 @@ ethnl_default_requests[__ETHTOOL_MSG_USER_CNT] = { [ETHTOOL_MSG_PHC_VCLOCKS_GET] = ðnl_phc_vclocks_request_ops, [ETHTOOL_MSG_MODULE_GET] = ðnl_module_request_ops, [ETHTOOL_MSG_PSE_GET] = ðnl_pse_request_ops, + [ETHTOOL_MSG_RSS_GET] = ðnl_rss_request_ops, }; static struct ethnl_dump_ctx *ethnl_dump_context(struct netlink_callback *cb) @@ -1040,6 +1041,12 @@ static const struct genl_ops ethtool_genl_ops[] = { .policy = ethnl_pse_set_policy, .maxattr = ARRAY_SIZE(ethnl_pse_set_policy) - 1, }, + { + .cmd = ETHTOOL_MSG_RSS_GET, + .doit = ethnl_default_doit, + .policy = ethnl_rss_get_policy, + .maxattr = ARRAY_SIZE(ethnl_rss_get_policy) - 1, + }, }; static const struct genl_multicast_group ethtool_nl_mcgrps[] = { diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 1bfd374f9718..3753787ba233 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -346,6 +346,7 @@ extern const struct ethnl_request_ops ethnl_stats_request_ops; extern const struct ethnl_request_ops ethnl_phc_vclocks_request_ops; extern const struct ethnl_request_ops ethnl_module_request_ops; extern const struct ethnl_request_ops ethnl_pse_request_ops; +extern const struct ethnl_request_ops ethnl_rss_request_ops; extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_FLAGS + 1]; extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_FLAGS + 1]; @@ -386,6 +387,7 @@ extern const struct nla_policy ethnl_module_get_policy[ETHTOOL_A_MODULE_HEADER + extern const struct nla_policy ethnl_module_set_policy[ETHTOOL_A_MODULE_POWER_MODE_POLICY + 1]; extern const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1]; extern const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1]; +extern const struct nla_policy ethnl_rss_get_policy[ETHTOOL_A_RSS_CONTEXT + 1]; int ethnl_set_linkinfo(struct sk_buff *skb, struct genl_info *info); int ethnl_set_linkmodes(struct sk_buff *skb, struct genl_info *info); diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c new file mode 100644 index 000000000000..ebe6145aed3f --- /dev/null +++ b/net/ethtool/rss.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "netlink.h" +#include "common.h" + +struct rss_req_info { + struct ethnl_req_info base; + u32 rss_context; +}; + +struct rss_reply_data { + struct ethnl_reply_data base; + u32 indir_size; + u32 hkey_size; + u32 hfunc; + u32 *indir_table; + u8 *hkey; +}; + +#define RSS_REQINFO(__req_base) \ + container_of(__req_base, struct rss_req_info, base) + +#define RSS_REPDATA(__reply_base) \ + container_of(__reply_base, struct rss_reply_data, base) + +const struct nla_policy ethnl_rss_get_policy[] = { + [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), + [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32 }, +}; + +static int +rss_parse_request(struct ethnl_req_info *req_info, struct nlattr **tb, + struct netlink_ext_ack *extack) +{ + struct rss_req_info *request = RSS_REQINFO(req_info); + + if (tb[ETHTOOL_A_RSS_CONTEXT]) + request->rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]); + + return 0; +} + +static int +rss_prepare_data(const struct ethnl_req_info *req_base, + struct ethnl_reply_data *reply_base, struct genl_info *info) +{ + struct rss_reply_data *data = RSS_REPDATA(reply_base); + struct rss_req_info *request = RSS_REQINFO(req_base); + struct net_device *dev = reply_base->dev; + const struct ethtool_ops *ops; + u32 total_size, indir_bytes; + u8 dev_hfunc = 0; + u8 *rss_config; + int ret; + + ops = dev->ethtool_ops; + if (!ops->get_rxfh) + return -EOPNOTSUPP; + + /* Some drivers don't handle rss_context */ + if (request->rss_context && !ops->get_rxfh_context) + return -EOPNOTSUPP; + + ret = ethnl_ops_begin(dev); + if (ret < 0) + return ret; + + data->indir_size = 0; + data->hkey_size = 0; + if (ops->get_rxfh_indir_size) + data->indir_size = ops->get_rxfh_indir_size(dev); + if (ops->get_rxfh_key_size) + data->hkey_size = ops->get_rxfh_key_size(dev); + + indir_bytes = data->indir_size * sizeof(u32); + total_size = indir_bytes + data->hkey_size; + rss_config = kzalloc(total_size, GFP_KERNEL); + if (!rss_config) { + ret = -ENOMEM; + goto out_ops; + } + + if (data->indir_size) + data->indir_table = (u32 *)rss_config; + + if (data->hkey_size) + data->hkey = rss_config + indir_bytes; + + if (request->rss_context) + ret = ops->get_rxfh_context(dev, data->indir_table, data->hkey, + &dev_hfunc, request->rss_context); + else + ret = ops->get_rxfh(dev, data->indir_table, data->hkey, + &dev_hfunc); + + if (ret) + goto out_ops; + + data->hfunc = dev_hfunc; +out_ops: + ethnl_ops_complete(dev); + return ret; +} + +static int +rss_reply_size(const struct ethnl_req_info *req_base, + const struct ethnl_reply_data *reply_base) +{ + const struct rss_reply_data *data = RSS_REPDATA(reply_base); + int len; + + len = nla_total_size(sizeof(u32)) + /* _RSS_HFUNC */ + nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */ + nla_total_size(data->hkey_size); /* _RSS_HKEY */ + + return len; +} + +static int +rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, + const struct ethnl_reply_data *reply_base) +{ + const struct rss_reply_data *data = RSS_REPDATA(reply_base); + + if (nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc) || + nla_put(skb, ETHTOOL_A_RSS_INDIR, + sizeof(u32) * data->indir_size, data->indir_table) || + nla_put(skb, ETHTOOL_A_RSS_HKEY, data->hkey_size, data->hkey)) + return -EMSGSIZE; + + return 0; +} + +static void rss_cleanup_data(struct ethnl_reply_data *reply_base) +{ + const struct rss_reply_data *data = RSS_REPDATA(reply_base); + + kfree(data->indir_table); +} + +const struct ethnl_request_ops ethnl_rss_request_ops = { + .request_cmd = ETHTOOL_MSG_RSS_GET, + .reply_cmd = ETHTOOL_MSG_RSS_GET_REPLY, + .hdr_attr = ETHTOOL_A_RSS_HEADER, + .req_info_size = sizeof(struct rss_req_info), + .reply_data_size = sizeof(struct rss_reply_data), + + .parse_request = rss_parse_request, + .prepare_data = rss_prepare_data, + .reply_size = rss_reply_size, + .fill_reply = rss_fill_reply, + .cleanup_data = rss_cleanup_data, +}; diff --git a/net/hsr/hsr_debugfs.c b/net/hsr/hsr_debugfs.c index de476a417631..1a195efc79cd 100644 --- a/net/hsr/hsr_debugfs.c +++ b/net/hsr/hsr_debugfs.c @@ -9,7 +9,6 @@ #include <linux/module.h> #include <linux/errno.h> #include <linux/debugfs.h> -#include <linux/jhash.h> #include "hsr_main.h" #include "hsr_framereg.h" @@ -21,7 +20,6 @@ hsr_node_table_show(struct seq_file *sfp, void *data) { struct hsr_priv *priv = (struct hsr_priv *)sfp->private; struct hsr_node *node; - int i; seq_printf(sfp, "Node Table entries for (%s) device\n", (priv->prot_version == PRP_V1 ? "PRP" : "HSR")); @@ -33,28 +31,22 @@ hsr_node_table_show(struct seq_file *sfp, void *data) seq_puts(sfp, "DAN-H\n"); rcu_read_lock(); - - for (i = 0 ; i < priv->hash_buckets; i++) { - hlist_for_each_entry_rcu(node, &priv->node_db[i], mac_list) { - /* skip self node */ - if (hsr_addr_is_self(priv, node->macaddress_A)) - continue; - seq_printf(sfp, "%pM ", &node->macaddress_A[0]); - seq_printf(sfp, "%pM ", &node->macaddress_B[0]); - seq_printf(sfp, "%10lx, ", - node->time_in[HSR_PT_SLAVE_A]); - seq_printf(sfp, "%10lx, ", - node->time_in[HSR_PT_SLAVE_B]); - seq_printf(sfp, "%14x, ", node->addr_B_port); - - if (priv->prot_version == PRP_V1) - seq_printf(sfp, "%5x, %5x, %5x\n", - node->san_a, node->san_b, - (node->san_a == 0 && - node->san_b == 0)); - else - seq_printf(sfp, "%5x\n", 1); - } + list_for_each_entry_rcu(node, &priv->node_db, mac_list) { + /* skip self node */ + if (hsr_addr_is_self(priv, node->macaddress_A)) + continue; + seq_printf(sfp, "%pM ", &node->macaddress_A[0]); + seq_printf(sfp, "%pM ", &node->macaddress_B[0]); + seq_printf(sfp, "%10lx, ", node->time_in[HSR_PT_SLAVE_A]); + seq_printf(sfp, "%10lx, ", node->time_in[HSR_PT_SLAVE_B]); + seq_printf(sfp, "%14x, ", node->addr_B_port); + + if (priv->prot_version == PRP_V1) + seq_printf(sfp, "%5x, %5x, %5x\n", + node->san_a, node->san_b, + (node->san_a == 0 && node->san_b == 0)); + else + seq_printf(sfp, "%5x\n", 1); } rcu_read_unlock(); return 0; diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 6ffef47e9be5..5a236aae2366 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -219,7 +219,9 @@ static netdev_tx_t hsr_dev_xmit(struct sk_buff *skb, struct net_device *dev) skb->dev = master->dev; skb_reset_mac_header(skb); skb_reset_mac_len(skb); + spin_lock_bh(&hsr->seqnr_lock); hsr_forward_skb(skb, master); + spin_unlock_bh(&hsr->seqnr_lock); } else { dev_core_stats_tx_dropped_inc(dev); dev_kfree_skb_any(skb); @@ -278,7 +280,6 @@ static void send_hsr_supervision_frame(struct hsr_port *master, __u8 type = HSR_TLV_LIFE_CHECK; struct hsr_sup_payload *hsr_sp; struct hsr_sup_tag *hsr_stag; - unsigned long irqflags; struct sk_buff *skb; *interval = msecs_to_jiffies(HSR_LIFE_CHECK_INTERVAL); @@ -299,7 +300,7 @@ static void send_hsr_supervision_frame(struct hsr_port *master, set_hsr_stag_HSR_ver(hsr_stag, hsr->prot_version); /* From HSRv1 on we have separate supervision sequence numbers. */ - spin_lock_irqsave(&master->hsr->seqnr_lock, irqflags); + spin_lock_bh(&hsr->seqnr_lock); if (hsr->prot_version > 0) { hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr); hsr->sup_sequence_nr++; @@ -307,7 +308,6 @@ static void send_hsr_supervision_frame(struct hsr_port *master, hsr_stag->sequence_nr = htons(hsr->sequence_nr); hsr->sequence_nr++; } - spin_unlock_irqrestore(&master->hsr->seqnr_lock, irqflags); hsr_stag->tlv.HSR_TLV_type = type; /* TODO: Why 12 in HSRv0? */ @@ -318,11 +318,13 @@ static void send_hsr_supervision_frame(struct hsr_port *master, hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload)); ether_addr_copy(hsr_sp->macaddress_A, master->dev->dev_addr); - if (skb_put_padto(skb, ETH_ZLEN)) + if (skb_put_padto(skb, ETH_ZLEN)) { + spin_unlock_bh(&hsr->seqnr_lock); return; + } hsr_forward_skb(skb, master); - + spin_unlock_bh(&hsr->seqnr_lock); return; } @@ -332,7 +334,6 @@ static void send_prp_supervision_frame(struct hsr_port *master, struct hsr_priv *hsr = master->hsr; struct hsr_sup_payload *hsr_sp; struct hsr_sup_tag *hsr_stag; - unsigned long irqflags; struct sk_buff *skb; skb = hsr_init_skb(master); @@ -347,7 +348,7 @@ static void send_prp_supervision_frame(struct hsr_port *master, set_hsr_stag_HSR_ver(hsr_stag, (hsr->prot_version ? 1 : 0)); /* From HSRv1 on we have separate supervision sequence numbers. */ - spin_lock_irqsave(&master->hsr->seqnr_lock, irqflags); + spin_lock_bh(&hsr->seqnr_lock); hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr); hsr->sup_sequence_nr++; hsr_stag->tlv.HSR_TLV_type = PRP_TLV_LIFE_CHECK_DD; @@ -358,13 +359,12 @@ static void send_prp_supervision_frame(struct hsr_port *master, ether_addr_copy(hsr_sp->macaddress_A, master->dev->dev_addr); if (skb_put_padto(skb, ETH_ZLEN)) { - spin_unlock_irqrestore(&master->hsr->seqnr_lock, irqflags); + spin_unlock_bh(&hsr->seqnr_lock); return; } - spin_unlock_irqrestore(&master->hsr->seqnr_lock, irqflags); - hsr_forward_skb(skb, master); + spin_unlock_bh(&hsr->seqnr_lock); } /* Announce (supervision frame) timer function @@ -444,7 +444,7 @@ void hsr_dev_setup(struct net_device *dev) dev->header_ops = &hsr_header_ops; dev->netdev_ops = &hsr_device_ops; SET_NETDEV_DEVTYPE(dev, &hsr_type); - dev->priv_flags |= IFF_NO_QUEUE; + dev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; dev->needs_free_netdev = true; @@ -485,16 +485,11 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], { bool unregister = false; struct hsr_priv *hsr; - int res, i; + int res; hsr = netdev_priv(hsr_dev); INIT_LIST_HEAD(&hsr->ports); - INIT_HLIST_HEAD(&hsr->self_node_db); - hsr->hash_buckets = HSR_HSIZE; - get_random_bytes(&hsr->hash_seed, sizeof(hsr->hash_seed)); - for (i = 0; i < hsr->hash_buckets; i++) - INIT_HLIST_HEAD(&hsr->node_db[i]); - + INIT_LIST_HEAD(&hsr->node_db); spin_lock_init(&hsr->list_lock); eth_hw_addr_set(hsr_dev, slave[0]->dev_addr); diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index 56bb27d67a2e..629daacc9607 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -500,7 +500,6 @@ static void handle_std_frame(struct sk_buff *skb, { struct hsr_port *port = frame->port_rcv; struct hsr_priv *hsr = port->hsr; - unsigned long irqflags; frame->skb_hsr = NULL; frame->skb_prp = NULL; @@ -510,10 +509,9 @@ static void handle_std_frame(struct sk_buff *skb, frame->is_from_san = true; } else { /* Sequence nr for the master node */ - spin_lock_irqsave(&hsr->seqnr_lock, irqflags); + lockdep_assert_held(&hsr->seqnr_lock); frame->sequence_nr = hsr->sequence_nr; hsr->sequence_nr++; - spin_unlock_irqrestore(&hsr->seqnr_lock, irqflags); } } @@ -571,23 +569,20 @@ static int fill_frame_info(struct hsr_frame_info *frame, struct ethhdr *ethhdr; __be16 proto; int ret; - u32 hash; /* Check if skb contains ethhdr */ if (skb->mac_len < sizeof(struct ethhdr)) return -EINVAL; memset(frame, 0, sizeof(*frame)); - - ethhdr = (struct ethhdr *)skb_mac_header(skb); - hash = hsr_mac_hash(port->hsr, ethhdr->h_source); frame->is_supervision = is_supervision_frame(port->hsr, skb); - frame->node_src = hsr_get_node(port, &hsr->node_db[hash], skb, + frame->node_src = hsr_get_node(port, &hsr->node_db, skb, frame->is_supervision, port->type); if (!frame->node_src) return -1; /* Unknown node and !is_supervision, or no mem */ + ethhdr = (struct ethhdr *)skb_mac_header(skb); frame->is_vlan = false; proto = ethhdr->h_proto; @@ -617,11 +612,13 @@ void hsr_forward_skb(struct sk_buff *skb, struct hsr_port *port) { struct hsr_frame_info frame; + rcu_read_lock(); if (fill_frame_info(&frame, skb, port) < 0) goto out_drop; hsr_register_frame_in(frame.node_src, port, frame.sequence_nr); hsr_forward_do(&frame); + rcu_read_unlock(); /* Gets called for ingress frames as well as egress from master port. * So check and increment stats for master port only here. */ @@ -636,6 +633,7 @@ void hsr_forward_skb(struct sk_buff *skb, struct hsr_port *port) return; out_drop: + rcu_read_unlock(); port->dev->stats.tx_dropped++; kfree_skb(skb); } diff --git a/net/hsr/hsr_framereg.c b/net/hsr/hsr_framereg.c index 584e21788799..00db74d96583 100644 --- a/net/hsr/hsr_framereg.c +++ b/net/hsr/hsr_framereg.c @@ -15,37 +15,10 @@ #include <linux/etherdevice.h> #include <linux/slab.h> #include <linux/rculist.h> -#include <linux/jhash.h> #include "hsr_main.h" #include "hsr_framereg.h" #include "hsr_netlink.h" -#ifdef CONFIG_LOCKDEP -int lockdep_hsr_is_held(spinlock_t *lock) -{ - return lockdep_is_held(lock); -} -#endif - -u32 hsr_mac_hash(struct hsr_priv *hsr, const unsigned char *addr) -{ - u32 hash = jhash(addr, ETH_ALEN, hsr->hash_seed); - - return reciprocal_scale(hash, hsr->hash_buckets); -} - -struct hsr_node *hsr_node_get_first(struct hlist_head *head, spinlock_t *lock) -{ - struct hlist_node *first; - - first = rcu_dereference_bh_check(hlist_first_rcu(head), - lockdep_hsr_is_held(lock)); - if (first) - return hlist_entry(first, struct hsr_node, mac_list); - - return NULL; -} - /* seq_nr_after(a, b) - return true if a is after (higher in sequence than) b, * false otherwise. */ @@ -65,30 +38,32 @@ static bool seq_nr_after(u16 a, u16 b) bool hsr_addr_is_self(struct hsr_priv *hsr, unsigned char *addr) { - struct hsr_node *node; + struct hsr_self_node *sn; + bool ret = false; - node = hsr_node_get_first(&hsr->self_node_db, &hsr->list_lock); - if (!node) { + rcu_read_lock(); + sn = rcu_dereference(hsr->self_node); + if (!sn) { WARN_ONCE(1, "HSR: No self node\n"); - return false; + goto out; } - if (ether_addr_equal(addr, node->macaddress_A)) - return true; - if (ether_addr_equal(addr, node->macaddress_B)) - return true; - - return false; + if (ether_addr_equal(addr, sn->macaddress_A) || + ether_addr_equal(addr, sn->macaddress_B)) + ret = true; +out: + rcu_read_unlock(); + return ret; } /* Search for mac entry. Caller must hold rcu read lock. */ -static struct hsr_node *find_node_by_addr_A(struct hlist_head *node_db, +static struct hsr_node *find_node_by_addr_A(struct list_head *node_db, const unsigned char addr[ETH_ALEN]) { struct hsr_node *node; - hlist_for_each_entry_rcu(node, node_db, mac_list) { + list_for_each_entry_rcu(node, node_db, mac_list) { if (ether_addr_equal(node->macaddress_A, addr)) return node; } @@ -96,58 +71,51 @@ static struct hsr_node *find_node_by_addr_A(struct hlist_head *node_db, return NULL; } -/* Helper for device init; the self_node_db is used in hsr_rcv() to recognize +/* Helper for device init; the self_node is used in hsr_rcv() to recognize * frames from self that's been looped over the HSR ring. */ int hsr_create_self_node(struct hsr_priv *hsr, const unsigned char addr_a[ETH_ALEN], const unsigned char addr_b[ETH_ALEN]) { - struct hlist_head *self_node_db = &hsr->self_node_db; - struct hsr_node *node, *oldnode; + struct hsr_self_node *sn, *old; - node = kmalloc(sizeof(*node), GFP_KERNEL); - if (!node) + sn = kmalloc(sizeof(*sn), GFP_KERNEL); + if (!sn) return -ENOMEM; - ether_addr_copy(node->macaddress_A, addr_a); - ether_addr_copy(node->macaddress_B, addr_b); + ether_addr_copy(sn->macaddress_A, addr_a); + ether_addr_copy(sn->macaddress_B, addr_b); spin_lock_bh(&hsr->list_lock); - oldnode = hsr_node_get_first(self_node_db, &hsr->list_lock); - if (oldnode) { - hlist_replace_rcu(&oldnode->mac_list, &node->mac_list); - spin_unlock_bh(&hsr->list_lock); - kfree_rcu(oldnode, rcu_head); - } else { - hlist_add_tail_rcu(&node->mac_list, self_node_db); - spin_unlock_bh(&hsr->list_lock); - } + old = rcu_replace_pointer(hsr->self_node, sn, + lockdep_is_held(&hsr->list_lock)); + spin_unlock_bh(&hsr->list_lock); + if (old) + kfree_rcu(old, rcu_head); return 0; } void hsr_del_self_node(struct hsr_priv *hsr) { - struct hlist_head *self_node_db = &hsr->self_node_db; - struct hsr_node *node; + struct hsr_self_node *old; spin_lock_bh(&hsr->list_lock); - node = hsr_node_get_first(self_node_db, &hsr->list_lock); - if (node) { - hlist_del_rcu(&node->mac_list); - kfree_rcu(node, rcu_head); - } + old = rcu_replace_pointer(hsr->self_node, NULL, + lockdep_is_held(&hsr->list_lock)); spin_unlock_bh(&hsr->list_lock); + if (old) + kfree_rcu(old, rcu_head); } -void hsr_del_nodes(struct hlist_head *node_db) +void hsr_del_nodes(struct list_head *node_db) { struct hsr_node *node; - struct hlist_node *tmp; + struct hsr_node *tmp; - hlist_for_each_entry_safe(node, tmp, node_db, mac_list) - kfree_rcu(node, rcu_head); + list_for_each_entry_safe(node, tmp, node_db, mac_list) + kfree(node); } void prp_handle_san_frame(bool san, enum hsr_port_type port, @@ -168,7 +136,7 @@ void prp_handle_san_frame(bool san, enum hsr_port_type port, * originating from the newly added node. */ static struct hsr_node *hsr_add_node(struct hsr_priv *hsr, - struct hlist_head *node_db, + struct list_head *node_db, unsigned char addr[], u16 seq_out, bool san, enum hsr_port_type rx_port) @@ -182,6 +150,7 @@ static struct hsr_node *hsr_add_node(struct hsr_priv *hsr, return NULL; ether_addr_copy(new_node->macaddress_A, addr); + spin_lock_init(&new_node->seq_out_lock); /* We are only interested in time diffs here, so use current jiffies * as initialization. (0 could trigger an spurious ring error warning). @@ -198,14 +167,14 @@ static struct hsr_node *hsr_add_node(struct hsr_priv *hsr, hsr->proto_ops->handle_san_frame(san, rx_port, new_node); spin_lock_bh(&hsr->list_lock); - hlist_for_each_entry_rcu(node, node_db, mac_list, - lockdep_hsr_is_held(&hsr->list_lock)) { + list_for_each_entry_rcu(node, node_db, mac_list, + lockdep_is_held(&hsr->list_lock)) { if (ether_addr_equal(node->macaddress_A, addr)) goto out; if (ether_addr_equal(node->macaddress_B, addr)) goto out; } - hlist_add_tail_rcu(&new_node->mac_list, node_db); + list_add_tail_rcu(&new_node->mac_list, node_db); spin_unlock_bh(&hsr->list_lock); return new_node; out: @@ -225,7 +194,7 @@ void prp_update_san_info(struct hsr_node *node, bool is_sup) /* Get the hsr_node from which 'skb' was sent. */ -struct hsr_node *hsr_get_node(struct hsr_port *port, struct hlist_head *node_db, +struct hsr_node *hsr_get_node(struct hsr_port *port, struct list_head *node_db, struct sk_buff *skb, bool is_sup, enum hsr_port_type rx_port) { @@ -241,7 +210,7 @@ struct hsr_node *hsr_get_node(struct hsr_port *port, struct hlist_head *node_db, ethhdr = (struct ethhdr *)skb_mac_header(skb); - hlist_for_each_entry_rcu(node, node_db, mac_list) { + list_for_each_entry_rcu(node, node_db, mac_list) { if (ether_addr_equal(node->macaddress_A, ethhdr->h_source)) { if (hsr->proto_ops->update_san_info) hsr->proto_ops->update_san_info(node, is_sup); @@ -291,12 +260,11 @@ void hsr_handle_sup_frame(struct hsr_frame_info *frame) struct hsr_sup_tlv *hsr_sup_tlv; struct hsr_node *node_real; struct sk_buff *skb = NULL; - struct hlist_head *node_db; + struct list_head *node_db; struct ethhdr *ethhdr; int i; unsigned int pull_size = 0; unsigned int total_pull_size = 0; - u32 hash; /* Here either frame->skb_hsr or frame->skb_prp should be * valid as supervision frame always will have protocol @@ -334,13 +302,11 @@ void hsr_handle_sup_frame(struct hsr_frame_info *frame) hsr_sp = (struct hsr_sup_payload *)skb->data; /* Merge node_curr (registered on macaddress_B) into node_real */ - node_db = port_rcv->hsr->node_db; - hash = hsr_mac_hash(hsr, hsr_sp->macaddress_A); - node_real = find_node_by_addr_A(&node_db[hash], hsr_sp->macaddress_A); + node_db = &port_rcv->hsr->node_db; + node_real = find_node_by_addr_A(node_db, hsr_sp->macaddress_A); if (!node_real) /* No frame received from AddrA of this node yet */ - node_real = hsr_add_node(hsr, &node_db[hash], - hsr_sp->macaddress_A, + node_real = hsr_add_node(hsr, node_db, hsr_sp->macaddress_A, HSR_SEQNR_START - 1, true, port_rcv->type); if (!node_real) @@ -374,14 +340,14 @@ void hsr_handle_sup_frame(struct hsr_frame_info *frame) hsr_sp = (struct hsr_sup_payload *)skb->data; /* Check if redbox mac and node mac are equal. */ - if (!ether_addr_equal(node_real->macaddress_A, - hsr_sp->macaddress_A)) { + if (!ether_addr_equal(node_real->macaddress_A, hsr_sp->macaddress_A)) { /* This is a redbox supervision frame for a VDAN! */ goto done; } } ether_addr_copy(node_real->macaddress_B, ethhdr->h_source); + spin_lock_bh(&node_real->seq_out_lock); for (i = 0; i < HSR_PT_PORTS; i++) { if (!node_curr->time_in_stale[i] && time_after(node_curr->time_in[i], node_real->time_in[i])) { @@ -392,12 +358,16 @@ void hsr_handle_sup_frame(struct hsr_frame_info *frame) if (seq_nr_after(node_curr->seq_out[i], node_real->seq_out[i])) node_real->seq_out[i] = node_curr->seq_out[i]; } + spin_unlock_bh(&node_real->seq_out_lock); node_real->addr_B_port = port_rcv->type; spin_lock_bh(&hsr->list_lock); - hlist_del_rcu(&node_curr->mac_list); + if (!node_curr->removed) { + list_del_rcu(&node_curr->mac_list); + node_curr->removed = true; + kfree_rcu(node_curr, rcu_head); + } spin_unlock_bh(&hsr->list_lock); - kfree_rcu(node_curr, rcu_head); done: /* Push back here */ @@ -433,7 +403,6 @@ void hsr_addr_subst_dest(struct hsr_node *node_src, struct sk_buff *skb, struct hsr_port *port) { struct hsr_node *node_dst; - u32 hash; if (!skb_mac_header_was_set(skb)) { WARN_ONCE(1, "%s: Mac header not set\n", __func__); @@ -443,8 +412,7 @@ void hsr_addr_subst_dest(struct hsr_node *node_src, struct sk_buff *skb, if (!is_unicast_ether_addr(eth_hdr(skb)->h_dest)) return; - hash = hsr_mac_hash(port->hsr, eth_hdr(skb)->h_dest); - node_dst = find_node_by_addr_A(&port->hsr->node_db[hash], + node_dst = find_node_by_addr_A(&port->hsr->node_db, eth_hdr(skb)->h_dest); if (!node_dst) { if (net_ratelimit()) @@ -484,13 +452,17 @@ void hsr_register_frame_in(struct hsr_node *node, struct hsr_port *port, int hsr_register_frame_out(struct hsr_port *port, struct hsr_node *node, u16 sequence_nr) { + spin_lock_bh(&node->seq_out_lock); if (seq_nr_before_or_eq(sequence_nr, node->seq_out[port->type]) && time_is_after_jiffies(node->time_out[port->type] + - msecs_to_jiffies(HSR_ENTRY_FORGET_TIME))) + msecs_to_jiffies(HSR_ENTRY_FORGET_TIME))) { + spin_unlock_bh(&node->seq_out_lock); return 1; + } node->time_out[port->type] = jiffies; node->seq_out[port->type] = sequence_nr; + spin_unlock_bh(&node->seq_out_lock); return 0; } @@ -520,71 +492,60 @@ static struct hsr_port *get_late_port(struct hsr_priv *hsr, void hsr_prune_nodes(struct timer_list *t) { struct hsr_priv *hsr = from_timer(hsr, t, prune_timer); - struct hlist_node *tmp; struct hsr_node *node; + struct hsr_node *tmp; struct hsr_port *port; unsigned long timestamp; unsigned long time_a, time_b; - int i; spin_lock_bh(&hsr->list_lock); + list_for_each_entry_safe(node, tmp, &hsr->node_db, mac_list) { + /* Don't prune own node. Neither time_in[HSR_PT_SLAVE_A] + * nor time_in[HSR_PT_SLAVE_B], will ever be updated for + * the master port. Thus the master node will be repeatedly + * pruned leading to packet loss. + */ + if (hsr_addr_is_self(hsr, node->macaddress_A)) + continue; + + /* Shorthand */ + time_a = node->time_in[HSR_PT_SLAVE_A]; + time_b = node->time_in[HSR_PT_SLAVE_B]; + + /* Check for timestamps old enough to risk wrap-around */ + if (time_after(jiffies, time_a + MAX_JIFFY_OFFSET / 2)) + node->time_in_stale[HSR_PT_SLAVE_A] = true; + if (time_after(jiffies, time_b + MAX_JIFFY_OFFSET / 2)) + node->time_in_stale[HSR_PT_SLAVE_B] = true; + + /* Get age of newest frame from node. + * At least one time_in is OK here; nodes get pruned long + * before both time_ins can get stale + */ + timestamp = time_a; + if (node->time_in_stale[HSR_PT_SLAVE_A] || + (!node->time_in_stale[HSR_PT_SLAVE_B] && + time_after(time_b, time_a))) + timestamp = time_b; + + /* Warn of ring error only as long as we get frames at all */ + if (time_is_after_jiffies(timestamp + + msecs_to_jiffies(1.5 * MAX_SLAVE_DIFF))) { + rcu_read_lock(); + port = get_late_port(hsr, node); + if (port) + hsr_nl_ringerror(hsr, node->macaddress_A, port); + rcu_read_unlock(); + } - for (i = 0; i < hsr->hash_buckets; i++) { - hlist_for_each_entry_safe(node, tmp, &hsr->node_db[i], - mac_list) { - /* Don't prune own node. - * Neither time_in[HSR_PT_SLAVE_A] - * nor time_in[HSR_PT_SLAVE_B], will ever be updated - * for the master port. Thus the master node will be - * repeatedly pruned leading to packet loss. - */ - if (hsr_addr_is_self(hsr, node->macaddress_A)) - continue; - - /* Shorthand */ - time_a = node->time_in[HSR_PT_SLAVE_A]; - time_b = node->time_in[HSR_PT_SLAVE_B]; - - /* Check for timestamps old enough to - * risk wrap-around - */ - if (time_after(jiffies, time_a + MAX_JIFFY_OFFSET / 2)) - node->time_in_stale[HSR_PT_SLAVE_A] = true; - if (time_after(jiffies, time_b + MAX_JIFFY_OFFSET / 2)) - node->time_in_stale[HSR_PT_SLAVE_B] = true; - - /* Get age of newest frame from node. - * At least one time_in is OK here; nodes get pruned - * long before both time_ins can get stale - */ - timestamp = time_a; - if (node->time_in_stale[HSR_PT_SLAVE_A] || - (!node->time_in_stale[HSR_PT_SLAVE_B] && - time_after(time_b, time_a))) - timestamp = time_b; - - /* Warn of ring error only as long as we get - * frames at all - */ - if (time_is_after_jiffies(timestamp + - msecs_to_jiffies(1.5 * MAX_SLAVE_DIFF))) { - rcu_read_lock(); - port = get_late_port(hsr, node); - if (port) - hsr_nl_ringerror(hsr, - node->macaddress_A, - port); - rcu_read_unlock(); - } - - /* Prune old entries */ - if (time_is_before_jiffies(timestamp + - msecs_to_jiffies(HSR_NODE_FORGET_TIME))) { - hsr_nl_nodedown(hsr, node->macaddress_A); - hlist_del_rcu(&node->mac_list); - /* Note that we need to free this - * entry later: - */ + /* Prune old entries */ + if (time_is_before_jiffies(timestamp + + msecs_to_jiffies(HSR_NODE_FORGET_TIME))) { + hsr_nl_nodedown(hsr, node->macaddress_A); + if (!node->removed) { + list_del_rcu(&node->mac_list); + node->removed = true; + /* Note that we need to free this entry later: */ kfree_rcu(node, rcu_head); } } @@ -600,20 +561,17 @@ void *hsr_get_next_node(struct hsr_priv *hsr, void *_pos, unsigned char addr[ETH_ALEN]) { struct hsr_node *node; - u32 hash; - - hash = hsr_mac_hash(hsr, addr); if (!_pos) { - node = hsr_node_get_first(&hsr->node_db[hash], - &hsr->list_lock); + node = list_first_or_null_rcu(&hsr->node_db, + struct hsr_node, mac_list); if (node) ether_addr_copy(addr, node->macaddress_A); return node; } node = _pos; - hlist_for_each_entry_continue_rcu(node, mac_list) { + list_for_each_entry_continue_rcu(node, &hsr->node_db, mac_list) { ether_addr_copy(addr, node->macaddress_A); return node; } @@ -633,11 +591,8 @@ int hsr_get_node_data(struct hsr_priv *hsr, struct hsr_node *node; struct hsr_port *port; unsigned long tdiff; - u32 hash; - - hash = hsr_mac_hash(hsr, addr); - node = find_node_by_addr_A(&hsr->node_db[hash], addr); + node = find_node_by_addr_A(&hsr->node_db, addr); if (!node) return -ENOENT; diff --git a/net/hsr/hsr_framereg.h b/net/hsr/hsr_framereg.h index f3762e9e42b5..b23556251d62 100644 --- a/net/hsr/hsr_framereg.h +++ b/net/hsr/hsr_framereg.h @@ -28,17 +28,9 @@ struct hsr_frame_info { bool is_from_san; }; -#ifdef CONFIG_LOCKDEP -int lockdep_hsr_is_held(spinlock_t *lock); -#else -#define lockdep_hsr_is_held(lock) 1 -#endif - -u32 hsr_mac_hash(struct hsr_priv *hsr, const unsigned char *addr); -struct hsr_node *hsr_node_get_first(struct hlist_head *head, spinlock_t *lock); void hsr_del_self_node(struct hsr_priv *hsr); -void hsr_del_nodes(struct hlist_head *node_db); -struct hsr_node *hsr_get_node(struct hsr_port *port, struct hlist_head *node_db, +void hsr_del_nodes(struct list_head *node_db); +struct hsr_node *hsr_get_node(struct hsr_port *port, struct list_head *node_db, struct sk_buff *skb, bool is_sup, enum hsr_port_type rx_port); void hsr_handle_sup_frame(struct hsr_frame_info *frame); @@ -76,7 +68,9 @@ void prp_handle_san_frame(bool san, enum hsr_port_type port, void prp_update_san_info(struct hsr_node *node, bool is_sup); struct hsr_node { - struct hlist_node mac_list; + struct list_head mac_list; + /* Protect R/W access to seq_out */ + spinlock_t seq_out_lock; unsigned char macaddress_A[ETH_ALEN]; unsigned char macaddress_B[ETH_ALEN]; /* Local slave through which AddrB frames are received from this node */ @@ -88,6 +82,7 @@ struct hsr_node { bool san_a; bool san_b; u16 seq_out[HSR_PT_PORTS]; + bool removed; struct rcu_head rcu_head; }; diff --git a/net/hsr/hsr_main.h b/net/hsr/hsr_main.h index b158ba409f9a..5584c80a5c79 100644 --- a/net/hsr/hsr_main.h +++ b/net/hsr/hsr_main.h @@ -47,9 +47,6 @@ #define HSR_V1_SUP_LSDUSIZE 52 -#define HSR_HSIZE_SHIFT 8 -#define HSR_HSIZE BIT(HSR_HSIZE_SHIFT) - /* The helper functions below assumes that 'path' occupies the 4 most * significant bits of the 16-bit field shared by 'path' and 'LSDU_size' (or * equivalently, the 4 most significant bits of HSR tag byte 14). @@ -185,11 +182,17 @@ struct hsr_proto_ops { void (*update_san_info)(struct hsr_node *node, bool is_sup); }; +struct hsr_self_node { + unsigned char macaddress_A[ETH_ALEN]; + unsigned char macaddress_B[ETH_ALEN]; + struct rcu_head rcu_head; +}; + struct hsr_priv { struct rcu_head rcu_head; struct list_head ports; - struct hlist_head node_db[HSR_HSIZE]; /* Known HSR nodes */ - struct hlist_head self_node_db; /* MACs of slaves */ + struct list_head node_db; /* Known HSR nodes */ + struct hsr_self_node __rcu *self_node; /* MACs of slaves */ struct timer_list announce_timer; /* Supervision frame dispatch */ struct timer_list prune_timer; int announce_count; @@ -199,8 +202,6 @@ struct hsr_priv { spinlock_t seqnr_lock; /* locking for sequence_nr */ spinlock_t list_lock; /* locking for node list */ struct hsr_proto_ops *proto_ops; - u32 hash_buckets; - u32 hash_seed; #define PRP_LAN_ID 0x5 /* 0x1010 for A and 0x1011 for B. Bit 0 is set * based on SLAVE_A or SLAVE_B */ diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c index 7174a9092900..78fe40eb9f01 100644 --- a/net/hsr/hsr_netlink.c +++ b/net/hsr/hsr_netlink.c @@ -105,7 +105,6 @@ static int hsr_newlink(struct net *src_net, struct net_device *dev, static void hsr_dellink(struct net_device *dev, struct list_head *head) { struct hsr_priv *hsr = netdev_priv(dev); - int i; del_timer_sync(&hsr->prune_timer); del_timer_sync(&hsr->announce_timer); @@ -114,8 +113,7 @@ static void hsr_dellink(struct net_device *dev, struct list_head *head) hsr_del_ports(hsr); hsr_del_self_node(hsr); - for (i = 0; i < hsr->hash_buckets; i++) - hsr_del_nodes(&hsr->node_db[i]); + hsr_del_nodes(&hsr->node_db); unregister_netdevice_queue(dev, head); } diff --git a/net/ieee802154/nl802154.c b/net/ieee802154/nl802154.c index b33d1b5eda87..248ad5e46969 100644 --- a/net/ieee802154/nl802154.c +++ b/net/ieee802154/nl802154.c @@ -26,10 +26,12 @@ static struct genl_family nl802154_fam; /* multicast groups */ enum nl802154_multicast_groups { NL802154_MCGRP_CONFIG, + NL802154_MCGRP_SCAN, }; static const struct genl_multicast_group nl802154_mcgrps[] = { [NL802154_MCGRP_CONFIG] = { .name = "config", }, + [NL802154_MCGRP_SCAN] = { .name = "scan", }, }; /* returns ERR_PTR values */ @@ -216,6 +218,9 @@ static const struct nla_policy nl802154_policy[NL802154_ATTR_MAX+1] = { [NL802154_ATTR_PID] = { .type = NLA_U32 }, [NL802154_ATTR_NETNS_FD] = { .type = NLA_U32 }, + + [NL802154_ATTR_COORDINATOR] = { .type = NLA_NESTED }, + #ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL [NL802154_ATTR_SEC_ENABLED] = { .type = NLA_U8, }, [NL802154_ATTR_SEC_OUT_LEVEL] = { .type = NLA_U32, }, @@ -1281,6 +1286,104 @@ static int nl802154_wpan_phy_netns(struct sk_buff *skb, struct genl_info *info) return err; } +static int nl802154_prep_scan_event_msg(struct sk_buff *msg, + struct cfg802154_registered_device *rdev, + struct wpan_dev *wpan_dev, + u32 portid, u32 seq, int flags, u8 cmd, + struct ieee802154_coord_desc *desc) +{ + struct nlattr *nla; + void *hdr; + + hdr = nl802154hdr_put(msg, portid, seq, flags, cmd); + if (!hdr) + return -ENOBUFS; + + if (nla_put_u32(msg, NL802154_ATTR_WPAN_PHY, rdev->wpan_phy_idx)) + goto nla_put_failure; + + if (wpan_dev->netdev && + nla_put_u32(msg, NL802154_ATTR_IFINDEX, wpan_dev->netdev->ifindex)) + goto nla_put_failure; + + if (nla_put_u64_64bit(msg, NL802154_ATTR_WPAN_DEV, + wpan_dev_id(wpan_dev), NL802154_ATTR_PAD)) + goto nla_put_failure; + + nla = nla_nest_start_noflag(msg, NL802154_ATTR_COORDINATOR); + if (!nla) + goto nla_put_failure; + + if (nla_put(msg, NL802154_COORD_PANID, IEEE802154_PAN_ID_LEN, + &desc->addr.pan_id)) + goto nla_put_failure; + + if (desc->addr.mode == IEEE802154_ADDR_SHORT) { + if (nla_put(msg, NL802154_COORD_ADDR, + IEEE802154_SHORT_ADDR_LEN, + &desc->addr.short_addr)) + goto nla_put_failure; + } else { + if (nla_put(msg, NL802154_COORD_ADDR, + IEEE802154_EXTENDED_ADDR_LEN, + &desc->addr.extended_addr)) + goto nla_put_failure; + } + + if (nla_put_u8(msg, NL802154_COORD_CHANNEL, desc->channel)) + goto nla_put_failure; + + if (nla_put_u8(msg, NL802154_COORD_PAGE, desc->page)) + goto nla_put_failure; + + if (nla_put_u16(msg, NL802154_COORD_SUPERFRAME_SPEC, + desc->superframe_spec)) + goto nla_put_failure; + + if (nla_put_u8(msg, NL802154_COORD_LINK_QUALITY, desc->link_quality)) + goto nla_put_failure; + + if (desc->gts_permit && nla_put_flag(msg, NL802154_COORD_GTS_PERMIT)) + goto nla_put_failure; + + /* TODO: NL802154_COORD_PAYLOAD_DATA if any */ + + nla_nest_end(msg, nla); + + genlmsg_end(msg, hdr); + + return 0; + + nla_put_failure: + genlmsg_cancel(msg, hdr); + + return -EMSGSIZE; +} + +int nl802154_scan_event(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, + struct ieee802154_coord_desc *desc) +{ + struct cfg802154_registered_device *rdev = wpan_phy_to_rdev(wpan_phy); + struct sk_buff *msg; + int ret; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC); + if (!msg) + return -ENOMEM; + + ret = nl802154_prep_scan_event_msg(msg, rdev, wpan_dev, 0, 0, 0, + NL802154_CMD_SCAN_EVENT, + desc); + if (ret < 0) { + nlmsg_free(msg); + return ret; + } + + return genlmsg_multicast_netns(&nl802154_fam, wpan_phy_net(wpan_phy), + msg, 0, NL802154_MCGRP_SCAN, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(nl802154_scan_event); + #ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL static const struct nla_policy nl802154_dev_addr_policy[NL802154_DEV_ADDR_ATTR_MAX + 1] = { [NL802154_DEV_ADDR_ATTR_PAN_ID] = { .type = NLA_U16 }, diff --git a/net/ieee802154/nl802154.h b/net/ieee802154/nl802154.h index 8c4b6d08954c..89b805500032 100644 --- a/net/ieee802154/nl802154.h +++ b/net/ieee802154/nl802154.h @@ -4,5 +4,7 @@ int nl802154_init(void); void nl802154_exit(void); +int nl802154_scan_event(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, + struct ieee802154_coord_desc *desc); #endif /* __IEEE802154_NL802154_H */ diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index f361d3d56be2..b5736ef16ed2 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -841,6 +841,9 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, return -EINVAL; } + if (!cfg->fc_table) + cfg->fc_table = RT_TABLE_MAIN; + return 0; errout: return err; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 19a662003eef..ce9ff3c62e84 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -423,6 +423,7 @@ static struct fib_info *fib_find_info(struct fib_info *nfi) nfi->fib_prefsrc == fi->fib_prefsrc && nfi->fib_priority == fi->fib_priority && nfi->fib_type == fi->fib_type && + nfi->fib_tb_id == fi->fib_tb_id && memcmp(nfi->fib_metrics, fi->fib_metrics, sizeof(u32) * RTAX_MAX) == 0 && !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) && diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index a4ccef3e6935..ffff46cdcb58 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1492,24 +1492,6 @@ static int ipgre_fill_info(struct sk_buff *skb, const struct net_device *dev) struct ip_tunnel_parm *p = &t->parms; __be16 o_flags = p->o_flags; - if (t->erspan_ver <= 2) { - if (t->erspan_ver != 0 && !t->collect_md) - o_flags |= TUNNEL_KEY; - - if (nla_put_u8(skb, IFLA_GRE_ERSPAN_VER, t->erspan_ver)) - goto nla_put_failure; - - if (t->erspan_ver == 1) { - if (nla_put_u32(skb, IFLA_GRE_ERSPAN_INDEX, t->index)) - goto nla_put_failure; - } else if (t->erspan_ver == 2) { - if (nla_put_u8(skb, IFLA_GRE_ERSPAN_DIR, t->dir)) - goto nla_put_failure; - if (nla_put_u16(skb, IFLA_GRE_ERSPAN_HWID, t->hwid)) - goto nla_put_failure; - } - } - if (nla_put_u32(skb, IFLA_GRE_LINK, p->link) || nla_put_be16(skb, IFLA_GRE_IFLAGS, gre_tnl_flags_to_gre_flags(p->i_flags)) || @@ -1550,6 +1532,34 @@ nla_put_failure: return -EMSGSIZE; } +static int erspan_fill_info(struct sk_buff *skb, const struct net_device *dev) +{ + struct ip_tunnel *t = netdev_priv(dev); + + if (t->erspan_ver <= 2) { + if (t->erspan_ver != 0 && !t->collect_md) + t->parms.o_flags |= TUNNEL_KEY; + + if (nla_put_u8(skb, IFLA_GRE_ERSPAN_VER, t->erspan_ver)) + goto nla_put_failure; + + if (t->erspan_ver == 1) { + if (nla_put_u32(skb, IFLA_GRE_ERSPAN_INDEX, t->index)) + goto nla_put_failure; + } else if (t->erspan_ver == 2) { + if (nla_put_u8(skb, IFLA_GRE_ERSPAN_DIR, t->dir)) + goto nla_put_failure; + if (nla_put_u16(skb, IFLA_GRE_ERSPAN_HWID, t->hwid)) + goto nla_put_failure; + } + } + + return ipgre_fill_info(skb, dev); + +nla_put_failure: + return -EMSGSIZE; +} + static void erspan_setup(struct net_device *dev) { struct ip_tunnel *t = netdev_priv(dev); @@ -1628,7 +1638,7 @@ static struct rtnl_link_ops erspan_link_ops __read_mostly = { .changelink = erspan_changelink, .dellink = ip_tunnel_dellink, .get_size = ipgre_get_size, - .fill_info = ipgre_fill_info, + .fill_info = erspan_fill_info, .get_link_net = ip_tunnel_get_link_net, }; diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index bb9854c2b7a1..409ec2a1f95b 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -49,6 +49,11 @@ #include <net/transp_v6.h> #endif +#define ping_portaddr_for_each_entry(__sk, node, list) \ + hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node) +#define ping_portaddr_for_each_entry_rcu(__sk, node, list) \ + hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node) + struct ping_table { struct hlist_nulls_head hash[PING_HTABLE_SIZE]; spinlock_t lock; @@ -192,7 +197,7 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) return NULL; } - ping_portaddr_for_each_entry(sk, hnode, hslot) { + ping_portaddr_for_each_entry_rcu(sk, hnode, hslot) { isk = inet_sk(sk); pr_debug("iterate\n"); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 24602a5184b0..001947136b0a 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4464,11 +4464,8 @@ bool tcp_alloc_md5sig_pool(void) if (unlikely(!READ_ONCE(tcp_md5sig_pool_populated))) { mutex_lock(&tcp_md5sig_mutex); - if (!tcp_md5sig_pool_populated) { + if (!tcp_md5sig_pool_populated) __tcp_alloc_md5sig_pool(); - if (tcp_md5sig_pool_populated) - static_branch_inc(&tcp_md5_needed); - } mutex_unlock(&tcp_md5sig_mutex); } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 1215fa4c1b9f..8320d0ecb13a 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1053,7 +1053,7 @@ static void tcp_v4_reqsk_destructor(struct request_sock *req) * We need to maintain these in the sk structure. */ -DEFINE_STATIC_KEY_FALSE(tcp_md5_needed); +DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ); EXPORT_SYMBOL(tcp_md5_needed); static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new) @@ -1161,10 +1161,25 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, } EXPORT_SYMBOL(tcp_v4_md5_lookup); +static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct tcp_md5sig_info *md5sig; + + md5sig = kmalloc(sizeof(*md5sig), gfp); + if (!md5sig) + return -ENOMEM; + + sk_gso_disable(sk); + INIT_HLIST_HEAD(&md5sig->head); + rcu_assign_pointer(tp->md5sig_info, md5sig); + return 0; +} + /* This can be called on a newly created socket, from other files */ -int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, - int family, u8 prefixlen, int l3index, u8 flags, - const u8 *newkey, u8 newkeylen, gfp_t gfp) +static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, u8 flags, + const u8 *newkey, u8 newkeylen, gfp_t gfp) { /* Add Key to the list */ struct tcp_md5sig_key *key; @@ -1193,15 +1208,6 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); - if (!md5sig) { - md5sig = kmalloc(sizeof(*md5sig), gfp); - if (!md5sig) - return -ENOMEM; - - sk_gso_disable(sk); - INIT_HLIST_HEAD(&md5sig->head); - rcu_assign_pointer(tp->md5sig_info, md5sig); - } key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO); if (!key) @@ -1223,8 +1229,59 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, hlist_add_head_rcu(&key->node, &md5sig->head); return 0; } + +int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, u8 flags, + const u8 *newkey, u8 newkeylen) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) { + if (tcp_md5sig_info_add(sk, GFP_KERNEL)) + return -ENOMEM; + + if (!static_branch_inc(&tcp_md5_needed.key)) { + struct tcp_md5sig_info *md5sig; + + md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); + rcu_assign_pointer(tp->md5sig_info, NULL); + kfree_rcu(md5sig, rcu); + return -EUSERS; + } + } + + return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags, + newkey, newkeylen, GFP_KERNEL); +} EXPORT_SYMBOL(tcp_md5_do_add); +int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, + struct tcp_md5sig_key *key) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) { + if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) + return -ENOMEM; + + if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) { + struct tcp_md5sig_info *md5sig; + + md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); + net_warn_ratelimited("Too many TCP-MD5 keys in the system\n"); + rcu_assign_pointer(tp->md5sig_info, NULL); + kfree_rcu(md5sig, rcu); + return -EUSERS; + } + } + + return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, + key->flags, key->key, key->keylen, + sk_gfp_mask(sk, GFP_ATOMIC)); +} +EXPORT_SYMBOL(tcp_md5_key_copy); + int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags) { @@ -1311,7 +1368,7 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, int optname, return -EINVAL; return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); } static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp, @@ -1562,14 +1619,8 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, addr = (union tcp_md5_addr *)&newinet->inet_daddr; key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET); if (key) { - /* - * We're using one, so create a matching key - * on the newsk structure. If we fail to get - * memory, then we end up not copying the key - * across. Shucks. - */ - tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index, key->flags, - key->key, key->keylen, GFP_ATOMIC); + if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key)) + goto put_and_exit; sk_gso_disable(newsk); } #endif @@ -2261,6 +2312,7 @@ void tcp_v4_destroy_sock(struct sock *sk) tcp_clear_md5_list(sk); kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu); tp->md5sig_info = NULL; + static_branch_slow_dec_deferred(&tcp_md5_needed); } #endif diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index c375f603a16c..e002f2e1d4f2 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -240,6 +240,40 @@ kill: } EXPORT_SYMBOL(tcp_timewait_state_process); +static void tcp_time_wait_init(struct sock *sk, struct tcp_timewait_sock *tcptw) +{ +#ifdef CONFIG_TCP_MD5SIG + const struct tcp_sock *tp = tcp_sk(sk); + struct tcp_md5sig_key *key; + + /* + * The timewait bucket does not have the key DB from the + * sock structure. We just make a quick copy of the + * md5 key being used (if indeed we are using one) + * so the timewait ack generating code has the key. + */ + tcptw->tw_md5_key = NULL; + if (!static_branch_unlikely(&tcp_md5_needed.key)) + return; + + key = tp->af_specific->md5_lookup(sk, sk); + if (key) { + tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC); + if (!tcptw->tw_md5_key) + return; + if (!tcp_alloc_md5sig_pool()) + goto out_free; + if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) + goto out_free; + } + return; +out_free: + WARN_ON_ONCE(1); + kfree(tcptw->tw_md5_key); + tcptw->tw_md5_key = NULL; +#endif +} + /* * Move a socket to time-wait or dead fin-wait-2 state. */ @@ -282,26 +316,7 @@ void tcp_time_wait(struct sock *sk, int state, int timeo) } #endif -#ifdef CONFIG_TCP_MD5SIG - /* - * The timewait bucket does not have the key DB from the - * sock structure. We just make a quick copy of the - * md5 key being used (if indeed we are using one) - * so the timewait ack generating code has the key. - */ - do { - tcptw->tw_md5_key = NULL; - if (static_branch_unlikely(&tcp_md5_needed)) { - struct tcp_md5sig_key *key; - - key = tp->af_specific->md5_lookup(sk, sk); - if (key) { - tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC); - BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); - } - } - } while (0); -#endif + tcp_time_wait_init(sk, tcptw); /* Get the TIME_WAIT timeout firing. */ if (timeo < rto) @@ -337,11 +352,13 @@ EXPORT_SYMBOL(tcp_time_wait); void tcp_twsk_destructor(struct sock *sk) { #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed)) { + if (static_branch_unlikely(&tcp_md5_needed.key)) { struct tcp_timewait_sock *twsk = tcp_twsk(sk); - if (twsk->tw_md5_key) + if (twsk->tw_md5_key) { kfree_rcu(twsk->tw_md5_key, rcu); + static_branch_slow_dec_deferred(&tcp_md5_needed); + } } #endif } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 894410dc9293..71d01cf3c13e 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -766,7 +766,7 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, *md5 = NULL; #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed) && + if (static_branch_unlikely(&tcp_md5_needed.key) && rcu_access_pointer(tp->md5sig_info)) { *md5 = tp->af_specific->md5_lookup(sk, sk); if (*md5) { @@ -922,7 +922,7 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb *md5 = NULL; #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed) && + if (static_branch_unlikely(&tcp_md5_needed.key) && rcu_access_pointer(tp->md5sig_info)) { *md5 = tp->af_specific->md5_lookup(sk, sk); if (*md5) { diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index aedde65e2268..1f01e15ca24f 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -387,7 +387,8 @@ static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, if (!pskb_may_pull(skb, sizeof(struct udphdr))) goto out; - if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + !skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) return __udp_gso_segment(skb, features, false); mss = skb_shinfo(skb)->gso_size; diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index e19507614f64..60fd91bb5171 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -920,6 +920,9 @@ int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, if (err < 0) goto fail; + /* We prevent @rt from being freed. */ + rcu_read_lock(); + for (;;) { /* Prepare header of the next frame, * before previous one went down. */ @@ -943,6 +946,7 @@ int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, if (err == 0) { IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), IPSTATS_MIB_FRAGOKS); + rcu_read_unlock(); return 0; } @@ -950,6 +954,7 @@ int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), IPSTATS_MIB_FRAGFAILS); + rcu_read_unlock(); return err; slow_path_clean: diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index f52b6f271a24..11b736a76bd7 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -665,12 +665,11 @@ static int tcp_v6_parse_md5_keys(struct sock *sk, int optname, if (ipv6_addr_v4mapped(&sin6->sin6_addr)) return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3], AF_INET, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, - GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr, AF_INET6, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); } static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp, @@ -1365,14 +1364,14 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * /* Copy over the MD5 key from the original socket */ key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index); if (key) { - /* We're using one, so create a matching key - * on the newsk structure. If we fail to get - * memory, then we end up not copying the key - * across. Shucks. - */ - tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newsk->sk_v6_daddr, - AF_INET6, 128, l3index, key->flags, key->key, key->keylen, - sk_gfp_mask(sk, GFP_ATOMIC)); + const union tcp_md5_addr *addr; + + addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr; + if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) { + inet_csk_prepare_forced_close(newsk); + tcp_done(newsk); + goto out; + } } #endif diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index e0e10f6bcdc1..c39c1e32f980 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -42,7 +42,8 @@ static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, if (!pskb_may_pull(skb, sizeof(struct udphdr))) goto out; - if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + !skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) return __udp_gso_segment(skb, features, true); mss = skb_shinfo(skb)->gso_size; diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index c848fe04dd44..8f9a2ab502b3 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -576,7 +576,7 @@ static struct ieee80211_key * ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id, u8 key_idx, bool pairwise, const u8 *mac_addr) { - struct ieee80211_local *local = sdata->local; + struct ieee80211_local *local __maybe_unused = sdata->local; struct ieee80211_link_data *link = &sdata->deflink; struct ieee80211_key *key; diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 7c4ce716c939..d49a5906a943 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -1849,8 +1849,7 @@ static int ieee80211_runtime_change_iftype(struct ieee80211_sub_if_data *sdata, ieee80211_stop_vif_queues(local, sdata, IEEE80211_QUEUE_STOP_REASON_IFTYPE_CHANGE); - synchronize_net(); - + /* do_stop will synchronize_rcu() first thing */ ieee80211_do_stop(sdata, false); ieee80211_teardown_sdata(sdata); @@ -2179,6 +2178,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name, ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; ndev->hw_features |= ndev->features & MAC80211_SUPPORTED_FEATURES_TX; + sdata->vif.netdev_features = local->hw.netdev_features; netdev_set_default_ethtool_ops(ndev, &ieee80211_ethtool_ops); @@ -2195,6 +2195,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name, ret = cfg80211_register_netdevice(ndev); if (ret) { + ieee80211_if_free(ndev); free_netdev(ndev); return ret; } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index a804e0220ed7..0aee2392dd29 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -3932,7 +3932,6 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, struct ieee80211_elems_parse_params parse_params = { .start = elem_start, .len = elem_len, - .bss = cbss, .link_id = link_id == assoc_data->assoc_link_id ? -1 : link_id, .from_ap = true, }; @@ -4017,6 +4016,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, parse_params.start = bss_ies->data; parse_params.len = bss_ies->len; + parse_params.bss = cbss; bss_elems = ieee802_11_parse_elems_full(&parse_params); if (!bss_elems) { ret = false; diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index c28c6fbf786e..7e3ab6e1b28f 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -2403,7 +2403,6 @@ static int ieee80211_802_1x_port_control(struct ieee80211_rx_data *rx) static int ieee80211_drop_unencrypted(struct ieee80211_rx_data *rx, __le16 fc) { - struct ieee80211_hdr *hdr = (void *)rx->skb->data; struct sk_buff *skb = rx->skb; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); @@ -2414,31 +2413,6 @@ static int ieee80211_drop_unencrypted(struct ieee80211_rx_data *rx, __le16 fc) if (status->flag & RX_FLAG_DECRYPTED) return 0; - /* check mesh EAPOL frames first */ - if (unlikely(rx->sta && ieee80211_vif_is_mesh(&rx->sdata->vif) && - ieee80211_is_data(fc))) { - struct ieee80211s_hdr *mesh_hdr; - u16 hdr_len = ieee80211_hdrlen(fc); - u16 ethertype_offset; - __be16 ethertype; - - if (!ether_addr_equal(hdr->addr1, rx->sdata->vif.addr)) - goto drop_check; - - /* make sure fixed part of mesh header is there, also checks skb len */ - if (!pskb_may_pull(rx->skb, hdr_len + 6)) - goto drop_check; - - mesh_hdr = (struct ieee80211s_hdr *)(skb->data + hdr_len); - ethertype_offset = hdr_len + ieee80211_get_mesh_hdrlen(mesh_hdr) + - sizeof(rfc1042_header); - - if (skb_copy_bits(rx->skb, ethertype_offset, ðertype, 2) == 0 && - ethertype == rx->sdata->control_port_protocol) - return 0; - } - -drop_check: /* Drop unencrypted frames if key is set. */ if (unlikely(!ieee80211_has_protected(fc) && !ieee80211_is_any_nullfunc(fc) && @@ -2892,8 +2866,16 @@ ieee80211_rx_h_mesh_fwding(struct ieee80211_rx_data *rx) hdr = (struct ieee80211_hdr *) skb->data; mesh_hdr = (struct ieee80211s_hdr *) (skb->data + hdrlen); - if (ieee80211_drop_unencrypted(rx, hdr->frame_control)) - return RX_DROP_MONITOR; + if (ieee80211_drop_unencrypted(rx, hdr->frame_control)) { + int offset = hdrlen + ieee80211_get_mesh_hdrlen(mesh_hdr) + + sizeof(rfc1042_header); + __be16 ethertype; + + if (!ether_addr_equal(hdr->addr1, rx->sdata->vif.addr) || + skb_copy_bits(rx->skb, offset, ðertype, 2) != 0 || + ethertype != rx->sdata->control_port_protocol) + return RX_DROP_MONITOR; + } /* frame is in RMC, don't forward */ if (ieee80211_is_data(hdr->frame_control) && diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 165ac0711d71..2171cd1ca807 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -1343,7 +1343,7 @@ static struct txq_info *ieee80211_get_txq(struct ieee80211_local *local, return NULL; txq = sta->sta.txq[tid]; - } else if (vif) { + } else { txq = vif->txq; } @@ -1355,7 +1355,11 @@ static struct txq_info *ieee80211_get_txq(struct ieee80211_local *local, static void ieee80211_set_skb_enqueue_time(struct sk_buff *skb) { - IEEE80211_SKB_CB(skb)->control.enqueue_time = codel_get_time(); + struct sk_buff *next; + codel_time_t now = codel_get_time(); + + skb_list_walk_safe(skb, skb, next) + IEEE80211_SKB_CB(skb)->control.enqueue_time = now; } static u32 codel_skb_len_func(const struct sk_buff *skb) @@ -3578,55 +3582,79 @@ ieee80211_xmit_fast_finish(struct ieee80211_sub_if_data *sdata, return TX_CONTINUE; } -static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, - struct sta_info *sta, - struct ieee80211_fast_tx *fast_tx, - struct sk_buff *skb) +static netdev_features_t +ieee80211_sdata_netdev_features(struct ieee80211_sub_if_data *sdata) { - struct ieee80211_local *local = sdata->local; - u16 ethertype = (skb->data[12] << 8) | skb->data[13]; - int extra_head = fast_tx->hdr_len - (ETH_HLEN - 2); - int hw_headroom = sdata->local->hw.extra_tx_headroom; - struct ethhdr eth; - struct ieee80211_tx_info *info; - struct ieee80211_hdr *hdr = (void *)fast_tx->hdr; - struct ieee80211_tx_data tx; - ieee80211_tx_result r; - struct tid_ampdu_tx *tid_tx = NULL; - u8 tid = IEEE80211_NUM_TIDS; + if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN) + return sdata->vif.netdev_features; - /* control port protocol needs a lot of special handling */ - if (cpu_to_be16(ethertype) == sdata->control_port_protocol) - return false; + if (!sdata->bss) + return 0; - /* only RFC 1042 SNAP */ - if (ethertype < ETH_P_802_3_MIN) - return false; + sdata = container_of(sdata->bss, struct ieee80211_sub_if_data, u.ap); + return sdata->vif.netdev_features; +} - /* don't handle TX status request here either */ - if (skb->sk && skb_shinfo(skb)->tx_flags & SKBTX_WIFI_STATUS) - return false; +static struct sk_buff * +ieee80211_tx_skb_fixup(struct sk_buff *skb, netdev_features_t features) +{ + if (skb_is_gso(skb)) { + struct sk_buff *segs; - if (hdr->frame_control & cpu_to_le16(IEEE80211_STYPE_QOS_DATA)) { - tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; - tid_tx = rcu_dereference(sta->ampdu_mlme.tid_tx[tid]); - if (tid_tx) { - if (!test_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state)) - return false; - if (tid_tx->timeout) - tid_tx->last_tx = jiffies; - } + segs = skb_gso_segment(skb, features); + if (!segs) + return skb; + if (IS_ERR(segs)) + goto free; + + consume_skb(skb); + return segs; } - /* after this point (skb is modified) we cannot return false */ + if (skb_needs_linearize(skb, features) && __skb_linearize(skb)) + goto free; + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + int ofs = skb_checksum_start_offset(skb); + + if (skb->encapsulation) + skb_set_inner_transport_header(skb, ofs); + else + skb_set_transport_header(skb, ofs); + + if (skb_csum_hwoffload_help(skb, features)) + goto free; + } + + skb_mark_not_on_list(skb); + return skb; + +free: + kfree_skb(skb); + return NULL; +} + +static void __ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, + struct sta_info *sta, + struct ieee80211_fast_tx *fast_tx, + struct sk_buff *skb, u8 tid, bool ampdu) +{ + struct ieee80211_local *local = sdata->local; + struct ieee80211_hdr *hdr = (void *)fast_tx->hdr; + struct ieee80211_tx_info *info; + struct ieee80211_tx_data tx; + ieee80211_tx_result r; + int hw_headroom = sdata->local->hw.extra_tx_headroom; + int extra_head = fast_tx->hdr_len - (ETH_HLEN - 2); + struct ethhdr eth; skb = skb_share_check(skb, GFP_ATOMIC); if (unlikely(!skb)) - return true; + return; if ((hdr->frame_control & cpu_to_le16(IEEE80211_STYPE_QOS_DATA)) && ieee80211_amsdu_aggregate(sdata, sta, fast_tx, skb)) - return true; + return; /* will not be crypto-handled beyond what we do here, so use false * as the may-encrypt argument for the resize to not account for @@ -3635,10 +3663,8 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, if (unlikely(ieee80211_skb_resize(sdata, skb, max_t(int, extra_head + hw_headroom - skb_headroom(skb), 0), - ENCRYPT_NO))) { - kfree_skb(skb); - return true; - } + ENCRYPT_NO))) + goto free; memcpy(ð, skb->data, ETH_HLEN - 2); hdr = skb_push(skb, extra_head); @@ -3652,7 +3678,7 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, info->control.vif = &sdata->vif; info->flags = IEEE80211_TX_CTL_FIRST_FRAGMENT | IEEE80211_TX_CTL_DONTFRAG | - (tid_tx ? IEEE80211_TX_CTL_AMPDU : 0); + (ampdu ? IEEE80211_TX_CTL_AMPDU : 0); info->control.flags = IEEE80211_TX_CTRL_FAST_XMIT | u32_encode_bits(IEEE80211_LINK_UNSPECIFIED, IEEE80211_TX_CTRL_MLO_LINK); @@ -3676,16 +3702,14 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, tx.key = fast_tx->key; if (ieee80211_queue_skb(local, sdata, sta, skb)) - return true; + return; tx.skb = skb; r = ieee80211_xmit_fast_finish(sdata, sta, fast_tx->pn_offs, fast_tx->key, &tx); tx.skb = NULL; - if (r == TX_DROP) { - kfree_skb(skb); - return true; - } + if (r == TX_DROP) + goto free; if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) sdata = container_of(sdata->bss, @@ -3693,6 +3717,56 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, __skb_queue_tail(&tx.skbs, skb); ieee80211_tx_frags(local, &sdata->vif, sta, &tx.skbs, false); + return; + +free: + kfree_skb(skb); +} + +static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, + struct sta_info *sta, + struct ieee80211_fast_tx *fast_tx, + struct sk_buff *skb) +{ + u16 ethertype = (skb->data[12] << 8) | skb->data[13]; + struct ieee80211_hdr *hdr = (void *)fast_tx->hdr; + struct tid_ampdu_tx *tid_tx = NULL; + struct sk_buff *next; + u8 tid = IEEE80211_NUM_TIDS; + + /* control port protocol needs a lot of special handling */ + if (cpu_to_be16(ethertype) == sdata->control_port_protocol) + return false; + + /* only RFC 1042 SNAP */ + if (ethertype < ETH_P_802_3_MIN) + return false; + + /* don't handle TX status request here either */ + if (skb->sk && skb_shinfo(skb)->tx_flags & SKBTX_WIFI_STATUS) + return false; + + if (hdr->frame_control & cpu_to_le16(IEEE80211_STYPE_QOS_DATA)) { + tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; + tid_tx = rcu_dereference(sta->ampdu_mlme.tid_tx[tid]); + if (tid_tx) { + if (!test_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state)) + return false; + if (tid_tx->timeout) + tid_tx->last_tx = jiffies; + } + } + + /* after this point (skb is modified) we cannot return false */ + skb = ieee80211_tx_skb_fixup(skb, ieee80211_sdata_netdev_features(sdata)); + if (!skb) + return true; + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + __ieee80211_xmit_fast(sdata, sta, fast_tx, skb, tid, tid_tx); + } + return true; } @@ -4192,31 +4266,14 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb, goto out; } - if (skb_is_gso(skb)) { - struct sk_buff *segs; - - segs = skb_gso_segment(skb, 0); - if (IS_ERR(segs)) { - goto out_free; - } else if (segs) { - consume_skb(skb); - skb = segs; - } - } else { - /* we cannot process non-linear frames on this path */ - if (skb_linearize(skb)) - goto out_free; - - /* the frame could be fragmented, software-encrypted, and other - * things so we cannot really handle checksum offload with it - - * fix it up in software before we handle anything else. - */ - if (skb->ip_summed == CHECKSUM_PARTIAL) { - skb_set_transport_header(skb, - skb_checksum_start_offset(skb)); - if (skb_checksum_help(skb)) - goto out_free; - } + /* the frame could be fragmented, software-encrypted, and other + * things so we cannot really handle checksum or GSO offload. + * fix it up in software before we handle anything else. + */ + skb = ieee80211_tx_skb_fixup(skb, 0); + if (!skb) { + len = 0; + goto out; } skb_list_walk_safe(skb, skb, next) { @@ -4434,9 +4491,11 @@ normal: return NETDEV_TX_OK; } -static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata, - struct sk_buff *skb, struct sta_info *sta, - bool txpending) + + +static bool __ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb, struct sta_info *sta, + bool txpending) { struct ieee80211_local *local = sdata->local; struct ieee80211_tx_control control = {}; @@ -4445,14 +4504,6 @@ static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata, unsigned long flags; int q = info->hw_queue; - if (sta) - sk_pacing_shift_update(skb->sk, local->hw.tx_sk_pacing_shift); - - ieee80211_tpt_led_trig_tx(local, skb->len); - - if (ieee80211_queue_skb(local, sdata, sta, skb)) - return true; - spin_lock_irqsave(&local->queue_stop_reason_lock, flags); if (local->queue_stop_reasons[q] || @@ -4479,6 +4530,26 @@ static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata, return true; } +static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb, struct sta_info *sta, + bool txpending) +{ + struct ieee80211_local *local = sdata->local; + struct sk_buff *next; + bool ret = true; + + if (ieee80211_queue_skb(local, sdata, sta, skb)) + return true; + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + if (!__ieee80211_tx_8023(sdata, skb, sta, txpending)) + ret = false; + } + + return ret; +} + static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, struct net_device *dev, struct sta_info *sta, struct ieee80211_key *key, struct sk_buff *skb) @@ -4486,9 +4557,13 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, struct ieee80211_tx_info *info; struct ieee80211_local *local = sdata->local; struct tid_ampdu_tx *tid_tx; + struct sk_buff *seg, *next; + unsigned int skbs = 0, len = 0; + u16 queue; u8 tid; - skb_set_queue_mapping(skb, ieee80211_select_queue(sdata, sta, skb)); + queue = ieee80211_select_queue(sdata, sta, skb); + skb_set_queue_mapping(skb, queue); if (unlikely(test_bit(SCAN_SW_SCANNING, &local->scanning)) && test_bit(SDATA_STATE_OFFCHANNEL, &sdata->state)) @@ -4498,9 +4573,6 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, if (unlikely(!skb)) return; - info = IEEE80211_SKB_CB(skb); - memset(info, 0, sizeof(*info)); - ieee80211_aggr_check(sdata, sta, skb); tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; @@ -4514,22 +4586,20 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, return; } - info->flags |= IEEE80211_TX_CTL_AMPDU; if (tid_tx->timeout) tid_tx->last_tx = jiffies; } - if (unlikely(skb->sk && - skb_shinfo(skb)->tx_flags & SKBTX_WIFI_STATUS)) - info->ack_frame_id = ieee80211_store_ack_skb(local, skb, - &info->flags, NULL); - - info->hw_queue = sdata->vif.hw_queue[skb_get_queue_mapping(skb)]; + skb = ieee80211_tx_skb_fixup(skb, ieee80211_sdata_netdev_features(sdata)); + if (!skb) + return; - dev_sw_netstats_tx_add(dev, 1, skb->len); + info = IEEE80211_SKB_CB(skb); + memset(info, 0, sizeof(*info)); + if (tid_tx) + info->flags |= IEEE80211_TX_CTL_AMPDU; - sta->deflink.tx_stats.bytes[skb_get_queue_mapping(skb)] += skb->len; - sta->deflink.tx_stats.packets[skb_get_queue_mapping(skb)]++; + info->hw_queue = sdata->vif.hw_queue[queue]; if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) sdata = container_of(sdata->bss, @@ -4541,6 +4611,24 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, if (key) info->control.hw_key = &key->conf; + skb_list_walk_safe(skb, seg, next) { + skbs++; + len += seg->len; + if (seg != skb) + memcpy(IEEE80211_SKB_CB(seg), info, sizeof(*info)); + } + + if (unlikely(skb->sk && + skb_shinfo(skb)->tx_flags & SKBTX_WIFI_STATUS)) + info->ack_frame_id = ieee80211_store_ack_skb(local, skb, + &info->flags, NULL); + + dev_sw_netstats_tx_add(dev, skbs, len); + sta->deflink.tx_stats.packets[queue] += skbs; + sta->deflink.tx_stats.bytes[queue] += len; + + ieee80211_tpt_led_trig_tx(local, len); + ieee80211_tx_8023(sdata, skb, sta, false); return; @@ -4582,6 +4670,7 @@ netdev_tx_t ieee80211_subif_start_xmit_8023(struct sk_buff *skb, key->conf.cipher == WLAN_CIPHER_SUITE_TKIP)) goto skip_offload; + sk_pacing_shift_update(skb->sk, sdata->local->hw.tx_sk_pacing_shift); ieee80211_8023_xmit(sdata, dev, sta, key, skb); goto out; @@ -4774,9 +4863,9 @@ static void __ieee80211_beacon_add_tim(struct ieee80211_sub_if_data *sdata, ps->dtim_count--; } - tim = pos = skb_put(skb, 6); + tim = pos = skb_put(skb, 5); *pos++ = WLAN_EID_TIM; - *pos++ = 4; + *pos++ = 3; *pos++ = ps->dtim_count; *pos++ = link_conf->dtim_period; @@ -4807,13 +4896,17 @@ static void __ieee80211_beacon_add_tim(struct ieee80211_sub_if_data *sdata, /* Bitmap control */ *pos++ = n1 | aid0; /* Part Virt Bitmap */ - skb_put(skb, n2 - n1); - memcpy(pos, ps->tim + n1, n2 - n1 + 1); + skb_put_data(skb, ps->tim + n1, n2 - n1 + 1); tim[1] = n2 - n1 + 4; } else { *pos++ = aid0; /* Bitmap control */ - *pos++ = 0; /* Part Virt Bitmap */ + + if (ieee80211_get_link_sband(link)->band != NL80211_BAND_S1GHZ) { + tim[1] = 4; + /* Part Virt Bitmap */ + skb_put_u8(skb, 0); + } } } diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index d9b50884d34e..ac0b28025fb0 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -254,7 +254,6 @@ ieee802154_check_concurrent_iface(struct ieee802154_sub_if_data *sdata, enum nl802154_iftype iftype) { struct ieee802154_local *local = sdata->local; - struct wpan_dev *wpan_dev = &sdata->wpan_dev; struct ieee802154_sub_if_data *nsdata; /* we hold the RTNL here so can safely walk the list */ @@ -262,13 +261,13 @@ ieee802154_check_concurrent_iface(struct ieee802154_sub_if_data *sdata, if (nsdata != sdata && ieee802154_sdata_running(nsdata)) { int ret; - /* TODO currently we don't support multiple node types - * we need to run skb_clone at rx path. Check if there - * exist really an use case if we need to support - * multiple node types at the same time. + /* TODO currently we don't support multiple node/coord + * types we need to run skb_clone at rx path. Check if + * there exist really an use case if we need to support + * multiple node/coord types at the same time. */ - if (wpan_dev->iftype == NL802154_IFTYPE_NODE && - nsdata->wpan_dev.iftype == NL802154_IFTYPE_NODE) + if (sdata->wpan_dev.iftype != NL802154_IFTYPE_MONITOR && + nsdata->wpan_dev.iftype != NL802154_IFTYPE_MONITOR) return -EBUSY; /* check all phy mac sublayer settings are the same. @@ -565,6 +564,7 @@ ieee802154_setup_sdata(struct ieee802154_sub_if_data *sdata, wpan_dev->short_addr = cpu_to_le16(IEEE802154_ADDR_BROADCAST); switch (type) { + case NL802154_IFTYPE_COORD: case NL802154_IFTYPE_NODE: ieee802154_be64_to_le64(&wpan_dev->extended_addr, sdata->dev->dev_addr); @@ -624,6 +624,7 @@ ieee802154_if_add(struct ieee802154_local *local, const char *name, ieee802154_le64_to_be64(ndev->perm_addr, &local->hw.phy->perm_extended_addr); switch (type) { + case NL802154_IFTYPE_COORD: case NL802154_IFTYPE_NODE: ndev->type = ARPHRD_IEEE802154; if (ieee802154_is_valid_extended_unicast_addr(extended_addr)) { @@ -650,6 +651,7 @@ ieee802154_if_add(struct ieee802154_local *local, const char *name, sdata->dev = ndev; sdata->wpan_dev.wpan_phy = local->hw.phy; sdata->local = local; + INIT_LIST_HEAD(&sdata->wpan_dev.list); /* setup type-dependent data */ ret = ieee802154_setup_sdata(sdata, type); diff --git a/net/mac802154/main.c b/net/mac802154/main.c index 40fab08df24b..3ed31daf7b9c 100644 --- a/net/mac802154/main.c +++ b/net/mac802154/main.c @@ -107,7 +107,7 @@ ieee802154_alloc_hw(size_t priv_data_len, const struct ieee802154_ops *ops) phy->supported.lbt = NL802154_SUPPORTED_BOOL_FALSE; /* always supported */ - phy->supported.iftypes = BIT(NL802154_IFTYPE_NODE); + phy->supported.iftypes = BIT(NL802154_IFTYPE_NODE) | BIT(NL802154_IFTYPE_COORD); return &local->hw; } diff --git a/net/mac802154/rx.c b/net/mac802154/rx.c index 0724aac8f48c..c2aae2a6d6a6 100644 --- a/net/mac802154/rx.c +++ b/net/mac802154/rx.c @@ -208,6 +208,7 @@ __ieee802154_rx_handle_packet(struct ieee802154_local *local, int ret; struct ieee802154_sub_if_data *sdata; struct ieee802154_hdr hdr; + struct sk_buff *skb2; ret = ieee802154_parse_frame_start(skb, &hdr); if (ret) { @@ -217,7 +218,7 @@ __ieee802154_rx_handle_packet(struct ieee802154_local *local, } list_for_each_entry_rcu(sdata, &local->interfaces, list) { - if (sdata->wpan_dev.iftype != NL802154_IFTYPE_NODE) + if (sdata->wpan_dev.iftype == NL802154_IFTYPE_MONITOR) continue; if (!ieee802154_sdata_running(sdata)) @@ -230,12 +231,12 @@ __ieee802154_rx_handle_packet(struct ieee802154_local *local, sdata->required_filtering == IEEE802154_FILTERING_4_FRAME_FIELDS) continue; - ieee802154_subif_frame(sdata, skb, &hdr); - skb = NULL; - break; + skb2 = skb_clone(skb, GFP_ATOMIC); + if (skb2) { + skb2->dev = sdata->dev; + ieee802154_subif_frame(sdata, skb2, &hdr); + } } - - kfree_skb(skb); } static void @@ -274,7 +275,7 @@ void ieee802154_rx(struct ieee802154_local *local, struct sk_buff *skb) WARN_ON_ONCE(softirq_count() == 0); if (local->suspended) - goto drop; + goto free_skb; /* TODO: When a transceiver omits the checksum here, we * add an own calculated one. This is currently an ugly @@ -292,20 +293,17 @@ void ieee802154_rx(struct ieee802154_local *local, struct sk_buff *skb) /* Level 1 filtering: Check the FCS by software when relevant */ if (local->hw.phy->filtering == IEEE802154_FILTERING_NONE) { crc = crc_ccitt(0, skb->data, skb->len); - if (crc) { - rcu_read_unlock(); + if (crc) goto drop; - } } /* remove crc */ skb_trim(skb, skb->len - 2); __ieee802154_rx_handle_packet(local, skb); - rcu_read_unlock(); - - return; drop: + rcu_read_unlock(); +free_skb: kfree_skb(skb); } diff --git a/net/mac802154/trace.h b/net/mac802154/trace.h index df855c33daf2..689396d6c76a 100644 --- a/net/mac802154/trace.h +++ b/net/mac802154/trace.h @@ -264,6 +264,31 @@ TRACE_EVENT(802154_drv_set_promiscuous_mode, BOOL_TO_STR(__entry->on)) ); +TRACE_EVENT(802154_new_scan_event, + TP_PROTO(struct ieee802154_coord_desc *desc), + TP_ARGS(desc), + TP_STRUCT__entry( + __field(__le16, pan_id) + __field(__le64, addr) + __field(u8, channel) + __field(u8, page) + ), + TP_fast_assign( + __entry->page = desc->page; + __entry->channel = desc->channel; + __entry->pan_id = desc->addr.pan_id; + __entry->addr = desc->addr.extended_addr; + ), + TP_printk("panid: %u, coord_addr: 0x%llx, page: %u, channel: %u", + __le16_to_cpu(__entry->pan_id), __le64_to_cpu(__entry->addr), + __entry->page, __entry->channel) +); + +DEFINE_EVENT(802154_new_scan_event, 802154_scan_event, + TP_PROTO(struct ieee802154_coord_desc *desc), + TP_ARGS(desc) +); + #endif /* !__MAC802154_DRIVER_TRACE || TRACE_HEADER_MULTI_READ */ #undef TRACE_INCLUDE_PATH diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c index d66fbd558263..2ea7eae43bdb 100644 --- a/net/mptcp/pm_netlink.c +++ b/net/mptcp/pm_netlink.c @@ -1029,6 +1029,8 @@ static int mptcp_pm_nl_create_listen_socket(struct sock *sk, if (err) return err; + mptcp_event_pm_listener(ssock->sk, MPTCP_EVENT_LISTENER_CREATED); + return 0; } @@ -1188,7 +1190,7 @@ static int mptcp_pm_parse_pm_addr_attr(struct nlattr *tb[], if (!tb[MPTCP_PM_ADDR_ATTR_FAMILY]) { if (!require_family) - return err; + return 0; NL_SET_ERR_MSG_ATTR(info->extack, attr, "missing family"); @@ -1222,7 +1224,7 @@ static int mptcp_pm_parse_pm_addr_attr(struct nlattr *tb[], if (tb[MPTCP_PM_ADDR_ATTR_PORT]) addr->port = htons(nla_get_u16(tb[MPTCP_PM_ADDR_ATTR_PORT])); - return err; + return 0; } int mptcp_pm_parse_addr(struct nlattr *attr, struct genl_info *info, @@ -2092,7 +2094,7 @@ void mptcp_event_addr_removed(const struct mptcp_sock *msk, uint8_t id) return; nla_put_failure: - kfree_skb(skb); + nlmsg_free(skb); } void mptcp_event_addr_announced(const struct sock *ssk, @@ -2149,7 +2151,59 @@ void mptcp_event_addr_announced(const struct sock *ssk, return; nla_put_failure: - kfree_skb(skb); + nlmsg_free(skb); +} + +void mptcp_event_pm_listener(const struct sock *ssk, + enum mptcp_event_type event) +{ + const struct inet_sock *issk = inet_sk(ssk); + struct net *net = sock_net(ssk); + struct nlmsghdr *nlh; + struct sk_buff *skb; + + if (!genl_has_listeners(&mptcp_genl_family, net, MPTCP_PM_EV_GRP_OFFSET)) + return; + + skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return; + + nlh = genlmsg_put(skb, 0, 0, &mptcp_genl_family, 0, event); + if (!nlh) + goto nla_put_failure; + + if (nla_put_u16(skb, MPTCP_ATTR_FAMILY, ssk->sk_family)) + goto nla_put_failure; + + if (nla_put_be16(skb, MPTCP_ATTR_SPORT, issk->inet_sport)) + goto nla_put_failure; + + switch (ssk->sk_family) { + case AF_INET: + if (nla_put_in_addr(skb, MPTCP_ATTR_SADDR4, issk->inet_saddr)) + goto nla_put_failure; + break; +#if IS_ENABLED(CONFIG_MPTCP_IPV6) + case AF_INET6: { + const struct ipv6_pinfo *np = inet6_sk(ssk); + + if (nla_put_in6_addr(skb, MPTCP_ATTR_SADDR6, &np->saddr)) + goto nla_put_failure; + break; + } +#endif + default: + WARN_ON_ONCE(1); + goto nla_put_failure; + } + + genlmsg_end(skb, nlh); + mptcp_nl_mcast_send(net, skb, GFP_KERNEL); + return; + +nla_put_failure: + nlmsg_free(skb); } void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, @@ -2197,6 +2251,9 @@ void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, if (mptcp_event_sub_closed(skb, msk, ssk) < 0) goto nla_put_failure; break; + case MPTCP_EVENT_LISTENER_CREATED: + case MPTCP_EVENT_LISTENER_CLOSED: + break; } genlmsg_end(skb, nlh); @@ -2204,7 +2261,7 @@ void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, return; nla_put_failure: - kfree_skb(skb); + nlmsg_free(skb); } static const struct genl_small_ops mptcp_pm_ops[] = { diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index b0d387be500a..f6f93957275b 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -2355,6 +2355,7 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, tcp_set_state(ssk, TCP_CLOSE); mptcp_subflow_queue_clean(ssk); inet_csk_listen_stop(ssk); + mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED); } __tcp_close(ssk, 0); @@ -3647,6 +3648,8 @@ static int mptcp_listen(struct socket *sock, int backlog) if (!err) mptcp_copy_inaddrs(sock->sk, ssock->sk); + mptcp_event_pm_listener(ssock->sk, MPTCP_EVENT_LISTENER_CREATED); + unlock: release_sock(sock->sk); return err; diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 8b4379a2cd85..955fb3d88eb3 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -839,6 +839,8 @@ void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, const struct sock *ssk, gfp_t gfp); void mptcp_event_addr_announced(const struct sock *ssk, const struct mptcp_addr_info *info); void mptcp_event_addr_removed(const struct mptcp_sock *msk, u8 id); +void mptcp_event_pm_listener(const struct sock *ssk, + enum mptcp_event_type event); bool mptcp_userspace_pm_active(const struct mptcp_sock *msk); void mptcp_fastopen_gen_msk_ackseq(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow, diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index a47423ebb33a..d4b1e6ec1b36 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -740,7 +740,7 @@ static int mptcp_setsockopt_v4_set_tos(struct mptcp_sock *msk, int optname, } release_sock(sk); - return err; + return 0; } static int mptcp_setsockopt_v4(struct mptcp_sock *msk, int optname, diff --git a/net/ncsi/ncsi-cmd.c b/net/ncsi/ncsi-cmd.c index dda8b76b7798..fd2236ee9a79 100644 --- a/net/ncsi/ncsi-cmd.c +++ b/net/ncsi/ncsi-cmd.c @@ -228,7 +228,8 @@ static int ncsi_cmd_handler_oem(struct sk_buff *skb, len += max(payload, padding_bytes); cmd = skb_put_zero(skb, len); - memcpy(&cmd->mfr_id, nca->data, nca->payload); + unsafe_memcpy(&cmd->mfr_id, nca->data, nca->payload, + /* skb allocated with enough to load the payload */); ncsi_cmd_build_header(&cmd->cmd.common, nca); return 0; diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig index 0846bd75b1da..f71b41c7ce2f 100644 --- a/net/netfilter/Kconfig +++ b/net/netfilter/Kconfig @@ -459,6 +459,9 @@ config NF_NAT_REDIRECT config NF_NAT_MASQUERADE bool +config NF_NAT_OVS + bool + config NETFILTER_SYNPROXY tristate diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index 1d4db1943936..3754eb06fb41 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -59,6 +59,7 @@ obj-$(CONFIG_NF_LOG_SYSLOG) += nf_log_syslog.o obj-$(CONFIG_NF_NAT) += nf_nat.o nf_nat-$(CONFIG_NF_NAT_REDIRECT) += nf_nat_redirect.o nf_nat-$(CONFIG_NF_NAT_MASQUERADE) += nf_nat_masquerade.o +nf_nat-$(CONFIG_NF_NAT_OVS) += nf_nat_ovs.o ifeq ($(CONFIG_NF_NAT),m) nf_nat-$(CONFIG_DEBUG_INFO_BTF_MODULES) += nf_nat_bpf.o diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index b96338b4bf36..5c3cf0834af0 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -887,7 +887,7 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct) zone = nf_ct_zone(ct); if (!nf_ct_ext_valid_pre(ct->ext)) { - NF_CT_STAT_INC(net, insert_failed); + NF_CT_STAT_INC_ATOMIC(net, insert_failed); return -ETIMEDOUT; } @@ -934,7 +934,7 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct) if (!nf_ct_ext_valid_post(ct->ext)) { nf_ct_kill(ct); - NF_CT_STAT_INC(net, drop); + NF_CT_STAT_INC_ATOMIC(net, drop); return -ETIMEDOUT; } @@ -1271,7 +1271,7 @@ chaintoolong: */ if (!nf_ct_ext_valid_post(ct->ext)) { nf_ct_kill(ct); - NF_CT_STAT_INC(net, drop); + NF_CT_STAT_INC_ATOMIC(net, drop); return NF_DROP; } diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index d71150a40fb0..1286ae7d4609 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -328,8 +328,13 @@ nla_put_failure: } #ifdef CONFIG_NF_CONNTRACK_MARK -static int ctnetlink_dump_mark(struct sk_buff *skb, u32 mark) +static int ctnetlink_dump_mark(struct sk_buff *skb, const struct nf_conn *ct) { + u32 mark = READ_ONCE(ct->mark); + + if (!mark) + return 0; + if (nla_put_be32(skb, CTA_MARK, htonl(mark))) goto nla_put_failure; return 0; @@ -543,7 +548,7 @@ static int ctnetlink_dump_extinfo(struct sk_buff *skb, static int ctnetlink_dump_info(struct sk_buff *skb, struct nf_conn *ct) { if (ctnetlink_dump_status(skb, ct) < 0 || - ctnetlink_dump_mark(skb, READ_ONCE(ct->mark)) < 0 || + ctnetlink_dump_mark(skb, ct) < 0 || ctnetlink_dump_secctx(skb, ct) < 0 || ctnetlink_dump_id(skb, ct) < 0 || ctnetlink_dump_use(skb, ct) < 0 || @@ -722,7 +727,6 @@ ctnetlink_conntrack_event(unsigned int events, const struct nf_ct_event *item) struct sk_buff *skb; unsigned int type; unsigned int flags = 0, group; - u32 mark; int err; if (events & (1 << IPCT_DESTROY)) { @@ -827,9 +831,8 @@ ctnetlink_conntrack_event(unsigned int events, const struct nf_ct_event *item) } #ifdef CONFIG_NF_CONNTRACK_MARK - mark = READ_ONCE(ct->mark); - if ((events & (1 << IPCT_MARK) || mark) && - ctnetlink_dump_mark(skb, mark) < 0) + if (events & (1 << IPCT_MARK) && + ctnetlink_dump_mark(skb, ct) < 0) goto nla_put_failure; #endif nlmsg_end(skb, nlh); @@ -2671,7 +2674,6 @@ static int __ctnetlink_glue_build(struct sk_buff *skb, struct nf_conn *ct) { const struct nf_conntrack_zone *zone; struct nlattr *nest_parms; - u32 mark; zone = nf_ct_zone(ct); @@ -2733,8 +2735,7 @@ static int __ctnetlink_glue_build(struct sk_buff *skb, struct nf_conn *ct) goto nla_put_failure; #ifdef CONFIG_NF_CONNTRACK_MARK - mark = READ_ONCE(ct->mark); - if (mark && ctnetlink_dump_mark(skb, mark) < 0) + if (ctnetlink_dump_mark(skb, ct) < 0) goto nla_put_failure; #endif if (ctnetlink_dump_labels(skb, ct) < 0) diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 00b522890d77..0fdcdb2c9ae4 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -997,13 +997,13 @@ static void flow_offload_queue_work(struct flow_offload_work *offload) struct net *net = read_pnet(&offload->flowtable->net); if (offload->cmd == FLOW_CLS_REPLACE) { - NF_FLOW_TABLE_STAT_INC(net, count_wq_add); + NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add); queue_work(nf_flow_offload_add_wq, &offload->work); } else if (offload->cmd == FLOW_CLS_DESTROY) { - NF_FLOW_TABLE_STAT_INC(net, count_wq_del); + NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del); queue_work(nf_flow_offload_del_wq, &offload->work); } else { - NF_FLOW_TABLE_STAT_INC(net, count_wq_stats); + NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats); queue_work(nf_flow_offload_stats_wq, &offload->work); } } diff --git a/net/netfilter/nf_nat_ovs.c b/net/netfilter/nf_nat_ovs.c new file mode 100644 index 000000000000..551abd2da614 --- /dev/null +++ b/net/netfilter/nf_nat_ovs.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Support nat functions for openvswitch and used by OVS and TC conntrack. */ + +#include <net/netfilter/nf_nat.h> + +/* Modelled after nf_nat_ipv[46]_fn(). + * range is only used for new, uninitialized NAT state. + * Returns either NF_ACCEPT or NF_DROP. + */ +static int nf_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, + enum ip_conntrack_info ctinfo, int *action, + const struct nf_nat_range2 *range, + enum nf_nat_manip_type maniptype) +{ + __be16 proto = skb_protocol(skb, true); + int hooknum, err = NF_ACCEPT; + + /* See HOOK2MANIP(). */ + if (maniptype == NF_NAT_MANIP_SRC) + hooknum = NF_INET_LOCAL_IN; /* Source NAT */ + else + hooknum = NF_INET_LOCAL_OUT; /* Destination NAT */ + + switch (ctinfo) { + case IP_CT_RELATED: + case IP_CT_RELATED_REPLY: + if (proto == htons(ETH_P_IP) && + ip_hdr(skb)->protocol == IPPROTO_ICMP) { + if (!nf_nat_icmp_reply_translation(skb, ct, ctinfo, + hooknum)) + err = NF_DROP; + goto out; + } else if (IS_ENABLED(CONFIG_IPV6) && proto == htons(ETH_P_IPV6)) { + __be16 frag_off; + u8 nexthdr = ipv6_hdr(skb)->nexthdr; + int hdrlen = ipv6_skip_exthdr(skb, + sizeof(struct ipv6hdr), + &nexthdr, &frag_off); + + if (hdrlen >= 0 && nexthdr == IPPROTO_ICMPV6) { + if (!nf_nat_icmpv6_reply_translation(skb, ct, + ctinfo, + hooknum, + hdrlen)) + err = NF_DROP; + goto out; + } + } + /* Non-ICMP, fall thru to initialize if needed. */ + fallthrough; + case IP_CT_NEW: + /* Seen it before? This can happen for loopback, retrans, + * or local packets. + */ + if (!nf_nat_initialized(ct, maniptype)) { + /* Initialize according to the NAT action. */ + err = (range && range->flags & NF_NAT_RANGE_MAP_IPS) + /* Action is set up to establish a new + * mapping. + */ + ? nf_nat_setup_info(ct, range, maniptype) + : nf_nat_alloc_null_binding(ct, hooknum); + if (err != NF_ACCEPT) + goto out; + } + break; + + case IP_CT_ESTABLISHED: + case IP_CT_ESTABLISHED_REPLY: + break; + + default: + err = NF_DROP; + goto out; + } + + err = nf_nat_packet(ct, ctinfo, hooknum, skb); + if (err == NF_ACCEPT) + *action |= BIT(maniptype); +out: + return err; +} + +int nf_ct_nat(struct sk_buff *skb, struct nf_conn *ct, + enum ip_conntrack_info ctinfo, int *action, + const struct nf_nat_range2 *range, bool commit) +{ + enum nf_nat_manip_type maniptype; + int err, ct_action = *action; + + *action = 0; + + /* Add NAT extension if not confirmed yet. */ + if (!nf_ct_is_confirmed(ct) && !nf_ct_nat_ext_add(ct)) + return NF_DROP; /* Can't NAT. */ + + if (ctinfo != IP_CT_NEW && (ct->status & IPS_NAT_MASK) && + (ctinfo != IP_CT_RELATED || commit)) { + /* NAT an established or related connection like before. */ + if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY) + /* This is the REPLY direction for a connection + * for which NAT was applied in the forward + * direction. Do the reverse NAT. + */ + maniptype = ct->status & IPS_SRC_NAT + ? NF_NAT_MANIP_DST : NF_NAT_MANIP_SRC; + else + maniptype = ct->status & IPS_SRC_NAT + ? NF_NAT_MANIP_SRC : NF_NAT_MANIP_DST; + } else if (ct_action & BIT(NF_NAT_MANIP_SRC)) { + maniptype = NF_NAT_MANIP_SRC; + } else if (ct_action & BIT(NF_NAT_MANIP_DST)) { + maniptype = NF_NAT_MANIP_DST; + } else { + return NF_ACCEPT; + } + + err = nf_ct_nat_execute(skb, ct, ctinfo, action, range, maniptype); + if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) { + if (ct->status & IPS_SRC_NAT) { + if (maniptype == NF_NAT_MANIP_SRC) + maniptype = NF_NAT_MANIP_DST; + else + maniptype = NF_NAT_MANIP_SRC; + + err = nf_ct_nat_execute(skb, ct, ctinfo, action, range, + maniptype); + } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) { + err = nf_ct_nat_execute(skb, ct, ctinfo, action, NULL, + NF_NAT_MANIP_SRC); + } + } + return err; +} +EXPORT_SYMBOL_GPL(nf_ct_nat); diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 4f9299b9dcdd..06d46d182634 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -1162,6 +1162,7 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, struct nft_pipapo_match *m = priv->clone; u8 genmask = nft_genmask_next(net); struct nft_pipapo_field *f; + const u8 *start_p, *end_p; int i, bsize_max, err = 0; if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END)) @@ -1202,9 +1203,9 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, } /* Validate */ + start_p = start; + end_p = end; nft_pipapo_for_each_field(f, i, m) { - const u8 *start_p = start, *end_p = end; - if (f->rules >= (unsigned long)NFT_PIPAPO_RULE0_MAX) return -ENOSPC; diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index 282c51051dcc..994a0a1efb58 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -240,6 +240,8 @@ static int nci_add_new_protocol(struct nci_dev *ndev, target->sens_res = nfca_poll->sens_res; target->sel_res = nfca_poll->sel_res; target->nfcid1_len = nfca_poll->nfcid1_len; + if (target->nfcid1_len > ARRAY_SIZE(target->nfcid1)) + return -EPROTO; if (target->nfcid1_len > 0) { memcpy(target->nfcid1, nfca_poll->nfcid1, target->nfcid1_len); @@ -248,6 +250,8 @@ static int nci_add_new_protocol(struct nci_dev *ndev, nfcb_poll = (struct rf_tech_specific_params_nfcb_poll *)params; target->sensb_res_len = nfcb_poll->sensb_res_len; + if (target->sensb_res_len > ARRAY_SIZE(target->sensb_res)) + return -EPROTO; if (target->sensb_res_len > 0) { memcpy(target->sensb_res, nfcb_poll->sensb_res, target->sensb_res_len); @@ -256,6 +260,8 @@ static int nci_add_new_protocol(struct nci_dev *ndev, nfcf_poll = (struct rf_tech_specific_params_nfcf_poll *)params; target->sensf_res_len = nfcf_poll->sensf_res_len; + if (target->sensf_res_len > ARRAY_SIZE(target->sensf_res)) + return -EPROTO; if (target->sensf_res_len > 0) { memcpy(target->sensf_res, nfcf_poll->sensf_res, target->sensf_res_len); diff --git a/net/openvswitch/Kconfig b/net/openvswitch/Kconfig index 15bd287f5cbd..747d537a3f06 100644 --- a/net/openvswitch/Kconfig +++ b/net/openvswitch/Kconfig @@ -15,6 +15,7 @@ config OPENVSWITCH select NET_MPLS_GSO select DST_CACHE select NET_NSH + select NF_NAT_OVS if NF_NAT help Open vSwitch is a multilayer Ethernet switch targeted at virtualized environments. In addition to supporting a variety of features diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index d78f0fc4337d..c8b137649ca4 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -726,147 +726,27 @@ static void ovs_nat_update_key(struct sw_flow_key *key, } } -/* Modelled after nf_nat_ipv[46]_fn(). - * range is only used for new, uninitialized NAT state. - * Returns either NF_ACCEPT or NF_DROP. - */ -static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - const struct nf_nat_range2 *range, - enum nf_nat_manip_type maniptype, struct sw_flow_key *key) -{ - int hooknum, nh_off, err = NF_ACCEPT; - - nh_off = skb_network_offset(skb); - skb_pull_rcsum(skb, nh_off); - - /* See HOOK2MANIP(). */ - if (maniptype == NF_NAT_MANIP_SRC) - hooknum = NF_INET_LOCAL_IN; /* Source NAT */ - else - hooknum = NF_INET_LOCAL_OUT; /* Destination NAT */ - - switch (ctinfo) { - case IP_CT_RELATED: - case IP_CT_RELATED_REPLY: - if (IS_ENABLED(CONFIG_NF_NAT) && - skb->protocol == htons(ETH_P_IP) && - ip_hdr(skb)->protocol == IPPROTO_ICMP) { - if (!nf_nat_icmp_reply_translation(skb, ct, ctinfo, - hooknum)) - err = NF_DROP; - goto push; - } else if (IS_ENABLED(CONFIG_IPV6) && - skb->protocol == htons(ETH_P_IPV6)) { - __be16 frag_off; - u8 nexthdr = ipv6_hdr(skb)->nexthdr; - int hdrlen = ipv6_skip_exthdr(skb, - sizeof(struct ipv6hdr), - &nexthdr, &frag_off); - - if (hdrlen >= 0 && nexthdr == IPPROTO_ICMPV6) { - if (!nf_nat_icmpv6_reply_translation(skb, ct, - ctinfo, - hooknum, - hdrlen)) - err = NF_DROP; - goto push; - } - } - /* Non-ICMP, fall thru to initialize if needed. */ - fallthrough; - case IP_CT_NEW: - /* Seen it before? This can happen for loopback, retrans, - * or local packets. - */ - if (!nf_nat_initialized(ct, maniptype)) { - /* Initialize according to the NAT action. */ - err = (range && range->flags & NF_NAT_RANGE_MAP_IPS) - /* Action is set up to establish a new - * mapping. - */ - ? nf_nat_setup_info(ct, range, maniptype) - : nf_nat_alloc_null_binding(ct, hooknum); - if (err != NF_ACCEPT) - goto push; - } - break; - - case IP_CT_ESTABLISHED: - case IP_CT_ESTABLISHED_REPLY: - break; - - default: - err = NF_DROP; - goto push; - } - - err = nf_nat_packet(ct, ctinfo, hooknum, skb); -push: - skb_push_rcsum(skb, nh_off); - - /* Update the flow key if NAT successful. */ - if (err == NF_ACCEPT) - ovs_nat_update_key(key, skb, maniptype); - - return err; -} - /* Returns NF_DROP if the packet should be dropped, NF_ACCEPT otherwise. */ static int ovs_ct_nat(struct net *net, struct sw_flow_key *key, const struct ovs_conntrack_info *info, struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo) { - enum nf_nat_manip_type maniptype; - int err; + int err, action = 0; - /* Add NAT extension if not confirmed yet. */ - if (!nf_ct_is_confirmed(ct) && !nf_ct_nat_ext_add(ct)) - return NF_ACCEPT; /* Can't NAT. */ + if (!(info->nat & OVS_CT_NAT)) + return NF_ACCEPT; + if (info->nat & OVS_CT_SRC_NAT) + action |= BIT(NF_NAT_MANIP_SRC); + if (info->nat & OVS_CT_DST_NAT) + action |= BIT(NF_NAT_MANIP_DST); - /* Determine NAT type. - * Check if the NAT type can be deduced from the tracked connection. - * Make sure new expected connections (IP_CT_RELATED) are NATted only - * when committing. - */ - if (info->nat & OVS_CT_NAT && ctinfo != IP_CT_NEW && - ct->status & IPS_NAT_MASK && - (ctinfo != IP_CT_RELATED || info->commit)) { - /* NAT an established or related connection like before. */ - if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY) - /* This is the REPLY direction for a connection - * for which NAT was applied in the forward - * direction. Do the reverse NAT. - */ - maniptype = ct->status & IPS_SRC_NAT - ? NF_NAT_MANIP_DST : NF_NAT_MANIP_SRC; - else - maniptype = ct->status & IPS_SRC_NAT - ? NF_NAT_MANIP_SRC : NF_NAT_MANIP_DST; - } else if (info->nat & OVS_CT_SRC_NAT) { - maniptype = NF_NAT_MANIP_SRC; - } else if (info->nat & OVS_CT_DST_NAT) { - maniptype = NF_NAT_MANIP_DST; - } else { - return NF_ACCEPT; /* Connection is not NATed. */ - } - err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, maniptype, key); - - if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) { - if (ct->status & IPS_SRC_NAT) { - if (maniptype == NF_NAT_MANIP_SRC) - maniptype = NF_NAT_MANIP_DST; - else - maniptype = NF_NAT_MANIP_SRC; - - err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, - maniptype, key); - } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) { - err = ovs_ct_nat_execute(skb, ct, ctinfo, NULL, - NF_NAT_MANIP_SRC, key); - } - } + err = nf_ct_nat(skb, ct, ctinfo, &action, &info->range, info->commit); + + if (action & BIT(NF_NAT_MANIP_SRC)) + ovs_nat_update_key(key, skb, NF_NAT_MANIP_SRC); + if (action & BIT(NF_NAT_MANIP_DST)) + ovs_nat_update_key(key, skb, NF_NAT_MANIP_DST); return err; } diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 861dfb8daf4a..932bcf766d63 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -209,6 +209,26 @@ static struct vport *new_vport(const struct vport_parms *parms) return vport; } +static void ovs_vport_update_upcall_stats(struct sk_buff *skb, + const struct dp_upcall_info *upcall_info, + bool upcall_result) +{ + struct vport *p = OVS_CB(skb)->input_vport; + struct vport_upcall_stats_percpu *stats; + + if (upcall_info->cmd != OVS_PACKET_CMD_MISS && + upcall_info->cmd != OVS_PACKET_CMD_ACTION) + return; + + stats = this_cpu_ptr(p->upcall_stats); + u64_stats_update_begin(&stats->syncp); + if (upcall_result) + u64_stats_inc(&stats->n_success); + else + u64_stats_inc(&stats->n_fail); + u64_stats_update_end(&stats->syncp); +} + void ovs_dp_detach_port(struct vport *p) { ASSERT_OVSL(); @@ -216,6 +236,9 @@ void ovs_dp_detach_port(struct vport *p) /* First drop references to device. */ hlist_del_rcu(&p->dp_hash_node); + /* Free percpu memory */ + free_percpu(p->upcall_stats); + /* Then destroy it. */ ovs_vport_del(p); } @@ -305,6 +328,8 @@ int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb, err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen); else err = queue_gso_packets(dp, skb, key, upcall_info, cutlen); + + ovs_vport_update_upcall_stats(skb, upcall_info, !err); if (err) goto err; @@ -1826,6 +1851,12 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) goto err_destroy_portids; } + vport->upcall_stats = netdev_alloc_pcpu_stats(struct vport_upcall_stats_percpu); + if (!vport->upcall_stats) { + err = -ENOMEM; + goto err_destroy_portids; + } + err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, info->snd_seq, 0, OVS_DP_CMD_NEW); BUG_ON(err < 0); @@ -2098,6 +2129,9 @@ static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb, OVS_VPORT_ATTR_PAD)) goto nla_put_failure; + if (ovs_vport_get_upcall_stats(vport, skb)) + goto nla_put_failure; + if (ovs_vport_get_upcall_portids(vport, skb)) goto nla_put_failure; @@ -2279,6 +2313,12 @@ restart: goto exit_unlock_free; } + vport->upcall_stats = netdev_alloc_pcpu_stats(struct vport_upcall_stats_percpu); + if (!vport->upcall_stats) { + err = -ENOMEM; + goto exit_unlock_free; + } + err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), info->snd_portid, info->snd_seq, 0, OVS_VPORT_CMD_NEW, GFP_KERNEL); @@ -2508,6 +2548,7 @@ static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = { [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED }, [OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 }, [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 }, + [OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED }, }; static const struct genl_small_ops dp_vport_genl_ops[] = { diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 82a74f998966..7e0f5c45b512 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c @@ -285,6 +285,56 @@ void ovs_vport_get_stats(struct vport *vport, struct ovs_vport_stats *stats) } /** + * ovs_vport_get_upcall_stats - retrieve upcall stats + * + * @vport: vport from which to retrieve the stats. + * @skb: sk_buff where upcall stats should be appended. + * + * Retrieves upcall stats for the given device. + * + * Must be called with ovs_mutex or rcu_read_lock. + */ +int ovs_vport_get_upcall_stats(struct vport *vport, struct sk_buff *skb) +{ + struct nlattr *nla; + int i; + + __u64 tx_success = 0; + __u64 tx_fail = 0; + + for_each_possible_cpu(i) { + const struct vport_upcall_stats_percpu *stats; + unsigned int start; + + stats = per_cpu_ptr(vport->upcall_stats, i); + do { + start = u64_stats_fetch_begin(&stats->syncp); + tx_success += u64_stats_read(&stats->n_success); + tx_fail += u64_stats_read(&stats->n_fail); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + } + + nla = nla_nest_start_noflag(skb, OVS_VPORT_ATTR_UPCALL_STATS); + if (!nla) + return -EMSGSIZE; + + if (nla_put_u64_64bit(skb, OVS_VPORT_UPCALL_ATTR_SUCCESS, tx_success, + OVS_VPORT_ATTR_PAD)) { + nla_nest_cancel(skb, nla); + return -EMSGSIZE; + } + + if (nla_put_u64_64bit(skb, OVS_VPORT_UPCALL_ATTR_FAIL, tx_fail, + OVS_VPORT_ATTR_PAD)) { + nla_nest_cancel(skb, nla); + return -EMSGSIZE; + } + nla_nest_end(skb, nla); + + return 0; +} + +/** * ovs_vport_get_options - retrieve device options * * @vport: vport from which to retrieve the options. diff --git a/net/openvswitch/vport.h b/net/openvswitch/vport.h index 6ff45e8a0868..3e71ca8ad8a7 100644 --- a/net/openvswitch/vport.h +++ b/net/openvswitch/vport.h @@ -32,6 +32,8 @@ struct vport *ovs_vport_locate(const struct net *net, const char *name); void ovs_vport_get_stats(struct vport *, struct ovs_vport_stats *); +int ovs_vport_get_upcall_stats(struct vport *vport, struct sk_buff *skb); + int ovs_vport_set_options(struct vport *, struct nlattr *options); int ovs_vport_get_options(const struct vport *, struct sk_buff *); @@ -65,6 +67,7 @@ struct vport_portids { * @hash_node: Element in @dev_table hash table in vport.c. * @dp_hash_node: Element in @datapath->ports hash table in datapath.c. * @ops: Class structure. + * @upcall_stats: Upcall stats of every ports. * @detach_list: list used for detaching vport in net-exit call. * @rcu: RCU callback head for deferred destruction. */ @@ -78,6 +81,7 @@ struct vport { struct hlist_node hash_node; struct hlist_node dp_hash_node; const struct vport_ops *ops; + struct vport_upcall_stats_percpu __percpu *upcall_stats; struct list_head detach_list; struct rcu_head rcu; @@ -137,6 +141,18 @@ struct vport_ops { struct list_head list; }; +/** + * struct vport_upcall_stats_percpu - per-cpu packet upcall statistics for + * a given vport. + * @n_success: Number of packets that upcall to userspace succeed. + * @n_fail: Number of packets that upcall to userspace failed. + */ +struct vport_upcall_stats_percpu { + struct u64_stats_sync syncp; + u64_stats_t n_success; + u64_stats_t n_fail; +}; + struct vport *ovs_vport_alloc(int priv_size, const struct vport_ops *, const struct vport_parms *); void ovs_vport_free(struct vport *); diff --git a/net/rxrpc/Kconfig b/net/rxrpc/Kconfig index accd35c05577..7ae023b37a83 100644 --- a/net/rxrpc/Kconfig +++ b/net/rxrpc/Kconfig @@ -58,4 +58,11 @@ config RXKAD See Documentation/networking/rxrpc.rst. +config RXPERF + tristate "RxRPC test service" + help + Provide an rxperf service tester. This listens on UDP port 7009 for + incoming calls from the rxperf program (an example of which can be + found in OpenAFS). + endif diff --git a/net/rxrpc/Makefile b/net/rxrpc/Makefile index fdeba488fc6e..e76d3459d78e 100644 --- a/net/rxrpc/Makefile +++ b/net/rxrpc/Makefile @@ -16,6 +16,7 @@ rxrpc-y := \ conn_service.o \ input.o \ insecure.o \ + io_thread.o \ key.o \ local_event.o \ local_object.o \ @@ -36,3 +37,6 @@ rxrpc-y := \ rxrpc-$(CONFIG_PROC_FS) += proc.o rxrpc-$(CONFIG_RXKAD) += rxkad.o rxrpc-$(CONFIG_SYSCTL) += sysctl.o + + +obj-$(CONFIG_RXPERF) += rxperf.o diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index aacdd96a9886..7ea576f6ba4b 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -194,8 +194,8 @@ static int rxrpc_bind(struct socket *sock, struct sockaddr *saddr, int len) service_in_use: write_unlock(&local->services_lock); - rxrpc_unuse_local(local); - rxrpc_put_local(local); + rxrpc_unuse_local(local, rxrpc_local_unuse_bind); + rxrpc_put_local(local, rxrpc_local_put_bind); ret = -EADDRINUSE; error_unlock: release_sock(&rx->sk); @@ -328,7 +328,7 @@ struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock, mutex_unlock(&call->user_mutex); } - rxrpc_put_peer(cp.peer); + rxrpc_put_peer(cp.peer, rxrpc_peer_put_discard_tmp); _leave(" = %p", call); return call; } @@ -359,9 +359,9 @@ void rxrpc_kernel_end_call(struct socket *sock, struct rxrpc_call *call) /* Make sure we're not going to call back into a kernel service */ if (call->notify_rx) { - spin_lock_bh(&call->notify_lock); + spin_lock(&call->notify_lock); call->notify_rx = rxrpc_dummy_notify_rx; - spin_unlock_bh(&call->notify_lock); + spin_unlock(&call->notify_lock); } mutex_unlock(&call->user_mutex); @@ -812,14 +812,12 @@ static int rxrpc_shutdown(struct socket *sock, int flags) lock_sock(sk); - spin_lock_bh(&sk->sk_receive_queue.lock); if (sk->sk_state < RXRPC_CLOSE) { sk->sk_state = RXRPC_CLOSE; sk->sk_shutdown = SHUTDOWN_MASK; } else { ret = -ESHUTDOWN; } - spin_unlock_bh(&sk->sk_receive_queue.lock); rxrpc_discard_prealloc(rx); @@ -872,9 +870,7 @@ static int rxrpc_release_sock(struct sock *sk) break; } - spin_lock_bh(&sk->sk_receive_queue.lock); sk->sk_state = RXRPC_CLOSE; - spin_unlock_bh(&sk->sk_receive_queue.lock); if (rx->local && rcu_access_pointer(rx->local->service) == rx) { write_lock(&rx->local->services_lock); @@ -888,8 +884,8 @@ static int rxrpc_release_sock(struct sock *sk) flush_workqueue(rxrpc_workqueue); rxrpc_purge_queue(&sk->sk_receive_queue); - rxrpc_unuse_local(rx->local); - rxrpc_put_local(rx->local); + rxrpc_unuse_local(rx->local, rxrpc_local_unuse_release_sock); + rxrpc_put_local(rx->local, rxrpc_local_put_release_sock); rx->local = NULL; key_put(rx->key); rx->key = NULL; diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index f5c538ce3e23..e7dccab7b741 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -36,6 +36,8 @@ struct rxrpc_txbuf; * to pass supplementary information. */ enum rxrpc_skb_mark { + RXRPC_SKB_MARK_PACKET, /* Received packet */ + RXRPC_SKB_MARK_ERROR, /* Error notification */ RXRPC_SKB_MARK_REJECT_BUSY, /* Reject with BUSY */ RXRPC_SKB_MARK_REJECT_ABORT, /* Reject with ABORT (code in skb->priority) */ }; @@ -76,7 +78,7 @@ struct rxrpc_net { bool kill_all_client_conns; atomic_t nr_client_conns; spinlock_t client_conn_cache_lock; /* Lock for ->*_client_conns */ - spinlock_t client_conn_discard_lock; /* Prevent multiple discarders */ + struct mutex client_conn_discard_lock; /* Prevent multiple discarders */ struct list_head idle_client_conns; struct work_struct client_conn_reaper; struct timer_list client_conn_reap_timer; @@ -99,6 +101,9 @@ struct rxrpc_net { atomic_t stat_tx_data_retrans; atomic_t stat_tx_data_send; atomic_t stat_tx_data_send_frag; + atomic_t stat_tx_data_send_fail; + atomic_t stat_tx_data_underflow; + atomic_t stat_tx_data_cwnd_reset; atomic_t stat_rx_data; atomic_t stat_rx_data_reqack; atomic_t stat_rx_data_jumbo; @@ -110,6 +115,8 @@ struct rxrpc_net { atomic_t stat_rx_acks[256]; atomic_t stat_why_req_ack[8]; + + atomic_t stat_io_loop; }; /* @@ -279,13 +286,11 @@ struct rxrpc_local { struct rxrpc_net *rxnet; /* The network ns in which this resides */ struct hlist_node link; struct socket *socket; /* my UDP socket */ - struct work_struct processor; - struct list_head ack_tx_queue; /* List of ACKs that need sending */ - spinlock_t ack_tx_lock; /* ACK list lock */ + struct task_struct *io_thread; struct rxrpc_sock __rcu *service; /* Service(s) listening on this endpoint */ struct rw_semaphore defrag_sem; /* control re-enablement of IP DF bit */ - struct sk_buff_head reject_queue; /* packets awaiting rejection */ - struct sk_buff_head event_queue; /* endpoint event packets awaiting processing */ + struct sk_buff_head rx_queue; /* Received packets */ + struct list_head call_attend_q; /* Calls requiring immediate attention */ struct rb_root client_bundles; /* Client connection bundles by socket params */ spinlock_t client_bundles_lock; /* Lock for client_bundles */ spinlock_t lock; /* access lock */ @@ -403,12 +408,18 @@ enum rxrpc_conn_proto_state { * RxRPC client connection bundle. */ struct rxrpc_bundle { - struct rxrpc_conn_parameters params; + struct rxrpc_local *local; /* Representation of local endpoint */ + struct rxrpc_peer *peer; /* Remote endpoint */ + struct key *key; /* Security details */ refcount_t ref; atomic_t active; /* Number of active users */ unsigned int debug_id; + u32 security_level; /* Security level selected */ + u16 service_id; /* Service ID for this connection */ bool try_upgrade; /* True if the bundle is attempting upgrade */ bool alloc_conn; /* True if someone's getting a conn */ + bool exclusive; /* T if conn is exclusive */ + bool upgrade; /* T if service ID can be upgraded */ short alloc_error; /* Error from last conn allocation */ spinlock_t channel_lock; struct rb_node local_node; /* Node in local->client_conns */ @@ -424,9 +435,13 @@ struct rxrpc_bundle { */ struct rxrpc_connection { struct rxrpc_conn_proto proto; - struct rxrpc_conn_parameters params; + struct rxrpc_local *local; /* Representation of local endpoint */ + struct rxrpc_peer *peer; /* Remote endpoint */ + struct rxrpc_net *rxnet; /* Network namespace to which call belongs */ + struct key *key; /* Security details */ refcount_t ref; + atomic_t active; /* Active count for service conns */ struct rcu_head rcu; struct list_head cache_link; @@ -447,6 +462,7 @@ struct rxrpc_connection { struct timer_list timer; /* Conn event timer */ struct work_struct processor; /* connection event processor */ + struct work_struct destructor; /* In-process-context destroyer */ struct rxrpc_bundle *bundle; /* Client connection bundle */ struct rb_node service_node; /* Node in peer->service_conns */ struct list_head proc_link; /* link in procfs list */ @@ -471,9 +487,13 @@ struct rxrpc_connection { atomic_t serial; /* packet serial number counter */ unsigned int hi_serial; /* highest serial number received */ u32 service_id; /* Service ID, possibly upgraded */ + u32 security_level; /* Security level selected */ u8 security_ix; /* security type */ u8 out_clientflag; /* RXRPC_CLIENT_INITIATED if we are client */ u8 bundle_shift; /* Index into bundle->avail_chans */ + bool exclusive; /* T if conn is exclusive */ + bool upgrade; /* T if service ID can be upgraded */ + u16 orig_service_id; /* Originally requested service ID */ short error; /* Local error code */ }; @@ -502,22 +522,19 @@ enum rxrpc_call_flag { RXRPC_CALL_RETRANS_TIMEOUT, /* Retransmission due to timeout occurred */ RXRPC_CALL_BEGAN_RX_TIMER, /* We began the expect_rx_by timer */ RXRPC_CALL_RX_HEARD, /* The peer responded at least once to this call */ - RXRPC_CALL_RX_UNDERRUN, /* Got data underrun */ RXRPC_CALL_DISCONNECTED, /* The call has been disconnected */ RXRPC_CALL_KERNEL, /* The call was made by the kernel */ RXRPC_CALL_UPGRADE, /* Service upgrade was requested for the call */ - RXRPC_CALL_DELAY_ACK_PENDING, /* DELAY ACK generation is pending */ - RXRPC_CALL_IDLE_ACK_PENDING, /* IDLE ACK generation is pending */ + RXRPC_CALL_EXCLUSIVE, /* The call uses a once-only connection */ + RXRPC_CALL_RX_IS_IDLE, /* Reception is idle - send an ACK */ }; /* * Events that can be raised on a call. */ enum rxrpc_call_event { - RXRPC_CALL_EV_ABORT, /* need to generate abort */ - RXRPC_CALL_EV_RESEND, /* Tx resend required */ - RXRPC_CALL_EV_EXPIRED, /* Expiry occurred */ RXRPC_CALL_EV_ACK_LOST, /* ACK may be lost, send ping */ + RXRPC_CALL_EV_INITIAL_PING, /* Send initial ping for a new service call */ }; /* @@ -570,10 +587,13 @@ struct rxrpc_call { struct rcu_head rcu; struct rxrpc_connection *conn; /* connection carrying call */ struct rxrpc_peer *peer; /* Peer record for remote address */ + struct rxrpc_local *local; /* Representation of local endpoint */ struct rxrpc_sock __rcu *socket; /* socket responsible */ struct rxrpc_net *rxnet; /* Network namespace to which call belongs */ + struct key *key; /* Security details */ const struct rxrpc_security *security; /* applied security module */ struct mutex user_mutex; /* User access mutex */ + struct sockaddr_rxrpc dest_srx; /* Destination address */ unsigned long delay_ack_at; /* When DELAY ACK needs to happen */ unsigned long ack_lost_at; /* When ACK is figured as lost */ unsigned long resend_at; /* When next resend needs to happen */ @@ -585,7 +605,7 @@ struct rxrpc_call { u32 next_rx_timo; /* Timeout for next Rx packet (jif) */ u32 next_req_timo; /* Timeout for next Rx request packet (jif) */ struct timer_list timer; /* Combined event timer */ - struct work_struct processor; /* Event processor */ + struct work_struct destroyer; /* In-process-context destroyer */ rxrpc_notify_rx_t notify_rx; /* kernel service Rx notification function */ struct list_head link; /* link in master call list */ struct list_head chan_wait_link; /* Link in conn->bundle->waiting_calls */ @@ -594,6 +614,7 @@ struct rxrpc_call { struct list_head recvmsg_link; /* Link in rx->recvmsg_q */ struct list_head sock_link; /* Link in rx->sock_calls */ struct rb_node sock_node; /* Node in rx->calls */ + struct list_head attend_link; /* Link in local->call_attend_q */ struct rxrpc_txbuf *tx_pending; /* Tx buffer being filled */ wait_queue_head_t waitq; /* Wait queue for channel or Tx */ s64 tx_total_len; /* Total length left to be transmitted (or -1) */ @@ -607,20 +628,22 @@ struct rxrpc_call { enum rxrpc_call_state state; /* current state of call */ enum rxrpc_call_completion completion; /* Call completion condition */ refcount_t ref; - u16 service_id; /* service ID */ u8 security_ix; /* Security type */ enum rxrpc_interruptibility interruptibility; /* At what point call may be interrupted */ u32 call_id; /* call ID on connection */ u32 cid; /* connection ID plus channel index */ + u32 security_level; /* Security level selected */ int debug_id; /* debug ID for printks */ unsigned short rx_pkt_offset; /* Current recvmsg packet offset */ unsigned short rx_pkt_len; /* Current recvmsg packet len */ /* Transmitted data tracking. */ spinlock_t tx_lock; /* Transmit queue lock */ + struct list_head tx_sendmsg; /* Sendmsg prepared packets */ struct list_head tx_buffer; /* Buffer of transmissible packets */ rxrpc_seq_t tx_bottom; /* First packet in buffer */ rxrpc_seq_t tx_transmitted; /* Highest packet transmitted */ + rxrpc_seq_t tx_prepared; /* Highest Tx slot prepared. */ rxrpc_seq_t tx_top; /* Highest Tx slot allocated. */ u16 tx_backoff; /* Delay to insert due to Tx failure */ u8 tx_winsize; /* Maximum size of Tx window */ @@ -635,13 +658,13 @@ struct rxrpc_call { rxrpc_seq_t rx_consumed; /* Highest packet consumed */ rxrpc_serial_t rx_serial; /* Highest serial received for this call */ u8 rx_winsize; /* Size of Rx window */ - spinlock_t input_lock; /* Lock for packet input to this call */ /* TCP-style slow-start congestion control [RFC5681]. Since the SMSS * is fixed, we keep these numbers in terms of segments (ie. DATA * packets) rather than bytes. */ #define RXRPC_TX_SMSS RXRPC_JUMBO_DATALEN +#define RXRPC_MIN_CWND (RXRPC_TX_SMSS > 2190 ? 2 : RXRPC_TX_SMSS > 1095 ? 3 : 4) u8 cong_cwnd; /* Congestion window size */ u8 cong_extra; /* Extra to send for congestion management */ u8 cong_ssthresh; /* Slow-start threshold */ @@ -676,11 +699,7 @@ struct rxrpc_call { rxrpc_seq_t acks_prev_seq; /* Highest previousPacket received */ rxrpc_seq_t acks_hard_ack; /* Latest hard-ack point */ rxrpc_seq_t acks_lowest_nak; /* Lowest NACK in the buffer (or ==tx_hard_ack) */ - rxrpc_seq_t acks_lost_top; /* tx_top at the time lost-ack ping sent */ - rxrpc_serial_t acks_lost_ping; /* Serial number of probe ACK */ rxrpc_serial_t acks_highest_serial; /* Highest serial number ACK'd */ - struct sk_buff *acks_soft_tbl; /* The last ACK packet with NAKs in it */ - spinlock_t acks_ack_lock; /* Access to ->acks_last_ack */ }; /* @@ -739,9 +758,8 @@ struct rxrpc_send_params { */ struct rxrpc_txbuf { struct rcu_head rcu; - struct list_head call_link; /* Link in call->tx_queue */ + struct list_head call_link; /* Link in call->tx_sendmsg/tx_buffer */ struct list_head tx_link; /* Link in live Enc queue or Tx queue */ - struct rxrpc_call *call; /* Call to which belongs */ ktime_t last_sent; /* Time at which last transmitted */ refcount_t ref; rxrpc_seq_t seq; /* Sequence number of this packet */ @@ -793,9 +811,9 @@ extern struct workqueue_struct *rxrpc_workqueue; */ int rxrpc_service_prealloc(struct rxrpc_sock *, gfp_t); void rxrpc_discard_prealloc(struct rxrpc_sock *); -struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *, - struct rxrpc_sock *, - struct sk_buff *); +bool rxrpc_new_incoming_call(struct rxrpc_local *, struct rxrpc_peer *, + struct rxrpc_connection *, struct sockaddr_rxrpc *, + struct sk_buff *); void rxrpc_accept_incoming_calls(struct rxrpc_local *); int rxrpc_user_charge_accept(struct rxrpc_sock *, unsigned long); @@ -808,14 +826,14 @@ void rxrpc_send_ACK(struct rxrpc_call *, u8, rxrpc_serial_t, enum rxrpc_propose_ void rxrpc_propose_delay_ACK(struct rxrpc_call *, rxrpc_serial_t, enum rxrpc_propose_ack_trace); void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *); -void rxrpc_process_call(struct work_struct *); +void rxrpc_resend(struct rxrpc_call *call, struct sk_buff *ack_skb); void rxrpc_reduce_call_timer(struct rxrpc_call *call, unsigned long expire_at, unsigned long now, enum rxrpc_timer_trace why); -void rxrpc_delete_call_timer(struct rxrpc_call *call); +void rxrpc_input_call_event(struct rxrpc_call *call, struct sk_buff *skb); /* * call_object.c @@ -824,6 +842,7 @@ extern const char *const rxrpc_call_states[]; extern const char *const rxrpc_call_completions[]; extern struct kmem_cache *rxrpc_call_jar; +void rxrpc_poke_call(struct rxrpc_call *call, enum rxrpc_call_poke_trace what); struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *, unsigned long); struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *, gfp_t, unsigned int); struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *, @@ -835,10 +854,8 @@ void rxrpc_incoming_call(struct rxrpc_sock *, struct rxrpc_call *, struct sk_buff *); void rxrpc_release_call(struct rxrpc_sock *, struct rxrpc_call *); void rxrpc_release_calls_on_socket(struct rxrpc_sock *); -bool __rxrpc_queue_call(struct rxrpc_call *); -bool rxrpc_queue_call(struct rxrpc_call *); -void rxrpc_see_call(struct rxrpc_call *); -bool rxrpc_try_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op); +void rxrpc_see_call(struct rxrpc_call *, enum rxrpc_call_trace); +struct rxrpc_call *rxrpc_try_get_call(struct rxrpc_call *, enum rxrpc_call_trace); void rxrpc_get_call(struct rxrpc_call *, enum rxrpc_call_trace); void rxrpc_put_call(struct rxrpc_call *, enum rxrpc_call_trace); void rxrpc_cleanup_call(struct rxrpc_call *); @@ -863,14 +880,14 @@ extern unsigned long rxrpc_conn_idle_client_fast_expiry; extern struct idr rxrpc_client_conn_ids; void rxrpc_destroy_client_conn_ids(void); -struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *); -void rxrpc_put_bundle(struct rxrpc_bundle *); +struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *, enum rxrpc_bundle_trace); +void rxrpc_put_bundle(struct rxrpc_bundle *, enum rxrpc_bundle_trace); int rxrpc_connect_call(struct rxrpc_sock *, struct rxrpc_call *, struct rxrpc_conn_parameters *, struct sockaddr_rxrpc *, gfp_t); void rxrpc_expose_client_call(struct rxrpc_call *); void rxrpc_disconnect_client_call(struct rxrpc_bundle *, struct rxrpc_call *); -void rxrpc_put_client_conn(struct rxrpc_connection *); +void rxrpc_put_client_conn(struct rxrpc_connection *, enum rxrpc_conn_trace); void rxrpc_discard_expired_client_conns(struct work_struct *); void rxrpc_destroy_all_client_connections(struct rxrpc_net *); void rxrpc_clean_up_local_conns(struct rxrpc_local *); @@ -880,6 +897,7 @@ void rxrpc_clean_up_local_conns(struct rxrpc_local *); */ void rxrpc_process_connection(struct work_struct *); void rxrpc_process_delayed_final_acks(struct rxrpc_connection *, bool); +int rxrpc_input_conn_packet(struct rxrpc_connection *conn, struct sk_buff *skb); /* * conn_object.c @@ -887,18 +905,20 @@ void rxrpc_process_delayed_final_acks(struct rxrpc_connection *, bool); extern unsigned int rxrpc_connection_expiry; extern unsigned int rxrpc_closed_conn_expiry; -struct rxrpc_connection *rxrpc_alloc_connection(gfp_t); -struct rxrpc_connection *rxrpc_find_connection_rcu(struct rxrpc_local *, - struct sk_buff *, - struct rxrpc_peer **); +struct rxrpc_connection *rxrpc_alloc_connection(struct rxrpc_net *, gfp_t); +struct rxrpc_connection *rxrpc_find_client_connection_rcu(struct rxrpc_local *, + struct sockaddr_rxrpc *, + struct sk_buff *); void __rxrpc_disconnect_call(struct rxrpc_connection *, struct rxrpc_call *); void rxrpc_disconnect_call(struct rxrpc_call *); -void rxrpc_kill_connection(struct rxrpc_connection *); -bool rxrpc_queue_conn(struct rxrpc_connection *); -void rxrpc_see_connection(struct rxrpc_connection *); -struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *); -struct rxrpc_connection *rxrpc_get_connection_maybe(struct rxrpc_connection *); -void rxrpc_put_service_conn(struct rxrpc_connection *); +void rxrpc_kill_client_conn(struct rxrpc_connection *); +void rxrpc_queue_conn(struct rxrpc_connection *, enum rxrpc_conn_trace); +void rxrpc_see_connection(struct rxrpc_connection *, enum rxrpc_conn_trace); +struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *, + enum rxrpc_conn_trace); +struct rxrpc_connection *rxrpc_get_connection_maybe(struct rxrpc_connection *, + enum rxrpc_conn_trace); +void rxrpc_put_connection(struct rxrpc_connection *, enum rxrpc_conn_trace); void rxrpc_service_connection_reaper(struct work_struct *); void rxrpc_destroy_all_connections(struct rxrpc_net *); @@ -912,17 +932,6 @@ static inline bool rxrpc_conn_is_service(const struct rxrpc_connection *conn) return !rxrpc_conn_is_client(conn); } -static inline void rxrpc_put_connection(struct rxrpc_connection *conn) -{ - if (!conn) - return; - - if (rxrpc_conn_is_client(conn)) - rxrpc_put_client_conn(conn); - else - rxrpc_put_service_conn(conn); -} - static inline void rxrpc_reduce_conn_timer(struct rxrpc_connection *conn, unsigned long expire_at) { @@ -942,7 +951,20 @@ void rxrpc_unpublish_service_conn(struct rxrpc_connection *); /* * input.c */ -int rxrpc_input_packet(struct sock *, struct sk_buff *); +void rxrpc_congestion_degrade(struct rxrpc_call *); +void rxrpc_input_call_packet(struct rxrpc_call *, struct sk_buff *); +void rxrpc_implicit_end_call(struct rxrpc_call *, struct sk_buff *); + +/* + * io_thread.c + */ +int rxrpc_encap_rcv(struct sock *, struct sk_buff *); +void rxrpc_error_report(struct sock *); +int rxrpc_io_thread(void *data); +static inline void rxrpc_wake_up_io_thread(struct rxrpc_local *local) +{ + wake_up_process(local->io_thread); +} /* * insecure.c @@ -961,28 +983,41 @@ int rxrpc_get_server_data_key(struct rxrpc_connection *, const void *, time64_t, /* * local_event.c */ -extern void rxrpc_process_local_events(struct rxrpc_local *); +void rxrpc_send_version_request(struct rxrpc_local *local, + struct rxrpc_host_header *hdr, + struct sk_buff *skb); /* * local_object.c */ struct rxrpc_local *rxrpc_lookup_local(struct net *, const struct sockaddr_rxrpc *); -struct rxrpc_local *rxrpc_get_local(struct rxrpc_local *); -struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *); -void rxrpc_put_local(struct rxrpc_local *); -struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *); -void rxrpc_unuse_local(struct rxrpc_local *); -void rxrpc_queue_local(struct rxrpc_local *); +struct rxrpc_local *rxrpc_get_local(struct rxrpc_local *, enum rxrpc_local_trace); +struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *, enum rxrpc_local_trace); +void rxrpc_put_local(struct rxrpc_local *, enum rxrpc_local_trace); +struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *, enum rxrpc_local_trace); +void rxrpc_unuse_local(struct rxrpc_local *, enum rxrpc_local_trace); +void rxrpc_destroy_local(struct rxrpc_local *local); void rxrpc_destroy_all_locals(struct rxrpc_net *); -static inline bool __rxrpc_unuse_local(struct rxrpc_local *local) +static inline bool __rxrpc_use_local(struct rxrpc_local *local, + enum rxrpc_local_trace why) { - return atomic_dec_return(&local->active_users) == 0; + int r, u; + + r = refcount_read(&local->ref); + u = atomic_fetch_add_unless(&local->active_users, 1, 0); + trace_rxrpc_local(local->debug_id, why, r, u); + return u != 0; } -static inline bool __rxrpc_use_local(struct rxrpc_local *local) +static inline void rxrpc_see_local(struct rxrpc_local *local, + enum rxrpc_local_trace why) { - return atomic_fetch_add_unless(&local->active_users, 1, 0) != 0; + int r, u; + + r = refcount_read(&local->ref); + u = atomic_read(&local->active_users); + trace_rxrpc_local(local->debug_id, why, r, u); } /* @@ -1009,16 +1044,17 @@ static inline struct rxrpc_net *rxrpc_net(struct net *net) /* * output.c */ -void rxrpc_transmit_ack_packets(struct rxrpc_local *); +int rxrpc_send_ack_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb); int rxrpc_send_abort_packet(struct rxrpc_call *); int rxrpc_send_data_packet(struct rxrpc_call *, struct rxrpc_txbuf *); -void rxrpc_reject_packets(struct rxrpc_local *); +void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb); void rxrpc_send_keepalive(struct rxrpc_peer *); +void rxrpc_transmit_one(struct rxrpc_call *call, struct rxrpc_txbuf *txb); /* * peer_event.c */ -void rxrpc_error_report(struct sock *); +void rxrpc_input_error(struct rxrpc_local *, struct sk_buff *); void rxrpc_peer_keepalive_worker(struct work_struct *); /* @@ -1028,14 +1064,15 @@ struct rxrpc_peer *rxrpc_lookup_peer_rcu(struct rxrpc_local *, const struct sockaddr_rxrpc *); struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *, struct rxrpc_local *, struct sockaddr_rxrpc *, gfp_t); -struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *, gfp_t); +struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *, gfp_t, + enum rxrpc_peer_trace); void rxrpc_new_incoming_peer(struct rxrpc_sock *, struct rxrpc_local *, struct rxrpc_peer *); void rxrpc_destroy_all_peers(struct rxrpc_net *); -struct rxrpc_peer *rxrpc_get_peer(struct rxrpc_peer *); -struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *); -void rxrpc_put_peer(struct rxrpc_peer *); -void rxrpc_put_peer_locked(struct rxrpc_peer *); +struct rxrpc_peer *rxrpc_get_peer(struct rxrpc_peer *, enum rxrpc_peer_trace); +struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *, enum rxrpc_peer_trace); +void rxrpc_put_peer(struct rxrpc_peer *, enum rxrpc_peer_trace); +void rxrpc_put_peer_locked(struct rxrpc_peer *, enum rxrpc_peer_trace); /* * proc.c @@ -1097,6 +1134,7 @@ extern const struct rxrpc_security rxkad; int __init rxrpc_init_security(void); const struct rxrpc_security *rxrpc_security_lookup(u8); void rxrpc_exit_security(void); +int rxrpc_init_client_call_security(struct rxrpc_call *); int rxrpc_init_client_conn_security(struct rxrpc_connection *); const struct rxrpc_security *rxrpc_get_incoming_security(struct rxrpc_sock *, struct sk_buff *); @@ -1119,7 +1157,6 @@ int rxrpc_server_keyring(struct rxrpc_sock *, sockptr_t, int); * skbuff.c */ void rxrpc_kernel_data_consumed(struct rxrpc_call *, struct sk_buff *); -void rxrpc_packet_destructor(struct sk_buff *); void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_eaten_skb(struct sk_buff *, enum rxrpc_skb_trace); @@ -1190,23 +1227,17 @@ extern unsigned int rxrpc_debug; #define kenter(FMT,...) dbgprintk("==> %s("FMT")",__func__ ,##__VA_ARGS__) #define kleave(FMT,...) dbgprintk("<== %s()"FMT"",__func__ ,##__VA_ARGS__) #define kdebug(FMT,...) dbgprintk(" "FMT ,##__VA_ARGS__) -#define kproto(FMT,...) dbgprintk("### "FMT ,##__VA_ARGS__) -#define knet(FMT,...) dbgprintk("@@@ "FMT ,##__VA_ARGS__) #if defined(__KDEBUG) #define _enter(FMT,...) kenter(FMT,##__VA_ARGS__) #define _leave(FMT,...) kleave(FMT,##__VA_ARGS__) #define _debug(FMT,...) kdebug(FMT,##__VA_ARGS__) -#define _proto(FMT,...) kproto(FMT,##__VA_ARGS__) -#define _net(FMT,...) knet(FMT,##__VA_ARGS__) #elif defined(CONFIG_AF_RXRPC_DEBUG) #define RXRPC_DEBUG_KENTER 0x01 #define RXRPC_DEBUG_KLEAVE 0x02 #define RXRPC_DEBUG_KDEBUG 0x04 -#define RXRPC_DEBUG_KPROTO 0x08 -#define RXRPC_DEBUG_KNET 0x10 #define _enter(FMT,...) \ do { \ @@ -1226,24 +1257,10 @@ do { \ kdebug(FMT,##__VA_ARGS__); \ } while (0) -#define _proto(FMT,...) \ -do { \ - if (unlikely(rxrpc_debug & RXRPC_DEBUG_KPROTO)) \ - kproto(FMT,##__VA_ARGS__); \ -} while (0) - -#define _net(FMT,...) \ -do { \ - if (unlikely(rxrpc_debug & RXRPC_DEBUG_KNET)) \ - knet(FMT,##__VA_ARGS__); \ -} while (0) - #else #define _enter(FMT,...) no_printk("==> %s("FMT")",__func__ ,##__VA_ARGS__) #define _leave(FMT,...) no_printk("<== %s()"FMT"",__func__ ,##__VA_ARGS__) #define _debug(FMT,...) no_printk(" "FMT ,##__VA_ARGS__) -#define _proto(FMT,...) no_printk("### "FMT ,##__VA_ARGS__) -#define _net(FMT,...) no_printk("@@@ "FMT ,##__VA_ARGS__) #endif /* diff --git a/net/rxrpc/call_accept.c b/net/rxrpc/call_accept.c index 48790ee77019..d1850863507f 100644 --- a/net/rxrpc/call_accept.c +++ b/net/rxrpc/call_accept.c @@ -38,7 +38,6 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, unsigned long user_call_ID, gfp_t gfp, unsigned int debug_id) { - const void *here = __builtin_return_address(0); struct rxrpc_call *call, *xcall; struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk)); struct rb_node *parent, **pp; @@ -70,7 +69,9 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, head = b->peer_backlog_head; tail = READ_ONCE(b->peer_backlog_tail); if (CIRC_CNT(head, tail, size) < max) { - struct rxrpc_peer *peer = rxrpc_alloc_peer(rx->local, gfp); + struct rxrpc_peer *peer; + + peer = rxrpc_alloc_peer(rx->local, gfp, rxrpc_peer_new_prealloc); if (!peer) return -ENOMEM; b->peer_backlog[head] = peer; @@ -89,9 +90,6 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, b->conn_backlog[head] = conn; smp_store_release(&b->conn_backlog_head, (head + 1) & (size - 1)); - - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_service, - refcount_read(&conn->ref), here); } /* Now it gets complicated, because calls get registered with the @@ -102,10 +100,10 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, return -ENOMEM; call->flags |= (1 << RXRPC_CALL_IS_SERVICE); call->state = RXRPC_CALL_SERVER_PREALLOC; + __set_bit(RXRPC_CALL_EV_INITIAL_PING, &call->events); - trace_rxrpc_call(call->debug_id, rxrpc_call_new_service, - refcount_read(&call->ref), - here, (const void *)user_call_ID); + trace_rxrpc_call(call->debug_id, refcount_read(&call->ref), + user_call_ID, rxrpc_call_new_prealloc_service); write_lock(&rx->call_lock); @@ -126,11 +124,11 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, call->user_call_ID = user_call_ID; call->notify_rx = notify_rx; if (user_attach_call) { - rxrpc_get_call(call, rxrpc_call_got_kernel); + rxrpc_get_call(call, rxrpc_call_get_kernel_service); user_attach_call(call, user_call_ID); } - rxrpc_get_call(call, rxrpc_call_got_userid); + rxrpc_get_call(call, rxrpc_call_get_userid); rb_link_node(&call->sock_node, parent, pp); rb_insert_color(&call->sock_node, &rx->calls); set_bit(RXRPC_CALL_HAS_USERID, &call->flags); @@ -140,9 +138,9 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, write_unlock(&rx->call_lock); rxnet = call->rxnet; - spin_lock_bh(&rxnet->call_lock); + spin_lock(&rxnet->call_lock); list_add_tail_rcu(&call->link, &rxnet->calls); - spin_unlock_bh(&rxnet->call_lock); + spin_unlock(&rxnet->call_lock); b->call_backlog[call_head] = call; smp_store_release(&b->call_backlog_head, (call_head + 1) & (size - 1)); @@ -190,14 +188,14 @@ void rxrpc_discard_prealloc(struct rxrpc_sock *rx) /* Make sure that there aren't any incoming calls in progress before we * clear the preallocation buffers. */ - spin_lock_bh(&rx->incoming_lock); - spin_unlock_bh(&rx->incoming_lock); + spin_lock(&rx->incoming_lock); + spin_unlock(&rx->incoming_lock); head = b->peer_backlog_head; tail = b->peer_backlog_tail; while (CIRC_CNT(head, tail, size) > 0) { struct rxrpc_peer *peer = b->peer_backlog[tail]; - rxrpc_put_local(peer->local); + rxrpc_put_local(peer->local, rxrpc_local_put_prealloc_conn); kfree(peer); tail = (tail + 1) & (size - 1); } @@ -230,7 +228,7 @@ void rxrpc_discard_prealloc(struct rxrpc_sock *rx) } rxrpc_call_completed(call); rxrpc_release_call(rx, call); - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_discard_prealloc); tail = (tail + 1) & (size - 1); } @@ -238,21 +236,6 @@ void rxrpc_discard_prealloc(struct rxrpc_sock *rx) } /* - * Ping the other end to fill our RTT cache and to retrieve the rwind - * and MTU parameters. - */ -static void rxrpc_send_ping(struct rxrpc_call *call, struct sk_buff *skb) -{ - struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - ktime_t now = skb->tstamp; - - if (call->peer->rtt_count < 3 || - ktime_before(ktime_add_ms(call->peer->rtt_last_req, 1000), now)) - rxrpc_send_ACK(call, RXRPC_ACK_PING, sp->hdr.serial, - rxrpc_propose_ack_ping_for_params); -} - -/* * Allocate a new incoming call from the prealloc pool, along with a connection * and a peer as necessary. */ @@ -261,6 +244,7 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx, struct rxrpc_peer *peer, struct rxrpc_connection *conn, const struct rxrpc_security *sec, + struct sockaddr_rxrpc *peer_srx, struct sk_buff *skb) { struct rxrpc_backlog *b = rx->backlog; @@ -286,12 +270,11 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx, return NULL; if (!conn) { - if (peer && !rxrpc_get_peer_maybe(peer)) + if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_service_conn)) peer = NULL; if (!peer) { peer = b->peer_backlog[peer_tail]; - if (rxrpc_extract_addr_from_skb(&peer->srx, skb) < 0) - return NULL; + peer->srx = *peer_srx; b->peer_backlog[peer_tail] = NULL; smp_store_release(&b->peer_backlog_tail, (peer_tail + 1) & @@ -305,12 +288,13 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx, b->conn_backlog[conn_tail] = NULL; smp_store_release(&b->conn_backlog_tail, (conn_tail + 1) & (RXRPC_BACKLOG_MAX - 1)); - conn->params.local = rxrpc_get_local(local); - conn->params.peer = peer; - rxrpc_see_connection(conn); + conn->local = rxrpc_get_local(local, rxrpc_local_get_prealloc_conn); + conn->peer = peer; + rxrpc_see_connection(conn, rxrpc_conn_see_new_service_conn); rxrpc_new_incoming_connection(rx, conn, sec, skb); } else { - rxrpc_get_connection(conn); + rxrpc_get_connection(conn, rxrpc_conn_get_service_conn); + atomic_inc(&conn->active); } /* And now we can allocate and set up a new call */ @@ -319,43 +303,69 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx, smp_store_release(&b->call_backlog_tail, (call_tail + 1) & (RXRPC_BACKLOG_MAX - 1)); - rxrpc_see_call(call); + rxrpc_see_call(call, rxrpc_call_see_accept); + call->local = rxrpc_get_local(conn->local, rxrpc_local_get_call); call->conn = conn; call->security = conn->security; call->security_ix = conn->security_ix; - call->peer = rxrpc_get_peer(conn->params.peer); + call->peer = rxrpc_get_peer(conn->peer, rxrpc_peer_get_accept); + call->dest_srx = peer->srx; call->cong_ssthresh = call->peer->cong_ssthresh; call->tx_last_sent = ktime_get_real(); return call; } /* - * Set up a new incoming call. Called in BH context with the RCU read lock - * held. + * Set up a new incoming call. Called from the I/O thread. * * If this is for a kernel service, when we allocate the call, it will have * three refs on it: (1) the kernel service, (2) the user_call_ID tree, (3) the * retainer ref obtained from the backlog buffer. Prealloc calls for userspace - * services only have the ref from the backlog buffer. We want to pass this - * ref to non-BH context to dispose of. + * services only have the ref from the backlog buffer. * * If we want to report an error, we mark the skb with the packet type and - * abort code and return NULL. - * - * The call is returned with the user access mutex held. + * abort code and return false. */ -struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local, - struct rxrpc_sock *rx, - struct sk_buff *skb) +bool rxrpc_new_incoming_call(struct rxrpc_local *local, + struct rxrpc_peer *peer, + struct rxrpc_connection *conn, + struct sockaddr_rxrpc *peer_srx, + struct sk_buff *skb) { - struct rxrpc_skb_priv *sp = rxrpc_skb(skb); const struct rxrpc_security *sec = NULL; - struct rxrpc_connection *conn; - struct rxrpc_peer *peer = NULL; + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_call *call = NULL; + struct rxrpc_sock *rx; _enter(""); + /* Don't set up a call for anything other than the first DATA packet. */ + if (sp->hdr.seq != 1 || + sp->hdr.type != RXRPC_PACKET_TYPE_DATA) + return true; /* Just discard */ + + rcu_read_lock(); + + /* Weed out packets to services we're not offering. Packets that would + * begin a call are explicitly rejected and the rest are just + * discarded. + */ + rx = rcu_dereference(local->service); + if (!rx || (sp->hdr.serviceId != rx->srx.srx_service && + sp->hdr.serviceId != rx->second_service) + ) { + if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA && + sp->hdr.seq == 1) + goto unsupported_service; + goto discard; + } + + if (!conn) { + sec = rxrpc_get_incoming_security(rx, skb); + if (!sec) + goto reject; + } + spin_lock(&rx->incoming_lock); if (rx->sk.sk_state == RXRPC_SERVER_LISTEN_DISABLED || rx->sk.sk_state == RXRPC_CLOSE) { @@ -366,20 +376,8 @@ struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local, goto no_call; } - /* The peer, connection and call may all have sprung into existence due - * to a duplicate packet being handled on another CPU in parallel, so - * we have to recheck the routing. However, we're now holding - * rx->incoming_lock, so the values should remain stable. - */ - conn = rxrpc_find_connection_rcu(local, skb, &peer); - - if (!conn) { - sec = rxrpc_get_incoming_security(rx, skb); - if (!sec) - goto no_call; - } - - call = rxrpc_alloc_incoming_call(rx, local, peer, conn, sec, skb); + call = rxrpc_alloc_incoming_call(rx, local, peer, conn, sec, peer_srx, + skb); if (!call) { skb->mark = RXRPC_SKB_MARK_REJECT_BUSY; goto no_call; @@ -396,50 +394,41 @@ struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local, rx->notify_new_call(&rx->sk, call, call->user_call_ID); spin_lock(&conn->state_lock); - switch (conn->state) { - case RXRPC_CONN_SERVICE_UNSECURED: + if (conn->state == RXRPC_CONN_SERVICE_UNSECURED) { conn->state = RXRPC_CONN_SERVICE_CHALLENGING; set_bit(RXRPC_CONN_EV_CHALLENGE, &call->conn->events); - rxrpc_queue_conn(call->conn); - break; - - case RXRPC_CONN_SERVICE: - write_lock(&call->state_lock); - if (call->state < RXRPC_CALL_COMPLETE) - call->state = RXRPC_CALL_SERVER_RECV_REQUEST; - write_unlock(&call->state_lock); - break; - - case RXRPC_CONN_REMOTELY_ABORTED: - rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED, - conn->abort_code, conn->error); - break; - case RXRPC_CONN_LOCALLY_ABORTED: - rxrpc_abort_call("CON", call, sp->hdr.seq, - conn->abort_code, conn->error); - break; - default: - BUG(); + rxrpc_queue_conn(call->conn, rxrpc_conn_queue_challenge); } spin_unlock(&conn->state_lock); - spin_unlock(&rx->incoming_lock); - rxrpc_send_ping(call, skb); + spin_unlock(&rx->incoming_lock); + rcu_read_unlock(); - /* We have to discard the prealloc queue's ref here and rely on a - * combination of the RCU read lock and refs held either by the socket - * (recvmsg queue, to-be-accepted queue or user ID tree) or the kernel - * service to prevent the call from being deallocated too early. - */ - rxrpc_put_call(call, rxrpc_call_put); + if (hlist_unhashed(&call->error_link)) { + spin_lock(&call->peer->lock); + hlist_add_head(&call->error_link, &call->peer->error_targets); + spin_unlock(&call->peer->lock); + } _leave(" = %p{%d}", call, call->debug_id); - return call; - + rxrpc_input_call_event(call, skb); + rxrpc_put_call(call, rxrpc_call_put_input); + return true; + +unsupported_service: + trace_rxrpc_abort(0, "INV", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, + RX_INVALID_OPERATION, EOPNOTSUPP); + skb->priority = RX_INVALID_OPERATION; + goto reject; no_call: spin_unlock(&rx->incoming_lock); - _leave(" = NULL [%u]", skb->mark); - return NULL; +reject: + rcu_read_unlock(); + _leave(" = f [%u]", skb->mark); + return false; +discard: + rcu_read_unlock(); + return true; } /* diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index 1e21a708390e..b2cf448fb02c 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -69,21 +69,15 @@ void rxrpc_propose_delay_ACK(struct rxrpc_call *call, rxrpc_serial_t serial, void rxrpc_send_ACK(struct rxrpc_call *call, u8 ack_reason, rxrpc_serial_t serial, enum rxrpc_propose_ack_trace why) { - struct rxrpc_local *local = call->conn->params.local; struct rxrpc_txbuf *txb; if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags)) return; - if (ack_reason == RXRPC_ACK_DELAY && - test_and_set_bit(RXRPC_CALL_DELAY_ACK_PENDING, &call->flags)) { - trace_rxrpc_drop_ack(call, why, ack_reason, serial, false); - return; - } rxrpc_inc_stat(call->rxnet, stat_tx_acks[ack_reason]); txb = rxrpc_alloc_txbuf(call, RXRPC_PACKET_TYPE_ACK, - in_softirq() ? GFP_ATOMIC | __GFP_NOWARN : GFP_NOFS); + rcu_read_lock_held() ? GFP_ATOMIC | __GFP_NOWARN : GFP_NOFS); if (!txb) { kleave(" = -ENOMEM"); return; @@ -101,22 +95,9 @@ void rxrpc_send_ACK(struct rxrpc_call *call, u8 ack_reason, txb->ack.reason = ack_reason; txb->ack.nAcks = 0; - if (!rxrpc_try_get_call(call, rxrpc_call_got)) { - rxrpc_put_txbuf(txb, rxrpc_txbuf_put_nomem); - return; - } - - spin_lock_bh(&local->ack_tx_lock); - list_add_tail(&txb->tx_link, &local->ack_tx_queue); - spin_unlock_bh(&local->ack_tx_lock); trace_rxrpc_send_ack(call, why, ack_reason, serial); - - if (in_task()) { - rxrpc_transmit_ack_packets(call->peer->local); - } else { - rxrpc_get_local(local); - rxrpc_queue_local(local); - } + rxrpc_send_ack_packet(call, txb); + rxrpc_put_txbuf(txb, rxrpc_txbuf_put_ack_tx); } /* @@ -130,11 +111,10 @@ static void rxrpc_congestion_timeout(struct rxrpc_call *call) /* * Perform retransmission of NAK'd and unack'd packets. */ -static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) +void rxrpc_resend(struct rxrpc_call *call, struct sk_buff *ack_skb) { struct rxrpc_ackpacket *ack = NULL; struct rxrpc_txbuf *txb; - struct sk_buff *ack_skb = NULL; unsigned long resend_at; rxrpc_seq_t transmitted = READ_ONCE(call->tx_transmitted); ktime_t now, max_age, oldest, ack_ts; @@ -148,32 +128,21 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) max_age = ktime_sub_us(now, jiffies_to_usecs(call->peer->rto_j)); oldest = now; - /* See if there's an ACK saved with a soft-ACK table in it. */ - if (call->acks_soft_tbl) { - spin_lock_bh(&call->acks_ack_lock); - ack_skb = call->acks_soft_tbl; - if (ack_skb) { - rxrpc_get_skb(ack_skb, rxrpc_skb_ack); - ack = (void *)ack_skb->data + sizeof(struct rxrpc_wire_header); - } - spin_unlock_bh(&call->acks_ack_lock); - } - if (list_empty(&call->tx_buffer)) goto no_resend; - spin_lock(&call->tx_lock); - if (list_empty(&call->tx_buffer)) goto no_further_resend; - trace_rxrpc_resend(call); + trace_rxrpc_resend(call, ack_skb); txb = list_first_entry(&call->tx_buffer, struct rxrpc_txbuf, call_link); /* Scan the soft ACK table without dropping the lock and resend any * explicitly NAK'd packets. */ - if (ack) { + if (ack_skb) { + ack = (void *)ack_skb->data + sizeof(struct rxrpc_wire_header); + for (i = 0; i < ack->nAcks; i++) { rxrpc_seq_t seq; @@ -197,8 +166,6 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked); if (list_empty(&txb->tx_link)) { - rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans); - rxrpc_get_call(call, rxrpc_call_got_tx); list_add_tail(&txb->tx_link, &retrans_queue); set_bit(RXRPC_TXBUF_RESENT, &txb->flags); } @@ -242,7 +209,6 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) do_resend: unacked = true; if (list_empty(&txb->tx_link)) { - rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans); list_add_tail(&txb->tx_link, &retrans_queue); set_bit(RXRPC_TXBUF_RESENT, &txb->flags); rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans); @@ -250,10 +216,7 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) } no_further_resend: - spin_unlock(&call->tx_lock); no_resend: - rxrpc_free_skb(ack_skb, rxrpc_skb_freed); - resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest))); resend_at += jiffies + rxrpc_get_rto_backoff(call->peer, !list_empty(&retrans_queue)); @@ -267,7 +230,7 @@ no_resend: * retransmitting data. */ if (list_empty(&retrans_queue)) { - rxrpc_reduce_call_timer(call, resend_at, now_j, + rxrpc_reduce_call_timer(call, resend_at, jiffies, rxrpc_timer_set_for_resend); ack_ts = ktime_sub(now, call->acks_latest_ts); if (ktime_to_us(ack_ts) < (call->peer->srtt_us >> 3)) @@ -277,76 +240,134 @@ no_resend: goto out; } + /* Retransmit the queue */ while ((txb = list_first_entry_or_null(&retrans_queue, struct rxrpc_txbuf, tx_link))) { list_del_init(&txb->tx_link); - rxrpc_send_data_packet(call, txb); - rxrpc_put_txbuf(txb, rxrpc_txbuf_put_trans); - - trace_rxrpc_retransmit(call, txb->seq, - ktime_to_ns(ktime_sub(txb->last_sent, - max_age))); + rxrpc_transmit_one(call, txb); } out: _leave(""); } +static bool rxrpc_tx_window_has_space(struct rxrpc_call *call) +{ + unsigned int winsize = min_t(unsigned int, call->tx_winsize, + call->cong_cwnd + call->cong_extra); + rxrpc_seq_t window = call->acks_hard_ack, wtop = window + winsize; + rxrpc_seq_t tx_top = call->tx_top; + int space; + + space = wtop - tx_top; + return space > 0; +} + +/* + * Decant some if the sendmsg prepared queue into the transmission buffer. + */ +static void rxrpc_decant_prepared_tx(struct rxrpc_call *call) +{ + struct rxrpc_txbuf *txb; + + if (rxrpc_is_client_call(call) && + !test_bit(RXRPC_CALL_EXPOSED, &call->flags)) + rxrpc_expose_client_call(call); + + while ((txb = list_first_entry_or_null(&call->tx_sendmsg, + struct rxrpc_txbuf, call_link))) { + spin_lock(&call->tx_lock); + list_del(&txb->call_link); + spin_unlock(&call->tx_lock); + + call->tx_top = txb->seq; + list_add_tail(&txb->call_link, &call->tx_buffer); + + rxrpc_transmit_one(call, txb); + + if (!rxrpc_tx_window_has_space(call)) + break; + } +} + +static void rxrpc_transmit_some_data(struct rxrpc_call *call) +{ + switch (call->state) { + case RXRPC_CALL_SERVER_ACK_REQUEST: + if (list_empty(&call->tx_sendmsg)) + return; + fallthrough; + + case RXRPC_CALL_SERVER_SEND_REPLY: + case RXRPC_CALL_SERVER_AWAIT_ACK: + case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_REPLY: + if (!rxrpc_tx_window_has_space(call)) + return; + if (list_empty(&call->tx_sendmsg)) { + rxrpc_inc_stat(call->rxnet, stat_tx_data_underflow); + return; + } + rxrpc_decant_prepared_tx(call); + break; + default: + return; + } +} + +/* + * Ping the other end to fill our RTT cache and to retrieve the rwind + * and MTU parameters. + */ +static void rxrpc_send_initial_ping(struct rxrpc_call *call) +{ + if (call->peer->rtt_count < 3 || + ktime_before(ktime_add_ms(call->peer->rtt_last_req, 1000), + ktime_get_real())) + rxrpc_send_ACK(call, RXRPC_ACK_PING, 0, + rxrpc_propose_ack_ping_for_params); +} + /* * Handle retransmission and deferred ACK/abort generation. */ -void rxrpc_process_call(struct work_struct *work) +void rxrpc_input_call_event(struct rxrpc_call *call, struct sk_buff *skb) { - struct rxrpc_call *call = - container_of(work, struct rxrpc_call, processor); unsigned long now, next, t; - unsigned int iterations = 0; rxrpc_serial_t ackr_serial; + bool resend = false, expired = false; - rxrpc_see_call(call); + rxrpc_see_call(call, rxrpc_call_see_input); //printk("\n--------------------\n"); _enter("{%d,%s,%lx}", call->debug_id, rxrpc_call_states[call->state], call->events); -recheck_state: - /* Limit the number of times we do this before returning to the manager */ - iterations++; - if (iterations > 5) - goto requeue; - - if (test_and_clear_bit(RXRPC_CALL_EV_ABORT, &call->events)) { - rxrpc_send_abort_packet(call); - goto recheck_state; - } - - if (READ_ONCE(call->acks_hard_ack) != call->tx_bottom) - rxrpc_shrink_call_tx_buffer(call); + if (call->state == RXRPC_CALL_COMPLETE) + goto out; - if (call->state == RXRPC_CALL_COMPLETE) { - rxrpc_delete_call_timer(call); - goto out_put; - } + if (skb && skb->mark == RXRPC_SKB_MARK_ERROR) + goto out; - /* Work out if any timeouts tripped */ + /* If we see our async-event poke, check for timeout trippage. */ now = jiffies; t = READ_ONCE(call->expect_rx_by); if (time_after_eq(now, t)) { trace_rxrpc_timer(call, rxrpc_timer_exp_normal, now); - set_bit(RXRPC_CALL_EV_EXPIRED, &call->events); + expired = true; } t = READ_ONCE(call->expect_req_by); if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST && time_after_eq(now, t)) { trace_rxrpc_timer(call, rxrpc_timer_exp_idle, now); - set_bit(RXRPC_CALL_EV_EXPIRED, &call->events); + expired = true; } t = READ_ONCE(call->expect_term_by); if (time_after_eq(now, t)) { trace_rxrpc_timer(call, rxrpc_timer_exp_hard, now); - set_bit(RXRPC_CALL_EV_EXPIRED, &call->events); + expired = true; } t = READ_ONCE(call->delay_ack_at); @@ -385,11 +406,26 @@ recheck_state: if (time_after_eq(now, t)) { trace_rxrpc_timer(call, rxrpc_timer_exp_resend, now); cmpxchg(&call->resend_at, t, now + MAX_JIFFY_OFFSET); - set_bit(RXRPC_CALL_EV_RESEND, &call->events); + resend = true; } + if (skb) + rxrpc_input_call_packet(call, skb); + + rxrpc_transmit_some_data(call); + + if (skb) { + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + + if (sp->hdr.type == RXRPC_PACKET_TYPE_ACK) + rxrpc_congestion_degrade(call); + } + + if (test_and_clear_bit(RXRPC_CALL_EV_INITIAL_PING, &call->events)) + rxrpc_send_initial_ping(call); + /* Process events */ - if (test_and_clear_bit(RXRPC_CALL_EV_EXPIRED, &call->events)) { + if (expired) { if (test_bit(RXRPC_CALL_RX_HEARD, &call->flags) && (int)call->conn->hi_serial - (int)call->rx_serial > 0) { trace_rxrpc_call_reset(call); @@ -397,52 +433,50 @@ recheck_state: } else { rxrpc_abort_call("EXP", call, 0, RX_CALL_TIMEOUT, -ETIME); } - set_bit(RXRPC_CALL_EV_ABORT, &call->events); - goto recheck_state; + rxrpc_send_abort_packet(call); + goto out; } - if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events)) { - call->acks_lost_top = call->tx_top; + if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events)) rxrpc_send_ACK(call, RXRPC_ACK_PING, 0, rxrpc_propose_ack_ping_for_lost_ack); - } - if (test_and_clear_bit(RXRPC_CALL_EV_RESEND, &call->events) && - call->state != RXRPC_CALL_CLIENT_RECV_REPLY) { - rxrpc_resend(call, now); - goto recheck_state; - } + if (resend && call->state != RXRPC_CALL_CLIENT_RECV_REPLY) + rxrpc_resend(call, NULL); + + if (test_and_clear_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags)) + rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0, + rxrpc_propose_ack_rx_idle); + + if (atomic_read(&call->ackr_nr_unacked) > 2) + rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0, + rxrpc_propose_ack_input_data); /* Make sure the timer is restarted */ - next = call->expect_rx_by; + if (call->state != RXRPC_CALL_COMPLETE) { + next = call->expect_rx_by; #define set(T) { t = READ_ONCE(T); if (time_before(t, next)) next = t; } - set(call->expect_req_by); - set(call->expect_term_by); - set(call->delay_ack_at); - set(call->ack_lost_at); - set(call->resend_at); - set(call->keepalive_at); - set(call->ping_at); - - now = jiffies; - if (time_after_eq(now, next)) - goto recheck_state; + set(call->expect_req_by); + set(call->expect_term_by); + set(call->delay_ack_at); + set(call->ack_lost_at); + set(call->resend_at); + set(call->keepalive_at); + set(call->ping_at); - rxrpc_reduce_call_timer(call, next, now, rxrpc_timer_restart); + now = jiffies; + if (time_after_eq(now, next)) + rxrpc_poke_call(call, rxrpc_call_poke_timer_now); - /* other events may have been raised since we started checking */ - if (call->events && call->state < RXRPC_CALL_COMPLETE) - goto requeue; + rxrpc_reduce_call_timer(call, next, now, rxrpc_timer_restart); + } -out_put: - rxrpc_put_call(call, rxrpc_call_put); out: + if (call->state == RXRPC_CALL_COMPLETE) + del_timer_sync(&call->timer); + if (call->acks_hard_ack != call->tx_bottom) + rxrpc_shrink_call_tx_buffer(call); _leave(""); - return; - -requeue: - __rxrpc_queue_call(call); - goto out; } diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index 1befe22cd301..be5eb8cdf549 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -45,6 +45,24 @@ static struct semaphore rxrpc_call_limiter = static struct semaphore rxrpc_kernel_call_limiter = __SEMAPHORE_INITIALIZER(rxrpc_kernel_call_limiter, 1000); +void rxrpc_poke_call(struct rxrpc_call *call, enum rxrpc_call_poke_trace what) +{ + struct rxrpc_local *local = call->local; + bool busy; + + if (call->state < RXRPC_CALL_COMPLETE) { + spin_lock_bh(&local->lock); + busy = !list_empty(&call->attend_link); + trace_rxrpc_poke_call(call, busy, what); + if (!busy) { + rxrpc_get_call(call, rxrpc_call_get_poke); + list_add_tail(&call->attend_link, &local->call_attend_q); + } + spin_unlock_bh(&local->lock); + rxrpc_wake_up_io_thread(local); + } +} + static void rxrpc_call_timer_expired(struct timer_list *t) { struct rxrpc_call *call = from_timer(call, t, timer); @@ -53,9 +71,7 @@ static void rxrpc_call_timer_expired(struct timer_list *t) if (call->state < RXRPC_CALL_COMPLETE) { trace_rxrpc_timer_expired(call, jiffies); - __rxrpc_queue_call(call); - } else { - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_poke_call(call, rxrpc_call_poke_timer); } } @@ -64,21 +80,14 @@ void rxrpc_reduce_call_timer(struct rxrpc_call *call, unsigned long now, enum rxrpc_timer_trace why) { - if (rxrpc_try_get_call(call, rxrpc_call_got_timer)) { - trace_rxrpc_timer(call, why, now); - if (timer_reduce(&call->timer, expire_at)) - rxrpc_put_call(call, rxrpc_call_put_notimer); - } -} - -void rxrpc_delete_call_timer(struct rxrpc_call *call) -{ - if (del_timer_sync(&call->timer)) - rxrpc_put_call(call, rxrpc_call_put_timer); + trace_rxrpc_timer(call, why, now); + timer_reduce(&call->timer, expire_at); } static struct lock_class_key rxrpc_call_user_mutex_lock_class_key; +static void rxrpc_destroy_call(struct work_struct *); + /* * find an extant server call * - called in process context with IRQs enabled @@ -110,7 +119,7 @@ struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *rx, return NULL; found_extant_call: - rxrpc_get_call(call, rxrpc_call_got); + rxrpc_get_call(call, rxrpc_call_get_sendmsg); read_unlock(&rx->call_lock); _leave(" = %p [%d]", call, refcount_read(&call->ref)); return call; @@ -139,20 +148,20 @@ struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *rx, gfp_t gfp, &rxrpc_call_user_mutex_lock_class_key); timer_setup(&call->timer, rxrpc_call_timer_expired, 0); - INIT_WORK(&call->processor, &rxrpc_process_call); + INIT_WORK(&call->destroyer, rxrpc_destroy_call); INIT_LIST_HEAD(&call->link); INIT_LIST_HEAD(&call->chan_wait_link); INIT_LIST_HEAD(&call->accept_link); INIT_LIST_HEAD(&call->recvmsg_link); INIT_LIST_HEAD(&call->sock_link); + INIT_LIST_HEAD(&call->attend_link); + INIT_LIST_HEAD(&call->tx_sendmsg); INIT_LIST_HEAD(&call->tx_buffer); skb_queue_head_init(&call->recvmsg_queue); skb_queue_head_init(&call->rx_oos_queue); init_waitqueue_head(&call->waitq); spin_lock_init(&call->notify_lock); spin_lock_init(&call->tx_lock); - spin_lock_init(&call->input_lock); - spin_lock_init(&call->acks_ack_lock); rwlock_init(&call->state_lock); refcount_set(&call->ref, 1); call->debug_id = debug_id; @@ -185,22 +194,45 @@ struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *rx, gfp_t gfp, */ static struct rxrpc_call *rxrpc_alloc_client_call(struct rxrpc_sock *rx, struct sockaddr_rxrpc *srx, + struct rxrpc_conn_parameters *cp, + struct rxrpc_call_params *p, gfp_t gfp, unsigned int debug_id) { struct rxrpc_call *call; ktime_t now; + int ret; _enter(""); call = rxrpc_alloc_call(rx, gfp, debug_id); if (!call) return ERR_PTR(-ENOMEM); - call->state = RXRPC_CALL_CLIENT_AWAIT_CONN; - call->service_id = srx->srx_service; now = ktime_get_real(); - call->acks_latest_ts = now; - call->cong_tstamp = now; + call->acks_latest_ts = now; + call->cong_tstamp = now; + call->state = RXRPC_CALL_CLIENT_AWAIT_CONN; + call->dest_srx = *srx; + call->interruptibility = p->interruptibility; + call->tx_total_len = p->tx_total_len; + call->key = key_get(cp->key); + call->local = rxrpc_get_local(cp->local, rxrpc_local_get_call); + if (p->kernel) + __set_bit(RXRPC_CALL_KERNEL, &call->flags); + if (cp->upgrade) + __set_bit(RXRPC_CALL_UPGRADE, &call->flags); + if (cp->exclusive) + __set_bit(RXRPC_CALL_EXCLUSIVE, &call->flags); + + ret = rxrpc_init_client_call_security(call); + if (ret < 0) { + __rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret); + rxrpc_put_call(call, rxrpc_call_put_discard_error); + return ERR_PTR(ret); + } + + trace_rxrpc_call(call->debug_id, refcount_read(&call->ref), + p->user_call_ID, rxrpc_call_new_client); _leave(" = %p", call); return call; @@ -218,6 +250,7 @@ static void rxrpc_start_call_timer(struct rxrpc_call *call) call->ack_lost_at = j; call->resend_at = j; call->ping_at = j; + call->keepalive_at = j; call->expect_rx_by = j; call->expect_req_by = j; call->expect_term_by = j; @@ -270,7 +303,6 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, struct rxrpc_net *rxnet; struct semaphore *limiter; struct rb_node *parent, **pp; - const void *here = __builtin_return_address(0); int ret; _enter("%p,%lx", rx, p->user_call_ID); @@ -281,7 +313,7 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, return ERR_PTR(-ERESTARTSYS); } - call = rxrpc_alloc_client_call(rx, srx, gfp, debug_id); + call = rxrpc_alloc_client_call(rx, srx, cp, p, gfp, debug_id); if (IS_ERR(call)) { release_sock(&rx->sk); up(limiter); @@ -289,14 +321,6 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, return call; } - call->interruptibility = p->interruptibility; - call->tx_total_len = p->tx_total_len; - trace_rxrpc_call(call->debug_id, rxrpc_call_new_client, - refcount_read(&call->ref), - here, (const void *)p->user_call_ID); - if (p->kernel) - __set_bit(RXRPC_CALL_KERNEL, &call->flags); - /* We need to protect a partially set up call against the user as we * will be acting outside the socket lock. */ @@ -322,7 +346,7 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, rcu_assign_pointer(call->socket, rx); call->user_call_ID = p->user_call_ID; __set_bit(RXRPC_CALL_HAS_USERID, &call->flags); - rxrpc_get_call(call, rxrpc_call_got_userid); + rxrpc_get_call(call, rxrpc_call_get_userid); rb_link_node(&call->sock_node, parent, pp); rb_insert_color(&call->sock_node, &rx->calls); list_add(&call->sock_link, &rx->sock_calls); @@ -330,9 +354,9 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, write_unlock(&rx->call_lock); rxnet = call->rxnet; - spin_lock_bh(&rxnet->call_lock); + spin_lock(&rxnet->call_lock); list_add_tail_rcu(&call->link, &rxnet->calls); - spin_unlock_bh(&rxnet->call_lock); + spin_unlock(&rxnet->call_lock); /* From this point on, the call is protected by its own lock. */ release_sock(&rx->sk); @@ -344,13 +368,10 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, if (ret < 0) goto error_attached_to_socket; - trace_rxrpc_call(call->debug_id, rxrpc_call_connected, - refcount_read(&call->ref), here, NULL); + rxrpc_see_call(call, rxrpc_call_see_connected); rxrpc_start_call_timer(call); - _net("CALL new %d on CONN %d", call->debug_id, call->conn->debug_id); - _leave(" = %p [new]", call); return call; @@ -364,11 +385,11 @@ error_dup_user_ID: release_sock(&rx->sk); __rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, RX_CALL_DEAD, -EEXIST); - trace_rxrpc_call(call->debug_id, rxrpc_call_error, - refcount_read(&call->ref), here, ERR_PTR(-EEXIST)); + trace_rxrpc_call(call->debug_id, refcount_read(&call->ref), 0, + rxrpc_call_see_userid_exists); rxrpc_release_call(rx, call); mutex_unlock(&call->user_mutex); - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_userid_exists); _leave(" = -EEXIST"); return ERR_PTR(-EEXIST); @@ -378,8 +399,8 @@ error_dup_user_ID: * leave the error to recvmsg() to deal with. */ error_attached_to_socket: - trace_rxrpc_call(call->debug_id, rxrpc_call_error, - refcount_read(&call->ref), here, ERR_PTR(ret)); + trace_rxrpc_call(call->debug_id, refcount_read(&call->ref), ret, + rxrpc_call_see_connect_failed); set_bit(RXRPC_CALL_DISCONNECTED, &call->flags); __rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, RX_CALL_DEAD, ret); @@ -403,11 +424,34 @@ void rxrpc_incoming_call(struct rxrpc_sock *rx, rcu_assign_pointer(call->socket, rx); call->call_id = sp->hdr.callNumber; - call->service_id = sp->hdr.serviceId; + call->dest_srx.srx_service = sp->hdr.serviceId; call->cid = sp->hdr.cid; call->state = RXRPC_CALL_SERVER_SECURING; call->cong_tstamp = skb->tstamp; + spin_lock(&conn->state_lock); + + switch (conn->state) { + case RXRPC_CONN_SERVICE_UNSECURED: + case RXRPC_CONN_SERVICE_CHALLENGING: + call->state = RXRPC_CALL_SERVER_SECURING; + break; + case RXRPC_CONN_SERVICE: + call->state = RXRPC_CALL_SERVER_RECV_REQUEST; + break; + + case RXRPC_CONN_REMOTELY_ABORTED: + __rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED, + conn->abort_code, conn->error); + break; + case RXRPC_CONN_LOCALLY_ABORTED: + __rxrpc_abort_call("CON", call, 1, + conn->abort_code, conn->error); + break; + default: + BUG(); + } + /* Set the channel for this call. We don't get channel_lock as we're * only defending against the data_ready handler (which we're called * from) and the RESPONSE packet parser (which is only really @@ -418,86 +462,48 @@ void rxrpc_incoming_call(struct rxrpc_sock *rx, conn->channels[chan].call_counter = call->call_id; conn->channels[chan].call_id = call->call_id; rcu_assign_pointer(conn->channels[chan].call, call); + spin_unlock(&conn->state_lock); - spin_lock(&conn->params.peer->lock); - hlist_add_head_rcu(&call->error_link, &conn->params.peer->error_targets); - spin_unlock(&conn->params.peer->lock); - - _net("CALL incoming %d on CONN %d", call->debug_id, call->conn->debug_id); + spin_lock(&conn->peer->lock); + hlist_add_head(&call->error_link, &conn->peer->error_targets); + spin_unlock(&conn->peer->lock); rxrpc_start_call_timer(call); _leave(""); } /* - * Queue a call's work processor, getting a ref to pass to the work queue. - */ -bool rxrpc_queue_call(struct rxrpc_call *call) -{ - const void *here = __builtin_return_address(0); - int n; - - if (!__refcount_inc_not_zero(&call->ref, &n)) - return false; - if (rxrpc_queue_work(&call->processor)) - trace_rxrpc_call(call->debug_id, rxrpc_call_queued, n + 1, - here, NULL); - else - rxrpc_put_call(call, rxrpc_call_put_noqueue); - return true; -} - -/* - * Queue a call's work processor, passing the callers ref to the work queue. - */ -bool __rxrpc_queue_call(struct rxrpc_call *call) -{ - const void *here = __builtin_return_address(0); - int n = refcount_read(&call->ref); - ASSERTCMP(n, >=, 1); - if (rxrpc_queue_work(&call->processor)) - trace_rxrpc_call(call->debug_id, rxrpc_call_queued_ref, n, - here, NULL); - else - rxrpc_put_call(call, rxrpc_call_put_noqueue); - return true; -} - -/* * Note the re-emergence of a call. */ -void rxrpc_see_call(struct rxrpc_call *call) +void rxrpc_see_call(struct rxrpc_call *call, enum rxrpc_call_trace why) { - const void *here = __builtin_return_address(0); if (call) { - int n = refcount_read(&call->ref); + int r = refcount_read(&call->ref); - trace_rxrpc_call(call->debug_id, rxrpc_call_seen, n, - here, NULL); + trace_rxrpc_call(call->debug_id, r, 0, why); } } -bool rxrpc_try_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op) +struct rxrpc_call *rxrpc_try_get_call(struct rxrpc_call *call, + enum rxrpc_call_trace why) { - const void *here = __builtin_return_address(0); - int n; + int r; - if (!__refcount_inc_not_zero(&call->ref, &n)) - return false; - trace_rxrpc_call(call->debug_id, op, n + 1, here, NULL); - return true; + if (!call || !__refcount_inc_not_zero(&call->ref, &r)) + return NULL; + trace_rxrpc_call(call->debug_id, r + 1, 0, why); + return call; } /* * Note the addition of a ref on a call. */ -void rxrpc_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op) +void rxrpc_get_call(struct rxrpc_call *call, enum rxrpc_call_trace why) { - const void *here = __builtin_return_address(0); - int n; + int r; - __refcount_inc(&call->ref, &n); - trace_rxrpc_call(call->debug_id, op, n + 1, here, NULL); + __refcount_inc(&call->ref, &r); + trace_rxrpc_call(call->debug_id, r + 1, 0, why); } /* @@ -514,15 +520,13 @@ static void rxrpc_cleanup_ring(struct rxrpc_call *call) */ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call) { - const void *here = __builtin_return_address(0); struct rxrpc_connection *conn = call->conn; bool put = false; _enter("{%d,%d}", call->debug_id, refcount_read(&call->ref)); - trace_rxrpc_call(call->debug_id, rxrpc_call_release, - refcount_read(&call->ref), - here, (const void *)call->flags); + trace_rxrpc_call(call->debug_id, refcount_read(&call->ref), + call->flags, rxrpc_call_see_release); ASSERTCMP(call->state, ==, RXRPC_CALL_COMPLETE); @@ -530,10 +534,10 @@ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call) BUG(); rxrpc_put_call_slot(call); - rxrpc_delete_call_timer(call); + del_timer_sync(&call->timer); /* Make sure we don't get any more notifications */ - write_lock_bh(&rx->recvmsg_lock); + write_lock(&rx->recvmsg_lock); if (!list_empty(&call->recvmsg_link)) { _debug("unlinking once-pending call %p { e=%lx f=%lx }", @@ -546,16 +550,16 @@ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call) call->recvmsg_link.next = NULL; call->recvmsg_link.prev = NULL; - write_unlock_bh(&rx->recvmsg_lock); + write_unlock(&rx->recvmsg_lock); if (put) - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_unnotify); write_lock(&rx->call_lock); if (test_and_clear_bit(RXRPC_CALL_HAS_USERID, &call->flags)) { rb_erase(&call->sock_node, &rx->calls); memset(&call->sock_node, 0xdd, sizeof(call->sock_node)); - rxrpc_put_call(call, rxrpc_call_put_userid); + rxrpc_put_call(call, rxrpc_call_put_userid_exists); } list_del(&call->sock_link); @@ -584,17 +588,17 @@ void rxrpc_release_calls_on_socket(struct rxrpc_sock *rx) struct rxrpc_call, accept_link); list_del(&call->accept_link); rxrpc_abort_call("SKR", call, 0, RX_CALL_DEAD, -ECONNRESET); - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_release_sock_tba); } while (!list_empty(&rx->sock_calls)) { call = list_entry(rx->sock_calls.next, struct rxrpc_call, sock_link); - rxrpc_get_call(call, rxrpc_call_got); + rxrpc_get_call(call, rxrpc_call_get_release_sock); rxrpc_abort_call("SKT", call, 0, RX_CALL_DEAD, -ECONNRESET); rxrpc_send_abort_packet(call); rxrpc_release_call(rx, call); - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_release_sock); } _leave(""); @@ -603,26 +607,24 @@ void rxrpc_release_calls_on_socket(struct rxrpc_sock *rx) /* * release a call */ -void rxrpc_put_call(struct rxrpc_call *call, enum rxrpc_call_trace op) +void rxrpc_put_call(struct rxrpc_call *call, enum rxrpc_call_trace why) { struct rxrpc_net *rxnet = call->rxnet; - const void *here = __builtin_return_address(0); unsigned int debug_id = call->debug_id; bool dead; - int n; + int r; ASSERT(call != NULL); - dead = __refcount_dec_and_test(&call->ref, &n); - trace_rxrpc_call(debug_id, op, n, here, NULL); + dead = __refcount_dec_and_test(&call->ref, &r); + trace_rxrpc_call(debug_id, r - 1, 0, why); if (dead) { - _debug("call %d dead", call->debug_id); ASSERTCMP(call->state, ==, RXRPC_CALL_COMPLETE); if (!list_empty(&call->link)) { - spin_lock_bh(&rxnet->call_lock); + spin_lock(&rxnet->call_lock); list_del_init(&call->link); - spin_unlock_bh(&rxnet->call_lock); + spin_unlock(&rxnet->call_lock); } rxrpc_cleanup_call(call); @@ -630,36 +632,45 @@ void rxrpc_put_call(struct rxrpc_call *call, enum rxrpc_call_trace op) } /* - * Final call destruction - but must be done in process context. + * Free up the call under RCU. */ -static void rxrpc_destroy_call(struct work_struct *work) +static void rxrpc_rcu_free_call(struct rcu_head *rcu) { - struct rxrpc_call *call = container_of(work, struct rxrpc_call, processor); - struct rxrpc_net *rxnet = call->rxnet; - - rxrpc_delete_call_timer(call); + struct rxrpc_call *call = container_of(rcu, struct rxrpc_call, rcu); + struct rxrpc_net *rxnet = READ_ONCE(call->rxnet); - rxrpc_put_connection(call->conn); - rxrpc_put_peer(call->peer); kmem_cache_free(rxrpc_call_jar, call); if (atomic_dec_and_test(&rxnet->nr_calls)) wake_up_var(&rxnet->nr_calls); } /* - * Final call destruction under RCU. + * Final call destruction - but must be done in process context. */ -static void rxrpc_rcu_destroy_call(struct rcu_head *rcu) +static void rxrpc_destroy_call(struct work_struct *work) { - struct rxrpc_call *call = container_of(rcu, struct rxrpc_call, rcu); + struct rxrpc_call *call = container_of(work, struct rxrpc_call, destroyer); + struct rxrpc_txbuf *txb; - if (in_softirq()) { - INIT_WORK(&call->processor, rxrpc_destroy_call); - if (!rxrpc_queue_work(&call->processor)) - BUG(); - } else { - rxrpc_destroy_call(&call->processor); + del_timer_sync(&call->timer); + + rxrpc_cleanup_ring(call); + while ((txb = list_first_entry_or_null(&call->tx_sendmsg, + struct rxrpc_txbuf, call_link))) { + list_del(&txb->call_link); + rxrpc_put_txbuf(txb, rxrpc_txbuf_put_cleaned); } + while ((txb = list_first_entry_or_null(&call->tx_buffer, + struct rxrpc_txbuf, call_link))) { + list_del(&txb->call_link); + rxrpc_put_txbuf(txb, rxrpc_txbuf_put_cleaned); + } + + rxrpc_put_txbuf(call->tx_pending, rxrpc_txbuf_put_cleaned); + rxrpc_put_connection(call->conn, rxrpc_conn_put_call); + rxrpc_put_peer(call->peer, rxrpc_peer_put_call); + rxrpc_put_local(call->local, rxrpc_local_put_call); + call_rcu(&call->rcu, rxrpc_rcu_free_call); } /* @@ -667,25 +678,20 @@ static void rxrpc_rcu_destroy_call(struct rcu_head *rcu) */ void rxrpc_cleanup_call(struct rxrpc_call *call) { - struct rxrpc_txbuf *txb; - - _net("DESTROY CALL %d", call->debug_id); - memset(&call->sock_node, 0xcd, sizeof(call->sock_node)); ASSERTCMP(call->state, ==, RXRPC_CALL_COMPLETE); ASSERT(test_bit(RXRPC_CALL_RELEASED, &call->flags)); - rxrpc_cleanup_ring(call); - while ((txb = list_first_entry_or_null(&call->tx_buffer, - struct rxrpc_txbuf, call_link))) { - list_del(&txb->call_link); - rxrpc_put_txbuf(txb, rxrpc_txbuf_put_cleaned); - } - rxrpc_put_txbuf(call->tx_pending, rxrpc_txbuf_put_cleaned); - rxrpc_free_skb(call->acks_soft_tbl, rxrpc_skb_cleaned); + del_timer(&call->timer); - call_rcu(&call->rcu, rxrpc_rcu_destroy_call); + if (rcu_read_lock_held()) + /* Can't use the rxrpc workqueue as we need to cancel/flush + * something that may be running/waiting there. + */ + schedule_work(&call->destroyer); + else + rxrpc_destroy_call(&call->destroyer); } /* @@ -700,14 +706,14 @@ void rxrpc_destroy_all_calls(struct rxrpc_net *rxnet) _enter(""); if (!list_empty(&rxnet->calls)) { - spin_lock_bh(&rxnet->call_lock); + spin_lock(&rxnet->call_lock); while (!list_empty(&rxnet->calls)) { call = list_entry(rxnet->calls.next, struct rxrpc_call, link); _debug("Zapping call %p", call); - rxrpc_see_call(call); + rxrpc_see_call(call, rxrpc_call_see_zap); list_del_init(&call->link); pr_err("Call %p still in use (%d,%s,%lx,%lx)!\n", @@ -715,12 +721,12 @@ void rxrpc_destroy_all_calls(struct rxrpc_net *rxnet) rxrpc_call_states[call->state], call->flags, call->events); - spin_unlock_bh(&rxnet->call_lock); + spin_unlock(&rxnet->call_lock); cond_resched(); - spin_lock_bh(&rxnet->call_lock); + spin_lock(&rxnet->call_lock); } - spin_unlock_bh(&rxnet->call_lock); + spin_unlock(&rxnet->call_lock); } atomic_dec(&rxnet->nr_calls); diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c index f11c97e28d2a..a08e33c9e54b 100644 --- a/net/rxrpc/conn_client.c +++ b/net/rxrpc/conn_client.c @@ -51,7 +51,7 @@ static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle); static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn, gfp_t gfp) { - struct rxrpc_net *rxnet = conn->params.local->rxnet; + struct rxrpc_net *rxnet = conn->rxnet; int id; _enter(""); @@ -122,37 +122,47 @@ static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp, bundle = kzalloc(sizeof(*bundle), gfp); if (bundle) { - bundle->params = *cp; - rxrpc_get_peer(bundle->params.peer); + bundle->local = cp->local; + bundle->peer = rxrpc_get_peer(cp->peer, rxrpc_peer_get_bundle); + bundle->key = cp->key; + bundle->exclusive = cp->exclusive; + bundle->upgrade = cp->upgrade; + bundle->service_id = cp->service_id; + bundle->security_level = cp->security_level; refcount_set(&bundle->ref, 1); atomic_set(&bundle->active, 1); spin_lock_init(&bundle->channel_lock); INIT_LIST_HEAD(&bundle->waiting_calls); + trace_rxrpc_bundle(bundle->debug_id, 1, rxrpc_bundle_new); } return bundle; } -struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle) +struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle, + enum rxrpc_bundle_trace why) { - refcount_inc(&bundle->ref); + int r; + + __refcount_inc(&bundle->ref, &r); + trace_rxrpc_bundle(bundle->debug_id, r + 1, why); return bundle; } static void rxrpc_free_bundle(struct rxrpc_bundle *bundle) { - rxrpc_put_peer(bundle->params.peer); + trace_rxrpc_bundle(bundle->debug_id, 1, rxrpc_bundle_free); + rxrpc_put_peer(bundle->peer, rxrpc_peer_put_bundle); kfree(bundle); } -void rxrpc_put_bundle(struct rxrpc_bundle *bundle) +void rxrpc_put_bundle(struct rxrpc_bundle *bundle, enum rxrpc_bundle_trace why) { - unsigned int d = bundle->debug_id; + unsigned int id = bundle->debug_id; bool dead; int r; dead = __refcount_dec_and_test(&bundle->ref, &r); - - _debug("PUT B=%x %d", d, r - 1); + trace_rxrpc_bundle(id, r - 1, why); if (dead) rxrpc_free_bundle(bundle); } @@ -164,12 +174,12 @@ static struct rxrpc_connection * rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp) { struct rxrpc_connection *conn; - struct rxrpc_net *rxnet = bundle->params.local->rxnet; + struct rxrpc_net *rxnet = bundle->local->rxnet; int ret; _enter(""); - conn = rxrpc_alloc_connection(gfp); + conn = rxrpc_alloc_connection(rxnet, gfp); if (!conn) { _leave(" = -ENOMEM"); return ERR_PTR(-ENOMEM); @@ -177,10 +187,16 @@ rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp) refcount_set(&conn->ref, 1); conn->bundle = bundle; - conn->params = bundle->params; + conn->local = bundle->local; + conn->peer = bundle->peer; + conn->key = bundle->key; + conn->exclusive = bundle->exclusive; + conn->upgrade = bundle->upgrade; + conn->orig_service_id = bundle->service_id; + conn->security_level = bundle->security_level; conn->out_clientflag = RXRPC_CLIENT_INITIATED; conn->state = RXRPC_CONN_CLIENT; - conn->service_id = conn->params.service_id; + conn->service_id = conn->orig_service_id; ret = rxrpc_get_client_connection_id(conn, gfp); if (ret < 0) @@ -195,14 +211,13 @@ rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp) list_add_tail(&conn->proc_link, &rxnet->conn_proc_list); write_unlock(&rxnet->conn_lock); - rxrpc_get_bundle(bundle); - rxrpc_get_peer(conn->params.peer); - rxrpc_get_local(conn->params.local); - key_get(conn->params.key); + rxrpc_get_bundle(bundle, rxrpc_bundle_get_client_conn); + rxrpc_get_peer(conn->peer, rxrpc_peer_get_client_conn); + rxrpc_get_local(conn->local, rxrpc_local_get_client_conn); + key_get(conn->key); - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client, - refcount_read(&conn->ref), - __builtin_return_address(0)); + trace_rxrpc_conn(conn->debug_id, refcount_read(&conn->ref), + rxrpc_conn_new_client); atomic_inc(&rxnet->nr_client_conns); trace_rxrpc_client(conn, -1, rxrpc_client_alloc); @@ -228,7 +243,7 @@ static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn) if (!conn) goto dont_reuse; - rxnet = conn->params.local->rxnet; + rxnet = conn->rxnet; if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags)) goto dont_reuse; @@ -285,7 +300,7 @@ static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *c while (p) { bundle = rb_entry(p, struct rxrpc_bundle, local_node); -#define cmp(X) ((long)bundle->params.X - (long)cp->X) +#define cmp(X) ((long)bundle->X - (long)cp->X) diff = (cmp(peer) ?: cmp(key) ?: cmp(security_level) ?: @@ -314,7 +329,7 @@ static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *c parent = *pp; bundle = rb_entry(parent, struct rxrpc_bundle, local_node); -#define cmp(X) ((long)bundle->params.X - (long)cp->X) +#define cmp(X) ((long)bundle->X - (long)cp->X) diff = (cmp(peer) ?: cmp(key) ?: cmp(security_level) ?: @@ -332,7 +347,7 @@ static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *c candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id); rb_link_node(&candidate->local_node, parent, pp); rb_insert_color(&candidate->local_node, &local->client_bundles); - rxrpc_get_bundle(candidate); + rxrpc_get_bundle(candidate, rxrpc_bundle_get_client_call); spin_unlock(&local->client_bundles_lock); _leave(" = %u [new]", candidate->debug_id); return candidate; @@ -340,7 +355,7 @@ static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *c found_bundle_free: rxrpc_free_bundle(candidate); found_bundle: - rxrpc_get_bundle(bundle); + rxrpc_get_bundle(bundle, rxrpc_bundle_get_client_call); atomic_inc(&bundle->active); spin_unlock(&local->client_bundles_lock); _leave(" = %u [found]", bundle->debug_id); @@ -456,10 +471,10 @@ static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp) if (candidate) { _debug("discard C=%x", candidate->debug_id); trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate); - rxrpc_put_connection(candidate); + rxrpc_put_connection(candidate, rxrpc_conn_put_discard); } - rxrpc_put_connection(old); + rxrpc_put_connection(old, rxrpc_conn_put_noreuse); _leave(""); } @@ -530,23 +545,21 @@ static void rxrpc_activate_one_channel(struct rxrpc_connection *conn, clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags); clear_bit(conn->bundle_shift + channel, &bundle->avail_chans); - rxrpc_see_call(call); + rxrpc_see_call(call, rxrpc_call_see_activate_client); list_del_init(&call->chan_wait_link); - call->peer = rxrpc_get_peer(conn->params.peer); - call->conn = rxrpc_get_connection(conn); + call->peer = rxrpc_get_peer(conn->peer, rxrpc_peer_get_activate_call); + call->conn = rxrpc_get_connection(conn, rxrpc_conn_get_activate_call); call->cid = conn->proto.cid | channel; call->call_id = call_id; call->security = conn->security; call->security_ix = conn->security_ix; - call->service_id = conn->service_id; + call->dest_srx.srx_service = conn->service_id; trace_rxrpc_connect_call(call); - _net("CONNECT call %08x:%08x as call %d on conn %d", - call->cid, call->call_id, call->debug_id, conn->debug_id); - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); call->state = RXRPC_CALL_CLIENT_SEND_REQUEST; - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); /* Paired with the read barrier in rxrpc_connect_call(). This orders * cid and epoch in the connection wrt to call_id without the need to @@ -571,7 +584,7 @@ static void rxrpc_activate_one_channel(struct rxrpc_connection *conn, */ static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn) { - struct rxrpc_net *rxnet = bundle->params.local->rxnet; + struct rxrpc_net *rxnet = bundle->local->rxnet; bool drop_ref; if (!list_empty(&conn->cache_link)) { @@ -583,7 +596,7 @@ static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connecti } spin_unlock(&rxnet->client_conn_cache_lock); if (drop_ref) - rxrpc_put_connection(conn); + rxrpc_put_connection(conn, rxrpc_conn_put_unidle); } } @@ -732,7 +745,7 @@ granted_channel: out_put_bundle: rxrpc_deactivate_bundle(bundle); - rxrpc_put_bundle(bundle); + rxrpc_put_bundle(bundle, rxrpc_bundle_get_client_call); out: _leave(" = %d", ret); return ret; @@ -773,6 +786,10 @@ void rxrpc_expose_client_call(struct rxrpc_call *call) if (chan->call_counter >= INT_MAX) set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); trace_rxrpc_client(conn, channel, rxrpc_client_exposed); + + spin_lock(&call->peer->lock); + hlist_add_head(&call->error_link, &call->peer->error_targets); + spin_unlock(&call->peer->lock); } } @@ -797,7 +814,7 @@ void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call { struct rxrpc_connection *conn; struct rxrpc_channel *chan = NULL; - struct rxrpc_net *rxnet = bundle->params.local->rxnet; + struct rxrpc_net *rxnet = bundle->local->rxnet; unsigned int channel; bool may_reuse; u32 cid; @@ -887,7 +904,7 @@ void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call trace_rxrpc_client(conn, channel, rxrpc_client_to_idle); conn->idle_timestamp = jiffies; - rxrpc_get_connection(conn); + rxrpc_get_connection(conn, rxrpc_conn_get_idle); spin_lock(&rxnet->client_conn_cache_lock); list_move_tail(&conn->cache_link, &rxnet->idle_client_conns); spin_unlock(&rxnet->client_conn_cache_lock); @@ -929,7 +946,7 @@ static void rxrpc_unbundle_conn(struct rxrpc_connection *conn) if (need_drop) { rxrpc_deactivate_bundle(bundle); - rxrpc_put_connection(conn); + rxrpc_put_connection(conn, rxrpc_conn_put_unbundle); } } @@ -938,11 +955,11 @@ static void rxrpc_unbundle_conn(struct rxrpc_connection *conn) */ static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle) { - struct rxrpc_local *local = bundle->params.local; + struct rxrpc_local *local = bundle->local; bool need_put = false; if (atomic_dec_and_lock(&bundle->active, &local->client_bundles_lock)) { - if (!bundle->params.exclusive) { + if (!bundle->exclusive) { _debug("erase bundle"); rb_erase(&bundle->local_node, &local->client_bundles); need_put = true; @@ -950,16 +967,16 @@ static void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle) spin_unlock(&local->client_bundles_lock); if (need_put) - rxrpc_put_bundle(bundle); + rxrpc_put_bundle(bundle, rxrpc_bundle_put_discard); } } /* * Clean up a dead client connection. */ -static void rxrpc_kill_client_conn(struct rxrpc_connection *conn) +void rxrpc_kill_client_conn(struct rxrpc_connection *conn) { - struct rxrpc_local *local = conn->params.local; + struct rxrpc_local *local = conn->local; struct rxrpc_net *rxnet = local->rxnet; _enter("C=%x", conn->debug_id); @@ -968,23 +985,6 @@ static void rxrpc_kill_client_conn(struct rxrpc_connection *conn) atomic_dec(&rxnet->nr_client_conns); rxrpc_put_client_connection_id(conn); - rxrpc_kill_connection(conn); -} - -/* - * Clean up a dead client connections. - */ -void rxrpc_put_client_conn(struct rxrpc_connection *conn) -{ - const void *here = __builtin_return_address(0); - unsigned int debug_id = conn->debug_id; - bool dead; - int r; - - dead = __refcount_dec_and_test(&conn->ref, &r); - trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, r - 1, here); - if (dead) - rxrpc_kill_client_conn(conn); } /* @@ -1010,7 +1010,7 @@ void rxrpc_discard_expired_client_conns(struct work_struct *work) } /* Don't double up on the discarding */ - if (!spin_trylock(&rxnet->client_conn_discard_lock)) { + if (!mutex_trylock(&rxnet->client_conn_discard_lock)) { _leave(" [already]"); return; } @@ -1038,7 +1038,7 @@ next: expiry = rxrpc_conn_idle_client_expiry; if (nr_conns > rxrpc_reap_client_connections) expiry = rxrpc_conn_idle_client_fast_expiry; - if (conn->params.local->service_closed) + if (conn->local->service_closed) expiry = rxrpc_closed_conn_expiry * HZ; conn_expires_at = conn->idle_timestamp + expiry; @@ -1048,13 +1048,15 @@ next: goto not_yet_expired; } + atomic_dec(&conn->active); trace_rxrpc_client(conn, -1, rxrpc_client_discard); list_del_init(&conn->cache_link); spin_unlock(&rxnet->client_conn_cache_lock); rxrpc_unbundle_conn(conn); - rxrpc_put_connection(conn); /* Drop the ->cache_link ref */ + /* Drop the ->cache_link ref */ + rxrpc_put_connection(conn, rxrpc_conn_put_discard_idle); nr_conns--; goto next; @@ -1073,7 +1075,7 @@ not_yet_expired: out: spin_unlock(&rxnet->client_conn_cache_lock); - spin_unlock(&rxnet->client_conn_discard_lock); + mutex_unlock(&rxnet->client_conn_discard_lock); _leave(""); } @@ -1112,7 +1114,8 @@ void rxrpc_clean_up_local_conns(struct rxrpc_local *local) list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns, cache_link) { - if (conn->params.local == local) { + if (conn->local == local) { + atomic_dec(&conn->active); trace_rxrpc_client(conn, -1, rxrpc_client_discard); list_move(&conn->cache_link, &graveyard); } @@ -1125,7 +1128,7 @@ void rxrpc_clean_up_local_conns(struct rxrpc_local *local) struct rxrpc_connection, cache_link); list_del_init(&conn->cache_link); rxrpc_unbundle_conn(conn); - rxrpc_put_connection(conn); + rxrpc_put_connection(conn, rxrpc_conn_put_local_dead); } _leave(" [culled]"); diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index aab069701398..480364bcbf85 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -52,8 +52,8 @@ static void rxrpc_conn_retransmit_call(struct rxrpc_connection *conn, if (skb && call_id != sp->hdr.callNumber) return; - msg.msg_name = &conn->params.peer->srx.transport; - msg.msg_namelen = conn->params.peer->srx.transport_len; + msg.msg_name = &conn->peer->srx.transport; + msg.msg_namelen = conn->peer->srx.transport_len; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = 0; @@ -86,8 +86,8 @@ static void rxrpc_conn_retransmit_call(struct rxrpc_connection *conn, break; case RXRPC_PACKET_TYPE_ACK: - mtu = conn->params.peer->if_mtu; - mtu -= conn->params.peer->hdrsize; + mtu = conn->peer->if_mtu; + mtu -= conn->peer->hdrsize; pkt.ack.bufferSpace = 0; pkt.ack.maxSkew = htons(skb ? skb->priority : 0); pkt.ack.firstPacket = htonl(chan->last_seq + 1); @@ -122,19 +122,17 @@ static void rxrpc_conn_retransmit_call(struct rxrpc_connection *conn, switch (chan->last_type) { case RXRPC_PACKET_TYPE_ABORT: - _proto("Tx ABORT %%%u { %d } [re]", serial, conn->abort_code); break; case RXRPC_PACKET_TYPE_ACK: trace_rxrpc_tx_ack(chan->call_debug_id, serial, ntohl(pkt.ack.firstPacket), ntohl(pkt.ack.serial), pkt.ack.reason, 0); - _proto("Tx ACK %%%u [re]", serial); break; } - ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, ioc, len); - conn->params.peer->last_tx_at = ktime_get_seconds(); + ret = kernel_sendmsg(conn->local->socket, &msg, iov, ioc, len); + conn->peer->last_tx_at = ktime_get_seconds(); if (ret < 0) trace_rxrpc_tx_fail(chan->call_debug_id, serial, ret, rxrpc_tx_point_call_final_resend); @@ -200,9 +198,9 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn, _enter("%d,,%u,%u", conn->debug_id, error, abort_code); /* generate a connection-level abort */ - spin_lock_bh(&conn->state_lock); + spin_lock(&conn->state_lock); if (conn->state >= RXRPC_CONN_REMOTELY_ABORTED) { - spin_unlock_bh(&conn->state_lock); + spin_unlock(&conn->state_lock); _leave(" = 0 [already dead]"); return 0; } @@ -211,10 +209,10 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn, conn->abort_code = abort_code; conn->state = RXRPC_CONN_LOCALLY_ABORTED; set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); - spin_unlock_bh(&conn->state_lock); + spin_unlock(&conn->state_lock); - msg.msg_name = &conn->params.peer->srx.transport; - msg.msg_namelen = conn->params.peer->srx.transport_len; + msg.msg_name = &conn->peer->srx.transport; + msg.msg_namelen = conn->peer->srx.transport_len; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = 0; @@ -242,9 +240,8 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn, serial = atomic_inc_return(&conn->serial); rxrpc_abort_calls(conn, RXRPC_CALL_LOCALLY_ABORTED, serial); whdr.serial = htonl(serial); - _proto("Tx CONN ABORT %%%u { %d }", serial, conn->abort_code); - ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len); + ret = kernel_sendmsg(conn->local->socket, &msg, iov, 2, len); if (ret < 0) { trace_rxrpc_tx_fail(conn->debug_id, serial, ret, rxrpc_tx_point_conn_abort); @@ -254,7 +251,7 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn, trace_rxrpc_tx_packet(conn->debug_id, &whdr, rxrpc_tx_point_conn_abort); - conn->params.peer->last_tx_at = ktime_get_seconds(); + conn->peer->last_tx_at = ktime_get_seconds(); _leave(" = 0"); return 0; @@ -268,12 +265,12 @@ static void rxrpc_call_is_secure(struct rxrpc_call *call) { _enter("%p", call); if (call) { - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); if (call->state == RXRPC_CALL_SERVER_SECURING) { call->state = RXRPC_CALL_SERVER_RECV_REQUEST; rxrpc_notify_socket(call); } - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); } } @@ -285,8 +282,6 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, u32 *_abort_code) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - __be32 wtmp; - u32 abort_code; int loop, ret; if (conn->state >= RXRPC_CONN_REMOTELY_ABORTED) { @@ -308,17 +303,8 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return 0; case RXRPC_PACKET_TYPE_ABORT: - if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), - &wtmp, sizeof(wtmp)) < 0) { - trace_rxrpc_rx_eproto(NULL, sp->hdr.serial, - tracepoint_string("bad_abort")); - return -EPROTO; - } - abort_code = ntohl(wtmp); - _proto("Rx ABORT %%%u { ac=%d }", sp->hdr.serial, abort_code); - conn->error = -ECONNABORTED; - conn->abort_code = abort_code; + conn->abort_code = skb->priority; conn->state = RXRPC_CONN_REMOTELY_ABORTED; set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); rxrpc_abort_calls(conn, RXRPC_CALL_REMOTELY_ABORTED, sp->hdr.serial); @@ -334,23 +320,23 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, return ret; ret = conn->security->init_connection_security( - conn, conn->params.key->payload.data[0]); + conn, conn->key->payload.data[0]); if (ret < 0) return ret; spin_lock(&conn->bundle->channel_lock); - spin_lock_bh(&conn->state_lock); + spin_lock(&conn->state_lock); if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) { conn->state = RXRPC_CONN_SERVICE; - spin_unlock_bh(&conn->state_lock); + spin_unlock(&conn->state_lock); for (loop = 0; loop < RXRPC_MAXCALLS; loop++) rxrpc_call_is_secure( rcu_dereference_protected( conn->channels[loop].call, lockdep_is_held(&conn->bundle->channel_lock))); } else { - spin_unlock_bh(&conn->state_lock); + spin_unlock(&conn->state_lock); } spin_unlock(&conn->bundle->channel_lock); @@ -451,7 +437,7 @@ static void rxrpc_do_process_connection(struct rxrpc_connection *conn) /* go through the conn-level event packets, releasing the ref on this * connection that each one has when we've finished with it */ while ((skb = skb_dequeue(&conn->rx_queue))) { - rxrpc_see_skb(skb, rxrpc_skb_seen); + rxrpc_see_skb(skb, rxrpc_skb_see_conn_work); ret = rxrpc_process_event(conn, skb, &abort_code); switch (ret) { case -EPROTO: @@ -463,7 +449,7 @@ static void rxrpc_do_process_connection(struct rxrpc_connection *conn) goto requeue_and_leave; case -ECONNABORTED: default: - rxrpc_free_skb(skb, rxrpc_skb_freed); + rxrpc_free_skb(skb, rxrpc_skb_put_conn_work); break; } } @@ -477,7 +463,7 @@ requeue_and_leave: protocol_error: if (rxrpc_abort_connection(conn, ret, abort_code) < 0) goto requeue_and_leave; - rxrpc_free_skb(skb, rxrpc_skb_freed); + rxrpc_free_skb(skb, rxrpc_skb_put_conn_work); return; } @@ -486,14 +472,70 @@ void rxrpc_process_connection(struct work_struct *work) struct rxrpc_connection *conn = container_of(work, struct rxrpc_connection, processor); - rxrpc_see_connection(conn); + rxrpc_see_connection(conn, rxrpc_conn_see_work); - if (__rxrpc_use_local(conn->params.local)) { + if (__rxrpc_use_local(conn->local, rxrpc_local_use_conn_work)) { rxrpc_do_process_connection(conn); - rxrpc_unuse_local(conn->params.local); + rxrpc_unuse_local(conn->local, rxrpc_local_unuse_conn_work); } +} - rxrpc_put_connection(conn); - _leave(""); - return; +/* + * post connection-level events to the connection + * - this includes challenges, responses, some aborts and call terminal packet + * retransmission. + */ +static void rxrpc_post_packet_to_conn(struct rxrpc_connection *conn, + struct sk_buff *skb) +{ + _enter("%p,%p", conn, skb); + + rxrpc_get_skb(skb, rxrpc_skb_get_conn_work); + skb_queue_tail(&conn->rx_queue, skb); + rxrpc_queue_conn(conn, rxrpc_conn_queue_rx_work); +} + +/* + * Input a connection-level packet. + */ +int rxrpc_input_conn_packet(struct rxrpc_connection *conn, struct sk_buff *skb) +{ + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + + if (conn->state >= RXRPC_CONN_REMOTELY_ABORTED) { + _leave(" = -ECONNABORTED [%u]", conn->state); + return -ECONNABORTED; + } + + _enter("{%d},{%u,%%%u},", conn->debug_id, sp->hdr.type, sp->hdr.serial); + + switch (sp->hdr.type) { + case RXRPC_PACKET_TYPE_DATA: + case RXRPC_PACKET_TYPE_ACK: + rxrpc_conn_retransmit_call(conn, skb, + sp->hdr.cid & RXRPC_CHANNELMASK); + return 0; + + case RXRPC_PACKET_TYPE_BUSY: + /* Just ignore BUSY packets for now. */ + return 0; + + case RXRPC_PACKET_TYPE_ABORT: + conn->error = -ECONNABORTED; + conn->abort_code = skb->priority; + conn->state = RXRPC_CONN_REMOTELY_ABORTED; + set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); + rxrpc_abort_calls(conn, RXRPC_CALL_REMOTELY_ABORTED, sp->hdr.serial); + return -ECONNABORTED; + + case RXRPC_PACKET_TYPE_CHALLENGE: + case RXRPC_PACKET_TYPE_RESPONSE: + rxrpc_post_packet_to_conn(conn, skb); + return 0; + + default: + trace_rxrpc_rx_eproto(NULL, sp->hdr.serial, + tracepoint_string("bad_conn_pkt")); + return -EPROTO; + } } diff --git a/net/rxrpc/conn_object.c b/net/rxrpc/conn_object.c index 156bd26daf74..3c8f83dacb2b 100644 --- a/net/rxrpc/conn_object.c +++ b/net/rxrpc/conn_object.c @@ -19,20 +19,23 @@ unsigned int __read_mostly rxrpc_connection_expiry = 10 * 60; unsigned int __read_mostly rxrpc_closed_conn_expiry = 10; -static void rxrpc_destroy_connection(struct rcu_head *); +static void rxrpc_clean_up_connection(struct work_struct *work); +static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet, + unsigned long reap_at); static void rxrpc_connection_timer(struct timer_list *timer) { struct rxrpc_connection *conn = container_of(timer, struct rxrpc_connection, timer); - rxrpc_queue_conn(conn); + rxrpc_queue_conn(conn, rxrpc_conn_queue_timer); } /* * allocate a new connection */ -struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp) +struct rxrpc_connection *rxrpc_alloc_connection(struct rxrpc_net *rxnet, + gfp_t gfp) { struct rxrpc_connection *conn; @@ -42,10 +45,12 @@ struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp) if (conn) { INIT_LIST_HEAD(&conn->cache_link); timer_setup(&conn->timer, &rxrpc_connection_timer, 0); - INIT_WORK(&conn->processor, &rxrpc_process_connection); + INIT_WORK(&conn->processor, rxrpc_process_connection); + INIT_WORK(&conn->destructor, rxrpc_clean_up_connection); INIT_LIST_HEAD(&conn->proc_link); INIT_LIST_HEAD(&conn->link); skb_queue_head_init(&conn->rx_queue); + conn->rxnet = rxnet; conn->security = &rxrpc_no_security; spin_lock_init(&conn->state_lock); conn->debug_id = atomic_inc_return(&rxrpc_debug_id); @@ -67,89 +72,55 @@ struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp) * * The caller must be holding the RCU read lock. */ -struct rxrpc_connection *rxrpc_find_connection_rcu(struct rxrpc_local *local, - struct sk_buff *skb, - struct rxrpc_peer **_peer) +struct rxrpc_connection *rxrpc_find_client_connection_rcu(struct rxrpc_local *local, + struct sockaddr_rxrpc *srx, + struct sk_buff *skb) { struct rxrpc_connection *conn; - struct rxrpc_conn_proto k; struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - struct sockaddr_rxrpc srx; struct rxrpc_peer *peer; _enter(",%x", sp->hdr.cid & RXRPC_CIDMASK); - if (rxrpc_extract_addr_from_skb(&srx, skb) < 0) - goto not_found; - - if (srx.transport.family != local->srx.transport.family && - (srx.transport.family == AF_INET && - local->srx.transport.family != AF_INET6)) { - pr_warn_ratelimited("AF_RXRPC: Protocol mismatch %u not %u\n", - srx.transport.family, - local->srx.transport.family); + /* Look up client connections by connection ID alone as their IDs are + * unique for this machine. + */ + conn = idr_find(&rxrpc_client_conn_ids, sp->hdr.cid >> RXRPC_CIDSHIFT); + if (!conn || refcount_read(&conn->ref) == 0) { + _debug("no conn"); goto not_found; } - k.epoch = sp->hdr.epoch; - k.cid = sp->hdr.cid & RXRPC_CIDMASK; - - if (rxrpc_to_server(sp)) { - /* We need to look up service connections by the full protocol - * parameter set. We look up the peer first as an intermediate - * step and then the connection from the peer's tree. - */ - peer = rxrpc_lookup_peer_rcu(local, &srx); - if (!peer) - goto not_found; - *_peer = peer; - conn = rxrpc_find_service_conn_rcu(peer, skb); - if (!conn || refcount_read(&conn->ref) == 0) - goto not_found; - _leave(" = %p", conn); - return conn; - } else { - /* Look up client connections by connection ID alone as their - * IDs are unique for this machine. - */ - conn = idr_find(&rxrpc_client_conn_ids, - sp->hdr.cid >> RXRPC_CIDSHIFT); - if (!conn || refcount_read(&conn->ref) == 0) { - _debug("no conn"); - goto not_found; - } + if (conn->proto.epoch != sp->hdr.epoch || + conn->local != local) + goto not_found; - if (conn->proto.epoch != k.epoch || - conn->params.local != local) + peer = conn->peer; + switch (srx->transport.family) { + case AF_INET: + if (peer->srx.transport.sin.sin_port != + srx->transport.sin.sin_port || + peer->srx.transport.sin.sin_addr.s_addr != + srx->transport.sin.sin_addr.s_addr) goto not_found; - - peer = conn->params.peer; - switch (srx.transport.family) { - case AF_INET: - if (peer->srx.transport.sin.sin_port != - srx.transport.sin.sin_port || - peer->srx.transport.sin.sin_addr.s_addr != - srx.transport.sin.sin_addr.s_addr) - goto not_found; - break; + break; #ifdef CONFIG_AF_RXRPC_IPV6 - case AF_INET6: - if (peer->srx.transport.sin6.sin6_port != - srx.transport.sin6.sin6_port || - memcmp(&peer->srx.transport.sin6.sin6_addr, - &srx.transport.sin6.sin6_addr, - sizeof(struct in6_addr)) != 0) - goto not_found; - break; + case AF_INET6: + if (peer->srx.transport.sin6.sin6_port != + srx->transport.sin6.sin6_port || + memcmp(&peer->srx.transport.sin6.sin6_addr, + &srx->transport.sin6.sin6_addr, + sizeof(struct in6_addr)) != 0) + goto not_found; + break; #endif - default: - BUG(); - } - - _leave(" = %p", conn); - return conn; + default: + BUG(); } + _leave(" = %p", conn); + return conn; + not_found: _leave(" = NULL"); return NULL; @@ -210,9 +181,9 @@ void rxrpc_disconnect_call(struct rxrpc_call *call) call->peer->cong_ssthresh = call->cong_ssthresh; if (!hlist_unhashed(&call->error_link)) { - spin_lock_bh(&call->peer->lock); - hlist_del_rcu(&call->error_link); - spin_unlock_bh(&call->peer->lock); + spin_lock(&call->peer->lock); + hlist_del_init(&call->error_link); + spin_unlock(&call->peer->lock); } if (rxrpc_is_client_call(call)) @@ -224,79 +195,45 @@ void rxrpc_disconnect_call(struct rxrpc_call *call) set_bit(RXRPC_CALL_DISCONNECTED, &call->flags); conn->idle_timestamp = jiffies; -} - -/* - * Kill off a connection. - */ -void rxrpc_kill_connection(struct rxrpc_connection *conn) -{ - struct rxrpc_net *rxnet = conn->params.local->rxnet; - - ASSERT(!rcu_access_pointer(conn->channels[0].call) && - !rcu_access_pointer(conn->channels[1].call) && - !rcu_access_pointer(conn->channels[2].call) && - !rcu_access_pointer(conn->channels[3].call)); - ASSERT(list_empty(&conn->cache_link)); - - write_lock(&rxnet->conn_lock); - list_del_init(&conn->proc_link); - write_unlock(&rxnet->conn_lock); - - /* Drain the Rx queue. Note that even though we've unpublished, an - * incoming packet could still be being added to our Rx queue, so we - * will need to drain it again in the RCU cleanup handler. - */ - rxrpc_purge_queue(&conn->rx_queue); - - /* Leave final destruction to RCU. The connection processor work item - * must carry a ref on the connection to prevent us getting here whilst - * it is queued or running. - */ - call_rcu(&conn->rcu, rxrpc_destroy_connection); + if (atomic_dec_and_test(&conn->active)) + rxrpc_set_service_reap_timer(conn->rxnet, + jiffies + rxrpc_connection_expiry); } /* * Queue a connection's work processor, getting a ref to pass to the work * queue. */ -bool rxrpc_queue_conn(struct rxrpc_connection *conn) +void rxrpc_queue_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why) { - const void *here = __builtin_return_address(0); - int r; - - if (!__refcount_inc_not_zero(&conn->ref, &r)) - return false; - if (rxrpc_queue_work(&conn->processor)) - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_queued, r + 1, here); - else - rxrpc_put_connection(conn); - return true; + if (atomic_read(&conn->active) >= 0 && + rxrpc_queue_work(&conn->processor)) + rxrpc_see_connection(conn, why); } /* * Note the re-emergence of a connection. */ -void rxrpc_see_connection(struct rxrpc_connection *conn) +void rxrpc_see_connection(struct rxrpc_connection *conn, + enum rxrpc_conn_trace why) { - const void *here = __builtin_return_address(0); if (conn) { - int n = refcount_read(&conn->ref); + int r = refcount_read(&conn->ref); - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_seen, n, here); + trace_rxrpc_conn(conn->debug_id, r, why); } } /* * Get a ref on a connection. */ -struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *conn) +struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *conn, + enum rxrpc_conn_trace why) { - const void *here = __builtin_return_address(0); int r; __refcount_inc(&conn->ref, &r); - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_got, r, here); + trace_rxrpc_conn(conn->debug_id, r + 1, why); return conn; } @@ -304,14 +241,14 @@ struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *conn) * Try to get a ref on a connection. */ struct rxrpc_connection * -rxrpc_get_connection_maybe(struct rxrpc_connection *conn) +rxrpc_get_connection_maybe(struct rxrpc_connection *conn, + enum rxrpc_conn_trace why) { - const void *here = __builtin_return_address(0); int r; if (conn) { if (__refcount_inc_not_zero(&conn->ref, &r)) - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_got, r + 1, here); + trace_rxrpc_conn(conn->debug_id, r + 1, why); else conn = NULL; } @@ -329,49 +266,95 @@ static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet, } /* - * Release a service connection + * destroy a virtual connection */ -void rxrpc_put_service_conn(struct rxrpc_connection *conn) +static void rxrpc_rcu_free_connection(struct rcu_head *rcu) { - const void *here = __builtin_return_address(0); - unsigned int debug_id = conn->debug_id; - int r; + struct rxrpc_connection *conn = + container_of(rcu, struct rxrpc_connection, rcu); + struct rxrpc_net *rxnet = conn->rxnet; - __refcount_dec(&conn->ref, &r); - trace_rxrpc_conn(debug_id, rxrpc_conn_put_service, r - 1, here); - if (r - 1 == 1) - rxrpc_set_service_reap_timer(conn->params.local->rxnet, - jiffies + rxrpc_connection_expiry); + _enter("{%d,u=%d}", conn->debug_id, refcount_read(&conn->ref)); + + trace_rxrpc_conn(conn->debug_id, refcount_read(&conn->ref), + rxrpc_conn_free); + kfree(conn); + + if (atomic_dec_and_test(&rxnet->nr_conns)) + wake_up_var(&rxnet->nr_conns); } /* - * destroy a virtual connection + * Clean up a dead connection. */ -static void rxrpc_destroy_connection(struct rcu_head *rcu) +static void rxrpc_clean_up_connection(struct work_struct *work) { struct rxrpc_connection *conn = - container_of(rcu, struct rxrpc_connection, rcu); + container_of(work, struct rxrpc_connection, destructor); + struct rxrpc_net *rxnet = conn->rxnet; - _enter("{%d,u=%d}", conn->debug_id, refcount_read(&conn->ref)); + ASSERT(!rcu_access_pointer(conn->channels[0].call) && + !rcu_access_pointer(conn->channels[1].call) && + !rcu_access_pointer(conn->channels[2].call) && + !rcu_access_pointer(conn->channels[3].call)); + ASSERT(list_empty(&conn->cache_link)); - ASSERTCMP(refcount_read(&conn->ref), ==, 0); + del_timer_sync(&conn->timer); + cancel_work_sync(&conn->processor); /* Processing may restart the timer */ + del_timer_sync(&conn->timer); - _net("DESTROY CONN %d", conn->debug_id); + write_lock(&rxnet->conn_lock); + list_del_init(&conn->proc_link); + write_unlock(&rxnet->conn_lock); - del_timer_sync(&conn->timer); rxrpc_purge_queue(&conn->rx_queue); + rxrpc_kill_client_conn(conn); + conn->security->clear(conn); - key_put(conn->params.key); - rxrpc_put_bundle(conn->bundle); - rxrpc_put_peer(conn->params.peer); + key_put(conn->key); + rxrpc_put_bundle(conn->bundle, rxrpc_bundle_put_conn); + rxrpc_put_peer(conn->peer, rxrpc_peer_put_conn); + rxrpc_put_local(conn->local, rxrpc_local_put_kill_conn); + + /* Drain the Rx queue. Note that even though we've unpublished, an + * incoming packet could still be being added to our Rx queue, so we + * will need to drain it again in the RCU cleanup handler. + */ + rxrpc_purge_queue(&conn->rx_queue); - if (atomic_dec_and_test(&conn->params.local->rxnet->nr_conns)) - wake_up_var(&conn->params.local->rxnet->nr_conns); - rxrpc_put_local(conn->params.local); + call_rcu(&conn->rcu, rxrpc_rcu_free_connection); +} - kfree(conn); - _leave(""); +/* + * Drop a ref on a connection. + */ +void rxrpc_put_connection(struct rxrpc_connection *conn, + enum rxrpc_conn_trace why) +{ + unsigned int debug_id; + bool dead; + int r; + + if (!conn) + return; + + debug_id = conn->debug_id; + dead = __refcount_dec_and_test(&conn->ref, &r); + trace_rxrpc_conn(debug_id, r - 1, why); + if (dead) { + del_timer(&conn->timer); + cancel_work(&conn->processor); + + if (in_softirq() || work_busy(&conn->processor) || + timer_pending(&conn->timer)) + /* Can't use the rxrpc workqueue as we need to cancel/flush + * something that may be running/waiting there. + */ + schedule_work(&conn->destructor); + else + rxrpc_clean_up_connection(&conn->destructor); + } } /* @@ -383,6 +366,7 @@ void rxrpc_service_connection_reaper(struct work_struct *work) struct rxrpc_net *rxnet = container_of(work, struct rxrpc_net, service_conn_reaper); unsigned long expire_at, earliest, idle_timestamp, now; + int active; LIST_HEAD(graveyard); @@ -393,20 +377,20 @@ void rxrpc_service_connection_reaper(struct work_struct *work) write_lock(&rxnet->conn_lock); list_for_each_entry_safe(conn, _p, &rxnet->service_conns, link) { - ASSERTCMP(refcount_read(&conn->ref), >, 0); - if (likely(refcount_read(&conn->ref) > 1)) + ASSERTCMP(atomic_read(&conn->active), >=, 0); + if (likely(atomic_read(&conn->active) > 0)) continue; if (conn->state == RXRPC_CONN_SERVICE_PREALLOC) continue; - if (rxnet->live && !conn->params.local->dead) { + if (rxnet->live && !conn->local->dead) { idle_timestamp = READ_ONCE(conn->idle_timestamp); expire_at = idle_timestamp + rxrpc_connection_expiry * HZ; - if (conn->params.local->service_closed) + if (conn->local->service_closed) expire_at = idle_timestamp + rxrpc_closed_conn_expiry * HZ; - _debug("reap CONN %d { u=%d,t=%ld }", - conn->debug_id, refcount_read(&conn->ref), + _debug("reap CONN %d { a=%d,t=%ld }", + conn->debug_id, atomic_read(&conn->active), (long)expire_at - (long)now); if (time_before(now, expire_at)) { @@ -416,12 +400,13 @@ void rxrpc_service_connection_reaper(struct work_struct *work) } } - /* The usage count sits at 1 whilst the object is unused on the - * list; we reduce that to 0 to make the object unavailable. + /* The activity count sits at 0 whilst the conn is unused on + * the list; we reduce that to -1 to make the conn unavailable. */ - if (!refcount_dec_if_one(&conn->ref)) + active = 0; + if (!atomic_try_cmpxchg(&conn->active, &active, -1)) continue; - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_reap_service, 0, NULL); + rxrpc_see_connection(conn, rxrpc_conn_see_reap_service); if (rxrpc_conn_is_client(conn)) BUG(); @@ -443,8 +428,8 @@ void rxrpc_service_connection_reaper(struct work_struct *work) link); list_del_init(&conn->link); - ASSERTCMP(refcount_read(&conn->ref), ==, 0); - rxrpc_kill_connection(conn); + ASSERTCMP(atomic_read(&conn->active), ==, -1); + rxrpc_put_connection(conn, rxrpc_conn_put_service_reaped); } _leave(""); diff --git a/net/rxrpc/conn_service.c b/net/rxrpc/conn_service.c index 6e6aa02c6f9e..2a55a88b2a5b 100644 --- a/net/rxrpc/conn_service.c +++ b/net/rxrpc/conn_service.c @@ -73,7 +73,7 @@ static void rxrpc_publish_service_conn(struct rxrpc_peer *peer, struct rxrpc_conn_proto k = conn->proto; struct rb_node **pp, *parent; - write_seqlock_bh(&peer->service_conn_lock); + write_seqlock(&peer->service_conn_lock); pp = &peer->service_conns.rb_node; parent = NULL; @@ -94,14 +94,14 @@ static void rxrpc_publish_service_conn(struct rxrpc_peer *peer, rb_insert_color(&conn->service_node, &peer->service_conns); conn_published: set_bit(RXRPC_CONN_IN_SERVICE_CONNS, &conn->flags); - write_sequnlock_bh(&peer->service_conn_lock); + write_sequnlock(&peer->service_conn_lock); _leave(" = %d [new]", conn->debug_id); return; found_extant_conn: if (refcount_read(&cursor->ref) == 0) goto replace_old_connection; - write_sequnlock_bh(&peer->service_conn_lock); + write_sequnlock(&peer->service_conn_lock); /* We should not be able to get here. rxrpc_incoming_connection() is * called in a non-reentrant context, so there can't be a race to * insert a new connection. @@ -125,7 +125,7 @@ replace_old_connection: struct rxrpc_connection *rxrpc_prealloc_service_connection(struct rxrpc_net *rxnet, gfp_t gfp) { - struct rxrpc_connection *conn = rxrpc_alloc_connection(gfp); + struct rxrpc_connection *conn = rxrpc_alloc_connection(rxnet, gfp); if (conn) { /* We maintain an extra ref on the connection whilst it is on @@ -133,7 +133,8 @@ struct rxrpc_connection *rxrpc_prealloc_service_connection(struct rxrpc_net *rxn */ conn->state = RXRPC_CONN_SERVICE_PREALLOC; refcount_set(&conn->ref, 2); - conn->bundle = rxrpc_get_bundle(&rxrpc_service_dummy_bundle); + conn->bundle = rxrpc_get_bundle(&rxrpc_service_dummy_bundle, + rxrpc_bundle_get_service_conn); atomic_inc(&rxnet->nr_conns); write_lock(&rxnet->conn_lock); @@ -141,9 +142,7 @@ struct rxrpc_connection *rxrpc_prealloc_service_connection(struct rxrpc_net *rxn list_add_tail(&conn->proc_link, &rxnet->conn_proc_list); write_unlock(&rxnet->conn_lock); - trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_service, - refcount_read(&conn->ref), - __builtin_return_address(0)); + rxrpc_see_connection(conn, rxrpc_conn_new_service); } return conn; @@ -164,7 +163,7 @@ void rxrpc_new_incoming_connection(struct rxrpc_sock *rx, conn->proto.epoch = sp->hdr.epoch; conn->proto.cid = sp->hdr.cid & RXRPC_CIDMASK; - conn->params.service_id = sp->hdr.serviceId; + conn->orig_service_id = sp->hdr.serviceId; conn->service_id = sp->hdr.serviceId; conn->security_ix = sp->hdr.securityIndex; conn->out_clientflag = 0; @@ -182,10 +181,10 @@ void rxrpc_new_incoming_connection(struct rxrpc_sock *rx, conn->service_id == rx->service_upgrade.from) conn->service_id = rx->service_upgrade.to; - /* Make the connection a target for incoming packets. */ - rxrpc_publish_service_conn(conn->params.peer, conn); + atomic_set(&conn->active, 1); - _net("CONNECTION new %d {%x}", conn->debug_id, conn->proto.cid); + /* Make the connection a target for incoming packets. */ + rxrpc_publish_service_conn(conn->peer, conn); } /* @@ -194,10 +193,10 @@ void rxrpc_new_incoming_connection(struct rxrpc_sock *rx, */ void rxrpc_unpublish_service_conn(struct rxrpc_connection *conn) { - struct rxrpc_peer *peer = conn->params.peer; + struct rxrpc_peer *peer = conn->peer; - write_seqlock_bh(&peer->service_conn_lock); + write_seqlock(&peer->service_conn_lock); if (test_and_clear_bit(RXRPC_CONN_IN_SERVICE_CONNS, &conn->flags)) rb_erase(&conn->service_node, &peer->service_conns); - write_sequnlock_bh(&peer->service_conn_lock); + write_sequnlock(&peer->service_conn_lock); } diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c index bdf70b81addc..d0e20e946e48 100644 --- a/net/rxrpc/input.c +++ b/net/rxrpc/input.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* RxRPC packet reception +/* Processing of received RxRPC packets * - * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ @@ -12,10 +12,8 @@ static void rxrpc_proto_abort(const char *why, struct rxrpc_call *call, rxrpc_seq_t seq) { - if (rxrpc_abort_call(why, call, seq, RX_PROTOCOL_ERROR, -EBADMSG)) { - set_bit(RXRPC_CALL_EV_ABORT, &call->events); - rxrpc_queue_call(call); - } + if (rxrpc_abort_call(why, call, seq, RX_PROTOCOL_ERROR, -EBADMSG)) + rxrpc_send_abort_packet(call); } /* @@ -58,25 +56,6 @@ static void rxrpc_congestion_management(struct rxrpc_call *call, summary->cumulative_acks = cumulative_acks; summary->dup_acks = call->cong_dup_acks; - /* If we haven't transmitted anything for >1RTT, we should reset the - * congestion management state. - */ - if ((call->cong_mode == RXRPC_CALL_SLOW_START || - call->cong_mode == RXRPC_CALL_CONGEST_AVOIDANCE) && - ktime_before(ktime_add_us(call->tx_last_sent, - call->peer->srtt_us >> 3), - ktime_get_real()) - ) { - change = rxrpc_cong_idle_reset; - summary->mode = RXRPC_CALL_SLOW_START; - if (RXRPC_TX_SMSS > 2190) - summary->cwnd = 2; - else if (RXRPC_TX_SMSS > 1095) - summary->cwnd = 3; - else - summary->cwnd = 4; - } - switch (call->cong_mode) { case RXRPC_CALL_SLOW_START: if (summary->saw_nacks) @@ -174,8 +153,8 @@ out_no_clear_ca: call->cong_cwnd = cwnd; call->cong_cumul_acks = cumulative_acks; trace_rxrpc_congest(call, summary, acked_serial, change); - if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events)) - rxrpc_queue_call(call); + if (resend) + rxrpc_resend(call, skb); return; packet_loss_detected: @@ -197,6 +176,33 @@ send_extra_data: } /* + * Degrade the congestion window if we haven't transmitted a packet for >1RTT. + */ +void rxrpc_congestion_degrade(struct rxrpc_call *call) +{ + ktime_t rtt, now; + + if (call->cong_mode != RXRPC_CALL_SLOW_START && + call->cong_mode != RXRPC_CALL_CONGEST_AVOIDANCE) + return; + if (call->state == RXRPC_CALL_CLIENT_AWAIT_REPLY) + return; + + rtt = ns_to_ktime(call->peer->srtt_us * (1000 / 8)); + now = ktime_get_real(); + if (!ktime_before(ktime_add(call->tx_last_sent, rtt), now)) + return; + + trace_rxrpc_reset_cwnd(call, now); + rxrpc_inc_stat(call->rxnet, stat_tx_data_cwnd_reset); + call->tx_last_sent = now; + call->cong_mode = RXRPC_CALL_SLOW_START; + call->cong_ssthresh = max_t(unsigned int, call->cong_ssthresh, + call->cong_cwnd * 3 / 4); + call->cong_cwnd = max_t(unsigned int, call->cong_cwnd / 2, RXRPC_MIN_CWND); +} + +/* * Apply a hard ACK by advancing the Tx window. */ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, @@ -338,7 +344,8 @@ static void rxrpc_input_queue_data(struct rxrpc_call *call, struct sk_buff *skb, /* * Process a DATA packet. */ -static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb) +static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb, + bool *_notify) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct sk_buff *oos; @@ -361,7 +368,7 @@ static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb) if (test_and_set_bit(RXRPC_CALL_RX_LAST, &call->flags) && seq + 1 != wtop) { rxrpc_proto_abort("LSN", call, seq); - goto err_free; + return; } } else { if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) && @@ -369,7 +376,7 @@ static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb) pr_warn("Packet beyond last: c=%x q=%x window=%x-%x wlimit=%x\n", call->debug_id, seq, window, wtop, wlimit); rxrpc_proto_abort("LSA", call, seq); - goto err_free; + return; } } @@ -397,14 +404,18 @@ static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb) /* Send an immediate ACK if we fill in a hole */ else if (!skb_queue_empty(&call->rx_oos_queue)) ack_reason = RXRPC_ACK_DELAY; + else + atomic_inc_return(&call->ackr_nr_unacked); window++; if (after(window, wtop)) wtop = window; + rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg); + spin_lock(&call->recvmsg_queue.lock); rxrpc_input_queue_data(call, skb, window, wtop, rxrpc_receive_queue); - skb = NULL; + *_notify = true; while ((oos = skb_peek(&call->rx_oos_queue))) { struct rxrpc_skb_priv *osp = rxrpc_skb(oos); @@ -456,36 +467,26 @@ static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb) struct rxrpc_skb_priv *osp = rxrpc_skb(oos); if (after(osp->hdr.seq, seq)) { + rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg_oos); __skb_queue_before(&call->rx_oos_queue, oos, skb); goto oos_queued; } } + rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg_oos); __skb_queue_tail(&call->rx_oos_queue, skb); oos_queued: trace_rxrpc_receive(call, last ? rxrpc_receive_oos_last : rxrpc_receive_oos, sp->hdr.serial, sp->hdr.seq); - skb = NULL; } send_ack: - if (ack_reason < 0 && - atomic_inc_return(&call->ackr_nr_unacked) > 2 && - test_and_set_bit(RXRPC_CALL_IDLE_ACK_PENDING, &call->flags)) { - ack_reason = RXRPC_ACK_IDLE; - } else if (ack_reason >= 0) { - set_bit(RXRPC_CALL_IDLE_ACK_PENDING, &call->flags); - } - if (ack_reason >= 0) rxrpc_send_ACK(call, ack_reason, serial, rxrpc_propose_ack_input_data); else rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_input_data); - -err_free: - rxrpc_free_skb(skb, rxrpc_skb_freed); } /* @@ -498,6 +499,7 @@ static bool rxrpc_input_split_jumbo(struct rxrpc_call *call, struct sk_buff *skb struct sk_buff *jskb; unsigned int offset = sizeof(struct rxrpc_wire_header); unsigned int len = skb->len - offset; + bool notify = false; while (sp->hdr.flags & RXRPC_JUMBO_PACKET) { if (len < RXRPC_JUMBO_SUBPKTLEN) @@ -508,16 +510,17 @@ static bool rxrpc_input_split_jumbo(struct rxrpc_call *call, struct sk_buff *skb &jhdr, sizeof(jhdr)) < 0) goto protocol_error; - jskb = skb_clone(skb, GFP_ATOMIC); + jskb = skb_clone(skb, GFP_NOFS); if (!jskb) { kdebug("couldn't clone"); return false; } - rxrpc_new_skb(jskb, rxrpc_skb_cloned_jumbo); + rxrpc_new_skb(jskb, rxrpc_skb_new_jumbo_subpacket); jsp = rxrpc_skb(jskb); jsp->offset = offset; jsp->len = RXRPC_JUMBO_DATALEN; - rxrpc_input_data_one(call, jskb); + rxrpc_input_data_one(call, jskb, ¬ify); + rxrpc_free_skb(jskb, rxrpc_skb_put_jumbo_subpacket); sp->hdr.flags = jhdr.flags; sp->hdr._rsvd = ntohs(jhdr._rsvd); @@ -529,7 +532,11 @@ static bool rxrpc_input_split_jumbo(struct rxrpc_call *call, struct sk_buff *skb sp->offset = offset; sp->len = len; - rxrpc_input_data_one(call, skb); + rxrpc_input_data_one(call, skb, ¬ify); + if (notify) { + trace_rxrpc_notify_socket(call->debug_id, sp->hdr.serial); + rxrpc_notify_socket(call); + } return true; protocol_error: @@ -551,32 +558,9 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb) atomic64_read(&call->ackr_window), call->rx_highest_seq, skb->len, seq0); - _proto("Rx DATA %%%u { #%u f=%02x }", - sp->hdr.serial, seq0, sp->hdr.flags); - state = READ_ONCE(call->state); - if (state >= RXRPC_CALL_COMPLETE) { - rxrpc_free_skb(skb, rxrpc_skb_freed); + if (state >= RXRPC_CALL_COMPLETE) return; - } - - /* Unshare the packet so that it can be modified for in-place - * decryption. - */ - if (sp->hdr.securityIndex != 0) { - struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC); - if (!nskb) { - rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem); - return; - } - - if (nskb != skb) { - rxrpc_eaten_skb(skb, rxrpc_skb_received); - skb = nskb; - rxrpc_new_skb(skb, rxrpc_skb_unshared); - sp = rxrpc_skb(skb); - } - } if (state == RXRPC_CALL_SERVER_RECV_REQUEST) { unsigned long timo = READ_ONCE(call->next_req_timo); @@ -591,28 +575,23 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb) } } - spin_lock(&call->input_lock); - /* Received data implicitly ACKs all of the request packets we sent * when we're acting as a client. */ if ((state == RXRPC_CALL_CLIENT_SEND_REQUEST || state == RXRPC_CALL_CLIENT_AWAIT_REPLY) && !rxrpc_receiving_reply(call)) - goto out; + goto out_notify; if (!rxrpc_input_split_jumbo(call, skb)) { rxrpc_proto_abort("VLD", call, sp->hdr.seq); - goto out; + goto out_notify; } skb = NULL; -out: +out_notify: trace_rxrpc_notify_socket(call->debug_id, serial); rxrpc_notify_socket(call); - - spin_unlock(&call->input_lock); - rxrpc_free_skb(skb, rxrpc_skb_freed); _leave(" [queued]"); } @@ -671,32 +650,6 @@ static void rxrpc_complete_rtt_probe(struct rxrpc_call *call, } /* - * Process the response to a ping that we sent to find out if we lost an ACK. - * - * If we got back a ping response that indicates a lower tx_top than what we - * had at the time of the ping transmission, we adjudge all the DATA packets - * sent between the response tx_top and the ping-time tx_top to have been lost. - */ -static void rxrpc_input_check_for_lost_ack(struct rxrpc_call *call) -{ - if (after(call->acks_lost_top, call->acks_prev_seq) && - !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events)) - rxrpc_queue_call(call); -} - -/* - * Process a ping response. - */ -static void rxrpc_input_ping_response(struct rxrpc_call *call, - ktime_t resp_time, - rxrpc_serial_t acked_serial, - rxrpc_serial_t ack_serial) -{ - if (acked_serial == call->acks_lost_ping) - rxrpc_input_check_for_lost_ack(call); -} - -/* * Process the extra information that may be appended to an ACK packet */ static void rxrpc_input_ackinfo(struct rxrpc_call *call, struct sk_buff *skb, @@ -708,11 +661,6 @@ static void rxrpc_input_ackinfo(struct rxrpc_call *call, struct sk_buff *skb, bool wake = false; u32 rwind = ntohl(ackinfo->rwind); - _proto("Rx ACK %%%u Info { rx=%u max=%u rwin=%u jm=%u }", - sp->hdr.serial, - ntohl(ackinfo->rxMTU), ntohl(ackinfo->maxMTU), - rwind, ntohl(ackinfo->jumbo_max)); - if (rwind > RXRPC_TX_MAX_WINDOW) rwind = RXRPC_TX_MAX_WINDOW; if (call->tx_winsize != rwind) { @@ -729,11 +677,10 @@ static void rxrpc_input_ackinfo(struct rxrpc_call *call, struct sk_buff *skb, peer = call->peer; if (mtu < peer->maxdata) { - spin_lock_bh(&peer->lock); + spin_lock(&peer->lock); peer->maxdata = mtu; peer->mtu = mtu + peer->hdrsize; - spin_unlock_bh(&peer->lock); - _net("Net MTU %u (maxdata %u)", peer->mtu, peer->maxdata); + spin_unlock(&peer->lock); } if (wake) @@ -810,7 +757,6 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) struct rxrpc_ackpacket ack; struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_ackinfo info; - struct sk_buff *skb_old = NULL, *skb_put = skb; rxrpc_serial_t ack_serial, acked_serial; rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt; int nr_acks, offset, ioffset; @@ -818,10 +764,8 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) _enter(""); offset = sizeof(struct rxrpc_wire_header); - if (skb_copy_bits(skb, offset, &ack, sizeof(ack)) < 0) { - rxrpc_proto_abort("XAK", call, 0); - goto out_not_locked; - } + if (skb_copy_bits(skb, offset, &ack, sizeof(ack)) < 0) + return rxrpc_proto_abort("XAK", call, 0); offset += sizeof(ack); ack_serial = sp->hdr.serial; @@ -855,7 +799,6 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) } if (ack.reason == RXRPC_ACK_PING) { - _proto("Rx ACK %%%u PING Request", ack_serial); rxrpc_send_ACK(call, RXRPC_ACK_PING_RESPONSE, ack_serial, rxrpc_propose_ack_respond_to_ping); } else if (sp->hdr.flags & RXRPC_REQUEST_ACK) { @@ -895,41 +838,25 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial, first_soft_ack, call->acks_first_seq, prev_pkt, call->acks_prev_seq); - goto out_not_locked; + return; } info.rxMTU = 0; ioffset = offset + nr_acks + 3; if (skb->len >= ioffset + sizeof(info) && - skb_copy_bits(skb, ioffset, &info, sizeof(info)) < 0) { - rxrpc_proto_abort("XAI", call, 0); - goto out_not_locked; - } + skb_copy_bits(skb, ioffset, &info, sizeof(info)) < 0) + return rxrpc_proto_abort("XAI", call, 0); if (nr_acks > 0) skb_condense(skb); - spin_lock(&call->input_lock); - - /* Discard any out-of-order or duplicate ACKs (inside lock). */ - if (!rxrpc_is_ack_valid(call, first_soft_ack, prev_pkt)) { - trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial, - first_soft_ack, call->acks_first_seq, - prev_pkt, call->acks_prev_seq); - goto out; - } call->acks_latest_ts = skb->tstamp; - call->acks_first_seq = first_soft_ack; call->acks_prev_seq = prev_pkt; switch (ack.reason) { case RXRPC_ACK_PING: break; - case RXRPC_ACK_PING_RESPONSE: - rxrpc_input_ping_response(call, skb->tstamp, acked_serial, - ack_serial); - fallthrough; default: if (after(acked_serial, call->acks_highest_serial)) call->acks_highest_serial = acked_serial; @@ -940,10 +867,8 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) if (info.rxMTU) rxrpc_input_ackinfo(call, skb, &info); - if (first_soft_ack == 0) { - rxrpc_proto_abort("AK0", call, 0); - goto out; - } + if (first_soft_ack == 0) + return rxrpc_proto_abort("AK0", call, 0); /* Ignore ACKs unless we are or have just been transmitting. */ switch (READ_ONCE(call->state)) { @@ -953,45 +878,27 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) case RXRPC_CALL_SERVER_AWAIT_ACK: break; default: - goto out; + return; } if (before(hard_ack, call->acks_hard_ack) || - after(hard_ack, call->tx_top)) { - rxrpc_proto_abort("AKW", call, 0); - goto out; - } - if (nr_acks > call->tx_top - hard_ack) { - rxrpc_proto_abort("AKN", call, 0); - goto out; - } + after(hard_ack, call->tx_top)) + return rxrpc_proto_abort("AKW", call, 0); + if (nr_acks > call->tx_top - hard_ack) + return rxrpc_proto_abort("AKN", call, 0); if (after(hard_ack, call->acks_hard_ack)) { if (rxrpc_rotate_tx_window(call, hard_ack, &summary)) { rxrpc_end_tx_phase(call, false, "ETA"); - goto out; + return; } } if (nr_acks > 0) { - if (offset > (int)skb->len - nr_acks) { - rxrpc_proto_abort("XSA", call, 0); - goto out; - } - - spin_lock(&call->acks_ack_lock); - skb_old = call->acks_soft_tbl; - call->acks_soft_tbl = skb; - spin_unlock(&call->acks_ack_lock); - + if (offset > (int)skb->len - nr_acks) + return rxrpc_proto_abort("XSA", call, 0); rxrpc_input_soft_acks(call, skb->data + offset, first_soft_ack, nr_acks, &summary); - skb_put = NULL; - } else if (call->acks_soft_tbl) { - spin_lock(&call->acks_ack_lock); - skb_old = call->acks_soft_tbl; - call->acks_soft_tbl = NULL; - spin_unlock(&call->acks_ack_lock); } if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) && @@ -1001,11 +908,6 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) rxrpc_propose_ack_ping_for_lost_reply); rxrpc_congestion_management(call, skb, &summary, acked_serial); -out: - spin_unlock(&call->input_lock); -out_not_locked: - rxrpc_free_skb(skb_put, rxrpc_skb_freed); - rxrpc_free_skb(skb_old, rxrpc_skb_freed); } /* @@ -1014,16 +916,9 @@ out_not_locked: static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_ack_summary summary = { 0 }; - struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - - _proto("Rx ACKALL %%%u", sp->hdr.serial); - - spin_lock(&call->input_lock); if (rxrpc_rotate_tx_window(call, call->tx_top, &summary)) rxrpc_end_tx_phase(call, false, "ETL"); - - spin_unlock(&call->input_lock); } /* @@ -1032,35 +927,30 @@ static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb) static void rxrpc_input_abort(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - __be32 wtmp; - u32 abort_code = RX_CALL_DEAD; - - _enter(""); - - if (skb->len >= 4 && - skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), - &wtmp, sizeof(wtmp)) >= 0) - abort_code = ntohl(wtmp); - trace_rxrpc_rx_abort(call, sp->hdr.serial, abort_code); - - _proto("Rx ABORT %%%u { %x }", sp->hdr.serial, abort_code); + trace_rxrpc_rx_abort(call, sp->hdr.serial, skb->priority); rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED, - abort_code, -ECONNABORTED); + skb->priority, -ECONNABORTED); } /* * Process an incoming call packet. */ -static void rxrpc_input_call_packet(struct rxrpc_call *call, - struct sk_buff *skb) +void rxrpc_input_call_packet(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); unsigned long timo; _enter("%p,%p", call, skb); + if (sp->hdr.serviceId != call->dest_srx.srx_service) + call->dest_srx.srx_service = sp->hdr.serviceId; + if ((int)sp->hdr.serial - (int)call->rx_serial > 0) + call->rx_serial = sp->hdr.serial; + if (!test_bit(RXRPC_CALL_RX_HEARD, &call->flags)) + set_bit(RXRPC_CALL_RX_HEARD, &call->flags); + timo = READ_ONCE(call->next_rx_timo); if (timo) { unsigned long now = jiffies, expect_rx_by; @@ -1074,15 +964,13 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, switch (sp->hdr.type) { case RXRPC_PACKET_TYPE_DATA: rxrpc_input_data(call, skb); - goto no_free; + break; case RXRPC_PACKET_TYPE_ACK: rxrpc_input_ack(call, skb); - goto no_free; + break; case RXRPC_PACKET_TYPE_BUSY: - _proto("Rx BUSY %%%u", sp->hdr.serial); - /* Just ignore BUSY packets from the server; the retry and * lifespan timers will take care of business. BUSY packets * from the client don't make sense. @@ -1100,10 +988,6 @@ static void rxrpc_input_call_packet(struct rxrpc_call *call, default: break; } - - rxrpc_free_skb(skb, rxrpc_skb_freed); -no_free: - _leave(""); } /* @@ -1112,10 +996,10 @@ no_free: * * TODO: If callNumber > call_id + 1, renegotiate security. */ -static void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx, - struct rxrpc_connection *conn, - struct rxrpc_call *call) +void rxrpc_implicit_end_call(struct rxrpc_call *call, struct sk_buff *skb) { + struct rxrpc_connection *conn = call->conn; + switch (READ_ONCE(call->state)) { case RXRPC_CALL_SERVER_AWAIT_ACK: rxrpc_call_completed(call); @@ -1123,360 +1007,15 @@ static void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx, case RXRPC_CALL_COMPLETE: break; default: - if (rxrpc_abort_call("IMP", call, 0, RX_CALL_DEAD, -ESHUTDOWN)) { - set_bit(RXRPC_CALL_EV_ABORT, &call->events); - rxrpc_queue_call(call); - } + if (rxrpc_abort_call("IMP", call, 0, RX_CALL_DEAD, -ESHUTDOWN)) + rxrpc_send_abort_packet(call); trace_rxrpc_improper_term(call); break; } - spin_lock(&rx->incoming_lock); - __rxrpc_disconnect_call(conn, call); - spin_unlock(&rx->incoming_lock); -} - -/* - * post connection-level events to the connection - * - this includes challenges, responses, some aborts and call terminal packet - * retransmission. - */ -static void rxrpc_post_packet_to_conn(struct rxrpc_connection *conn, - struct sk_buff *skb) -{ - _enter("%p,%p", conn, skb); - - skb_queue_tail(&conn->rx_queue, skb); - rxrpc_queue_conn(conn); -} - -/* - * post endpoint-level events to the local endpoint - * - this includes debug and version messages - */ -static void rxrpc_post_packet_to_local(struct rxrpc_local *local, - struct sk_buff *skb) -{ - _enter("%p,%p", local, skb); - - if (rxrpc_get_local_maybe(local)) { - skb_queue_tail(&local->event_queue, skb); - rxrpc_queue_local(local); - } else { - rxrpc_free_skb(skb, rxrpc_skb_freed); - } -} - -/* - * put a packet up for transport-level abort - */ -static void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb) -{ - if (rxrpc_get_local_maybe(local)) { - skb_queue_tail(&local->reject_queue, skb); - rxrpc_queue_local(local); - } else { - rxrpc_free_skb(skb, rxrpc_skb_freed); - } -} - -/* - * Extract the wire header from a packet and translate the byte order. - */ -static noinline -int rxrpc_extract_header(struct rxrpc_skb_priv *sp, struct sk_buff *skb) -{ - struct rxrpc_wire_header whdr; - - /* dig out the RxRPC connection details */ - if (skb_copy_bits(skb, 0, &whdr, sizeof(whdr)) < 0) { - trace_rxrpc_rx_eproto(NULL, sp->hdr.serial, - tracepoint_string("bad_hdr")); - return -EBADMSG; - } - - memset(sp, 0, sizeof(*sp)); - sp->hdr.epoch = ntohl(whdr.epoch); - sp->hdr.cid = ntohl(whdr.cid); - sp->hdr.callNumber = ntohl(whdr.callNumber); - sp->hdr.seq = ntohl(whdr.seq); - sp->hdr.serial = ntohl(whdr.serial); - sp->hdr.flags = whdr.flags; - sp->hdr.type = whdr.type; - sp->hdr.userStatus = whdr.userStatus; - sp->hdr.securityIndex = whdr.securityIndex; - sp->hdr._rsvd = ntohs(whdr._rsvd); - sp->hdr.serviceId = ntohs(whdr.serviceId); - return 0; -} - -/* - * handle data received on the local endpoint - * - may be called in interrupt context - * - * [!] Note that as this is called from the encap_rcv hook, the socket is not - * held locked by the caller and nothing prevents sk_user_data on the UDP from - * being cleared in the middle of processing this function. - * - * Called with the RCU read lock held from the IP layer via UDP. - */ -int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb) -{ - struct rxrpc_local *local = rcu_dereference_sk_user_data(udp_sk); - struct rxrpc_connection *conn; - struct rxrpc_channel *chan; - struct rxrpc_call *call = NULL; - struct rxrpc_skb_priv *sp; - struct rxrpc_peer *peer = NULL; - struct rxrpc_sock *rx = NULL; - unsigned int channel; - - _enter("%p", udp_sk); - - if (unlikely(!local)) { - kfree_skb(skb); - return 0; - } - if (skb->tstamp == 0) - skb->tstamp = ktime_get_real(); - - rxrpc_new_skb(skb, rxrpc_skb_received); - - skb_pull(skb, sizeof(struct udphdr)); - - /* The UDP protocol already released all skb resources; - * we are free to add our own data there. - */ - sp = rxrpc_skb(skb); - - /* dig out the RxRPC connection details */ - if (rxrpc_extract_header(sp, skb) < 0) - goto bad_message; - - if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) { - static int lose; - if ((lose++ & 7) == 7) { - trace_rxrpc_rx_lose(sp); - rxrpc_free_skb(skb, rxrpc_skb_lost); - return 0; - } - } - - if (skb->tstamp == 0) - skb->tstamp = ktime_get_real(); - trace_rxrpc_rx_packet(sp); - - switch (sp->hdr.type) { - case RXRPC_PACKET_TYPE_VERSION: - if (rxrpc_to_client(sp)) - goto discard; - rxrpc_post_packet_to_local(local, skb); - goto out; - - case RXRPC_PACKET_TYPE_BUSY: - if (rxrpc_to_server(sp)) - goto discard; - fallthrough; - case RXRPC_PACKET_TYPE_ACK: - case RXRPC_PACKET_TYPE_ACKALL: - if (sp->hdr.callNumber == 0) - goto bad_message; - fallthrough; - case RXRPC_PACKET_TYPE_ABORT: - break; - - case RXRPC_PACKET_TYPE_DATA: - if (sp->hdr.callNumber == 0 || - sp->hdr.seq == 0) - goto bad_message; - - /* Unshare the packet so that it can be modified for in-place - * decryption. - */ - if (sp->hdr.securityIndex != 0) { - struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC); - if (!nskb) { - rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem); - goto out; - } - - if (nskb != skb) { - rxrpc_eaten_skb(skb, rxrpc_skb_received); - skb = nskb; - rxrpc_new_skb(skb, rxrpc_skb_unshared); - sp = rxrpc_skb(skb); - } - } - break; - - case RXRPC_PACKET_TYPE_CHALLENGE: - if (rxrpc_to_server(sp)) - goto discard; - break; - case RXRPC_PACKET_TYPE_RESPONSE: - if (rxrpc_to_client(sp)) - goto discard; - break; - - /* Packet types 9-11 should just be ignored. */ - case RXRPC_PACKET_TYPE_PARAMS: - case RXRPC_PACKET_TYPE_10: - case RXRPC_PACKET_TYPE_11: - goto discard; - - default: - _proto("Rx Bad Packet Type %u", sp->hdr.type); - goto bad_message; - } - - if (sp->hdr.serviceId == 0) - goto bad_message; - - if (rxrpc_to_server(sp)) { - /* Weed out packets to services we're not offering. Packets - * that would begin a call are explicitly rejected and the rest - * are just discarded. - */ - rx = rcu_dereference(local->service); - if (!rx || (sp->hdr.serviceId != rx->srx.srx_service && - sp->hdr.serviceId != rx->second_service)) { - if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA && - sp->hdr.seq == 1) - goto unsupported_service; - goto discard; - } - } - - conn = rxrpc_find_connection_rcu(local, skb, &peer); - if (conn) { - if (sp->hdr.securityIndex != conn->security_ix) - goto wrong_security; - - if (sp->hdr.serviceId != conn->service_id) { - int old_id; - - if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) - goto reupgrade; - old_id = cmpxchg(&conn->service_id, conn->params.service_id, - sp->hdr.serviceId); - - if (old_id != conn->params.service_id && - old_id != sp->hdr.serviceId) - goto reupgrade; - } - - if (sp->hdr.callNumber == 0) { - /* Connection-level packet */ - _debug("CONN %p {%d}", conn, conn->debug_id); - rxrpc_post_packet_to_conn(conn, skb); - goto out; - } - - if ((int)sp->hdr.serial - (int)conn->hi_serial > 0) - conn->hi_serial = sp->hdr.serial; - - /* Call-bound packets are routed by connection channel. */ - channel = sp->hdr.cid & RXRPC_CHANNELMASK; - chan = &conn->channels[channel]; - - /* Ignore really old calls */ - if (sp->hdr.callNumber < chan->last_call) - goto discard; - - if (sp->hdr.callNumber == chan->last_call) { - if (chan->call || - sp->hdr.type == RXRPC_PACKET_TYPE_ABORT) - goto discard; - - /* For the previous service call, if completed - * successfully, we discard all further packets. - */ - if (rxrpc_conn_is_service(conn) && - chan->last_type == RXRPC_PACKET_TYPE_ACK) - goto discard; - - /* But otherwise we need to retransmit the final packet - * from data cached in the connection record. - */ - if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA) - trace_rxrpc_rx_data(chan->call_debug_id, - sp->hdr.seq, - sp->hdr.serial, - sp->hdr.flags); - rxrpc_post_packet_to_conn(conn, skb); - goto out; - } - - call = rcu_dereference(chan->call); - - if (sp->hdr.callNumber > chan->call_id) { - if (rxrpc_to_client(sp)) - goto reject_packet; - if (call) - rxrpc_input_implicit_end_call(rx, conn, call); - call = NULL; - } - - if (call) { - if (sp->hdr.serviceId != call->service_id) - call->service_id = sp->hdr.serviceId; - if ((int)sp->hdr.serial - (int)call->rx_serial > 0) - call->rx_serial = sp->hdr.serial; - if (!test_bit(RXRPC_CALL_RX_HEARD, &call->flags)) - set_bit(RXRPC_CALL_RX_HEARD, &call->flags); - } - } - - if (!call || refcount_read(&call->ref) == 0) { - if (rxrpc_to_client(sp) || - sp->hdr.type != RXRPC_PACKET_TYPE_DATA) - goto bad_message; - if (sp->hdr.seq != 1) - goto discard; - call = rxrpc_new_incoming_call(local, rx, skb); - if (!call) - goto reject_packet; - } - - /* Process a call packet; this either discards or passes on the ref - * elsewhere. - */ - rxrpc_input_call_packet(call, skb); - goto out; + rxrpc_input_call_event(call, skb); -discard: - rxrpc_free_skb(skb, rxrpc_skb_freed); -out: - trace_rxrpc_rx_done(0, 0); - return 0; - -wrong_security: - trace_rxrpc_abort(0, "SEC", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, - RXKADINCONSISTENCY, EBADMSG); - skb->priority = RXKADINCONSISTENCY; - goto post_abort; - -unsupported_service: - trace_rxrpc_abort(0, "INV", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, - RX_INVALID_OPERATION, EOPNOTSUPP); - skb->priority = RX_INVALID_OPERATION; - goto post_abort; - -reupgrade: - trace_rxrpc_abort(0, "UPG", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, - RX_PROTOCOL_ERROR, EBADMSG); - goto protocol_error; - -bad_message: - trace_rxrpc_abort(0, "BAD", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, - RX_PROTOCOL_ERROR, EBADMSG); -protocol_error: - skb->priority = RX_PROTOCOL_ERROR; -post_abort: - skb->mark = RXRPC_SKB_MARK_REJECT_ABORT; -reject_packet: - trace_rxrpc_rx_done(skb->mark, skb->priority); - rxrpc_reject_packet(local, skb); - _leave(" [badmsg]"); - return 0; + spin_lock(&conn->bundle->channel_lock); + __rxrpc_disconnect_call(conn, call); + spin_unlock(&conn->bundle->channel_lock); } diff --git a/net/rxrpc/io_thread.c b/net/rxrpc/io_thread.c new file mode 100644 index 000000000000..d83ae3193032 --- /dev/null +++ b/net/rxrpc/io_thread.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* RxRPC packet reception + * + * Copyright (C) 2007, 2016, 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "ar-internal.h" + +static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn, + struct sockaddr_rxrpc *peer_srx, + struct sk_buff *skb); + +/* + * handle data received on the local endpoint + * - may be called in interrupt context + * + * [!] Note that as this is called from the encap_rcv hook, the socket is not + * held locked by the caller and nothing prevents sk_user_data on the UDP from + * being cleared in the middle of processing this function. + * + * Called with the RCU read lock held from the IP layer via UDP. + */ +int rxrpc_encap_rcv(struct sock *udp_sk, struct sk_buff *skb) +{ + struct rxrpc_local *local = rcu_dereference_sk_user_data(udp_sk); + + if (unlikely(!local)) { + kfree_skb(skb); + return 0; + } + if (skb->tstamp == 0) + skb->tstamp = ktime_get_real(); + + skb->mark = RXRPC_SKB_MARK_PACKET; + rxrpc_new_skb(skb, rxrpc_skb_new_encap_rcv); + skb_queue_tail(&local->rx_queue, skb); + rxrpc_wake_up_io_thread(local); + return 0; +} + +/* + * Handle an error received on the local endpoint. + */ +void rxrpc_error_report(struct sock *sk) +{ + struct rxrpc_local *local; + struct sk_buff *skb; + + rcu_read_lock(); + local = rcu_dereference_sk_user_data(sk); + if (unlikely(!local)) { + rcu_read_unlock(); + return; + } + + while ((skb = skb_dequeue(&sk->sk_error_queue))) { + skb->mark = RXRPC_SKB_MARK_ERROR; + rxrpc_new_skb(skb, rxrpc_skb_new_error_report); + skb_queue_tail(&local->rx_queue, skb); + } + + rxrpc_wake_up_io_thread(local); + rcu_read_unlock(); +} + +/* + * Process event packets targeted at a local endpoint. + */ +static void rxrpc_input_version(struct rxrpc_local *local, struct sk_buff *skb) +{ + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + char v; + + _enter(""); + + rxrpc_see_skb(skb, rxrpc_skb_see_version); + if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), &v, 1) >= 0) { + if (v == 0) + rxrpc_send_version_request(local, &sp->hdr, skb); + } +} + +/* + * Extract the wire header from a packet and translate the byte order. + */ +static noinline +int rxrpc_extract_header(struct rxrpc_skb_priv *sp, struct sk_buff *skb) +{ + struct rxrpc_wire_header whdr; + + /* dig out the RxRPC connection details */ + if (skb_copy_bits(skb, 0, &whdr, sizeof(whdr)) < 0) { + trace_rxrpc_rx_eproto(NULL, sp->hdr.serial, + tracepoint_string("bad_hdr")); + return -EBADMSG; + } + + memset(sp, 0, sizeof(*sp)); + sp->hdr.epoch = ntohl(whdr.epoch); + sp->hdr.cid = ntohl(whdr.cid); + sp->hdr.callNumber = ntohl(whdr.callNumber); + sp->hdr.seq = ntohl(whdr.seq); + sp->hdr.serial = ntohl(whdr.serial); + sp->hdr.flags = whdr.flags; + sp->hdr.type = whdr.type; + sp->hdr.userStatus = whdr.userStatus; + sp->hdr.securityIndex = whdr.securityIndex; + sp->hdr._rsvd = ntohs(whdr._rsvd); + sp->hdr.serviceId = ntohs(whdr.serviceId); + return 0; +} + +/* + * Extract the abort code from an ABORT packet and stash it in skb->priority. + */ +static bool rxrpc_extract_abort(struct sk_buff *skb) +{ + __be32 wtmp; + + if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), + &wtmp, sizeof(wtmp)) < 0) + return false; + skb->priority = ntohl(wtmp); + return true; +} + +/* + * Process packets received on the local endpoint + */ +static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb) +{ + struct rxrpc_connection *conn; + struct sockaddr_rxrpc peer_srx; + struct rxrpc_skb_priv *sp; + struct rxrpc_peer *peer = NULL; + struct sk_buff *skb = *_skb; + int ret = 0; + + skb_pull(skb, sizeof(struct udphdr)); + + sp = rxrpc_skb(skb); + + /* dig out the RxRPC connection details */ + if (rxrpc_extract_header(sp, skb) < 0) + goto bad_message; + + if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) { + static int lose; + if ((lose++ & 7) == 7) { + trace_rxrpc_rx_lose(sp); + return 0; + } + } + + trace_rxrpc_rx_packet(sp); + + switch (sp->hdr.type) { + case RXRPC_PACKET_TYPE_VERSION: + if (rxrpc_to_client(sp)) + return 0; + rxrpc_input_version(local, skb); + return 0; + + case RXRPC_PACKET_TYPE_BUSY: + if (rxrpc_to_server(sp)) + return 0; + fallthrough; + case RXRPC_PACKET_TYPE_ACK: + case RXRPC_PACKET_TYPE_ACKALL: + if (sp->hdr.callNumber == 0) + goto bad_message; + break; + case RXRPC_PACKET_TYPE_ABORT: + if (!rxrpc_extract_abort(skb)) + return 0; /* Just discard if malformed */ + break; + + case RXRPC_PACKET_TYPE_DATA: + if (sp->hdr.callNumber == 0 || + sp->hdr.seq == 0) + goto bad_message; + + /* Unshare the packet so that it can be modified for in-place + * decryption. + */ + if (sp->hdr.securityIndex != 0) { + skb = skb_unshare(skb, GFP_ATOMIC); + if (!skb) { + rxrpc_eaten_skb(*_skb, rxrpc_skb_eaten_by_unshare_nomem); + *_skb = NULL; + return 0; + } + + if (skb != *_skb) { + rxrpc_eaten_skb(*_skb, rxrpc_skb_eaten_by_unshare); + *_skb = skb; + rxrpc_new_skb(skb, rxrpc_skb_new_unshared); + sp = rxrpc_skb(skb); + } + } + break; + + case RXRPC_PACKET_TYPE_CHALLENGE: + if (rxrpc_to_server(sp)) + return 0; + break; + case RXRPC_PACKET_TYPE_RESPONSE: + if (rxrpc_to_client(sp)) + return 0; + break; + + /* Packet types 9-11 should just be ignored. */ + case RXRPC_PACKET_TYPE_PARAMS: + case RXRPC_PACKET_TYPE_10: + case RXRPC_PACKET_TYPE_11: + return 0; + + default: + goto bad_message; + } + + if (sp->hdr.serviceId == 0) + goto bad_message; + + if (WARN_ON_ONCE(rxrpc_extract_addr_from_skb(&peer_srx, skb) < 0)) + return true; /* Unsupported address type - discard. */ + + if (peer_srx.transport.family != local->srx.transport.family && + (peer_srx.transport.family == AF_INET && + local->srx.transport.family != AF_INET6)) { + pr_warn_ratelimited("AF_RXRPC: Protocol mismatch %u not %u\n", + peer_srx.transport.family, + local->srx.transport.family); + return true; /* Wrong address type - discard. */ + } + + if (rxrpc_to_client(sp)) { + rcu_read_lock(); + conn = rxrpc_find_client_connection_rcu(local, &peer_srx, skb); + conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_call_input); + rcu_read_unlock(); + if (!conn) { + trace_rxrpc_abort(0, "NCC", sp->hdr.cid, + sp->hdr.callNumber, sp->hdr.seq, + RXKADINCONSISTENCY, EBADMSG); + goto protocol_error; + } + + ret = rxrpc_input_packet_on_conn(conn, &peer_srx, skb); + rxrpc_put_connection(conn, rxrpc_conn_put_call_input); + return ret; + } + + /* We need to look up service connections by the full protocol + * parameter set. We look up the peer first as an intermediate step + * and then the connection from the peer's tree. + */ + rcu_read_lock(); + + peer = rxrpc_lookup_peer_rcu(local, &peer_srx); + if (!peer) { + rcu_read_unlock(); + return rxrpc_new_incoming_call(local, NULL, NULL, &peer_srx, skb); + } + + conn = rxrpc_find_service_conn_rcu(peer, skb); + conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_call_input); + if (conn) { + rcu_read_unlock(); + ret = rxrpc_input_packet_on_conn(conn, &peer_srx, skb); + rxrpc_put_connection(conn, rxrpc_conn_put_call_input); + return ret; + } + + peer = rxrpc_get_peer_maybe(peer, rxrpc_peer_get_input); + rcu_read_unlock(); + + ret = rxrpc_new_incoming_call(local, peer, NULL, &peer_srx, skb); + rxrpc_put_peer(peer, rxrpc_peer_put_input); + if (ret < 0) + goto reject_packet; + return 0; + +bad_message: + trace_rxrpc_abort(0, "BAD", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, + RX_PROTOCOL_ERROR, EBADMSG); +protocol_error: + skb->priority = RX_PROTOCOL_ERROR; + skb->mark = RXRPC_SKB_MARK_REJECT_ABORT; +reject_packet: + rxrpc_reject_packet(local, skb); + return ret; +} + +/* + * Deal with a packet that's associated with an extant connection. + */ +static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn, + struct sockaddr_rxrpc *peer_srx, + struct sk_buff *skb) +{ + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); + struct rxrpc_channel *chan; + struct rxrpc_call *call = NULL; + unsigned int channel; + + if (sp->hdr.securityIndex != conn->security_ix) + goto wrong_security; + + if (sp->hdr.serviceId != conn->service_id) { + int old_id; + + if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) + goto reupgrade; + old_id = cmpxchg(&conn->service_id, conn->orig_service_id, + sp->hdr.serviceId); + + if (old_id != conn->orig_service_id && + old_id != sp->hdr.serviceId) + goto reupgrade; + } + + if (after(sp->hdr.serial, conn->hi_serial)) + conn->hi_serial = sp->hdr.serial; + + /* It's a connection-level packet if the call number is 0. */ + if (sp->hdr.callNumber == 0) + return rxrpc_input_conn_packet(conn, skb); + + /* Call-bound packets are routed by connection channel. */ + channel = sp->hdr.cid & RXRPC_CHANNELMASK; + chan = &conn->channels[channel]; + + /* Ignore really old calls */ + if (sp->hdr.callNumber < chan->last_call) + return 0; + + if (sp->hdr.callNumber == chan->last_call) { + if (chan->call || + sp->hdr.type == RXRPC_PACKET_TYPE_ABORT) + return 0; + + /* For the previous service call, if completed successfully, we + * discard all further packets. + */ + if (rxrpc_conn_is_service(conn) && + chan->last_type == RXRPC_PACKET_TYPE_ACK) + return 0; + + /* But otherwise we need to retransmit the final packet from + * data cached in the connection record. + */ + if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA) + trace_rxrpc_rx_data(chan->call_debug_id, + sp->hdr.seq, + sp->hdr.serial, + sp->hdr.flags); + rxrpc_input_conn_packet(conn, skb); + return 0; + } + + rcu_read_lock(); + call = rxrpc_try_get_call(rcu_dereference(chan->call), + rxrpc_call_get_input); + rcu_read_unlock(); + + if (sp->hdr.callNumber > chan->call_id) { + if (rxrpc_to_client(sp)) { + rxrpc_put_call(call, rxrpc_call_put_input); + goto reject_packet; + } + + if (call) { + rxrpc_implicit_end_call(call, skb); + rxrpc_put_call(call, rxrpc_call_put_input); + call = NULL; + } + } + + if (!call) { + if (rxrpc_to_client(sp)) + goto bad_message; + if (rxrpc_new_incoming_call(conn->local, conn->peer, conn, + peer_srx, skb)) + return 0; + goto reject_packet; + } + + rxrpc_input_call_event(call, skb); + rxrpc_put_call(call, rxrpc_call_put_input); + return 0; + +wrong_security: + trace_rxrpc_abort(0, "SEC", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, + RXKADINCONSISTENCY, EBADMSG); + skb->priority = RXKADINCONSISTENCY; + goto post_abort; + +reupgrade: + trace_rxrpc_abort(0, "UPG", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, + RX_PROTOCOL_ERROR, EBADMSG); + goto protocol_error; + +bad_message: + trace_rxrpc_abort(0, "BAD", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq, + RX_PROTOCOL_ERROR, EBADMSG); +protocol_error: + skb->priority = RX_PROTOCOL_ERROR; +post_abort: + skb->mark = RXRPC_SKB_MARK_REJECT_ABORT; +reject_packet: + rxrpc_reject_packet(conn->local, skb); + return 0; +} + +/* + * I/O and event handling thread. + */ +int rxrpc_io_thread(void *data) +{ + struct sk_buff_head rx_queue; + struct rxrpc_local *local = data; + struct rxrpc_call *call; + struct sk_buff *skb; + + skb_queue_head_init(&rx_queue); + + set_user_nice(current, MIN_NICE); + + for (;;) { + rxrpc_inc_stat(local->rxnet, stat_io_loop); + + /* Deal with calls that want immediate attention. */ + if ((call = list_first_entry_or_null(&local->call_attend_q, + struct rxrpc_call, + attend_link))) { + spin_lock_bh(&local->lock); + list_del_init(&call->attend_link); + spin_unlock_bh(&local->lock); + + trace_rxrpc_call_poked(call); + rxrpc_input_call_event(call, NULL); + rxrpc_put_call(call, rxrpc_call_put_poke); + continue; + } + + /* Process received packets and errors. */ + if ((skb = __skb_dequeue(&rx_queue))) { + switch (skb->mark) { + case RXRPC_SKB_MARK_PACKET: + skb->priority = 0; + rxrpc_input_packet(local, &skb); + trace_rxrpc_rx_done(skb->mark, skb->priority); + rxrpc_free_skb(skb, rxrpc_skb_put_input); + break; + case RXRPC_SKB_MARK_ERROR: + rxrpc_input_error(local, skb); + rxrpc_free_skb(skb, rxrpc_skb_put_error_report); + break; + default: + WARN_ON_ONCE(1); + rxrpc_free_skb(skb, rxrpc_skb_put_unknown); + break; + } + continue; + } + + if (!skb_queue_empty(&local->rx_queue)) { + spin_lock_irq(&local->rx_queue.lock); + skb_queue_splice_tail_init(&local->rx_queue, &rx_queue); + spin_unlock_irq(&local->rx_queue.lock); + continue; + } + + set_current_state(TASK_INTERRUPTIBLE); + if (!skb_queue_empty(&local->rx_queue) || + !list_empty(&local->call_attend_q)) { + __set_current_state(TASK_RUNNING); + continue; + } + + if (kthread_should_stop()) + break; + schedule(); + } + + __set_current_state(TASK_RUNNING); + rxrpc_see_local(local, rxrpc_local_stop); + rxrpc_destroy_local(local); + local->io_thread = NULL; + rxrpc_see_local(local, rxrpc_local_stopped); + return 0; +} diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 8d2073e0e3da..8d53aded09c4 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -513,7 +513,7 @@ int rxrpc_get_server_data_key(struct rxrpc_connection *conn, if (ret < 0) goto error; - conn->params.key = key; + conn->key = key; _leave(" = 0 [%d]", key_serial(key)); return 0; @@ -602,7 +602,8 @@ static long rxrpc_read(const struct key *key, } _debug("token[%u]: toksize=%u", ntoks, toksize); - ASSERTCMP(toksize, <=, AFSTOKEN_LENGTH_MAX); + if (WARN_ON(toksize > AFSTOKEN_LENGTH_MAX)) + return -EIO; toksizes[ntoks++] = toksize; size += toksize + 4; /* each token has a length word */ @@ -679,8 +680,9 @@ static long rxrpc_read(const struct key *key, return -ENOPKG; } - ASSERTCMP((unsigned long)xdr - (unsigned long)oldxdr, ==, - toksize); + if (WARN_ON((unsigned long)xdr - (unsigned long)oldxdr == + toksize)) + return -EIO; } #undef ENCODE_STR @@ -688,8 +690,10 @@ static long rxrpc_read(const struct key *key, #undef ENCODE64 #undef ENCODE - ASSERTCMP(tok, ==, ntoks); - ASSERTCMP((char __user *) xdr - buffer, ==, size); + if (WARN_ON(tok != ntoks)) + return -EIO; + if (WARN_ON((unsigned long)xdr - (unsigned long)buffer != size)) + return -EIO; _leave(" = %zu", size); return size; } diff --git a/net/rxrpc/local_event.c b/net/rxrpc/local_event.c index 19e929c7c38b..5e69ea6b233d 100644 --- a/net/rxrpc/local_event.c +++ b/net/rxrpc/local_event.c @@ -21,9 +21,9 @@ static const char rxrpc_version_string[65] = "linux-" UTS_RELEASE " AF_RXRPC"; /* * Reply to a version request */ -static void rxrpc_send_version_request(struct rxrpc_local *local, - struct rxrpc_host_header *hdr, - struct sk_buff *skb) +void rxrpc_send_version_request(struct rxrpc_local *local, + struct rxrpc_host_header *hdr, + struct sk_buff *skb) { struct rxrpc_wire_header whdr; struct rxrpc_skb_priv *sp = rxrpc_skb(skb); @@ -63,8 +63,6 @@ static void rxrpc_send_version_request(struct rxrpc_local *local, len = iov[0].iov_len + iov[1].iov_len; - _proto("Tx VERSION (reply)"); - ret = kernel_sendmsg(local->socket, &msg, iov, 2, len); if (ret < 0) trace_rxrpc_tx_fail(local->debug_id, 0, ret, @@ -75,41 +73,3 @@ static void rxrpc_send_version_request(struct rxrpc_local *local, _leave(""); } - -/* - * Process event packets targeted at a local endpoint. - */ -void rxrpc_process_local_events(struct rxrpc_local *local) -{ - struct sk_buff *skb; - char v; - - _enter(""); - - skb = skb_dequeue(&local->event_queue); - if (skb) { - struct rxrpc_skb_priv *sp = rxrpc_skb(skb); - - rxrpc_see_skb(skb, rxrpc_skb_seen); - _debug("{%d},{%u}", local->debug_id, sp->hdr.type); - - switch (sp->hdr.type) { - case RXRPC_PACKET_TYPE_VERSION: - if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), - &v, 1) < 0) - return; - _proto("Rx VERSION { %02x }", v); - if (v == 0) - rxrpc_send_version_request(local, &sp->hdr, skb); - break; - - default: - /* Just ignore anything we don't understand */ - break; - } - - rxrpc_free_skb(skb, rxrpc_skb_freed); - } - - _leave(""); -} diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c index a943fdf91e24..44222923c0d1 100644 --- a/net/rxrpc/local_object.c +++ b/net/rxrpc/local_object.c @@ -20,7 +20,6 @@ #include <net/af_rxrpc.h> #include "ar-internal.h" -static void rxrpc_local_processor(struct work_struct *); static void rxrpc_local_rcu(struct rcu_head *); /* @@ -97,12 +96,9 @@ static struct rxrpc_local *rxrpc_alloc_local(struct rxrpc_net *rxnet, atomic_set(&local->active_users, 1); local->rxnet = rxnet; INIT_HLIST_NODE(&local->link); - INIT_WORK(&local->processor, rxrpc_local_processor); - INIT_LIST_HEAD(&local->ack_tx_queue); - spin_lock_init(&local->ack_tx_lock); init_rwsem(&local->defrag_sem); - skb_queue_head_init(&local->reject_queue); - skb_queue_head_init(&local->event_queue); + skb_queue_head_init(&local->rx_queue); + INIT_LIST_HEAD(&local->call_attend_q); local->client_bundles = RB_ROOT; spin_lock_init(&local->client_bundles_lock); spin_lock_init(&local->lock); @@ -110,7 +106,7 @@ static struct rxrpc_local *rxrpc_alloc_local(struct rxrpc_net *rxnet, local->debug_id = atomic_inc_return(&rxrpc_debug_id); memcpy(&local->srx, srx, sizeof(*srx)); local->srx.srx_service = 0; - trace_rxrpc_local(local->debug_id, rxrpc_local_new, 1, NULL); + trace_rxrpc_local(local->debug_id, rxrpc_local_new, 1, 1); } _leave(" = %p", local); @@ -126,6 +122,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) struct udp_tunnel_sock_cfg tuncfg = {NULL}; struct sockaddr_rxrpc *srx = &local->srx; struct udp_port_cfg udp_conf = {0}; + struct task_struct *io_thread; struct sock *usk; int ret; @@ -152,7 +149,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) } tuncfg.encap_type = UDP_ENCAP_RXRPC; - tuncfg.encap_rcv = rxrpc_input_packet; + tuncfg.encap_rcv = rxrpc_encap_rcv; tuncfg.encap_err_rcv = rxrpc_encap_err_rcv; tuncfg.sk_user_data = local; setup_udp_tunnel_sock(net, local->socket, &tuncfg); @@ -185,8 +182,23 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) BUG(); } + io_thread = kthread_run(rxrpc_io_thread, local, + "krxrpcio/%u", ntohs(udp_conf.local_udp_port)); + if (IS_ERR(io_thread)) { + ret = PTR_ERR(io_thread); + goto error_sock; + } + + local->io_thread = io_thread; _leave(" = 0"); return 0; + +error_sock: + kernel_sock_shutdown(local->socket, SHUT_RDWR); + local->socket->sk->sk_user_data = NULL; + sock_release(local->socket); + local->socket = NULL; + return ret; } /* @@ -198,7 +210,6 @@ struct rxrpc_local *rxrpc_lookup_local(struct net *net, struct rxrpc_local *local; struct rxrpc_net *rxnet = rxrpc_net(net); struct hlist_node *cursor; - const char *age; long diff; int ret; @@ -229,10 +240,9 @@ struct rxrpc_local *rxrpc_lookup_local(struct net *net, * we're attempting to use a local address that the dying * object is still using. */ - if (!rxrpc_use_local(local)) + if (!rxrpc_use_local(local, rxrpc_local_use_lookup)) break; - age = "old"; goto found; } @@ -250,14 +260,9 @@ struct rxrpc_local *rxrpc_lookup_local(struct net *net, } else { hlist_add_head_rcu(&local->link, &rxnet->local_endpoints); } - age = "new"; found: mutex_unlock(&rxnet->local_mutex); - - _net("LOCAL %s %d {%pISp}", - age, local->debug_id, &local->srx.transport); - _leave(" = %p", local); return local; @@ -279,64 +284,49 @@ addr_in_use: /* * Get a ref on a local endpoint. */ -struct rxrpc_local *rxrpc_get_local(struct rxrpc_local *local) +struct rxrpc_local *rxrpc_get_local(struct rxrpc_local *local, + enum rxrpc_local_trace why) { - const void *here = __builtin_return_address(0); - int r; + int r, u; + u = atomic_read(&local->active_users); __refcount_inc(&local->ref, &r); - trace_rxrpc_local(local->debug_id, rxrpc_local_got, r + 1, here); + trace_rxrpc_local(local->debug_id, why, r + 1, u); return local; } /* * Get a ref on a local endpoint unless its usage has already reached 0. */ -struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *local) +struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *local, + enum rxrpc_local_trace why) { - const void *here = __builtin_return_address(0); - int r; + int r, u; - if (local) { - if (__refcount_inc_not_zero(&local->ref, &r)) - trace_rxrpc_local(local->debug_id, rxrpc_local_got, - r + 1, here); - else - local = NULL; + if (local && __refcount_inc_not_zero(&local->ref, &r)) { + u = atomic_read(&local->active_users); + trace_rxrpc_local(local->debug_id, why, r + 1, u); + return local; } - return local; -} -/* - * Queue a local endpoint and pass the caller's reference to the work item. - */ -void rxrpc_queue_local(struct rxrpc_local *local) -{ - const void *here = __builtin_return_address(0); - unsigned int debug_id = local->debug_id; - int r = refcount_read(&local->ref); - - if (rxrpc_queue_work(&local->processor)) - trace_rxrpc_local(debug_id, rxrpc_local_queued, r + 1, here); - else - rxrpc_put_local(local); + return NULL; } /* * Drop a ref on a local endpoint. */ -void rxrpc_put_local(struct rxrpc_local *local) +void rxrpc_put_local(struct rxrpc_local *local, enum rxrpc_local_trace why) { - const void *here = __builtin_return_address(0); unsigned int debug_id; bool dead; - int r; + int r, u; if (local) { debug_id = local->debug_id; + u = atomic_read(&local->active_users); dead = __refcount_dec_and_test(&local->ref, &r); - trace_rxrpc_local(debug_id, rxrpc_local_put, r, here); + trace_rxrpc_local(debug_id, why, r, u); if (dead) call_rcu(&local->rcu, rxrpc_local_rcu); @@ -346,14 +336,15 @@ void rxrpc_put_local(struct rxrpc_local *local) /* * Start using a local endpoint. */ -struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *local) +struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *local, + enum rxrpc_local_trace why) { - local = rxrpc_get_local_maybe(local); + local = rxrpc_get_local_maybe(local, rxrpc_local_get_for_use); if (!local) return NULL; - if (!__rxrpc_use_local(local)) { - rxrpc_put_local(local); + if (!__rxrpc_use_local(local, why)) { + rxrpc_put_local(local, rxrpc_local_put_for_use); return NULL; } @@ -362,15 +353,19 @@ struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *local) /* * Cease using a local endpoint. Once the number of active users reaches 0, we - * start the closure of the transport in the work processor. + * start the closure of the transport in the I/O thread.. */ -void rxrpc_unuse_local(struct rxrpc_local *local) +void rxrpc_unuse_local(struct rxrpc_local *local, enum rxrpc_local_trace why) { + unsigned int debug_id = local->debug_id; + int r, u; + if (local) { - if (__rxrpc_unuse_local(local)) { - rxrpc_get_local(local); - rxrpc_queue_local(local); - } + r = refcount_read(&local->ref); + u = atomic_dec_return(&local->active_users); + trace_rxrpc_local(debug_id, why, r, u); + if (u == 0) + kthread_stop(local->io_thread); } } @@ -381,7 +376,7 @@ void rxrpc_unuse_local(struct rxrpc_local *local) * Closing the socket cannot be done from bottom half context or RCU callback * context because it might sleep. */ -static void rxrpc_local_destroyer(struct rxrpc_local *local) +void rxrpc_destroy_local(struct rxrpc_local *local) { struct socket *socket = local->socket; struct rxrpc_net *rxnet = local->rxnet; @@ -408,52 +403,7 @@ static void rxrpc_local_destroyer(struct rxrpc_local *local) /* At this point, there should be no more packets coming in to the * local endpoint. */ - rxrpc_purge_queue(&local->reject_queue); - rxrpc_purge_queue(&local->event_queue); -} - -/* - * Process events on an endpoint. The work item carries a ref which - * we must release. - */ -static void rxrpc_local_processor(struct work_struct *work) -{ - struct rxrpc_local *local = - container_of(work, struct rxrpc_local, processor); - bool again; - - if (local->dead) - return; - - trace_rxrpc_local(local->debug_id, rxrpc_local_processing, - refcount_read(&local->ref), NULL); - - do { - again = false; - if (!__rxrpc_use_local(local)) { - rxrpc_local_destroyer(local); - break; - } - - if (!list_empty(&local->ack_tx_queue)) { - rxrpc_transmit_ack_packets(local); - again = true; - } - - if (!skb_queue_empty(&local->reject_queue)) { - rxrpc_reject_packets(local); - again = true; - } - - if (!skb_queue_empty(&local->event_queue)) { - rxrpc_process_local_events(local); - again = true; - } - - __rxrpc_unuse_local(local); - } while (again); - - rxrpc_put_local(local); + rxrpc_purge_queue(&local->rx_queue); } /* @@ -463,13 +413,8 @@ static void rxrpc_local_rcu(struct rcu_head *rcu) { struct rxrpc_local *local = container_of(rcu, struct rxrpc_local, rcu); - _enter("%d", local->debug_id); - - ASSERT(!work_pending(&local->processor)); - - _net("DESTROY LOCAL %d", local->debug_id); + rxrpc_see_local(local, rxrpc_local_free); kfree(local); - _leave(""); } /* diff --git a/net/rxrpc/net_ns.c b/net/rxrpc/net_ns.c index 84242c0e467c..5905530e2f33 100644 --- a/net/rxrpc/net_ns.c +++ b/net/rxrpc/net_ns.c @@ -65,7 +65,7 @@ static __net_init int rxrpc_init_net(struct net *net) atomic_set(&rxnet->nr_client_conns, 0); rxnet->kill_all_client_conns = false; spin_lock_init(&rxnet->client_conn_cache_lock); - spin_lock_init(&rxnet->client_conn_discard_lock); + mutex_init(&rxnet->client_conn_discard_lock); INIT_LIST_HEAD(&rxnet->idle_client_conns); INIT_WORK(&rxnet->client_conn_reaper, rxrpc_discard_expired_client_conns); diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c index c5eed0e83e47..3d8c9f830ee0 100644 --- a/net/rxrpc/output.c +++ b/net/rxrpc/output.c @@ -142,8 +142,8 @@ retry: txb->ack.reason = RXRPC_ACK_IDLE; } - mtu = conn->params.peer->if_mtu; - mtu -= conn->params.peer->hdrsize; + mtu = conn->peer->if_mtu; + mtu -= conn->peer->hdrsize; jmax = rxrpc_rx_jumbo_max; qsize = (window - 1) - call->rx_consumed; rsize = max_t(int, call->rx_winsize - qsize, 0); @@ -203,12 +203,11 @@ static void rxrpc_cancel_rtt_probe(struct rxrpc_call *call, } /* - * Send an ACK call packet. + * Transmit an ACK packet. */ -static int rxrpc_send_ack_packet(struct rxrpc_local *local, struct rxrpc_txbuf *txb) +int rxrpc_send_ack_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) { struct rxrpc_connection *conn; - struct rxrpc_call *call = txb->call; struct msghdr msg; struct kvec iov[1]; rxrpc_serial_t serial; @@ -229,11 +228,6 @@ static int rxrpc_send_ack_packet(struct rxrpc_local *local, struct rxrpc_txbuf * if (txb->ack.reason == RXRPC_ACK_PING) txb->wire.flags |= RXRPC_REQUEST_ACK; - if (txb->ack.reason == RXRPC_ACK_DELAY) - clear_bit(RXRPC_CALL_DELAY_ACK_PENDING, &call->flags); - if (txb->ack.reason == RXRPC_ACK_IDLE) - clear_bit(RXRPC_CALL_IDLE_ACK_PENDING, &call->flags); - n = rxrpc_fill_out_ack(conn, call, txb); if (n == 0) return 0; @@ -247,8 +241,6 @@ static int rxrpc_send_ack_packet(struct rxrpc_local *local, struct rxrpc_txbuf * trace_rxrpc_tx_ack(call->debug_id, serial, ntohl(txb->ack.firstPacket), ntohl(txb->ack.serial), txb->ack.reason, txb->ack.nAcks); - if (txb->ack_why == rxrpc_propose_ack_ping_for_lost_ack) - call->acks_lost_ping = serial; if (txb->ack.reason == RXRPC_ACK_PING) rtt_slot = rxrpc_begin_rtt_probe(call, serial, rxrpc_rtt_tx_ping); @@ -259,7 +251,7 @@ static int rxrpc_send_ack_packet(struct rxrpc_local *local, struct rxrpc_txbuf * txb->ack.previousPacket = htonl(call->rx_highest_seq); iov_iter_kvec(&msg.msg_iter, WRITE, iov, 1, len); - ret = do_udp_sendmsg(conn->params.local->socket, &msg, len); + ret = do_udp_sendmsg(conn->local->socket, &msg, len); call->peer->last_tx_at = ktime_get_seconds(); if (ret < 0) trace_rxrpc_tx_fail(call->debug_id, serial, ret, @@ -279,44 +271,6 @@ static int rxrpc_send_ack_packet(struct rxrpc_local *local, struct rxrpc_txbuf * } /* - * ACK transmitter for a local endpoint. The UDP socket locks around each - * transmission, so we can only transmit one packet at a time, ACK, DATA or - * otherwise. - */ -void rxrpc_transmit_ack_packets(struct rxrpc_local *local) -{ - LIST_HEAD(queue); - int ret; - - trace_rxrpc_local(local->debug_id, rxrpc_local_tx_ack, - refcount_read(&local->ref), NULL); - - if (list_empty(&local->ack_tx_queue)) - return; - - spin_lock_bh(&local->ack_tx_lock); - list_splice_tail_init(&local->ack_tx_queue, &queue); - spin_unlock_bh(&local->ack_tx_lock); - - while (!list_empty(&queue)) { - struct rxrpc_txbuf *txb = - list_entry(queue.next, struct rxrpc_txbuf, tx_link); - - ret = rxrpc_send_ack_packet(local, txb); - if (ret < 0 && ret != -ECONNRESET) { - spin_lock_bh(&local->ack_tx_lock); - list_splice_init(&queue, &local->ack_tx_queue); - spin_unlock_bh(&local->ack_tx_lock); - break; - } - - list_del_init(&txb->tx_link); - rxrpc_put_call(txb->call, rxrpc_call_put); - rxrpc_put_txbuf(txb, rxrpc_txbuf_put_ack_tx); - } -} - -/* * Send an ABORT call packet. */ int rxrpc_send_abort_packet(struct rxrpc_call *call) @@ -358,7 +312,7 @@ int rxrpc_send_abort_packet(struct rxrpc_call *call) pkt.whdr.userStatus = 0; pkt.whdr.securityIndex = call->security_ix; pkt.whdr._rsvd = 0; - pkt.whdr.serviceId = htons(call->service_id); + pkt.whdr.serviceId = htons(call->dest_srx.srx_service); pkt.abort_code = htonl(call->abort_code); iov[0].iov_base = &pkt; @@ -368,8 +322,8 @@ int rxrpc_send_abort_packet(struct rxrpc_call *call) pkt.whdr.serial = htonl(serial); iov_iter_kvec(&msg.msg_iter, WRITE, iov, 1, sizeof(pkt)); - ret = do_udp_sendmsg(conn->params.local->socket, &msg, sizeof(pkt)); - conn->params.peer->last_tx_at = ktime_get_seconds(); + ret = do_udp_sendmsg(conn->local->socket, &msg, sizeof(pkt)); + conn->peer->last_tx_at = ktime_get_seconds(); if (ret < 0) trace_rxrpc_tx_fail(call->debug_id, serial, ret, rxrpc_tx_point_call_abort); @@ -395,12 +349,6 @@ int rxrpc_send_data_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) _enter("%x,{%d}", txb->seq, txb->len); - if (hlist_unhashed(&call->error_link)) { - spin_lock_bh(&call->peer->lock); - hlist_add_head_rcu(&call->error_link, &call->peer->error_targets); - spin_unlock_bh(&call->peer->lock); - } - /* Each transmission of a Tx packet needs a new serial number */ serial = atomic_inc_return(&conn->serial); txb->wire.serial = htonl(serial); @@ -466,6 +414,14 @@ dont_set_request_ack: trace_rxrpc_tx_data(call, txb->seq, serial, txb->wire.flags, test_bit(RXRPC_TXBUF_RESENT, &txb->flags), false); + + /* Track what we've attempted to transmit at least once so that the + * retransmission algorithm doesn't try to resend what we haven't sent + * yet. However, this can race as we can receive an ACK before we get + * to this point. But, OTOH, if we won't get an ACK mentioning this + * packet unless the far side received it (though it could have + * discarded it anyway and NAK'd it). + */ cmpxchg(&call->tx_transmitted, txb->seq - 1, txb->seq); /* send the packet with the don't fragment bit set if we currently @@ -473,7 +429,7 @@ dont_set_request_ack: if (txb->len >= call->peer->maxdata) goto send_fragmentable; - down_read(&conn->params.local->defrag_sem); + down_read(&conn->local->defrag_sem); txb->last_sent = ktime_get_real(); if (txb->wire.flags & RXRPC_REQUEST_ACK) @@ -486,11 +442,12 @@ dont_set_request_ack: * message and update the peer record */ rxrpc_inc_stat(call->rxnet, stat_tx_data_send); - ret = do_udp_sendmsg(conn->params.local->socket, &msg, len); - conn->params.peer->last_tx_at = ktime_get_seconds(); + ret = do_udp_sendmsg(conn->local->socket, &msg, len); + conn->peer->last_tx_at = ktime_get_seconds(); - up_read(&conn->params.local->defrag_sem); + up_read(&conn->local->defrag_sem); if (ret < 0) { + rxrpc_inc_stat(call->rxnet, stat_tx_data_send_fail); rxrpc_cancel_rtt_probe(call, serial, rtt_slot); trace_rxrpc_tx_fail(call->debug_id, serial, ret, rxrpc_tx_point_call_data_nofrag); @@ -549,22 +506,22 @@ send_fragmentable: /* attempt to send this message with fragmentation enabled */ _debug("send fragment"); - down_write(&conn->params.local->defrag_sem); + down_write(&conn->local->defrag_sem); txb->last_sent = ktime_get_real(); if (txb->wire.flags & RXRPC_REQUEST_ACK) rtt_slot = rxrpc_begin_rtt_probe(call, serial, rxrpc_rtt_tx_data); - switch (conn->params.local->srx.transport.family) { + switch (conn->local->srx.transport.family) { case AF_INET6: case AF_INET: - ip_sock_set_mtu_discover(conn->params.local->socket->sk, + ip_sock_set_mtu_discover(conn->local->socket->sk, IP_PMTUDISC_DONT); rxrpc_inc_stat(call->rxnet, stat_tx_data_send_frag); - ret = do_udp_sendmsg(conn->params.local->socket, &msg, len); - conn->params.peer->last_tx_at = ktime_get_seconds(); + ret = do_udp_sendmsg(conn->local->socket, &msg, len); + conn->peer->last_tx_at = ktime_get_seconds(); - ip_sock_set_mtu_discover(conn->params.local->socket->sk, + ip_sock_set_mtu_discover(conn->local->socket->sk, IP_PMTUDISC_DO); break; @@ -573,6 +530,7 @@ send_fragmentable: } if (ret < 0) { + rxrpc_inc_stat(call->rxnet, stat_tx_data_send_fail); rxrpc_cancel_rtt_probe(call, serial, rtt_slot); trace_rxrpc_tx_fail(call->debug_id, serial, ret, rxrpc_tx_point_call_data_frag); @@ -582,26 +540,25 @@ send_fragmentable: } rxrpc_tx_backoff(call, ret); - up_write(&conn->params.local->defrag_sem); + up_write(&conn->local->defrag_sem); goto done; } /* - * reject packets through the local endpoint + * Reject a packet through the local endpoint. */ -void rxrpc_reject_packets(struct rxrpc_local *local) +void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb) { - struct sockaddr_rxrpc srx; - struct rxrpc_skb_priv *sp; struct rxrpc_wire_header whdr; - struct sk_buff *skb; + struct sockaddr_rxrpc srx; + struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct msghdr msg; struct kvec iov[2]; size_t size; __be32 code; int ret, ioc; - _enter("%d", local->debug_id); + rxrpc_see_skb(skb, rxrpc_skb_see_reject); iov[0].iov_base = &whdr; iov[0].iov_len = sizeof(whdr); @@ -615,52 +572,42 @@ void rxrpc_reject_packets(struct rxrpc_local *local) memset(&whdr, 0, sizeof(whdr)); - while ((skb = skb_dequeue(&local->reject_queue))) { - rxrpc_see_skb(skb, rxrpc_skb_seen); - sp = rxrpc_skb(skb); + switch (skb->mark) { + case RXRPC_SKB_MARK_REJECT_BUSY: + whdr.type = RXRPC_PACKET_TYPE_BUSY; + size = sizeof(whdr); + ioc = 1; + break; + case RXRPC_SKB_MARK_REJECT_ABORT: + whdr.type = RXRPC_PACKET_TYPE_ABORT; + code = htonl(skb->priority); + size = sizeof(whdr) + sizeof(code); + ioc = 2; + break; + default: + return; + } - switch (skb->mark) { - case RXRPC_SKB_MARK_REJECT_BUSY: - whdr.type = RXRPC_PACKET_TYPE_BUSY; - size = sizeof(whdr); - ioc = 1; - break; - case RXRPC_SKB_MARK_REJECT_ABORT: - whdr.type = RXRPC_PACKET_TYPE_ABORT; - code = htonl(skb->priority); - size = sizeof(whdr) + sizeof(code); - ioc = 2; - break; - default: - rxrpc_free_skb(skb, rxrpc_skb_freed); - continue; - } + if (rxrpc_extract_addr_from_skb(&srx, skb) == 0) { + msg.msg_namelen = srx.transport_len; - if (rxrpc_extract_addr_from_skb(&srx, skb) == 0) { - msg.msg_namelen = srx.transport_len; - - whdr.epoch = htonl(sp->hdr.epoch); - whdr.cid = htonl(sp->hdr.cid); - whdr.callNumber = htonl(sp->hdr.callNumber); - whdr.serviceId = htons(sp->hdr.serviceId); - whdr.flags = sp->hdr.flags; - whdr.flags ^= RXRPC_CLIENT_INITIATED; - whdr.flags &= RXRPC_CLIENT_INITIATED; - - iov_iter_kvec(&msg.msg_iter, WRITE, iov, ioc, size); - ret = do_udp_sendmsg(local->socket, &msg, size); - if (ret < 0) - trace_rxrpc_tx_fail(local->debug_id, 0, ret, - rxrpc_tx_point_reject); - else - trace_rxrpc_tx_packet(local->debug_id, &whdr, - rxrpc_tx_point_reject); - } + whdr.epoch = htonl(sp->hdr.epoch); + whdr.cid = htonl(sp->hdr.cid); + whdr.callNumber = htonl(sp->hdr.callNumber); + whdr.serviceId = htons(sp->hdr.serviceId); + whdr.flags = sp->hdr.flags; + whdr.flags ^= RXRPC_CLIENT_INITIATED; + whdr.flags &= RXRPC_CLIENT_INITIATED; - rxrpc_free_skb(skb, rxrpc_skb_freed); + iov_iter_kvec(&msg.msg_iter, WRITE, iov, ioc, size); + ret = do_udp_sendmsg(local->socket, &msg, size); + if (ret < 0) + trace_rxrpc_tx_fail(local->debug_id, 0, ret, + rxrpc_tx_point_reject); + else + trace_rxrpc_tx_packet(local->debug_id, &whdr, + rxrpc_tx_point_reject); } - - _leave(""); } /* @@ -701,8 +648,6 @@ void rxrpc_send_keepalive(struct rxrpc_peer *peer) len = iov[0].iov_len + iov[1].iov_len; - _proto("Tx VERSION (keepalive)"); - iov_iter_kvec(&msg.msg_iter, WRITE, iov, 2, len); ret = do_udp_sendmsg(peer->local->socket, &msg, len); if (ret < 0) @@ -715,3 +660,43 @@ void rxrpc_send_keepalive(struct rxrpc_peer *peer) peer->last_tx_at = ktime_get_seconds(); _leave(""); } + +/* + * Schedule an instant Tx resend. + */ +static inline void rxrpc_instant_resend(struct rxrpc_call *call, + struct rxrpc_txbuf *txb) +{ + if (call->state < RXRPC_CALL_COMPLETE) + kdebug("resend"); +} + +/* + * Transmit one packet. + */ +void rxrpc_transmit_one(struct rxrpc_call *call, struct rxrpc_txbuf *txb) +{ + int ret; + + ret = rxrpc_send_data_packet(call, txb); + if (ret < 0) { + switch (ret) { + case -ENETUNREACH: + case -EHOSTUNREACH: + case -ECONNREFUSED: + rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, + 0, ret); + break; + default: + _debug("need instant resend %d", ret); + rxrpc_instant_resend(call, txb); + } + } else { + unsigned long now = jiffies; + unsigned long resend_at = now + call->peer->rto_j; + + WRITE_ONCE(call->resend_at, resend_at); + rxrpc_reduce_call_timer(call, resend_at, now, + rxrpc_timer_set_for_send); + } +} diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c index cda3890657a9..6685bf917aa6 100644 --- a/net/rxrpc/peer_event.c +++ b/net/rxrpc/peer_event.c @@ -18,9 +18,9 @@ #include <net/ip.h> #include "ar-internal.h" -static void rxrpc_store_error(struct rxrpc_peer *, struct sock_exterr_skb *); -static void rxrpc_distribute_error(struct rxrpc_peer *, int, - enum rxrpc_call_completion); +static void rxrpc_store_error(struct rxrpc_peer *, struct sk_buff *); +static void rxrpc_distribute_error(struct rxrpc_peer *, struct sk_buff *, + enum rxrpc_call_completion, int); /* * Find the peer associated with a local error. @@ -48,13 +48,11 @@ static struct rxrpc_peer *rxrpc_lookup_peer_local_rcu(struct rxrpc_local *local, srx->transport.sin.sin_port = serr->port; switch (serr->ee.ee_origin) { case SO_EE_ORIGIN_ICMP: - _net("Rx ICMP"); memcpy(&srx->transport.sin.sin_addr, skb_network_header(skb) + serr->addr_offset, sizeof(struct in_addr)); break; case SO_EE_ORIGIN_ICMP6: - _net("Rx ICMP6 on v4 sock"); memcpy(&srx->transport.sin.sin_addr, skb_network_header(skb) + serr->addr_offset + 12, sizeof(struct in_addr)); @@ -70,14 +68,12 @@ static struct rxrpc_peer *rxrpc_lookup_peer_local_rcu(struct rxrpc_local *local, case AF_INET6: switch (serr->ee.ee_origin) { case SO_EE_ORIGIN_ICMP6: - _net("Rx ICMP6"); srx->transport.sin6.sin6_port = serr->port; memcpy(&srx->transport.sin6.sin6_addr, skb_network_header(skb) + serr->addr_offset, sizeof(struct in6_addr)); break; case SO_EE_ORIGIN_ICMP: - _net("Rx ICMP on v6 sock"); srx->transport_len = sizeof(srx->transport.sin); srx->transport.family = AF_INET; srx->transport.sin.sin_port = serr->port; @@ -106,13 +102,9 @@ static struct rxrpc_peer *rxrpc_lookup_peer_local_rcu(struct rxrpc_local *local, */ static void rxrpc_adjust_mtu(struct rxrpc_peer *peer, unsigned int mtu) { - _net("Rx ICMP Fragmentation Needed (%d)", mtu); - /* wind down the local interface MTU */ - if (mtu > 0 && peer->if_mtu == 65535 && mtu < peer->if_mtu) { + if (mtu > 0 && peer->if_mtu == 65535 && mtu < peer->if_mtu) peer->if_mtu = mtu; - _net("I/F MTU %u", mtu); - } if (mtu == 0) { /* they didn't give us a size, estimate one */ @@ -129,63 +121,36 @@ static void rxrpc_adjust_mtu(struct rxrpc_peer *peer, unsigned int mtu) } if (mtu < peer->mtu) { - spin_lock_bh(&peer->lock); + spin_lock(&peer->lock); peer->mtu = mtu; peer->maxdata = peer->mtu - peer->hdrsize; - spin_unlock_bh(&peer->lock); - _net("Net MTU %u (maxdata %u)", - peer->mtu, peer->maxdata); + spin_unlock(&peer->lock); } } /* * Handle an error received on the local endpoint. */ -void rxrpc_error_report(struct sock *sk) +void rxrpc_input_error(struct rxrpc_local *local, struct sk_buff *skb) { - struct sock_exterr_skb *serr; + struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); struct sockaddr_rxrpc srx; - struct rxrpc_local *local; struct rxrpc_peer *peer = NULL; - struct sk_buff *skb; - rcu_read_lock(); - local = rcu_dereference_sk_user_data(sk); - if (unlikely(!local)) { - rcu_read_unlock(); - return; - } - _enter("%p{%d}", sk, local->debug_id); - - /* Clear the outstanding error value on the socket so that it doesn't - * cause kernel_sendmsg() to return it later. - */ - sock_error(sk); + _enter("L=%x", local->debug_id); - skb = sock_dequeue_err_skb(sk); - if (!skb) { - rcu_read_unlock(); - _leave("UDP socket errqueue empty"); - return; - } - rxrpc_new_skb(skb, rxrpc_skb_received); - serr = SKB_EXT_ERR(skb); if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) { _leave("UDP empty message"); - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); return; } + rcu_read_lock(); peer = rxrpc_lookup_peer_local_rcu(local, skb, &srx); - if (peer && !rxrpc_get_peer_maybe(peer)) + if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_input_error)) peer = NULL; - if (!peer) { - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); - _leave(" [no peer]"); + rcu_read_unlock(); + if (!peer) return; - } trace_rxrpc_rx_icmp(peer, &serr->ee, &srx); @@ -196,72 +161,26 @@ void rxrpc_error_report(struct sock *sk) goto out; } - rxrpc_store_error(peer, serr); + rxrpc_store_error(peer, skb); out: - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); - rxrpc_put_peer(peer); - - _leave(""); + rxrpc_put_peer(peer, rxrpc_peer_put_input_error); } /* * Map an error report to error codes on the peer record. */ -static void rxrpc_store_error(struct rxrpc_peer *peer, - struct sock_exterr_skb *serr) +static void rxrpc_store_error(struct rxrpc_peer *peer, struct sk_buff *skb) { enum rxrpc_call_completion compl = RXRPC_CALL_NETWORK_ERROR; - struct sock_extended_err *ee; - int err; + struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); + struct sock_extended_err *ee = &serr->ee; + int err = ee->ee_errno; _enter(""); - ee = &serr->ee; - - err = ee->ee_errno; - switch (ee->ee_origin) { - case SO_EE_ORIGIN_ICMP: - switch (ee->ee_type) { - case ICMP_DEST_UNREACH: - switch (ee->ee_code) { - case ICMP_NET_UNREACH: - _net("Rx Received ICMP Network Unreachable"); - break; - case ICMP_HOST_UNREACH: - _net("Rx Received ICMP Host Unreachable"); - break; - case ICMP_PORT_UNREACH: - _net("Rx Received ICMP Port Unreachable"); - break; - case ICMP_NET_UNKNOWN: - _net("Rx Received ICMP Unknown Network"); - break; - case ICMP_HOST_UNKNOWN: - _net("Rx Received ICMP Unknown Host"); - break; - default: - _net("Rx Received ICMP DestUnreach code=%u", - ee->ee_code); - break; - } - break; - - case ICMP_TIME_EXCEEDED: - _net("Rx Received ICMP TTL Exceeded"); - break; - - default: - _proto("Rx Received ICMP error { type=%u code=%u }", - ee->ee_type, ee->ee_code); - break; - } - break; - case SO_EE_ORIGIN_NONE: case SO_EE_ORIGIN_LOCAL: - _proto("Rx Received local error { error=%d }", err); compl = RXRPC_CALL_LOCAL_ERROR; break; @@ -269,26 +188,40 @@ static void rxrpc_store_error(struct rxrpc_peer *peer, if (err == EACCES) err = EHOSTUNREACH; fallthrough; + case SO_EE_ORIGIN_ICMP: default: - _proto("Rx Received error report { orig=%u }", ee->ee_origin); break; } - rxrpc_distribute_error(peer, err, compl); + rxrpc_distribute_error(peer, skb, compl, err); } /* * Distribute an error that occurred on a peer. */ -static void rxrpc_distribute_error(struct rxrpc_peer *peer, int error, - enum rxrpc_call_completion compl) +static void rxrpc_distribute_error(struct rxrpc_peer *peer, struct sk_buff *skb, + enum rxrpc_call_completion compl, int err) { struct rxrpc_call *call; + HLIST_HEAD(error_targets); - hlist_for_each_entry_rcu(call, &peer->error_targets, error_link) { - rxrpc_see_call(call); - rxrpc_set_call_completion(call, compl, 0, -error); + spin_lock(&peer->lock); + hlist_move_list(&peer->error_targets, &error_targets); + + while (!hlist_empty(&error_targets)) { + call = hlist_entry(error_targets.first, + struct rxrpc_call, error_link); + hlist_del_init(&call->error_link); + spin_unlock(&peer->lock); + + rxrpc_see_call(call, rxrpc_call_see_distribute_error); + rxrpc_set_call_completion(call, compl, 0, -err); + rxrpc_input_call_event(call, skb); + + spin_lock(&peer->lock); } + + spin_unlock(&peer->lock); } /* @@ -304,18 +237,18 @@ static void rxrpc_peer_keepalive_dispatch(struct rxrpc_net *rxnet, time64_t keepalive_at; int slot; - spin_lock_bh(&rxnet->peer_hash_lock); + spin_lock(&rxnet->peer_hash_lock); while (!list_empty(collector)) { peer = list_entry(collector->next, struct rxrpc_peer, keepalive_link); list_del_init(&peer->keepalive_link); - if (!rxrpc_get_peer_maybe(peer)) + if (!rxrpc_get_peer_maybe(peer, rxrpc_peer_get_keepalive)) continue; - if (__rxrpc_use_local(peer->local)) { - spin_unlock_bh(&rxnet->peer_hash_lock); + if (__rxrpc_use_local(peer->local, rxrpc_local_use_peer_keepalive)) { + spin_unlock(&rxnet->peer_hash_lock); keepalive_at = peer->last_tx_at + RXRPC_KEEPALIVE_TIME; slot = keepalive_at - base; @@ -334,15 +267,15 @@ static void rxrpc_peer_keepalive_dispatch(struct rxrpc_net *rxnet, */ slot += cursor; slot &= mask; - spin_lock_bh(&rxnet->peer_hash_lock); + spin_lock(&rxnet->peer_hash_lock); list_add_tail(&peer->keepalive_link, &rxnet->peer_keepalive[slot & mask]); - rxrpc_unuse_local(peer->local); + rxrpc_unuse_local(peer->local, rxrpc_local_unuse_peer_keepalive); } - rxrpc_put_peer_locked(peer); + rxrpc_put_peer_locked(peer, rxrpc_peer_put_keepalive); } - spin_unlock_bh(&rxnet->peer_hash_lock); + spin_unlock(&rxnet->peer_hash_lock); } /* @@ -372,7 +305,7 @@ void rxrpc_peer_keepalive_worker(struct work_struct *work) * second; the bucket at cursor + 1 goes at now + 1s and so * on... */ - spin_lock_bh(&rxnet->peer_hash_lock); + spin_lock(&rxnet->peer_hash_lock); list_splice_init(&rxnet->peer_keepalive_new, &collector); stop = cursor + ARRAY_SIZE(rxnet->peer_keepalive); @@ -384,7 +317,7 @@ void rxrpc_peer_keepalive_worker(struct work_struct *work) } base = now; - spin_unlock_bh(&rxnet->peer_hash_lock); + spin_unlock(&rxnet->peer_hash_lock); rxnet->peer_keepalive_base = base; rxnet->peer_keepalive_cursor = cursor; diff --git a/net/rxrpc/peer_object.c b/net/rxrpc/peer_object.c index 041a51225c5f..608946dcc505 100644 --- a/net/rxrpc/peer_object.c +++ b/net/rxrpc/peer_object.c @@ -138,10 +138,8 @@ struct rxrpc_peer *rxrpc_lookup_peer_rcu(struct rxrpc_local *local, unsigned long hash_key = rxrpc_peer_hash_key(local, srx); peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key); - if (peer) { - _net("PEER %d {%pISp}", peer->debug_id, &peer->srx.transport); + if (peer) _leave(" = %p {u=%d}", peer, refcount_read(&peer->ref)); - } return peer; } @@ -207,9 +205,9 @@ static void rxrpc_assess_MTU_size(struct rxrpc_sock *rx, /* * Allocate a peer. */ -struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *local, gfp_t gfp) +struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *local, gfp_t gfp, + enum rxrpc_peer_trace why) { - const void *here = __builtin_return_address(0); struct rxrpc_peer *peer; _enter(""); @@ -217,7 +215,7 @@ struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *local, gfp_t gfp) peer = kzalloc(sizeof(struct rxrpc_peer), gfp); if (peer) { refcount_set(&peer->ref, 1); - peer->local = rxrpc_get_local(local); + peer->local = rxrpc_get_local(local, rxrpc_local_get_peer); INIT_HLIST_HEAD(&peer->error_targets); peer->service_conns = RB_ROOT; seqlock_init(&peer->service_conn_lock); @@ -228,7 +226,7 @@ struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *local, gfp_t gfp) rxrpc_peer_init_rtt(peer); peer->cong_ssthresh = RXRPC_TX_MAX_WINDOW; - trace_rxrpc_peer(peer->debug_id, rxrpc_peer_new, 1, here); + trace_rxrpc_peer(peer->debug_id, why, 1); } _leave(" = %p", peer); @@ -284,7 +282,7 @@ static struct rxrpc_peer *rxrpc_create_peer(struct rxrpc_sock *rx, _enter(""); - peer = rxrpc_alloc_peer(local, gfp); + peer = rxrpc_alloc_peer(local, gfp, rxrpc_peer_new_client); if (peer) { memcpy(&peer->srx, srx, sizeof(*srx)); rxrpc_init_peer(rx, peer, hash_key); @@ -296,7 +294,8 @@ static struct rxrpc_peer *rxrpc_create_peer(struct rxrpc_sock *rx, static void rxrpc_free_peer(struct rxrpc_peer *peer) { - rxrpc_put_local(peer->local); + trace_rxrpc_peer(peer->debug_id, 0, rxrpc_peer_free); + rxrpc_put_local(peer->local, rxrpc_local_put_peer); kfree_rcu(peer, rcu); } @@ -336,7 +335,7 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx, /* search the peer list first */ rcu_read_lock(); peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key); - if (peer && !rxrpc_get_peer_maybe(peer)) + if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_lookup_client)) peer = NULL; rcu_read_unlock(); @@ -350,11 +349,11 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx, return NULL; } - spin_lock_bh(&rxnet->peer_hash_lock); + spin_lock(&rxnet->peer_hash_lock); /* Need to check that we aren't racing with someone else */ peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key); - if (peer && !rxrpc_get_peer_maybe(peer)) + if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_lookup_client)) peer = NULL; if (!peer) { hash_add_rcu(rxnet->peer_hash, @@ -363,7 +362,7 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx, &rxnet->peer_keepalive_new); } - spin_unlock_bh(&rxnet->peer_hash_lock); + spin_unlock(&rxnet->peer_hash_lock); if (peer) rxrpc_free_peer(candidate); @@ -371,8 +370,6 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx, peer = candidate; } - _net("PEER %d {%pISp}", peer->debug_id, &peer->srx.transport); - _leave(" = %p {u=%d}", peer, refcount_read(&peer->ref)); return peer; } @@ -380,27 +377,26 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx, /* * Get a ref on a peer record. */ -struct rxrpc_peer *rxrpc_get_peer(struct rxrpc_peer *peer) +struct rxrpc_peer *rxrpc_get_peer(struct rxrpc_peer *peer, enum rxrpc_peer_trace why) { - const void *here = __builtin_return_address(0); int r; __refcount_inc(&peer->ref, &r); - trace_rxrpc_peer(peer->debug_id, rxrpc_peer_got, r + 1, here); + trace_rxrpc_peer(peer->debug_id, why, r + 1); return peer; } /* * Get a ref on a peer record unless its usage has already reached 0. */ -struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *peer) +struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *peer, + enum rxrpc_peer_trace why) { - const void *here = __builtin_return_address(0); int r; if (peer) { if (__refcount_inc_not_zero(&peer->ref, &r)) - trace_rxrpc_peer(peer->debug_id, rxrpc_peer_got, r + 1, here); + trace_rxrpc_peer(peer->debug_id, r + 1, why); else peer = NULL; } @@ -416,10 +412,10 @@ static void __rxrpc_put_peer(struct rxrpc_peer *peer) ASSERT(hlist_empty(&peer->error_targets)); - spin_lock_bh(&rxnet->peer_hash_lock); + spin_lock(&rxnet->peer_hash_lock); hash_del_rcu(&peer->hash_link); list_del_init(&peer->keepalive_link); - spin_unlock_bh(&rxnet->peer_hash_lock); + spin_unlock(&rxnet->peer_hash_lock); rxrpc_free_peer(peer); } @@ -427,9 +423,8 @@ static void __rxrpc_put_peer(struct rxrpc_peer *peer) /* * Drop a ref on a peer record. */ -void rxrpc_put_peer(struct rxrpc_peer *peer) +void rxrpc_put_peer(struct rxrpc_peer *peer, enum rxrpc_peer_trace why) { - const void *here = __builtin_return_address(0); unsigned int debug_id; bool dead; int r; @@ -437,7 +432,7 @@ void rxrpc_put_peer(struct rxrpc_peer *peer) if (peer) { debug_id = peer->debug_id; dead = __refcount_dec_and_test(&peer->ref, &r); - trace_rxrpc_peer(debug_id, rxrpc_peer_put, r - 1, here); + trace_rxrpc_peer(debug_id, r - 1, why); if (dead) __rxrpc_put_peer(peer); } @@ -447,15 +442,14 @@ void rxrpc_put_peer(struct rxrpc_peer *peer) * Drop a ref on a peer record where the caller already holds the * peer_hash_lock. */ -void rxrpc_put_peer_locked(struct rxrpc_peer *peer) +void rxrpc_put_peer_locked(struct rxrpc_peer *peer, enum rxrpc_peer_trace why) { - const void *here = __builtin_return_address(0); unsigned int debug_id = peer->debug_id; bool dead; int r; dead = __refcount_dec_and_test(&peer->ref, &r); - trace_rxrpc_peer(debug_id, rxrpc_peer_put, r - 1, here); + trace_rxrpc_peer(debug_id, r - 1, why); if (dead) { hash_del_rcu(&peer->hash_link); list_del_init(&peer->keepalive_link); diff --git a/net/rxrpc/proc.c b/net/rxrpc/proc.c index fae22a8b38d6..3a59591ec061 100644 --- a/net/rxrpc/proc.c +++ b/net/rxrpc/proc.c @@ -49,8 +49,6 @@ static void rxrpc_call_seq_stop(struct seq_file *seq, void *v) static int rxrpc_call_seq_show(struct seq_file *seq, void *v) { struct rxrpc_local *local; - struct rxrpc_sock *rx; - struct rxrpc_peer *peer; struct rxrpc_call *call; struct rxrpc_net *rxnet = rxrpc_net(seq_file_net(seq)); unsigned long timeout = 0; @@ -63,28 +61,19 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) "Proto Local " " Remote " " SvID ConnID CallID End Use State Abort " - " DebugId TxSeq TW RxSeq RW RxSerial RxTimo\n"); + " DebugId TxSeq TW RxSeq RW RxSerial CW RxTimo\n"); return 0; } call = list_entry(v, struct rxrpc_call, link); - rx = rcu_dereference(call->socket); - if (rx) { - local = READ_ONCE(rx->local); - if (local) - sprintf(lbuff, "%pISpc", &local->srx.transport); - else - strcpy(lbuff, "no_local"); - } else { - strcpy(lbuff, "no_socket"); - } - - peer = call->peer; - if (peer) - sprintf(rbuff, "%pISpc", &peer->srx.transport); + local = call->local; + if (local) + sprintf(lbuff, "%pISpc", &local->srx.transport); else - strcpy(rbuff, "no_connection"); + strcpy(lbuff, "no_local"); + + sprintf(rbuff, "%pISpc", &call->dest_srx.transport); if (call->state != RXRPC_CALL_SERVER_PREALLOC) { timeout = READ_ONCE(call->expect_rx_by); @@ -95,10 +84,10 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) wtmp = atomic64_read_acquire(&call->ackr_window); seq_printf(seq, "UDP %-47.47s %-47.47s %4x %08x %08x %s %3u" - " %-8.8s %08x %08x %08x %02x %08x %02x %08x %06lx\n", + " %-8.8s %08x %08x %08x %02x %08x %02x %08x %02x %06lx\n", lbuff, rbuff, - call->service_id, + call->dest_srx.srx_service, call->cid, call->call_id, rxrpc_is_service_call(call) ? "Svc" : "Clt", @@ -109,6 +98,7 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v) acks_hard_ack, READ_ONCE(call->tx_top) - acks_hard_ack, lower_32_bits(wtmp), upper_32_bits(wtmp) - lower_32_bits(wtmp), call->rx_serial, + call->cong_cwnd, timeout); return 0; @@ -159,7 +149,7 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) seq_puts(seq, "Proto Local " " Remote " - " SvID ConnID End Use State Key " + " SvID ConnID End Ref Act State Key " " Serial ISerial CallId0 CallId1 CallId2 CallId3\n" ); return 0; @@ -172,12 +162,12 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v) goto print; } - sprintf(lbuff, "%pISpc", &conn->params.local->srx.transport); + sprintf(lbuff, "%pISpc", &conn->local->srx.transport); - sprintf(rbuff, "%pISpc", &conn->params.peer->srx.transport); + sprintf(rbuff, "%pISpc", &conn->peer->srx.transport); print: seq_printf(seq, - "UDP %-47.47s %-47.47s %4x %08x %s %3u" + "UDP %-47.47s %-47.47s %4x %08x %s %3u %3d" " %s %08x %08x %08x %08x %08x %08x %08x\n", lbuff, rbuff, @@ -185,8 +175,9 @@ print: conn->proto.cid, rxrpc_conn_is_service(conn) ? "Svc" : "Clt", refcount_read(&conn->ref), + atomic_read(&conn->active), rxrpc_conn_states[conn->state], - key_serial(conn->params.key), + key_serial(conn->key), atomic_read(&conn->serial), conn->hi_serial, conn->channels[0].call_id, @@ -341,7 +332,7 @@ static int rxrpc_local_seq_show(struct seq_file *seq, void *v) if (v == SEQ_START_TOKEN) { seq_puts(seq, "Proto Local " - " Use Act\n"); + " Use Act RxQ\n"); return 0; } @@ -350,10 +341,11 @@ static int rxrpc_local_seq_show(struct seq_file *seq, void *v) sprintf(lbuff, "%pISpc", &local->srx.transport); seq_printf(seq, - "UDP %-47.47s %3u %3u\n", + "UDP %-47.47s %3u %3u %3u\n", lbuff, refcount_read(&local->ref), - atomic_read(&local->active_users)); + atomic_read(&local->active_users), + local->rx_queue.qlen); return 0; } @@ -407,13 +399,16 @@ int rxrpc_stats_show(struct seq_file *seq, void *v) struct rxrpc_net *rxnet = rxrpc_net(seq_file_single_net(seq)); seq_printf(seq, - "Data : send=%u sendf=%u\n", + "Data : send=%u sendf=%u fail=%u\n", atomic_read(&rxnet->stat_tx_data_send), - atomic_read(&rxnet->stat_tx_data_send_frag)); + atomic_read(&rxnet->stat_tx_data_send_frag), + atomic_read(&rxnet->stat_tx_data_send_fail)); seq_printf(seq, - "Data-Tx : nr=%u retrans=%u\n", + "Data-Tx : nr=%u retrans=%u uf=%u cwr=%u\n", atomic_read(&rxnet->stat_tx_data), - atomic_read(&rxnet->stat_tx_data_retrans)); + atomic_read(&rxnet->stat_tx_data_retrans), + atomic_read(&rxnet->stat_tx_data_underflow), + atomic_read(&rxnet->stat_tx_data_cwnd_reset)); seq_printf(seq, "Data-Rx : nr=%u reqack=%u jumbo=%u\n", atomic_read(&rxnet->stat_rx_data), @@ -462,6 +457,9 @@ int rxrpc_stats_show(struct seq_file *seq, void *v) "Buffers : txb=%u rxb=%u\n", atomic_read(&rxrpc_nr_txbuf), atomic_read(&rxrpc_n_rx_skbs)); + seq_printf(seq, + "IO-thread: loops=%u\n", + atomic_read(&rxnet->stat_io_loop)); return 0; } @@ -478,8 +476,11 @@ int rxrpc_stats_clear(struct file *file, char *buf, size_t size) atomic_set(&rxnet->stat_tx_data, 0); atomic_set(&rxnet->stat_tx_data_retrans, 0); + atomic_set(&rxnet->stat_tx_data_underflow, 0); + atomic_set(&rxnet->stat_tx_data_cwnd_reset, 0); atomic_set(&rxnet->stat_tx_data_send, 0); atomic_set(&rxnet->stat_tx_data_send_frag, 0); + atomic_set(&rxnet->stat_tx_data_send_fail, 0); atomic_set(&rxnet->stat_rx_data, 0); atomic_set(&rxnet->stat_rx_data_reqack, 0); atomic_set(&rxnet->stat_rx_data_jumbo, 0); @@ -491,5 +492,7 @@ int rxrpc_stats_clear(struct file *file, char *buf, size_t size) memset(&rxnet->stat_rx_acks, 0, sizeof(rxnet->stat_rx_acks)); memset(&rxnet->stat_why_req_ack, 0, sizeof(rxnet->stat_why_req_ack)); + + atomic_set(&rxnet->stat_io_loop, 0); return size; } diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index efb85f983657..36b25d003cf0 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -36,16 +36,16 @@ void rxrpc_notify_socket(struct rxrpc_call *call) sk = &rx->sk; if (rx && sk->sk_state < RXRPC_CLOSE) { if (call->notify_rx) { - spin_lock_bh(&call->notify_lock); + spin_lock(&call->notify_lock); call->notify_rx(sk, call, call->user_call_ID); - spin_unlock_bh(&call->notify_lock); + spin_unlock(&call->notify_lock); } else { - write_lock_bh(&rx->recvmsg_lock); + write_lock(&rx->recvmsg_lock); if (list_empty(&call->recvmsg_link)) { - rxrpc_get_call(call, rxrpc_call_got); + rxrpc_get_call(call, rxrpc_call_get_notify_socket); list_add_tail(&call->recvmsg_link, &rx->recvmsg_q); } - write_unlock_bh(&rx->recvmsg_lock); + write_unlock(&rx->recvmsg_lock); if (!sock_flag(sk, SOCK_DEAD)) { _debug("call %ps", sk->sk_data_ready); @@ -87,9 +87,9 @@ bool rxrpc_set_call_completion(struct rxrpc_call *call, bool ret = false; if (call->state < RXRPC_CALL_COMPLETE) { - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); ret = __rxrpc_set_call_completion(call, compl, abort_code, error); - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); } return ret; } @@ -107,9 +107,9 @@ bool rxrpc_call_completed(struct rxrpc_call *call) bool ret = false; if (call->state < RXRPC_CALL_COMPLETE) { - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); ret = __rxrpc_call_completed(call); - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); } return ret; } @@ -131,9 +131,9 @@ bool rxrpc_abort_call(const char *why, struct rxrpc_call *call, { bool ret; - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); ret = __rxrpc_abort_call(why, call, seq, abort_code, error); - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); return ret; } @@ -193,23 +193,23 @@ static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial) if (call->state == RXRPC_CALL_CLIENT_RECV_REPLY) rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_terminal_ack); - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); switch (call->state) { case RXRPC_CALL_CLIENT_RECV_REPLY: __rxrpc_call_completed(call); - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); break; case RXRPC_CALL_SERVER_RECV_REQUEST: call->state = RXRPC_CALL_SERVER_ACK_REQUEST; call->expect_req_by = jiffies + MAX_JIFFY_OFFSET; - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_processing_op); break; default: - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); break; } } @@ -228,9 +228,8 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call) _enter("%d", call->debug_id); -further_rotation: skb = skb_dequeue(&call->recvmsg_queue); - rxrpc_see_skb(skb, rxrpc_skb_rotated); + rxrpc_see_skb(skb, rxrpc_skb_see_rotate); sp = rxrpc_skb(skb); tseq = sp->hdr.seq; @@ -241,7 +240,7 @@ further_rotation: if (after(tseq, call->rx_consumed)) smp_store_release(&call->rx_consumed, tseq); - rxrpc_free_skb(skb, rxrpc_skb_freed); + rxrpc_free_skb(skb, rxrpc_skb_put_rotate); trace_rxrpc_receive(call, last ? rxrpc_receive_rotate_last : rxrpc_receive_rotate, serial, call->rx_consumed); @@ -250,26 +249,12 @@ further_rotation: return; } - /* The next packet on the queue might entirely overlap with the one we - * just consumed; if so, rotate that away also. - */ - skb = skb_peek(&call->recvmsg_queue); - if (skb) { - sp = rxrpc_skb(skb); - if (sp->hdr.seq != call->rx_consumed && - after_eq(call->rx_consumed, sp->hdr.seq)) - goto further_rotation; - } - /* Check to see if there's an ACK that needs sending. */ acked = atomic_add_return(call->rx_consumed - old_consumed, &call->ackr_nr_consumed); if (acked > 2 && - !test_and_set_bit(RXRPC_CALL_IDLE_ACK_PENDING, &call->flags)) { - rxrpc_send_ACK(call, RXRPC_ACK_IDLE, serial, - rxrpc_propose_ack_rotate_rx); - rxrpc_transmit_ack_packets(call->peer->local); - } + !test_and_set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags)) + rxrpc_poke_call(call, rxrpc_call_poke_idle); } /* @@ -314,15 +299,10 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call, */ skb = skb_peek(&call->recvmsg_queue); while (skb) { - rxrpc_see_skb(skb, rxrpc_skb_seen); + rxrpc_see_skb(skb, rxrpc_skb_see_recvmsg); sp = rxrpc_skb(skb); seq = sp->hdr.seq; - if (after_eq(call->rx_consumed, seq)) { - kdebug("obsolete %x %x", call->rx_consumed, seq); - goto skip_obsolete; - } - if (!(flags & MSG_PEEK)) trace_rxrpc_receive(call, rxrpc_receive_front, sp->hdr.serial, seq); @@ -340,7 +320,6 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call, ret = ret2; goto out; } - rxrpc_transmit_ack_packets(call->peer->local); } else { trace_rxrpc_recvdata(call, rxrpc_recvmsg_cont, seq, rx_pkt_offset, rx_pkt_len, 0); @@ -373,7 +352,6 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call, break; } - skip_obsolete: /* The whole packet has been transferred. */ if (sp->hdr.flags & RXRPC_LAST_PACKET) ret = 1; @@ -395,7 +373,7 @@ done: trace_rxrpc_recvdata(call, rxrpc_recvmsg_data_return, seq, rx_pkt_offset, rx_pkt_len, ret); if (ret == -EAGAIN) - set_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags); + set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags); return ret; } @@ -463,14 +441,14 @@ try_again: /* Find the next call and dequeue it if we're not just peeking. If we * do dequeue it, that comes with a ref that we will need to release. */ - write_lock_bh(&rx->recvmsg_lock); + write_lock(&rx->recvmsg_lock); l = rx->recvmsg_q.next; call = list_entry(l, struct rxrpc_call, recvmsg_link); if (!(flags & MSG_PEEK)) list_del_init(&call->recvmsg_link); else - rxrpc_get_call(call, rxrpc_call_got); - write_unlock_bh(&rx->recvmsg_lock); + rxrpc_get_call(call, rxrpc_call_get_recvmsg); + write_unlock(&rx->recvmsg_lock); trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0); @@ -508,11 +486,9 @@ try_again: } if (msg->msg_name && call->peer) { - struct sockaddr_rxrpc *srx = msg->msg_name; - size_t len = sizeof(call->peer->srx); + size_t len = sizeof(call->dest_srx); - memcpy(msg->msg_name, &call->peer->srx, len); - srx->srx_service = call->service_id; + memcpy(msg->msg_name, &call->dest_srx, len); msg->msg_namelen = len; } @@ -525,7 +501,6 @@ try_again: if (ret == -EAGAIN) ret = 0; - rxrpc_transmit_ack_packets(call->peer->local); if (!skb_queue_empty(&call->recvmsg_queue)) rxrpc_notify_socket(call); break; @@ -555,18 +530,18 @@ try_again: error_unlock_call: mutex_unlock(&call->user_mutex); - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_recvmsg); trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, ret); return ret; error_requeue_call: if (!(flags & MSG_PEEK)) { - write_lock_bh(&rx->recvmsg_lock); + write_lock(&rx->recvmsg_lock); list_add(&call->recvmsg_link, &rx->recvmsg_q); - write_unlock_bh(&rx->recvmsg_lock); + write_unlock(&rx->recvmsg_lock); trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0); } else { - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_recvmsg); } error_no_call: release_sock(&rx->sk); @@ -655,9 +630,8 @@ int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call, read_phase_complete: ret = 1; out: - rxrpc_transmit_ack_packets(call->peer->local); if (_service) - *_service = call->service_id; + *_service = call->dest_srx.srx_service; mutex_unlock(&call->user_mutex); _leave(" = %d [%zu,%d]", ret, iov_iter_count(iter), *_abort); return ret; diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 110a5550c0a6..d1233720e05f 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -103,7 +103,7 @@ static int rxkad_init_connection_security(struct rxrpc_connection *conn, struct crypto_sync_skcipher *ci; int ret; - _enter("{%d},{%x}", conn->debug_id, key_serial(conn->params.key)); + _enter("{%d},{%x}", conn->debug_id, key_serial(conn->key)); conn->security_ix = token->security_index; @@ -118,7 +118,7 @@ static int rxkad_init_connection_security(struct rxrpc_connection *conn, sizeof(token->kad->session_key)) < 0) BUG(); - switch (conn->params.security_level) { + switch (conn->security_level) { case RXRPC_SECURITY_PLAIN: case RXRPC_SECURITY_AUTH: case RXRPC_SECURITY_ENCRYPT: @@ -150,7 +150,7 @@ static int rxkad_how_much_data(struct rxrpc_call *call, size_t remain, { size_t shdr, buf_size, chunk; - switch (call->conn->params.security_level) { + switch (call->conn->security_level) { default: buf_size = chunk = min_t(size_t, remain, RXRPC_JUMBO_DATALEN); shdr = 0; @@ -192,7 +192,7 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, _enter(""); - if (!conn->params.key) + if (!conn->key) return 0; tmpbuf = kmalloc(tmpsize, GFP_KERNEL); @@ -205,7 +205,7 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn, return -ENOMEM; } - token = conn->params.key->payload.data[0]; + token = conn->key->payload.data[0]; memcpy(&iv, token->kad->session_key, sizeof(iv)); tmpbuf[0] = htonl(conn->proto.epoch); @@ -317,7 +317,7 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call, } /* encrypt from the session key */ - token = call->conn->params.key->payload.data[0]; + token = call->conn->key->payload.data[0]; memcpy(&iv, token->kad->session_key, sizeof(iv)); sg_init_one(&sg, txb->data, txb->len); @@ -344,13 +344,13 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) int ret; _enter("{%d{%x}},{#%u},%u,", - call->debug_id, key_serial(call->conn->params.key), + call->debug_id, key_serial(call->conn->key), txb->seq, txb->len); if (!call->conn->rxkad.cipher) return 0; - ret = key_validate(call->conn->params.key); + ret = key_validate(call->conn->key); if (ret < 0) return ret; @@ -380,7 +380,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb) y = 1; /* zero checksums are not permitted */ txb->wire.cksum = htons(y); - switch (call->conn->params.security_level) { + switch (call->conn->security_level) { case RXRPC_SECURITY_PLAIN: ret = 0; break; @@ -525,7 +525,7 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, } /* decrypt from the session key */ - token = call->conn->params.key->payload.data[0]; + token = call->conn->key->payload.data[0]; memcpy(&iv, token->kad->session_key, sizeof(iv)); skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher); @@ -596,7 +596,7 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb) u32 x, y; _enter("{%d{%x}},{#%u}", - call->debug_id, key_serial(call->conn->params.key), seq); + call->debug_id, key_serial(call->conn->key), seq); if (!call->conn->rxkad.cipher) return 0; @@ -632,7 +632,7 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb) goto protocol_error; } - switch (call->conn->params.security_level) { + switch (call->conn->security_level) { case RXRPC_SECURITY_PLAIN: ret = 0; break; @@ -678,8 +678,8 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn) challenge.min_level = htonl(0); challenge.__padding = 0; - msg.msg_name = &conn->params.peer->srx.transport; - msg.msg_namelen = conn->params.peer->srx.transport_len; + msg.msg_name = &conn->peer->srx.transport; + msg.msg_namelen = conn->peer->srx.transport_len; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = 0; @@ -704,16 +704,15 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn) serial = atomic_inc_return(&conn->serial); whdr.serial = htonl(serial); - _proto("Tx CHALLENGE %%%u", serial); - ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len); + ret = kernel_sendmsg(conn->local->socket, &msg, iov, 2, len); if (ret < 0) { trace_rxrpc_tx_fail(conn->debug_id, serial, ret, rxrpc_tx_point_rxkad_challenge); return -EAGAIN; } - conn->params.peer->last_tx_at = ktime_get_seconds(); + conn->peer->last_tx_at = ktime_get_seconds(); trace_rxrpc_tx_packet(conn->debug_id, &whdr, rxrpc_tx_point_rxkad_challenge); _leave(" = 0"); @@ -737,8 +736,8 @@ static int rxkad_send_response(struct rxrpc_connection *conn, _enter(""); - msg.msg_name = &conn->params.peer->srx.transport; - msg.msg_namelen = conn->params.peer->srx.transport_len; + msg.msg_name = &conn->peer->srx.transport; + msg.msg_namelen = conn->peer->srx.transport_len; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = 0; @@ -762,16 +761,15 @@ static int rxkad_send_response(struct rxrpc_connection *conn, serial = atomic_inc_return(&conn->serial); whdr.serial = htonl(serial); - _proto("Tx RESPONSE %%%u", serial); - ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 3, len); + ret = kernel_sendmsg(conn->local->socket, &msg, iov, 3, len); if (ret < 0) { trace_rxrpc_tx_fail(conn->debug_id, serial, ret, rxrpc_tx_point_rxkad_response); return -EAGAIN; } - conn->params.peer->last_tx_at = ktime_get_seconds(); + conn->peer->last_tx_at = ktime_get_seconds(); _leave(" = 0"); return 0; } @@ -834,15 +832,15 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, u32 version, nonce, min_level, abort_code; int ret; - _enter("{%d,%x}", conn->debug_id, key_serial(conn->params.key)); + _enter("{%d,%x}", conn->debug_id, key_serial(conn->key)); eproto = tracepoint_string("chall_no_key"); abort_code = RX_PROTOCOL_ERROR; - if (!conn->params.key) + if (!conn->key) goto protocol_error; abort_code = RXKADEXPIRED; - ret = key_validate(conn->params.key); + ret = key_validate(conn->key); if (ret < 0) goto other_error; @@ -856,8 +854,7 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, nonce = ntohl(challenge.nonce); min_level = ntohl(challenge.min_level); - _proto("Rx CHALLENGE %%%u { v=%u n=%u ml=%u }", - sp->hdr.serial, version, nonce, min_level); + trace_rxrpc_rx_challenge(conn, sp->hdr.serial, version, nonce, min_level); eproto = tracepoint_string("chall_ver"); abort_code = RXKADINCONSISTENCY; @@ -866,10 +863,10 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, abort_code = RXKADLEVELFAIL; ret = -EACCES; - if (conn->params.security_level < min_level) + if (conn->security_level < min_level) goto other_error; - token = conn->params.key->payload.data[0]; + token = conn->key->payload.data[0]; /* build the response packet */ resp = kzalloc(sizeof(struct rxkad_response), GFP_NOFS); @@ -881,7 +878,7 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn, resp->encrypted.cid = htonl(conn->proto.cid); resp->encrypted.securityIndex = htonl(conn->security_ix); resp->encrypted.inc_nonce = htonl(nonce + 1); - resp->encrypted.level = htonl(conn->params.security_level); + resp->encrypted.level = htonl(conn->security_level); resp->kvno = htonl(token->kad->kvno); resp->ticket_len = htonl(token->kad->ticket_len); resp->encrypted.call_id[0] = htonl(conn->channels[0].call_counter); @@ -1139,8 +1136,8 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, version = ntohl(response->version); ticket_len = ntohl(response->ticket_len); kvno = ntohl(response->kvno); - _proto("Rx RESPONSE %%%u { v=%u kv=%u tl=%u }", - sp->hdr.serial, version, kvno, ticket_len); + + trace_rxrpc_rx_response(conn, sp->hdr.serial, version, kvno, ticket_len); eproto = tracepoint_string("rxkad_rsp_ver"); abort_code = RXKADINCONSISTENCY; @@ -1229,7 +1226,7 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, level = ntohl(response->encrypted.level); if (level > RXRPC_SECURITY_ENCRYPT) goto protocol_error_free; - conn->params.security_level = level; + conn->security_level = level; /* create a key to hold the security data and expiration time - after * this the connection security can be handled in exactly the same way diff --git a/net/rxrpc/rxperf.c b/net/rxrpc/rxperf.c new file mode 100644 index 000000000000..66f5eea291ff --- /dev/null +++ b/net/rxrpc/rxperf.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* In-kernel rxperf server for testing purposes. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) "rxperf: " fmt +#include <linux/module.h> +#include <linux/slab.h> +#include <net/sock.h> +#include <net/af_rxrpc.h> + +MODULE_DESCRIPTION("rxperf test server (afs)"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +#define RXPERF_PORT 7009 +#define RX_PERF_SERVICE 147 +#define RX_PERF_VERSION 3 +#define RX_PERF_SEND 0 +#define RX_PERF_RECV 1 +#define RX_PERF_RPC 3 +#define RX_PERF_FILE 4 +#define RX_PERF_MAGIC_COOKIE 0x4711 + +struct rxperf_proto_params { + __be32 version; + __be32 type; + __be32 rsize; + __be32 wsize; +} __packed; + +static const u8 rxperf_magic_cookie[] = { 0x00, 0x00, 0x47, 0x11 }; +static const u8 secret[8] = { 0xa7, 0x83, 0x8a, 0xcb, 0xc7, 0x83, 0xec, 0x94 }; + +enum rxperf_call_state { + RXPERF_CALL_SV_AWAIT_PARAMS, /* Server: Awaiting parameter block */ + RXPERF_CALL_SV_AWAIT_REQUEST, /* Server: Awaiting request data */ + RXPERF_CALL_SV_REPLYING, /* Server: Replying */ + RXPERF_CALL_SV_AWAIT_ACK, /* Server: Awaiting final ACK */ + RXPERF_CALL_COMPLETE, /* Completed or failed */ +}; + +struct rxperf_call { + struct rxrpc_call *rxcall; + struct iov_iter iter; + struct kvec kvec[1]; + struct work_struct work; + const char *type; + size_t iov_len; + size_t req_len; /* Size of request blob */ + size_t reply_len; /* Size of reply blob */ + unsigned int debug_id; + unsigned int operation_id; + struct rxperf_proto_params params; + __be32 tmp[2]; + s32 abort_code; + enum rxperf_call_state state; + short error; + unsigned short unmarshal; + u16 service_id; + int (*deliver)(struct rxperf_call *call); + void (*processor)(struct work_struct *work); +}; + +static struct socket *rxperf_socket; +static struct key *rxperf_sec_keyring; /* Ring of security/crypto keys */ +static struct workqueue_struct *rxperf_workqueue; + +static void rxperf_deliver_to_call(struct work_struct *work); +static int rxperf_deliver_param_block(struct rxperf_call *call); +static int rxperf_deliver_request(struct rxperf_call *call); +static int rxperf_process_call(struct rxperf_call *call); +static void rxperf_charge_preallocation(struct work_struct *work); + +static DECLARE_WORK(rxperf_charge_preallocation_work, + rxperf_charge_preallocation); + +static inline void rxperf_set_call_state(struct rxperf_call *call, + enum rxperf_call_state to) +{ + call->state = to; +} + +static inline void rxperf_set_call_complete(struct rxperf_call *call, + int error, s32 remote_abort) +{ + if (call->state != RXPERF_CALL_COMPLETE) { + call->abort_code = remote_abort; + call->error = error; + call->state = RXPERF_CALL_COMPLETE; + } +} + +static void rxperf_rx_discard_new_call(struct rxrpc_call *rxcall, + unsigned long user_call_ID) +{ + kfree((struct rxperf_call *)user_call_ID); +} + +static void rxperf_rx_new_call(struct sock *sk, struct rxrpc_call *rxcall, + unsigned long user_call_ID) +{ + queue_work(rxperf_workqueue, &rxperf_charge_preallocation_work); +} + +static void rxperf_queue_call_work(struct rxperf_call *call) +{ + queue_work(rxperf_workqueue, &call->work); +} + +static void rxperf_notify_rx(struct sock *sk, struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + struct rxperf_call *call = (struct rxperf_call *)call_user_ID; + + if (call->state != RXPERF_CALL_COMPLETE) + rxperf_queue_call_work(call); +} + +static void rxperf_rx_attach(struct rxrpc_call *rxcall, unsigned long user_call_ID) +{ + struct rxperf_call *call = (struct rxperf_call *)user_call_ID; + + call->rxcall = rxcall; +} + +static void rxperf_notify_end_reply_tx(struct sock *sock, + struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + rxperf_set_call_state((struct rxperf_call *)call_user_ID, + RXPERF_CALL_SV_AWAIT_ACK); +} + +/* + * Charge the incoming call preallocation. + */ +static void rxperf_charge_preallocation(struct work_struct *work) +{ + struct rxperf_call *call; + + for (;;) { + call = kzalloc(sizeof(*call), GFP_KERNEL); + if (!call) + break; + + call->type = "unset"; + call->debug_id = atomic_inc_return(&rxrpc_debug_id); + call->deliver = rxperf_deliver_param_block; + call->state = RXPERF_CALL_SV_AWAIT_PARAMS; + call->service_id = RX_PERF_SERVICE; + call->iov_len = sizeof(call->params); + call->kvec[0].iov_len = sizeof(call->params); + call->kvec[0].iov_base = &call->params; + iov_iter_kvec(&call->iter, READ, call->kvec, 1, call->iov_len); + INIT_WORK(&call->work, rxperf_deliver_to_call); + + if (rxrpc_kernel_charge_accept(rxperf_socket, + rxperf_notify_rx, + rxperf_rx_attach, + (unsigned long)call, + GFP_KERNEL, + call->debug_id) < 0) + break; + call = NULL; + } + + kfree(call); +} + +/* + * Open an rxrpc socket and bind it to be a server for callback notifications + * - the socket is left in blocking mode and non-blocking ops use MSG_DONTWAIT + */ +static int rxperf_open_socket(void) +{ + struct sockaddr_rxrpc srx; + struct socket *socket; + int ret; + + ret = sock_create_kern(&init_net, AF_RXRPC, SOCK_DGRAM, PF_INET6, + &socket); + if (ret < 0) + goto error_1; + + socket->sk->sk_allocation = GFP_NOFS; + + /* bind the callback manager's address to make this a server socket */ + memset(&srx, 0, sizeof(srx)); + srx.srx_family = AF_RXRPC; + srx.srx_service = RX_PERF_SERVICE; + srx.transport_type = SOCK_DGRAM; + srx.transport_len = sizeof(srx.transport.sin6); + srx.transport.sin6.sin6_family = AF_INET6; + srx.transport.sin6.sin6_port = htons(RXPERF_PORT); + + ret = rxrpc_sock_set_min_security_level(socket->sk, + RXRPC_SECURITY_ENCRYPT); + if (ret < 0) + goto error_2; + + ret = rxrpc_sock_set_security_keyring(socket->sk, rxperf_sec_keyring); + + ret = kernel_bind(socket, (struct sockaddr *)&srx, sizeof(srx)); + if (ret < 0) + goto error_2; + + rxrpc_kernel_new_call_notification(socket, rxperf_rx_new_call, + rxperf_rx_discard_new_call); + + ret = kernel_listen(socket, INT_MAX); + if (ret < 0) + goto error_2; + + rxperf_socket = socket; + rxperf_charge_preallocation(&rxperf_charge_preallocation_work); + return 0; + +error_2: + sock_release(socket); +error_1: + pr_err("Can't set up rxperf socket: %d\n", ret); + return ret; +} + +/* + * close the rxrpc socket rxperf was using + */ +static void rxperf_close_socket(void) +{ + kernel_listen(rxperf_socket, 0); + kernel_sock_shutdown(rxperf_socket, SHUT_RDWR); + flush_workqueue(rxperf_workqueue); + sock_release(rxperf_socket); +} + +/* + * Log remote abort codes that indicate that we have a protocol disagreement + * with the server. + */ +static void rxperf_log_error(struct rxperf_call *call, s32 remote_abort) +{ + static int max = 0; + const char *msg; + int m; + + switch (remote_abort) { + case RX_EOF: msg = "unexpected EOF"; break; + case RXGEN_CC_MARSHAL: msg = "client marshalling"; break; + case RXGEN_CC_UNMARSHAL: msg = "client unmarshalling"; break; + case RXGEN_SS_MARSHAL: msg = "server marshalling"; break; + case RXGEN_SS_UNMARSHAL: msg = "server unmarshalling"; break; + case RXGEN_DECODE: msg = "opcode decode"; break; + case RXGEN_SS_XDRFREE: msg = "server XDR cleanup"; break; + case RXGEN_CC_XDRFREE: msg = "client XDR cleanup"; break; + case -32: msg = "insufficient data"; break; + default: + return; + } + + m = max; + if (m < 3) { + max = m + 1; + pr_info("Peer reported %s failure on %s\n", msg, call->type); + } +} + +/* + * deliver messages to a call + */ +static void rxperf_deliver_to_call(struct work_struct *work) +{ + struct rxperf_call *call = container_of(work, struct rxperf_call, work); + enum rxperf_call_state state; + u32 abort_code, remote_abort = 0; + int ret; + + if (call->state == RXPERF_CALL_COMPLETE) + return; + + while (state = call->state, + state == RXPERF_CALL_SV_AWAIT_PARAMS || + state == RXPERF_CALL_SV_AWAIT_REQUEST || + state == RXPERF_CALL_SV_AWAIT_ACK + ) { + if (state == RXPERF_CALL_SV_AWAIT_ACK) { + if (!rxrpc_kernel_check_life(rxperf_socket, call->rxcall)) + goto call_complete; + return; + } + + ret = call->deliver(call); + if (ret == 0) + ret = rxperf_process_call(call); + + switch (ret) { + case 0: + continue; + case -EINPROGRESS: + case -EAGAIN: + return; + case -ECONNABORTED: + rxperf_log_error(call, call->abort_code); + goto call_complete; + case -EOPNOTSUPP: + abort_code = RXGEN_OPCODE; + rxrpc_kernel_abort_call(rxperf_socket, call->rxcall, + abort_code, ret, "GOP"); + goto call_complete; + case -ENOTSUPP: + abort_code = RX_USER_ABORT; + rxrpc_kernel_abort_call(rxperf_socket, call->rxcall, + abort_code, ret, "GUA"); + goto call_complete; + case -EIO: + pr_err("Call %u in bad state %u\n", + call->debug_id, call->state); + fallthrough; + case -ENODATA: + case -EBADMSG: + case -EMSGSIZE: + case -ENOMEM: + case -EFAULT: + rxrpc_kernel_abort_call(rxperf_socket, call->rxcall, + RXGEN_SS_UNMARSHAL, ret, "GUM"); + goto call_complete; + default: + rxrpc_kernel_abort_call(rxperf_socket, call->rxcall, + RX_CALL_DEAD, ret, "GER"); + goto call_complete; + } + } + +call_complete: + rxperf_set_call_complete(call, ret, remote_abort); + /* The call may have been requeued */ + rxrpc_kernel_end_call(rxperf_socket, call->rxcall); + cancel_work(&call->work); + kfree(call); +} + +/* + * Extract a piece of data from the received data socket buffers. + */ +static int rxperf_extract_data(struct rxperf_call *call, bool want_more) +{ + u32 remote_abort = 0; + int ret; + + ret = rxrpc_kernel_recv_data(rxperf_socket, call->rxcall, &call->iter, + &call->iov_len, want_more, &remote_abort, + &call->service_id); + pr_debug("Extract i=%zu l=%zu m=%u ret=%d\n", + iov_iter_count(&call->iter), call->iov_len, want_more, ret); + if (ret == 0 || ret == -EAGAIN) + return ret; + + if (ret == 1) { + switch (call->state) { + case RXPERF_CALL_SV_AWAIT_REQUEST: + rxperf_set_call_state(call, RXPERF_CALL_SV_REPLYING); + break; + case RXPERF_CALL_COMPLETE: + pr_debug("premature completion %d", call->error); + return call->error; + default: + break; + } + return 0; + } + + rxperf_set_call_complete(call, ret, remote_abort); + return ret; +} + +/* + * Grab the operation ID from an incoming manager call. + */ +static int rxperf_deliver_param_block(struct rxperf_call *call) +{ + u32 version; + int ret; + + /* Extract the parameter block */ + ret = rxperf_extract_data(call, true); + if (ret < 0) + return ret; + + version = ntohl(call->params.version); + call->operation_id = ntohl(call->params.type); + call->deliver = rxperf_deliver_request; + + if (version != RX_PERF_VERSION) { + pr_info("Version mismatch %x\n", version); + return -ENOTSUPP; + } + + switch (call->operation_id) { + case RX_PERF_SEND: + call->type = "send"; + call->reply_len = 0; + call->iov_len = 4; /* Expect req size */ + break; + case RX_PERF_RECV: + call->type = "recv"; + call->req_len = 0; + call->iov_len = 4; /* Expect reply size */ + break; + case RX_PERF_RPC: + call->type = "rpc"; + call->iov_len = 8; /* Expect req size and reply size */ + break; + case RX_PERF_FILE: + call->type = "file"; + fallthrough; + default: + return -EOPNOTSUPP; + } + + rxperf_set_call_state(call, RXPERF_CALL_SV_AWAIT_REQUEST); + return call->deliver(call); +} + +/* + * Deliver the request data. + */ +static int rxperf_deliver_request(struct rxperf_call *call) +{ + int ret; + + switch (call->unmarshal) { + case 0: + call->kvec[0].iov_len = call->iov_len; + call->kvec[0].iov_base = call->tmp; + iov_iter_kvec(&call->iter, READ, call->kvec, 1, call->iov_len); + call->unmarshal++; + fallthrough; + case 1: + ret = rxperf_extract_data(call, true); + if (ret < 0) + return ret; + + switch (call->operation_id) { + case RX_PERF_SEND: + call->type = "send"; + call->req_len = ntohl(call->tmp[0]); + call->reply_len = 0; + break; + case RX_PERF_RECV: + call->type = "recv"; + call->req_len = 0; + call->reply_len = ntohl(call->tmp[0]); + break; + case RX_PERF_RPC: + call->type = "rpc"; + call->req_len = ntohl(call->tmp[0]); + call->reply_len = ntohl(call->tmp[1]); + break; + default: + pr_info("Can't parse extra params\n"); + return -EIO; + } + + pr_debug("CALL op=%s rq=%zx rp=%zx\n", + call->type, call->req_len, call->reply_len); + + call->iov_len = call->req_len; + iov_iter_discard(&call->iter, READ, call->req_len); + call->unmarshal++; + fallthrough; + case 2: + ret = rxperf_extract_data(call, false); + if (ret < 0) + return ret; + call->unmarshal++; + fallthrough; + default: + return 0; + } +} + +/* + * Process a call for which we've received the request. + */ +static int rxperf_process_call(struct rxperf_call *call) +{ + struct msghdr msg = {}; + struct bio_vec bv[1]; + struct kvec iov[1]; + ssize_t n; + size_t reply_len = call->reply_len, len; + + rxrpc_kernel_set_tx_length(rxperf_socket, call->rxcall, + reply_len + sizeof(rxperf_magic_cookie)); + + while (reply_len > 0) { + len = min_t(size_t, reply_len, PAGE_SIZE); + bv[0].bv_page = ZERO_PAGE(0); + bv[0].bv_offset = 0; + bv[0].bv_len = len; + iov_iter_bvec(&msg.msg_iter, WRITE, bv, 1, len); + msg.msg_flags = MSG_MORE; + n = rxrpc_kernel_send_data(rxperf_socket, call->rxcall, &msg, + len, rxperf_notify_end_reply_tx); + if (n < 0) + return n; + if (n == 0) + return -EIO; + reply_len -= n; + } + + len = sizeof(rxperf_magic_cookie); + iov[0].iov_base = (void *)rxperf_magic_cookie; + iov[0].iov_len = len; + iov_iter_kvec(&msg.msg_iter, WRITE, iov, 1, len); + msg.msg_flags = 0; + n = rxrpc_kernel_send_data(rxperf_socket, call->rxcall, &msg, len, + rxperf_notify_end_reply_tx); + if (n >= 0) + return 0; /* Success */ + + if (n == -ENOMEM) + rxrpc_kernel_abort_call(rxperf_socket, call->rxcall, + RXGEN_SS_MARSHAL, -ENOMEM, "GOM"); + return n; +} + +/* + * Add a key to the security keyring. + */ +static int rxperf_add_key(struct key *keyring) +{ + key_ref_t kref; + int ret; + + kref = key_create_or_update(make_key_ref(keyring, true), + "rxrpc_s", + __stringify(RX_PERF_SERVICE) ":2", + secret, + sizeof(secret), + KEY_POS_VIEW | KEY_POS_READ | KEY_POS_SEARCH + | KEY_USR_VIEW, + KEY_ALLOC_NOT_IN_QUOTA); + + if (IS_ERR(kref)) { + pr_err("Can't allocate rxperf server key: %ld\n", PTR_ERR(kref)); + return PTR_ERR(kref); + } + + ret = key_link(keyring, key_ref_to_ptr(kref)); + if (ret < 0) + pr_err("Can't link rxperf server key: %d\n", ret); + key_ref_put(kref); + return ret; +} + +/* + * Initialise the rxperf server. + */ +static int __init rxperf_init(void) +{ + struct key *keyring; + int ret = -ENOMEM; + + pr_info("Server registering\n"); + + rxperf_workqueue = alloc_workqueue("rxperf", 0, 0); + if (!rxperf_workqueue) + goto error_workqueue; + + keyring = keyring_alloc("rxperf_server", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), + KEY_POS_VIEW | KEY_POS_READ | KEY_POS_SEARCH | + KEY_POS_WRITE | + KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH | + KEY_USR_WRITE | + KEY_OTH_VIEW | KEY_OTH_READ | KEY_OTH_SEARCH, + KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("Can't allocate rxperf server keyring: %ld\n", + PTR_ERR(keyring)); + goto error_keyring; + } + rxperf_sec_keyring = keyring; + ret = rxperf_add_key(keyring); + if (ret < 0) + goto error_key; + + ret = rxperf_open_socket(); + if (ret < 0) + goto error_socket; + return 0; + +error_socket: +error_key: + key_put(rxperf_sec_keyring); +error_keyring: + destroy_workqueue(rxperf_workqueue); + rcu_barrier(); +error_workqueue: + pr_err("Failed to register: %d\n", ret); + return ret; +} +late_initcall(rxperf_init); /* Must be called after net/ to create socket */ + +static void __exit rxperf_exit(void) +{ + pr_info("Server unregistering.\n"); + + rxperf_close_socket(); + key_put(rxperf_sec_keyring); + destroy_workqueue(rxperf_workqueue); + rcu_barrier(); +} +module_exit(rxperf_exit); + diff --git a/net/rxrpc/security.c b/net/rxrpc/security.c index 50cb5f1ee0c0..209f2c25a0da 100644 --- a/net/rxrpc/security.c +++ b/net/rxrpc/security.c @@ -63,13 +63,43 @@ const struct rxrpc_security *rxrpc_security_lookup(u8 security_index) } /* + * Initialise the security on a client call. + */ +int rxrpc_init_client_call_security(struct rxrpc_call *call) +{ + const struct rxrpc_security *sec; + struct rxrpc_key_token *token; + struct key *key = call->key; + int ret; + + if (!key) + return 0; + + ret = key_validate(key); + if (ret < 0) + return ret; + + for (token = key->payload.data[0]; token; token = token->next) { + sec = rxrpc_security_lookup(token->security_index); + if (sec) + goto found; + } + return -EKEYREJECTED; + +found: + call->security = sec; + _leave(" = 0"); + return 0; +} + +/* * initialise the security on a client connection */ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn) { const struct rxrpc_security *sec; struct rxrpc_key_token *token; - struct key *key = conn->params.key; + struct key *key = conn->key; int ret; _enter("{%d},{%x}", conn->debug_id, key_serial(key)); @@ -163,7 +193,7 @@ struct key *rxrpc_look_up_server_security(struct rxrpc_connection *conn, rcu_read_lock(); - rx = rcu_dereference(conn->params.local->service); + rx = rcu_dereference(conn->local->service); if (!rx) goto out; diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index e5fd8a95bf71..9fa7e37f7155 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -22,30 +22,9 @@ */ static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win) { - unsigned int win_size; - rxrpc_seq_t tx_win = smp_load_acquire(&call->acks_hard_ack); - - /* If we haven't transmitted anything for >1RTT, we should reset the - * congestion management state. - */ - if (ktime_before(ktime_add_us(call->tx_last_sent, - call->peer->srtt_us >> 3), - ktime_get_real())) { - if (RXRPC_TX_SMSS > 2190) - win_size = 2; - else if (RXRPC_TX_SMSS > 1095) - win_size = 3; - else - win_size = 4; - win_size += call->cong_extra; - } else { - win_size = min_t(unsigned int, call->tx_winsize, - call->cong_cwnd + call->cong_extra); - } - if (_tx_win) - *_tx_win = tx_win; - return call->tx_top - tx_win < win_size; + *_tx_win = call->tx_bottom; + return call->tx_prepared - call->tx_bottom < 256; } /* @@ -66,11 +45,6 @@ static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx, if (signal_pending(current)) return sock_intr_errno(*timeo); - if (READ_ONCE(call->acks_hard_ack) != call->tx_bottom) { - rxrpc_shrink_call_tx_buffer(call); - continue; - } - trace_rxrpc_txqueue(call, rxrpc_txqueue_wait); *timeo = schedule_timeout(*timeo); } @@ -107,11 +81,6 @@ static int rxrpc_wait_for_tx_window_waitall(struct rxrpc_sock *rx, tx_win == tx_start && signal_pending(current)) return -EINTR; - if (READ_ONCE(call->acks_hard_ack) != call->tx_bottom) { - rxrpc_shrink_call_tx_buffer(call); - continue; - } - if (tx_win != tx_start) { timeout = rtt; tx_start = tx_win; @@ -137,11 +106,6 @@ static int rxrpc_wait_for_tx_window_nonintr(struct rxrpc_sock *rx, if (call->state >= RXRPC_CALL_COMPLETE) return call->error; - if (READ_ONCE(call->acks_hard_ack) != call->tx_bottom) { - rxrpc_shrink_call_tx_buffer(call); - continue; - } - trace_rxrpc_txqueue(call, rxrpc_txqueue_wait); *timeo = schedule_timeout(*timeo); } @@ -206,33 +170,32 @@ static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call, { unsigned long now; rxrpc_seq_t seq = txb->seq; - bool last = test_bit(RXRPC_TXBUF_LAST, &txb->flags); - int ret; + bool last = test_bit(RXRPC_TXBUF_LAST, &txb->flags), poke; rxrpc_inc_stat(call->rxnet, stat_tx_data); - ASSERTCMP(seq, ==, call->tx_top + 1); + ASSERTCMP(txb->seq, ==, call->tx_prepared + 1); /* We have to set the timestamp before queueing as the retransmit * algorithm can see the packet as soon as we queue it. */ txb->last_sent = ktime_get_real(); - /* Add the packet to the call's output buffer */ - rxrpc_get_txbuf(txb, rxrpc_txbuf_get_buffer); - spin_lock(&call->tx_lock); - list_add_tail(&txb->call_link, &call->tx_buffer); - call->tx_top = seq; - spin_unlock(&call->tx_lock); - if (last) trace_rxrpc_txqueue(call, rxrpc_txqueue_queue_last); else trace_rxrpc_txqueue(call, rxrpc_txqueue_queue); + /* Add the packet to the call's output buffer */ + spin_lock(&call->tx_lock); + poke = list_empty(&call->tx_sendmsg); + list_add_tail(&txb->call_link, &call->tx_sendmsg); + call->tx_prepared = seq; + spin_unlock(&call->tx_lock); + if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) { _debug("________awaiting reply/ACK__________"); - write_lock_bh(&call->state_lock); + write_lock(&call->state_lock); switch (call->state) { case RXRPC_CALL_CLIENT_SEND_REQUEST: call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY; @@ -255,33 +218,11 @@ static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call, default: break; } - write_unlock_bh(&call->state_lock); + write_unlock(&call->state_lock); } - if (seq == 1 && rxrpc_is_client_call(call)) - rxrpc_expose_client_call(call); - - ret = rxrpc_send_data_packet(call, txb); - if (ret < 0) { - switch (ret) { - case -ENETUNREACH: - case -EHOSTUNREACH: - case -ECONNREFUSED: - rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, - 0, ret); - goto out; - } - } else { - unsigned long now = jiffies; - unsigned long resend_at = now + call->peer->rto_j; - - WRITE_ONCE(call->resend_at, resend_at); - rxrpc_reduce_call_timer(call, resend_at, now, - rxrpc_timer_set_for_send); - } - -out: - rxrpc_put_txbuf(txb, rxrpc_txbuf_put_trans); + if (poke) + rxrpc_poke_call(call, rxrpc_call_poke_start); } /* @@ -335,8 +276,6 @@ reload: rxrpc_see_txbuf(txb, rxrpc_txbuf_see_send_more); do { - rxrpc_transmit_ack_packets(call->peer->local); - if (!txb) { size_t remain, bufsize, chunk, offset; @@ -416,10 +355,10 @@ reload: success: ret = copied; if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE) { - read_lock_bh(&call->state_lock); + read_lock(&call->state_lock); if (call->error < 0) ret = call->error; - read_unlock_bh(&call->state_lock); + read_unlock(&call->state_lock); } out: call->tx_pending = txb; @@ -604,7 +543,7 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, atomic_inc_return(&rxrpc_debug_id)); /* The socket is now unlocked */ - rxrpc_put_peer(cp.peer); + rxrpc_put_peer(cp.peer, rxrpc_peer_put_discard_tmp); _leave(" = %p\n", call); return call; } @@ -667,7 +606,7 @@ int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len) case RXRPC_CALL_CLIENT_AWAIT_CONN: case RXRPC_CALL_SERVER_PREALLOC: case RXRPC_CALL_SERVER_SECURING: - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_sendmsg); ret = -EBUSY; goto error_release_sock; default: @@ -737,7 +676,7 @@ out_put_unlock: if (!dropped_lock) mutex_unlock(&call->user_mutex); error_put: - rxrpc_put_call(call, rxrpc_call_put); + rxrpc_put_call(call, rxrpc_call_put_sendmsg); _leave(" = %d", ret); return ret; @@ -784,9 +723,9 @@ int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call, notify_end_tx, &dropped_lock); break; case RXRPC_CALL_COMPLETE: - read_lock_bh(&call->state_lock); + read_lock(&call->state_lock); ret = call->error; - read_unlock_bh(&call->state_lock); + read_unlock(&call->state_lock); break; default: /* Request phase complete for this client call */ diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c index ee269e0e6ee8..e51940589ee5 100644 --- a/net/rxrpc/server_key.c +++ b/net/rxrpc/server_key.c @@ -144,3 +144,28 @@ int rxrpc_server_keyring(struct rxrpc_sock *rx, sockptr_t optval, int optlen) _leave(" = 0 [key %x]", key->serial); return 0; } + +/** + * rxrpc_sock_set_security_keyring - Set the security keyring for a kernel service + * @sk: The socket to set the keyring on + * @keyring: The keyring to set + * + * Set the server security keyring on an rxrpc socket. This is used to provide + * the encryption keys for a kernel service. + */ +int rxrpc_sock_set_security_keyring(struct sock *sk, struct key *keyring) +{ + struct rxrpc_sock *rx = rxrpc_sk(sk); + int ret = 0; + + lock_sock(sk); + if (rx->securities) + ret = -EINVAL; + else if (rx->sk.sk_state != RXRPC_UNBOUND) + ret = -EISCONN; + else + rx->securities = key_get(keyring); + release_sock(sk); + return ret; +} +EXPORT_SYMBOL(rxrpc_sock_set_security_keyring); diff --git a/net/rxrpc/skbuff.c b/net/rxrpc/skbuff.c index 0c827d5bb2b8..ebe0c75e7b07 100644 --- a/net/rxrpc/skbuff.c +++ b/net/rxrpc/skbuff.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* ar-skbuff.c: socket buffer destruction handling +/* Socket buffer accounting * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) @@ -19,56 +19,50 @@ /* * Note the allocation or reception of a socket buffer. */ -void rxrpc_new_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) +void rxrpc_new_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) { - const void *here = __builtin_return_address(0); int n = atomic_inc_return(select_skb_count(skb)); - trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here); + trace_rxrpc_skb(skb, refcount_read(&skb->users), n, why); } /* * Note the re-emergence of a socket buffer from a queue or buffer. */ -void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) +void rxrpc_see_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) { - const void *here = __builtin_return_address(0); if (skb) { int n = atomic_read(select_skb_count(skb)); - trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here); + trace_rxrpc_skb(skb, refcount_read(&skb->users), n, why); } } /* * Note the addition of a ref on a socket buffer. */ -void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) +void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) { - const void *here = __builtin_return_address(0); int n = atomic_inc_return(select_skb_count(skb)); - trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here); + trace_rxrpc_skb(skb, refcount_read(&skb->users), n, why); skb_get(skb); } /* * Note the dropping of a ref on a socket buffer by the core. */ -void rxrpc_eaten_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) +void rxrpc_eaten_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) { - const void *here = __builtin_return_address(0); int n = atomic_inc_return(&rxrpc_n_rx_skbs); - trace_rxrpc_skb(skb, op, 0, n, here); + trace_rxrpc_skb(skb, 0, n, why); } /* * Note the destruction of a socket buffer. */ -void rxrpc_free_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) +void rxrpc_free_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) { - const void *here = __builtin_return_address(0); if (skb) { - int n; - n = atomic_dec_return(select_skb_count(skb)); - trace_rxrpc_skb(skb, op, refcount_read(&skb->users), n, here); + int n = atomic_dec_return(select_skb_count(skb)); + trace_rxrpc_skb(skb, refcount_read(&skb->users), n, why); kfree_skb(skb); } } @@ -78,12 +72,12 @@ void rxrpc_free_skb(struct sk_buff *skb, enum rxrpc_skb_trace op) */ void rxrpc_purge_queue(struct sk_buff_head *list) { - const void *here = __builtin_return_address(0); struct sk_buff *skb; + while ((skb = skb_dequeue((list))) != NULL) { int n = atomic_dec_return(select_skb_count(skb)); - trace_rxrpc_skb(skb, rxrpc_skb_purged, - refcount_read(&skb->users), n, here); + trace_rxrpc_skb(skb, refcount_read(&skb->users), n, + rxrpc_skb_put_purge); kfree_skb(skb); } } diff --git a/net/rxrpc/txbuf.c b/net/rxrpc/txbuf.c index 96bfee89927b..d2cf2aac3adb 100644 --- a/net/rxrpc/txbuf.c +++ b/net/rxrpc/txbuf.c @@ -26,7 +26,6 @@ struct rxrpc_txbuf *rxrpc_alloc_txbuf(struct rxrpc_call *call, u8 packet_type, INIT_LIST_HEAD(&txb->call_link); INIT_LIST_HEAD(&txb->tx_link); refcount_set(&txb->ref, 1); - txb->call = call; txb->call_debug_id = call->debug_id; txb->debug_id = atomic_inc_return(&rxrpc_txbuf_debug_ids); txb->space = sizeof(txb->data); @@ -34,7 +33,7 @@ struct rxrpc_txbuf *rxrpc_alloc_txbuf(struct rxrpc_call *call, u8 packet_type, txb->offset = 0; txb->flags = 0; txb->ack_why = 0; - txb->seq = call->tx_top + 1; + txb->seq = call->tx_prepared + 1; txb->wire.epoch = htonl(call->conn->proto.epoch); txb->wire.cid = htonl(call->cid); txb->wire.callNumber = htonl(call->call_id); @@ -44,7 +43,7 @@ struct rxrpc_txbuf *rxrpc_alloc_txbuf(struct rxrpc_call *call, u8 packet_type, txb->wire.userStatus = 0; txb->wire.securityIndex = call->security_ix; txb->wire._rsvd = 0; - txb->wire.serviceId = htons(call->service_id); + txb->wire.serviceId = htons(call->dest_srx.srx_service); trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 1, @@ -107,6 +106,7 @@ void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call) { struct rxrpc_txbuf *txb; rxrpc_seq_t hard_ack = smp_load_acquire(&call->acks_hard_ack); + bool wake = false; _enter("%x/%x/%x", call->tx_bottom, call->acks_hard_ack, call->tx_top); @@ -120,8 +120,10 @@ void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call) if (before(hard_ack, txb->seq)) break; + if (txb->seq != call->tx_bottom + 1) + rxrpc_see_txbuf(txb, rxrpc_txbuf_see_out_of_step); ASSERTCMP(txb->seq, ==, call->tx_bottom + 1); - call->tx_bottom++; + smp_store_release(&call->tx_bottom, call->tx_bottom + 1); list_del_rcu(&txb->call_link); trace_rxrpc_txqueue(call, rxrpc_txqueue_dequeue); @@ -129,7 +131,12 @@ void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call) spin_unlock(&call->tx_lock); rxrpc_put_txbuf(txb, rxrpc_txbuf_put_rotated); + if (after(call->acks_hard_ack, call->tx_bottom + 128)) + wake = true; } spin_unlock(&call->tx_lock); + + if (wake) + wake_up(&call->waitq); } diff --git a/net/sched/Kconfig b/net/sched/Kconfig index 4662a6ce8a7e..777d6b50505c 100644 --- a/net/sched/Kconfig +++ b/net/sched/Kconfig @@ -977,6 +977,7 @@ config NET_ACT_TUNNEL_KEY config NET_ACT_CT tristate "connection tracking tc action" depends on NET_CLS_ACT && NF_CONNTRACK && (!NF_NAT || NF_NAT) && NF_FLOW_TABLE + select NF_NAT_OVS if NF_NAT help Say Y here to allow sending the packets to conntrack module. diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 9b31a10cc639..5b3c0ac495be 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -23,6 +23,7 @@ #include <net/act_api.h> #include <net/netlink.h> #include <net/flow_offload.h> +#include <net/tc_wrapper.h> #ifdef CONFIG_INET DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count); @@ -1080,7 +1081,7 @@ restart_act_graph: repeat_ttl = 32; repeat: - ret = a->ops->act(skb, a, res); + ret = tc_act(skb, a, res); if (unlikely(ret == TC_ACT_REPEAT)) { if (--repeat_ttl != 0) goto repeat; diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index b79eee44e24e..b0455fda7d0b 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -18,6 +18,7 @@ #include <linux/tc_act/tc_bpf.h> #include <net/tc_act/tc_bpf.h> +#include <net/tc_wrapper.h> #define ACT_BPF_NAME_LEN 256 @@ -31,8 +32,9 @@ struct tcf_bpf_cfg { static struct tc_action_ops act_bpf_ops; -static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, + const struct tc_action *act, + struct tcf_result *res) { bool at_ingress = skb_at_tc_ingress(skb); struct tcf_bpf *prog = to_bpf(act); diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c index d41002e4613f..7e63ff7e3ed7 100644 --- a/net/sched/act_connmark.c +++ b/net/sched/act_connmark.c @@ -20,6 +20,7 @@ #include <net/pkt_cls.h> #include <uapi/linux/tc_act/tc_connmark.h> #include <net/tc_act/tc_connmark.h> +#include <net/tc_wrapper.h> #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_core.h> @@ -27,8 +28,9 @@ static struct tc_action_ops act_connmark_ops; -static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_connmark_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { const struct nf_conntrack_tuple_hash *thash; struct nf_conntrack_tuple tuple; diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 1366adf9b909..95e9304024b7 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -32,6 +32,7 @@ #include <linux/tc_act/tc_csum.h> #include <net/tc_act/tc_csum.h> +#include <net/tc_wrapper.h> static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = { [TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), }, @@ -563,8 +564,9 @@ fail: return 0; } -static int tcf_csum_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_csum *p = to_tcf_csum(a); bool orig_vlan_tag_present = false; diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index dd5ae7551956..0ca2bb8ed026 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -24,6 +24,7 @@ #include <net/ipv6_frag.h> #include <uapi/linux/tc_act/tc_ct.h> #include <net/tc_act/tc_ct.h> +#include <net/tc_wrapper.h> #include <net/netfilter/nf_flow_table.h> #include <net/netfilter/nf_conntrack.h> @@ -863,90 +864,6 @@ static void tcf_ct_params_free_rcu(struct rcu_head *head) tcf_ct_params_free(params); } -#if IS_ENABLED(CONFIG_NF_NAT) -/* Modelled after nf_nat_ipv[46]_fn(). - * range is only used for new, uninitialized NAT state. - * Returns either NF_ACCEPT or NF_DROP. - */ -static int ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - const struct nf_nat_range2 *range, - enum nf_nat_manip_type maniptype) -{ - __be16 proto = skb_protocol(skb, true); - int hooknum, err = NF_ACCEPT; - - /* See HOOK2MANIP(). */ - if (maniptype == NF_NAT_MANIP_SRC) - hooknum = NF_INET_LOCAL_IN; /* Source NAT */ - else - hooknum = NF_INET_LOCAL_OUT; /* Destination NAT */ - - switch (ctinfo) { - case IP_CT_RELATED: - case IP_CT_RELATED_REPLY: - if (proto == htons(ETH_P_IP) && - ip_hdr(skb)->protocol == IPPROTO_ICMP) { - if (!nf_nat_icmp_reply_translation(skb, ct, ctinfo, - hooknum)) - err = NF_DROP; - goto out; - } else if (IS_ENABLED(CONFIG_IPV6) && proto == htons(ETH_P_IPV6)) { - __be16 frag_off; - u8 nexthdr = ipv6_hdr(skb)->nexthdr; - int hdrlen = ipv6_skip_exthdr(skb, - sizeof(struct ipv6hdr), - &nexthdr, &frag_off); - - if (hdrlen >= 0 && nexthdr == IPPROTO_ICMPV6) { - if (!nf_nat_icmpv6_reply_translation(skb, ct, - ctinfo, - hooknum, - hdrlen)) - err = NF_DROP; - goto out; - } - } - /* Non-ICMP, fall thru to initialize if needed. */ - fallthrough; - case IP_CT_NEW: - /* Seen it before? This can happen for loopback, retrans, - * or local packets. - */ - if (!nf_nat_initialized(ct, maniptype)) { - /* Initialize according to the NAT action. */ - err = (range && range->flags & NF_NAT_RANGE_MAP_IPS) - /* Action is set up to establish a new - * mapping. - */ - ? nf_nat_setup_info(ct, range, maniptype) - : nf_nat_alloc_null_binding(ct, hooknum); - if (err != NF_ACCEPT) - goto out; - } - break; - - case IP_CT_ESTABLISHED: - case IP_CT_ESTABLISHED_REPLY: - break; - - default: - err = NF_DROP; - goto out; - } - - err = nf_nat_packet(ct, ctinfo, hooknum, skb); - if (err == NF_ACCEPT) { - if (maniptype == NF_NAT_MANIP_SRC) - tc_skb_cb(skb)->post_ct_snat = 1; - if (maniptype == NF_NAT_MANIP_DST) - tc_skb_cb(skb)->post_ct_dnat = 1; - } -out: - return err; -} -#endif /* CONFIG_NF_NAT */ - static void tcf_ct_act_set_mark(struct nf_conn *ct, u32 mark, u32 mask) { #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) @@ -986,60 +903,30 @@ static int tcf_ct_act_nat(struct sk_buff *skb, bool commit) { #if IS_ENABLED(CONFIG_NF_NAT) - int err; - enum nf_nat_manip_type maniptype; + int err, action = 0; if (!(ct_action & TCA_CT_ACT_NAT)) return NF_ACCEPT; + if (ct_action & TCA_CT_ACT_NAT_SRC) + action |= BIT(NF_NAT_MANIP_SRC); + if (ct_action & TCA_CT_ACT_NAT_DST) + action |= BIT(NF_NAT_MANIP_DST); - /* Add NAT extension if not confirmed yet. */ - if (!nf_ct_is_confirmed(ct) && !nf_ct_nat_ext_add(ct)) - return NF_DROP; /* Can't NAT. */ - - if (ctinfo != IP_CT_NEW && (ct->status & IPS_NAT_MASK) && - (ctinfo != IP_CT_RELATED || commit)) { - /* NAT an established or related connection like before. */ - if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY) - /* This is the REPLY direction for a connection - * for which NAT was applied in the forward - * direction. Do the reverse NAT. - */ - maniptype = ct->status & IPS_SRC_NAT - ? NF_NAT_MANIP_DST : NF_NAT_MANIP_SRC; - else - maniptype = ct->status & IPS_SRC_NAT - ? NF_NAT_MANIP_SRC : NF_NAT_MANIP_DST; - } else if (ct_action & TCA_CT_ACT_NAT_SRC) { - maniptype = NF_NAT_MANIP_SRC; - } else if (ct_action & TCA_CT_ACT_NAT_DST) { - maniptype = NF_NAT_MANIP_DST; - } else { - return NF_ACCEPT; - } + err = nf_ct_nat(skb, ct, ctinfo, &action, range, commit); + + if (action & BIT(NF_NAT_MANIP_SRC)) + tc_skb_cb(skb)->post_ct_snat = 1; + if (action & BIT(NF_NAT_MANIP_DST)) + tc_skb_cb(skb)->post_ct_dnat = 1; - err = ct_nat_execute(skb, ct, ctinfo, range, maniptype); - if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) { - if (ct->status & IPS_SRC_NAT) { - if (maniptype == NF_NAT_MANIP_SRC) - maniptype = NF_NAT_MANIP_DST; - else - maniptype = NF_NAT_MANIP_SRC; - - err = ct_nat_execute(skb, ct, ctinfo, range, - maniptype); - } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) { - err = ct_nat_execute(skb, ct, ctinfo, NULL, - NF_NAT_MANIP_SRC); - } - } return err; #else return NF_ACCEPT; #endif } -static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, + struct tcf_result *res) { struct net *net = dev_net(skb->dev); enum ip_conntrack_info ctinfo; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index eaa02f098d1c..4b1b59da5c0b 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -18,6 +18,7 @@ #include <net/pkt_cls.h> #include <uapi/linux/tc_act/tc_ctinfo.h> #include <net/tc_act/tc_ctinfo.h> +#include <net/tc_wrapper.h> #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_core.h> @@ -75,8 +76,9 @@ static void tcf_ctinfo_cpmark_set(struct nf_conn *ct, struct tcf_ctinfo *ca, skb->mark = READ_ONCE(ct->mark) & cp->cpmarkmask; } -static int tcf_ctinfo_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_ctinfo_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { const struct nf_conntrack_tuple_hash *thash = NULL; struct tcf_ctinfo *ca = to_ctinfo(a); diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index 62d682b96b88..54f1b13b2360 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -18,6 +18,7 @@ #include <net/pkt_cls.h> #include <linux/tc_act/tc_gact.h> #include <net/tc_act/tc_gact.h> +#include <net/tc_wrapper.h> static struct tc_action_ops act_gact_ops; @@ -145,8 +146,9 @@ release_idr: return err; } -static int tcf_gact_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_gact_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_gact *gact = to_gact(a); int action = READ_ONCE(gact->tcf_action); diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index 3049878e7315..9b8def0be41e 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -14,6 +14,7 @@ #include <net/netlink.h> #include <net/pkt_cls.h> #include <net/tc_act/tc_gate.h> +#include <net/tc_wrapper.h> static struct tc_action_ops act_gate_ops; @@ -113,8 +114,9 @@ static enum hrtimer_restart gate_timer_func(struct hrtimer *timer) return HRTIMER_RESTART; } -static int tcf_gate_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_gate_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_gate *gact = to_gate(a); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 41d63b33461d..bc7611b0744c 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -29,6 +29,7 @@ #include <net/tc_act/tc_ife.h> #include <linux/etherdevice.h> #include <net/ife.h> +#include <net/tc_wrapper.h> static int max_metacnt = IFE_META_MAX + 1; static struct tc_action_ops act_ife_ops; @@ -861,8 +862,9 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, return action; } -static int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_ife_info *ife = to_ife(a); struct tcf_ife_params *p; diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c index 1625e1037416..5d96ffebd40f 100644 --- a/net/sched/act_ipt.c +++ b/net/sched/act_ipt.c @@ -20,6 +20,7 @@ #include <net/pkt_sched.h> #include <linux/tc_act/tc_ipt.h> #include <net/tc_act/tc_ipt.h> +#include <net/tc_wrapper.h> #include <linux/netfilter_ipv4/ip_tables.h> @@ -216,8 +217,9 @@ static int tcf_xt_init(struct net *net, struct nlattr *nla, a, &act_xt_ops, tp, flags); } -static int tcf_ipt_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_ipt_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { int ret = 0, result = 0; struct tcf_ipt *ipt = to_ipt(a); diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index b8ad6ae282c0..7284bcea7b0b 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -24,6 +24,7 @@ #include <net/pkt_cls.h> #include <linux/tc_act/tc_mirred.h> #include <net/tc_act/tc_mirred.h> +#include <net/tc_wrapper.h> static LIST_HEAD(mirred_list); static DEFINE_SPINLOCK(mirred_list_lock); @@ -217,8 +218,9 @@ static int tcf_mirred_forward(bool want_ingress, struct sk_buff *skb) return err; } -static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_mirred *m = to_mirred(a); struct sk_buff *skb2 = skb; diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index 8ad25cc8ccd5..ff47ce4d3968 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -14,6 +14,7 @@ #include <net/pkt_sched.h> #include <net/pkt_cls.h> #include <net/tc_act/tc_mpls.h> +#include <net/tc_wrapper.h> static struct tc_action_ops act_mpls_ops; @@ -49,8 +50,9 @@ static __be32 tcf_mpls_get_lse(struct mpls_shim_hdr *lse, return cpu_to_be32(new_lse); } -static int tcf_mpls_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_mpls *m = to_mpls(a); struct tcf_mpls_params *p; diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c index 9265145f1040..74c74be33048 100644 --- a/net/sched/act_nat.c +++ b/net/sched/act_nat.c @@ -24,7 +24,7 @@ #include <net/tc_act/tc_nat.h> #include <net/tcp.h> #include <net/udp.h> - +#include <net/tc_wrapper.h> static struct tc_action_ops act_nat_ops; @@ -98,8 +98,9 @@ release_idr: return err; } -static int tcf_nat_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_nat_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_nat *p = to_tcf_nat(a); struct iphdr *iph; diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 94ed5857ce67..a0378e9f0121 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -20,6 +20,7 @@ #include <net/tc_act/tc_pedit.h> #include <uapi/linux/tc_act/tc_pedit.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> static struct tc_action_ops act_pedit_ops; @@ -319,8 +320,9 @@ static int pedit_skb_hdr_offset(struct sk_buff *skb, return ret; } -static int tcf_pedit_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_pedit *p = to_pedit(a); u32 max_offset; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index 0adb26e366a7..227cba58ce9f 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -19,6 +19,7 @@ #include <net/netlink.h> #include <net/pkt_cls.h> #include <net/tc_act/tc_police.h> +#include <net/tc_wrapper.h> /* Each policer is serialized by its individual spinlock */ @@ -242,8 +243,9 @@ static bool tcf_police_mtu_check(struct sk_buff *skb, u32 limit) return len <= limit; } -static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_police *police = to_police(a); s64 now, toks, ppstoks = 0, ptoks = 0; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 7a25477f5d99..98dea08c1764 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -20,6 +20,7 @@ #include <net/tc_act/tc_sample.h> #include <net/psample.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/if_arp.h> @@ -153,8 +154,9 @@ static bool tcf_sample_dev_ok_push(struct net_device *dev) } } -static int tcf_sample_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_sample_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_sample *s = to_sample(a); struct psample_group *psample_group; diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c index 18d376135461..4b84514534f3 100644 --- a/net/sched/act_simple.c +++ b/net/sched/act_simple.c @@ -14,6 +14,7 @@ #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/tc_act/tc_defact.h> #include <net/tc_act/tc_defact.h> @@ -21,8 +22,9 @@ static struct tc_action_ops act_simp_ops; #define SIMP_MAX_DATA 32 -static int tcf_simp_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_simp_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_defact *d = to_defact(a); diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 1710780c908a..ce7008cf291c 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -16,6 +16,7 @@ #include <net/ipv6.h> #include <net/dsfield.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/tc_act/tc_skbedit.h> #include <net/tc_act/tc_skbedit.h> @@ -36,8 +37,9 @@ static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params, return netdev_cap_txqueue(skb->dev, queue_mapping); } -static int tcf_skbedit_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_skbedit *d = to_skbedit(a); struct tcf_skbedit_params *params; diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index d98758a63934..dffa990a9629 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -15,14 +15,16 @@ #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/tc_act/tc_skbmod.h> #include <net/tc_act/tc_skbmod.h> static struct tc_action_ops act_skbmod_ops; -static int tcf_skbmod_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_skbmod *d = to_skbmod(a); int action, max_edit_len, err; diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 2691a3d8e451..2d12d2626415 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -16,14 +16,16 @@ #include <net/pkt_sched.h> #include <net/dst.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/tc_act/tc_tunnel_key.h> #include <net/tc_act/tc_tunnel_key.h> static struct tc_action_ops act_tunnel_key_ops; -static int tunnel_key_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tunnel_key_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_tunnel_key *t = to_tunnel_key(a); struct tcf_tunnel_key_params *params; diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c index 7b24e898a3e6..0251442f5f29 100644 --- a/net/sched/act_vlan.c +++ b/net/sched/act_vlan.c @@ -12,14 +12,16 @@ #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <linux/tc_act/tc_vlan.h> #include <net/tc_act/tc_vlan.h> static struct tc_action_ops act_vlan_ops; -static int tcf_vlan_act(struct sk_buff *skb, const struct tc_action *a, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcf_vlan_act(struct sk_buff *skb, + const struct tc_action *a, + struct tcf_result *res) { struct tcf_vlan *v = to_vlan(a); struct tcf_vlan_params *p; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 23d1cfa4f58c..668130f08903 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -40,6 +40,7 @@ #include <net/tc_act/tc_mpls.h> #include <net/tc_act/tc_gate.h> #include <net/flow_offload.h> +#include <net/tc_wrapper.h> extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; @@ -1564,7 +1565,7 @@ reclassify: tp->protocol != htons(ETH_P_ALL)) continue; - err = tp->classify(skb, tp, res); + err = tc_classify(skb, tp, res); #ifdef CONFIG_NET_CLS_ACT if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { first_tp = orig_tp; diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index d229ce99e554..1b92c33b5f81 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -18,6 +18,7 @@ #include <net/netlink.h> #include <net/act_api.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> struct basic_head { struct list_head flist; @@ -36,8 +37,9 @@ struct basic_filter { struct rcu_work rwork; }; -static int basic_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int basic_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { int r; struct basic_head *head = rcu_dereference_bh(tp->root); diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index bc317b3eac12..466c26df853a 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -19,6 +19,7 @@ #include <net/rtnetlink.h> #include <net/pkt_cls.h> #include <net/sock.h> +#include <net/tc_wrapper.h> MODULE_LICENSE("GPL"); MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>"); @@ -77,8 +78,9 @@ static int cls_bpf_exec_opcode(int code) } } -static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct cls_bpf_head *head = rcu_dereference_bh(tp->root); bool at_ingress = skb_at_tc_ingress(skb); diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c index ed00001b528a..bd9322d71910 100644 --- a/net/sched/cls_cgroup.c +++ b/net/sched/cls_cgroup.c @@ -13,6 +13,7 @@ #include <net/pkt_cls.h> #include <net/sock.h> #include <net/cls_cgroup.h> +#include <net/tc_wrapper.h> struct cls_cgroup_head { u32 handle; @@ -22,8 +23,9 @@ struct cls_cgroup_head { struct rcu_work rwork; }; -static int cls_cgroup_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int cls_cgroup_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct cls_cgroup_head *head = rcu_dereference_bh(tp->root); u32 classid = task_get_classid(skb); diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 014cd3de7b5d..535668e1f748 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -24,6 +24,7 @@ #include <net/ip.h> #include <net/route.h> #include <net/flow_dissector.h> +#include <net/tc_wrapper.h> #if IS_ENABLED(CONFIG_NF_CONNTRACK) #include <net/netfilter/nf_conntrack.h> @@ -292,8 +293,9 @@ static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow) (1 << FLOW_KEY_NFCT_PROTO_SRC) | \ (1 << FLOW_KEY_NFCT_PROTO_DST)) -static int flow_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int flow_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct flow_head *head = rcu_dereference_bh(tp->root); struct flow_filter *f; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 25bc57ee6ea1..0b15698b3531 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -27,6 +27,7 @@ #include <net/vxlan.h> #include <net/erspan.h> #include <net/gtp.h> +#include <net/tc_wrapper.h> #include <net/dst.h> #include <net/dst_metadata.h> @@ -305,8 +306,9 @@ static u16 fl_ct_info_to_flower_map[] = { TCA_FLOWER_KEY_CT_FLAGS_NEW, }; -static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct cls_fl_head *head = rcu_dereference_bh(tp->root); bool post_ct = tc_skb_cb(skb)->post_ct; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index a32351da968c..ae9439a6c56c 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -21,6 +21,7 @@ #include <net/act_api.h> #include <net/pkt_cls.h> #include <net/sch_generic.h> +#include <net/tc_wrapper.h> #define HTSIZE 256 @@ -47,8 +48,9 @@ static u32 fw_hash(u32 handle) return handle % HTSIZE; } -static int fw_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int fw_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct fw_head *head = rcu_dereference_bh(tp->root); struct fw_filter *f; diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 39a5d9c170de..705f63da2c21 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -12,6 +12,7 @@ #include <net/sch_generic.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> struct cls_mall_head { struct tcf_exts exts; @@ -24,8 +25,9 @@ struct cls_mall_head { bool deleting; }; -static int mall_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int mall_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct cls_mall_head *head = rcu_dereference_bh(tp->root); diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index 9e43b929d4ca..d0c53724d3e8 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -17,6 +17,7 @@ #include <net/netlink.h> #include <net/act_api.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> /* * 1. For now we assume that route tags < 256. @@ -121,8 +122,9 @@ static inline int route4_hash_wild(void) return 0; \ } -static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct route4_head *head = rcu_dereference_bh(tp->root); struct dst_entry *dst; diff --git a/net/sched/cls_rsvp.c b/net/sched/cls_rsvp.c index de1c1d4da597..03d8619bd9c6 100644 --- a/net/sched/cls_rsvp.c +++ b/net/sched/cls_rsvp.c @@ -15,10 +15,12 @@ #include <net/netlink.h> #include <net/act_api.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #define RSVP_DST_LEN 1 #define RSVP_ID "rsvp" #define RSVP_OPS cls_rsvp_ops +#define RSVP_CLS rsvp_classify #include "cls_rsvp.h" MODULE_LICENSE("GPL"); diff --git a/net/sched/cls_rsvp.h b/net/sched/cls_rsvp.h index b00a7dbd0587..869efba9f834 100644 --- a/net/sched/cls_rsvp.h +++ b/net/sched/cls_rsvp.h @@ -124,8 +124,8 @@ static inline unsigned int hash_src(__be32 *src) return r; \ } -static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int RSVP_CLS(struct sk_buff *skb, const struct tcf_proto *tp, + struct tcf_result *res) { struct rsvp_head *head = rcu_dereference_bh(tp->root); struct rsvp_session *s; @@ -738,7 +738,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q, static struct tcf_proto_ops RSVP_OPS __read_mostly = { .kind = RSVP_ID, - .classify = rsvp_classify, + .classify = RSVP_CLS, .init = rsvp_init, .destroy = rsvp_destroy, .get = rsvp_get, diff --git a/net/sched/cls_rsvp6.c b/net/sched/cls_rsvp6.c index 64078846000e..e627cc32d633 100644 --- a/net/sched/cls_rsvp6.c +++ b/net/sched/cls_rsvp6.c @@ -15,10 +15,12 @@ #include <net/act_api.h> #include <net/pkt_cls.h> #include <net/netlink.h> +#include <net/tc_wrapper.h> #define RSVP_DST_LEN 4 #define RSVP_ID "rsvp6" #define RSVP_OPS cls_rsvp6_ops +#define RSVP_CLS rsvp6_classify #include "cls_rsvp.h" MODULE_LICENSE("GPL"); diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c index 1c9eeb98d826..eb0e9458e722 100644 --- a/net/sched/cls_tcindex.c +++ b/net/sched/cls_tcindex.c @@ -16,6 +16,7 @@ #include <net/netlink.h> #include <net/pkt_cls.h> #include <net/sch_generic.h> +#include <net/tc_wrapper.h> /* * Passing parameters to the root seems to be done more awkwardly than really @@ -98,9 +99,9 @@ static struct tcindex_filter_result *tcindex_lookup(struct tcindex_data *p, return NULL; } - -static int tcindex_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int tcindex_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct tcindex_data *p = rcu_dereference_bh(tp->root); struct tcindex_filter_result *f; diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 34d25f7a0687..4e2e269f121f 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -39,6 +39,7 @@ #include <net/act_api.h> #include <net/pkt_cls.h> #include <linux/idr.h> +#include <net/tc_wrapper.h> struct tc_u_knode { struct tc_u_knode __rcu *next; @@ -100,8 +101,9 @@ static inline unsigned int u32_hash_fold(__be32 key, return h; } -static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp, - struct tcf_result *res) +TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) { struct { struct tc_u_knode *knode; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 4a27dfb1ba0f..2317db02c764 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -31,6 +31,7 @@ #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> +#include <net/tc_wrapper.h> #include <trace/events/qdisc.h> @@ -2273,6 +2274,8 @@ static struct pernet_operations psched_net_ops = { .exit = psched_net_exit, }; +DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper); + static int __init pktsched_init(void) { int err; @@ -2300,6 +2303,8 @@ static int __init pktsched_init(void) rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass, 0); + tc_wrapper_init(); + return 0; } diff --git a/net/sctp/stream_sched.c b/net/sctp/stream_sched.c index 7c8f9d89e16a..330067002deb 100644 --- a/net/sctp/stream_sched.c +++ b/net/sctp/stream_sched.c @@ -50,10 +50,6 @@ static void sctp_sched_fcfs_free_sid(struct sctp_stream *stream, __u16 sid) { } -static void sctp_sched_fcfs_free(struct sctp_stream *stream) -{ -} - static void sctp_sched_fcfs_enqueue(struct sctp_outq *q, struct sctp_datamsg *msg) { @@ -101,7 +97,6 @@ static struct sctp_sched_ops sctp_sched_fcfs = { .init = sctp_sched_fcfs_init, .init_sid = sctp_sched_fcfs_init_sid, .free_sid = sctp_sched_fcfs_free_sid, - .free = sctp_sched_fcfs_free, .enqueue = sctp_sched_fcfs_enqueue, .dequeue = sctp_sched_fcfs_dequeue, .dequeue_done = sctp_sched_fcfs_dequeue_done, @@ -131,6 +126,23 @@ void sctp_sched_ops_init(void) sctp_sched_ops_rr_init(); } +static void sctp_sched_free_sched(struct sctp_stream *stream) +{ + struct sctp_sched_ops *sched = sctp_sched_ops_from_stream(stream); + struct sctp_stream_out_ext *soute; + int i; + + sched->unsched_all(stream); + for (i = 0; i < stream->outcnt; i++) { + soute = SCTP_SO(stream, i)->ext; + if (!soute) + continue; + sched->free_sid(stream, i); + /* Give the next scheduler a clean slate. */ + memset_after(soute, 0, outq); + } +} + int sctp_sched_set_sched(struct sctp_association *asoc, enum sctp_sched_type sched) { @@ -146,18 +158,8 @@ int sctp_sched_set_sched(struct sctp_association *asoc, if (sched > SCTP_SS_MAX) return -EINVAL; - if (old) { - old->free(&asoc->stream); - - /* Give the next scheduler a clean slate. */ - for (i = 0; i < asoc->stream.outcnt; i++) { - struct sctp_stream_out_ext *ext = SCTP_SO(&asoc->stream, i)->ext; - - if (!ext) - continue; - memset_after(ext, 0, outq); - } - } + if (old) + sctp_sched_free_sched(&asoc->stream); asoc->outqueue.sched = n; n->init(&asoc->stream); @@ -181,7 +183,7 @@ int sctp_sched_set_sched(struct sctp_association *asoc, return ret; err: - n->free(&asoc->stream); + sctp_sched_free_sched(&asoc->stream); asoc->outqueue.sched = &sctp_sched_fcfs; /* Always safe */ return ret; diff --git a/net/sctp/stream_sched_prio.c b/net/sctp/stream_sched_prio.c index 4fc9f2923ed1..42d4800f263d 100644 --- a/net/sctp/stream_sched_prio.c +++ b/net/sctp/stream_sched_prio.c @@ -222,32 +222,6 @@ static void sctp_sched_prio_free_sid(struct sctp_stream *stream, __u16 sid) kfree(prio); } -static void sctp_sched_prio_free(struct sctp_stream *stream) -{ - struct sctp_stream_priorities *prio, *n; - LIST_HEAD(list); - int i; - - /* As we don't keep a list of priorities, to avoid multiple - * frees we have to do it in 3 steps: - * 1. unsched everyone, so the lists are free to use in 2. - * 2. build the list of the priorities - * 3. free the list - */ - sctp_sched_prio_unsched_all(stream); - for (i = 0; i < stream->outcnt; i++) { - if (!SCTP_SO(stream, i)->ext) - continue; - prio = SCTP_SO(stream, i)->ext->prio_head; - if (prio && list_empty(&prio->prio_sched)) - list_add(&prio->prio_sched, &list); - } - list_for_each_entry_safe(prio, n, &list, prio_sched) { - list_del_init(&prio->prio_sched); - kfree(prio); - } -} - static void sctp_sched_prio_enqueue(struct sctp_outq *q, struct sctp_datamsg *msg) { @@ -342,7 +316,6 @@ static struct sctp_sched_ops sctp_sched_prio = { .init = sctp_sched_prio_init, .init_sid = sctp_sched_prio_init_sid, .free_sid = sctp_sched_prio_free_sid, - .free = sctp_sched_prio_free, .enqueue = sctp_sched_prio_enqueue, .dequeue = sctp_sched_prio_dequeue, .dequeue_done = sctp_sched_prio_dequeue_done, diff --git a/net/sctp/stream_sched_rr.c b/net/sctp/stream_sched_rr.c index cc444fe0d67c..1f235e7f643a 100644 --- a/net/sctp/stream_sched_rr.c +++ b/net/sctp/stream_sched_rr.c @@ -94,11 +94,6 @@ static void sctp_sched_rr_free_sid(struct sctp_stream *stream, __u16 sid) { } -static void sctp_sched_rr_free(struct sctp_stream *stream) -{ - sctp_sched_rr_unsched_all(stream); -} - static void sctp_sched_rr_enqueue(struct sctp_outq *q, struct sctp_datamsg *msg) { @@ -182,7 +177,6 @@ static struct sctp_sched_ops sctp_sched_rr = { .init = sctp_sched_rr_init, .init_sid = sctp_sched_rr_init_sid, .free_sid = sctp_sched_rr_free_sid, - .free = sctp_sched_rr_free, .enqueue = sctp_sched_rr_enqueue, .dequeue = sctp_sched_rr_dequeue, .dequeue_done = sctp_sched_rr_dequeue_done, diff --git a/net/tipc/link.c b/net/tipc/link.c index e260c0d557f5..b3ce24823f50 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -2224,7 +2224,9 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb, if (tipc_own_addr(l->net) > msg_prevnode(hdr)) l->net_plane = msg_net_plane(hdr); - skb_linearize(skb); + if (skb_linearize(skb)) + goto exit; + hdr = buf_msg(skb); data = msg_data(hdr); diff --git a/net/tipc/node.c b/net/tipc/node.c index b48d97cbbe29..49ddc484c4fe 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1689,6 +1689,7 @@ int tipc_node_xmit(struct net *net, struct sk_buff_head *list, struct tipc_node *n; struct sk_buff_head xmitq; bool node_up = false; + struct net *peer_net; int bearer_id; int rc; @@ -1705,18 +1706,23 @@ int tipc_node_xmit(struct net *net, struct sk_buff_head *list, return -EHOSTUNREACH; } + rcu_read_lock(); tipc_node_read_lock(n); node_up = node_is_up(n); - if (node_up && n->peer_net && check_net(n->peer_net)) { + peer_net = n->peer_net; + tipc_node_read_unlock(n); + if (node_up && peer_net && check_net(peer_net)) { /* xmit inner linux container */ - tipc_lxc_xmit(n->peer_net, list); + tipc_lxc_xmit(peer_net, list); if (likely(skb_queue_empty(list))) { - tipc_node_read_unlock(n); + rcu_read_unlock(); tipc_node_put(n); return 0; } } + rcu_read_unlock(); + tipc_node_read_lock(n); bearer_id = n->active_links[selector & 1]; if (unlikely(bearer_id == INVALID_BEARER_ID)) { tipc_node_read_unlock(n); diff --git a/net/unix/diag.c b/net/unix/diag.c index 105f522a89fe..616b55c5b890 100644 --- a/net/unix/diag.c +++ b/net/unix/diag.c @@ -114,14 +114,16 @@ static int sk_diag_show_rqlen(struct sock *sk, struct sk_buff *nlskb) return nla_put(nlskb, UNIX_DIAG_RQLEN, sizeof(rql), &rql); } -static int sk_diag_dump_uid(struct sock *sk, struct sk_buff *nlskb) +static int sk_diag_dump_uid(struct sock *sk, struct sk_buff *nlskb, + struct user_namespace *user_ns) { - uid_t uid = from_kuid_munged(sk_user_ns(nlskb->sk), sock_i_uid(sk)); + uid_t uid = from_kuid_munged(user_ns, sock_i_uid(sk)); return nla_put(nlskb, UNIX_DIAG_UID, sizeof(uid_t), &uid); } static int sk_diag_fill(struct sock *sk, struct sk_buff *skb, struct unix_diag_req *req, - u32 portid, u32 seq, u32 flags, int sk_ino) + struct user_namespace *user_ns, + u32 portid, u32 seq, u32 flags, int sk_ino) { struct nlmsghdr *nlh; struct unix_diag_msg *rep; @@ -167,7 +169,7 @@ static int sk_diag_fill(struct sock *sk, struct sk_buff *skb, struct unix_diag_r goto out_nlmsg_trim; if ((req->udiag_show & UDIAG_SHOW_UID) && - sk_diag_dump_uid(sk, skb)) + sk_diag_dump_uid(sk, skb, user_ns)) goto out_nlmsg_trim; nlmsg_end(skb, nlh); @@ -179,7 +181,8 @@ out_nlmsg_trim: } static int sk_diag_dump(struct sock *sk, struct sk_buff *skb, struct unix_diag_req *req, - u32 portid, u32 seq, u32 flags) + struct user_namespace *user_ns, + u32 portid, u32 seq, u32 flags) { int sk_ino; @@ -190,7 +193,7 @@ static int sk_diag_dump(struct sock *sk, struct sk_buff *skb, struct unix_diag_r if (!sk_ino) return 0; - return sk_diag_fill(sk, skb, req, portid, seq, flags, sk_ino); + return sk_diag_fill(sk, skb, req, user_ns, portid, seq, flags, sk_ino); } static int unix_diag_dump(struct sk_buff *skb, struct netlink_callback *cb) @@ -214,7 +217,7 @@ static int unix_diag_dump(struct sk_buff *skb, struct netlink_callback *cb) goto next; if (!(req->udiag_states & (1 << sk->sk_state))) goto next; - if (sk_diag_dump(sk, skb, req, + if (sk_diag_dump(sk, skb, req, sk_user_ns(skb->sk), NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) { @@ -282,7 +285,8 @@ again: if (!rep) goto out; - err = sk_diag_fill(sk, rep, req, NETLINK_CB(in_skb).portid, + err = sk_diag_fill(sk, rep, req, sk_user_ns(NETLINK_CB(in_skb).sk), + NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, 0, req->udiag_ino); if (err < 0) { nlmsg_free(rep); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 1ad0326ff4dc..33a82ecab9d5 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -3868,6 +3868,9 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag struct cfg80211_chan_def chandef = {}; int ret; + if (!link) + goto nla_put_failure; + if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id)) goto nla_put_failure; if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, diff --git a/net/wireless/reg.c b/net/wireless/reg.c index c3d950d29432..4f3f31244e8b 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -4311,8 +4311,10 @@ static int __init regulatory_init_db(void) return -EINVAL; err = load_builtin_regdb_keys(); - if (err) + if (err) { + platform_device_unregister(reg_pdev); return err; + } /* We always try to get an update for the static regdomain */ err = regulatory_hint_core(cfg80211_world_regdom->alpha2); diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 26c74f5b473c..790bc31cf82e 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -158,9 +158,8 @@ static inline void bss_ref_put(struct cfg80211_registered_device *rdev, if (bss->pub.hidden_beacon_bss) { struct cfg80211_internal_bss *hbss; - hbss = container_of(bss->pub.hidden_beacon_bss, - struct cfg80211_internal_bss, - pub); + + hbss = bss_from_pub(bss->pub.hidden_beacon_bss); hbss->refcount--; if (hbss->refcount == 0) bss_free(hbss); @@ -169,9 +168,7 @@ static inline void bss_ref_put(struct cfg80211_registered_device *rdev, if (bss->pub.transmitted_bss) { struct cfg80211_internal_bss *tbss; - tbss = container_of(bss->pub.transmitted_bss, - struct cfg80211_internal_bss, - pub); + tbss = bss_from_pub(bss->pub.transmitted_bss); tbss->refcount--; if (tbss->refcount == 0) bss_free(tbss); @@ -1290,7 +1287,8 @@ static int cmp_bss(struct cfg80211_bss *a, int i, r; if (a->channel != b->channel) - return b->channel->center_freq - a->channel->center_freq; + return (b->channel->center_freq * 1000 + b->channel->freq_offset) - + (a->channel->center_freq * 1000 + a->channel->freq_offset); a_ies = rcu_access_pointer(a->ies); if (!a_ies) @@ -1791,13 +1789,8 @@ cfg80211_bss_update(struct cfg80211_registered_device *rdev, /* This must be before the call to bss_ref_get */ if (tmp->pub.transmitted_bss) { - struct cfg80211_internal_bss *pbss = - container_of(tmp->pub.transmitted_bss, - struct cfg80211_internal_bss, - pub); - new->pub.transmitted_bss = tmp->pub.transmitted_bss; - bss_ref_get(rdev, pbss); + bss_ref_get(rdev, bss_from_pub(tmp->pub.transmitted_bss)); } list_add_tail(&new->list, &rdev->bss_list); @@ -2575,15 +2568,12 @@ EXPORT_SYMBOL(cfg80211_inform_bss_frame_data); void cfg80211_ref_bss(struct wiphy *wiphy, struct cfg80211_bss *pub) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); - struct cfg80211_internal_bss *bss; if (!pub) return; - bss = container_of(pub, struct cfg80211_internal_bss, pub); - spin_lock_bh(&rdev->bss_lock); - bss_ref_get(rdev, bss); + bss_ref_get(rdev, bss_from_pub(pub)); spin_unlock_bh(&rdev->bss_lock); } EXPORT_SYMBOL(cfg80211_ref_bss); @@ -2591,15 +2581,12 @@ EXPORT_SYMBOL(cfg80211_ref_bss); void cfg80211_put_bss(struct wiphy *wiphy, struct cfg80211_bss *pub) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); - struct cfg80211_internal_bss *bss; if (!pub) return; - bss = container_of(pub, struct cfg80211_internal_bss, pub); - spin_lock_bh(&rdev->bss_lock); - bss_ref_put(rdev, bss); + bss_ref_put(rdev, bss_from_pub(pub)); spin_unlock_bh(&rdev->bss_lock); } EXPORT_SYMBOL(cfg80211_put_bss); @@ -2613,7 +2600,7 @@ void cfg80211_unlink_bss(struct wiphy *wiphy, struct cfg80211_bss *pub) if (WARN_ON(!pub)) return; - bss = container_of(pub, struct cfg80211_internal_bss, pub); + bss = bss_from_pub(pub); spin_lock_bh(&rdev->bss_lock); if (list_empty(&bss->list)) @@ -2622,8 +2609,7 @@ void cfg80211_unlink_bss(struct wiphy *wiphy, struct cfg80211_bss *pub) list_for_each_entry_safe(nontrans_bss, tmp, &pub->nontrans_list, nontrans_list) { - tmp1 = container_of(nontrans_bss, - struct cfg80211_internal_bss, pub); + tmp1 = bss_from_pub(nontrans_bss); if (__cfg80211_unlink_bss(rdev, tmp1)) rdev->bss_generation++; } @@ -2680,9 +2666,7 @@ void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, /* use transmitting bss */ if (cbss->pub.transmitted_bss) - cbss = container_of(cbss->pub.transmitted_bss, - struct cfg80211_internal_bss, - pub); + cbss = bss_from_pub(cbss->pub.transmitted_bss); cbss->pub.channel = chan; @@ -2711,8 +2695,7 @@ void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, list_for_each_entry_safe(nontrans_bss, tmp, &new->pub.nontrans_list, nontrans_list) { - bss = container_of(nontrans_bss, - struct cfg80211_internal_bss, pub); + bss = bss_from_pub(nontrans_bss); if (__cfg80211_unlink_bss(rdev, bss)) rdev->bss_generation++; } @@ -2729,8 +2712,7 @@ void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, list_for_each_entry_safe(nontrans_bss, tmp, &cbss->pub.nontrans_list, nontrans_list) { - bss = container_of(nontrans_bss, - struct cfg80211_internal_bss, pub); + bss = bss_from_pub(nontrans_bss); bss->pub.channel = chan; rb_erase(&bss->rbn, &rdev->bss_tree); rb_insert_bss(rdev, bss); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 21269e8f2db4..4aff76c6f12e 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -132,6 +132,16 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN) return skb; + /* The packet was sent to HW IPsec packet offload engine, + * but to wrong device. Drop the packet, so it won't skip + * XFRM stack. + */ + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) { + kfree_skb(skb); + dev_core_stats_tx_dropped_inc(dev); + return NULL; + } + /* This skb was already validated on the upper/virtual dev */ if ((x->xso.dev != dev) && (x->xso.real_dev == dev)) return skb; @@ -229,6 +239,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_dev_offload *xso = &x->xso; xfrm_address_t *saddr; xfrm_address_t *daddr; + bool is_packet_offload; if (!x->type_offload) { NL_SET_ERR_MSG(extack, "Type doesn't support offload"); @@ -241,11 +252,13 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, return -EINVAL; } - if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) { + if (xuo->flags & + ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) { NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); return -EINVAL; } + is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET; dev = dev_get_by_index(net, xuo->ifindex); if (!dev) { if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { @@ -260,7 +273,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, x->props.family, xfrm_smark_get(0, x)); if (IS_ERR(dst)) - return 0; + return (is_packet_offload) ? -EINVAL : 0; dev = dst->dev; @@ -271,7 +284,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { xso->dev = NULL; dev_put(dev); - return 0; + return (is_packet_offload) ? -EINVAL : 0; } if (x->props.flags & XFRM_STATE_ESN && @@ -291,14 +304,28 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, else xso->dir = XFRM_DEV_OFFLOAD_OUT; + if (is_packet_offload) + xso->type = XFRM_DEV_OFFLOAD_PACKET; + else + xso->type = XFRM_DEV_OFFLOAD_CRYPTO; + err = dev->xfrmdev_ops->xdo_dev_state_add(x); if (err) { xso->dev = NULL; xso->dir = 0; xso->real_dev = NULL; netdev_put(dev, &xso->dev_tracker); - - if (err != -EOPNOTSUPP) { + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + + /* User explicitly requested packet offload mode and configured + * policy in addition to the XFRM state. So be civil to users, + * and return an error instead of taking fallback path. + * + * This WARN_ON() can be seen as a documentation for driver + * authors to do not return -EOPNOTSUPP in packet offload mode. + */ + WARN_ON(err == -EOPNOTSUPP && is_packet_offload); + if (err != -EOPNOTSUPP || is_packet_offload) { NL_SET_ERR_MSG(extack, "Device failed to offload this state"); return err; } @@ -308,6 +335,69 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } EXPORT_SYMBOL_GPL(xfrm_dev_state_add); +int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack) +{ + struct xfrm_dev_offload *xdo = &xp->xdo; + struct net_device *dev; + int err; + + if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) { + /* We support only packet offload mode and it means + * that user must set XFRM_OFFLOAD_PACKET bit. + */ + NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); + return -EINVAL; + } + + dev = dev_get_by_index(net, xuo->ifindex); + if (!dev) + return -EINVAL; + + if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) { + xdo->dev = NULL; + dev_put(dev); + NL_SET_ERR_MSG(extack, "Policy offload is not supported"); + return -EINVAL; + } + + xdo->dev = dev; + netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC); + xdo->real_dev = dev; + xdo->type = XFRM_DEV_OFFLOAD_PACKET; + switch (dir) { + case XFRM_POLICY_IN: + xdo->dir = XFRM_DEV_OFFLOAD_IN; + break; + case XFRM_POLICY_OUT: + xdo->dir = XFRM_DEV_OFFLOAD_OUT; + break; + case XFRM_POLICY_FWD: + xdo->dir = XFRM_DEV_OFFLOAD_FWD; + break; + default: + xdo->dev = NULL; + dev_put(dev); + NL_SET_ERR_MSG(extack, "Unrecognized offload direction"); + return -EINVAL; + } + + err = dev->xfrmdev_ops->xdo_dev_policy_add(xp); + if (err) { + xdo->dev = NULL; + xdo->real_dev = NULL; + xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + xdo->dir = 0; + netdev_put(dev, &xdo->dev_tracker); + NL_SET_ERR_MSG(extack, "Device failed to offload this policy"); + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xfrm_dev_policy_add); + bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) { int mtu; @@ -318,8 +408,9 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) if (!x->type_offload || x->encap) return false; - if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && - (!xdst->child->xfrm)) { + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET || + ((!dev || (dev == xfrm_dst_path(dst)->dev)) && + !xdst->child->xfrm)) { mtu = xfrm_state_mtu(x, xdst->child_mtu_cached); if (skb->len <= mtu) goto ok; @@ -410,8 +501,10 @@ static int xfrm_api_check(struct net_device *dev) static int xfrm_dev_down(struct net_device *dev) { - if (dev->features & NETIF_F_HW_ESP) + if (dev->features & NETIF_F_HW_ESP) { xfrm_dev_state_flush(dev_net(dev), dev, true); + xfrm_dev_policy_flush(dev_net(dev), dev, true); + } return NOTIFY_DONE; } diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index 78cb8d0a6a18..ff114d68cc43 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -492,7 +492,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) struct xfrm_state *x = dst->xfrm; struct net *net = xs_net(x); - if (err <= 0) + if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET) goto resume; do { @@ -717,6 +717,16 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) break; } + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) { + if (!xfrm_dev_offload_ok(skb, x)) { + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); + kfree_skb(skb); + return -EHOSTUNREACH; + } + + return xfrm_output_resume(sk, skb, 0); + } + secpath_reset(skb); if (xfrm_dev_offload_ok(skb, x)) { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 9b9e2765363d..e9eb82c5457d 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -425,6 +425,7 @@ void xfrm_policy_destroy(struct xfrm_policy *policy) if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) BUG(); + xfrm_dev_policy_free(policy); call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); } EXPORT_SYMBOL(xfrm_policy_destroy); @@ -535,7 +536,7 @@ redo: __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, pol->family, nhashmask, dbits, sbits); - if (!entry0) { + if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { hlist_del_rcu(&pol->bydst); hlist_add_head_rcu(&pol->bydst, ndsttable + h); h0 = h; @@ -866,7 +867,7 @@ static void xfrm_policy_inexact_list_reinsert(struct net *net, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, newpos); else hlist_add_head_rcu(&policy->bydst, &n->hhead); @@ -1347,7 +1348,7 @@ static void xfrm_hash_rebuild(struct work_struct *work) else break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, newpos); else hlist_add_head_rcu(&policy->bydst, chain); @@ -1524,7 +1525,7 @@ static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); else hlist_add_head_rcu(&policy->bydst_inexact_list, chain); @@ -1561,9 +1562,12 @@ static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); else + /* Packet offload policies enter to the head + * to speed-up lookups. + */ hlist_add_head_rcu(&policy->bydst, chain); return delpol; @@ -1769,12 +1773,41 @@ xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) } return err; } + +static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, + struct net_device *dev, + bool task_valid) +{ + struct xfrm_policy *pol; + int err = 0; + + list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { + if (pol->walk.dead || + xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || + pol->xdo.dev != dev) + continue; + + err = security_xfrm_policy_delete(pol->security); + if (err) { + xfrm_audit_policy_delete(pol, 0, task_valid); + return err; + } + } + return err; +} #else static inline int xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) { return 0; } + +static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, + struct net_device *dev, + bool task_valid) +{ + return 0; +} #endif int xfrm_policy_flush(struct net *net, u8 type, bool task_valid) @@ -1814,6 +1847,44 @@ out: } EXPORT_SYMBOL(xfrm_policy_flush); +int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, + bool task_valid) +{ + int dir, err = 0, cnt = 0; + struct xfrm_policy *pol; + + spin_lock_bh(&net->xfrm.xfrm_policy_lock); + + err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid); + if (err) + goto out; + +again: + list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { + dir = xfrm_policy_id2dir(pol->index); + if (pol->walk.dead || + dir >= XFRM_POLICY_MAX || + pol->xdo.dev != dev) + continue; + + __xfrm_policy_unlink(pol, dir); + spin_unlock_bh(&net->xfrm.xfrm_policy_lock); + cnt++; + xfrm_audit_policy_delete(pol, 1, task_valid); + xfrm_policy_kill(pol); + spin_lock_bh(&net->xfrm.xfrm_policy_lock); + goto again; + } + if (cnt) + __xfrm_policy_inexact_flush(net); + else + err = -ESRCH; +out: + spin_unlock_bh(&net->xfrm.xfrm_policy_lock); + return err; +} +EXPORT_SYMBOL(xfrm_dev_policy_flush); + int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk, int (*func)(struct xfrm_policy *, int, int, void*), void *data) @@ -2113,6 +2184,9 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, break; } } + if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) + goto skip_inexact; + bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr, daddr)) @@ -2245,6 +2319,7 @@ int xfrm_policy_delete(struct xfrm_policy *pol, int dir) pol = __xfrm_policy_unlink(pol, dir); spin_unlock_bh(&net->xfrm.xfrm_policy_lock); if (pol) { + xfrm_dev_policy_delete(pol); xfrm_policy_kill(pol); return 0; } diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 9ec481fbfb63..cc1d0ea42672 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -84,6 +84,25 @@ static unsigned int xfrm_seq_hash(struct net *net, u32 seq) return __xfrm_seq_hash(seq, net->xfrm.state_hmask); } +#define XFRM_STATE_INSERT(by, _n, _h, _type) \ + { \ + struct xfrm_state *_x = NULL; \ + \ + if (_type != XFRM_DEV_OFFLOAD_PACKET) { \ + hlist_for_each_entry_rcu(_x, _h, by) { \ + if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ + continue; \ + break; \ + } \ + } \ + \ + if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ + /* SAD is empty or consist from HW SAs only */ \ + hlist_add_head_rcu(_n, _h); \ + else \ + hlist_add_before_rcu(_n, &_x->by); \ + } + static void xfrm_hash_transfer(struct hlist_head *list, struct hlist_head *ndsttable, struct hlist_head *nsrctable, @@ -100,23 +119,25 @@ static void xfrm_hash_transfer(struct hlist_head *list, h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, x->props.reqid, x->props.family, nhashmask); - hlist_add_head_rcu(&x->bydst, ndsttable + h); + XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type); h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, x->props.family, nhashmask); - hlist_add_head_rcu(&x->bysrc, nsrctable + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type); if (x->id.spi) { h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto, x->props.family, nhashmask); - hlist_add_head_rcu(&x->byspi, nspitable + h); + XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h, + x->xso.type); } if (x->km.seq) { h = __xfrm_seq_hash(x->km.seq, nhashmask); - hlist_add_head_rcu(&x->byseq, nseqtable + h); + XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, + x->xso.type); } } } @@ -549,6 +570,8 @@ static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) int err = 0; spin_lock(&x->lock); + xfrm_dev_state_update_curlft(x); + if (x->km.state == XFRM_STATE_DEAD) goto out; if (x->km.state == XFRM_STATE_EXPIRED) @@ -951,6 +974,49 @@ xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, x->props.family = tmpl->encap_family; } +static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark, + const xfrm_address_t *daddr, + __be32 spi, u8 proto, + unsigned short family, + struct xfrm_dev_offload *xdo) +{ + unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); + struct xfrm_state *x; + + hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { +#ifdef CONFIG_XFRM_OFFLOAD + if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (xdo->dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif + if (x->props.family != family || + x->id.spi != spi || + x->id.proto != proto || + !xfrm_addr_equal(&x->id.daddr, daddr, family)) + continue; + + if ((mark & x->mark.m) != x->mark.v) + continue; + if (!xfrm_state_hold_rcu(x)) + continue; + return x; + } + + return NULL; +} + static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, u8 proto, @@ -1092,6 +1158,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, rcu_read_lock(); h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) { +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (pol->xdo.dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif if (x->props.family == encap_family && x->props.reqid == tmpl->reqid && (mark & x->mark.m) == x->mark.v && @@ -1109,6 +1192,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family); hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) { +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (pol->xdo.dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif if (x->props.family == encap_family && x->props.reqid == tmpl->reqid && (mark & x->mark.m) == x->mark.v && @@ -1126,8 +1226,10 @@ found: x = best; if (!x && !error && !acquire_in_progress) { if (tmpl->id.spi && - (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi, - tmpl->id.proto, encap_family)) != NULL) { + (x0 = __xfrm_state_lookup_all(net, mark, daddr, + tmpl->id.spi, tmpl->id.proto, + encap_family, + &pol->xdo)) != NULL) { to_put = x0; error = -EEXIST; goto out; @@ -1161,21 +1263,53 @@ found: x = NULL; goto out; } - +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + struct xfrm_dev_offload *xdo = &pol->xdo; + struct xfrm_dev_offload *xso = &x->xso; + + xso->type = XFRM_DEV_OFFLOAD_PACKET; + xso->dir = xdo->dir; + xso->dev = xdo->dev; + xso->real_dev = xdo->real_dev; + netdev_tracker_alloc(xso->dev, &xso->dev_tracker, + GFP_ATOMIC); + error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x); + if (error) { + xso->dir = 0; + netdev_put(xso->dev, &xso->dev_tracker); + xso->dev = NULL; + xso->real_dev = NULL; + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + x->km.state = XFRM_STATE_DEAD; + to_put = x; + x = NULL; + goto out; + } + } +#endif if (km_query(x, tmpl, pol) == 0) { spin_lock_bh(&net->xfrm.xfrm_state_lock); x->km.state = XFRM_STATE_ACQ; list_add(&x->km.all, &net->xfrm.state_all); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, + net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, daddr, saddr, encap_family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, + net->xfrm.state_bysrc + h, + x->xso.type); if (x->id.spi) { h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, + net->xfrm.state_byspi + h, + x->xso.type); } if (x->km.seq) { h = xfrm_seq_hash(net, x->km.seq); - hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h); + XFRM_STATE_INSERT(byseq, &x->byseq, + net->xfrm.state_byseq + h, + x->xso.type); } x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; hrtimer_start(&x->mtimer, @@ -1185,6 +1319,18 @@ found: xfrm_hash_grow_check(net, x->bydst.next != NULL); spin_unlock_bh(&net->xfrm.xfrm_state_lock); } else { +#ifdef CONFIG_XFRM_OFFLOAD + struct xfrm_dev_offload *xso = &x->xso; + + if (xso->type == XFRM_DEV_OFFLOAD_PACKET) { + xso->dev->xfrmdev_ops->xdo_dev_state_delete(x); + xso->dir = 0; + netdev_put(xso->dev, &xso->dev_tracker); + xso->dev = NULL; + xso->real_dev = NULL; + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + } +#endif x->km.state = XFRM_STATE_DEAD; to_put = x; x = NULL; @@ -1280,22 +1426,26 @@ static void __xfrm_state_insert(struct xfrm_state *x) h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, x->props.reqid, x->props.family); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, + x->xso.type); if (x->id.spi) { h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, + x->xso.type); } if (x->km.seq) { h = xfrm_seq_hash(net, x->km.seq); - hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h); + XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h, + x->xso.type); } hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); @@ -1409,9 +1559,11 @@ static struct xfrm_state *__find_acq_core(struct net *net, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL_SOFT); list_add(&x->km.all, &net->xfrm.state_all); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, daddr, saddr, family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, + x->xso.type); net->xfrm.state_num++; @@ -1786,6 +1938,8 @@ EXPORT_SYMBOL(xfrm_state_update); int xfrm_state_check_expire(struct xfrm_state *x) { + xfrm_dev_state_update_curlft(x); + if (!x->curlft.use_time) x->curlft.use_time = ktime_get_real_seconds(); @@ -2094,7 +2248,8 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, spin_lock_bh(&net->xfrm.xfrm_state_lock); x->id.spi = newspi; h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, + x->xso.type); spin_unlock_bh(&net->xfrm.xfrm_state_lock); err = 0; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 0eb4696661c8..cf5172d4ce68 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -956,6 +956,8 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) xuo->ifindex = xso->dev->ifindex; if (xso->dir == XFRM_DEV_OFFLOAD_IN) xuo->flags = XFRM_OFFLOAD_INBOUND; + if (xso->type == XFRM_DEV_OFFLOAD_PACKET) + xuo->flags |= XFRM_OFFLOAD_PACKET; return 0; } @@ -1890,6 +1892,15 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, if (attrs[XFRMA_IF_ID]) xp->if_id = nla_get_u32(attrs[XFRMA_IF_ID]); + /* configure the hardware if offload is requested */ + if (attrs[XFRMA_OFFLOAD_DEV]) { + err = xfrm_dev_policy_add(net, xp, + nla_data(attrs[XFRMA_OFFLOAD_DEV]), + p->dir, extack); + if (err) + goto error; + } + return xp; error: *errp = err; @@ -1929,6 +1940,7 @@ static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh, xfrm_audit_policy_add(xp, err ? 0 : 1, true); if (err) { + xfrm_dev_policy_delete(xp); security_xfrm_policy_free(xp->security); kfree(xp); return err; @@ -2041,6 +2053,8 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3379,6 +3393,8 @@ static int build_acquire(struct sk_buff *skb, struct xfrm_state *x, err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3497,6 +3513,8 @@ static int build_polexpire(struct sk_buff *skb, struct xfrm_policy *xp, err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3580,6 +3598,8 @@ static int xfrm_notify_policy(struct xfrm_policy *xp, int dir, const struct km_e err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) goto out_free_skb; diff --git a/sound/firewire/dice/dice-stream.c b/sound/firewire/dice/dice-stream.c index f99e00083141..4c677c8546c7 100644 --- a/sound/firewire/dice/dice-stream.c +++ b/sound/firewire/dice/dice-stream.c @@ -59,7 +59,7 @@ int snd_dice_stream_get_rate_mode(struct snd_dice *dice, unsigned int rate, static int select_clock(struct snd_dice *dice, unsigned int rate) { - __be32 reg; + __be32 reg, new; u32 data; int i; int err; @@ -83,15 +83,17 @@ static int select_clock(struct snd_dice *dice, unsigned int rate) if (completion_done(&dice->clock_accepted)) reinit_completion(&dice->clock_accepted); - reg = cpu_to_be32(data); + new = cpu_to_be32(data); err = snd_dice_transaction_write_global(dice, GLOBAL_CLOCK_SELECT, - ®, sizeof(reg)); + &new, sizeof(new)); if (err < 0) return err; if (wait_for_completion_timeout(&dice->clock_accepted, - msecs_to_jiffies(NOTIFICATION_TIMEOUT_MS)) == 0) - return -ETIMEDOUT; + msecs_to_jiffies(NOTIFICATION_TIMEOUT_MS)) == 0) { + if (reg != new) + return -ETIMEDOUT; + } return 0; } diff --git a/sound/soc/codecs/cs42l51.c b/sound/soc/codecs/cs42l51.c index 51721edd8f53..e88d9ff95cdf 100644 --- a/sound/soc/codecs/cs42l51.c +++ b/sound/soc/codecs/cs42l51.c @@ -143,7 +143,7 @@ static const struct snd_kcontrol_new cs42l51_snd_controls[] = { 0, 0xA0, 96, adc_att_tlv), SOC_DOUBLE_R_SX_TLV("PGA Volume", CS42L51_ALC_PGA_CTL, CS42L51_ALC_PGB_CTL, - 0, 0x19, 30, pga_tlv), + 0, 0x1A, 30, pga_tlv), SOC_SINGLE("Playback Deemphasis Switch", CS42L51_DAC_CTL, 3, 1, 0), SOC_SINGLE("Auto-Mute Switch", CS42L51_DAC_CTL, 2, 1, 0), SOC_SINGLE("Soft Ramp Switch", CS42L51_DAC_CTL, 1, 1, 0), diff --git a/sound/soc/codecs/tlv320adc3xxx.c b/sound/soc/codecs/tlv320adc3xxx.c index a969547708d4..52bb55724724 100644 --- a/sound/soc/codecs/tlv320adc3xxx.c +++ b/sound/soc/codecs/tlv320adc3xxx.c @@ -14,6 +14,7 @@ #include <dt-bindings/sound/tlv320adc3xxx.h> #include <linux/clk.h> +#include <linux/gpio/consumer.h> #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/io.h> @@ -1025,7 +1026,9 @@ static const struct gpio_chip adc3xxx_gpio_chip = { static void adc3xxx_free_gpio(struct adc3xxx *adc3xxx) { +#ifdef CONFIG_GPIOLIB gpiochip_remove(&adc3xxx->gpio_chip); +#endif } static void adc3xxx_init_gpio(struct adc3xxx *adc3xxx) diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c index 79ef4e269bc9..4b86ef82fd93 100644 --- a/sound/soc/fsl/fsl_micfil.c +++ b/sound/soc/fsl/fsl_micfil.c @@ -194,6 +194,25 @@ static int fsl_micfil_reset(struct device *dev) if (ret) return ret; + /* + * SRES is self-cleared bit, but REG_MICFIL_CTRL1 is defined + * as non-volatile register, so SRES still remain in regmap + * cache after set, that every update of REG_MICFIL_CTRL1, + * software reset happens. so clear it explicitly. + */ + ret = regmap_clear_bits(micfil->regmap, REG_MICFIL_CTRL1, + MICFIL_CTRL1_SRES); + if (ret) + return ret; + + /* + * Set SRES should clear CHnF flags, But even add delay here + * the CHnF may not be cleared sometimes, so clear CHnF explicitly. + */ + ret = regmap_write_bits(micfil->regmap, REG_MICFIL_STAT, 0xFF, 0xFF); + if (ret) + return ret; + return 0; } diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index bd88de056358..55b009d3c681 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -452,7 +452,7 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, val = ucontrol->value.integer.value[0]; if (mc->platform_max && val > mc->platform_max) return -EINVAL; - if (val > max - min) + if (val > max) return -EINVAL; val_mask = mask << shift; val = (val + min) & mask; @@ -464,10 +464,15 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, ret = err; if (snd_soc_volsw_is_stereo(mc)) { - unsigned int val2; + unsigned int val2 = ucontrol->value.integer.value[1]; + + if (mc->platform_max && val2 > mc->platform_max) + return -EINVAL; + if (val2 > max) + return -EINVAL; val_mask = mask << rshift; - val2 = (ucontrol->value.integer.value[1] + min) & mask; + val2 = (val2 + min) & mask; val2 = val2 << rshift; err = snd_soc_component_update_bits(component, reg2, val_mask, diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index f07aef7c592c..b57b091d8026 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -48,6 +48,7 @@ TARGETS += nci TARGETS += net TARGETS += net/af_unix TARGETS += net/forwarding +TARGETS += net/hsr TARGETS += net/mptcp TARGETS += net/openvswitch TARGETS += netfilter diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/devlink_trap_tunnel_ipip6.sh b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_ipip6.sh index f62ce479c266..878125041fc3 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/devlink_trap_tunnel_ipip6.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_ipip6.sh @@ -31,7 +31,7 @@ # | 2001:db8:10::2/64 | # +-------------------------+ -lib_dir=$(dirname $0)/../../../../net/forwarding +lib_dir=$(dirname $0)/../../../net/forwarding ALL_TESTS=" decap_error_test diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index ee38ca888244..9cc84114741d 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -2,6 +2,7 @@ bind_bhash csum cmsg_sender +diag_uid fin_ack_lat gro hwtstamp_config diff --git a/tools/testing/selftests/net/af_unix/Makefile b/tools/testing/selftests/net/af_unix/Makefile index 969620ae9928..1e4b397cece6 100644 --- a/tools/testing/selftests/net/af_unix/Makefile +++ b/tools/testing/selftests/net/af_unix/Makefile @@ -1,3 +1,3 @@ -TEST_GEN_PROGS := test_unix_oob unix_connect +TEST_GEN_PROGS := diag_uid test_unix_oob unix_connect include ../../lib.mk diff --git a/tools/testing/selftests/net/af_unix/diag_uid.c b/tools/testing/selftests/net/af_unix/diag_uid.c new file mode 100644 index 000000000000..5b88f7129fea --- /dev/null +++ b/tools/testing/selftests/net/af_unix/diag_uid.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright Amazon.com Inc. or its affiliates. */ + +#define _GNU_SOURCE +#include <sched.h> + +#include <unistd.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> +#include <linux/sock_diag.h> +#include <linux/unix_diag.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/un.h> + +#include "../../kselftest_harness.h" + +FIXTURE(diag_uid) +{ + int netlink_fd; + int unix_fd; + __u32 inode; + __u64 cookie; +}; + +FIXTURE_VARIANT(diag_uid) +{ + int unshare; + int udiag_show; +}; + +FIXTURE_VARIANT_ADD(diag_uid, uid) +{ + .unshare = 0, + .udiag_show = UDIAG_SHOW_UID +}; + +FIXTURE_VARIANT_ADD(diag_uid, uid_unshare) +{ + .unshare = CLONE_NEWUSER, + .udiag_show = UDIAG_SHOW_UID +}; + +FIXTURE_SETUP(diag_uid) +{ + struct stat file_stat; + socklen_t optlen; + int ret; + + if (variant->unshare) + ASSERT_EQ(unshare(variant->unshare), 0); + + self->netlink_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG); + ASSERT_NE(self->netlink_fd, -1); + + self->unix_fd = socket(AF_UNIX, SOCK_STREAM, 0); + ASSERT_NE(self->unix_fd, -1); + + ret = fstat(self->unix_fd, &file_stat); + ASSERT_EQ(ret, 0); + + self->inode = file_stat.st_ino; + + optlen = sizeof(self->cookie); + ret = getsockopt(self->unix_fd, SOL_SOCKET, SO_COOKIE, &self->cookie, &optlen); + ASSERT_EQ(ret, 0); +} + +FIXTURE_TEARDOWN(diag_uid) +{ + close(self->netlink_fd); + close(self->unix_fd); +} + +int send_request(struct __test_metadata *_metadata, + FIXTURE_DATA(diag_uid) *self, + const FIXTURE_VARIANT(diag_uid) *variant) +{ + struct { + struct nlmsghdr nlh; + struct unix_diag_req udr; + } req = { + .nlh = { + .nlmsg_len = sizeof(req), + .nlmsg_type = SOCK_DIAG_BY_FAMILY, + .nlmsg_flags = NLM_F_REQUEST + }, + .udr = { + .sdiag_family = AF_UNIX, + .udiag_ino = self->inode, + .udiag_cookie = { + (__u32)self->cookie, + (__u32)(self->cookie >> 32) + }, + .udiag_show = variant->udiag_show + } + }; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK + }; + struct iovec iov = { + .iov_base = &req, + .iov_len = sizeof(req) + }; + struct msghdr msg = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1 + }; + + return sendmsg(self->netlink_fd, &msg, 0); +} + +void render_response(struct __test_metadata *_metadata, + struct unix_diag_req *udr, __u32 len) +{ + unsigned int rta_len = len - NLMSG_LENGTH(sizeof(*udr)); + struct rtattr *attr; + uid_t uid; + + ASSERT_GT(len, sizeof(*udr)); + ASSERT_EQ(udr->sdiag_family, AF_UNIX); + + attr = (struct rtattr *)(udr + 1); + ASSERT_NE(RTA_OK(attr, rta_len), 0); + ASSERT_EQ(attr->rta_type, UNIX_DIAG_UID); + + uid = *(uid_t *)RTA_DATA(attr); + ASSERT_EQ(uid, getuid()); +} + +void receive_response(struct __test_metadata *_metadata, + FIXTURE_DATA(diag_uid) *self) +{ + long buf[8192 / sizeof(long)]; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK + }; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf) + }; + struct msghdr msg = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1 + }; + struct unix_diag_req *udr; + struct nlmsghdr *nlh; + int ret; + + ret = recvmsg(self->netlink_fd, &msg, 0); + ASSERT_GT(ret, 0); + + nlh = (struct nlmsghdr *)buf; + ASSERT_NE(NLMSG_OK(nlh, ret), 0); + ASSERT_EQ(nlh->nlmsg_type, SOCK_DIAG_BY_FAMILY); + + render_response(_metadata, NLMSG_DATA(nlh), nlh->nlmsg_len); + + nlh = NLMSG_NEXT(nlh, ret); + ASSERT_EQ(NLMSG_OK(nlh, ret), 0); +} + +TEST_F(diag_uid, 1) +{ + int ret; + + ret = send_request(_metadata, self, variant); + ASSERT_GT(ret, 0); + + receive_response(_metadata, self); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/net/bpf/Makefile b/tools/testing/selftests/net/bpf/Makefile index a26cb94354f6..4abaf16d2077 100644 --- a/tools/testing/selftests/net/bpf/Makefile +++ b/tools/testing/selftests/net/bpf/Makefile @@ -12,7 +12,7 @@ CCINCLUDE += -I$(SCRATCH_DIR)/include BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a -MAKE_DIRS := $(BUILD_DIR)/libbpf +MAKE_DIRS := $(BUILD_DIR)/libbpf $(OUTPUT)/bpf $(MAKE_DIRS): mkdir -p $@ @@ -37,8 +37,8 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) -$(TEST_CUSTOM_PROGS): $(BPFOBJ) - $(CLANG) -O2 -target bpf -c $(@:.o=.c) $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ +$(TEST_CUSTOM_PROGS): $(OUTPUT)/%.o: %.c $(BPFOBJ) | $(MAKE_DIRS) + $(CLANG) -O2 -target bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ $(APIDIR)/linux/bpf.h \ diff --git a/tools/testing/selftests/net/config b/tools/testing/selftests/net/config index ead7963b9bf0..bd89198cd817 100644 --- a/tools/testing/selftests/net/config +++ b/tools/testing/selftests/net/config @@ -43,5 +43,5 @@ CONFIG_NET_ACT_TUNNEL_KEY=m CONFIG_NET_ACT_MIRRED=m CONFIG_BAREUDP=m CONFIG_IPV6_IOAM6_LWTUNNEL=y -CONFIG_CRYPTO_SM4=y +CONFIG_CRYPTO_SM4_GENERIC=y CONFIG_AMT=m diff --git a/tools/testing/selftests/net/fib_tests.sh b/tools/testing/selftests/net/fib_tests.sh index 2271a8727f62..5637b5dadabd 100755 --- a/tools/testing/selftests/net/fib_tests.sh +++ b/tools/testing/selftests/net/fib_tests.sh @@ -1711,13 +1711,21 @@ ipv4_del_addr_test() $IP addr add dev dummy1 172.16.104.1/24 $IP addr add dev dummy1 172.16.104.11/24 + $IP addr add dev dummy1 172.16.104.12/24 + $IP addr add dev dummy1 172.16.104.13/24 $IP addr add dev dummy2 172.16.104.1/24 $IP addr add dev dummy2 172.16.104.11/24 + $IP addr add dev dummy2 172.16.104.12/24 $IP route add 172.16.105.0/24 via 172.16.104.2 src 172.16.104.11 + $IP route add 172.16.106.0/24 dev lo src 172.16.104.12 + $IP route add table 0 172.16.107.0/24 via 172.16.104.2 src 172.16.104.13 $IP route add vrf red 172.16.105.0/24 via 172.16.104.2 src 172.16.104.11 + $IP route add vrf red 172.16.106.0/24 dev lo src 172.16.104.12 set +e # removing address from device in vrf should only remove route from vrf table + echo " Regular FIB info" + $IP addr del dev dummy2 172.16.104.11/24 $IP ro ls vrf red | grep -q 172.16.105.0/24 log_test $? 1 "Route removed from VRF when source address deleted" @@ -1735,6 +1743,35 @@ ipv4_del_addr_test() $IP ro ls vrf red | grep -q 172.16.105.0/24 log_test $? 0 "Route in VRF is not removed by address delete" + # removing address from device in vrf should only remove route from vrf + # table even when the associated fib info only differs in table ID + echo " Identical FIB info with different table ID" + + $IP addr del dev dummy2 172.16.104.12/24 + $IP ro ls vrf red | grep -q 172.16.106.0/24 + log_test $? 1 "Route removed from VRF when source address deleted" + + $IP ro ls | grep -q 172.16.106.0/24 + log_test $? 0 "Route in default VRF not removed" + + $IP addr add dev dummy2 172.16.104.12/24 + $IP route add vrf red 172.16.106.0/24 dev lo src 172.16.104.12 + + $IP addr del dev dummy1 172.16.104.12/24 + $IP ro ls | grep -q 172.16.106.0/24 + log_test $? 1 "Route removed in default VRF when source address deleted" + + $IP ro ls vrf red | grep -q 172.16.106.0/24 + log_test $? 0 "Route in VRF is not removed by address delete" + + # removing address from device in default vrf should remove route from + # the default vrf even when route was inserted with a table ID of 0. + echo " Table ID 0" + + $IP addr del dev dummy1 172.16.104.13/24 + $IP ro ls | grep -q 172.16.107.0/24 + log_test $? 1 "Route removed in default VRF when source address deleted" + $IP li del dummy1 $IP li del dummy2 cleanup diff --git a/tools/testing/selftests/net/hsr/Makefile b/tools/testing/selftests/net/hsr/Makefile new file mode 100644 index 000000000000..92c1d9d080cd --- /dev/null +++ b/tools/testing/selftests/net/hsr/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +top_srcdir = ../../../../.. + +TEST_PROGS := hsr_ping.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/net/hsr/config b/tools/testing/selftests/net/hsr/config new file mode 100644 index 000000000000..22061204fb69 --- /dev/null +++ b/tools/testing/selftests/net/hsr/config @@ -0,0 +1,4 @@ +CONFIG_IPV6=y +CONFIG_NET_SCH_NETEM=m +CONFIG_HSR=y +CONFIG_VETH=y diff --git a/tools/testing/selftests/net/hsr/hsr_ping.sh b/tools/testing/selftests/net/hsr/hsr_ping.sh new file mode 100755 index 000000000000..df9143538708 --- /dev/null +++ b/tools/testing/selftests/net/hsr/hsr_ping.sh @@ -0,0 +1,256 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +ret=0 +ksft_skip=4 +ipv6=true + +optstring="h4" +usage() { + echo "Usage: $0 [OPTION]" + echo -e "\t-4: IPv4 only: disable IPv6 tests (default: test both IPv4 and IPv6)" +} + +while getopts "$optstring" option;do + case "$option" in + "h") + usage $0 + exit 0 + ;; + "4") + ipv6=false + ;; + "?") + usage $0 + exit 1 + ;; +esac +done + +sec=$(date +%s) +rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) +ns1="ns1-$rndh" +ns2="ns2-$rndh" +ns3="ns3-$rndh" + +cleanup() +{ + local netns + for netns in "$ns1" "$ns2" "$ns3" ;do + ip netns del $netns + done +} + +ip -Version > /dev/null 2>&1 +if [ $? -ne 0 ];then + echo "SKIP: Could not run test without ip tool" + exit $ksft_skip +fi + +trap cleanup EXIT + +for i in "$ns1" "$ns2" "$ns3" ;do + ip netns add $i || exit $ksft_skip + ip -net $i link set lo up +done + +echo "INFO: preparing interfaces." +# Three HSR nodes. Each node has one link to each of its neighbour, two links in total. +# +# ns1eth1 ----- ns2eth1 +# hsr1 hsr2 +# ns1eth2 ns2eth2 +# | | +# ns3eth1 ns3eth2 +# \ / +# hsr3 +# +# Interfaces +ip link add ns1eth1 netns "$ns1" type veth peer name ns2eth1 netns "$ns2" +ip link add ns1eth2 netns "$ns1" type veth peer name ns3eth1 netns "$ns3" +ip link add ns3eth2 netns "$ns3" type veth peer name ns2eth2 netns "$ns2" + +# HSRv0. +ip -net "$ns1" link add name hsr1 type hsr slave1 ns1eth1 slave2 ns1eth2 supervision 45 version 0 proto 0 +ip -net "$ns2" link add name hsr2 type hsr slave1 ns2eth1 slave2 ns2eth2 supervision 45 version 0 proto 0 +ip -net "$ns3" link add name hsr3 type hsr slave1 ns3eth1 slave2 ns3eth2 supervision 45 version 0 proto 0 + +# IP for HSR +ip -net "$ns1" addr add 100.64.0.1/24 dev hsr1 +ip -net "$ns1" addr add dead:beef:1::1/64 dev hsr1 nodad +ip -net "$ns2" addr add 100.64.0.2/24 dev hsr2 +ip -net "$ns2" addr add dead:beef:1::2/64 dev hsr2 nodad +ip -net "$ns3" addr add 100.64.0.3/24 dev hsr3 +ip -net "$ns3" addr add dead:beef:1::3/64 dev hsr3 nodad + +# All Links up +ip -net "$ns1" link set ns1eth1 up +ip -net "$ns1" link set ns1eth2 up +ip -net "$ns1" link set hsr1 up + +ip -net "$ns2" link set ns2eth1 up +ip -net "$ns2" link set ns2eth2 up +ip -net "$ns2" link set hsr2 up + +ip -net "$ns3" link set ns3eth1 up +ip -net "$ns3" link set ns3eth2 up +ip -net "$ns3" link set hsr3 up + +# $1: IP address +is_v6() +{ + [ -z "${1##*:*}" ] +} + +do_ping() +{ + local netns="$1" + local connect_addr="$2" + local ping_args="-q -c 2" + + if is_v6 "${connect_addr}"; then + $ipv6 || return 0 + ping_args="${ping_args} -6" + fi + + ip netns exec ${netns} ping ${ping_args} $connect_addr >/dev/null + if [ $? -ne 0 ] ; then + echo "$netns -> $connect_addr connectivity [ FAIL ]" 1>&2 + ret=1 + return 1 + fi + + return 0 +} + +do_ping_long() +{ + local netns="$1" + local connect_addr="$2" + local ping_args="-q -c 10" + + if is_v6 "${connect_addr}"; then + $ipv6 || return 0 + ping_args="${ping_args} -6" + fi + + OUT="$(LANG=C ip netns exec ${netns} ping ${ping_args} $connect_addr | grep received)" + if [ $? -ne 0 ] ; then + echo "$netns -> $connect_addr ping [ FAIL ]" 1>&2 + ret=1 + return 1 + fi + + VAL="$(echo $OUT | cut -d' ' -f1-8)" + if [ "$VAL" != "10 packets transmitted, 10 received, 0% packet loss," ] + then + echo "$netns -> $connect_addr ping TEST [ FAIL ]" + echo "Expect to send and receive 10 packets and no duplicates." + echo "Full message: ${OUT}." + ret=1 + return 1 + fi + + return 0 +} + +stop_if_error() +{ + local msg="$1" + + if [ ${ret} -ne 0 ]; then + echo "FAIL: ${msg}" 1>&2 + exit ${ret} + fi +} + + +echo "INFO: Initial validation ping." +# Each node has to be able each one. +do_ping "$ns1" 100.64.0.2 +do_ping "$ns2" 100.64.0.1 +do_ping "$ns3" 100.64.0.1 +stop_if_error "Initial validation failed." + +do_ping "$ns1" 100.64.0.3 +do_ping "$ns2" 100.64.0.3 +do_ping "$ns3" 100.64.0.2 + +do_ping "$ns1" dead:beef:1::2 +do_ping "$ns1" dead:beef:1::3 +do_ping "$ns2" dead:beef:1::1 +do_ping "$ns2" dead:beef:1::2 +do_ping "$ns3" dead:beef:1::1 +do_ping "$ns3" dead:beef:1::2 + +stop_if_error "Initial validation failed." + +# Wait until supervisor all supervision frames have been processed and the node +# entries have been merged. Otherwise duplicate frames will be observed which is +# valid at this stage. +WAIT=5 +while [ ${WAIT} -gt 0 ] +do + grep 00:00:00:00:00:00 /sys/kernel/debug/hsr/hsr*/node_table + if [ $? -ne 0 ] + then + break + fi + sleep 1 + let WAIT = WAIT - 1 +done + +# Just a safety delay in case the above check didn't handle it. +sleep 1 + +echo "INFO: Longer ping test." +do_ping_long "$ns1" 100.64.0.2 +do_ping_long "$ns1" dead:beef:1::2 +do_ping_long "$ns1" 100.64.0.3 +do_ping_long "$ns1" dead:beef:1::3 + +stop_if_error "Longer ping test failed." + +do_ping_long "$ns2" 100.64.0.1 +do_ping_long "$ns2" dead:beef:1::1 +do_ping_long "$ns2" 100.64.0.3 +do_ping_long "$ns2" dead:beef:1::2 +stop_if_error "Longer ping test failed." + +do_ping_long "$ns3" 100.64.0.1 +do_ping_long "$ns3" dead:beef:1::1 +do_ping_long "$ns3" 100.64.0.2 +do_ping_long "$ns3" dead:beef:1::2 +stop_if_error "Longer ping test failed." + +echo "INFO: Cutting one link." +do_ping_long "$ns1" 100.64.0.3 & + +sleep 3 +ip -net "$ns3" link set ns3eth1 down +wait + +ip -net "$ns3" link set ns3eth1 up + +stop_if_error "Failed with one link down." + +echo "INFO: Delay the link and drop a few packages." +tc -net "$ns3" qdisc add dev ns3eth1 root netem delay 50ms +tc -net "$ns2" qdisc add dev ns2eth1 root netem delay 5ms loss 25% + +do_ping_long "$ns1" 100.64.0.2 +do_ping_long "$ns1" 100.64.0.3 + +stop_if_error "Failed with delay and packetloss." + +do_ping_long "$ns2" 100.64.0.1 +do_ping_long "$ns2" 100.64.0.3 + +stop_if_error "Failed with delay and packetloss." + +do_ping_long "$ns3" 100.64.0.1 +do_ping_long "$ns3" 100.64.0.2 +stop_if_error "Failed with delay and packetloss." + +echo "INFO: All good." +exit $ret diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh index 515859a5168b..24bcd7b9bdb2 100755 --- a/tools/testing/selftests/net/mptcp/diag.sh +++ b/tools/testing/selftests/net/mptcp/diag.sh @@ -1,6 +1,7 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +sec=$(date +%s) rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) ns="ns1-$rndh" ksft_skip=4 diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh index 60198b91a530..a43d3e2f59bb 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh @@ -274,8 +274,7 @@ check_transfer() check_mptcp_disabled() { - local disabled_ns - disabled_ns="ns_disabled-$sech-$(mktemp -u XXXXXX)" + local disabled_ns="ns_disabled-$rndh" ip netns add ${disabled_ns} || exit $ksft_skip # net.mptcp.enabled should be enabled by default @@ -781,7 +780,6 @@ run_tests_mptfo() run_tests_disconnect() { - local peekmode="$1" local old_cin=$cin local old_sin=$sin @@ -789,7 +787,6 @@ run_tests_disconnect() # force do_transfer to cope with the multiple tranmissions sin="$cin.disconnect" - sin_disconnect=$old_sin cin="$cin.disconnect" cin_disconnect="$old_cin" connect_per_transfer=3 @@ -800,7 +797,6 @@ run_tests_disconnect() # restore previous status sin=$old_sin - sin_disconnect="$cout".disconnect cin=$old_cin cin_disconnect="$cin".disconnect connect_per_transfer=1 diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 2eeaf4aca644..d11d3d566608 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -26,6 +26,10 @@ ip_mptcp=0 check_invert=0 validate_checksum=0 init=0 +evts_ns1="" +evts_ns2="" +evts_ns1_pid=0 +evts_ns2_pid=0 declare -A all_tests declare -a only_tests_ids @@ -59,8 +63,9 @@ init_partial() { capout=$(mktemp) - local rndh - rndh=$(mktemp -u XXXXXX) + local sec rndh + sec=$(date +%s) + rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) ns1="ns1-$rndh" ns2="ns2-$rndh" @@ -153,6 +158,8 @@ init() { cin=$(mktemp) cinsent=$(mktemp) cout=$(mktemp) + evts_ns1=$(mktemp) + evts_ns2=$(mktemp) trap cleanup EXIT @@ -164,6 +171,7 @@ cleanup() { rm -f "$cin" "$cout" "$sinfail" rm -f "$sin" "$sout" "$cinsent" "$cinfail" + rm -rf $evts_ns1 $evts_ns2 cleanup_partial } @@ -319,6 +327,18 @@ reset_with_fail() index 100 || exit 1 } +reset_with_events() +{ + reset "${1}" || return 1 + + :> "$evts_ns1" + :> "$evts_ns2" + ip netns exec $ns1 ./pm_nl_ctl events >> "$evts_ns1" 2>&1 & + evts_ns1_pid=$! + ip netns exec $ns2 ./pm_nl_ctl events >> "$evts_ns2" 2>&1 & + evts_ns2_pid=$! +} + fail_test() { ret=1 @@ -472,6 +492,12 @@ kill_wait() wait $1 2>/dev/null } +kill_events_pids() +{ + kill_wait $evts_ns1_pid + kill_wait $evts_ns2_pid +} + pm_nl_set_limits() { local ns=$1 @@ -672,10 +698,6 @@ do_transfer() local port=$((10000 + TEST_COUNT - 1)) local cappid local userspace_pm=0 - local evts_ns1 - local evts_ns1_pid - local evts_ns2 - local evts_ns2_pid :> "$cout" :> "$sout" @@ -752,17 +774,6 @@ do_transfer() addr_nr_ns2=${addr_nr_ns2:9} fi - if [ $userspace_pm -eq 1 ]; then - evts_ns1=$(mktemp) - evts_ns2=$(mktemp) - :> "$evts_ns1" - :> "$evts_ns2" - ip netns exec ${listener_ns} ./pm_nl_ctl events >> "$evts_ns1" 2>&1 & - evts_ns1_pid=$! - ip netns exec ${connector_ns} ./pm_nl_ctl events >> "$evts_ns2" 2>&1 & - evts_ns2_pid=$! - fi - local local_addr if is_v6 "${connect_addr}"; then local_addr="::" @@ -829,7 +840,8 @@ do_transfer() if [ $userspace_pm -eq 0 ]; then pm_nl_add_endpoint $ns1 $addr flags signal else - tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns1") + tk=$(grep "type:1," "$evts_ns1" | + sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q') ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id sleep 1 ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id @@ -980,12 +992,6 @@ do_transfer() kill $cappid fi - if [ $userspace_pm -eq 1 ]; then - kill_wait $evts_ns1_pid - kill_wait $evts_ns2_pid - rm -rf $evts_ns1 $evts_ns2 - fi - NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \ nstat | grep Tcp > /tmp/${listener_ns}.out NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \ @@ -2507,6 +2513,57 @@ backup_tests() fi } +LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED +LISTENER_CLOSED=16 #MPTCP_EVENT_LISTENER_CLOSED + +AF_INET=2 +AF_INET6=10 + +verify_listener_events() +{ + local evt=$1 + local e_type=$2 + local e_family=$3 + local e_saddr=$4 + local e_sport=$5 + local type + local family + local saddr + local sport + + if [ $e_type = $LISTENER_CREATED ]; then + stdbuf -o0 -e0 printf "\t\t\t\t\t CREATE_LISTENER %s:%s"\ + $e_saddr $e_sport + elif [ $e_type = $LISTENER_CLOSED ]; then + stdbuf -o0 -e0 printf "\t\t\t\t\t CLOSE_LISTENER %s:%s "\ + $e_saddr $e_sport + fi + + type=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q') + family=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q') + sport=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q') + if [ $family ] && [ $family = $AF_INET6 ]; then + saddr=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q') + else + saddr=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q') + fi + + if [ $type ] && [ $type = $e_type ] && + [ $family ] && [ $family = $e_family ] && + [ $saddr ] && [ $saddr = $e_saddr ] && + [ $sport ] && [ $sport = $e_sport ]; then + stdbuf -o0 -e0 printf "[ ok ]\n" + return 0 + fi + fail_test + stdbuf -o0 -e0 printf "[fail]\n" +} + add_addr_ports_tests() { # signal address with port @@ -2531,7 +2588,8 @@ add_addr_ports_tests() fi # single address with port, remove - if reset "remove single address with port"; then + # pm listener events + if reset_with_events "remove single address with port"; then pm_nl_set_limits $ns1 0 1 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100 pm_nl_set_limits $ns2 1 1 @@ -2539,6 +2597,10 @@ add_addr_ports_tests() chk_join_nr 1 1 1 chk_add_nr 1 1 1 chk_rm_nr 1 1 invert + + verify_listener_events $evts_ns1 $LISTENER_CREATED $AF_INET 10.0.2.1 10100 + verify_listener_events $evts_ns1 $LISTENER_CLOSED $AF_INET 10.0.2.1 10100 + kill_events_pids fi # subflow and signal with port, remove @@ -2959,22 +3021,24 @@ userspace_tests() fi # userspace pm add & remove address - if reset "userspace pm add & remove address"; then + if reset_with_events "userspace pm add & remove address"; then set_userspace_pm $ns1 pm_nl_set_limits $ns2 1 1 run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow chk_join_nr 1 1 1 chk_add_nr 1 1 chk_rm_nr 1 1 invert + kill_events_pids fi # userspace pm create destroy subflow - if reset "userspace pm create destroy subflow"; then + if reset_with_events "userspace pm create destroy subflow"; then set_userspace_pm $ns2 pm_nl_set_limits $ns1 0 1 run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow chk_join_nr 1 1 1 chk_rm_nr 0 1 + kill_events_pids fi } diff --git a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh index 80d36f7cfee8..1b70c0a304ce 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh @@ -10,13 +10,19 @@ ksft_skip=4 timeout_poll=30 timeout_test=$((timeout_poll * 2 + 1)) mptcp_connect="" -do_all_tests=1 + +sec=$(date +%s) +rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) +ns1="ns1-$rndh" +ns2="ns2-$rndh" +ns_sbox="ns_sbox-$rndh" add_mark_rules() { local ns=$1 local m=$2 + local t for t in iptables ip6tables; do # just to debug: check we have multiple subflows connection requests ip netns exec $ns $t -A OUTPUT -p tcp --syn -m mark --mark $m -j ACCEPT @@ -31,12 +37,7 @@ add_mark_rules() init() { - rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) - - ns1="ns1-$rndh" - ns2="ns2-$rndh" - ns_sbox="ns_sbox-$rndh" - + local netns for netns in "$ns1" "$ns2" "$ns_sbox";do ip netns add $netns || exit $ksft_skip ip -net $netns link set lo up @@ -45,6 +46,7 @@ init() ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0 done + local i for i in `seq 1 4`; do ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2" ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i @@ -74,6 +76,7 @@ init() cleanup() { + local netns for netns in "$ns1" "$ns2" "$ns_sbox"; do ip netns del $netns done @@ -104,15 +107,17 @@ check_mark() local ns=$1 local af=$2 - tables=iptables + local tables=iptables if [ $af -eq 6 ];then tables=ip6tables fi + local counters values counters=$(ip netns exec $ns $tables -v -L OUTPUT | grep DROP) values=${counters%DROP*} + local v for v in $values; do if [ $v -ne 0 ]; then echo "FAIL: got $tables $values in ns $ns , not 0 - not all expected packets marked" 1>&2 @@ -132,9 +137,9 @@ print_file_err() check_transfer() { - in=$1 - out=$2 - what=$3 + local in=$1 + local out=$2 + local what=$3 cmp "$in" "$out" > /dev/null 2>&1 if [ $? -ne 0 ] ;then @@ -157,18 +162,18 @@ is_v6() do_transfer() { - listener_ns="$1" - connector_ns="$2" - cl_proto="$3" - srv_proto="$4" - connect_addr="$5" + local listener_ns="$1" + local connector_ns="$2" + local cl_proto="$3" + local srv_proto="$4" + local connect_addr="$5" - port=12001 + local port=12001 :> "$cout" :> "$sout" - mptcp_connect="./mptcp_connect -r 20" + local mptcp_connect="./mptcp_connect -r 20" local local_addr if is_v6 "${connect_addr}"; then @@ -181,7 +186,7 @@ do_transfer() ip netns exec ${listener_ns} \ $mptcp_connect -t ${timeout_poll} -l -M 1 -p $port -s ${srv_proto} -c TIMESTAMPNS,TCPINQ \ ${local_addr} < "$sin" > "$sout" & - spid=$! + local spid=$! sleep 1 @@ -190,12 +195,12 @@ do_transfer() $mptcp_connect -t ${timeout_poll} -M 2 -p $port -s ${cl_proto} -c TIMESTAMPNS,TCPINQ \ $connect_addr < "$cin" > "$cout" & - cpid=$! + local cpid=$! wait $cpid - retc=$? + local retc=$? wait $spid - rets=$? + local rets=$? if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then echo " client exit code $retc, server $rets" 1>&2 @@ -230,9 +235,9 @@ do_transfer() make_file() { - name=$1 - who=$2 - size=$3 + local name=$1 + local who=$2 + local size=$3 dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name" @@ -265,9 +270,9 @@ do_mptcp_sockopt_tests() run_tests() { - listener_ns="$1" - connector_ns="$2" - connect_addr="$3" + local listener_ns="$1" + local connector_ns="$2" + local connect_addr="$3" local lret=0 do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} @@ -282,8 +287,8 @@ run_tests() do_tcpinq_test() { - ip netns exec "$ns1" ./mptcp_inq "$@" - lret=$? + ip netns exec "$ns_sbox" ./mptcp_inq "$@" + local lret=$? if [ $lret -ne 0 ];then ret=$lret echo "FAIL: mptcp_inq $@" 1>&2 @@ -298,9 +303,7 @@ do_tcpinq_tests() { local lret=0 - ip netns exec "$ns1" iptables -F - ip netns exec "$ns1" ip6tables -F - + local args for args in "-t tcp" "-r tcp"; do do_tcpinq_test $args lret=$? diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index a5aeefd58ab3..9f22f7e5027d 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -1,6 +1,7 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +sec=$(date +%s) rndh=$(printf %x $sec)-$(mktemp -u XXXXXX) ns1="ns1-$rndh" ns2="ns2-$rndh" @@ -148,9 +149,6 @@ do_transfer() :> "$sout" :> "$capout" - local addr_port - addr_port=$(printf "%s:%d" ${connect_addr} ${port}) - if $capture; then local capuser if [ -z $SUDO_USER ] ; then diff --git a/tools/testing/selftests/net/mptcp/userspace_pm.sh b/tools/testing/selftests/net/mptcp/userspace_pm.sh index 3229725b64b0..a29deb9fa024 100755 --- a/tools/testing/selftests/net/mptcp/userspace_pm.sh +++ b/tools/testing/selftests/net/mptcp/userspace_pm.sh @@ -11,11 +11,17 @@ ANNOUNCED=6 # MPTCP_EVENT_ANNOUNCED REMOVED=7 # MPTCP_EVENT_REMOVED SUB_ESTABLISHED=10 # MPTCP_EVENT_SUB_ESTABLISHED SUB_CLOSED=11 # MPTCP_EVENT_SUB_CLOSED +LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED +LISTENER_CLOSED=16 #MPTCP_EVENT_LISTENER_CLOSED AF_INET=2 AF_INET6=10 -evts_pid=0 +file="" +server_evts="" +client_evts="" +server_evts_pid=0 +client_evts_pid=0 client4_pid=0 server4_pid=0 client6_pid=0 @@ -33,7 +39,7 @@ client_addr_id=${RANDOM:0:2} server_addr_id=${RANDOM:0:2} sec=$(date +%s) -rndh=$(stdbuf -o0 -e0 printf %x "$sec")-$(mktemp -u XXXXXX) +rndh=$(printf %x "$sec")-$(mktemp -u XXXXXX) ns1="ns1-$rndh" ns2="ns2-$rndh" @@ -47,7 +53,7 @@ cleanup() { echo "cleanup" - rm -rf $file + rm -rf $file $client_evts $server_evts # Terminate the MPTCP connection and related processes if [ $client4_pid -ne 0 ]; then @@ -62,8 +68,11 @@ cleanup() if [ $server6_pid -ne 0 ]; then kill_wait $server6_pid fi - if [ $evts_pid -ne 0 ]; then - kill_wait $evts_pid + if [ $server_evts_pid -ne 0 ]; then + kill_wait $server_evts_pid + fi + if [ $client_evts_pid -ne 0 ]; then + kill_wait $client_evts_pid fi local netns for netns in "$ns1" "$ns2" ;do @@ -113,8 +122,9 @@ make_file() make_connection() { - local file - file=$(mktemp) + if [ -z "$file" ]; then + file=$(mktemp) + fi make_file "$file" "client" local is_v6=$1 @@ -132,16 +142,24 @@ make_connection() # Capture netlink events over the two network namespaces running # the MPTCP client and server - local client_evts - client_evts=$(mktemp) + if [ -z "$client_evts" ]; then + client_evts=$(mktemp) + fi :>"$client_evts" + if [ $client_evts_pid -ne 0 ]; then + kill_wait $client_evts_pid + fi ip netns exec "$ns2" ./pm_nl_ctl events >> "$client_evts" 2>&1 & - local client_evts_pid=$! - local server_evts - server_evts=$(mktemp) + client_evts_pid=$! + if [ -z "$server_evts" ]; then + server_evts=$(mktemp) + fi :>"$server_evts" + if [ $server_evts_pid -ne 0 ]; then + kill_wait $server_evts_pid + fi ip netns exec "$ns1" ./pm_nl_ctl events >> "$server_evts" 2>&1 & - local server_evts_pid=$! + server_evts_pid=$! sleep 0.5 # Run the server @@ -159,7 +177,6 @@ make_connection() sleep 1 # Capture client/server attributes from MPTCP connection netlink events - kill_wait $client_evts_pid local client_token local client_port @@ -171,11 +188,10 @@ make_connection() client_port=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$client_evts") client_serverside=$(sed --unbuffered -n 's/.*\(server_side:\)\([[:digit:]]*\).*$/\2/p;q'\ "$client_evts") - kill_wait $server_evts_pid - server_token=$(sed --unbuffered -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") - server_serverside=$(sed --unbuffered -n 's/.*\(server_side:\)\([[:digit:]]*\).*$/\2/p;q'\ - "$server_evts") - rm -f "$client_evts" "$server_evts" "$file" + server_token=$(grep "type:1," "$server_evts" | + sed --unbuffered -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q') + server_serverside=$(grep "type:1," "$server_evts" | + sed --unbuffered -n 's/.*\(server_side:\)\([[:digit:]]*\).*$/\2/p;q') if [ "$client_token" != "" ] && [ "$server_token" != "" ] && [ "$client_serverside" = 0 ] && [ "$server_serverside" = 1 ] @@ -239,13 +255,8 @@ verify_announce_event() test_announce() { - local evts - evts=$(mktemp) # Capture events on the network namespace running the server - :>"$evts" - ip netns exec "$ns1" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$server_evts" # ADD_ADDR using an invalid token should result in no action local invalid_token=$(( client4_token - 1)) @@ -253,7 +264,7 @@ test_announce() $client_addr_id dev ns2eth1 > /dev/null 2>&1 local type - type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") stdbuf -o0 -e0 printf "ADD_ADDR 10.0.2.2 (ns2) => ns1, invalid token \t\t" if [ "$type" = "" ] then @@ -264,71 +275,63 @@ test_announce() fi # ADD_ADDR from the client to server machine reusing the subflow port - :>"$evts" + :>"$server_evts" ip netns exec "$ns2"\ ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id $client_addr_id dev\ ns2eth1 > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.2 (ns2) => ns1, reuse port \t\t" $client_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$server4_token" "10.0.2.2" "$client_addr_id"\ + verify_announce_event $server_evts $ANNOUNCED $server4_token "10.0.2.2" $client_addr_id \ "$client4_port" # ADD_ADDR6 from the client to server machine reusing the subflow port - :>"$evts" + :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann\ dead:beef:2::2 token "$client6_token" id $client_addr_id dev ns2eth1 > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR6 id:%d dead:beef:2::2 (ns2) => ns1, reuse port\t\t" $client_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$server6_token" "dead:beef:2::2"\ + verify_announce_event "$server_evts" "$ANNOUNCED" "$server6_token" "dead:beef:2::2"\ "$client_addr_id" "$client6_port" "v6" # ADD_ADDR from the client to server machine using a new port - :>"$evts" + :>"$server_evts" client_addr_id=$((client_addr_id+1)) ip netns exec "$ns2" ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id\ $client_addr_id dev ns2eth1 port $new4_port > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.2 (ns2) => ns1, new port \t\t\t" $client_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$server4_token" "10.0.2.2"\ + verify_announce_event "$server_evts" "$ANNOUNCED" "$server4_token" "10.0.2.2"\ "$client_addr_id" "$new4_port" - kill_wait $evts_pid - # Capture events on the network namespace running the client - :>"$evts" - ip netns exec "$ns2" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$client_evts" # ADD_ADDR from the server to client machine reusing the subflow port ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ $server_addr_id dev ns1eth2 > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.1 (ns1) => ns2, reuse port \t\t" $server_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ + verify_announce_event "$client_evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ "$server_addr_id" "$app4_port" # ADD_ADDR6 from the server to client machine reusing the subflow port - :>"$evts" + :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann dead:beef:2::1 token "$server6_token" id\ $server_addr_id dev ns1eth2 > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR6 id:%d dead:beef:2::1 (ns1) => ns2, reuse port\t\t" $server_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$client6_token" "dead:beef:2::1"\ + verify_announce_event "$client_evts" "$ANNOUNCED" "$client6_token" "dead:beef:2::1"\ "$server_addr_id" "$app6_port" "v6" # ADD_ADDR from the server to client machine using a new port - :>"$evts" + :>"$client_evts" server_addr_id=$((server_addr_id+1)) ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ $server_addr_id dev ns1eth2 port $new4_port > /dev/null 2>&1 stdbuf -o0 -e0 printf "ADD_ADDR id:%d 10.0.2.1 (ns1) => ns2, new port \t\t\t" $server_addr_id sleep 0.5 - verify_announce_event "$evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ + verify_announce_event "$client_evts" "$ANNOUNCED" "$client4_token" "10.0.2.1"\ "$server_addr_id" "$new4_port" - - kill_wait $evts_pid - rm -f "$evts" } verify_remove_event() @@ -356,14 +359,8 @@ verify_remove_event() test_remove() { - local evts - evts=$(mktemp) - # Capture events on the network namespace running the server - :>"$evts" - ip netns exec "$ns1" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$server_evts" # RM_ADDR using an invalid token should result in no action local invalid_token=$(( client4_token - 1 )) @@ -372,7 +369,7 @@ test_remove() stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1, invalid token \t"\ $client_addr_id local type - type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") if [ "$type" = "" ] then stdbuf -o0 -e0 printf "[OK]\n" @@ -386,7 +383,7 @@ test_remove() $invalid_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1, invalid id \t"\ $invalid_id - type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + type=$(sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") if [ "$type" = "" ] then stdbuf -o0 -e0 printf "[OK]\n" @@ -395,40 +392,35 @@ test_remove() fi # RM_ADDR from the client to server machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl rem token "$client4_token" id\ $client_addr_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1 \t"\ $client_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$server4_token" "$client_addr_id" + verify_remove_event "$server_evts" "$REMOVED" "$server4_token" "$client_addr_id" # RM_ADDR from the client to server machine - :>"$evts" + :>"$server_evts" client_addr_id=$(( client_addr_id - 1 )) ip netns exec "$ns2" ./pm_nl_ctl rem token "$client4_token" id\ $client_addr_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR id:%d ns2 => ns1 \t"\ $client_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$server4_token" "$client_addr_id" + verify_remove_event "$server_evts" "$REMOVED" "$server4_token" "$client_addr_id" # RM_ADDR6 from the client to server machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl rem token "$client6_token" id\ $client_addr_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR6 id:%d ns2 => ns1 \t"\ $client_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$server6_token" "$client_addr_id" - - kill_wait $evts_pid + verify_remove_event "$server_evts" "$REMOVED" "$server6_token" "$client_addr_id" # Capture events on the network namespace running the client - :>"$evts" - ip netns exec "$ns2" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$client_evts" # RM_ADDR from the server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem token "$server4_token" id\ @@ -436,27 +428,24 @@ test_remove() stdbuf -o0 -e0 printf "RM_ADDR id:%d ns1 => ns2 \t"\ $server_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$client4_token" "$server_addr_id" + verify_remove_event "$client_evts" "$REMOVED" "$client4_token" "$server_addr_id" # RM_ADDR from the server to client machine - :>"$evts" + :>"$client_evts" server_addr_id=$(( server_addr_id - 1 )) ip netns exec "$ns1" ./pm_nl_ctl rem token "$server4_token" id\ $server_addr_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR id:%d ns1 => ns2 \t" $server_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$client4_token" "$server_addr_id" + verify_remove_event "$client_evts" "$REMOVED" "$client4_token" "$server_addr_id" # RM_ADDR6 from the server to client machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl rem token "$server6_token" id\ $server_addr_id > /dev/null 2>&1 stdbuf -o0 -e0 printf "RM_ADDR6 id:%d ns1 => ns2 \t" $server_addr_id sleep 0.5 - verify_remove_event "$evts" "$REMOVED" "$client6_token" "$server_addr_id" - - kill_wait $evts_pid - rm -f "$evts" + verify_remove_event "$client_evts" "$REMOVED" "$client6_token" "$server_addr_id" } verify_subflow_events() @@ -532,13 +521,8 @@ verify_subflow_events() test_subflows() { - local evts - evts=$(mktemp) # Capture events on the network namespace running the server - :>"$evts" - ip netns exec "$ns1" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$server_evts" # Attempt to add a listener at 10.0.2.2:<subflow-port> ip netns exec "$ns2" ./pm_nl_ctl listen 10.0.2.2\ @@ -551,25 +535,25 @@ test_subflows() sleep 0.5 # CREATE_SUBFLOW from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2\ rport "$client4_port" token "$server4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$server4_token" "$AF_INET" "10.0.2.1"\ + verify_subflow_events $server_evts $SUB_ESTABLISHED $server4_token $AF_INET "10.0.2.1" \ "10.0.2.2" "$client4_port" "23" "$client_addr_id" "ns1" "ns2" # Delete the listener from the client ns, if one was created kill_wait $listener_pid local sport - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") # DESTROY_SUBFLOW from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip 10.0.2.1 lport "$sport" rip 10.0.2.2 rport\ "$client4_port" token "$server4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ + verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ "10.0.2.2" "$client4_port" "23" "$client_addr_id" "ns1" "ns2" # RM_ADDR from client to server machine @@ -583,31 +567,31 @@ test_subflows() listener_pid=$! # ADD_ADDR6 from client to server machine reusing the subflow port - :>"$evts" + :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann dead:beef:2::2 token "$client6_token" id\ $client_addr_id > /dev/null 2>&1 sleep 0.5 # CREATE_SUBFLOW6 from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip dead:beef:2::1 lid 23 rip\ dead:beef:2::2 rport "$client6_port" token "$server6_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$server6_token" "$AF_INET6"\ + verify_subflow_events "$server_evts" "$SUB_ESTABLISHED" "$server6_token" "$AF_INET6"\ "dead:beef:2::1" "dead:beef:2::2" "$client6_port" "23"\ "$client_addr_id" "ns1" "ns2" # Delete the listener from the client ns, if one was created kill_wait $listener_pid - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") # DESTROY_SUBFLOW6 from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip dead:beef:2::1 lport "$sport" rip\ dead:beef:2::2 rport "$client6_port" token "$server6_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$server6_token" "$AF_INET6"\ + verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server6_token" "$AF_INET6"\ "dead:beef:2::1" "dead:beef:2::2" "$client6_port" "23"\ "$client_addr_id" "ns1" "ns2" @@ -622,44 +606,39 @@ test_subflows() listener_pid=$! # ADD_ADDR from client to server machine using a new port - :>"$evts" + :>"$server_evts" ip netns exec "$ns2" ./pm_nl_ctl ann 10.0.2.2 token "$client4_token" id\ $client_addr_id port $new4_port > /dev/null 2>&1 sleep 0.5 # CREATE_SUBFLOW from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2 rport\ $new4_port token "$server4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$server4_token" "$AF_INET"\ + verify_subflow_events "$server_evts" "$SUB_ESTABLISHED" "$server4_token" "$AF_INET"\ "10.0.2.1" "10.0.2.2" "$new4_port" "23"\ "$client_addr_id" "ns1" "ns2" # Delete the listener from the client ns, if one was created kill_wait $listener_pid - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$server_evts") # DESTROY_SUBFLOW from server to client machine - :>"$evts" + :>"$server_evts" ip netns exec "$ns1" ./pm_nl_ctl dsf lip 10.0.2.1 lport "$sport" rip 10.0.2.2 rport\ $new4_port token "$server4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ + verify_subflow_events "$server_evts" "$SUB_CLOSED" "$server4_token" "$AF_INET" "10.0.2.1"\ "10.0.2.2" "$new4_port" "23" "$client_addr_id" "ns1" "ns2" # RM_ADDR from client to server machine ip netns exec "$ns2" ./pm_nl_ctl rem id $client_addr_id token\ "$client4_token" > /dev/null 2>&1 - kill_wait $evts_pid - # Capture events on the network namespace running the client - :>"$evts" - ip netns exec "$ns2" ./pm_nl_ctl events >> "$evts" 2>&1 & - evts_pid=$! - sleep 0.5 + :>"$client_evts" # Attempt to add a listener at 10.0.2.1:<subflow-port> ip netns exec "$ns1" ./pm_nl_ctl listen 10.0.2.1\ @@ -672,24 +651,24 @@ test_subflows() sleep 0.5 # CREATE_SUBFLOW from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ $app4_port token "$client4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$client4_token" "$AF_INET" "10.0.2.2"\ + verify_subflow_events $client_evts $SUB_ESTABLISHED $client4_token $AF_INET "10.0.2.2"\ "10.0.2.1" "$app4_port" "23" "$server_addr_id" "ns2" "ns1" # Delete the listener from the server ns, if one was created kill_wait $listener_pid - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$client_evts") # DESTROY_SUBFLOW from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ $app4_port token "$client4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ + verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ "10.0.2.1" "$app4_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR from server to client machine @@ -703,17 +682,17 @@ test_subflows() listener_pid=$! # ADD_ADDR6 from server to client machine reusing the subflow port - :>"$evts" + :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann dead:beef:2::1 token "$server6_token" id\ $server_addr_id > /dev/null 2>&1 sleep 0.5 # CREATE_SUBFLOW6 from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip dead:beef:2::2 lid 23 rip\ dead:beef:2::1 rport $app6_port token "$client6_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$client6_token"\ + verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client6_token"\ "$AF_INET6" "dead:beef:2::2"\ "dead:beef:2::1" "$app6_port" "23"\ "$server_addr_id" "ns2" "ns1" @@ -721,14 +700,14 @@ test_subflows() # Delete the listener from the server ns, if one was created kill_wait $listener_pid - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$client_evts") # DESTROY_SUBFLOW6 from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip dead:beef:2::2 lport "$sport" rip\ dead:beef:2::1 rport $app6_port token "$client6_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$client6_token" "$AF_INET6" "dead:beef:2::2"\ + verify_subflow_events $client_evts $SUB_CLOSED $client6_token $AF_INET6 "dead:beef:2::2"\ "dead:beef:2::1" "$app6_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR6 from server to client machine @@ -742,38 +721,35 @@ test_subflows() listener_pid=$! # ADD_ADDR from server to client machine using a new port - :>"$evts" + :>"$client_evts" ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server4_token" id\ $server_addr_id port $new4_port > /dev/null 2>&1 sleep 0.5 # CREATE_SUBFLOW from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ $new4_port token "$client4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_ESTABLISHED" "$client4_token" "$AF_INET"\ + verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client4_token" "$AF_INET"\ "10.0.2.2" "10.0.2.1" "$new4_port" "23" "$server_addr_id" "ns2" "ns1" # Delete the listener from the server ns, if one was created kill_wait $listener_pid - sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts") + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$client_evts") # DESTROY_SUBFLOW from client to server machine - :>"$evts" + :>"$client_evts" ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ $new4_port token "$client4_token" > /dev/null 2>&1 sleep 0.5 - verify_subflow_events "$evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ + verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client4_token" "$AF_INET" "10.0.2.2"\ "10.0.2.1" "$new4_port" "23" "$server_addr_id" "ns2" "ns1" # RM_ADDR from server to client machine ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ "$server4_token" > /dev/null 2>&1 - - kill_wait $evts_pid - rm -f "$evts" } test_prio() @@ -807,11 +783,85 @@ test_prio() fi } +verify_listener_events() +{ + local evt=$1 + local e_type=$2 + local e_family=$3 + local e_saddr=$4 + local e_sport=$5 + local type + local family + local saddr + local sport + + if [ $e_type = $LISTENER_CREATED ]; then + stdbuf -o0 -e0 printf "CREATE_LISTENER %s:%s\t\t\t\t\t"\ + $e_saddr $e_sport + elif [ $e_type = $LISTENER_CLOSED ]; then + stdbuf -o0 -e0 printf "CLOSE_LISTENER %s:%s\t\t\t\t\t"\ + $e_saddr $e_sport + fi + + type=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q') + family=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q') + sport=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q') + if [ $family ] && [ $family = $AF_INET6 ]; then + saddr=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q') + else + saddr=$(grep "type:$e_type," $evt | + sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q') + fi + + if [ $type ] && [ $type = $e_type ] && + [ $family ] && [ $family = $e_family ] && + [ $saddr ] && [ $saddr = $e_saddr ] && + [ $sport ] && [ $sport = $e_sport ]; then + stdbuf -o0 -e0 printf "[OK]\n" + return 0 + fi + stdbuf -o0 -e0 printf "[FAIL]\n" + exit 1 +} + +test_listener() +{ + # Capture events on the network namespace running the client + :>$client_evts + + # Attempt to add a listener at 10.0.2.2:<subflow-port> + ip netns exec $ns2 ./pm_nl_ctl listen 10.0.2.2\ + $client4_port > /dev/null 2>&1 & + local listener_pid=$! + + verify_listener_events $client_evts $LISTENER_CREATED $AF_INET 10.0.2.2 $client4_port + + # ADD_ADDR from client to server machine reusing the subflow port + ip netns exec $ns2 ./pm_nl_ctl ann 10.0.2.2 token $client4_token id\ + $client_addr_id > /dev/null 2>&1 + sleep 0.5 + + # CREATE_SUBFLOW from server to client machine + ip netns exec $ns1 ./pm_nl_ctl csf lip 10.0.2.1 lid 23 rip 10.0.2.2\ + rport $client4_port token $server4_token > /dev/null 2>&1 + sleep 0.5 + + # Delete the listener from the client ns, if one was created + kill_wait $listener_pid + + verify_listener_events $client_evts $LISTENER_CLOSED $AF_INET 10.0.2.2 $client4_port +} + make_connection make_connection "v6" test_announce test_remove test_subflows test_prio +test_listener exit 0 diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh index 0900c5438fbb..275491be3da2 100755 --- a/tools/testing/selftests/net/rtnetlink.sh +++ b/tools/testing/selftests/net/rtnetlink.sh @@ -782,7 +782,7 @@ kci_test_ipsec_offload() tmpl proto esp src $srcip dst $dstip spi 9 \ mode transport reqid 42 check_err $? - ip x p add dir out src $dstip/24 dst $srcip/24 \ + ip x p add dir in src $dstip/24 dst $srcip/24 \ tmpl proto esp src $dstip dst $srcip spi 9 \ mode transport reqid 42 check_err $? diff --git a/tools/testing/selftests/net/toeplitz.sh b/tools/testing/selftests/net/toeplitz.sh index 0a49907cd4fe..da5bfd834eff 100755 --- a/tools/testing/selftests/net/toeplitz.sh +++ b/tools/testing/selftests/net/toeplitz.sh @@ -32,7 +32,7 @@ DEV="eth0" # This is determined by reading the RSS indirection table using ethtool. get_rss_cfg_num_rxqs() { echo $(ethtool -x "${DEV}" | - egrep [[:space:]]+[0-9]+:[[:space:]]+ | + grep -E [[:space:]]+[0-9]+:[[:space:]]+ | cut -d: -f2- | awk '{$1=$1};1' | tr ' ' '\n' | diff --git a/tools/vm/slabinfo-gnuplot.sh b/tools/vm/slabinfo-gnuplot.sh index 26e193ffd2a2..873a892147e5 100644 --- a/tools/vm/slabinfo-gnuplot.sh +++ b/tools/vm/slabinfo-gnuplot.sh @@ -150,7 +150,7 @@ do_preprocess() let lines=3 out=`basename "$in"`"-slabs-by-loss" `cat "$in" | grep -A "$lines" 'Slabs sorted by loss' |\ - egrep -iv '\-\-|Name|Slabs'\ + grep -E -iv '\-\-|Name|Slabs'\ | awk '{print $1" "$4+$2*$3" "$4}' > "$out"` if [ $? -eq 0 ]; then do_slabs_plotting "$out" @@ -159,7 +159,7 @@ do_preprocess() let lines=3 out=`basename "$in"`"-slabs-by-size" `cat "$in" | grep -A "$lines" 'Slabs sorted by size' |\ - egrep -iv '\-\-|Name|Slabs'\ + grep -E -iv '\-\-|Name|Slabs'\ | awk '{print $1" "$4" "$4-$2*$3}' > "$out"` if [ $? -eq 0 ]; then do_slabs_plotting "$out" |