Gitiles
Code Review Sign In
git01.mediatek.com / openwrt / feeds / mtk-openwrt-feeds / 61b20f81461dc60eda4d2ba1f709b616939093b3 / . / autobuild_mac80211_release / target / linux / mediatek / patches-5.4 / 999-3003-add-wed.patch
blob: 09502e951f1351659cc7566213b1e74584efeaf6 [file] [log] [blame]
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]1From cdb36beeb6725bcef3faad499c017c26bc17fd4a Mon Sep 17 00:00:00 2001
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2From: Sujuan Chen <sujuan.chen@mediatek.com>
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]3Date: Mon, 18 Sep 2023 10:56:21 +0800
4Subject: [PATCH 04/22] add wed
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]5
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]6---
7 arch/arm64/boot/dts/mediatek/mt7622.dtsi | 32 +-
8 drivers/net/ethernet/mediatek/Kconfig | 4 +
9 drivers/net/ethernet/mediatek/Makefile | 5 +
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]10 drivers/net/ethernet/mediatek/mtk_eth_soc.c | 39 +-
11 drivers/net/ethernet/mediatek/mtk_eth_soc.h | 5 +-
12 drivers/net/ethernet/mediatek/mtk_ppe.c | 378 +++++++-
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]13 drivers/net/ethernet/mediatek/mtk_ppe.h | 89 +-
14 .../net/ethernet/mediatek/mtk_ppe_debugfs.c | 4 +-
15 .../net/ethernet/mediatek/mtk_ppe_offload.c | 167 +++-
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]16 drivers/net/ethernet/mediatek/mtk_wed.c | 876 ++++++++++++++++++
17 drivers/net/ethernet/mediatek/mtk_wed.h | 135 +++
18 .../net/ethernet/mediatek/mtk_wed_debugfs.c | 175 ++++
19 drivers/net/ethernet/mediatek/mtk_wed_ops.c | 8 +
20 drivers/net/ethernet/mediatek/mtk_wed_regs.h | 251 +++++
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]21 include/linux/netdevice.h | 7 +
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]22 include/linux/soc/mediatek/mtk_wed.h | 131 +++
23 net/core/dev.c | 4 +
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]24 17 files changed, 2207 insertions(+), 103 deletions(-)
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]25 mode change 100755 => 100644 drivers/net/ethernet/mediatek/Kconfig
26 mode change 100755 => 100644 drivers/net/ethernet/mediatek/Makefile
27 mode change 100755 => 100644 drivers/net/ethernet/mediatek/mtk_eth_soc.c
28 mode change 100755 => 100644 drivers/net/ethernet/mediatek/mtk_eth_soc.h
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]29 mode change 100644 => 100755 drivers/net/ethernet/mediatek/mtk_ppe.c
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]30 create mode 100644 drivers/net/ethernet/mediatek/mtk_wed.c
31 create mode 100644 drivers/net/ethernet/mediatek/mtk_wed.h
32 create mode 100644 drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
33 create mode 100644 drivers/net/ethernet/mediatek/mtk_wed_ops.c
34 create mode 100644 drivers/net/ethernet/mediatek/mtk_wed_regs.h
35 create mode 100644 include/linux/soc/mediatek/mtk_wed.h
36
37diff --git a/arch/arm64/boot/dts/mediatek/mt7622.dtsi b/arch/arm64/boot/dts/mediatek/mt7622.dtsi
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]38index 753a97a..e46566a 100644
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]39--- a/arch/arm64/boot/dts/mediatek/mt7622.dtsi
40+++ b/arch/arm64/boot/dts/mediatek/mt7622.dtsi
41@@ -338,7 +338,7 @@
42 };
43
44 cci_control2: slave-if@5000 {
45- compatible = "arm,cci-400-ctrl-if";
46+ compatible = "arm,cci-400-ctrl-if", "syscon";
47 interface-type = "ace";
48 reg = <0x5000 0x1000>;
49 };
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]50@@ -921,6 +921,11 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]51 };
52 };
53
54+ hifsys: syscon@1af00000 {
55+ compatible = "mediatek,mt7622-hifsys", "syscon";
56+ reg = <0 0x1af00000 0 0x70>;
57+ };
58+
59 ethsys: syscon@1b000000 {
60 compatible = "mediatek,mt7622-ethsys",
61 "syscon";
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]62@@ -939,6 +944,26 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]63 #dma-cells = <1>;
64 };
65
66+ pcie_mirror: pcie-mirror@10000400 {
67+ compatible = "mediatek,mt7622-pcie-mirror",
68+ "syscon";
69+ reg = <0 0x10000400 0 0x10>;
70+ };
71+
72+ wed0: wed@1020a000 {
73+ compatible = "mediatek,mt7622-wed",
74+ "syscon";
75+ reg = <0 0x1020a000 0 0x1000>;
76+ interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_LOW>;
77+ };
78+
79+ wed1: wed@1020b000 {
80+ compatible = "mediatek,mt7622-wed",
81+ "syscon";
82+ reg = <0 0x1020b000 0 0x1000>;
83+ interrupts = <GIC_SPI 215 IRQ_TYPE_LEVEL_LOW>;
84+ };
85+
86 eth: ethernet@1b100000 {
87 compatible = "mediatek,mt7622-eth",
88 "mediatek,mt2701-eth",
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]89@@ -965,6 +990,11 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]90 power-domains = <&scpsys MT7622_POWER_DOMAIN_ETHSYS>;
91 mediatek,ethsys = <&ethsys>;
92 mediatek,sgmiisys = <&sgmiisys>;
93+ mediatek,cci-control = <&cci_control2>;
94+ mediatek,wed = <&wed0>, <&wed1>;
95+ mediatek,pcie-mirror = <&pcie_mirror>;
96+ mediatek,hifsys = <&hifsys>;
97+ dma-coherent;
98 #address-cells = <1>;
99 #size-cells = <0>;
100 status = "disabled";
101diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
102old mode 100755
103new mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]104index 7bfc78b..1574af9
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]105--- a/drivers/net/ethernet/mediatek/Kconfig
106+++ b/drivers/net/ethernet/mediatek/Kconfig
107@@ -7,6 +7,10 @@ config NET_VENDOR_MEDIATEK
108
109 if NET_VENDOR_MEDIATEK
110
111+config NET_MEDIATEK_SOC_WED
112+ depends on ARCH_MEDIATEK || COMPILE_TEST
113+ def_bool NET_MEDIATEK_SOC != n
114+
115 config NET_MEDIATEK_SOC
116 tristate "MediaTek SoC Gigabit Ethernet support"
117 select PHYLINK
118diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile
119old mode 100755
120new mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]121index 5f342f4..4090132
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]122--- a/drivers/net/ethernet/mediatek/Makefile
123+++ b/drivers/net/ethernet/mediatek/Makefile
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]124@@ -6,4 +6,9 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]125 obj-$(CONFIG_NET_MEDIATEK_SOC) += mtk_eth.o
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]126 mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_usxgmii.o mtk_eth_path.o mtk_eth_dbg.o mtk_eth_reset.o \
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]127 mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o
128+mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed.o
129+ifdef CONFIG_DEBUG_FS
130+mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_debugfs.o
131+endif
132+obj-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_ops.o
133 obj-$(CONFIG_NET_MEDIATEK_HNAT) += mtk_hnat/
134diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
135old mode 100755
136new mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]137index 9c85e16..88b38e2
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]138--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
139+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]140@@ -21,11 +21,13 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]141 #include <linux/pinctrl/devinfo.h>
142 #include <linux/phylink.h>
developer926f9162022-07-05 10:55:37 +0800[diff] [blame]143 #include <linux/gpio/consumer.h>
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]144+#include <linux/bitfield.h>
145 #include <net/dsa.h>
146
147 #include "mtk_eth_soc.h"
148 #include "mtk_eth_dbg.h"
149 #include "mtk_eth_reset.h"
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]150+#include "mtk_wed.h"
151
152 #if defined(CONFIG_NET_MEDIATEK_HNAT) || defined(CONFIG_NET_MEDIATEK_HNAT_MODULE)
153 #include "mtk_hnat/nf_hnat_mtk.h"
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]154@@ -2191,6 +2193,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
155 unsigned int pktlen, *rxdcsum;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]156 struct net_device *netdev = NULL;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]157 dma_addr_t dma_addr = 0;
158+ u32 hash, reason;
159 int mac = 0;
160
161 if (eth->hwlro)
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]162@@ -2282,6 +2285,17 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]163 skb_checksum_none_assert(skb);
164 skb->protocol = eth_type_trans(skb, netdev);
165
166+ hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
167+ if (hash != MTK_RXD4_FOE_ENTRY) {
168+ hash = jhash_1word(hash, 0);
169+ skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
170+ }
171+
172+ reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
173+ if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
174+ mtk_ppe_check_skb(eth->ppe, skb,
175+ trxd.rxd4 & MTK_RXD4_FOE_ENTRY);
176+
177 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]178 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_RX_V2)) {
179 if (trxd.rxd3 & RX_DMA_VTAG_V2)
180@@ -3735,7 +3749,7 @@ static int mtk_stop(struct net_device *dev)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]181 mtk_dma_free(eth);
182
183 if (eth->soc->offload_version)
184- mtk_ppe_stop(&eth->ppe);
185+ mtk_ppe_stop(eth->ppe);
186
187 return 0;
188 }
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]189@@ -5044,6 +5058,22 @@ static int mtk_probe(struct platform_device *pdev)
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]190 }
191 }
192
193+ for (i = 0;; i++) {
194+ struct device_node *np = of_parse_phandle(pdev->dev.of_node,
195+ "mediatek,wed", i);
196+ static const u32 wdma_regs[] = {
197+ MTK_WDMA0_BASE,
198+ MTK_WDMA1_BASE
199+ };
200+ void __iomem *wdma;
201+
202+ if (!np || i >= ARRAY_SIZE(wdma_regs))
203+ break;
204+
205+ wdma = eth->base + wdma_regs[i];
206+ mtk_wed_add_hw(np, eth, wdma, i);
207+ }
208+
developer46ed9492023-10-31 15:19:05 +0800[diff] [blame]209 if (MTK_HAS_CAPS(eth->soc->caps, MTK_PDMA_INT)) {
210 for (i = 0; i < MTK_PDMA_IRQ_NUM; i++)
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]211
212@@ -5170,10 +5200,11 @@ static int mtk_probe(struct platform_device *pdev)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]213 }
214
215 if (eth->soc->offload_version) {
216- err = mtk_ppe_init(&eth->ppe, eth->dev,
217- eth->base + MTK_ETH_PPE_BASE, 2);
218- if (err)
219+ eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2);
220+ if (!eth->ppe) {
221+ err = -ENOMEM;
222 goto err_free_dev;
223+ }
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]224
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]225 err = mtk_eth_offload_init(eth);
226 if (err)
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]227diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
228old mode 100755
229new mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]230index a87e46d..15337d3
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]231--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
232+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]233@@ -578,6 +578,9 @@
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]234 #define RX_DMA_SPORT_MASK 0x7
developerdca0fde2022-12-14 11:40:35 +0800[diff] [blame]235 #define RX_DMA_SPORT_MASK_V2 0xf
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]236
237+#define MTK_WDMA0_BASE 0x2800
238+#define MTK_WDMA1_BASE 0x2c00
239+
240 /* QDMA descriptor txd4 */
241 #define TX_DMA_CHKSUM (0x7 << 29)
242 #define TX_DMA_TSO BIT(28)
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]243@@ -1859,7 +1862,7 @@ struct mtk_eth {
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]244 spinlock_t syscfg0_lock;
245 struct timer_list mtk_dma_monitor_timer;
246
247- struct mtk_ppe ppe;
248+ struct mtk_ppe *ppe;
249 struct rhashtable flow_table;
250 };
251
252diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
253old mode 100644
254new mode 100755
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]255index 27b5be5..86741bf
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]256--- a/drivers/net/ethernet/mediatek/mtk_ppe.c
257+++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
258@@ -6,9 +6,22 @@
259 #include <linux/iopoll.h>
260 #include <linux/etherdevice.h>
261 #include <linux/platform_device.h>
262+#include <linux/if_ether.h>
263+#include <linux/if_vlan.h>
264+#include <net/dsa.h>
265+#include "mtk_eth_soc.h"
266 #include "mtk_ppe.h"
267 #include "mtk_ppe_regs.h"
268
269+static DEFINE_SPINLOCK(ppe_lock);
270+
271+static const struct rhashtable_params mtk_flow_l2_ht_params = {
272+ .head_offset = offsetof(struct mtk_flow_entry, l2_node),
273+ .key_offset = offsetof(struct mtk_flow_entry, data.bridge),
274+ .key_len = offsetof(struct mtk_foe_bridge, key_end),
275+ .automatic_shrinking = true,
276+};
277+
278 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
279 {
280 writel(val, ppe->base + reg);
281@@ -41,6 +54,11 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
282 return ppe_m32(ppe, reg, val, 0);
283 }
284
285+static u32 mtk_eth_timestamp(struct mtk_eth *eth)
286+{
287+ return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
288+}
289+
290 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
291 {
292 int ret;
293@@ -76,13 +94,6 @@ static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e)
294 u32 hash;
295
296 switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) {
297- case MTK_PPE_PKT_TYPE_BRIDGE:
298- hv1 = e->bridge.src_mac_lo;
299- hv1 ^= ((e->bridge.src_mac_hi & 0xffff) << 16);
300- hv2 = e->bridge.src_mac_hi >> 16;
301- hv2 ^= e->bridge.dest_mac_lo;
302- hv3 = e->bridge.dest_mac_hi;
303- break;
304 case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
305 case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
306 hv1 = e->ipv4.orig.ports;
307@@ -122,6 +133,9 @@ mtk_foe_entry_l2(struct mtk_foe_entry *entry)
308 {
309 int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
310
311+ if (type == MTK_PPE_PKT_TYPE_BRIDGE)
312+ return &entry->bridge.l2;
313+
314 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
315 return &entry->ipv6.l2;
316
317@@ -133,6 +147,9 @@ mtk_foe_entry_ib2(struct mtk_foe_entry *entry)
318 {
319 int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
320
321+ if (type == MTK_PPE_PKT_TYPE_BRIDGE)
322+ return &entry->bridge.ib2;
323+
324 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
325 return &entry->ipv6.ib2;
326
327@@ -167,7 +184,12 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
328 if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
329 entry->ipv6.ports = ports_pad;
330
331- if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
332+ if (type == MTK_PPE_PKT_TYPE_BRIDGE) {
333+ ether_addr_copy(entry->bridge.src_mac, src_mac);
334+ ether_addr_copy(entry->bridge.dest_mac, dest_mac);
335+ entry->bridge.ib2 = val;
336+ l2 = &entry->bridge.l2;
337+ } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
338 entry->ipv6.ib2 = val;
339 l2 = &entry->ipv6.l2;
340 } else {
341@@ -329,32 +351,168 @@ int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid)
342 return 0;
343 }
344
345+int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
346+ int bss, int wcid)
347+{
348+ struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
349+ u32 *ib2 = mtk_foe_entry_ib2(entry);
350+
351+ *ib2 &= ~MTK_FOE_IB2_PORT_MG;
352+ *ib2 |= MTK_FOE_IB2_WDMA_WINFO;
353+ if (wdma_idx)
354+ *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
355+
356+ l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
357+ FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
358+ FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
359+
360+ return 0;
361+}
362+
363 static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry)
364 {
365 return !(entry->ib1 & MTK_FOE_IB1_STATIC) &&
366 FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;
367 }
368
369-int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
370- u16 timestamp)
371+static bool
372+mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data)
373+{
374+ int type, len;
375+
376+ if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
377+ return false;
378+
379+ type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
380+ if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
381+ len = offsetof(struct mtk_foe_entry, ipv6._rsv);
382+ else
383+ len = offsetof(struct mtk_foe_entry, ipv4.ib2);
384+
385+ return !memcmp(&entry->data.data, &data->data, len - 4);
386+}
387+
388+static void
389+__mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
390+{
391+ struct hlist_head *head;
392+ struct hlist_node *tmp;
393+
394+ if (entry->type == MTK_FLOW_TYPE_L2) {
395+ rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node,
396+ mtk_flow_l2_ht_params);
397+
398+ head = &entry->l2_flows;
399+ hlist_for_each_entry_safe(entry, tmp, head, l2_data.list)
400+ __mtk_foe_entry_clear(ppe, entry);
401+ return;
402+ }
403+
404+ hlist_del_init(&entry->list);
405+ if (entry->hash != 0xffff) {
406+ ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE;
407+ ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE,
408+ MTK_FOE_STATE_INVALID);
409+ dma_wmb();
410+ mtk_ppe_cache_clear(ppe);
411+ }
412+ entry->hash = 0xffff;
413+
414+ if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW)
415+ return;
416+
417+ hlist_del_init(&entry->l2_data.list);
418+ kfree(entry);
419+}
420+
421+static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
422+{
423+ u16 timestamp;
424+ u16 now;
425+
426+ now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP;
427+ timestamp = ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
428+
429+ if (timestamp > now)
430+ return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now;
431+ else
432+ return now - timestamp;
433+}
434+
435+static void
436+mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
437 {
438+ struct mtk_flow_entry *cur;
439 struct mtk_foe_entry *hwe;
440- u32 hash;
441+ struct hlist_node *tmp;
442+ int idle;
443+
444+ idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
445+ hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) {
446+ int cur_idle;
447+ u32 ib1;
448+
449+ hwe = &ppe->foe_table[cur->hash];
450+ ib1 = READ_ONCE(hwe->ib1);
451+
452+ if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) {
453+ cur->hash = 0xffff;
454+ __mtk_foe_entry_clear(ppe, cur);
455+ continue;
456+ }
457+
458+ cur_idle = __mtk_foe_entry_idle_time(ppe, ib1);
459+ if (cur_idle >= idle)
460+ continue;
461+
462+ idle = cur_idle;
463+ entry->data.ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
464+ entry->data.ib1 |= hwe->ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
465+ }
466+}
467+
468+static void
469+mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
470+{
471+ struct mtk_foe_entry *hwe;
472+ struct mtk_foe_entry foe;
473+
474+ spin_lock_bh(&ppe_lock);
475+
476+ if (entry->type == MTK_FLOW_TYPE_L2) {
477+ mtk_flow_entry_update_l2(ppe, entry);
478+ goto out;
479+ }
480+
481+ if (entry->hash == 0xffff)
482+ goto out;
483+
484+ hwe = &ppe->foe_table[entry->hash];
485+ memcpy(&foe, hwe, sizeof(foe));
486+ if (!mtk_flow_entry_match(entry, &foe)) {
487+ entry->hash = 0xffff;
488+ goto out;
489+ }
490+
491+ entry->data.ib1 = foe.ib1;
492+
493+out:
494+ spin_unlock_bh(&ppe_lock);
495+}
496+
497+static void
498+__mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
499+ u16 hash)
500+{
501+ struct mtk_foe_entry *hwe;
502+ u16 timestamp;
503
504+ timestamp = mtk_eth_timestamp(ppe->eth);
505 timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
506 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
507 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
508
509- hash = mtk_ppe_hash_entry(entry);
510 hwe = &ppe->foe_table[hash];
511- if (!mtk_foe_entry_usable(hwe)) {
512- hwe++;
513- hash++;
514-
515- if (!mtk_foe_entry_usable(hwe))
516- return -ENOSPC;
517- }
518-
519 memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
520 wmb();
521 hwe->ib1 = entry->ib1;
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]522@@ -362,32 +520,201 @@ int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]523 dma_wmb();
524
525 mtk_ppe_cache_clear(ppe);
526+}
527
528- return hash;
529+void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
530+{
531+ spin_lock_bh(&ppe_lock);
532+ __mtk_foe_entry_clear(ppe, entry);
533+ spin_unlock_bh(&ppe_lock);
534+}
535+
536+static int
537+mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
538+{
539+ entry->type = MTK_FLOW_TYPE_L2;
540+
541+ return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node,
542+ mtk_flow_l2_ht_params);
543+}
544+
545+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
546+{
547+ int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
548+ u32 hash;
549+
550+ if (type == MTK_PPE_PKT_TYPE_BRIDGE)
551+ return mtk_foe_entry_commit_l2(ppe, entry);
552+
553+ hash = mtk_ppe_hash_entry(&entry->data);
554+ entry->hash = 0xffff;
555+ spin_lock_bh(&ppe_lock);
556+ hlist_add_head(&entry->list, &ppe->foe_flow[hash / 4]);
557+ spin_unlock_bh(&ppe_lock);
558+
559+ return 0;
560+}
561+
562+static void
563+mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
564+ u16 hash)
565+{
566+ struct mtk_flow_entry *flow_info;
567+ struct mtk_foe_entry foe, *hwe;
568+ struct mtk_foe_mac_info *l2;
569+ u32 ib1_mask = MTK_FOE_IB1_PACKET_TYPE | MTK_FOE_IB1_UDP;
570+ int type;
571+
572+ flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end),
573+ GFP_ATOMIC);
574+ if (!flow_info)
575+ return;
576+
577+ flow_info->l2_data.base_flow = entry;
578+ flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW;
579+ flow_info->hash = hash;
580+ hlist_add_head(&flow_info->list, &ppe->foe_flow[hash / 4]);
581+ hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows);
582+
583+ hwe = &ppe->foe_table[hash];
584+ memcpy(&foe, hwe, sizeof(foe));
585+ foe.ib1 &= ib1_mask;
586+ foe.ib1 |= entry->data.ib1 & ~ib1_mask;
587+
588+ l2 = mtk_foe_entry_l2(&foe);
589+ memcpy(l2, &entry->data.bridge.l2, sizeof(*l2));
590+
591+ type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, foe.ib1);
592+ if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT)
593+ memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new));
594+ else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP)
595+ l2->etype = ETH_P_IPV6;
596+
597+ *mtk_foe_entry_ib2(&foe) = entry->data.bridge.ib2;
598+
599+ __mtk_foe_entry_commit(ppe, &foe, hash);
600 }
601
602-int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base,
603+void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
604+{
605+ struct hlist_head *head = &ppe->foe_flow[hash / 4];
606+ struct mtk_foe_entry *hwe = &ppe->foe_table[hash];
607+ struct mtk_flow_entry *entry;
608+ struct mtk_foe_bridge key = {};
609+ struct hlist_node *n;
610+ struct ethhdr *eh;
611+ bool found = false;
612+ u8 *tag;
613+
614+ spin_lock_bh(&ppe_lock);
615+
616+ if (hash >= MTK_PPE_ENTRIES)
617+ goto out;
618+
619+ if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND)
620+ goto out;
621+
622+ hlist_for_each_entry_safe(entry, n, head, list) {
623+ if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) {
624+ if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) ==
625+ MTK_FOE_STATE_BIND))
626+ continue;
627+
628+ entry->hash = 0xffff;
629+ __mtk_foe_entry_clear(ppe, entry);
630+ continue;
631+ }
632+
633+ if (found || !mtk_flow_entry_match(entry, hwe)) {
634+ if (entry->hash != 0xffff)
635+ entry->hash = 0xffff;
636+ continue;
637+ }
638+
639+ entry->hash = hash;
640+ __mtk_foe_entry_commit(ppe, &entry->data, hash);
641+ found = true;
642+ }
643+
644+ if (found)
645+ goto out;
646+
647+ if (!skb)
648+ goto out;
649+
650+ eh = eth_hdr(skb);
651+ ether_addr_copy(key.dest_mac, eh->h_dest);
652+ ether_addr_copy(key.src_mac, eh->h_source);
653+ tag = skb->data - 2;
654+ key.vlan = 0;
655+ switch (skb->protocol) {
656+#if IS_ENABLED(CONFIG_NET_DSA)
657+ case htons(ETH_P_XDSA):
658+ if (!netdev_uses_dsa(skb->dev) ||
659+ skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
660+ goto out;
661+
662+ tag += 4;
663+ if (get_unaligned_be16(tag) != ETH_P_8021Q)
664+ break;
665+
666+ fallthrough;
667+#endif
668+ case htons(ETH_P_8021Q):
669+ key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK;
670+ break;
671+ default:
672+ break;
673+ }
674+
675+ entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params);
676+ if (!entry)
677+ goto out;
678+
679+ mtk_foe_entry_commit_subflow(ppe, entry, hash);
680+
681+out:
682+ spin_unlock_bh(&ppe_lock);
683+}
684+
685+int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
686+{
687+ mtk_flow_entry_update(ppe, entry);
688+
689+ return __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
690+}
691+
692+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
693 int version)
694 {
695+ struct device *dev = eth->dev;
696 struct mtk_foe_entry *foe;
697+ struct mtk_ppe *ppe;
698+
699+ ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
700+ if (!ppe)
701+ return NULL;
702+
703+ rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params);
704
705 /* need to allocate a separate device, since it PPE DMA access is
706 * not coherent.
707 */
708 ppe->base = base;
709+ ppe->eth = eth;
710 ppe->dev = dev;
711 ppe->version = version;
712
713 foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe),
714 &ppe->foe_phys, GFP_KERNEL);
715 if (!foe)
716- return -ENOMEM;
717+ return NULL;
718
719 ppe->foe_table = foe;
720
721 mtk_ppe_debugfs_init(ppe);
722
723- return 0;
724+ return ppe;
725 }
726
727 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]728@@ -395,7 +722,7 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]729 static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
730 int i, k;
731
732- memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(*ppe->foe_table));
733+ memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(ppe->foe_table));
734
735 if (!IS_ENABLED(CONFIG_SOC_MT7621))
736 return;
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]737@@ -443,7 +770,6 @@ int mtk_ppe_start(struct mtk_ppe *ppe)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]738 MTK_PPE_FLOW_CFG_IP4_NAT |
739 MTK_PPE_FLOW_CFG_IP4_NAPT |
740 MTK_PPE_FLOW_CFG_IP4_DSLITE |
741- MTK_PPE_FLOW_CFG_L2_BRIDGE |
742 MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
743 ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
744
745diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]746index 242fb8f..1f5cf1c 100644
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]747--- a/drivers/net/ethernet/mediatek/mtk_ppe.h
748+++ b/drivers/net/ethernet/mediatek/mtk_ppe.h
749@@ -6,6 +6,7 @@
750
751 #include <linux/kernel.h>
752 #include <linux/bitfield.h>
753+#include <linux/rhashtable.h>
754
755 #define MTK_ETH_PPE_BASE 0xc00
756
757@@ -48,9 +49,9 @@ enum {
758 #define MTK_FOE_IB2_DEST_PORT GENMASK(7, 5)
759 #define MTK_FOE_IB2_MULTICAST BIT(8)
760
761-#define MTK_FOE_IB2_WHNAT_QID2 GENMASK(13, 12)
762-#define MTK_FOE_IB2_WHNAT_DEVIDX BIT(16)
763-#define MTK_FOE_IB2_WHNAT_NAT BIT(17)
764+#define MTK_FOE_IB2_WDMA_QID2 GENMASK(13, 12)
765+#define MTK_FOE_IB2_WDMA_DEVIDX BIT(16)
766+#define MTK_FOE_IB2_WDMA_WINFO BIT(17)
767
768 #define MTK_FOE_IB2_PORT_MG GENMASK(17, 12)
769
770@@ -58,9 +59,9 @@ enum {
771
772 #define MTK_FOE_IB2_DSCP GENMASK(31, 24)
773
774-#define MTK_FOE_VLAN2_WHNAT_BSS GEMMASK(5, 0)
775-#define MTK_FOE_VLAN2_WHNAT_WCID GENMASK(13, 6)
776-#define MTK_FOE_VLAN2_WHNAT_RING GENMASK(15, 14)
777+#define MTK_FOE_VLAN2_WINFO_BSS GENMASK(5, 0)
778+#define MTK_FOE_VLAN2_WINFO_WCID GENMASK(13, 6)
779+#define MTK_FOE_VLAN2_WINFO_RING GENMASK(15, 14)
780
781 enum {
782 MTK_FOE_STATE_INVALID,
783@@ -84,19 +85,16 @@ struct mtk_foe_mac_info {
784 u16 src_mac_lo;
785 };
786
787+/* software-only entry type */
788 struct mtk_foe_bridge {
789- u32 dest_mac_hi;
790+ u8 dest_mac[ETH_ALEN];
791+ u8 src_mac[ETH_ALEN];
792+ u16 vlan;
793
794- u16 src_mac_lo;
795- u16 dest_mac_lo;
796-
797- u32 src_mac_hi;
798+ struct {} key_end;
799
800 u32 ib2;
801
802- u32 _rsv[5];
803-
804- u32 udf_tsid;
805 struct mtk_foe_mac_info l2;
806 };
807
808@@ -235,7 +233,37 @@ enum {
809 MTK_PPE_CPU_REASON_INVALID = 0x1f,
810 };
811
812+enum {
813+ MTK_FLOW_TYPE_L4,
814+ MTK_FLOW_TYPE_L2,
815+ MTK_FLOW_TYPE_L2_SUBFLOW,
816+};
817+
818+struct mtk_flow_entry {
819+ union {
820+ struct hlist_node list;
821+ struct {
822+ struct rhash_head l2_node;
823+ struct hlist_head l2_flows;
824+ };
825+ };
826+ u8 type;
827+ s8 wed_index;
828+ u16 hash;
829+ union {
830+ struct mtk_foe_entry data;
831+ struct {
832+ struct mtk_flow_entry *base_flow;
833+ struct hlist_node list;
834+ struct {} end;
835+ } l2_data;
836+ };
837+ struct rhash_head node;
838+ unsigned long cookie;
839+};
840+
841 struct mtk_ppe {
842+ struct mtk_eth *eth;
843 struct device *dev;
844 void __iomem *base;
845 int version;
846@@ -243,19 +271,35 @@ struct mtk_ppe {
847 struct mtk_foe_entry *foe_table;
848 dma_addr_t foe_phys;
849
850+ u16 foe_check_time[MTK_PPE_ENTRIES];
851+ struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2];
852+
853+ struct rhashtable l2_flows;
854+
855 void *acct_table;
856 };
857
858-int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base,
859- int version);
860+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version);
861 int mtk_ppe_start(struct mtk_ppe *ppe);
862 int mtk_ppe_stop(struct mtk_ppe *ppe);
863
864+void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash);
865+
866 static inline void
867-mtk_foe_entry_clear(struct mtk_ppe *ppe, u16 hash)
868+mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
869 {
870- ppe->foe_table[hash].ib1 = 0;
871- dma_wmb();
872+ u16 now, diff;
873+
874+ if (!ppe)
875+ return;
876+
877+ now = (u16)jiffies;
878+ diff = now - ppe->foe_check_time[hash];
879+ if (diff < HZ / 10)
880+ return;
881+
882+ ppe->foe_check_time[hash] = now;
883+ __mtk_ppe_check_skb(ppe, skb, hash);
884 }
885
886 static inline int
887@@ -281,8 +325,11 @@ int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
888 int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port);
889 int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid);
890 int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);
891-int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
892- u16 timestamp);
893+int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
894+ int bss, int wcid);
895+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
896+void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
897+int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
898 int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);
899
900 #endif
901diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]902index d4b4823..a591ab1 100644
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]903--- a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
904+++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
905@@ -32,7 +32,6 @@ static const char *mtk_foe_pkt_type_str(int type)
906 static const char * const type_str[] = {
907 [MTK_PPE_PKT_TYPE_IPV4_HNAPT] = "IPv4 5T",
908 [MTK_PPE_PKT_TYPE_IPV4_ROUTE] = "IPv4 3T",
909- [MTK_PPE_PKT_TYPE_BRIDGE] = "L2",
910 [MTK_PPE_PKT_TYPE_IPV4_DSLITE] = "DS-LITE",
911 [MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T] = "IPv6 3T",
912 [MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T] = "IPv6 5T",
913@@ -207,6 +206,9 @@ int mtk_ppe_debugfs_init(struct mtk_ppe *ppe)
914 struct dentry *root;
915
916 root = debugfs_create_dir("mtk_ppe", NULL);
917+ if (!root)
918+ return -ENOMEM;
919+
920 debugfs_create_file("entries", S_IRUGO, root, ppe, &fops_all);
921 debugfs_create_file("bind", S_IRUGO, root, ppe, &fops_bind);
922
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]923diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]924index 1380ef0..8a28572 100644
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]925--- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
926+++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
927@@ -11,6 +11,7 @@
928 #include <net/pkt_cls.h>
929 #include <net/dsa.h>
930 #include "mtk_eth_soc.h"
931+#include "mtk_wed.h"
932
933 struct mtk_flow_data {
934 struct ethhdr eth;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]935@@ -30,6 +31,8 @@ struct mtk_flow_data {
936 __be16 src_port;
937 __be16 dst_port;
938
939+ u16 vlan_in;
940+
941 struct {
942 u16 id;
943 __be16 proto;
944@@ -41,12 +44,6 @@ struct mtk_flow_data {
945 } pppoe;
946 };
947
948-struct mtk_flow_entry {
949- struct rhash_head node;
950- unsigned long cookie;
951- u16 hash;
952-};
953-
954 static const struct rhashtable_params mtk_flow_ht_params = {
955 .head_offset = offsetof(struct mtk_flow_entry, node),
956 .key_offset = offsetof(struct mtk_flow_entry, cookie),
957@@ -54,12 +51,6 @@ static const struct rhashtable_params mtk_flow_ht_params = {
958 .automatic_shrinking = true,
959 };
960
961-static u32
962-mtk_eth_timestamp(struct mtk_eth *eth)
963-{
964- return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
965-}
966-
967 static int
968 mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
969 bool egress)
970@@ -94,6 +85,35 @@ mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth)
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]971 memcpy(dest, src, act->mangle.mask ? 2 : 4);
972 }
973
974+static int
975+mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_info *info)
976+{
977+ struct net_device_path_ctx ctx = {
978+ .dev = dev,
979+ };
980+ struct net_device_path path = {};
981+
982+ if (!IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED))
983+ return -1;
984+
985+ if (!dev->netdev_ops->ndo_fill_forward_path)
986+ return -1;
987+
988+ memcpy(ctx.daddr, addr, sizeof(ctx.daddr));
989+ if (dev->netdev_ops->ndo_fill_forward_path(&ctx, &path))
990+ return -1;
991+
992+ if (path.type != DEV_PATH_MTK_WDMA)
993+ return -1;
994+
995+ info->wdma_idx = path.mtk_wdma.wdma_idx;
996+ info->queue = path.mtk_wdma.queue;
997+ info->bss = path.mtk_wdma.bss;
998+ info->wcid = path.mtk_wdma.wcid;
999+
1000+ return 0;
1001+}
1002+
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1003
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1004 static int
1005 mtk_flow_mangle_ports(const struct flow_action_entry *act,
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1006@@ -163,10 +183,20 @@ mtk_flow_get_dsa_port(struct net_device **dev)
1007
1008 static int
1009 mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
1010- struct net_device *dev)
1011+ struct net_device *dev, const u8 *dest_mac,
1012+ int *wed_index)
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1013 {
1014+ struct mtk_wdma_info info = {};
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1015 int pse_port, dsa_port;
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1016
1017+ if (mtk_flow_get_wdma_info(dev, dest_mac, &info) == 0) {
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1018+ mtk_foe_entry_set_wdma(foe, info.wdma_idx, info.queue, info.bss,
1019+ info.wcid);
1020+ pse_port = PSE_PPE0_PORT;
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1021+ *wed_index = info.wdma_idx;
1022+ goto out;
1023+ }
1024+
1025 dsa_port = mtk_flow_get_dsa_port(&dev);
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1026 if (dsa_port >= 0)
1027 mtk_foe_entry_set_dsa(foe, dsa_port);
1028@@ -178,6 +208,7 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
1029 else
1030 return -EOPNOTSUPP;
1031
1032+out:
1033 mtk_foe_entry_set_pse_port(foe, pse_port);
1034
1035 return 0;
1036@@ -193,11 +224,10 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1037 struct net_device *odev = NULL;
1038 struct mtk_flow_entry *entry;
1039 int offload_type = 0;
1040+ int wed_index = -1;
1041 u16 addr_type = 0;
1042- u32 timestamp;
1043 u8 l4proto = 0;
1044 int err = 0;
1045- int hash;
1046 int i;
1047
1048 if (rhashtable_lookup(&eth->flow_table, &f->cookie, mtk_flow_ht_params))
1049@@ -229,9 +259,45 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1050 return -EOPNOTSUPP;
1051 }
1052
1053+ switch (addr_type) {
1054+ case 0:
1055+ offload_type = MTK_PPE_PKT_TYPE_BRIDGE;
1056+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1057+ struct flow_match_eth_addrs match;
1058+
1059+ flow_rule_match_eth_addrs(rule, &match);
1060+ memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN);
1061+ memcpy(data.eth.h_source, match.key->src, ETH_ALEN);
1062+ } else {
1063+ return -EOPNOTSUPP;
1064+ }
1065+
1066+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
1067+ struct flow_match_vlan match;
1068+
1069+ flow_rule_match_vlan(rule, &match);
1070+
1071+ if (match.key->vlan_tpid != cpu_to_be16(ETH_P_8021Q))
1072+ return -EOPNOTSUPP;
1073+
1074+ data.vlan_in = match.key->vlan_id;
1075+ }
1076+ break;
1077+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1078+ offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT;
1079+ break;
1080+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1081+ offload_type = MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T;
1082+ break;
1083+ default:
1084+ return -EOPNOTSUPP;
1085+ }
1086+
1087 flow_action_for_each(i, act, &rule->action) {
1088 switch (act->id) {
1089 case FLOW_ACTION_MANGLE:
1090+ if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE)
1091+ return -EOPNOTSUPP;
1092 if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH)
1093 mtk_flow_offload_mangle_eth(act, &data.eth);
1094 break;
1095@@ -263,17 +329,6 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1096 }
1097 }
1098
1099- switch (addr_type) {
1100- case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1101- offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT;
1102- break;
1103- case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1104- offload_type = MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T;
1105- break;
1106- default:
1107- return -EOPNOTSUPP;
1108- }
1109-
1110 if (!is_valid_ether_addr(data.eth.h_source) ||
1111 !is_valid_ether_addr(data.eth.h_dest))
1112 return -EINVAL;
1113@@ -287,10 +342,13 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1114 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
1115 struct flow_match_ports ports;
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1116
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1117+ if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE)
1118+ return -EOPNOTSUPP;
1119+
1120 flow_rule_match_ports(rule, &ports);
1121 data.src_port = ports.key->src;
1122 data.dst_port = ports.key->dst;
1123- } else {
1124+ } else if (offload_type != MTK_PPE_PKT_TYPE_BRIDGE) {
1125 return -EOPNOTSUPP;
1126 }
1127
1128@@ -320,6 +378,9 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1129 if (act->id != FLOW_ACTION_MANGLE)
1130 continue;
1131
1132+ if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE)
1133+ return -EOPNOTSUPP;
1134+
1135 switch (act->mangle.htype) {
1136 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
1137 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
1138@@ -345,6 +406,9 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1139 return err;
1140 }
1141
1142+ if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE)
1143+ foe.bridge.vlan = data.vlan_in;
1144+
1145 if (data.vlan.num == 1) {
1146 if (data.vlan.proto != htons(ETH_P_8021Q))
1147 return -EOPNOTSUPP;
1148@@ -354,33 +418,38 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
1149 if (data.pppoe.num == 1)
1150 mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid);
1151
1152- err = mtk_flow_set_output_device(eth, &foe, odev);
1153+ err = mtk_flow_set_output_device(eth, &foe, odev, data.eth.h_dest,
1154+ &wed_index);
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1155 if (err)
1156 return err;
1157
1158+ if (wed_index >= 0 && (err = mtk_wed_flow_add(wed_index)) < 0)
1159+ return err;
1160+
1161 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1162 if (!entry)
1163 return -ENOMEM;
1164
1165 entry->cookie = f->cookie;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1166- timestamp = mtk_eth_timestamp(eth);
1167- hash = mtk_foe_entry_commit(&eth->ppe, &foe, timestamp);
1168- if (hash < 0) {
1169- err = hash;
1170+ memcpy(&entry->data, &foe, sizeof(entry->data));
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1171+ entry->wed_index = wed_index;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1172+
1173+ if (mtk_foe_entry_commit(eth->ppe, entry) < 0)
1174 goto free;
1175- }
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1176
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1177- entry->hash = hash;
1178 err = rhashtable_insert_fast(&eth->flow_table, &entry->node,
1179 mtk_flow_ht_params);
1180 if (err < 0)
1181- goto clear_flow;
1182+ goto clear;
1183
1184 return 0;
1185-clear_flow:
1186- mtk_foe_entry_clear(&eth->ppe, hash);
1187+
1188+clear:
1189+ mtk_foe_entry_clear(eth->ppe, entry);
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1190 free:
1191 kfree(entry);
1192+ if (wed_index >= 0)
1193+ mtk_wed_flow_remove(wed_index);
1194 return err;
1195 }
1196
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1197@@ -394,9 +463,11 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f)
1198 if (!entry)
1199 return -ENOENT;
1200
1201- mtk_foe_entry_clear(&eth->ppe, entry->hash);
1202+ mtk_foe_entry_clear(eth->ppe, entry);
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1203 rhashtable_remove_fast(&eth->flow_table, &entry->node,
1204 mtk_flow_ht_params);
1205+ if (entry->wed_index >= 0)
1206+ mtk_wed_flow_remove(entry->wed_index);
1207 kfree(entry);
1208
1209 return 0;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1210@@ -406,7 +477,6 @@ static int
1211 mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
1212 {
1213 struct mtk_flow_entry *entry;
1214- int timestamp;
1215 u32 idle;
1216
1217 entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
1218@@ -414,11 +484,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
1219 if (!entry)
1220 return -ENOENT;
1221
1222- timestamp = mtk_foe_entry_timestamp(&eth->ppe, entry->hash);
1223- if (timestamp < 0)
1224- return -ETIMEDOUT;
1225-
1226- idle = mtk_eth_timestamp(eth) - timestamp;
1227+ idle = mtk_foe_entry_idle_time(eth->ppe, entry);
1228 f->stats.lastused = jiffies - idle * HZ;
1229
1230 return 0;
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]1231@@ -471,7 +537,7 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1232 flow_setup_cb_t *cb;
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]1233 int err = 0;
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1234
1235- if (!eth->ppe.foe_table)
1236+ if (!eth->ppe || !eth->ppe->foe_table)
1237 return -EOPNOTSUPP;
1238
1239 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]1240@@ -520,15 +586,18 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]1241 int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
1242 void *type_data)
1243 {
1244- if (type == TC_SETUP_FT)
1245+ switch (type) {
1246+ case TC_SETUP_BLOCK:
1247+ case TC_SETUP_FT:
1248 return mtk_eth_setup_tc_block(dev, type_data);
1249-
1250- return -EOPNOTSUPP;
1251+ default:
1252+ return -EOPNOTSUPP;
1253+ }
1254 }
1255
1256 int mtk_eth_offload_init(struct mtk_eth *eth)
1257 {
1258- if (!eth->ppe.foe_table)
1259+ if (!eth->ppe || !eth->ppe->foe_table)
1260 return 0;
1261
1262 return rhashtable_init(&eth->flow_table, &mtk_flow_ht_params);
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1263diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c
1264new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]1265index 0000000..ea1cbdf
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]1266--- /dev/null
1267+++ b/drivers/net/ethernet/mediatek/mtk_wed.c
1268@@ -0,0 +1,876 @@
1269+// SPDX-License-Identifier: GPL-2.0-only
1270+/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */
1271+
1272+#include <linux/kernel.h>
1273+#include <linux/slab.h>
1274+#include <linux/module.h>
1275+#include <linux/bitfield.h>
1276+#include <linux/dma-mapping.h>
1277+#include <linux/skbuff.h>
1278+#include <linux/of_platform.h>
1279+#include <linux/of_address.h>
1280+#include <linux/mfd/syscon.h>
1281+#include <linux/debugfs.h>
1282+#include <linux/iopoll.h>
1283+#include <linux/soc/mediatek/mtk_wed.h>
1284+#include "mtk_eth_soc.h"
1285+#include "mtk_wed_regs.h"
1286+#include "mtk_wed.h"
1287+#include "mtk_ppe.h"
1288+
1289+#define MTK_PCIE_BASE(n) (0x1a143000 + (n) * 0x2000)
1290+
1291+#define MTK_WED_PKT_SIZE 1900
1292+#define MTK_WED_BUF_SIZE 2048
1293+#define MTK_WED_BUF_PER_PAGE (PAGE_SIZE / 2048)
1294+
1295+#define MTK_WED_TX_RING_SIZE 2048
1296+#define MTK_WED_WDMA_RING_SIZE 1024
1297+
1298+static struct mtk_wed_hw *hw_list[2];
1299+static DEFINE_MUTEX(hw_lock);
1300+
1301+static void
1302+wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
1303+{
1304+ regmap_update_bits(dev->hw->regs, reg, mask | val, val);
1305+}
1306+
1307+static void
1308+wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
1309+{
1310+ return wed_m32(dev, reg, 0, mask);
1311+}
1312+
1313+static void
1314+wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)
1315+{
1316+ return wed_m32(dev, reg, mask, 0);
1317+}
1318+
1319+static void
1320+wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
1321+{
1322+ wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val);
1323+}
1324+
1325+static void
1326+wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
1327+{
1328+ wdma_m32(dev, reg, 0, mask);
1329+}
1330+
1331+static u32
1332+mtk_wed_read_reset(struct mtk_wed_device *dev)
1333+{
1334+ return wed_r32(dev, MTK_WED_RESET);
1335+}
1336+
1337+static void
1338+mtk_wed_reset(struct mtk_wed_device *dev, u32 mask)
1339+{
1340+ u32 status;
1341+
1342+ wed_w32(dev, MTK_WED_RESET, mask);
1343+ if (readx_poll_timeout(mtk_wed_read_reset, dev, status,
1344+ !(status & mask), 0, 1000))
1345+ WARN_ON_ONCE(1);
1346+}
1347+
1348+static struct mtk_wed_hw *
1349+mtk_wed_assign(struct mtk_wed_device *dev)
1350+{
1351+ struct mtk_wed_hw *hw;
1352+
1353+ hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)];
1354+ if (!hw || hw->wed_dev)
1355+ return NULL;
1356+
1357+ hw->wed_dev = dev;
1358+ return hw;
1359+}
1360+
1361+static int
1362+mtk_wed_buffer_alloc(struct mtk_wed_device *dev)
1363+{
1364+ struct mtk_wdma_desc *desc;
1365+ dma_addr_t desc_phys;
1366+ void **page_list;
1367+ int token = dev->wlan.token_start;
1368+ int ring_size;
1369+ int n_pages;
1370+ int i, page_idx;
1371+
1372+ ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);
1373+ n_pages = ring_size / MTK_WED_BUF_PER_PAGE;
1374+
1375+ page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);
1376+ if (!page_list)
1377+ return -ENOMEM;
1378+
1379+ dev->buf_ring.size = ring_size;
1380+ dev->buf_ring.pages = page_list;
1381+
1382+ desc = dma_alloc_coherent(dev->hw->dev, ring_size * sizeof(*desc),
1383+ &desc_phys, GFP_KERNEL);
1384+ if (!desc)
1385+ return -ENOMEM;
1386+
1387+ dev->buf_ring.desc = desc;
1388+ dev->buf_ring.desc_phys = desc_phys;
1389+
1390+ for (i = 0, page_idx = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) {
1391+ dma_addr_t page_phys, buf_phys;
1392+ struct page *page;
1393+ void *buf;
1394+ int s;
1395+
1396+ page = __dev_alloc_pages(GFP_KERNEL, 0);
1397+ if (!page)
1398+ return -ENOMEM;
1399+
1400+ page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,
1401+ DMA_BIDIRECTIONAL);
1402+ if (dma_mapping_error(dev->hw->dev, page_phys)) {
1403+ __free_page(page);
1404+ return -ENOMEM;
1405+ }
1406+
1407+ page_list[page_idx++] = page;
1408+ dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,
1409+ DMA_BIDIRECTIONAL);
1410+
1411+ buf = page_to_virt(page);
1412+ buf_phys = page_phys;
1413+
1414+ for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) {
1415+ u32 txd_size;
1416+
1417+ txd_size = dev->wlan.init_buf(buf, buf_phys, token++);
1418+
1419+ desc->buf0 = buf_phys;
1420+ desc->buf1 = buf_phys + txd_size;
1421+ desc->ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0,
1422+ txd_size) |
1423+ FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1,
1424+ MTK_WED_BUF_SIZE - txd_size) |
1425+ MTK_WDMA_DESC_CTRL_LAST_SEG1;
1426+ desc->info = 0;
1427+ desc++;
1428+
1429+ buf += MTK_WED_BUF_SIZE;
1430+ buf_phys += MTK_WED_BUF_SIZE;
1431+ }
1432+
1433+ dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,
1434+ DMA_BIDIRECTIONAL);
1435+ }
1436+
1437+ return 0;
1438+}
1439+
1440+static void
1441+mtk_wed_free_buffer(struct mtk_wed_device *dev)
1442+{
1443+ struct mtk_wdma_desc *desc = dev->buf_ring.desc;
1444+ void **page_list = dev->buf_ring.pages;
1445+ int page_idx;
1446+ int i;
1447+
1448+ if (!page_list)
1449+ return;
1450+
1451+ if (!desc)
1452+ goto free_pagelist;
1453+
1454+ for (i = 0, page_idx = 0; i < dev->buf_ring.size; i += MTK_WED_BUF_PER_PAGE) {
1455+ void *page = page_list[page_idx++];
1456+
1457+ if (!page)
1458+ break;
1459+
1460+ dma_unmap_page(dev->hw->dev, desc[i].buf0,
1461+ PAGE_SIZE, DMA_BIDIRECTIONAL);
1462+ __free_page(page);
1463+ }
1464+
1465+ dma_free_coherent(dev->hw->dev, dev->buf_ring.size * sizeof(*desc),
1466+ desc, dev->buf_ring.desc_phys);
1467+
1468+free_pagelist:
1469+ kfree(page_list);
1470+}
1471+
1472+static void
1473+mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring)
1474+{
1475+ if (!ring->desc)
1476+ return;
1477+
1478+ dma_free_coherent(dev->hw->dev, ring->size * sizeof(*ring->desc),
1479+ ring->desc, ring->desc_phys);
1480+}
1481+
1482+static void
1483+mtk_wed_free_tx_rings(struct mtk_wed_device *dev)
1484+{
1485+ int i;
1486+
1487+ for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++)
1488+ mtk_wed_free_ring(dev, &dev->tx_ring[i]);
1489+ for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)
1490+ mtk_wed_free_ring(dev, &dev->tx_wdma[i]);
1491+}
1492+
1493+static void
1494+mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en)
1495+{
1496+ u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
1497+
1498+ if (!dev->hw->num_flows)
1499+ mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
1500+
1501+ wed_w32(dev, MTK_WED_EXT_INT_MASK, en ? mask : 0);
1502+ wed_r32(dev, MTK_WED_EXT_INT_MASK);
1503+}
1504+
1505+static void
1506+mtk_wed_stop(struct mtk_wed_device *dev)
1507+{
1508+ regmap_write(dev->hw->mirror, dev->hw->index * 4, 0);
1509+ mtk_wed_set_ext_int(dev, false);
1510+
1511+ wed_clr(dev, MTK_WED_CTRL,
1512+ MTK_WED_CTRL_WDMA_INT_AGENT_EN |
1513+ MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
1514+ MTK_WED_CTRL_WED_TX_BM_EN |
1515+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1516+ wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0);
1517+ wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0);
1518+ wdma_w32(dev, MTK_WDMA_INT_MASK, 0);
1519+ wdma_w32(dev, MTK_WDMA_INT_GRP2, 0);
1520+ wed_w32(dev, MTK_WED_WPDMA_INT_MASK, 0);
1521+
1522+ wed_clr(dev, MTK_WED_GLO_CFG,
1523+ MTK_WED_GLO_CFG_TX_DMA_EN |
1524+ MTK_WED_GLO_CFG_RX_DMA_EN);
1525+ wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
1526+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
1527+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
1528+ wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
1529+ MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
1530+}
1531+
1532+static void
1533+mtk_wed_detach(struct mtk_wed_device *dev)
1534+{
1535+ struct device_node *wlan_node = dev->wlan.pci_dev->dev.of_node;
1536+ struct mtk_wed_hw *hw = dev->hw;
1537+
1538+ mutex_lock(&hw_lock);
1539+
1540+ mtk_wed_stop(dev);
1541+
1542+ wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX);
1543+ wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);
1544+
1545+ mtk_wed_reset(dev, MTK_WED_RESET_WED);
1546+
1547+ mtk_wed_free_buffer(dev);
1548+ mtk_wed_free_tx_rings(dev);
1549+
1550+ if (of_dma_is_coherent(wlan_node))
1551+ regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
1552+ BIT(hw->index), BIT(hw->index));
1553+
1554+ if (!hw_list[!hw->index]->wed_dev &&
1555+ hw->eth->dma_dev != hw->eth->dev)
1556+ mtk_eth_set_dma_device(hw->eth, hw->eth->dev);
1557+
1558+ memset(dev, 0, sizeof(*dev));
1559+ module_put(THIS_MODULE);
1560+
1561+ hw->wed_dev = NULL;
1562+ mutex_unlock(&hw_lock);
1563+}
1564+
1565+static void
1566+mtk_wed_hw_init_early(struct mtk_wed_device *dev)
1567+{
1568+ u32 mask, set;
1569+ u32 offset;
1570+
1571+ mtk_wed_stop(dev);
1572+ mtk_wed_reset(dev, MTK_WED_RESET_WED);
1573+
1574+ mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE |
1575+ MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE |
1576+ MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE;
1577+ set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2) |
1578+ MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP |
1579+ MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY;
1580+ wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set);
1581+
1582+ wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_INFO_PRERES);
1583+
1584+ offset = dev->hw->index ? 0x04000400 : 0;
1585+ wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset);
1586+ wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset);
1587+
1588+ wed_w32(dev, MTK_WED_PCIE_CFG_BASE, MTK_PCIE_BASE(dev->hw->index));
1589+ wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys);
1590+}
1591+
1592+static void
1593+mtk_wed_hw_init(struct mtk_wed_device *dev)
1594+{
1595+ if (dev->init_done)
1596+ return;
1597+
1598+ dev->init_done = true;
1599+ mtk_wed_set_ext_int(dev, false);
1600+ wed_w32(dev, MTK_WED_TX_BM_CTRL,
1601+ MTK_WED_TX_BM_CTRL_PAUSE |
1602+ FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,
1603+ dev->buf_ring.size / 128) |
1604+ FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,
1605+ MTK_WED_TX_RING_SIZE / 256));
1606+
1607+ wed_w32(dev, MTK_WED_TX_BM_BASE, dev->buf_ring.desc_phys);
1608+
1609+ wed_w32(dev, MTK_WED_TX_BM_TKID,
1610+ FIELD_PREP(MTK_WED_TX_BM_TKID_START,
1611+ dev->wlan.token_start) |
1612+ FIELD_PREP(MTK_WED_TX_BM_TKID_END,
1613+ dev->wlan.token_start + dev->wlan.nbuf - 1));
1614+
1615+ wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE);
1616+
1617+ wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
1618+ FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |
1619+ MTK_WED_TX_BM_DYN_THR_HI);
1620+
1621+ mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
1622+
1623+ wed_set(dev, MTK_WED_CTRL,
1624+ MTK_WED_CTRL_WED_TX_BM_EN |
1625+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1626+
1627+ wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE);
1628+}
1629+
1630+static void
1631+mtk_wed_ring_reset(struct mtk_wdma_desc *desc, int size)
1632+{
1633+ int i;
1634+
1635+ for (i = 0; i < size; i++) {
1636+ desc[i].buf0 = 0;
1637+ desc[i].ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
1638+ desc[i].buf1 = 0;
1639+ desc[i].info = 0;
1640+ }
1641+}
1642+
1643+static u32
1644+mtk_wed_check_busy(struct mtk_wed_device *dev)
1645+{
1646+ if (wed_r32(dev, MTK_WED_GLO_CFG) & MTK_WED_GLO_CFG_TX_DMA_BUSY)
1647+ return true;
1648+
1649+ if (wed_r32(dev, MTK_WED_WPDMA_GLO_CFG) &
1650+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY)
1651+ return true;
1652+
1653+ if (wed_r32(dev, MTK_WED_CTRL) & MTK_WED_CTRL_WDMA_INT_AGENT_BUSY)
1654+ return true;
1655+
1656+ if (wed_r32(dev, MTK_WED_WDMA_GLO_CFG) &
1657+ MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY)
1658+ return true;
1659+
1660+ if (wdma_r32(dev, MTK_WDMA_GLO_CFG) &
1661+ MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY)
1662+ return true;
1663+
1664+ if (wed_r32(dev, MTK_WED_CTRL) &
1665+ (MTK_WED_CTRL_WED_TX_BM_BUSY | MTK_WED_CTRL_WED_TX_FREE_AGENT_BUSY))
1666+ return true;
1667+
1668+ return false;
1669+}
1670+
1671+static int
1672+mtk_wed_poll_busy(struct mtk_wed_device *dev)
1673+{
1674+ int sleep = 15000;
1675+ int timeout = 100 * sleep;
1676+ u32 val;
1677+
1678+ return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep,
1679+ timeout, false, dev);
1680+}
1681+
1682+static void
1683+mtk_wed_reset_dma(struct mtk_wed_device *dev)
1684+{
1685+ bool busy = false;
1686+ u32 val;
1687+ int i;
1688+
1689+ for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {
1690+ struct mtk_wdma_desc *desc = dev->tx_ring[i].desc;
1691+
1692+ if (!desc)
1693+ continue;
1694+
1695+ mtk_wed_ring_reset(desc, MTK_WED_TX_RING_SIZE);
1696+ }
1697+
1698+ if (mtk_wed_poll_busy(dev))
1699+ busy = mtk_wed_check_busy(dev);
1700+
1701+ if (busy) {
1702+ mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA);
1703+ } else {
1704+ wed_w32(dev, MTK_WED_RESET_IDX,
1705+ MTK_WED_RESET_IDX_TX |
1706+ MTK_WED_RESET_IDX_RX);
1707+ wed_w32(dev, MTK_WED_RESET_IDX, 0);
1708+ }
1709+
1710+ wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX);
1711+ wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);
1712+
1713+ if (busy) {
1714+ mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT);
1715+ mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV);
1716+ } else {
1717+ wed_w32(dev, MTK_WED_WDMA_RESET_IDX,
1718+ MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV);
1719+ wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0);
1720+
1721+ wed_set(dev, MTK_WED_WDMA_GLO_CFG,
1722+ MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
1723+
1724+ wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
1725+ MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
1726+ }
1727+
1728+ for (i = 0; i < 100; i++) {
1729+ val = wed_r32(dev, MTK_WED_TX_BM_INTF);
1730+ if (FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP, val) == 0x40)
1731+ break;
1732+ }
1733+
1734+ mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT);
1735+ mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
1736+
1737+ if (busy) {
1738+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);
1739+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV);
1740+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV);
1741+ } else {
1742+ wed_w32(dev, MTK_WED_WPDMA_RESET_IDX,
1743+ MTK_WED_WPDMA_RESET_IDX_TX |
1744+ MTK_WED_WPDMA_RESET_IDX_RX);
1745+ wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0);
1746+ }
1747+
1748+}
1749+
1750+static int
1751+mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
1752+ int size)
1753+{
1754+ ring->desc = dma_alloc_coherent(dev->hw->dev,
1755+ size * sizeof(*ring->desc),
1756+ &ring->desc_phys, GFP_KERNEL);
1757+ if (!ring->desc)
1758+ return -ENOMEM;
1759+
1760+ ring->size = size;
1761+ mtk_wed_ring_reset(ring->desc, size);
1762+
1763+ return 0;
1764+}
1765+
1766+static int
1767+mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size)
1768+{
1769+ struct mtk_wed_ring *wdma = &dev->tx_wdma[idx];
1770+
1771+ if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE))
1772+ return -ENOMEM;
1773+
1774+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
1775+ wdma->desc_phys);
1776+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,
1777+ size);
1778+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
1779+
1780+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
1781+ wdma->desc_phys);
1782+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,
1783+ size);
1784+
1785+ return 0;
1786+}
1787+
1788+static void
1789+mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
1790+{
1791+ u32 wdma_mask;
1792+ u32 val;
1793+ int i;
1794+
1795+ for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)
1796+ if (!dev->tx_wdma[i].desc)
1797+ mtk_wed_wdma_ring_setup(dev, i, 16);
1798+
1799+ wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));
1800+
1801+ mtk_wed_hw_init(dev);
1802+
1803+ wed_set(dev, MTK_WED_CTRL,
1804+ MTK_WED_CTRL_WDMA_INT_AGENT_EN |
1805+ MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
1806+ MTK_WED_CTRL_WED_TX_BM_EN |
1807+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
1808+
1809+ wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, MTK_WED_PCIE_INT_TRIGGER_STATUS);
1810+
1811+ wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,
1812+ MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |
1813+ MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);
1814+
1815+ wed_set(dev, MTK_WED_WPDMA_INT_CTRL,
1816+ MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);
1817+
1818+ wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask);
1819+ wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);
1820+
1821+ wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask);
1822+ wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask);
1823+
1824+ wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);
1825+ wed_w32(dev, MTK_WED_INT_MASK, irq_mask);
1826+
1827+ wed_set(dev, MTK_WED_GLO_CFG,
1828+ MTK_WED_GLO_CFG_TX_DMA_EN |
1829+ MTK_WED_GLO_CFG_RX_DMA_EN);
1830+ wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
1831+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
1832+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
1833+ wed_set(dev, MTK_WED_WDMA_GLO_CFG,
1834+ MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
1835+
1836+ mtk_wed_set_ext_int(dev, true);
1837+ val = dev->wlan.wpdma_phys |
1838+ MTK_PCIE_MIRROR_MAP_EN |
1839+ FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, dev->hw->index);
1840+
1841+ if (dev->hw->index)
1842+ val |= BIT(1);
1843+ val |= BIT(0);
1844+ regmap_write(dev->hw->mirror, dev->hw->index * 4, val);
1845+
1846+ dev->running = true;
1847+}
1848+
1849+static int
1850+mtk_wed_attach(struct mtk_wed_device *dev)
1851+ __releases(RCU)
1852+{
1853+ struct mtk_wed_hw *hw;
1854+ int ret = 0;
1855+
1856+ RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1857+ "mtk_wed_attach without holding the RCU read lock");
1858+
1859+ if (pci_domain_nr(dev->wlan.pci_dev->bus) > 1 ||
1860+ !try_module_get(THIS_MODULE))
1861+ ret = -ENODEV;
1862+
1863+ rcu_read_unlock();
1864+
1865+ if (ret)
1866+ return ret;
1867+
1868+ mutex_lock(&hw_lock);
1869+
1870+ hw = mtk_wed_assign(dev);
1871+ if (!hw) {
1872+ module_put(THIS_MODULE);
1873+ ret = -ENODEV;
1874+ goto out;
1875+ }
1876+
1877+ dev_info(&dev->wlan.pci_dev->dev, "attaching wed device %d\n", hw->index);
1878+
1879+ dev->hw = hw;
1880+ dev->dev = hw->dev;
1881+ dev->irq = hw->irq;
1882+ dev->wdma_idx = hw->index;
1883+
1884+ if (hw->eth->dma_dev == hw->eth->dev &&
1885+ of_dma_is_coherent(hw->eth->dev->of_node))
1886+ mtk_eth_set_dma_device(hw->eth, hw->dev);
1887+
1888+ ret = mtk_wed_buffer_alloc(dev);
1889+ if (ret) {
1890+ mtk_wed_detach(dev);
1891+ goto out;
1892+ }
1893+
1894+ mtk_wed_hw_init_early(dev);
1895+ regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, BIT(hw->index), 0);
1896+
1897+out:
1898+ mutex_unlock(&hw_lock);
1899+
1900+ return ret;
1901+}
1902+
1903+static int
1904+mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs)
1905+{
1906+ struct mtk_wed_ring *ring = &dev->tx_ring[idx];
1907+
1908+ /*
1909+ * Tx ring redirection:
1910+ * Instead of configuring the WLAN PDMA TX ring directly, the WLAN
1911+ * driver allocated DMA ring gets configured into WED MTK_WED_RING_TX(n)
1912+ * registers.
1913+ *
1914+ * WED driver posts its own DMA ring as WLAN PDMA TX and configures it
1915+ * into MTK_WED_WPDMA_RING_TX(n) registers.
1916+ * It gets filled with packets picked up from WED TX ring and from
1917+ * WDMA RX.
1918+ */
1919+
1920+ BUG_ON(idx > ARRAY_SIZE(dev->tx_ring));
1921+
1922+ if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE))
1923+ return -ENOMEM;
1924+
1925+ if (mtk_wed_wdma_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE))
1926+ return -ENOMEM;
1927+
1928+ ring->reg_base = MTK_WED_RING_TX(idx);
1929+ ring->wpdma = regs;
1930+
1931+ /* WED -> WPDMA */
1932+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);
1933+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE);
1934+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0);
1935+
1936+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,
1937+ ring->desc_phys);
1938+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,
1939+ MTK_WED_TX_RING_SIZE);
1940+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
1941+
1942+ return 0;
1943+}
1944+
1945+static int
1946+mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
1947+{
1948+ struct mtk_wed_ring *ring = &dev->txfree_ring;
1949+ int i;
1950+
1951+ /*
1952+ * For txfree event handling, the same DMA ring is shared between WED
1953+ * and WLAN. The WLAN driver accesses the ring index registers through
1954+ * WED
1955+ */
1956+ ring->reg_base = MTK_WED_RING_RX(1);
1957+ ring->wpdma = regs;
1958+
1959+ for (i = 0; i < 12; i += 4) {
1960+ u32 val = readl(regs + i);
1961+
1962+ wed_w32(dev, MTK_WED_RING_RX(1) + i, val);
1963+ wed_w32(dev, MTK_WED_WPDMA_RING_RX(1) + i, val);
1964+ }
1965+
1966+ return 0;
1967+}
1968+
1969+static u32
1970+mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask)
1971+{
1972+ u32 val;
1973+
1974+ val = wed_r32(dev, MTK_WED_EXT_INT_STATUS);
1975+ wed_w32(dev, MTK_WED_EXT_INT_STATUS, val);
1976+ val &= MTK_WED_EXT_INT_STATUS_ERROR_MASK;
1977+ if (!dev->hw->num_flows)
1978+ val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
1979+ if (val && net_ratelimit())
1980+ pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val);
1981+
1982+ val = wed_r32(dev, MTK_WED_INT_STATUS);
1983+ val &= mask;
1984+ wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */
1985+
1986+ return val;
1987+}
1988+
1989+static void
1990+mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask)
1991+{
1992+ if (!dev->running)
1993+ return;
1994+
1995+ mtk_wed_set_ext_int(dev, !!mask);
1996+ wed_w32(dev, MTK_WED_INT_MASK, mask);
1997+}
1998+
1999+int mtk_wed_flow_add(int index)
2000+{
2001+ struct mtk_wed_hw *hw = hw_list[index];
2002+ int ret;
2003+
2004+ if (!hw || !hw->wed_dev)
2005+ return -ENODEV;
2006+
2007+ if (hw->num_flows) {
2008+ hw->num_flows++;
2009+ return 0;
2010+ }
2011+
2012+ mutex_lock(&hw_lock);
2013+ if (!hw->wed_dev) {
2014+ ret = -ENODEV;
2015+ goto out;
2016+ }
2017+
2018+ ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev);
2019+ if (!ret)
2020+ hw->num_flows++;
2021+ mtk_wed_set_ext_int(hw->wed_dev, true);
2022+
2023+out:
2024+ mutex_unlock(&hw_lock);
2025+
2026+ return ret;
2027+}
2028+
2029+void mtk_wed_flow_remove(int index)
2030+{
2031+ struct mtk_wed_hw *hw = hw_list[index];
2032+
2033+ if (!hw)
2034+ return;
2035+
2036+ if (--hw->num_flows)
2037+ return;
2038+
2039+ mutex_lock(&hw_lock);
2040+ if (!hw->wed_dev)
2041+ goto out;
2042+
2043+ hw->wed_dev->wlan.offload_disable(hw->wed_dev);
2044+ mtk_wed_set_ext_int(hw->wed_dev, true);
2045+
2046+out:
2047+ mutex_unlock(&hw_lock);
2048+}
2049+
2050+void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
2051+ void __iomem *wdma, int index)
2052+{
2053+ static const struct mtk_wed_ops wed_ops = {
2054+ .attach = mtk_wed_attach,
2055+ .tx_ring_setup = mtk_wed_tx_ring_setup,
2056+ .txfree_ring_setup = mtk_wed_txfree_ring_setup,
2057+ .start = mtk_wed_start,
2058+ .stop = mtk_wed_stop,
2059+ .reset_dma = mtk_wed_reset_dma,
2060+ .reg_read = wed_r32,
2061+ .reg_write = wed_w32,
2062+ .irq_get = mtk_wed_irq_get,
2063+ .irq_set_mask = mtk_wed_irq_set_mask,
2064+ .detach = mtk_wed_detach,
2065+ };
2066+ struct device_node *eth_np = eth->dev->of_node;
2067+ struct platform_device *pdev;
2068+ struct mtk_wed_hw *hw;
2069+ struct regmap *regs;
2070+ int irq;
2071+
2072+ if (!np)
2073+ return;
2074+
2075+ pdev = of_find_device_by_node(np);
2076+ if (!pdev)
2077+ return;
2078+
2079+ get_device(&pdev->dev);
2080+ irq = platform_get_irq(pdev, 0);
2081+ if (irq < 0)
2082+ return;
2083+
2084+ regs = syscon_regmap_lookup_by_phandle(np, NULL);
2085+ if (!regs)
2086+ return;
2087+
2088+ rcu_assign_pointer(mtk_soc_wed_ops, &wed_ops);
2089+
2090+ mutex_lock(&hw_lock);
2091+
2092+ if (WARN_ON(hw_list[index]))
2093+ goto unlock;
2094+
2095+ hw = kzalloc(sizeof(*hw), GFP_KERNEL);
2096+ hw->node = np;
2097+ hw->regs = regs;
2098+ hw->eth = eth;
2099+ hw->dev = &pdev->dev;
2100+ hw->wdma = wdma;
2101+ hw->index = index;
2102+ hw->irq = irq;
2103+ hw->mirror = syscon_regmap_lookup_by_phandle(eth_np,
2104+ "mediatek,pcie-mirror");
2105+ hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np,
2106+ "mediatek,hifsys");
2107+ if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) {
2108+ kfree(hw);
2109+ goto unlock;
2110+ }
2111+
2112+ if (!index) {
2113+ regmap_write(hw->mirror, 0, 0);
2114+ regmap_write(hw->mirror, 4, 0);
2115+ }
2116+ mtk_wed_hw_add_debugfs(hw);
2117+
2118+ hw_list[index] = hw;
2119+
2120+unlock:
2121+ mutex_unlock(&hw_lock);
2122+}
2123+
2124+void mtk_wed_exit(void)
2125+{
2126+ int i;
2127+
2128+ rcu_assign_pointer(mtk_soc_wed_ops, NULL);
2129+
2130+ synchronize_rcu();
2131+
2132+ for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
2133+ struct mtk_wed_hw *hw;
2134+
2135+ hw = hw_list[i];
2136+ if (!hw)
2137+ continue;
2138+
2139+ hw_list[i] = NULL;
2140+ debugfs_remove(hw->debugfs_dir);
2141+ put_device(hw->dev);
2142+ kfree(hw);
2143+ }
2144+}
2145diff --git a/drivers/net/ethernet/mediatek/mtk_wed.h b/drivers/net/ethernet/mediatek/mtk_wed.h
2146new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2147index 0000000..981ec61
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2148--- /dev/null
2149+++ b/drivers/net/ethernet/mediatek/mtk_wed.h
2150@@ -0,0 +1,135 @@
2151+// SPDX-License-Identifier: GPL-2.0-only
2152+/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */
2153+
2154+#ifndef __MTK_WED_PRIV_H
2155+#define __MTK_WED_PRIV_H
2156+
2157+#include <linux/soc/mediatek/mtk_wed.h>
2158+#include <linux/debugfs.h>
2159+#include <linux/regmap.h>
2160+#include <linux/netdevice.h>
2161+
2162+struct mtk_eth;
2163+
2164+struct mtk_wed_hw {
2165+ struct device_node *node;
2166+ struct mtk_eth *eth;
2167+ struct regmap *regs;
2168+ struct regmap *hifsys;
2169+ struct device *dev;
2170+ void __iomem *wdma;
2171+ struct regmap *mirror;
2172+ struct dentry *debugfs_dir;
2173+ struct mtk_wed_device *wed_dev;
2174+ u32 debugfs_reg;
2175+ u32 num_flows;
2176+ char dirname[5];
2177+ int irq;
2178+ int index;
2179+};
2180+
2181+struct mtk_wdma_info {
2182+ u8 wdma_idx;
2183+ u8 queue;
2184+ u16 wcid;
2185+ u8 bss;
2186+};
2187+
2188+#ifdef CONFIG_NET_MEDIATEK_SOC_WED
2189+static inline void
2190+wed_w32(struct mtk_wed_device *dev, u32 reg, u32 val)
2191+{
2192+ regmap_write(dev->hw->regs, reg, val);
2193+}
2194+
2195+static inline u32
2196+wed_r32(struct mtk_wed_device *dev, u32 reg)
2197+{
2198+ unsigned int val;
2199+
2200+ regmap_read(dev->hw->regs, reg, &val);
2201+
2202+ return val;
2203+}
2204+
2205+static inline void
2206+wdma_w32(struct mtk_wed_device *dev, u32 reg, u32 val)
2207+{
2208+ writel(val, dev->hw->wdma + reg);
2209+}
2210+
2211+static inline u32
2212+wdma_r32(struct mtk_wed_device *dev, u32 reg)
2213+{
2214+ return readl(dev->hw->wdma + reg);
2215+}
2216+
2217+static inline u32
2218+wpdma_tx_r32(struct mtk_wed_device *dev, int ring, u32 reg)
2219+{
2220+ if (!dev->tx_ring[ring].wpdma)
2221+ return 0;
2222+
2223+ return readl(dev->tx_ring[ring].wpdma + reg);
2224+}
2225+
2226+static inline void
2227+wpdma_tx_w32(struct mtk_wed_device *dev, int ring, u32 reg, u32 val)
2228+{
2229+ if (!dev->tx_ring[ring].wpdma)
2230+ return;
2231+
2232+ writel(val, dev->tx_ring[ring].wpdma + reg);
2233+}
2234+
2235+static inline u32
2236+wpdma_txfree_r32(struct mtk_wed_device *dev, u32 reg)
2237+{
2238+ if (!dev->txfree_ring.wpdma)
2239+ return 0;
2240+
2241+ return readl(dev->txfree_ring.wpdma + reg);
2242+}
2243+
2244+static inline void
2245+wpdma_txfree_w32(struct mtk_wed_device *dev, u32 reg, u32 val)
2246+{
2247+ if (!dev->txfree_ring.wpdma)
2248+ return;
2249+
2250+ writel(val, dev->txfree_ring.wpdma + reg);
2251+}
2252+
2253+void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
2254+ void __iomem *wdma, int index);
2255+void mtk_wed_exit(void);
2256+int mtk_wed_flow_add(int index);
2257+void mtk_wed_flow_remove(int index);
2258+#else
2259+static inline void
2260+mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
2261+ void __iomem *wdma, int index)
2262+{
2263+}
2264+static inline void
2265+mtk_wed_exit(void)
2266+{
2267+}
2268+static inline int mtk_wed_flow_add(int index)
2269+{
2270+ return -EINVAL;
2271+}
2272+static inline void mtk_wed_flow_remove(int index)
2273+{
2274+}
2275+#endif
2276+
2277+#ifdef CONFIG_DEBUG_FS
2278+void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw);
2279+#else
2280+static inline void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw)
2281+{
2282+}
2283+#endif
2284+
2285+#endif
2286diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
2287new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2288index 0000000..a81d3fd
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2289--- /dev/null
2290+++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
2291@@ -0,0 +1,175 @@
2292+// SPDX-License-Identifier: GPL-2.0-only
2293+/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */
2294+
2295+#include <linux/seq_file.h>
2296+#include "mtk_wed.h"
2297+#include "mtk_wed_regs.h"
2298+
2299+struct reg_dump {
2300+ const char *name;
2301+ u16 offset;
2302+ u8 type;
2303+ u8 base;
2304+};
2305+
2306+enum {
2307+ DUMP_TYPE_STRING,
2308+ DUMP_TYPE_WED,
2309+ DUMP_TYPE_WDMA,
2310+ DUMP_TYPE_WPDMA_TX,
2311+ DUMP_TYPE_WPDMA_TXFREE,
2312+};
2313+
2314+#define DUMP_STR(_str) { _str, 0, DUMP_TYPE_STRING }
2315+#define DUMP_REG(_reg, ...) { #_reg, MTK_##_reg, __VA_ARGS__ }
2316+#define DUMP_RING(_prefix, _base, ...) \
2317+ { _prefix " BASE", _base, __VA_ARGS__ }, \
2318+ { _prefix " CNT", _base + 0x4, __VA_ARGS__ }, \
2319+ { _prefix " CIDX", _base + 0x8, __VA_ARGS__ }, \
2320+ { _prefix " DIDX", _base + 0xc, __VA_ARGS__ }
2321+
2322+#define DUMP_WED(_reg) DUMP_REG(_reg, DUMP_TYPE_WED)
2323+#define DUMP_WED_RING(_base) DUMP_RING(#_base, MTK_##_base, DUMP_TYPE_WED)
2324+
2325+#define DUMP_WDMA(_reg) DUMP_REG(_reg, DUMP_TYPE_WDMA)
2326+#define DUMP_WDMA_RING(_base) DUMP_RING(#_base, MTK_##_base, DUMP_TYPE_WDMA)
2327+
2328+#define DUMP_WPDMA_TX_RING(_n) DUMP_RING("WPDMA_TX" #_n, 0, DUMP_TYPE_WPDMA_TX, _n)
2329+#define DUMP_WPDMA_TXFREE_RING DUMP_RING("WPDMA_RX1", 0, DUMP_TYPE_WPDMA_TXFREE)
2330+
2331+static void
2332+print_reg_val(struct seq_file *s, const char *name, u32 val)
2333+{
2334+ seq_printf(s, "%-32s %08x\n", name, val);
2335+}
2336+
2337+static void
2338+dump_wed_regs(struct seq_file *s, struct mtk_wed_device *dev,
2339+ const struct reg_dump *regs, int n_regs)
2340+{
2341+ const struct reg_dump *cur;
2342+ u32 val;
2343+
2344+ for (cur = regs; cur < &regs[n_regs]; cur++) {
2345+ switch (cur->type) {
2346+ case DUMP_TYPE_STRING:
2347+ seq_printf(s, "%s======== %s:\n",
2348+ cur > regs ? "\n" : "",
2349+ cur->name);
2350+ continue;
2351+ case DUMP_TYPE_WED:
2352+ val = wed_r32(dev, cur->offset);
2353+ break;
2354+ case DUMP_TYPE_WDMA:
2355+ val = wdma_r32(dev, cur->offset);
2356+ break;
2357+ case DUMP_TYPE_WPDMA_TX:
2358+ val = wpdma_tx_r32(dev, cur->base, cur->offset);
2359+ break;
2360+ case DUMP_TYPE_WPDMA_TXFREE:
2361+ val = wpdma_txfree_r32(dev, cur->offset);
2362+ break;
2363+ }
2364+ print_reg_val(s, cur->name, val);
2365+ }
2366+}
2367+
2368+
2369+static int
2370+wed_txinfo_show(struct seq_file *s, void *data)
2371+{
2372+ static const struct reg_dump regs[] = {
2373+ DUMP_STR("WED TX"),
2374+ DUMP_WED(WED_TX_MIB(0)),
2375+ DUMP_WED_RING(WED_RING_TX(0)),
2376+
2377+ DUMP_WED(WED_TX_MIB(1)),
2378+ DUMP_WED_RING(WED_RING_TX(1)),
2379+
2380+ DUMP_STR("WPDMA TX"),
2381+ DUMP_WED(WED_WPDMA_TX_MIB(0)),
2382+ DUMP_WED_RING(WED_WPDMA_RING_TX(0)),
2383+ DUMP_WED(WED_WPDMA_TX_COHERENT_MIB(0)),
2384+
2385+ DUMP_WED(WED_WPDMA_TX_MIB(1)),
2386+ DUMP_WED_RING(WED_WPDMA_RING_TX(1)),
2387+ DUMP_WED(WED_WPDMA_TX_COHERENT_MIB(1)),
2388+
2389+ DUMP_STR("WPDMA TX"),
2390+ DUMP_WPDMA_TX_RING(0),
2391+ DUMP_WPDMA_TX_RING(1),
2392+
2393+ DUMP_STR("WED WDMA RX"),
2394+ DUMP_WED(WED_WDMA_RX_MIB(0)),
2395+ DUMP_WED_RING(WED_WDMA_RING_RX(0)),
2396+ DUMP_WED(WED_WDMA_RX_THRES(0)),
2397+ DUMP_WED(WED_WDMA_RX_RECYCLE_MIB(0)),
2398+ DUMP_WED(WED_WDMA_RX_PROCESSED_MIB(0)),
2399+
2400+ DUMP_WED(WED_WDMA_RX_MIB(1)),
2401+ DUMP_WED_RING(WED_WDMA_RING_RX(1)),
2402+ DUMP_WED(WED_WDMA_RX_THRES(1)),
2403+ DUMP_WED(WED_WDMA_RX_RECYCLE_MIB(1)),
2404+ DUMP_WED(WED_WDMA_RX_PROCESSED_MIB(1)),
2405+
2406+ DUMP_STR("WDMA RX"),
2407+ DUMP_WDMA(WDMA_GLO_CFG),
2408+ DUMP_WDMA_RING(WDMA_RING_RX(0)),
2409+ DUMP_WDMA_RING(WDMA_RING_RX(1)),
2410+ };
2411+ struct mtk_wed_hw *hw = s->private;
2412+ struct mtk_wed_device *dev = hw->wed_dev;
2413+
2414+ if (!dev)
2415+ return 0;
2416+
2417+ dump_wed_regs(s, dev, regs, ARRAY_SIZE(regs));
2418+
2419+ return 0;
2420+}
2421+DEFINE_SHOW_ATTRIBUTE(wed_txinfo);
2422+
2423+
2424+static int
2425+mtk_wed_reg_set(void *data, u64 val)
2426+{
2427+ struct mtk_wed_hw *hw = data;
2428+
2429+ regmap_write(hw->regs, hw->debugfs_reg, val);
2430+
2431+ return 0;
2432+}
2433+
2434+static int
2435+mtk_wed_reg_get(void *data, u64 *val)
2436+{
2437+ struct mtk_wed_hw *hw = data;
2438+ unsigned int regval;
2439+ int ret;
2440+
2441+ ret = regmap_read(hw->regs, hw->debugfs_reg, &regval);
2442+ if (ret)
2443+ return ret;
2444+
2445+ *val = regval;
2446+
2447+ return 0;
2448+}
2449+
2450+DEFINE_DEBUGFS_ATTRIBUTE(fops_regval, mtk_wed_reg_get, mtk_wed_reg_set,
2451+ "0x%08llx\n");
2452+
2453+void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw)
2454+{
2455+ struct dentry *dir;
2456+
2457+ snprintf(hw->dirname, sizeof(hw->dirname), "wed%d", hw->index);
2458+ dir = debugfs_create_dir(hw->dirname, NULL);
2459+ if (!dir)
2460+ return;
2461+
2462+ hw->debugfs_dir = dir;
2463+ debugfs_create_u32("regidx", 0600, dir, &hw->debugfs_reg);
2464+ debugfs_create_file_unsafe("regval", 0600, dir, hw, &fops_regval);
2465+ debugfs_create_file_unsafe("txinfo", 0400, dir, hw, &wed_txinfo_fops);
2466+}
2467diff --git a/drivers/net/ethernet/mediatek/mtk_wed_ops.c b/drivers/net/ethernet/mediatek/mtk_wed_ops.c
2468new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2469index 0000000..a5d9d8a
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2470--- /dev/null
2471+++ b/drivers/net/ethernet/mediatek/mtk_wed_ops.c
2472@@ -0,0 +1,8 @@
2473+// SPDX-License-Identifier: GPL-2.0-only
2474+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
2475+
2476+#include <linux/kernel.h>
2477+#include <linux/soc/mediatek/mtk_wed.h>
2478+
2479+const struct mtk_wed_ops __rcu *mtk_soc_wed_ops;
2480+EXPORT_SYMBOL_GPL(mtk_soc_wed_ops);
2481diff --git a/drivers/net/ethernet/mediatek/mtk_wed_regs.h b/drivers/net/ethernet/mediatek/mtk_wed_regs.h
2482new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2483index 0000000..0a0465e
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2484--- /dev/null
2485+++ b/drivers/net/ethernet/mediatek/mtk_wed_regs.h
2486@@ -0,0 +1,251 @@
2487+// SPDX-License-Identifier: GPL-2.0-only
2488+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
2489+
2490+#ifndef __MTK_WED_REGS_H
2491+#define __MTK_WED_REGS_H
2492+
2493+#define MTK_WDMA_DESC_CTRL_LEN1 GENMASK(14, 0)
2494+#define MTK_WDMA_DESC_CTRL_LAST_SEG1 BIT(15)
2495+#define MTK_WDMA_DESC_CTRL_BURST BIT(16)
2496+#define MTK_WDMA_DESC_CTRL_LEN0 GENMASK(29, 16)
2497+#define MTK_WDMA_DESC_CTRL_LAST_SEG0 BIT(30)
2498+#define MTK_WDMA_DESC_CTRL_DMA_DONE BIT(31)
2499+
2500+struct mtk_wdma_desc {
2501+ __le32 buf0;
2502+ __le32 ctrl;
2503+ __le32 buf1;
2504+ __le32 info;
2505+} __packed __aligned(4);
2506+
2507+#define MTK_WED_RESET 0x008
2508+#define MTK_WED_RESET_TX_BM BIT(0)
2509+#define MTK_WED_RESET_TX_FREE_AGENT BIT(4)
2510+#define MTK_WED_RESET_WPDMA_TX_DRV BIT(8)
2511+#define MTK_WED_RESET_WPDMA_RX_DRV BIT(9)
2512+#define MTK_WED_RESET_WPDMA_INT_AGENT BIT(11)
2513+#define MTK_WED_RESET_WED_TX_DMA BIT(12)
2514+#define MTK_WED_RESET_WDMA_RX_DRV BIT(17)
2515+#define MTK_WED_RESET_WDMA_INT_AGENT BIT(19)
2516+#define MTK_WED_RESET_WED BIT(31)
2517+
2518+#define MTK_WED_CTRL 0x00c
2519+#define MTK_WED_CTRL_WPDMA_INT_AGENT_EN BIT(0)
2520+#define MTK_WED_CTRL_WPDMA_INT_AGENT_BUSY BIT(1)
2521+#define MTK_WED_CTRL_WDMA_INT_AGENT_EN BIT(2)
2522+#define MTK_WED_CTRL_WDMA_INT_AGENT_BUSY BIT(3)
2523+#define MTK_WED_CTRL_WED_TX_BM_EN BIT(8)
2524+#define MTK_WED_CTRL_WED_TX_BM_BUSY BIT(9)
2525+#define MTK_WED_CTRL_WED_TX_FREE_AGENT_EN BIT(10)
2526+#define MTK_WED_CTRL_WED_TX_FREE_AGENT_BUSY BIT(11)
2527+#define MTK_WED_CTRL_RESERVE_EN BIT(12)
2528+#define MTK_WED_CTRL_RESERVE_BUSY BIT(13)
2529+#define MTK_WED_CTRL_FINAL_DIDX_READ BIT(24)
2530+#define MTK_WED_CTRL_MIB_READ_CLEAR BIT(28)
2531+
2532+#define MTK_WED_EXT_INT_STATUS 0x020
2533+#define MTK_WED_EXT_INT_STATUS_TF_LEN_ERR BIT(0)
2534+#define MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD BIT(1)
2535+#define MTK_WED_EXT_INT_STATUS_TKID_TITO_INVALID BIT(4)
2536+#define MTK_WED_EXT_INT_STATUS_TX_FBUF_LO_TH BIT(8)
2537+#define MTK_WED_EXT_INT_STATUS_TX_FBUF_HI_TH BIT(9)
2538+#define MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH BIT(12)
2539+#define MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH BIT(13)
2540+#define MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR BIT(16)
2541+#define MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR BIT(17)
2542+#define MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT BIT(18)
2543+#define MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN BIT(19)
2544+#define MTK_WED_EXT_INT_STATUS_RX_DRV_BM_DMAD_COHERENT BIT(20)
2545+#define MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR BIT(21)
2546+#define MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR BIT(22)
2547+#define MTK_WED_EXT_INT_STATUS_RX_DRV_DMA_RECYCLE BIT(24)
2548+#define MTK_WED_EXT_INT_STATUS_ERROR_MASK (MTK_WED_EXT_INT_STATUS_TF_LEN_ERR | \
2549+ MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD | \
2550+ MTK_WED_EXT_INT_STATUS_TKID_TITO_INVALID | \
2551+ MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR | \
2552+ MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR | \
2553+ MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN | \
2554+ MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR | \
2555+ MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR)
2556+
2557+#define MTK_WED_EXT_INT_MASK 0x028
2558+
2559+#define MTK_WED_STATUS 0x060
2560+#define MTK_WED_STATUS_TX GENMASK(15, 8)
2561+
2562+#define MTK_WED_TX_BM_CTRL 0x080
2563+#define MTK_WED_TX_BM_CTRL_VLD_GRP_NUM GENMASK(6, 0)
2564+#define MTK_WED_TX_BM_CTRL_RSV_GRP_NUM GENMASK(22, 16)
2565+#define MTK_WED_TX_BM_CTRL_PAUSE BIT(28)
2566+
2567+#define MTK_WED_TX_BM_BASE 0x084
2568+
2569+#define MTK_WED_TX_BM_TKID 0x088
2570+#define MTK_WED_TX_BM_TKID_START GENMASK(15, 0)
2571+#define MTK_WED_TX_BM_TKID_END GENMASK(31, 16)
2572+
2573+#define MTK_WED_TX_BM_BUF_LEN 0x08c
2574+
2575+#define MTK_WED_TX_BM_INTF 0x09c
2576+#define MTK_WED_TX_BM_INTF_TKID GENMASK(15, 0)
2577+#define MTK_WED_TX_BM_INTF_TKFIFO_FDEP GENMASK(23, 16)
2578+#define MTK_WED_TX_BM_INTF_TKID_VALID BIT(28)
2579+#define MTK_WED_TX_BM_INTF_TKID_READ BIT(29)
2580+
2581+#define MTK_WED_TX_BM_DYN_THR 0x0a0
2582+#define MTK_WED_TX_BM_DYN_THR_LO GENMASK(6, 0)
2583+#define MTK_WED_TX_BM_DYN_THR_HI GENMASK(22, 16)
2584+
2585+#define MTK_WED_INT_STATUS 0x200
2586+#define MTK_WED_INT_MASK 0x204
2587+
2588+#define MTK_WED_GLO_CFG 0x208
2589+#define MTK_WED_GLO_CFG_TX_DMA_EN BIT(0)
2590+#define MTK_WED_GLO_CFG_TX_DMA_BUSY BIT(1)
2591+#define MTK_WED_GLO_CFG_RX_DMA_EN BIT(2)
2592+#define MTK_WED_GLO_CFG_RX_DMA_BUSY BIT(3)
2593+#define MTK_WED_GLO_CFG_RX_BT_SIZE GENMASK(5, 4)
2594+#define MTK_WED_GLO_CFG_TX_WB_DDONE BIT(6)
2595+#define MTK_WED_GLO_CFG_BIG_ENDIAN BIT(7)
2596+#define MTK_WED_GLO_CFG_DIS_BT_SIZE_ALIGN BIT(8)
2597+#define MTK_WED_GLO_CFG_TX_BT_SIZE_LO BIT(9)
2598+#define MTK_WED_GLO_CFG_MULTI_DMA_EN GENMASK(11, 10)
2599+#define MTK_WED_GLO_CFG_FIFO_LITTLE_ENDIAN BIT(12)
2600+#define MTK_WED_GLO_CFG_MI_DEPTH_RD GENMASK(21, 13)
2601+#define MTK_WED_GLO_CFG_TX_BT_SIZE_HI GENMASK(23, 22)
2602+#define MTK_WED_GLO_CFG_SW_RESET BIT(24)
2603+#define MTK_WED_GLO_CFG_FIRST_TOKEN_ONLY BIT(26)
2604+#define MTK_WED_GLO_CFG_OMIT_RX_INFO BIT(27)
2605+#define MTK_WED_GLO_CFG_OMIT_TX_INFO BIT(28)
2606+#define MTK_WED_GLO_CFG_BYTE_SWAP BIT(29)
2607+#define MTK_WED_GLO_CFG_RX_2B_OFFSET BIT(31)
2608+
2609+#define MTK_WED_RESET_IDX 0x20c
2610+#define MTK_WED_RESET_IDX_TX GENMASK(3, 0)
2611+#define MTK_WED_RESET_IDX_RX GENMASK(17, 16)
2612+
2613+#define MTK_WED_TX_MIB(_n) (0x2a0 + (_n) * 4)
2614+
2615+#define MTK_WED_RING_TX(_n) (0x300 + (_n) * 0x10)
2616+
2617+#define MTK_WED_RING_RX(_n) (0x400 + (_n) * 0x10)
2618+
2619+#define MTK_WED_WPDMA_INT_TRIGGER 0x504
2620+#define MTK_WED_WPDMA_INT_TRIGGER_RX_DONE BIT(1)
2621+#define MTK_WED_WPDMA_INT_TRIGGER_TX_DONE GENMASK(5, 4)
2622+
2623+#define MTK_WED_WPDMA_GLO_CFG 0x508
2624+#define MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN BIT(0)
2625+#define MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY BIT(1)
2626+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN BIT(2)
2627+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY BIT(3)
2628+#define MTK_WED_WPDMA_GLO_CFG_RX_BT_SIZE GENMASK(5, 4)
2629+#define MTK_WED_WPDMA_GLO_CFG_TX_WB_DDONE BIT(6)
2630+#define MTK_WED_WPDMA_GLO_CFG_BIG_ENDIAN BIT(7)
2631+#define MTK_WED_WPDMA_GLO_CFG_DIS_BT_SIZE_ALIGN BIT(8)
2632+#define MTK_WED_WPDMA_GLO_CFG_TX_BT_SIZE_LO BIT(9)
2633+#define MTK_WED_WPDMA_GLO_CFG_MULTI_DMA_EN GENMASK(11, 10)
2634+#define MTK_WED_WPDMA_GLO_CFG_FIFO_LITTLE_ENDIAN BIT(12)
2635+#define MTK_WED_WPDMA_GLO_CFG_MI_DEPTH_RD GENMASK(21, 13)
2636+#define MTK_WED_WPDMA_GLO_CFG_TX_BT_SIZE_HI GENMASK(23, 22)
2637+#define MTK_WED_WPDMA_GLO_CFG_SW_RESET BIT(24)
2638+#define MTK_WED_WPDMA_GLO_CFG_FIRST_TOKEN_ONLY BIT(26)
2639+#define MTK_WED_WPDMA_GLO_CFG_OMIT_RX_INFO BIT(27)
2640+#define MTK_WED_WPDMA_GLO_CFG_OMIT_TX_INFO BIT(28)
2641+#define MTK_WED_WPDMA_GLO_CFG_BYTE_SWAP BIT(29)
2642+#define MTK_WED_WPDMA_GLO_CFG_RX_2B_OFFSET BIT(31)
2643+
2644+#define MTK_WED_WPDMA_RESET_IDX 0x50c
2645+#define MTK_WED_WPDMA_RESET_IDX_TX GENMASK(3, 0)
2646+#define MTK_WED_WPDMA_RESET_IDX_RX GENMASK(17, 16)
2647+
2648+#define MTK_WED_WPDMA_INT_CTRL 0x520
2649+#define MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV BIT(21)
2650+
2651+#define MTK_WED_WPDMA_INT_MASK 0x524
2652+
2653+#define MTK_WED_PCIE_CFG_BASE 0x560
2654+
2655+#define MTK_WED_PCIE_INT_TRIGGER 0x570
2656+#define MTK_WED_PCIE_INT_TRIGGER_STATUS BIT(16)
2657+
2658+#define MTK_WED_WPDMA_CFG_BASE 0x580
2659+
2660+#define MTK_WED_WPDMA_TX_MIB(_n) (0x5a0 + (_n) * 4)
2661+#define MTK_WED_WPDMA_TX_COHERENT_MIB(_n) (0x5d0 + (_n) * 4)
2662+
2663+#define MTK_WED_WPDMA_RING_TX(_n) (0x600 + (_n) * 0x10)
2664+#define MTK_WED_WPDMA_RING_RX(_n) (0x700 + (_n) * 0x10)
2665+#define MTK_WED_WDMA_RING_RX(_n) (0x900 + (_n) * 0x10)
2666+#define MTK_WED_WDMA_RX_THRES(_n) (0x940 + (_n) * 0x4)
2667+
2668+#define MTK_WED_WDMA_GLO_CFG 0xa04
2669+#define MTK_WED_WDMA_GLO_CFG_TX_DRV_EN BIT(0)
2670+#define MTK_WED_WDMA_GLO_CFG_RX_DRV_EN BIT(2)
2671+#define MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY BIT(3)
2672+#define MTK_WED_WDMA_GLO_CFG_BT_SIZE GENMASK(5, 4)
2673+#define MTK_WED_WDMA_GLO_CFG_TX_WB_DDONE BIT(6)
2674+#define MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE BIT(13)
2675+#define MTK_WED_WDMA_GLO_CFG_WCOMPLETE_SEL BIT(16)
2676+#define MTK_WED_WDMA_GLO_CFG_INIT_PHASE_RXDMA_BYPASS BIT(17)
2677+#define MTK_WED_WDMA_GLO_CFG_INIT_PHASE_BYPASS BIT(18)
2678+#define MTK_WED_WDMA_GLO_CFG_FSM_RETURN_IDLE BIT(19)
2679+#define MTK_WED_WDMA_GLO_CFG_WAIT_COHERENT BIT(20)
2680+#define MTK_WED_WDMA_GLO_CFG_AXI_W_AFTER_AW BIT(21)
2681+#define MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY_SINGLE_W BIT(22)
2682+#define MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY BIT(23)
2683+#define MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP BIT(24)
2684+#define MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE BIT(25)
2685+#define MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE BIT(26)
2686+#define MTK_WED_WDMA_GLO_CFG_RXDRV_CLKGATE_BYPASS BIT(30)
2687+
2688+#define MTK_WED_WDMA_RESET_IDX 0xa08
2689+#define MTK_WED_WDMA_RESET_IDX_RX GENMASK(17, 16)
2690+#define MTK_WED_WDMA_RESET_IDX_DRV GENMASK(25, 24)
2691+
2692+#define MTK_WED_WDMA_INT_TRIGGER 0xa28
2693+#define MTK_WED_WDMA_INT_TRIGGER_RX_DONE GENMASK(17, 16)
2694+
2695+#define MTK_WED_WDMA_INT_CTRL 0xa2c
2696+#define MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL GENMASK(17, 16)
2697+
2698+#define MTK_WED_WDMA_OFFSET0 0xaa4
2699+#define MTK_WED_WDMA_OFFSET1 0xaa8
2700+
2701+#define MTK_WED_WDMA_RX_MIB(_n) (0xae0 + (_n) * 4)
2702+#define MTK_WED_WDMA_RX_RECYCLE_MIB(_n) (0xae8 + (_n) * 4)
2703+#define MTK_WED_WDMA_RX_PROCESSED_MIB(_n) (0xaf0 + (_n) * 4)
2704+
2705+#define MTK_WED_RING_OFS_BASE 0x00
2706+#define MTK_WED_RING_OFS_COUNT 0x04
2707+#define MTK_WED_RING_OFS_CPU_IDX 0x08
2708+#define MTK_WED_RING_OFS_DMA_IDX 0x0c
2709+
2710+#define MTK_WDMA_RING_RX(_n) (0x100 + (_n) * 0x10)
2711+
2712+#define MTK_WDMA_GLO_CFG 0x204
2713+#define MTK_WDMA_GLO_CFG_RX_INFO_PRERES GENMASK(28, 26)
2714+
2715+#define MTK_WDMA_RESET_IDX 0x208
2716+#define MTK_WDMA_RESET_IDX_TX GENMASK(3, 0)
2717+#define MTK_WDMA_RESET_IDX_RX GENMASK(17, 16)
2718+
2719+#define MTK_WDMA_INT_MASK 0x228
2720+#define MTK_WDMA_INT_MASK_TX_DONE GENMASK(3, 0)
2721+#define MTK_WDMA_INT_MASK_RX_DONE GENMASK(17, 16)
2722+#define MTK_WDMA_INT_MASK_TX_DELAY BIT(28)
2723+#define MTK_WDMA_INT_MASK_TX_COHERENT BIT(29)
2724+#define MTK_WDMA_INT_MASK_RX_DELAY BIT(30)
2725+#define MTK_WDMA_INT_MASK_RX_COHERENT BIT(31)
2726+
2727+#define MTK_WDMA_INT_GRP1 0x250
2728+#define MTK_WDMA_INT_GRP2 0x254
2729+
2730+#define MTK_PCIE_MIRROR_MAP(n) ((n) ? 0x4 : 0x0)
2731+#define MTK_PCIE_MIRROR_MAP_EN BIT(0)
2732+#define MTK_PCIE_MIRROR_MAP_WED_ID BIT(1)
2733+
2734+/* DMA channel mapping */
2735+#define HIFSYS_DMA_AG_MAP 0x008
2736+
2737+#endif
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]2738diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2739index ef44d9a..59a3e96 100644
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]2740--- a/include/linux/netdevice.h
2741+++ b/include/linux/netdevice.h
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2742@@ -844,6 +844,7 @@ enum net_device_path_type {
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]2743 DEV_PATH_BRIDGE,
2744 DEV_PATH_PPPOE,
2745 DEV_PATH_DSA,
2746+ DEV_PATH_MTK_WDMA,
2747 };
2748
2749 struct net_device_path {
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2750@@ -869,6 +870,12 @@ struct net_device_path {
developeree39bcf2023-06-16 08:03:30 +0800[diff] [blame]2751 int port;
2752 u16 proto;
2753 } dsa;
2754+ struct {
2755+ u8 wdma_idx;
2756+ u8 queue;
2757+ u16 wcid;
2758+ u8 bss;
2759+ } mtk_wdma;
2760 };
2761 };
2762
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2763diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h
2764new file mode 100644
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2765index 0000000..7e00cca
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2766--- /dev/null
2767+++ b/include/linux/soc/mediatek/mtk_wed.h
2768@@ -0,0 +1,131 @@
2769+#ifndef __MTK_WED_H
2770+#define __MTK_WED_H
2771+
2772+#include <linux/kernel.h>
2773+#include <linux/rcupdate.h>
2774+#include <linux/regmap.h>
2775+#include <linux/pci.h>
2776+
2777+#define MTK_WED_TX_QUEUES 2
2778+
2779+struct mtk_wed_hw;
2780+struct mtk_wdma_desc;
2781+
2782+struct mtk_wed_ring {
2783+ struct mtk_wdma_desc *desc;
2784+ dma_addr_t desc_phys;
2785+ int size;
2786+
2787+ u32 reg_base;
2788+ void __iomem *wpdma;
2789+};
2790+
2791+struct mtk_wed_device {
2792+#ifdef CONFIG_NET_MEDIATEK_SOC_WED
2793+ const struct mtk_wed_ops *ops;
2794+ struct device *dev;
2795+ struct mtk_wed_hw *hw;
2796+ bool init_done, running;
2797+ int wdma_idx;
2798+ int irq;
2799+
2800+ struct mtk_wed_ring tx_ring[MTK_WED_TX_QUEUES];
2801+ struct mtk_wed_ring txfree_ring;
2802+ struct mtk_wed_ring tx_wdma[MTK_WED_TX_QUEUES];
2803+
2804+ struct {
2805+ int size;
2806+ void **pages;
2807+ struct mtk_wdma_desc *desc;
2808+ dma_addr_t desc_phys;
2809+ } buf_ring;
2810+
2811+ /* filled by driver: */
2812+ struct {
2813+ struct pci_dev *pci_dev;
2814+
2815+ u32 wpdma_phys;
2816+
2817+ u16 token_start;
2818+ unsigned int nbuf;
2819+
2820+ u32 (*init_buf)(void *ptr, dma_addr_t phys, int token_id);
2821+ int (*offload_enable)(struct mtk_wed_device *wed);
2822+ void (*offload_disable)(struct mtk_wed_device *wed);
2823+ } wlan;
2824+#endif
2825+};
2826+
2827+struct mtk_wed_ops {
2828+ int (*attach)(struct mtk_wed_device *dev);
2829+ int (*tx_ring_setup)(struct mtk_wed_device *dev, int ring,
2830+ void __iomem *regs);
2831+ int (*txfree_ring_setup)(struct mtk_wed_device *dev,
2832+ void __iomem *regs);
2833+ void (*detach)(struct mtk_wed_device *dev);
2834+
2835+ void (*stop)(struct mtk_wed_device *dev);
2836+ void (*start)(struct mtk_wed_device *dev, u32 irq_mask);
2837+ void (*reset_dma)(struct mtk_wed_device *dev);
2838+
2839+ u32 (*reg_read)(struct mtk_wed_device *dev, u32 reg);
2840+ void (*reg_write)(struct mtk_wed_device *dev, u32 reg, u32 val);
2841+
2842+ u32 (*irq_get)(struct mtk_wed_device *dev, u32 mask);
2843+ void (*irq_set_mask)(struct mtk_wed_device *dev, u32 mask);
2844+};
2845+
2846+extern const struct mtk_wed_ops __rcu *mtk_soc_wed_ops;
2847+
2848+static inline int
2849+mtk_wed_device_attach(struct mtk_wed_device *dev)
2850+{
2851+ int ret = -ENODEV;
2852+
2853+#ifdef CONFIG_NET_MEDIATEK_SOC_WED
2854+ rcu_read_lock();
2855+ dev->ops = rcu_dereference(mtk_soc_wed_ops);
2856+ if (dev->ops)
2857+ ret = dev->ops->attach(dev);
2858+ else
2859+ rcu_read_unlock();
2860+
2861+ if (ret)
2862+ dev->ops = NULL;
2863+#endif
2864+
2865+ return ret;
2866+}
2867+
2868+#ifdef CONFIG_NET_MEDIATEK_SOC_WED
2869+#define mtk_wed_device_active(_dev) !!(_dev)->ops
2870+#define mtk_wed_device_detach(_dev) (_dev)->ops->detach(_dev)
2871+#define mtk_wed_device_start(_dev, _mask) (_dev)->ops->start(_dev, _mask)
2872+#define mtk_wed_device_tx_ring_setup(_dev, _ring, _regs) \
2873+ (_dev)->ops->tx_ring_setup(_dev, _ring, _regs)
2874+#define mtk_wed_device_txfree_ring_setup(_dev, _regs) \
2875+ (_dev)->ops->txfree_ring_setup(_dev, _regs)
2876+#define mtk_wed_device_reg_read(_dev, _reg) \
2877+ (_dev)->ops->reg_read(_dev, _reg)
2878+#define mtk_wed_device_reg_write(_dev, _reg, _val) \
2879+ (_dev)->ops->reg_write(_dev, _reg, _val)
2880+#define mtk_wed_device_irq_get(_dev, _mask) \
2881+ (_dev)->ops->irq_get(_dev, _mask)
2882+#define mtk_wed_device_irq_set_mask(_dev, _mask) \
2883+ (_dev)->ops->irq_set_mask(_dev, _mask)
2884+#else
2885+static inline bool mtk_wed_device_active(struct mtk_wed_device *dev)
2886+{
2887+ return false;
2888+}
2889+#define mtk_wed_device_detach(_dev) do {} while (0)
2890+#define mtk_wed_device_start(_dev, _mask) do {} while (0)
2891+#define mtk_wed_device_tx_ring_setup(_dev, _ring, _regs) -ENODEV
2892+#define mtk_wed_device_txfree_ring_setup(_dev, _ring, _regs) -ENODEV
2893+#define mtk_wed_device_reg_read(_dev, _reg) 0
2894+#define mtk_wed_device_reg_write(_dev, _reg, _val) do {} while (0)
2895+#define mtk_wed_device_irq_get(_dev, _mask) 0
2896+#define mtk_wed_device_irq_set_mask(_dev, _mask) do {} while (0)
2897+#endif
2898+
2899+#endif
2900diff --git a/net/core/dev.c b/net/core/dev.c
developer58aa0682023-09-18 14:02:26 +0800[diff] [blame]2901index a117bd0..1b6d42b 100644
developer8cb3ac72022-07-04 10:55:14 +0800[diff] [blame]2902--- a/net/core/dev.c
2903+++ b/net/core/dev.c
2904@@ -675,6 +675,10 @@ int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
2905 if (WARN_ON_ONCE(last_dev == ctx.dev))
2906 return -1;
2907 }
2908+
2909+ if (!ctx.dev)
2910+ return ret;
2911+
2912 path = dev_fwd_path(stack);
2913 if (!path)
2914 return -1;
2915--
29162.18.0
2917