Banana Pi BPI-R4 Wifi 7 router board with MediaTek MT7988A (Filogic 880),4G RAM and 8G eMMC

The reason for PPE not working on the R4 (but working fine on MT7988 reference board) is that the R4 got 4 GiB of RAM and hence requires DMA addressing beyond 32-bit boundaries (memory starts from 0x40000000, so only up to 3 GiB [0x40000000~0xFFFFFFFF] can be addressed in 32-bits. For the Ethernet DMA I’ve already adapted some changes found in MTK SDK to support 36-bit DMA, however, something is still missing in that regard for PPE to work as well.

In the meantime, and while @LorenzoBianconi and me are working on finding a good solution for the driver in mainline Linux and OpenWrt, you can test the Ethernet performance of the R4 with NAT offloading via PPE by limiting memory e.g. to 2GB by appending mem=2048M to the bootargs U-Boot environment variable.

2 Likes

The one single thing not entirely Open Source but needed for a tight trusted platform implementation is the anti-rollback protection part of the SoC’s bootrom using the efuse, for that proprietary tools are still needed.

Thanks a lot, I will try that.

I was trying to expand on @Dale patch to include more changes besides enabling 3 ppe, but it looks like the reason was 4G RAM.

--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1620,6 +1620,8 @@ struct net_device_ops {
 	struct net_device *	(*ndo_get_peer_dev)(struct net_device *dev);
 	int                     (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
                                                          struct net_device_path *path);
+	int                     (*ndo_fill_receive_path)(struct net_device_path_ctx *ctx,
+							 struct net_device_path *path);
 	ktime_t			(*ndo_get_tstamp)(struct net_device *dev,
 						  const struct skb_shared_hwtstamps *hwtstamps,
 						  bool cycles);
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index a84e70e..a122e50 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -2306,7 +2306,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 		}
 
 		if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
-			mtk_ppe_check_skb(eth->ppe[0], skb, hash);
+			mtk_ppe_check_skb(eth->ppe[eth->mac[mac]->ppe_idx], skb, hash);
 
 		skb_record_rx_queue(skb, 0);
 		napi_gro_receive(napi, skb);
@@ -3512,6 +3512,15 @@ static int mtk_open(struct net_device *dev)
 
 		gdm_config = soc->offload_version ? soc->reg_map->gdma_to_ppe
 						  : MTK_GDMA_TO_PDMA;
+
+		if (mac->id == 2) {
+			mac->ppe_idx = 2;
+			gdm_config = MTK_GDMA_TO_PPE2;
+		} else if (mac->id == 1) {
+			mac->ppe_idx = 1;
+			gdm_config = MTK_GDMA_TO_PPE1;
+		}
+
 		mtk_gdm_config(eth, gdm_config);
 
 		napi_enable(&eth->tx_napi);
@@ -4601,6 +4610,7 @@ static const struct net_device_ops mtk_netdev_ops = {
 	.ndo_get_stats64        = mtk_get_stats64,
 	.ndo_fix_features	= mtk_fix_features,
 	.ndo_set_features	= mtk_set_features,
+	.ndo_fill_receive_path	= mtk_eth_fill_receive_path,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= mtk_poll_controller,
 #endif
@@ -4849,6 +4859,16 @@ static int mtk_sgmii_init(struct mtk_eth *eth)
 	return 0;
 }
 
+int mtk_eth_fill_receive_path(struct net_device_path_ctx *ctx,
+			      struct net_device_path *path)
+{
+	struct mtk_mac *mac = netdev_priv(ctx->dev);
+
+	path->mtk_wdma.wdma_idx = mac->ppe_idx;
+
+	return 0;
+}
+
 static int mtk_probe(struct platform_device *pdev)
 {
 	struct resource *res = NULL, *res_sram;
@@ -5078,10 +5098,12 @@ static int mtk_probe(struct platform_device *pdev)
 
 	if (eth->soc->offload_version) {
 		u32 num_ppe = mtk_is_netsys_v2_or_greater(eth) ? 2 : 1;
+		if(mtk_is_netsys_v3_or_greater(eth))
+			num_ppe = 3;
 
 		num_ppe = min_t(u32, ARRAY_SIZE(eth->ppe), num_ppe);
 		for (i = 0; i < num_ppe; i++) {
-			u32 ppe_addr = eth->soc->reg_map->ppe_base + i * 0x400;
+			u32 ppe_addr = eth->soc->reg_map->ppe_base + (i == 2 ? 0xC00 : i * 0x400);
 
 			eth->ppe[i] = mtk_ppe_init(eth, eth->base + ppe_addr, i);
 
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index e80f6fb..1232acf 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -125,6 +125,10 @@
 #define MTK_GDMA_TO_PDMA	0x0
 #define MTK_GDMA_DROP_ALL       0x7777
 
+#define MTK_GDMA_TO_PPE0	0x3333
+#define MTK_GDMA_TO_PPE1	0x4444
+#define MTK_GDMA_TO_PPE2	0xcccc
+
 /* GDM Egress Control Register */
 #define MTK_GDMA_EG_CTRL(x)	({ typeof(x) _x = (x); (_x == MTK_GMAC3_ID) ?	\
 				   0x544 : 0x504 + (_x * 0x1000); })
@@ -1440,7 +1444,7 @@ struct mtk_eth {
 
 	struct metadata_dst		*dsa_meta[MTK_MAX_DSA_PORTS];
 
-	struct mtk_ppe			*ppe[2];
+	struct mtk_ppe			*ppe[3];
 	struct rhashtable		flow_table;
 
 	struct bpf_prog			__rcu *prog;
@@ -1465,6 +1469,7 @@ struct mtk_eth {
 struct mtk_mac {
 	int				id;
 	phy_interface_t			interface;
+	unsigned int			ppe_idx;
 	int				speed;
 	struct device_node		*of_node;
 	struct phylink			*phylink;
@@ -1625,6 +1630,8 @@ int mtk_flow_offload_cmd(struct mtk_eth *eth, struct flow_cls_offload *cls,
 			 int ppe_index);
 void mtk_flow_offload_cleanup(struct mtk_eth *eth, struct list_head *list);
 void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev);
+int mtk_eth_fill_receive_path(struct net_device_path_ctx *ctx,
+			      struct net_device_path *path);
 
 static inline int mtk_mac2xgmii_id(struct mtk_eth *eth, int mac_id)
 {
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
index 0fc2222..c54b986 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
@@ -123,8 +123,12 @@ struct mtk_foe_accounting *mtk_ppe_mib_entry_read(struct mtk_ppe *ppe, u16 index
 	if (mtk_is_netsys_v3_or_greater(ppe->eth)) {
 		/* 64 bit for each counter */
 		u32 cnt_r3 = readl(ppe->base + MTK_PPE_MIB_SER_R3);
-		acct->bytes += ((u64)cnt_r1 << 32) | cnt_r0;
-		acct->packets += ((u64)cnt_r3 << 32) | cnt_r2;
+		u32 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0);
+		u32 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH_V2, cnt_r1);
+		u32 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_LOW_V2, cnt_r2);
+		u32 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R3_PKT_CNT_HIGH, cnt_r3);
+		acct->bytes += ((u64)byte_cnt_high << 32) | byte_cnt_low;
+		acct->packets += ((u64)pkt_cnt_high << 32) | pkt_cnt_low;
 	} else {
 		/* 48 bit byte counter, 40 bit packet counter */
 		u32 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0);
@@ -1061,6 +1065,11 @@ void mtk_ppe_start(struct mtk_ppe *ppe)
 	      MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
 	      MTK_PPE_GLO_CFG_IP4_CS_DROP |
 	      MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
+
+	if (mtk_is_netsys_v3_or_greater(ppe->eth)) {
+		val |= MTK_PPE_GLO_CFG_CS0_PIPE_EN;
+	    val |= MTK_PPE_GLO_CFG_SRH_CACHE_FIRST_EN;
+	}
 	ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
 
 	ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
index 3ce088e..695653e 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
@@ -18,6 +18,8 @@
 #define MTK_PPE_GLO_CFG_UDP_LITE_EN		BIT(10)
 #define MTK_PPE_GLO_CFG_UDP_LEN_DROP		BIT(11)
 #define MTK_PPE_GLO_CFG_MCAST_ENTRIES		GNEMASK(13, 12)
+#define MTK_PPE_GLO_CFG_CS0_PIPE_EN		BIT(29)
+#define MTK_PPE_GLO_CFG_SRH_CACHE_FIRST_EN	BIT(30)
 #define MTK_PPE_GLO_CFG_BUSY			BIT(31)
 
 #define MTK_PPE_FLOW_CFG			0x204
@@ -159,11 +161,14 @@ enum {
 #define MTK_PPE_MIB_SER_R1			0x344
 #define MTK_PPE_MIB_SER_R1_PKT_CNT_LOW		GENMASK(31, 16)
 #define MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH	GENMASK(15, 0)
+#define MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH_V2	GENMASK(31, 0)
 
 #define MTK_PPE_MIB_SER_R2			0x348
 #define MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH		GENMASK(23, 0)
+#define MTK_PPE_MIB_SER_R2_PKT_CNT_LOW_V2	GENMASK(31, 0)
 
 #define MTK_PPE_MIB_SER_R3			0x34c
+#define MTK_PPE_MIB_SER_R3_PKT_CNT_HIGH		GENMASK(31, 0)
 
 #define MTK_PPE_MIB_CACHE_CTL			0x350
 #define MTK_PPE_MIB_CACHE_CTL_EN		BIT(0)
--
libgit2 1.7.1

No, DOCSIS requires specialized chips that only MaxLinear and Broadcom make. You’ll at the very least need a standalone cable modem.

Thank you! You helped me saving a lot of time for searching!

1 Like

I’ve just tried the M.2 Key-M socket on the back of the board with a cheap NVMe SSD (S3+ S3SSD240) and trying to read AT24 I2C EEPROM in the same bus which is also used as SMBus (MFG Data and MFG Clock) of the M.2 socket (see schematics excerpt below):

image

image

The problem here is that the presence of the SSD doesn’t only result in the at24 EEPROM (and probably also the RTC, I didn’t try) not being accessible any more, but also all other busses on the mux stop working and hence SFP modules can no longer be probed.

As MFG Data and MFG Clock signals are hardly ever needed, I suggest to not populate R228 and R230 in future versions.

please make sure you use the right revision in feeds.conf before compiling the image.

I checked the datasheets. In the Platform Datasheet, I found one reference:

In the registers datasheet part 1, I found several references, but I’m not sure if they’re relevant:

first one is only the register for accessing the efuse corresponding to the dt node

second seems are parts of efuse used by other subsystems (netsys,infracfg), but no information about how to write the efuse…maybe there is a hint in the first part (set bit X to have write access and then push data to Y)

@moore I’ve followed the steps exactly for Non-MLO SDK Release (20240112), all revisions are correct but it’s throwing error when building kernel

scripts/Makefile.build:42: /scripts/basic/Makefile: No such file or directory
make[5]: *** No rule to make target '/scripts/basic/Makefile'.  Stop.
make[5]: Leaving directory '/home/graphine/Desktop/mtk-openwrt/openwrt/build_dir/target-aarch64_cortex-a53_musl/linux-mediatek_mt7988/linux-5.4.260'
make[5]: *** [Makefile:530: scripts_basic] Error 2
make[4]: *** [Makefile:32: /home/graphine/Desktop/mtk-openwrt/openwrt/build_dir/target-aarch64_cortex-a53_musl/linux-mediatek_mt7988/linux-5.4.260/.configured] Error 2
make[4]: Leaving directory '/home/graphine/Desktop/mtk-openwrt/openwrt/target/linux/mediatek'
make[3]: *** [Makefile:11: compile] Error 2
make[3]: Leaving directory '/home/graphine/Desktop/mtk-openwrt/openwrt/target/linux'
time: target/linux/compile#7.43#2.90#9.17
    ERROR: target/linux failed to build.

I can build images successfully by using the ubuntu 18.04. I guess it’s a build environment issue.

I’ve modified the ugly patch. Ppe seems to be working now even with 4GB. Although I expected all 3 PPEs to work, and only ppe0 works. I tried with different gmacs.

I don’t expect this to be merged anywhere .

@dangowrt

1 Like

Has anyone ran some nvme ssd speed test? Can we expect to reach 10G speeds via SMB/NFS?

10G from SSD will not be possible as there is only a PCIe Gen-3 (8 GT/s) x1. Yes, only one lane! On R3 it was PCIe Gen-2 (5 GT/s) x2 which ends up a bit faster than the single Gen-3 lane on the R4.

1 Like

I see, since it’s gen 3 link, we should hopefully see around 7. I just installed a 980 formatted as f2fs in the slot, and I’m getting around 65 MB/s over samba 4 and connected to one of the 1G ports of the switch. Not able to test 10G port yet, but for 1G port it uses around 20% CPU. Not sure if the PPE comes into play to offload some parts.

No youll only see around 1GB/s on the PCIe port on the BPI-R4, to me it’s a waste to put Samsung 980 in it, id just get some cheaper one from other well known brands, afaik PPE isn’t working on dangowrt build because of 4gb ram

PPE works on my bpi-r4 branch, but even without it I was able to get 9.5gbps~ with Mellanox connectx3 on the other end

Can you share a link

2 Likes

I’ve looked at the patch and the part relevant to get PPE working with 4 GiB boils down to

Using all 3 PPE units is still on another page, of course…

I’ve also figured out why 100M/10M speeds were not working and got a fix for that as well

Please re-test with my updated tree on [WIP/RFC] mediatek: switch to Linux 6.1 and add BananaPi BPi-R4 by dangowrt · Pull Request #14140 · openwrt/openwrt · GitHub or build from mt7988-for-next branch of GitHub - dangowrt/linux: Linux kernel source tree .

2 Likes