I believe my changes were not propagated. I’ve created a new patch and I’m rebuilding everything. I’ll update about the result.
Btw. I’ve found wrong/missing config: PCIE_MEDIATEK_GEN3. PCIE_MEDIATEK supports only Gen2.
I have good and bad news.
My changes work. Kind of 
At least nvme is an available command.
MT7988> pci enum; nvme scan;nvme info
returns nothing.
MT7988> nvme detail
as well.
MT7988> dm tree
Class Seq Probed Driver Name
-----------------------------------------------------------
root 0 [ + ] root_driver root_driver
clk 0 [ ] fixed_clock |-- dummy40m
clk 1 [ + ] fixed_clock |-- dummy208m
firmware 0 [ ] psci |-- psci
syscon 0 [ ] syscon |-- hwver
syscon 1 [ + ] syscon |-- watchdog@1001c000
clk 2 [ + ] mt7988-clock-fixed-p |-- apmixedsys@1001e000
clk 3 [ + ] mt7988-clock-topckge |-- topckgen@1001b000
pinctrl 0 [ + ] mt7988_pinctrl |-- pinctrl@1001f000
gpio 0 [ + ] mediatek_gpio | |-- mediatek_gpio
pinconfig 0 [ ] pinconfig | |-- i2c1-pins
pinconfig 1 [ ] pinconfig | | `-- mux
pinconfig 2 [ ] pinconfig | |-- pwm-pins
pinconfig 3 [ ] pinconfig | | `-- mux
pinconfig 4 [ + ] pinconfig | |-- spi0-pins
pinconfig 5 [ ] pinconfig | | `-- mux
pinconfig 6 [ ] pinconfig | |-- mmc0default
pinconfig 7 [ ] pinconfig | | |-- mux
pinconfig 8 [ ] pinconfig | | |-- conf-cmd-dat
pinconfig 9 [ ] pinconfig | | |-- conf-clk
pinconfig 10 [ ] pinconfig | | |-- conf-dsl
pinconfig 11 [ ] pinconfig | | `-- conf-rst
pinconfig 12 [ + ] pinconfig | `-- mmc1default
pinconfig 13 [ ] pinconfig | |-- mux
pinconfig 14 [ ] pinconfig | |-- conf-cmd-dat
pinconfig 15 [ ] pinconfig | `-- conf-clk
clk 4 [ ] mt7988-clock-sgmiisy |-- syscon@10060000
clk 5 [ ] mt7988-clock-sgmiisy |-- syscon@10070000
syscon 2 [ + ] syscon |-- syscon@10080000
syscon 3 [ ] syscon |-- syscon@10081000
clk 6 [ ] fixed_clock |-- dummy12m
usb 0 [ ] xhci-mtk |-- xhci@11200000
phy 0 [ ] mtk-tphy |-- usb-phy@11c50000
syscon 4 [ + ] syscon |-- syscon@11f20000
syscon 5 [ ] syscon |-- syscon@11f30000
syscon 6 [ + ] syscon |-- syscon@11f40000
syscon 7 [ + ] syscon |-- topmisc@11d10000
clk 7 [ + ] mt7988-clock-infracf |-- infracfg@10001000
serial 0 [ + ] serial_mtk |-- serial@11000000
pwm 0 [ ] mtk_pwm |-- pwm@10048000
spi 0 [ + ] mtk_spim |-- spi@1100a000
mtd 0 [ + ] spi_nand | `-- spi_nand@0
mmc 0 [ + ] mtk_sd |-- mmc@11230000
blk 0 [ + ] mmc_blk | |-- [email protected]
partition 0 [ + ] blk_partition | | |-- [email protected]:1
partition 1 [ + ] blk_partition | | |-- [email protected]:2
partition 2 [ + ] blk_partition | | |-- [email protected]:3
partition 3 [ + ] blk_partition | | |-- [email protected]:4
partition 4 [ + ] blk_partition | | |-- [email protected]:5
partition 5 [ + ] blk_partition | | |-- [email protected]:6
partition 6 [ + ] blk_partition | | `-- [email protected]:7
bootdev 0 [ ] mmc_bootdev | `-- [email protected]
clk 8 [ + ] mt7988-clock-ethdma |-- syscon@15000000
reset 0 [ + ] mediatek_reset | |-- reset
syscon 8 [ + ] syscon | `-- syscon@15000000
clk 9 [ + ] mt7988-clock-ethwarp |-- syscon@15031000
reset 1 [ + ] mediatek_reset | |-- reset
syscon 9 [ + ] syscon | `-- syscon@15031000
ethernet 0 [ + ] mtk-eth |-- ethernet@15110100
bootdev 1 [ ] eth_bootdev | `-- [email protected]
regulator 0 [ + ] regulator_fixed |-- regulator-3p3v
regulator 1 [ + ] regulator_fixed |-- regulator-1p8v
button 0 [ + ] button_gpio |-- keys
button 1 [ + ] button_gpio | `-- wps
nop 0 [ ] gpio_led_wrap |-- leds
led 0 [ ] gpio_led | |-- led-green
led 1 [ ] gpio_led | `-- led-blue
bootstd 0 [ ] bootstd_drv |-- bootstd
bootmeth 0 [ ] bootmeth_extlinux | |-- extlinux
bootmeth 1 [ ] bootmeth_efi_mgr | |-- efi_mgr
bootmeth 2 [ ] bootmeth_efi | |-- efi
bootmeth 3 [ ] bootmeth_pxe | |-- pxe
bootmeth 4 [ ] vbe_simple | `-- vbe_simple
efi 0 [ ] EFI block driver |-- efi
simple_bus 0 [ ] cpu_bus `-- cpus
cpu 0 [ ] mtk-cpu |-- cpu@0
cpu 1 [ ] mtk-cpu |-- cpu@1
cpu 2 [ ] mtk-cpu |-- cpu@2
cpu 3 [ ] mtk-cpu `-- cpu@3
My changes: package/boot/uboot-mediatek/patches/450a-add-bpi-r4-nvme-pcie.patch:
diff --git a/configs/mt7988a_bananapi_bpi-r4-emmc_defconfig b/configs/mt7988a_bananapi_bpi-r4-emmc_defconfig
index 87c0932..a2b81ad 100644
--- a/configs/mt7988a_bananapi_bpi-r4-emmc_defconfig
+++ b/configs/mt7988a_bananapi_bpi-r4-emmc_defconfig
@@ -16,6 +16,9 @@ CONFIG_DEBUG_UART_BASE=0x11000000
CONFIG_DEBUG_UART_CLOCK=40000000
CONFIG_ENV_OFFSET_REDUND=0x440000
CONFIG_PCI=y
+CONFIG_NVME=y
+CONFIG_NVME_PCI=y
+CONFIG_BLK=y
CONFIG_DEBUG_UART=y
CONFIG_AHCI=y
CONFIG_FIT=y
@@ -73,6 +76,7 @@ CONFIG_CMD_FS_UUID=y
CONFIG_CMD_UBI=y
CONFIG_CMD_UBI_RENAME=y
CONFIG_CMD_MEMSIZE=y
+CONFIG_CMD_NVME=y
CONFIG_OF_EMBED=y
CONFIG_ENV_OVERWRITE=y
CONFIG_ENV_IS_IN_MMC=y
@@ -112,6 +116,7 @@ CONFIG_MTD_UBI_FASTMAP=y
CONFIG_PHY_FIXED=y
CONFIG_MEDIATEK_ETH=y
CONFIG_PCIE_MEDIATEK=y
+CONFIG_PCIE_MEDIATEK_GEN3=y
CONFIG_PHY=y
CONFIG_PHY_MTK_TPHY=y
CONFIG_PINCTRL=y
@@ -123,6 +128,7 @@ CONFIG_DM_REGULATOR=y
CONFIG_DM_REGULATOR_FIXED=y
CONFIG_DM_REGULATOR_GPIO=y
CONFIG_DM_PWM=y
+CONFIG_DM_PCI=y
CONFIG_PWM_MTK=y
CONFIG_RAM=y
CONFIG_SCSI=y
diff --git a/configs/mt7988a_bananapi_bpi-r4-sdmmc_defconfig b/configs/mt7988a_bananapi_bpi-r4-sdmmc_defconfig
index 027b100..325e2ad 100644
--- a/configs/mt7988a_bananapi_bpi-r4-sdmmc_defconfig
+++ b/configs/mt7988a_bananapi_bpi-r4-sdmmc_defconfig
@@ -16,6 +16,9 @@ CONFIG_DEBUG_UART_BASE=0x11000000
CONFIG_DEBUG_UART_CLOCK=40000000
CONFIG_ENV_OFFSET_REDUND=0x440000
CONFIG_PCI=y
+CONFIG_NVME=y
+CONFIG_NVME_PCI=y
+CONFIG_BLK=y
CONFIG_DEBUG_UART=y
CONFIG_AHCI=y
CONFIG_FIT=y
@@ -72,6 +75,7 @@ CONFIG_CMD_FS_GENERIC=y
CONFIG_CMD_FS_UUID=y
CONFIG_CMD_UBI=y
CONFIG_CMD_UBI_RENAME=y
+CONFIG_CMD_NVME=y
CONFIG_OF_EMBED=y
CONFIG_ENV_OVERWRITE=y
CONFIG_ENV_IS_IN_MMC=y
@@ -111,6 +115,7 @@ CONFIG_MTD_UBI_FASTMAP=y
CONFIG_PHY_FIXED=y
CONFIG_MEDIATEK_ETH=y
CONFIG_PCIE_MEDIATEK=y
+CONFIG_PCIE_MEDIATEK_GEN3=y
CONFIG_PHY=y
CONFIG_PHY_MTK_TPHY=y
CONFIG_PINCTRL=y
@@ -122,6 +127,7 @@ CONFIG_DM_REGULATOR=y
CONFIG_DM_REGULATOR_FIXED=y
CONFIG_DM_REGULATOR_GPIO=y
CONFIG_DM_PWM=y
+CONFIG_DM_PCI=y
CONFIG_PWM_MTK=y
CONFIG_RAM=y
CONFIG_SCSI=y
diff --git a/configs/mt7988a_bananapi_bpi-r4-snand_defconfig b/configs/mt7988a_bananapi_bpi-r4-snand_defconfig
index fb7924f..b296574 100644
--- a/configs/mt7988a_bananapi_bpi-r4-snand_defconfig
+++ b/configs/mt7988a_bananapi_bpi-r4-snand_defconfig
@@ -13,6 +13,9 @@ CONFIG_PRE_CON_BUF_ADDR=0x4007EF00
CONFIG_DEBUG_UART_BASE=0x11000000
CONFIG_DEBUG_UART_CLOCK=40000000
CONFIG_PCI=y
+CONFIG_NVME=y
+CONFIG_NVME_PCI=y
+CONFIG_BLK=y
CONFIG_DEBUG_UART=y
CONFIG_AHCI=y
CONFIG_FIT=y
@@ -70,6 +73,7 @@ CONFIG_CMD_FS_UUID=y
CONFIG_CMD_UBI=y
CONFIG_CMD_UBI_RENAME=y
CONFIG_CMD_MEMSIZE=y
+CONFIG_CMD_NVME=y
CONFIG_OF_EMBED=y
CONFIG_ENV_OVERWRITE=y
CONFIG_ENV_IS_IN_UBI=y
@@ -112,6 +116,7 @@ CONFIG_MTD_UBI_FASTMAP=y
CONFIG_PHY_FIXED=y
CONFIG_MEDIATEK_ETH=y
CONFIG_PCIE_MEDIATEK=y
+CONFIG_PCIE_MEDIATEK_GEN3=y
CONFIG_PHY=y
CONFIG_PHY_MTK_TPHY=y
CONFIG_PINCTRL=y
@@ -123,6 +128,7 @@ CONFIG_DM_REGULATOR=y
CONFIG_DM_REGULATOR_FIXED=y
CONFIG_DM_REGULATOR_GPIO=y
CONFIG_DM_PWM=y
+CONFIG_DM_PCI=y
CONFIG_PWM_MTK=y
CONFIG_RAM=y
CONFIG_SCSI=y
Any suggestions?
is the mtk_pcie_gen3 driver enabled in uboot?
else check dt nodes if all except the pcie2 (key-b) are enabled
Also enable dm_pci and cmd_pci so you can look at lower layer
I do not see enabling of pcie nodes in the r4 uboot dts…also there is no patch adding the pcie support to mt7988,but maybe it is already upstream.
Make sure this is there in your uboot source and enabling also for the right r4 dts (here only rfb)
1 Like
if you just want to be able to boot from NVME - my suggestion is to take an existing image (it could be with openwrt, or linux , doesnt matter ), find the one that supports nvme , update bootloader from this image or flash it to nand/emmc , adjust uboot config with your specific boot sequence for your specific setup
btw. i try to get nvme working in openwrt for R4+R4Pro
my own uboot works so far here for R4pro with nvme in cn14 (cn13 depends on pcie2 with special phy so not available in uboot)… looks like i miss something in openwrt yet (possibly the gpio-hog)
BPI-R4P> pci enum
BPI-R4P> pci scan
BusDevFun VendorId DeviceId Device Class Sub-Class
_____________________________________________________________
00.00.00 0x14c3 0x7988 Bridge device 0x04
01.00.00 0x1e4b 0x1202 Mass storage controller 0x08
BPI-R4P> nvme scan
BPI-R4P> nvme info
Device 0: Vendor: 0x1e4b Rev: GTdecda1 Prod: 1OXUW7Y39WJPT7YGL4QE
Type: Hard Disk
Capacity: 122104.3 MB = 119.2 GB (250069680 x 512)
BPI-R4P>
works now in my openwrt-uboot too 
and linux now too
here i have added nvme patch for R4 too (but untested), so you can try building R4 tree from here:
mhm, have flashed my uboot now (previously i did a chainload of same u-boot.bin) and now pcie does not find anything again ;( something strange, but at least openwrt works now
1 Like
I was looking for a good reference and have not found any device that could be a role model here. I haven’t found any config with CMD_NVME enabled in u-boot.
Your changes works with my BPI-R4.
Thank you
MT7988> pci enum
MT7988> pci scan
BusDevFun VendorId DeviceId Device Class Sub-Class
_____________________________________________________________
00.00.00 0x14c3 0x7988 Bridge device 0x04
01.00.00 0x2646 0x5028 Mass storage controller 0x08
02.00.00 0x14c3 0x7988 Bridge device 0x04
03.00.00 0x14c3 0x7990 Network controller 0x80
04.00.00 0x14c3 0x7988 Bridge device 0x04
05.00.00 0x14c3 0x7991 Network controller 0x80
MT7988> nvme scan
MT7988> nvme info
Device 0: Vendor: 0x2646 Rev: SDU00103 Prod: 50026B73XXXXXXX
Type: Hard Disk
Capacity: 953869.7 MB = 931.5 GB (1953525168 x 512)
I’ll try to experiment with upstream OpenWrt to make it working. I guess the best approach would be to upstream changes to OpenWrt, u-boot and kernel.
I was able to move those changes to upstream OpenWrt. Let me know if it’s fine from your end.
What exactly you want to get ? I have R4 running ubuntu from a nvme drive - which configuration do you need ? I can share
1 Like
It is ok, but i guess defconfig change will be rejected. You can install kmod-nvme in openwrt,but uboot change should be good so far.
Not exactly, it needs to be included to kernel in order to boot rootfs. There is also option with initram (initrd successor) and put kmod-nvme three, but my understanding is that it requires serial console and TFTP server:
I’ve noticed that configuration e.g. for rockchip and few other SoCs:
I have also done it like that because of this and also mentioned that loading a file from rootfs which is not available due to missing nvme driver
But lets see which responses you get.
1 Like
@Valentin_G please share how to configure that. My goal is to install upstream OpenWrt on nvme and boot from it. But of course I take into account other distros as well. That can help to resolve a primary goal.
Sorry for the delay
I use eMMC as my base boot option, with following U-Boot version
U-Boot 2025.01-bpi-geb2e9f450076-dirty (Apr 26 2025 - 08:25:19 +0000)
Which I got from one of @frank-w’s sdcard image with ubuntu and flashed the bootloader directly to eMMC partition. You can download couple of images and try them out booting from SDcard , without actual flashing. Bassically any image that has any NVME options and actually able to list nvme devices will be good to start with. I have my environment file ( uEnv.txt) on the boot partition of the eMMC, it is loaded on start. Here is the content of the file
askbootnetfit=run lstftp askkernel;if printenv kernelinput; then setenv bootfile "${kernelinput}"; run bootnetfit;fi
askbootnvme=run usenvme; if test "$device" = "nvme"; then run askkernel;if printenv kernelinput; then setenv fit $kernelinput;run newboot;fi;fi
askkernel=askenv kernelinput "enter kernel name:";
board=bpi-r4
boot0=run lskernel;run askkernel;if printenv kernelinput ;then setenv fit ${kernelinput}; run newboot; fi;
bootcmd=setenv bootdelay 3; run loadenv;bootmenu;
bootdelay=0
bootdevice=sd
bootenv=uEnv.txt
bootfile=bpi-r4.itb
bootmedia=emmc
bootmenu_0=1. Enter kernel-name to boot from SD/EMMC.=run boot0
bootmenu_1=2. Boot kernel from TFTP.=run askbootnetfit
bootmenu_2=3. Boot from SD/EMMC.=run newboot
bootmenu_3=4. Boot kernel from NVME.=run askbootnvme
bootmenu_4=5. Boot debian from nvme.=run bootnvmedebian
bootmenu_5=6. Boot ubuntu from nvme.=run bootnvmeubuntu
bootmenu_6=7. Update nand bootloader from nvme.=run nandnvmeupdate
bootmenu_default=5
bootnetfit=run checkrd; run setbootconf;run buildargs;if tftp $kaddr ${bootfile}; then bootm ${kaddr}${bootconf};fi
bootnvmedebian=run usenvme; setenv root $nvme_root_debian; run buildargs; ext4load nvme 0:1 $kaddr $debian_nvme_kernel;bootm $kaddr
bootnvmeubuntu=run usenvme; setenv root $nvme_root_ubuntu; run buildargs; ext4load nvme 0:1 $kaddr $ununtu_nvme_kernel;bootm $kaddr
bootopts=debug=7
buildargs=setenv bootargs "board=${board} console=${console} root=${root} ${bootopts}
check2g5=if test -n "$is2g5" && test $is2g5 -eq 1;then setenv variant "-2g5"; fi
checkenv=test -e ${device} ${partition} ${bootenv}
checkmmc=if mmc partconf 0; then echo "emmc available";mmcdev="emmc";else echo "sd available";mmcdev="sd";fi
checkrd=if printenv initrd;then if fatload ${device} ${partition} ${rdaddr} ${initrd};then run useinitrd;fi;fi
checksata=if test -n "$sata_en" && test $sata_en -eq 1;then setenv satacfg "#sata"; fi
checkwifi=if test -n "$wifi_en" && test $wifi_en -eq 1;then setenv wificfg "#wifi"; fi
console=ttyS0,115200n1 earlycon=uart8250,mmio32,0x11000000
debian_nvme_kernel=bpi-r4.itb
device=mmc
ethaddr=0a:5c:0a:d5:1e:bd
fdtcontroladdr=ff7fbda0
fileaddr=46000000
filesize=1392
fit=bpi-r4.itb
ipaddr=192.168.0.19
kaddr=0x46000000
loadaddr=0x46000000
loadenv=if run checkenv;then run loadenvfile;fi
loadenvfile=if fatload ${device} ${partition} ${loadaddr} ${bootenv};then env import -t ${loadaddr} ${filesize};else echo "fatload (${bootenv}) failed";fi
loadkernel=fatload $device $partition $kaddr ${kernel}
lskernel=ls ${device} ${partition};
lstftp=if tftp ${loadaddr} ${serverip}:r4-itbfiles.lst;then setexpr listend ${loadaddr} + ${filesize};mw.b ${listend} 00 2;strings ${loadaddr};else echo "tftp download failed";fi
nandnvmeupdate=run usenvme;if printenv nvme_bootloader_file;then if ext4load nvme 0:1 $loadaddr $nvme_bootloader_file;then mtd erase spi-nand0 0x0 0x100000;mtd write spi-nand0 $loadaddr 0x0 0x100000;else echo "loading bl2 $bl2file failed";fi;else echo "nvme_bootloader_file not defined";fi
netmask=255.255.255.0
newboot=run checkrd; run setbootconf; run buildargs;if printenv fit;then setenv kernel ${fit};if run loadkernel; then bootm ${kaddr}${bootconf}; fi;fi;
nvme_boot=0:1
nvme_bootloader_file=bl2_NAND.bin
nvme_root=/dev/nvme0n1p2
nvme_root_debian=/dev/nvme0n1p2
nvme_root_ubuntu=/dev/nvme0n1p3
partition=0:5
rdaddr=0x48000000
resetenv=env default -a;printenv;
root=/dev/mmcblk0p6 rootfstype=ext4 rootwait
sata_en=0
serverip=192.168.0.10
setbootconf=run checkmmc checksata checkwifi;setenv bootconf "#conf-$mmcdev$version$wificfg$satacfg";printenv bootconf
stderr=serial@11000000
stdin=serial@11000000
stdout=serial@11000000
uaddr=0x41e00000
ununtu_nvme_kernel=bpi-r4.itb
usb_part=0:1
useinitrd=setenv bootopts "${bootopts} initrd=${rdaddr},100M";
usenvme=pci enum; nvme scan;nvme info; if ls nvme $nvme_boot;then setenv device "nvme";setenv partition $nvme_boot;setenv root $nvme_root;else echo "nvme partition not found";fi
useusb=if usb info; then usb reset;else usb start;fi; if ls usb $usb_part $usb_path; then setenv device usb;setenv partition $usb_part;fi
wifi_en=1
wremmc=if printenv bl2file;then mmc partconf 0 1 1 1;if fatload $device $partition $loadaddr $bl2file;then mmc erase 0x0 0x400;mmc write ${loadaddr} 0x0 0x400;else echo "loading bl2 $bl2file failed";fi;else echo "bl2file not defined";fi;mmc partconf 0 1 1 0;if printenv fipfile;then if fatload $device $partition $loadaddr $fipfile;then mmc erase 0x3400 0x1000;mmc write ${loadaddr} 0x3400 0x1000;else echo "loading fip $fipfile failed";fi;else echo "fipfile not defined";fi
wrspimnand=if printenv bl2file;then if fatload $device $partition $loadaddr $bl2file;then mtd erase spi-nand0 0x0 0x100000;mtd write spi-nand0 $loadaddr 0x0 0x100000;else echo "loading bl2 $bl2file failed";fi;else echo "bl2file not defined";fi;if printenv fipfile;then if fatload $device $partition $loadaddr $fipfile;then mtd erase spi-nand0 0x580000 0x200000;mtd write spi-nand0 $loadaddr 0x580000 0x200000;else echo "loading fip $fipfile failed";fi;else echo "fipfile not defined";fi
line that is responsible for nvme boot is
run usenvme; setenv root $nvme_root_ubuntu; run buildargs; ext4load nvme 0:1 $kaddr $ununtu_nvme_kernel;bootm $kaddr
it does loading of an itb file from first partition of nvme drive which is formated as ext4 I hope this helps
Also - a small lifihack - I’m using a Bluetooth ↔ UART bridge to access system console , which is more convinient than using wired adapter , and I can route this connection over ssh and it becomes something like Serial Over Lan
1 Like
This looks really great.
Which bluetooth adapter have you used? As mtk filogic had some issues with most uart adapters, mainly ftdi is used and afaik there are no bluetooth adapters with ftdi backend (i only know the hc0x) it would be interesting for me too.
I’ve used this adapter - Grove - Serial Bluetooth v3.0 | Seeed Studio Wiki works great. the base is actually the same as HC0x modules, there are modules wit BLE chips - and those are not good for serial port forwarding, really hard to get it working
The datasheet for bc417 mentions weak internal pull-up/pulldown, apparantly weak enough not to disturb wifi. Or does the R4 not have this problem anyway? I only tried ft232rl.