Weird behaviour of PCIe controller on RPI4
Posted: Thu Jan 30, 2020 9:19 am
I want to get PCIe controller running on RPI4. It's almost working, but not exactly right. There is a weird problem with accessing configuration space. USB controller (which is, I believe, the only PCIe device in RPI4, apart from root complex) is responding to every even device number on bus 0. Odd numbers all return all FFs in their configuration space (which is expected). But the weirdest part is that I get SErr exceptions (which is basically bus error in AArch64) when accessing odd numbered devices AFTER accessing even numbered devices or accessing odd numbered device twice. I can access even devices as much as I want without generating exception. There is also that delay (about 12 seconds) before exceptions is signalled after bad access.
Here is my initialization code
I don't fully understand all of it, because, as it always is with Broadcom devices, there no hint of documentation to be found anywhere. This code is based on 2 only pieces of Broadcom PCIe code I found on the Internet. One is Linux driver and second is Plan 9 driver.
I also don't fully understand used terminology either. What is MDIO, GISB, SCB, inbound and outbound memory?
And here is how I access configuration space
All controller MMIO space (0xFE500000 to 0xFE50A000) is configured as nGnRE device memory. Caches and MMU are enabled.
Here is my initialization code
Code: Select all
static void writeField(ulong addr, u32 mask, unsigned shift, u32 val)
{
u32 tmp = CPU::MMIORead32(addr);
tmp &= ~mask;
tmp |= (val << shift) & mask;
CPU::MMIOWrite32(addr, val);
}
static void setGen(unsigned gen)
{
u32 lnkcap = CPU::MMIORead32(pcieBase + CAP_REGS + PCI_EXP_LNKCAP);
u16 lnkctl2 = CPU::MMIORead16(pcieBase + CAP_REGS + PCI_EXP_LNKCTL2);
lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen;
CPU::MMIOWrite32(pcieBase + CAP_REGS + PCI_EXP_LNKCAP, lnkcap);
lnkctl2 = (lnkctl2 & ~0xFu) | static_cast<u16>(gen);
CPU::MMIOWrite16(pcieBase + CAP_REGS + PCI_EXP_LNKCTL2, lnkctl2);
}
static inline unsigned makeIdx(unsigned bus, unsigned dev, unsigned fun)
{
bus &= BUS_MASK; dev &= DEV_MASK; fun &= FUN_MASK;
return bus << 20 | dev << 15 | fun << 12;
}
errcode PCI::Initialize()
{
errcode stat = ESUCCESS;
pcieBase = Paging::MapMMIO(&stat, PCIE_BASE, PCIE_SIZE);
if(stat != ESUCCESS)
{
pcieBase = 0;
return stat;
}
// assert bridge reset
writeField(pcieBase + RGR1_SW_INIT_1, RGR1_SW_INIT_1_INIT_MASK, RGR1_SW_INIT_1_INIT_SHIFT, 1);
Time::Sleep(1, false);
// assert fundamental reset
writeField(pcieBase + RGR1_SW_INIT_1, RGR1_SW_INIT_1_PERST_MASK, RGR1_SW_INIT_1_PERST_SHIFT, 1);
Time::Sleep(1, false);
// deassert bridge reset
writeField(pcieBase + RGR1_SW_INIT_1, RGR1_SW_INIT_1_INIT_MASK, RGR1_SW_INIT_1_INIT_SHIFT, 0);
Time::Sleep(1, false);
// enable serdes
writeField(pcieBase + HARD_PCIE_HARD_DEBUG, HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK,
HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_SHIFT, 0);
Time::Sleep(1, false);
// get hardware revision
hwRev = CPU::MMIORead32(pcieBase + PCIE_REVISION) & 0xFFFFu;
// disable and clear any pending interrupts
CPU::MMIOWrite32(pcieBase + MSI_INTR2_BASE + INTR_CLR, 0xFFFFFFFFu);
CPU::MMIOWrite32(pcieBase + MSI_INTR2_BASE + INTR_MASK_SET, 0xFFFFFFFFu);
// what does this exactly do ???
CPU::MMIOWrite32(pcieBase + CPU_2_PCIE_MEM_WIN0_LO, 0);
CPU::MMIOWrite32(pcieBase + CPU_2_PCIE_MEM_WIN0_HI, 0);
CPU::MMIOWrite32(pcieBase + CPU_2_PCIE_MEM_WIN0_BASE_LIMIT, 0);
CPU::MMIOWrite32(pcieBase + CPU_2_PCIE_MEM_WIN0_BASE_HI, 0);
CPU::MMIOWrite32(pcieBase + CPU_2_PCIE_MEM_WIN0_LIMIT_HI, 0);
// initialize SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN and CTRL_SCB0_SIZE
CPU::MMIOWrite32(pcieBase + MISC_CTRL,
CTRL_SCB0_SIZE(LOG2DMASIZE - 15) |
MAX_BURST_SIZE(BURST_SIZE_128) |
CFG_READ_UR_MODE(1) |
SCB_ACCESS_EN(1));
// setup inbound memory view
CPU::MMIOWrite32(pcieBase + RC_BAR2_CONFIG_LO, (LOG2DMASIZE - 15));
CPU::MMIOWrite32(pcieBase + RC_BAR2_CONFIG_HI, 0);
// disable PCIe->GISB and PCIe->SCB
CPU::MMIOWrite32(pcieBase + RC_BAR1_CONFIG_LO, 0);
CPU::MMIOWrite32(pcieBase + RC_BAR3_CONFIG_LO, 0);
// setup MSIs
CPU::MMIOWrite32(pcieBase + MSI_BAR_CONFIG_LO, (MSI_TARGET_ADDR & 0xFFFFFFFFu) | 1);
CPU::MMIOWrite32(pcieBase + MSI_BAR_CONFIG_HI, MSI_TARGET_ADDR >> 32);
CPU::MMIOWrite32(pcieBase + MSI_DATA_CONFIG, hwRev >= HW_REV_33 ? 0xffe06540 : 0xFFF86540);
// TODO: add MSI handler registration here
// cap controller to Gen2
setGen(2);
// deassert fundamental reset
writeField(pcieBase + RGR1_SW_INIT_1, RGR1_SW_INIT_1_PERST_MASK, RGR1_SW_INIT_1_PERST_SHIFT, 0);
for(unsigned int i = 0; i < 10; ++i)
{
if((CPU::MMIORead32(pcieBase + PCIE_STATUS) & 0x30) == 0x30)
break;
Time::Sleep(100, false);
}
// check if link is up
if((CPU::MMIORead32(pcieBase + PCIE_STATUS) & 0x30) != 0x30)
{
Debug::PutFmt("PCIe link is down\n");
return EERROR;
}
Debug::PutFmt("PCIe link is up\n");
// check if controller is running in root complex mode
if((CPU::MMIORead32(pcieBase + PCIE_STATUS) & 0x80) != 0x80)
{
Debug::PutFmt("PCIe controller is not running in root complex mode\n");
return EERROR;
}
// set proper class Id
CPU::MMIOWrite32(pcieBase + RC_CFG_PRIV1_ID_VAL3, 0x060400);
// set proper endian
writeField(pcieBase + RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1,
RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK,
RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_SHIFT,
DATA_ENDIAN);
// set debug mode
writeField(pcieBase + HARD_PCIE_HARD_DEBUG, HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK,
HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_SHIFT, 1);
return ESUCCESS;
}
I also don't fully understand used terminology either. What is MDIO, GISB, SCB, inbound and outbound memory?
And here is how I access configuration space
Code: Select all
u32 PCI::ConfigRead32(unsigned bus, unsigned dev, unsigned fun, unsigned reg)
{
cfgLock.Lock();
u32 res = 0;
if(!bus && !dev)
res = CPU::MMIORead32(pcieBase + reg);
else
{
CPU::MMIOWrite32(pcieBase + EXT_CFG_INDEX, makeIdx(bus, dev, fun));
res = CPU::MMIORead32(pcieBase + EXT_CFG_DATA + reg);
}
cfgLock.Unlock();
return res;
}
void PCI::ConfigWrite32(unsigned bus, unsigned dev, unsigned fun, unsigned reg, u32 val)
{
cfgLock.Lock();
if(!bus && !dev)
CPU::MMIOWrite32(pcieBase + reg, val);
else
{
CPU::MMIOWrite32(pcieBase + EXT_CFG_INDEX, makeIdx(bus, dev, fun));
CPU::MMIOWrite32(pcieBase + EXT_CFG_DATA + reg, val);
}
cfgLock.Unlock();
}
It must be the range then?
). Keep up the good work!