/* * IwxTrans.cpp * Intel AX210/AX211 PCIe transport: MMIO access, MSI-X, DMA rings, * context-info-gen3 firmware boot, host commands and RX processing. * * The bring-up sequence follows Intel's iwlwifi/OpenBSD iwx flow: * prepare card -> sw reset -> APM init -> MSI-X config -> rfkill check * -> context info gen3 (firmware self-load) -> ALIVE -> PNVM -> init cmds * * Everything here runs in process/idle context. Hard interrupts only latch * a "work pending" flag; all ring processing happens in IwxProcessEvents(), * which the idle loop and the command wait loops pump. This mirrors the * Bluetooth/E1000E split in this kernel: the network and IPC layers take * process-context mutexes that must never be entered from an IRQ. * * Copyright (c) 2026 Daniel Hammer */ #include "Iwx.hpp" #include "Ieee80211.hpp" #include #include #include #include #include #include #include #include #include #include using namespace Kt; namespace Drivers::Net::Wifi { IwxState g_iwx; // MSI-X/MSI interrupt slot. Slots in use elsewhere: 24 (E1000E), // 27 (Intel HDA), 28 (GPU vblank), 29 (xHCI). static constexpr uint8_t MSI_IRQ = 25; static constexpr uint32_t MSI_VECTOR = 57; // IRQ_VECTOR_BASE + MSI_IRQ static constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000; // Cached MSI-X masks (a cause is ENABLED when its mask bit is clear). static uint32_t g_fhInitMask = 0; static uint32_t g_hwInitMask = 0; static uint32_t g_fhMask = 0; static uint32_t g_hwMask = 0; static bool g_msix = false; // ========================================================================= // Register access // ========================================================================= uint32_t IwxRead32(uint32_t reg) { return *(volatile uint32_t*)(g_iwx.Mmio + reg); } void IwxWrite32(uint32_t reg, uint32_t val) { *(volatile uint32_t*)(g_iwx.Mmio + reg) = val; } void IwxWrite8(uint32_t reg, uint8_t val) { *(volatile uint8_t*)(g_iwx.Mmio + reg) = val; } void IwxSetBits(uint32_t reg, uint32_t bits) { IwxWrite32(reg, IwxRead32(reg) | bits); } void IwxClearBits(uint32_t reg, uint32_t bits) { IwxWrite32(reg, IwxRead32(reg) & ~bits); } // Approximate busy-wait. The kernel has no sub-millisecond clock source // exposed here; the loop count is calibrated the same way IntelHda does it. void IwxDelayUs(uint32_t us) { for (volatile uint32_t i = 0; i < us * 100u; i++) { asm volatile("pause" ::: "memory"); } } void IwxDelayMs(uint32_t ms) { uint64_t start = Timekeeping::GetMilliseconds(); while (Timekeeping::GetMilliseconds() - start < ms) { asm volatile("pause" ::: "memory"); } } bool IwxPollBit(uint32_t reg, uint32_t bits, uint32_t mask, int timeoutUs) { for (int t = 0; t < timeoutUs; t += 10) { if ((IwxRead32(reg) & mask) == bits) return true; IwxDelayUs(10); } return false; } // ========================================================================= // NIC lock (MAC wakeup for PRPH / internal register access) // ========================================================================= bool IwxNicLock() { if (g_iwx.NicLockCount > 0) { g_iwx.NicLockCount++; return true; } IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); IwxDelayUs(2); if (IwxPollBit(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | IWX_CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP, 150000)) { g_iwx.NicLockCount = 1; return true; } KernelLogStream(WARNING, "WiFi") << "Timeout waiting for NIC access"; return false; } void IwxNicUnlock() { if (g_iwx.NicLockCount > 0) { if (--g_iwx.NicLockCount == 0) IwxClearBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); } } // PRPH accessors. Bits 24-25 of the address register encode "number of // bytes - 1"; the driver only ever does dword accesses (3). // // The address field is 24 bits wide on the AX210 family (20 bits on older // parts). This matters: UMAC PRPH registers are reached by adding // IWX_UMAC_PRPH_OFFSET (0x300000), so e.g. UREG_CPU_INIT_RUN becomes // 0xd05c44 -- a 20-bit mask truncates that to 0x005c44 and the access // silently lands on an unrelated register. With the "kick firmware // self-load" write going nowhere, the firmware never boots and ALIVE // never arrives. static uint32_t PrphAddr(uint32_t addr) { return ((addr & 0x00FFFFFF) | (3 << 24)); } uint32_t IwxReadPrph(uint32_t addr) { IwxWrite32(IWX_HBUS_TARG_PRPH_RADDR, PrphAddr(addr)); asm volatile("" ::: "memory"); return IwxRead32(IWX_HBUS_TARG_PRPH_RDAT); } void IwxWritePrph(uint32_t addr, uint32_t val) { IwxWrite32(IWX_HBUS_TARG_PRPH_WADDR, PrphAddr(addr)); asm volatile("" ::: "memory"); IwxWrite32(IWX_HBUS_TARG_PRPH_WDAT, val); } uint32_t IwxReadUmacPrph(uint32_t addr) { return IwxReadPrph(addr + IWX_UMAC_PRPH_OFFSET); } void IwxWriteUmacPrph(uint32_t addr, uint32_t val) { IwxWritePrph(addr + IWX_UMAC_PRPH_OFFSET, val); } // ========================================================================= // DMA allocation // ========================================================================= bool IwxDmaAlloc(IwxDma& dma, uint64_t bytes) { if (bytes == 0) return false; uint32_t pages = (uint32_t)((bytes + 0xFFF) / 0x1000); void* base = Memory::g_pfa->Allocate(); if (!base) return false; if (pages > 1) { base = Memory::g_pfa->ReallocConsecutive(base, (int)pages); if (!base) return false; } memset(base, 0, (size_t)pages * 0x1000); dma.Virt = base; dma.Phys = Memory::SubHHDM(base); dma.Pages = pages; return true; } void IwxDmaFree(IwxDma& dma) { if (!dma.Virt) return; if (dma.Pages > 1) Memory::g_pfa->Free(dma.Virt, (int)dma.Pages); else Memory::g_pfa->Free(dma.Virt); dma.Virt = nullptr; dma.Phys = 0; dma.Pages = 0; } // ========================================================================= // Interrupt configuration // ========================================================================= static void IwxDisableInterrupts() { if (!g_msix) { IwxWrite32(IWX_CSR_INT_MASK, 0); IwxWrite32(IWX_CSR_INT, ~0u); IwxWrite32(IWX_CSR_FH_INT_STATUS, ~0u); } else { IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, g_fhInitMask); IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, g_hwInitMask); } } static void IwxEnableInterrupts() { if (!g_msix) { // Non-MSI-X path: enable the causes we act on. constexpr uint32_t INT_BIT_FH_RX = 1u << 31; constexpr uint32_t INT_BIT_HW_ERR = 1u << 29; constexpr uint32_t INT_BIT_FH_TX = 1u << 27; constexpr uint32_t INT_BIT_SW_ERR = 1u << 25; constexpr uint32_t INT_BIT_RF_KILL = 1u << 7; constexpr uint32_t INT_BIT_SW_RX = 1u << 3; constexpr uint32_t INT_BIT_ALIVE = 1u << 0; IwxWrite32(IWX_CSR_INT_MASK, INT_BIT_FH_RX | INT_BIT_HW_ERR | INT_BIT_FH_TX | INT_BIT_SW_ERR | INT_BIT_RF_KILL | INT_BIT_SW_RX | INT_BIT_ALIVE); } else { g_hwMask = g_hwInitMask; g_fhMask = g_fhInitMask; IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, ~g_fhMask); IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, ~g_hwMask); } } // Map every cause to vector 0 and unmask the ones we handle. `stopped` // skips the UREG_CHICK write, which needs a live MAC. static void IwxConfMsixHw(bool stopped) { constexpr uint8_t vector = 0; if (!g_msix) { if (!stopped && IwxNicLock()) { IwxWriteUmacPrph(IWX_UREG_CHICK, IWX_UREG_CHICK_MSI_ENABLE); IwxNicUnlock(); } return; } if (!stopped && IwxNicLock()) { IwxWriteUmacPrph(IWX_UREG_CHICK, IWX_UREG_CHICK_MSIX_ENABLE); IwxNicUnlock(); } IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, ~0u); IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, ~0u); // RX queues 0 (command/fallback) and 1 (RSS data) -> our vector. IwxWrite8(IWX_CSR_MSIX_RX_IVAR(0), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE); IwxWrite8(IWX_CSR_MSIX_RX_IVAR(1), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE); static const uint32_t causes[] = { IWX_MSIX_IVAR_CAUSE_D2S_CH0_NUM, IWX_MSIX_IVAR_CAUSE_D2S_CH1_NUM, IWX_MSIX_IVAR_CAUSE_S2D, IWX_MSIX_IVAR_CAUSE_FH_ERR, IWX_MSIX_IVAR_CAUSE_REG_ALIVE, IWX_MSIX_IVAR_CAUSE_REG_WAKEUP, IWX_MSIX_IVAR_CAUSE_REG_RESET_DONE, IWX_MSIX_IVAR_CAUSE_REG_CT_KILL, IWX_MSIX_IVAR_CAUSE_REG_RF_KILL, IWX_MSIX_IVAR_CAUSE_REG_PERIODIC, IWX_MSIX_IVAR_CAUSE_REG_SW_ERR, IWX_MSIX_IVAR_CAUSE_REG_SW_ERR_V2, IWX_MSIX_IVAR_CAUSE_REG_SCD, IWX_MSIX_IVAR_CAUSE_REG_FH_TX, IWX_MSIX_IVAR_CAUSE_REG_HW_ERR, IWX_MSIX_IVAR_CAUSE_REG_HAP, }; for (uint32_t c : causes) IwxWrite8(IWX_CSR_MSIX_IVAR(c), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE); IwxClearBits(IWX_CSR_MSIX_FH_INT_MASK_AD, IWX_MSIX_FH_INT_CAUSES_Q0 | IWX_MSIX_FH_INT_CAUSES_Q1 | IWX_MSIX_FH_INT_CAUSES_D2S_CH0_NUM | IWX_MSIX_FH_INT_CAUSES_D2S_CH1_NUM | IWX_MSIX_FH_INT_CAUSES_S2D | IWX_MSIX_FH_INT_CAUSES_FH_ERR); IwxClearBits(IWX_CSR_MSIX_HW_INT_MASK_AD, IWX_MSIX_HW_INT_CAUSES_REG_ALIVE | IWX_MSIX_HW_INT_CAUSES_REG_WAKEUP | IWX_MSIX_HW_INT_CAUSES_REG_RESET_DONE | IWX_MSIX_HW_INT_CAUSES_REG_CT_KILL | IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL | IWX_MSIX_HW_INT_CAUSES_REG_PERIODIC | IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR | IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR_V2 | IWX_MSIX_HW_INT_CAUSES_REG_SCD | IWX_MSIX_HW_INT_CAUSES_REG_FH_TX | IWX_MSIX_HW_INT_CAUSES_REG_HW_ERR | IWX_MSIX_HW_INT_CAUSES_REG_HAP); } static void IwxInitMsixHw() { IwxConfMsixHw(false); if (!g_msix) return; g_fhInitMask = ~IwxRead32(IWX_CSR_MSIX_FH_INT_MASK_AD); g_fhMask = g_fhInitMask; g_hwInitMask = ~IwxRead32(IWX_CSR_MSIX_HW_INT_MASK_AD); g_hwMask = g_hwInitMask; } static void IwxEnableRfkillInt() { if (!g_msix) { IwxWrite32(IWX_CSR_INT_MASK, 1u << 7); // IWX_CSR_INT_BIT_RF_KILL } else { IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, g_fhInitMask); IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, ~IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL); g_hwMask = IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL; } IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_RFKILL_WAKE_L1A_EN); } bool IwxCheckRfKill() { // Documented as "state of the hardware RF-Kill switch", but the bit // reads 0 when the radio is DISABLED (Intel's own comment says the // documentation is unhelpful here). uint32_t v = IwxRead32(IWX_CSR_GP_CNTRL); return (v & IWX_CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW) == 0; } // The hard interrupt only latches work. All ring/notification handling // takes heap and (indirectly) IPC locks, so it runs from the idle loop. static void HandleInterrupt(uint8_t, bool) { g_iwx.WorkPending = true; } // ========================================================================= // APM (adapter power management) bring-up // ========================================================================= static void IwxApmConfig() { // L0s is unstable on these parts and unsupported on newer ones; the // Linux and OpenBSD drivers both disable it unconditionally. IwxSetBits(IWX_CSR_GIO_REG, IWX_CSR_GIO_REG_VAL_L0S_DISABLED); } static bool IwxApmInit() { IwxSetBits(IWX_CSR_GIO_CHICKEN_BITS, IWX_CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX); IwxSetBits(IWX_CSR_DBG_HPET_MEM_REG, IWX_CSR_DBG_HPET_MEM_REG_VAL); IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG, IWX_CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A); IwxApmConfig(); // D0U* -> D0A*: "initialization complete". IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_INIT_DONE); if (!IwxPollBit(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000)) { KernelLogStream(ERROR, "WiFi") << "Timeout waiting for clock stabilization"; return false; } return true; } static void IwxApmStop() { IwxSetBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG, IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED); IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG, IWX_CSR_HW_IF_CONFIG_REG_PREPARE | IWX_CSR_HW_IF_CONFIG_REG_ENABLE_PME); IwxDelayUs(1000); IwxClearBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG, IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED); IwxDelayUs(5000); IwxSetBits(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_STOP_MASTER); if (!IwxPollBit(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_MASTER_DISABLED, IWX_CSR_RESET_REG_FLAG_MASTER_DISABLED, 100)) KernelLogStream(WARNING, "WiFi") << "Timeout waiting for bus master"; IwxClearBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_INIT_DONE); } static void IwxSwReset() { IwxSetBits(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_SW_RESET); IwxDelayUs(5000); } static bool IwxSetHwReady() { IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG, IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY); bool ready = IwxPollBit(IWX_CSR_HW_IF_CONFIG_REG, IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY, IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY, 50); if (ready) IwxSetBits(IWX_CSR_MBOX_SET_REG, IWX_CSR_MBOX_SET_REG_OS_ALIVE); return ready; } static bool IwxPrepareCardHw() { if (IwxSetHwReady()) return true; IwxSetBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG, IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED); IwxDelayUs(1000); for (int tries = 0; tries < 10; tries++) { IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG, IWX_CSR_HW_IF_CONFIG_REG_PREPARE); int t = 0; do { if (IwxSetHwReady()) return true; IwxDelayUs(200); t += 200; } while (t < 150000); IwxDelayUs(25000); } return false; } // ========================================================================= // Ring allocation // ========================================================================= static bool IwxAllocRxRing() { IwxRxRing& ring = g_iwx.RxQ; ring.Cur = 0; if (!IwxDmaAlloc(ring.FreeDescs, sizeof(IwxRxTransferDesc) * IWX_RX_MQ_RING_COUNT)) return false; if (!IwxDmaAlloc(ring.Stat, 64)) return false; if (!IwxDmaAlloc(ring.UsedDescs, sizeof(IwxRxCompletionDesc) * IWX_RX_MQ_RING_COUNT)) return false; // One 4 KiB page per RX buffer (IWX_RBUF_SIZE == PAGE_SIZE). for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) { void* p = Memory::g_pfa->AllocateZeroed(); if (!p) return false; ring.Buf[i] = (uint8_t*)p; ring.BufPhys[i] = Memory::SubHHDM(p); } return true; } static void IwxFreeRxRing() { IwxRxRing& ring = g_iwx.RxQ; IwxDmaFree(ring.FreeDescs); IwxDmaFree(ring.Stat); IwxDmaFree(ring.UsedDescs); for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) { if (ring.Buf[i]) { Memory::g_pfa->Free(ring.Buf[i]); ring.Buf[i] = nullptr; } } } // Publish every RX buffer into the free ring. The firmware configures the // RFH itself and only accepts these once it signals ALIVE. static void IwxUpdateRxDescs() { auto* descs = (IwxRxTransferDesc*)g_iwx.RxQ.FreeDescs.Virt; for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) { descs[i].rbid = (uint16_t)(i & 0xffff); descs[i].addr = g_iwx.RxQ.BufPhys[i]; } asm volatile("" ::: "memory"); } // `stageSlots` reserves one page per concurrently queued frame; pass 0 for // queues that only ever carry host commands. static bool IwxAllocTxRing(IwxTxRing& ring, int qid, uint32_t stageSlots = 0) { ring.Qid = qid; ring.Cur = 0; ring.CurHw = 0; ring.Queued = 0; ring.StageSlots = 0; ring.Active = false; if (!IwxDmaAlloc(ring.Desc, sizeof(IwxTfhTfd) * IWX_TX_RING_COUNT)) return false; if (!IwxDmaAlloc(ring.BcTbl, sizeof(IwxGen3BcTblEntry) * IWX_TFD_QUEUE_BC_SIZE_GEN3_AX210)) return false; if (!IwxDmaAlloc(ring.Cmd, sizeof(IwxDeviceCmd) * IWX_TX_RING_COUNT)) return false; // Commands larger than IWX_DEF_CMD_PAYLOAD_SIZE (e.g. SCAN_REQ_UMAC) // are staged in this page instead of the per-slot command area. if (!IwxDmaAlloc(ring.Bounce, 4096)) return false; if (stageSlots > IWX_TX_STAGE_SLOTS) stageSlots = IWX_TX_STAGE_SLOTS; for (uint32_t i = 0; i < stageSlots; i++) { void* p = Memory::g_pfa->AllocateZeroed(); if (!p) return false; ring.Stage[i] = (uint8_t*)p; ring.StagePhys[i] = Memory::SubHHDM(p); ring.StageSlots = i + 1; } return true; } static void IwxFreeTxRing(IwxTxRing& ring) { IwxDmaFree(ring.Desc); IwxDmaFree(ring.BcTbl); IwxDmaFree(ring.Cmd); IwxDmaFree(ring.Bounce); for (uint32_t i = 0; i < ring.StageSlots; i++) { if (ring.Stage[i]) { Memory::g_pfa->Free(ring.Stage[i]); ring.Stage[i] = nullptr; } } ring.StageSlots = 0; ring.Active = false; } static void IwxResetTxRing(IwxTxRing& ring) { if (ring.BcTbl.Virt) memset(ring.BcTbl.Virt, 0, ring.BcTbl.Pages * 0x1000); if (ring.Desc.Virt) memset(ring.Desc.Virt, 0, ring.Desc.Pages * 0x1000); ring.Cur = 0; ring.CurHw = 0; ring.Queued = 0; } // ========================================================================= // Firmware section staging + context info (gen3) // ========================================================================= static void IwxFreeFwSecDma() { for (int i = 0; i < g_iwx.FwSecDmaCount; i++) IwxDmaFree(g_iwx.FwSecDma[i]); g_iwx.FwSecDmaCount = 0; } static void IwxFreePagingDma() { for (int i = 0; i < g_iwx.PagingCount; i++) IwxDmaFree(g_iwx.PagingDma[i]); g_iwx.PagingCount = 0; } // Count sections until the next separator marker. static int IwxNumSections(int start) { int n = 0; while (start < g_iwx.Fw.SectionCount && g_iwx.Fw.Sections[start].DevOff != IWX_CPU1_CPU2_SEPARATOR_SECTION && g_iwx.Fw.Sections[start].DevOff != IWX_PAGING_SEPARATOR_SECTION) { start++; n++; } return n; } // Copy LMAC/UMAC/paging sections into DMA and record their addresses in the // firmware's DRAM map. The image is laid out as // [lmac...] SEP [umac...] SEP [paging...]. static bool IwxInitFwSec(IwxContextInfoDram* dram) { int lmacCnt = IwxNumSections(0); int umacCnt = IwxNumSections(lmacCnt + 1); int pagingCnt = IwxNumSections(lmacCnt + umacCnt + 2); if (lmacCnt <= 0 || umacCnt <= 0) { KernelLogStream(ERROR, "WiFi") << "Firmware image has no LMAC/UMAC sections"; return false; } if (lmacCnt + umacCnt + pagingCnt > IWX_MAX_FW_SECTIONS) return false; int fwCnt = 0; for (int i = 0; i < lmacCnt; i++) { IwxDma& d = g_iwx.FwSecDma[fwCnt]; if (!IwxDmaAlloc(d, g_iwx.Fw.Sections[i].Len)) return false; memcpy(d.Virt, g_iwx.Fw.Sections[i].Data, g_iwx.Fw.Sections[i].Len); dram->lmac_img[i] = d.Phys; g_iwx.FwSecDmaCount = ++fwCnt; } for (int i = 0; i < umacCnt; i++) { IwxDma& d = g_iwx.FwSecDma[fwCnt]; const IwxFwSection& sec = g_iwx.Fw.Sections[fwCnt + 1]; // skip separator if (!IwxDmaAlloc(d, sec.Len)) return false; memcpy(d.Virt, sec.Data, sec.Len); dram->umac_img[i] = d.Phys; g_iwx.FwSecDmaCount = ++fwCnt; } // Paging sections are tracked separately: unlike the LMAC/UMAC images, // the firmware keeps demand-paging from them for as long as it runs, so // they must survive past ALIVE. if (pagingCnt > (int)IWX_MAX_DRAM_ENTRY) pagingCnt = IWX_MAX_DRAM_ENTRY; IwxFreePagingDma(); for (int i = 0; i < pagingCnt; i++) { IwxDma& d = g_iwx.PagingDma[i]; const IwxFwSection& sec = g_iwx.Fw.Sections[fwCnt + i + 2]; if (!IwxDmaAlloc(d, sec.Len)) return false; memcpy(d.Virt, sec.Data, sec.Len); dram->virtual_img[i] = d.Phys; g_iwx.PagingCount = i + 1; } return true; } static void IwxSetLtr() { uint32_t ltrVal = IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_REQ | ((IWX_CSR_LTR_LONG_VAL_AD_SCALE_USEC << IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_SCALE_SHIFT) & IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_SCALE_MASK) | ((250u << IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_VAL_SHIFT) & IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_VAL_MASK) | IWX_CSR_LTR_LONG_VAL_AD_SNOOP_REQ | ((IWX_CSR_LTR_LONG_VAL_AD_SCALE_USEC << IWX_CSR_LTR_LONG_VAL_AD_SNOOP_SCALE_SHIFT) & IWX_CSR_LTR_LONG_VAL_AD_SNOOP_SCALE_MASK) | (250u & IWX_CSR_LTR_LONG_VAL_AD_SNOOP_VAL); // The AX211 in this laptop is a CNVi (integrated) part of the AX210 // family; only discrete parts take the CSR write here (the Qu-only // HPM_*_LTR path does not apply to AX210). (void)ltrVal; } // Fill the gen3 context info and kick the firmware's self-load. static bool IwxCtxtInfoGen3Init() { if (!g_iwx.Fw.Iml || g_iwx.Fw.ImlLen == 0) { KernelLogStream(ERROR, "WiFi") << "Firmware has no image loader (IML)"; return false; } if (!IwxDmaAlloc(g_iwx.ImlDma, g_iwx.Fw.ImlLen)) return false; auto* scratch = (IwxPrphScratch*)g_iwx.PrphScratch.Virt; memset(scratch, 0, sizeof(*scratch)); auto* ctrl = &scratch->ctrl_cfg; ctrl->version.version = 0; ctrl->version.mac_id = (uint16_t)IwxRead32(IWX_CSR_HW_REV); ctrl->version.size = (uint16_t)(sizeof(*scratch) / 4); ctrl->control.control_flags = IWX_PRPH_SCRATCH_RB_SIZE_4K | IWX_PRPH_SCRATCH_MTR_MODE | (IWX_PRPH_MTR_FORMAT_256B & IWX_PRPH_SCRATCH_MTR_FORMAT); ctrl->rbd_cfg.free_rbd_addr = g_iwx.RxQ.FreeDescs.Phys; if (!IwxInitFwSec(&scratch->dram)) { IwxDmaFree(g_iwx.ImlDma); IwxFreeFwSecDma(); IwxFreePagingDma(); return false; } auto* ci = (IwxContextInfoGen3*)g_iwx.CtxtInfo.Virt; memset(ci, 0, sizeof(*ci)); ci->prph_info_base_addr = g_iwx.PrphInfo.Phys; ci->prph_scratch_base_addr = g_iwx.PrphScratch.Phys; ci->prph_scratch_size = sizeof(IwxPrphScratch); ci->cr_head_idx_arr_base_addr = g_iwx.RxQ.Stat.Phys; // The driver does not use the TR/CR tail arrays, but the device still // writes there; point them at the unused half of the prph-info page. ci->tr_tail_idx_arr_base_addr = g_iwx.PrphInfo.Phys + 0x1000 / 2; ci->cr_tail_idx_arr_base_addr = g_iwx.PrphInfo.Phys + 3 * 0x1000 / 4; ci->mtr_base_addr = g_iwx.CmdQ.Desc.Phys; ci->mcr_base_addr = g_iwx.RxQ.UsedDescs.Phys; ci->mtr_size = (uint16_t)IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT); ci->mcr_size = (uint16_t)IWX_RX_QUEUE_CB_SIZE(IWX_RX_MQ_RING_COUNT); memcpy(g_iwx.ImlDma.Virt, g_iwx.Fw.Iml, g_iwx.Fw.ImlLen); asm volatile("" ::: "memory"); uint64_t paddr = g_iwx.CtxtInfo.Phys; IwxWrite32(IWX_CSR_CTXT_INFO_ADDR, (uint32_t)(paddr & 0xffffffff)); IwxWrite32(IWX_CSR_CTXT_INFO_ADDR + 4, (uint32_t)(paddr >> 32)); paddr = g_iwx.ImlDma.Phys; IwxWrite32(IWX_CSR_IML_DATA_ADDR, (uint32_t)(paddr & 0xffffffff)); IwxWrite32(IWX_CSR_IML_DATA_ADDR + 4, (uint32_t)(paddr >> 32)); IwxWrite32(IWX_CSR_IML_SIZE_ADDR, g_iwx.Fw.ImlLen); IwxSetBits(IWX_CSR_CTXT_INFO_BOOT_CTRL, IWX_CSR_AUTO_FUNC_BOOT_ENA); if (!IwxNicLock()) { IwxDmaFree(g_iwx.ImlDma); IwxFreeFwSecDma(); IwxFreePagingDma(); return false; } IwxSetLtr(); IwxWriteUmacPrph(IWX_UREG_CPU_INIT_RUN, 1); IwxNicUnlock(); return true; } // ========================================================================= // NIC init + firmware start // ========================================================================= static bool IwxNicInit() { if (!IwxApmInit()) return false; // AX210 firmware configures the RFH itself; only interrupt coalescing // and the shadow-register control need programming here. IwxWrite8(IWX_CSR_INT_COALESCING, IWX_HOST_INT_TIMEOUT_DEF); IwxSetBits(IWX_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff); return true; } bool IwxStartHw() { if (!IwxPrepareCardHw()) { KernelLogStream(ERROR, "WiFi") << "Could not prepare card hardware"; return false; } IwxSwReset(); if (!IwxApmInit()) return false; IwxInitMsixHw(); IwxEnableRfkillInt(); if (IwxCheckRfKill()) { KernelLogStream(WARNING, "WiFi") << "Radio is disabled by hardware switch"; g_iwx.State = IwxFwState::RfKill; } return true; } void IwxStopDevice() { IwxDisableInterrupts(); if (IwxNicLock()) { IwxWriteUmacPrph(IWX_RFH_RXF_DMA_CFG_GEN3, 0); for (int i = 0; i < 1000; i++) { if (IwxReadUmacPrph(IWX_RFH_GEN_STATUS_GEN3) & IWX_RXF_DMA_IDLE) break; IwxDelayUs(10); } IwxNicUnlock(); } IwxResetTxRing(g_iwx.CmdQ); IwxResetTxRing(g_iwx.MgmtQ); g_iwx.RxQ.Cur = 0; if (g_iwx.RxQ.Stat.Virt) *(volatile uint16_t*)g_iwx.RxQ.Stat.Virt = 0; IwxClearBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); g_iwx.NicLockCount = 0; IwxApmStop(); IwxSwReset(); // The IVAR table is wiped by the reset; reprogram it so an RF-kill // interrupt still lands on our vector. IwxConfMsixHw(true); IwxDisableInterrupts(); IwxEnableRfkillInt(); IwxCheckRfKill(); IwxPrepareCardHw(); IwxFreeFwSecDma(); IwxFreePagingDma(); IwxDmaFree(g_iwx.PnvmDma); for (int i = 0; i < g_iwx.PnvmSegs; i++) IwxDmaFree(g_iwx.PnvmSegDma[i]); g_iwx.PnvmSegs = 0; g_iwx.PnvmSize = 0; } bool IwxStartFirmware() { g_iwx.AliveIntr = false; g_iwx.AliveOk = false; g_iwx.InitComplete = 0; IwxWrite32(IWX_CSR_INT, ~0u); IwxDisableInterrupts(); // Clear the rfkill handshake bits left over from a previous run. IwxWrite32(IWX_CSR_UCODE_DRV_GP1_CLR, IWX_CSR_UCODE_SW_BIT_RFKILL); IwxWrite32(IWX_CSR_UCODE_DRV_GP1_CLR, IWX_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED); IwxWrite32(IWX_CSR_INT, ~0u); if (!IwxNicInit()) { KernelLogStream(ERROR, "WiFi") << "Unable to init NIC"; return false; } IwxEnableInterrupts(); if (!IwxCtxtInfoGen3Init()) return false; // Wait for the ALIVE notification (firmware self-load takes ~100 ms). uint64_t start = Timekeeping::GetMilliseconds(); while (!g_iwx.AliveIntr && Timekeeping::GetMilliseconds() - start < 2000) { IwxProcessEvents(); IwxDelayUs(200); } IwxDmaFree(g_iwx.ImlDma); IwxFreeFwSecDma(); if (!g_iwx.AliveOk) { KernelLogStream(ERROR, "WiFi") << "Firmware did not come alive" << (g_iwx.AliveIntr ? " (bad ALIVE response)" : " (timeout)"); // The firmware mirrors its boot progress into the prph-info page, // so a stalled self-load can be told apart from one that ran but // whose notification never reached us. auto* info = (IwxPrphInfo*)g_iwx.PrphInfo.Virt; KernelLogStream(INFO, "WiFi") << " boot_stage=" << base::hex << (uint64_t)info->boot_stage_mirror << " ipc_status=" << (uint64_t)info->ipc_status_mirror << " sleep_notif=" << (uint64_t)info->sleep_notif << base::dec; KernelLogStream(INFO, "WiFi") << " fh_causes=" << base::hex << (uint64_t)IwxRead32(IWX_CSR_MSIX_FH_INT_CAUSES_AD) << " hw_causes=" << (uint64_t)IwxRead32(IWX_CSR_MSIX_HW_INT_CAUSES_AD) << " csr_int=" << (uint64_t)IwxRead32(IWX_CSR_INT) << " gp_cntrl=" << (uint64_t)IwxRead32(IWX_CSR_GP_CNTRL) << base::dec << (g_msix ? " [MSI-X]" : " [MSI]"); KernelLogStream(INFO, "WiFi") << " rx_used_widx=" << (uint64_t)(*(volatile uint16_t*)g_iwx.RxQ.Stat.Virt & 0xfff) << " rx_cur=" << (uint64_t)g_iwx.RxQ.Cur; return false; } KernelLogStream(OK, "WiFi") << "Firmware alive: " << g_iwx.Fw.Version; return true; } // Point the firmware at the staged PNVM image and wait for its ack. bool IwxLoadPnvm() { if (g_iwx.SkuId.data[0] == 0 && g_iwx.SkuId.data[1] == 0 && g_iwx.SkuId.data[2] == 0) return true; // no PNVM needed if (g_iwx.PnvmDma.Virt == nullptr && g_iwx.Fw.PnvmData) { // Parse the PNVM TLV blob for our SKU; a missing match is not // fatal, the firmware then runs with built-in defaults. IwxPnvmParse(g_iwx.Fw.PnvmData, g_iwx.Fw.PnvmLen); } if (g_iwx.PnvmDma.Virt) { auto* scratch = (IwxPrphScratch*)g_iwx.PrphScratch.Virt; scratch->ctrl_cfg.pnvm_cfg.pnvm_base_addr = g_iwx.PnvmDma.Phys; scratch->ctrl_cfg.pnvm_cfg.pnvm_size = g_iwx.PnvmSize; asm volatile("" ::: "memory"); } if (!IwxNicLock()) return false; IwxWriteUmacPrph(IWX_UREG_DOORBELL_TO_ISR6, IWX_UREG_DOORBELL_TO_ISR6_PNVM); IwxNicUnlock(); uint64_t start = Timekeeping::GetMilliseconds(); while (!(g_iwx.InitComplete & 0x2) && Timekeeping::GetMilliseconds() - start < 2000) { IwxProcessEvents(); IwxDelayUs(200); } if (!(g_iwx.InitComplete & 0x2)) { KernelLogStream(WARNING, "WiFi") << "No PNVM complete notification"; return false; } return true; } // ========================================================================= // Host command submission // ========================================================================= int IwxLookupCmdVer(uint8_t group, uint8_t cmd) { for (int i = 0; i < g_iwx.Fw.NumCmdVersions; i++) { if (g_iwx.Fw.CmdVersions[i].group == group && g_iwx.Fw.CmdVersions[i].cmd == cmd) return g_iwx.Fw.CmdVersions[i].cmd_ver; } return -1; } int IwxLookupNotifVer(uint8_t group, uint8_t cmd) { for (int i = 0; i < g_iwx.Fw.NumCmdVersions; i++) { if (g_iwx.Fw.CmdVersions[i].group == group && g_iwx.Fw.CmdVersions[i].cmd == cmd) return g_iwx.Fw.CmdVersions[i].notif_ver; } return -1; } void IwxDumpFwError(); static uint32_t g_cmdTimeouts = 0; // consecutive unanswered commands bool IwxSendCmd(IwxHostCmd& hcmd) { if (g_iwx.State == IwxFwState::Error) return false; IwxTxRing& ring = g_iwx.CmdQ; g_iwx.CmdLock.Acquire(); uint32_t idx = ring.Cur; uint32_t code = hcmd.Id; // Firmware API >= 50 rejects group-0 commands; the legacy opcodes must // be re-tagged into LONG_GROUP. Responses come back tagged the same // way, so notification dispatch strips the group again. if (IwxCmdGroupId(code) == 0) code = IWX_WIDE_ID(IWX_LONG_GROUP, code); uint32_t hdrLen = sizeof(IwxCmdHeaderWide); uint32_t datasz = sizeof(IwxDeviceCmd) - hdrLen; if (hcmd.Len > IWX_MAX_CMD_PAYLOAD_SIZE) { g_iwx.CmdLock.Release(); KernelLogStream(ERROR, "WiFi") << "Firmware command too long (" << (uint64_t)hcmd.Len << " bytes)"; return false; } uint8_t* cmdBuf; uint64_t cmdPhys; if (hcmd.Len > datasz) { cmdBuf = (uint8_t*)ring.Bounce.Virt; cmdPhys = ring.Bounce.Phys; } else { cmdBuf = (uint8_t*)ring.Cmd.Virt + (uint64_t)idx * sizeof(IwxDeviceCmd); cmdPhys = ring.Cmd.Phys + (uint64_t)idx * sizeof(IwxDeviceCmd); } memset(cmdBuf, 0, hdrLen + hcmd.Len); auto* hdr = (IwxCmdHeaderWide*)cmdBuf; hdr->opcode = IwxCmdOpcode(code); hdr->group_id = IwxCmdGroupId(code); hdr->qid = (uint8_t)ring.Qid; hdr->idx = (uint8_t)idx; hdr->length = (uint16_t)hcmd.Len; hdr->version = 0; if (hcmd.Len && hcmd.Data) memcpy(cmdBuf + hdrLen, hcmd.Data, hcmd.Len); auto* desc = &((IwxTfhTfd*)ring.Desc.Virt)[idx]; memset(desc, 0, sizeof(*desc)); uint32_t total = hdrLen + hcmd.Len; uint16_t firstLen = (uint16_t)(total < IWX_FIRST_TB_SIZE ? total : IWX_FIRST_TB_SIZE); desc->tbs[0].tb_len = firstLen; desc->tbs[0].addr = cmdPhys; if (total > IWX_FIRST_TB_SIZE) { desc->tbs[1].tb_len = (uint16_t)(total - IWX_FIRST_TB_SIZE); desc->tbs[1].addr = cmdPhys + IWX_FIRST_TB_SIZE; desc->num_tbs = 2; } else { desc->num_tbs = 1; } asm volatile("" ::: "memory"); g_iwx.CmdDone = false; g_iwx.LastCmdId = code; // Keep the payload so a firmware assert can show exactly what it // choked on -- struct mismatches are invisible without the bytes. g_iwx.LastCmdLen = hcmd.Len; uint32_t keep = hcmd.Len < sizeof(g_iwx.LastCmdPayload) ? hcmd.Len : (uint32_t)sizeof(g_iwx.LastCmdPayload); if (hcmd.Data && keep) memcpy(g_iwx.LastCmdPayload, hcmd.Data, keep); g_iwx.CmdWantResp = hcmd.WantResp; g_iwx.CmdRespLen = 0; g_iwx.CmdIdx = idx; ring.Queued++; ring.Cur = (ring.Cur + 1) % IWX_TX_RING_COUNT; ring.CurHw = (ring.CurHw + 1) % IWX_TFD_QUEUE_SIZE_MAX_GEN3; IwxWrite32(IWX_HBUS_TARG_WRPTR, ((uint32_t)ring.Qid << 16) | ring.CurHw); // Wait for the firmware's response/ack. Commands are serialized by // CmdLock, so exactly one can be in flight and the completion is // unambiguous. // Two independent bounds. The wall clock is the intended one, but it // is driven by the timer interrupt, so anything that leaves this loop // running with interrupts disabled would spin forever and take the // whole machine down with it -- the spin cap makes that impossible. // Bail immediately if the firmware has asserted, because it will never // answer this or any later command. constexpr uint32_t MAX_SPINS = 20000; // ~2 s at 100 us bool ok = false; bool died = false; uint64_t start = Timekeeping::GetMilliseconds(); for (uint32_t spins = 0; spins < MAX_SPINS; spins++) { IwxProcessEvents(); if (g_iwx.CmdDone) { ok = true; break; } if (g_iwx.State == IwxFwState::Error) { died = true; break; } if (Timekeeping::GetMilliseconds() - start >= 1000) break; IwxDelayUs(100); } if (!ok) { KernelLogStream(WARNING, "WiFi") << "Command 0x" << base::hex << (uint64_t)code << base::dec << (died ? " abandoned: firmware has stopped responding" : " timed out"); if (ring.Queued > 0) ring.Queued--; if (!died) { // Dump on the first silence: the firmware's error table names // the command that asserted, and it is overwritten as later // commands go unanswered. if (++g_cmdTimeouts == 1) IwxDumpFwError(); // Repeated silence means it is wedged and every later command // would burn the same timeout, so stop trying. A single late // response is not worth disabling the adapter over. if (g_cmdTimeouts >= 3) { KernelLogStream(ERROR, "WiFi") << "Firmware stopped responding to host commands"; g_iwx.FwErrors++; g_iwx.State = IwxFwState::Error; } } } else { g_cmdTimeouts = 0; } g_iwx.CmdWantResp = false; g_iwx.CmdLock.Release(); return ok; } bool IwxSendCmdPdu(uint32_t id, const void* data, uint32_t len) { IwxHostCmd cmd; cmd.Id = id; cmd.Data = data; cmd.Len = len; return IwxSendCmd(cmd); } bool IwxSendCmdStatus(uint32_t id, const void* data, uint32_t len, uint32_t* statusOut) { IwxHostCmd cmd; cmd.Id = id; cmd.Data = data; cmd.Len = len; cmd.WantResp = true; if (!IwxSendCmd(cmd)) return false; if (g_iwx.CmdRespLen < sizeof(IwxRxPacket) + sizeof(IwxCmdResponse)) return false; auto* pkt = (IwxRxPacket*)g_iwx.CmdRespBuf; auto* resp = (IwxCmdResponse*)pkt->data; if (statusOut) *statusOut = resp->status; return true; } // ========================================================================= // TX queue configuration (used by the connect path) // ========================================================================= bool IwxEnableTxq(IwxTxRing& ring, int staId, int qid, int tid) { IwxResetTxRing(ring); ring.Qid = qid; ring.Active = false; int cmdVer = IwxLookupCmdVer(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD); IwxHostCmd hcmd; hcmd.WantResp = true; // AX211 firmware 89 advertises v3 (DATA_PATH group). Older images // expose no version at all and take the legacy SCD_QUEUE_CFG command. IwxTxQueueCfgCmd cmdV0 = {}; IwxScdQueueCfgCmd cmdV3 = {}; if (cmdVer == 3) { cmdV3.operation = IWX_SCD_QUEUE_ADD; cmdV3.u.add.tfdq_dram_addr = ring.Desc.Phys; cmdV3.u.add.bc_dram_addr = ring.BcTbl.Phys; cmdV3.u.add.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT); cmdV3.u.add.flags = 0; cmdV3.u.add.sta_mask = 1u << staId; cmdV3.u.add.tid = (uint8_t)tid; hcmd.Id = IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD); hcmd.Data = &cmdV3; hcmd.Len = sizeof(cmdV3); } else if (cmdVer <= 0) { cmdV0.sta_id = (uint8_t)staId; cmdV0.tid = (uint8_t)tid; cmdV0.flags = IWX_TX_QUEUE_CFG_ENABLE_QUEUE; cmdV0.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT); cmdV0.byte_cnt_addr = ring.BcTbl.Phys; cmdV0.tfdq_addr = ring.Desc.Phys; hcmd.Id = IWX_SCD_QUEUE_CFG; hcmd.Data = &cmdV0; hcmd.Len = sizeof(cmdV0); } else { KernelLogStream(WARNING, "WiFi") << "Unsupported SCD_QUEUE_CONFIG version " << (uint64_t)cmdVer; return false; } if (!IwxSendCmd(hcmd)) return false; if (g_iwx.CmdRespLen < sizeof(IwxRxPacket) + sizeof(IwxTxQueueCfgRsp)) return false; auto* pkt = (IwxRxPacket*)g_iwx.CmdRespBuf; auto* resp = (IwxTxQueueCfgRsp*)pkt->data; // On the v3 data-path API the firmware owns queue assignment: `qid` is // only a hint and the response names the queue we actually got. if (resp->queue_number != qid) { KernelLogStream(INFO, "WiFi") << "Firmware assigned TX queue " << (uint64_t)resp->queue_number << " (asked for " << (uint64_t)qid << ")"; } ring.Qid = resp->queue_number; ring.Active = true; return true; } void IwxDisableTxq(IwxTxRing& ring, int staId, int tid) { if (!ring.Active) return; ring.Active = false; int cmdVer = IwxLookupCmdVer(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD); if (cmdVer == 3) { IwxScdQueueCfgCmd cmd = {}; cmd.operation = IWX_SCD_QUEUE_REMOVE; cmd.u.remove.sta_mask = 1u << staId; cmd.u.remove.tid = (uint32_t)tid; IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD), &cmd, sizeof(cmd)); } else { IwxTxQueueCfgCmd cmd = {}; cmd.sta_id = (uint8_t)staId; cmd.tid = (uint8_t)tid; cmd.flags = 0; // clear ENABLE_QUEUE cmd.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT); cmd.byte_cnt_addr = ring.BcTbl.Phys; cmd.tfdq_addr = ring.Desc.Phys; IwxSendCmdPdu(IWX_SCD_QUEUE_CFG, &cmd, sizeof(cmd)); } IwxResetTxRing(ring); } // ========================================================================= // Frame transmission // ========================================================================= // Lowest usable transmit antenna, as a rate_n_flags antenna field. static uint32_t IwxTxAntBits() { uint8_t ant = (uint8_t)((g_iwx.Fw.PhyConfig & IWX_FW_PHY_CFG_TX_CHAIN) >> IWX_FW_PHY_CFG_TX_CHAIN_POS); if (g_iwx.Nvm.ValidTxAnt) ant &= g_iwx.Nvm.ValidTxAnt; if (!ant) ant = 1; uint8_t lowest = (uint8_t)(ant & (uint8_t)(~ant + 1)); // isolate low bit return (uint32_t)lowest << IWX_RATE_MCS_ANT_POS; } // The lowest basic rate for the current band, in whichever rate_n_flags // encoding the firmware advertises: 1 Mbps CCK on 2.4 GHz, 6 Mbps OFDM on // 5 GHz. Management frames go out at this rate because rate control has // no table for the station until it is associated. static uint32_t IwxLowestRate() { uint32_t ant = IwxTxAntBits(); // Firmware exposing TX_CMD notification version 7 or later (equally, // command version 9+) uses the "version 2" rate layout. int notifVer = IwxLookupNotifVer(IWX_LONG_GROUP, IWX_TX_CMD); int cmdVer = IwxLookupCmdVer(IWX_LONG_GROUP, IWX_TX_CMD); bool v2 = notifVer > 6 || cmdVer >= 9; if (v2) { uint32_t mod = g_iwx.Is5GHz ? IWX_RATE_MCS_MOD_LEGACY_OFDM : IWX_RATE_MCS_MOD_CCK; return ant | mod | IWX_RATE_MCS_CHAN_WIDTH_20 | 0u; // index 0 } uint32_t plcp = g_iwx.Is5GHz ? IWX_RATE_6M_PLCP : IWX_RATE_1M_PLCP; uint32_t cck = g_iwx.Is5GHz ? 0 : IWX_RATE_MCS_CCK_MSK_V1; return ant | cck | plcp; } // Recover a queue whose completions stopped arriving. Without this a // single lost TX response would permanently consume a slot and, after // StageSlots of them, wedge the queue. static uint64_t g_txStallMs = 0; static bool IwxTxQueueHasRoom(IwxTxRing& ring) { if (ring.Queued < ring.StageSlots) { g_txStallMs = 0; return true; } uint64_t now = Timekeeping::GetMilliseconds(); if (g_txStallMs == 0) { g_txStallMs = now; return false; } if (now - g_txStallMs < 2000) return false; KernelLogStream(WARNING, "WiFi") << "TX queue " << (uint64_t)ring.Qid << " stopped completing; resetting its outstanding count"; ring.Queued = 0; g_txStallMs = 0; return true; } bool IwxTxFrame(IwxTxRing& ring, const uint8_t* hdr, uint32_t hdrLen, const uint8_t* payload, uint32_t payloadLen, bool encrypt, bool fixedRate) { if (g_iwx.State != IwxFwState::Running) return false; if (!ring.Active || ring.StageSlots == 0) return false; if (!hdr || hdrLen < IEEE80211_HDR_LEN || hdrLen > 32) return false; // The 802.11 header is padded to a dword boundary before the payload; // TX_CMD_OFFLD_PAD tells the firmware to skip those bytes. uint32_t padded = (hdrLen + 3) & ~3u; uint32_t head = (uint32_t)(sizeof(IwxCmdHeader) + sizeof(IwxTxCmdGen3)) + padded; if (head + payloadLen > 4096) return false; g_iwx.TxLock.Acquire(); if (!IwxTxQueueHasRoom(ring)) { g_iwx.TxLock.Release(); return false; } uint32_t idx = ring.Cur; uint32_t slot = idx % ring.StageSlots; uint8_t* buf = ring.Stage[slot]; uint64_t phys = ring.StagePhys[slot]; memset(buf, 0, head); auto* ch = (IwxCmdHeader*)buf; ch->code = IWX_TX_CMD; ch->flags = 0; // TX_CMD stays in the legacy group ch->idx = (uint8_t)idx; ch->qid = (uint8_t)ring.Qid; auto* tx = (IwxTxCmdGen3*)(buf + sizeof(IwxCmdHeader)); tx->len = (uint16_t)(hdrLen + payloadLen); uint16_t flags = 0; if (!encrypt) flags |= IWX_TX_FLAGS_ENCRYPT_DIS; if (fixedRate) { flags |= IWX_TX_FLAGS_CMD_RATE; tx->rate_n_flags = IwxLowestRate(); } tx->flags = flags; uint32_t offload = ((hdrLen / 2) & IWX_TX_CMD_OFFLD_MH_MASK) << IWX_TX_CMD_OFFLD_MH_SIZE_POS; if (hdrLen % 4) offload |= IWX_TX_CMD_OFFLD_PAD; tx->offload_assist = offload; uint8_t* body = buf + sizeof(IwxCmdHeader) + sizeof(IwxTxCmdGen3); memcpy(body, hdr, hdrLen); if (payloadLen) memcpy(body + padded, payload, payloadLen); auto* desc = &((IwxTfhTfd*)ring.Desc.Virt)[idx]; memset(desc, 0, sizeof(*desc)); desc->tbs[0].tb_len = (uint16_t)IWX_FIRST_TB_SIZE; desc->tbs[0].addr = phys; desc->tbs[1].tb_len = (uint16_t)(head - IWX_FIRST_TB_SIZE); desc->tbs[1].addr = phys + IWX_FIRST_TB_SIZE; uint16_t numTbs = 2; if (payloadLen) { desc->tbs[2].tb_len = (uint16_t)payloadLen; desc->tbs[2].addr = phys + head; numTbs = 3; } desc->num_tbs = numTbs; // Byte-count table: AX210 wants the frame length in bytes plus the // number of extra 64-byte chunks the firmware must fetch for the TFD. uint32_t filled = (uint32_t)(sizeof(uint16_t) + numTbs * sizeof(IwxTfhTb)); uint32_t chunks = ((filled + 63) / 64) - 1; auto* bc = (IwxGen3BcTblEntry*)ring.BcTbl.Virt; bc[idx].tfd_offset = (uint16_t)((hdrLen + payloadLen) | (chunks << 14)); asm volatile("" ::: "memory"); ring.Queued++; ring.Cur = (ring.Cur + 1) % IWX_TX_RING_COUNT; ring.CurHw = (ring.CurHw + 1) % IWX_TFD_QUEUE_SIZE_MAX_GEN3; IwxWrite32(IWX_HBUS_TARG_WRPTR, ((uint32_t)ring.Qid << 16) | ring.CurHw); g_iwx.TxLock.Release(); return true; } void IwxTxComplete(int qid, int idx, uint32_t status) { (void)idx; IwxTxRing* ring = nullptr; if (g_iwx.MgmtQ.Active && qid == g_iwx.MgmtQ.Qid) ring = &g_iwx.MgmtQ; if (!ring) return; g_iwx.TxLock.Acquire(); if (ring->Queued > 0) ring->Queued--; g_iwx.TxLock.Release(); if (status == IWX_TX_STATUS_SUCCESS || status == IWX_TX_STATUS_DIRECT_DONE) g_iwx.TxPackets++; else g_iwx.TxFailures++; } // ========================================================================= // Hardware key installation // ========================================================================= // Keys go in and out through SEC_KEY_CMD: the MLD firmware does not // implement the legacy ADD_STA_KEY, like the rest of the legacy station // API. Values mirror the Linux trace: PTK as {sta_mask 1, key_id 0, // flags CIPHER}, GTK as {sta_mask 1, key_id N, flags CIPHER|MCAST}. static uint32_t SecKeyFlags(uint8_t cipher, uint32_t keyLen, bool pairwise) { uint32_t flags; switch (cipher) { case RSN_CIPHER_CCMP: case RSN_CIPHER_CCMP_256: flags = IWX_SEC_KEY_FLAG_CIPHER_CCMP; break; case RSN_CIPHER_GCMP: case RSN_CIPHER_GCMP_256: flags = IWX_SEC_KEY_FLAG_CIPHER_GCMP; break; default: return 0; } if (keyLen == 32) flags |= IWX_SEC_KEY_FLAG_KEY_SIZE; if (!pairwise) flags |= IWX_SEC_KEY_FLAG_MCAST_KEY; return flags; } bool IwxSetKey(const uint8_t* key, uint32_t keyLen, uint8_t keyIdx, bool pairwise, uint8_t cipher, const uint8_t* rsc) { if (!key || (keyLen != 16 && keyLen != 32)) return false; uint32_t flags = SecKeyFlags(cipher, keyLen, pairwise); if (!flags) { KernelLogStream(WARNING, "WiFi") << "Cannot install a key for cipher " << (uint64_t)cipher; return false; } IwxSecKeyCmd cmd = {}; cmd.action = IWX_FW_CTXT_ACTION_ADD; cmd.sta_mask = 1u << IWX_STATION_ID; cmd.key_id = keyIdx; cmd.key_flags = flags; memcpy(cmd.key, key, keyLen); // The EAPOL RSC carries the AP's packet number for the group key, // lowest byte first; it becomes the initial receive counter. if (rsc) { uint64_t pn = 0; for (int i = 5; i >= 0; i--) pn = (pn << 8) | rsc[i]; cmd.rx_seq = pn; } return IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SEC_KEY_CMD), &cmd, sizeof(cmd)); } bool IwxRemoveKey(uint8_t keyIdx, bool pairwise, uint8_t cipher, uint32_t keyLen) { uint32_t flags = SecKeyFlags(cipher, keyLen, pairwise); if (!flags) return false; IwxSecKeyCmd cmd = {}; cmd.action = IWX_FW_CTXT_ACTION_REMOVE; cmd.sta_mask = 1u << IWX_STATION_ID; cmd.key_id = keyIdx; cmd.key_flags = flags; return IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SEC_KEY_CMD), &cmd, sizeof(cmd)); } // ========================================================================= // RX / notification processing // ========================================================================= static bool IwxRxPacketValid(const IwxRxPacket* pkt) { int qid = pkt->hdr.qid & ~0x80; int idx = pkt->hdr.idx; uint32_t code = ((uint32_t)pkt->hdr.flags << 8) | pkt->hdr.code; return !(qid == 0 && idx == 0 && code == 0) && pkt->len_n_flags != IWX_FH_RSCSR_FRAME_INVALID; } // Capture a command response for the in-flight synchronous command. static void IwxCaptureCmdResp(const IwxRxPacket* pkt) { uint32_t len = sizeof(pkt->len_n_flags) + IwxRxPacketLen(pkt); if (len > sizeof(g_iwx.CmdRespBuf)) len = sizeof(g_iwx.CmdRespBuf); memcpy(g_iwx.CmdRespBuf, pkt, len); g_iwx.CmdRespLen = len; } static void IwxHandleAlive(const IwxRxPacket* pkt) { uint32_t payload = IwxRxPacketPayloadLen(pkt); g_iwx.AliveOk = false; if (payload >= sizeof(IwxAliveRespV6)) { auto* r = (const IwxAliveRespV6*)pkt->data; g_iwx.SkuId = r->sku_id; g_iwx.UmacErrorTable = r->umac_data.dbg_ptrs.error_info_addr; g_iwx.LmacErrorTable = r->lmac_data[0].dbg_ptrs.error_event_table_ptr; if (r->status == IWX_ALIVE_STATUS_OK) g_iwx.AliveOk = true; } else if (payload >= sizeof(IwxAliveRespV5)) { auto* r = (const IwxAliveRespV5*)pkt->data; g_iwx.SkuId = r->sku_id; g_iwx.UmacErrorTable = r->umac_data.dbg_ptrs.error_info_addr; g_iwx.LmacErrorTable = r->lmac_data[0].dbg_ptrs.error_event_table_ptr; if (r->status == IWX_ALIVE_STATUS_OK) g_iwx.AliveOk = true; } else { KernelLogStream(WARNING, "WiFi") << "Unexpected ALIVE payload size " << (uint64_t)payload; } g_iwx.AliveIntr = true; } // Read dwords out of device-internal SRAM (needs the MAC awake). static bool IwxReadMem(uint32_t addr, uint32_t* out, int dwords) { if (!IwxNicLock()) return false; IwxWrite32(IWX_HBUS_TARG_MEM_RADDR, addr); asm volatile("" ::: "memory"); for (int i = 0; i < dwords; i++) out[i] = IwxRead32(IWX_HBUS_TARG_MEM_RDAT); IwxNicUnlock(); return true; } // Dump the firmware's own error log after an assert. error_id identifies // the assert and cmd_header names the last command the UMAC accepted -- // between them they pin down which host command the firmware rejected. void IwxDumpFwError() { KernelLogStream(ERROR, "WiFi-FW") << "Firmware assert; last command sent: 0x" << base::hex << (uint64_t)g_iwx.LastCmdId << base::dec << " (" << (uint64_t)g_iwx.LastCmdLen << " byte payload)"; // Dump the payload: a struct that does not match the firmware's // expected layout is otherwise impossible to spot from the log. { uint32_t n = g_iwx.LastCmdLen; if (n > sizeof(g_iwx.LastCmdPayload)) n = sizeof(g_iwx.LastCmdPayload); for (uint32_t off = 0; off < n; off += 32) { auto line = KernelLogStream(INFO, "WiFi-FW"); line << " +" << (uint64_t)off << ": " << base::hex; for (uint32_t i = off; i < n && i < off + 32; i++) { if (g_iwx.LastCmdPayload[i] < 0x10) line << "0"; line << (uint64_t)g_iwx.LastCmdPayload[i]; } line << base::dec; } } uint32_t base_ = g_iwx.UmacErrorTable; if (base_ < 0x400000) { KernelLogStream(WARNING, "WiFi-FW") << " no valid UMAC error log pointer (0x" << base::hex << (uint64_t)base_ << base::dec << ")"; return; } IwxUmacErrorEventTable t = {}; if (!IwxReadMem(base_, (uint32_t*)&t, sizeof(t) / sizeof(uint32_t))) { KernelLogStream(WARNING, "WiFi-FW") << " could not read the error log"; return; } KernelLogStream(ERROR, "WiFi-FW") << " UMAC error 0x" << base::hex << (uint64_t)t.error_id << " last host cmd 0x" << (uint64_t)t.cmd_header << base::dec << " (valid=" << (uint64_t)t.valid << ")"; KernelLogStream(INFO, "WiFi-FW") << " data 0x" << base::hex << (uint64_t)t.data1 << " 0x" << (uint64_t)t.data2 << " 0x" << (uint64_t)t.data3 << " pc 0x" << (uint64_t)t.frame_pointer << " sp 0x" << (uint64_t)t.stack_pointer << base::dec; KernelLogStream(INFO, "WiFi-FW") << " umac ver " << (uint64_t)t.umac_major << "." << (uint64_t)t.umac_minor << " blink 0x" << base::hex << (uint64_t)t.blink1 << "/0x" << (uint64_t)t.blink2 << " ilink 0x" << (uint64_t)t.ilink1 << "/0x" << (uint64_t)t.ilink2 << base::dec; } // Process one firmware packet out of an RX buffer. static void IwxRxPkt(uint8_t* buf) { const size_t minsz = sizeof(uint32_t) + sizeof(IwxCmdHeader); auto* pkt = (IwxRxPacket*)buf; if (!IwxRxPacketValid(pkt)) return; uint32_t len = sizeof(pkt->len_n_flags) + IwxRxPacketLen(pkt); if (len < minsz || len > IWX_RBUF_SIZE) return; int qid = pkt->hdr.qid; uint32_t code = ((uint32_t)pkt->hdr.flags << 8) | pkt->hdr.code; // Legacy commands were re-tagged into LONG_GROUP on the way out; undo // that so the dispatcher sees the original opcode. if (IwxCmdGroupId(code) == IWX_LONG_GROUP) code = IwxCmdOpcode(code); switch (code) { case IWX_ALIVE: IwxHandleAlive(pkt); break; case IWX_INIT_COMPLETE_NOTIF: g_iwx.InitComplete |= 0x1; break; case IWX_WIDE_ID(IWX_REGULATORY_AND_NVM_GROUP, IWX_PNVM_INIT_COMPLETE): g_iwx.InitComplete |= 0x2; break; case IWX_TX_CMD: { // TX completion for a frame we queued on a data/mgmt queue. uint32_t status = 0; if (IwxRxPacketPayloadLen(pkt) >= sizeof(IwxTxResp)) { auto* r = (const IwxTxResp*)pkt->data; status = r->status & IWX_TX_STATUS_MSK; } IwxTxComplete(qid & ~0x80, pkt->hdr.idx, status); break; } case IWX_REPLY_ERROR: { if (IwxRxPacketPayloadLen(pkt) >= 8) { uint32_t errType = *(const uint32_t*)pkt->data; uint8_t cmdId = pkt->data[4]; KernelLogStream(ERROR, "WiFi") << "Firmware error 0x" << base::hex << (uint64_t)errType << ", cmd 0x" << (uint64_t)cmdId << base::dec; } g_iwx.FwErrors++; break; } default: // Everything else (scan results, NVM/MCC responses, per-command // acks, statistics) is handled by the MVM layer. IwxHandleNotification(pkt, buf, IWX_RBUF_SIZE); break; } // Capture the payload for a pending WANT_RESP command before signalling // completion. Bit 7 of qid marks firmware-originated notifications, // which never complete a command, and only the command queue carries // command acks at all -- a data-queue packet with a colliding index // must not release the waiter. if (!(qid & 0x80) && (qid & ~0x80) == IWX_DQA_CMD_QUEUE) { if (g_iwx.CmdWantResp && pkt->hdr.idx == g_iwx.CmdIdx && !(pkt->hdr.flags & IWX_CMD_FAILED_MSK)) IwxCaptureCmdResp(pkt); if (pkt->hdr.idx == g_iwx.CmdIdx) { if (g_iwx.CmdQ.Queued > 0) g_iwx.CmdQ.Queued--; g_iwx.CmdDone = true; } } } // Drain the used (completion) ring: for each completed RBD, process the // firmware packet it points at and hand the buffer back. static void IwxNotifIntr() { if (!g_iwx.RxQ.Stat.Virt) return; uint16_t hw = *(volatile uint16_t*)g_iwx.RxQ.Stat.Virt & 0xfff; hw &= (IWX_RX_MQ_RING_COUNT - 1); auto* used = (IwxRxCompletionDesc*)g_iwx.RxQ.UsedDescs.Virt; uint32_t processed = 0; while (g_iwx.RxQ.Cur != hw && processed < IWX_RX_MQ_RING_COUNT) { uint16_t rbid = used[g_iwx.RxQ.Cur].rbid & (IWX_RX_MQ_RING_COUNT - 1); uint8_t* buf = g_iwx.RxQ.Buf[rbid]; if (buf) { IwxRxPkt(buf); g_iwx.RxPackets++; } g_iwx.RxQ.Cur = (g_iwx.RxQ.Cur + 1) % IWX_RX_MQ_RING_COUNT; processed++; } if (processed == 0) return; // Tell the firmware how far we have consumed. The write index must be // 8-aligned; the hardware misbehaves otherwise (documented quirk). uint16_t widx = (hw == 0) ? (uint16_t)(IWX_RX_MQ_RING_COUNT - 1) : (uint16_t)(hw - 1); IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, widx & ~7u); } void IwxProcessEvents() { if (!g_iwx.Mmio) return; // Never nest. With interrupts enabled during command waits this is a // genuine multi-core race, so it has to be an atomic test-and-set. if (g_iwx.InProcessEvents.test_and_set(std::memory_order_acquire)) return; if (g_msix) { uint32_t fh = IwxRead32(IWX_CSR_MSIX_FH_INT_CAUSES_AD); uint32_t hwc = IwxRead32(IWX_CSR_MSIX_HW_INT_CAUSES_AD); IwxWrite32(IWX_CSR_MSIX_FH_INT_CAUSES_AD, fh); IwxWrite32(IWX_CSR_MSIX_HW_INT_CAUSES_AD, hwc); fh &= g_fhMask; hwc &= g_hwMask; if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_ALIVE) { // The firmware has configured the RFH; publish RX buffers. IwxUpdateRxDescs(); IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, 8); } if (fh & (IWX_MSIX_FH_INT_CAUSES_Q0 | IWX_MSIX_FH_INT_CAUSES_Q1)) IwxNotifIntr(); if ((fh & IWX_MSIX_FH_INT_CAUSES_FH_ERR) || (hwc & IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR) || (hwc & IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR_V2)) { KernelLogStream(ERROR, "WiFi") << "Fatal firmware error"; g_iwx.State = IwxFwState::Error; g_iwx.FwErrors++; IwxDumpFwError(); } if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_HW_ERR) { KernelLogStream(ERROR, "WiFi") << "Hardware error"; g_iwx.State = IwxFwState::Error; g_iwx.FwErrors++; } if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL) { if (IwxCheckRfKill()) { KernelLogStream(WARNING, "WiFi") << "Radio disabled by RF-kill"; g_iwx.State = IwxFwState::RfKill; } } IwxWrite32(IWX_CSR_MSIX_AUTOMASK_ST_AD, 1u << 0); } else { uint32_t r1 = IwxRead32(IWX_CSR_INT); if (r1 != 0 && r1 != 0xffffffff && (r1 & 0xfffffff0) != 0xa5a5a5a0) { uint32_t r2 = IwxRead32(IWX_CSR_FH_INT_STATUS); IwxWrite32(IWX_CSR_INT, r1); IwxWrite32(IWX_CSR_FH_INT_STATUS, r2); if (r1 & (1u << 0)) { // ALIVE IwxUpdateRxDescs(); IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, 8); } if (r1 & (1u << 25)) { // SW_ERR KernelLogStream(ERROR, "WiFi") << "Fatal firmware error"; g_iwx.State = IwxFwState::Error; g_iwx.FwErrors++; IwxDumpFwError(); } if (r1 & ((1u << 31) | (1u << 3) | (1u << 28))) // FH_RX/SW_RX/periodic IwxNotifIntr(); } } // The status word is updated by DMA and the interrupt may already have // been consumed by another poller, so always sweep the ring. IwxNotifIntr(); g_iwx.WorkPending = false; g_iwx.InProcessEvents.clear(std::memory_order_release); } // ========================================================================= // MSI-X / MSI setup // ========================================================================= static bool SetupMsix(uint8_t bus, uint8_t dev, uint8_t func) { uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSIX); if (cap == 0) return false; Pci::DisableInterruptDelivery(bus, dev, func); uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); uint32_t tableOff = Pci::LegacyRead32(bus, dev, func, cap + 4); uint8_t tableBir = (uint8_t)(tableOff & 0x7); tableOff &= ~0x7u; if (tableBir != 0) { KernelLogStream(INFO, "WiFi") << "MSI-X table lives in BAR " << (uint64_t)tableBir << " (only BAR0 supported); falling back to MSI"; return false; } // Program entry 0: all causes are mapped to vector 0 by IwxConfMsixHw. volatile uint32_t* entry = (volatile uint32_t*)(g_iwx.Mmio + tableOff); entry[0] = MSI_ADDR_BASE; // message address (BSP, physical, fixed) entry[1] = 0; // upper address entry[2] = MSI_VECTOR; // message data entry[3] = 0; // vector control: unmasked msgCtrl |= (1 << 15); // MSI-X enable msgCtrl &= ~(1 << 14); // function not masked Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl); uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND); pciCmd |= Pci::PCI_CMD_INTX_DISABLE; Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd); Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt); KernelLogStream(OK, "WiFi") << "MSI-X enabled: vector " << (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"; return true; } static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); if (cap == 0) return false; Pci::DisableInterruptDelivery(bus, dev, func); uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); bool is64 = (msgCtrl & (1 << 7)) != 0; Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE); if (is64) { Pci::LegacyWrite32(bus, dev, func, cap + 8, 0); Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR); } else { Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR); } msgCtrl &= ~0x70; // single message msgCtrl |= (1 << 0); // MSI enable Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl); uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND); pciCmd |= Pci::PCI_CMD_INTX_DISABLE; Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd); Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt); KernelLogStream(OK, "WiFi") << "MSI enabled: vector " << (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"; return true; } // ========================================================================= // Probe // ========================================================================= bool IwxProbe(const Pci::PciDevice& dev) { if (g_iwx.State != IwxFwState::Absent) return false; uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function); if (mmioPhys == 0) { KernelLogStream(WARNING, "WiFi") << "Device has no memory BAR0"; return false; } // The CSR/MSI-X window is 16 KiB on these parts; map a bit more so the // MSI-X table (which lives inside BAR0) is always covered. constexpr uint64_t MmioSize = 0x8000; for (uint64_t off = 0; off < MmioSize; off += 0x1000) Memory::VMM::g_paging->MapMMIO(mmioPhys + off, Memory::HHDM(mmioPhys + off)); g_iwx.Mmio = (volatile uint8_t*)Memory::HHDM(mmioPhys); g_iwx.Bus = dev.Bus; g_iwx.Dev = dev.Device; g_iwx.Func = dev.Function; Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function); // LTR_CONFIG may only be sent when the PCIe link actually advertises // Latency Tolerance Reporting (Device Control 2, bit 10); firmware // rejects it otherwise. constexpr uint8_t PCI_CAP_PCIE = 0x10; constexpr uint8_t PCIE_DCSR2 = 0x28; uint8_t pcieCap = Pci::FindCapability(dev.Bus, dev.Device, dev.Function, PCI_CAP_PCIE); if (pcieCap) { uint32_t dcsr2 = Pci::LegacyRead32(dev.Bus, dev.Device, dev.Function, (uint8_t)(pcieCap + PCIE_DCSR2)); g_iwx.LtrEnabled = (dcsr2 & (1u << 10)) != 0; } // Disable the PCI retry timeout (0x41) so Tx retries cannot interfere // with deep CPU C-states. uint32_t reg40 = Pci::LegacyRead32(dev.Bus, dev.Device, dev.Function, 0x40); Pci::LegacyWrite32(dev.Bus, dev.Device, dev.Function, 0x40, reg40 & ~0xff00u); IwxWrite32(IWX_CSR_INT_MASK, 0); IwxWrite32(IWX_CSR_INT, ~0u); IwxWrite32(IWX_CSR_FH_INT_STATUS, ~0u); g_iwx.HwRev = IwxRead32(IWX_CSR_HW_REV); g_iwx.HwRfId = IwxRead32(IWX_CSR_HW_RF_ID); uint32_t macType = IWX_CSR_HW_REV_TYPE(g_iwx.HwRev); uint32_t rfType = IWX_CSR_HW_RFID_TYPE(g_iwx.HwRfId); KernelLogStream(OK, "WiFi") << "Intel Wi-Fi device " << base::hex << (uint64_t)dev.DeviceId << " at PCI " << (uint64_t)dev.Bus << ":" << (uint64_t)dev.Device << "." << (uint64_t)dev.Function << " hw_rev=" << (uint64_t)g_iwx.HwRev << " mac_type=" << (uint64_t)macType << " rf_type=" << (uint64_t)rfType << base::dec; if (rfType != IWX_CFG_RF_TYPE_GF) { KernelLogStream(WARNING, "WiFi") << "Unsupported RF type (only GF / AX211 firmware is bundled)"; g_iwx.Mmio = nullptr; return false; } g_msix = SetupMsix(dev.Bus, dev.Device, dev.Function); if (!g_msix && !SetupMsi(dev.Bus, dev.Device, dev.Function)) { KernelLogStream(WARNING, "WiFi") << "No MSI-X/MSI available; the device cannot be driven by polling alone"; g_iwx.Mmio = nullptr; return false; } // Boot-time DMA that persists for the device's lifetime. if (!IwxDmaAlloc(g_iwx.CtxtInfo, sizeof(IwxContextInfoGen3)) || !IwxDmaAlloc(g_iwx.PrphScratch, sizeof(IwxPrphScratch)) || !IwxDmaAlloc(g_iwx.PrphInfo, 4096) || !IwxAllocRxRing() || !IwxAllocTxRing(g_iwx.CmdQ, IWX_DQA_CMD_QUEUE) || !IwxAllocTxRing(g_iwx.MgmtQ, IWX_DQA_MGMT_QUEUE, IWX_TX_STAGE_SLOTS)) { KernelLogStream(ERROR, "WiFi") << "Could not allocate device DMA memory"; IwxFreeRxRing(); IwxFreeTxRing(g_iwx.CmdQ); IwxFreeTxRing(g_iwx.MgmtQ); IwxDmaFree(g_iwx.CtxtInfo); IwxDmaFree(g_iwx.PrphScratch); IwxDmaFree(g_iwx.PrphInfo); g_iwx.Mmio = nullptr; return false; } g_iwx.State = IwxFwState::Detected; KernelLogStream(INFO, "WiFi") << "Transport ready; deferring firmware load until the ramdisk is mounted"; return true; } }