daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bbe1df62fd7b66c6cd7cd78e2cb3430ee16be63c
MontaukOS/kernel/src/Drivers/Net/Wifi/IwxTrans.cpp
T

1811 lines
72 KiB
C++
Raw Blame History

/*
* IwxTrans.cpp
* Intel AX210/AX211 PCIe transport: MMIO access, MSI-X, DMA rings,
* context-info-gen3 firmware boot, host commands and RX processing.
*
* The bring-up sequence follows Intel's iwlwifi/OpenBSD iwx flow:
* prepare card -> sw reset -> APM init -> MSI-X config -> rfkill check
* -> context info gen3 (firmware self-load) -> ALIVE -> PNVM -> init cmds
*
* Everything here runs in process/idle context. Hard interrupts only latch
* a "work pending" flag; all ring processing happens in IwxProcessEvents(),
* which the idle loop and the command wait loops pump. This mirrors the
* Bluetooth/E1000E split in this kernel: the network and IPC layers take
* process-context mutexes that must never be entered from an IRQ.
*
* Copyright (c) 2026 Daniel Hammer
*/
#include "Iwx.hpp"
#include "Ieee80211.hpp"
#include <Pci/Pci.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Memory/Heap.hpp>
#include <Libraries/Memory.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
namespace Drivers::Net::Wifi {
IwxState g_iwx;
// MSI-X/MSI interrupt slot. Slots in use elsewhere: 24 (E1000E),
// 27 (Intel HDA), 28 (GPU vblank), 29 (xHCI).
static constexpr uint8_t MSI_IRQ = 25;
static constexpr uint32_t MSI_VECTOR = 57; // IRQ_VECTOR_BASE + MSI_IRQ
static constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// Cached MSI-X masks (a cause is ENABLED when its mask bit is clear).
static uint32_t g_fhInitMask = 0;
static uint32_t g_hwInitMask = 0;
static uint32_t g_fhMask = 0;
static uint32_t g_hwMask = 0;
static bool g_msix = false;
// =========================================================================
// Register access
// =========================================================================
uint32_t IwxRead32(uint32_t reg) {
return *(volatile uint32_t*)(g_iwx.Mmio + reg);
}
void IwxWrite32(uint32_t reg, uint32_t val) {
*(volatile uint32_t*)(g_iwx.Mmio + reg) = val;
}
void IwxWrite8(uint32_t reg, uint8_t val) {
*(volatile uint8_t*)(g_iwx.Mmio + reg) = val;
}
void IwxSetBits(uint32_t reg, uint32_t bits) {
IwxWrite32(reg, IwxRead32(reg) | bits);
}
void IwxClearBits(uint32_t reg, uint32_t bits) {
IwxWrite32(reg, IwxRead32(reg) & ~bits);
}
// Approximate busy-wait. The kernel has no sub-millisecond clock source
// exposed here; the loop count is calibrated the same way IntelHda does it.
void IwxDelayUs(uint32_t us) {
for (volatile uint32_t i = 0; i < us * 100u; i++) {
asm volatile("pause" ::: "memory");
}
}
void IwxDelayMs(uint32_t ms) {
uint64_t start = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - start < ms) {
asm volatile("pause" ::: "memory");
}
}
bool IwxPollBit(uint32_t reg, uint32_t bits, uint32_t mask, int timeoutUs) {
for (int t = 0; t < timeoutUs; t += 10) {
if ((IwxRead32(reg) & mask) == bits) return true;
IwxDelayUs(10);
}
return false;
}
// =========================================================================
// NIC lock (MAC wakeup for PRPH / internal register access)
// =========================================================================
bool IwxNicLock() {
if (g_iwx.NicLockCount > 0) {
g_iwx.NicLockCount++;
return true;
}
IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
IwxDelayUs(2);
if (IwxPollBit(IWX_CSR_GP_CNTRL,
IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY
| IWX_CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP, 150000)) {
g_iwx.NicLockCount = 1;
return true;
}
KernelLogStream(WARNING, "WiFi") << "Timeout waiting for NIC access";
return false;
}
void IwxNicUnlock() {
if (g_iwx.NicLockCount > 0) {
if (--g_iwx.NicLockCount == 0)
IwxClearBits(IWX_CSR_GP_CNTRL,
IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
}
}
// PRPH accessors. Bits 24-25 of the address register encode "number of
// bytes - 1"; the driver only ever does dword accesses (3).
//
// The address field is 24 bits wide on the AX210 family (20 bits on older
// parts). This matters: UMAC PRPH registers are reached by adding
// IWX_UMAC_PRPH_OFFSET (0x300000), so e.g. UREG_CPU_INIT_RUN becomes
// 0xd05c44 -- a 20-bit mask truncates that to 0x005c44 and the access
// silently lands on an unrelated register. With the "kick firmware
// self-load" write going nowhere, the firmware never boots and ALIVE
// never arrives.
static uint32_t PrphAddr(uint32_t addr) {
return ((addr & 0x00FFFFFF) | (3 << 24));
}
uint32_t IwxReadPrph(uint32_t addr) {
IwxWrite32(IWX_HBUS_TARG_PRPH_RADDR, PrphAddr(addr));
asm volatile("" ::: "memory");
return IwxRead32(IWX_HBUS_TARG_PRPH_RDAT);
}
void IwxWritePrph(uint32_t addr, uint32_t val) {
IwxWrite32(IWX_HBUS_TARG_PRPH_WADDR, PrphAddr(addr));
asm volatile("" ::: "memory");
IwxWrite32(IWX_HBUS_TARG_PRPH_WDAT, val);
}
uint32_t IwxReadUmacPrph(uint32_t addr) {
return IwxReadPrph(addr + IWX_UMAC_PRPH_OFFSET);
}
void IwxWriteUmacPrph(uint32_t addr, uint32_t val) {
IwxWritePrph(addr + IWX_UMAC_PRPH_OFFSET, val);
}
// =========================================================================
// DMA allocation
// =========================================================================
bool IwxDmaAlloc(IwxDma& dma, uint64_t bytes) {
if (bytes == 0) return false;
uint32_t pages = (uint32_t)((bytes + 0xFFF) / 0x1000);
void* base = Memory::g_pfa->Allocate();
if (!base) return false;
if (pages > 1) {
base = Memory::g_pfa->ReallocConsecutive(base, (int)pages);
if (!base) return false;
}
memset(base, 0, (size_t)pages * 0x1000);
dma.Virt = base;
dma.Phys = Memory::SubHHDM(base);
dma.Pages = pages;
return true;
}
void IwxDmaFree(IwxDma& dma) {
if (!dma.Virt) return;
if (dma.Pages > 1) Memory::g_pfa->Free(dma.Virt, (int)dma.Pages);
else Memory::g_pfa->Free(dma.Virt);
dma.Virt = nullptr;
dma.Phys = 0;
dma.Pages = 0;
}
// =========================================================================
// Interrupt configuration
// =========================================================================
static void IwxDisableInterrupts() {
if (!g_msix) {
IwxWrite32(IWX_CSR_INT_MASK, 0);
IwxWrite32(IWX_CSR_INT, ~0u);
IwxWrite32(IWX_CSR_FH_INT_STATUS, ~0u);
} else {
IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, g_fhInitMask);
IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, g_hwInitMask);
}
}
static void IwxEnableInterrupts() {
if (!g_msix) {
// Non-MSI-X path: enable the causes we act on.
constexpr uint32_t INT_BIT_FH_RX = 1u << 31;
constexpr uint32_t INT_BIT_HW_ERR = 1u << 29;
constexpr uint32_t INT_BIT_FH_TX = 1u << 27;
constexpr uint32_t INT_BIT_SW_ERR = 1u << 25;
constexpr uint32_t INT_BIT_RF_KILL = 1u << 7;
constexpr uint32_t INT_BIT_SW_RX = 1u << 3;
constexpr uint32_t INT_BIT_ALIVE = 1u << 0;
IwxWrite32(IWX_CSR_INT_MASK, INT_BIT_FH_RX | INT_BIT_HW_ERR
| INT_BIT_FH_TX | INT_BIT_SW_ERR | INT_BIT_RF_KILL
| INT_BIT_SW_RX | INT_BIT_ALIVE);
} else {
g_hwMask = g_hwInitMask;
g_fhMask = g_fhInitMask;
IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, ~g_fhMask);
IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, ~g_hwMask);
}
}
// Map every cause to vector 0 and unmask the ones we handle. `stopped`
// skips the UREG_CHICK write, which needs a live MAC.
static void IwxConfMsixHw(bool stopped) {
constexpr uint8_t vector = 0;
if (!g_msix) {
if (!stopped && IwxNicLock()) {
IwxWriteUmacPrph(IWX_UREG_CHICK, IWX_UREG_CHICK_MSI_ENABLE);
IwxNicUnlock();
}
return;
}
if (!stopped && IwxNicLock()) {
IwxWriteUmacPrph(IWX_UREG_CHICK, IWX_UREG_CHICK_MSIX_ENABLE);
IwxNicUnlock();
}
IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, ~0u);
IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD, ~0u);
// RX queues 0 (command/fallback) and 1 (RSS data) -> our vector.
IwxWrite8(IWX_CSR_MSIX_RX_IVAR(0), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE);
IwxWrite8(IWX_CSR_MSIX_RX_IVAR(1), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE);
static const uint32_t causes[] = {
IWX_MSIX_IVAR_CAUSE_D2S_CH0_NUM, IWX_MSIX_IVAR_CAUSE_D2S_CH1_NUM,
IWX_MSIX_IVAR_CAUSE_S2D, IWX_MSIX_IVAR_CAUSE_FH_ERR,
IWX_MSIX_IVAR_CAUSE_REG_ALIVE, IWX_MSIX_IVAR_CAUSE_REG_WAKEUP,
IWX_MSIX_IVAR_CAUSE_REG_RESET_DONE, IWX_MSIX_IVAR_CAUSE_REG_CT_KILL,
IWX_MSIX_IVAR_CAUSE_REG_RF_KILL, IWX_MSIX_IVAR_CAUSE_REG_PERIODIC,
IWX_MSIX_IVAR_CAUSE_REG_SW_ERR, IWX_MSIX_IVAR_CAUSE_REG_SW_ERR_V2,
IWX_MSIX_IVAR_CAUSE_REG_SCD, IWX_MSIX_IVAR_CAUSE_REG_FH_TX,
IWX_MSIX_IVAR_CAUSE_REG_HW_ERR, IWX_MSIX_IVAR_CAUSE_REG_HAP,
};
for (uint32_t c : causes)
IwxWrite8(IWX_CSR_MSIX_IVAR(c), vector | IWX_MSIX_NON_AUTO_CLEAR_CAUSE);
IwxClearBits(IWX_CSR_MSIX_FH_INT_MASK_AD,
IWX_MSIX_FH_INT_CAUSES_Q0 | IWX_MSIX_FH_INT_CAUSES_Q1
| IWX_MSIX_FH_INT_CAUSES_D2S_CH0_NUM
| IWX_MSIX_FH_INT_CAUSES_D2S_CH1_NUM
| IWX_MSIX_FH_INT_CAUSES_S2D
| IWX_MSIX_FH_INT_CAUSES_FH_ERR);
IwxClearBits(IWX_CSR_MSIX_HW_INT_MASK_AD,
IWX_MSIX_HW_INT_CAUSES_REG_ALIVE
| IWX_MSIX_HW_INT_CAUSES_REG_WAKEUP
| IWX_MSIX_HW_INT_CAUSES_REG_RESET_DONE
| IWX_MSIX_HW_INT_CAUSES_REG_CT_KILL
| IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL
| IWX_MSIX_HW_INT_CAUSES_REG_PERIODIC
| IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR
| IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR_V2
| IWX_MSIX_HW_INT_CAUSES_REG_SCD
| IWX_MSIX_HW_INT_CAUSES_REG_FH_TX
| IWX_MSIX_HW_INT_CAUSES_REG_HW_ERR
| IWX_MSIX_HW_INT_CAUSES_REG_HAP);
}
static void IwxInitMsixHw() {
IwxConfMsixHw(false);
if (!g_msix) return;
g_fhInitMask = ~IwxRead32(IWX_CSR_MSIX_FH_INT_MASK_AD);
g_fhMask = g_fhInitMask;
g_hwInitMask = ~IwxRead32(IWX_CSR_MSIX_HW_INT_MASK_AD);
g_hwMask = g_hwInitMask;
}
static void IwxEnableRfkillInt() {
if (!g_msix) {
IwxWrite32(IWX_CSR_INT_MASK, 1u << 7); // IWX_CSR_INT_BIT_RF_KILL
} else {
IwxWrite32(IWX_CSR_MSIX_FH_INT_MASK_AD, g_fhInitMask);
IwxWrite32(IWX_CSR_MSIX_HW_INT_MASK_AD,
~IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL);
g_hwMask = IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL;
}
IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_RFKILL_WAKE_L1A_EN);
}
bool IwxCheckRfKill() {
// Documented as "state of the hardware RF-Kill switch", but the bit
// reads 0 when the radio is DISABLED (Intel's own comment says the
// documentation is unhelpful here).
uint32_t v = IwxRead32(IWX_CSR_GP_CNTRL);
return (v & IWX_CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW) == 0;
}
// The hard interrupt only latches work. All ring/notification handling
// takes heap and (indirectly) IPC locks, so it runs from the idle loop.
static void HandleInterrupt(uint8_t, bool) {
g_iwx.WorkPending = true;
}
// =========================================================================
// APM (adapter power management) bring-up
// =========================================================================
static void IwxApmConfig() {
// L0s is unstable on these parts and unsupported on newer ones; the
// Linux and OpenBSD drivers both disable it unconditionally.
IwxSetBits(IWX_CSR_GIO_REG, IWX_CSR_GIO_REG_VAL_L0S_DISABLED);
}
static bool IwxApmInit() {
IwxSetBits(IWX_CSR_GIO_CHICKEN_BITS,
IWX_CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
IwxSetBits(IWX_CSR_DBG_HPET_MEM_REG, IWX_CSR_DBG_HPET_MEM_REG_VAL);
IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG,
IWX_CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
IwxApmConfig();
// D0U* -> D0A*: "initialization complete".
IwxSetBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
if (!IwxPollBit(IWX_CSR_GP_CNTRL,
IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
IWX_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000)) {
KernelLogStream(ERROR, "WiFi") << "Timeout waiting for clock stabilization";
return false;
}
return true;
}
static void IwxApmStop() {
IwxSetBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG,
IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED);
IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG,
IWX_CSR_HW_IF_CONFIG_REG_PREPARE
| IWX_CSR_HW_IF_CONFIG_REG_ENABLE_PME);
IwxDelayUs(1000);
IwxClearBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG,
IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED);
IwxDelayUs(5000);
IwxSetBits(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_STOP_MASTER);
if (!IwxPollBit(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_MASTER_DISABLED,
IWX_CSR_RESET_REG_FLAG_MASTER_DISABLED, 100))
KernelLogStream(WARNING, "WiFi") << "Timeout waiting for bus master";
IwxClearBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
}
static void IwxSwReset() {
IwxSetBits(IWX_CSR_RESET, IWX_CSR_RESET_REG_FLAG_SW_RESET);
IwxDelayUs(5000);
}
static bool IwxSetHwReady() {
IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG,
IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
bool ready = IwxPollBit(IWX_CSR_HW_IF_CONFIG_REG,
IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
IWX_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY, 50);
if (ready)
IwxSetBits(IWX_CSR_MBOX_SET_REG, IWX_CSR_MBOX_SET_REG_OS_ALIVE);
return ready;
}
static bool IwxPrepareCardHw() {
if (IwxSetHwReady()) return true;
IwxSetBits(IWX_CSR_DBG_LINK_PWR_MGMT_REG,
IWX_CSR_RESET_LINK_PWR_MGMT_DISABLED);
IwxDelayUs(1000);
for (int tries = 0; tries < 10; tries++) {
IwxSetBits(IWX_CSR_HW_IF_CONFIG_REG,
IWX_CSR_HW_IF_CONFIG_REG_PREPARE);
int t = 0;
do {
if (IwxSetHwReady()) return true;
IwxDelayUs(200);
t += 200;
} while (t < 150000);
IwxDelayUs(25000);
}
return false;
}
// =========================================================================
// Ring allocation
// =========================================================================
static bool IwxAllocRxRing() {
IwxRxRing& ring = g_iwx.RxQ;
ring.Cur = 0;
if (!IwxDmaAlloc(ring.FreeDescs,
sizeof(IwxRxTransferDesc) * IWX_RX_MQ_RING_COUNT))
return false;
if (!IwxDmaAlloc(ring.Stat, 64))
return false;
if (!IwxDmaAlloc(ring.UsedDescs,
sizeof(IwxRxCompletionDesc) * IWX_RX_MQ_RING_COUNT))
return false;
// One 4 KiB page per RX buffer (IWX_RBUF_SIZE == PAGE_SIZE).
for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) {
void* p = Memory::g_pfa->AllocateZeroed();
if (!p) return false;
ring.Buf[i] = (uint8_t*)p;
ring.BufPhys[i] = Memory::SubHHDM(p);
}
return true;
}
static void IwxFreeRxRing() {
IwxRxRing& ring = g_iwx.RxQ;
IwxDmaFree(ring.FreeDescs);
IwxDmaFree(ring.Stat);
IwxDmaFree(ring.UsedDescs);
for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) {
if (ring.Buf[i]) {
Memory::g_pfa->Free(ring.Buf[i]);
ring.Buf[i] = nullptr;
}
}
}
// Publish every RX buffer into the free ring. The firmware configures the
// RFH itself and only accepts these once it signals ALIVE.
static void IwxUpdateRxDescs() {
auto* descs = (IwxRxTransferDesc*)g_iwx.RxQ.FreeDescs.Virt;
for (uint32_t i = 0; i < IWX_RX_MQ_RING_COUNT; i++) {
descs[i].rbid = (uint16_t)(i & 0xffff);
descs[i].addr = g_iwx.RxQ.BufPhys[i];
}
asm volatile("" ::: "memory");
}
// `stageSlots` reserves one page per concurrently queued frame; pass 0 for
// queues that only ever carry host commands.
static bool IwxAllocTxRing(IwxTxRing& ring, int qid, uint32_t stageSlots = 0) {
ring.Qid = qid;
ring.Cur = 0;
ring.CurHw = 0;
ring.Queued = 0;
ring.StageSlots = 0;
ring.Active = false;
if (!IwxDmaAlloc(ring.Desc, sizeof(IwxTfhTfd) * IWX_TX_RING_COUNT))
return false;
if (!IwxDmaAlloc(ring.BcTbl,
sizeof(IwxGen3BcTblEntry) * IWX_TFD_QUEUE_BC_SIZE_GEN3_AX210))
return false;
if (!IwxDmaAlloc(ring.Cmd, sizeof(IwxDeviceCmd) * IWX_TX_RING_COUNT))
return false;
// Commands larger than IWX_DEF_CMD_PAYLOAD_SIZE (e.g. SCAN_REQ_UMAC)
// are staged in this page instead of the per-slot command area.
if (!IwxDmaAlloc(ring.Bounce, 4096))
return false;
if (stageSlots > IWX_TX_STAGE_SLOTS) stageSlots = IWX_TX_STAGE_SLOTS;
for (uint32_t i = 0; i < stageSlots; i++) {
void* p = Memory::g_pfa->AllocateZeroed();
if (!p) return false;
ring.Stage[i] = (uint8_t*)p;
ring.StagePhys[i] = Memory::SubHHDM(p);
ring.StageSlots = i + 1;
}
return true;
}
static void IwxFreeTxRing(IwxTxRing& ring) {
IwxDmaFree(ring.Desc);
IwxDmaFree(ring.BcTbl);
IwxDmaFree(ring.Cmd);
IwxDmaFree(ring.Bounce);
for (uint32_t i = 0; i < ring.StageSlots; i++) {
if (ring.Stage[i]) {
Memory::g_pfa->Free(ring.Stage[i]);
ring.Stage[i] = nullptr;
}
}
ring.StageSlots = 0;
ring.Active = false;
}
static void IwxResetTxRing(IwxTxRing& ring) {
if (ring.BcTbl.Virt) memset(ring.BcTbl.Virt, 0, ring.BcTbl.Pages * 0x1000);
if (ring.Desc.Virt) memset(ring.Desc.Virt, 0, ring.Desc.Pages * 0x1000);
ring.Cur = 0;
ring.CurHw = 0;
ring.Queued = 0;
}
// =========================================================================
// Firmware section staging + context info (gen3)
// =========================================================================
static void IwxFreeFwSecDma() {
for (int i = 0; i < g_iwx.FwSecDmaCount; i++) IwxDmaFree(g_iwx.FwSecDma[i]);
g_iwx.FwSecDmaCount = 0;
}
static void IwxFreePagingDma() {
for (int i = 0; i < g_iwx.PagingCount; i++) IwxDmaFree(g_iwx.PagingDma[i]);
g_iwx.PagingCount = 0;
}
// Count sections until the next separator marker.
static int IwxNumSections(int start) {
int n = 0;
while (start < g_iwx.Fw.SectionCount
&& g_iwx.Fw.Sections[start].DevOff != IWX_CPU1_CPU2_SEPARATOR_SECTION
&& g_iwx.Fw.Sections[start].DevOff != IWX_PAGING_SEPARATOR_SECTION) {
start++;
n++;
}
return n;
}
// Copy LMAC/UMAC/paging sections into DMA and record their addresses in the
// firmware's DRAM map. The image is laid out as
// [lmac...] SEP [umac...] SEP [paging...].
static bool IwxInitFwSec(IwxContextInfoDram* dram) {
int lmacCnt = IwxNumSections(0);
int umacCnt = IwxNumSections(lmacCnt + 1);
int pagingCnt = IwxNumSections(lmacCnt + umacCnt + 2);
if (lmacCnt <= 0 || umacCnt <= 0) {
KernelLogStream(ERROR, "WiFi") << "Firmware image has no LMAC/UMAC sections";
return false;
}
if (lmacCnt + umacCnt + pagingCnt > IWX_MAX_FW_SECTIONS) return false;
int fwCnt = 0;
for (int i = 0; i < lmacCnt; i++) {
IwxDma& d = g_iwx.FwSecDma[fwCnt];
if (!IwxDmaAlloc(d, g_iwx.Fw.Sections[i].Len)) return false;
memcpy(d.Virt, g_iwx.Fw.Sections[i].Data, g_iwx.Fw.Sections[i].Len);
dram->lmac_img[i] = d.Phys;
g_iwx.FwSecDmaCount = ++fwCnt;
}
for (int i = 0; i < umacCnt; i++) {
IwxDma& d = g_iwx.FwSecDma[fwCnt];
const IwxFwSection& sec = g_iwx.Fw.Sections[fwCnt + 1]; // skip separator
if (!IwxDmaAlloc(d, sec.Len)) return false;
memcpy(d.Virt, sec.Data, sec.Len);
dram->umac_img[i] = d.Phys;
g_iwx.FwSecDmaCount = ++fwCnt;
}
// Paging sections are tracked separately: unlike the LMAC/UMAC images,
// the firmware keeps demand-paging from them for as long as it runs, so
// they must survive past ALIVE.
if (pagingCnt > (int)IWX_MAX_DRAM_ENTRY) pagingCnt = IWX_MAX_DRAM_ENTRY;
IwxFreePagingDma();
for (int i = 0; i < pagingCnt; i++) {
IwxDma& d = g_iwx.PagingDma[i];
const IwxFwSection& sec = g_iwx.Fw.Sections[fwCnt + i + 2];
if (!IwxDmaAlloc(d, sec.Len)) return false;
memcpy(d.Virt, sec.Data, sec.Len);
dram->virtual_img[i] = d.Phys;
g_iwx.PagingCount = i + 1;
}
return true;
}
static void IwxSetLtr() {
uint32_t ltrVal =
IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_REQ
| ((IWX_CSR_LTR_LONG_VAL_AD_SCALE_USEC
<< IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_SCALE_SHIFT)
& IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_SCALE_MASK)
| ((250u << IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_VAL_SHIFT)
& IWX_CSR_LTR_LONG_VAL_AD_NO_SNOOP_VAL_MASK)
| IWX_CSR_LTR_LONG_VAL_AD_SNOOP_REQ
| ((IWX_CSR_LTR_LONG_VAL_AD_SCALE_USEC
<< IWX_CSR_LTR_LONG_VAL_AD_SNOOP_SCALE_SHIFT)
& IWX_CSR_LTR_LONG_VAL_AD_SNOOP_SCALE_MASK)
| (250u & IWX_CSR_LTR_LONG_VAL_AD_SNOOP_VAL);
// The AX211 in this laptop is a CNVi (integrated) part of the AX210
// family; only discrete parts take the CSR write here (the Qu-only
// HPM_*_LTR path does not apply to AX210).
(void)ltrVal;
}
// Fill the gen3 context info and kick the firmware's self-load.
static bool IwxCtxtInfoGen3Init() {
if (!g_iwx.Fw.Iml || g_iwx.Fw.ImlLen == 0) {
KernelLogStream(ERROR, "WiFi") << "Firmware has no image loader (IML)";
return false;
}
if (!IwxDmaAlloc(g_iwx.ImlDma, g_iwx.Fw.ImlLen)) return false;
auto* scratch = (IwxPrphScratch*)g_iwx.PrphScratch.Virt;
memset(scratch, 0, sizeof(*scratch));
auto* ctrl = &scratch->ctrl_cfg;
ctrl->version.version = 0;
ctrl->version.mac_id = (uint16_t)IwxRead32(IWX_CSR_HW_REV);
ctrl->version.size = (uint16_t)(sizeof(*scratch) / 4);
ctrl->control.control_flags = IWX_PRPH_SCRATCH_RB_SIZE_4K
| IWX_PRPH_SCRATCH_MTR_MODE
| (IWX_PRPH_MTR_FORMAT_256B
& IWX_PRPH_SCRATCH_MTR_FORMAT);
ctrl->rbd_cfg.free_rbd_addr = g_iwx.RxQ.FreeDescs.Phys;
if (!IwxInitFwSec(&scratch->dram)) {
IwxDmaFree(g_iwx.ImlDma);
IwxFreeFwSecDma();
IwxFreePagingDma();
return false;
}
auto* ci = (IwxContextInfoGen3*)g_iwx.CtxtInfo.Virt;
memset(ci, 0, sizeof(*ci));
ci->prph_info_base_addr = g_iwx.PrphInfo.Phys;
ci->prph_scratch_base_addr = g_iwx.PrphScratch.Phys;
ci->prph_scratch_size = sizeof(IwxPrphScratch);
ci->cr_head_idx_arr_base_addr = g_iwx.RxQ.Stat.Phys;
// The driver does not use the TR/CR tail arrays, but the device still
// writes there; point them at the unused half of the prph-info page.
ci->tr_tail_idx_arr_base_addr = g_iwx.PrphInfo.Phys + 0x1000 / 2;
ci->cr_tail_idx_arr_base_addr = g_iwx.PrphInfo.Phys + 3 * 0x1000 / 4;
ci->mtr_base_addr = g_iwx.CmdQ.Desc.Phys;
ci->mcr_base_addr = g_iwx.RxQ.UsedDescs.Phys;
ci->mtr_size = (uint16_t)IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT);
ci->mcr_size = (uint16_t)IWX_RX_QUEUE_CB_SIZE(IWX_RX_MQ_RING_COUNT);
memcpy(g_iwx.ImlDma.Virt, g_iwx.Fw.Iml, g_iwx.Fw.ImlLen);
asm volatile("" ::: "memory");
uint64_t paddr = g_iwx.CtxtInfo.Phys;
IwxWrite32(IWX_CSR_CTXT_INFO_ADDR, (uint32_t)(paddr & 0xffffffff));
IwxWrite32(IWX_CSR_CTXT_INFO_ADDR + 4, (uint32_t)(paddr >> 32));
paddr = g_iwx.ImlDma.Phys;
IwxWrite32(IWX_CSR_IML_DATA_ADDR, (uint32_t)(paddr & 0xffffffff));
IwxWrite32(IWX_CSR_IML_DATA_ADDR + 4, (uint32_t)(paddr >> 32));
IwxWrite32(IWX_CSR_IML_SIZE_ADDR, g_iwx.Fw.ImlLen);
IwxSetBits(IWX_CSR_CTXT_INFO_BOOT_CTRL, IWX_CSR_AUTO_FUNC_BOOT_ENA);
if (!IwxNicLock()) {
IwxDmaFree(g_iwx.ImlDma);
IwxFreeFwSecDma();
IwxFreePagingDma();
return false;
}
IwxSetLtr();
IwxWriteUmacPrph(IWX_UREG_CPU_INIT_RUN, 1);
IwxNicUnlock();
return true;
}
// =========================================================================
// NIC init + firmware start
// =========================================================================
static bool IwxNicInit() {
if (!IwxApmInit()) return false;
// AX210 firmware configures the RFH itself; only interrupt coalescing
// and the shadow-register control need programming here.
IwxWrite8(IWX_CSR_INT_COALESCING, IWX_HOST_INT_TIMEOUT_DEF);
IwxSetBits(IWX_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
return true;
}
bool IwxStartHw() {
if (!IwxPrepareCardHw()) {
KernelLogStream(ERROR, "WiFi") << "Could not prepare card hardware";
return false;
}
IwxSwReset();
if (!IwxApmInit()) return false;
IwxInitMsixHw();
IwxEnableRfkillInt();
if (IwxCheckRfKill()) {
KernelLogStream(WARNING, "WiFi") << "Radio is disabled by hardware switch";
g_iwx.State = IwxFwState::RfKill;
}
return true;
}
void IwxStopDevice() {
IwxDisableInterrupts();
if (IwxNicLock()) {
IwxWriteUmacPrph(IWX_RFH_RXF_DMA_CFG_GEN3, 0);
for (int i = 0; i < 1000; i++) {
if (IwxReadUmacPrph(IWX_RFH_GEN_STATUS_GEN3) & IWX_RXF_DMA_IDLE)
break;
IwxDelayUs(10);
}
IwxNicUnlock();
}
IwxResetTxRing(g_iwx.CmdQ);
IwxResetTxRing(g_iwx.MgmtQ);
g_iwx.RxQ.Cur = 0;
if (g_iwx.RxQ.Stat.Virt) *(volatile uint16_t*)g_iwx.RxQ.Stat.Virt = 0;
IwxClearBits(IWX_CSR_GP_CNTRL, IWX_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
g_iwx.NicLockCount = 0;
IwxApmStop();
IwxSwReset();
// The IVAR table is wiped by the reset; reprogram it so an RF-kill
// interrupt still lands on our vector.
IwxConfMsixHw(true);
IwxDisableInterrupts();
IwxEnableRfkillInt();
IwxCheckRfKill();
IwxPrepareCardHw();
IwxFreeFwSecDma();
IwxFreePagingDma();
IwxDmaFree(g_iwx.PnvmDma);
for (int i = 0; i < g_iwx.PnvmSegs; i++) IwxDmaFree(g_iwx.PnvmSegDma[i]);
g_iwx.PnvmSegs = 0;
g_iwx.PnvmSize = 0;
}
bool IwxStartFirmware() {
g_iwx.AliveIntr = false;
g_iwx.AliveOk = false;
g_iwx.InitComplete = 0;
IwxWrite32(IWX_CSR_INT, ~0u);
IwxDisableInterrupts();
// Clear the rfkill handshake bits left over from a previous run.
IwxWrite32(IWX_CSR_UCODE_DRV_GP1_CLR, IWX_CSR_UCODE_SW_BIT_RFKILL);
IwxWrite32(IWX_CSR_UCODE_DRV_GP1_CLR, IWX_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
IwxWrite32(IWX_CSR_INT, ~0u);
if (!IwxNicInit()) {
KernelLogStream(ERROR, "WiFi") << "Unable to init NIC";
return false;
}
IwxEnableInterrupts();
if (!IwxCtxtInfoGen3Init()) return false;
// Wait for the ALIVE notification (firmware self-load takes ~100 ms).
uint64_t start = Timekeeping::GetMilliseconds();
while (!g_iwx.AliveIntr && Timekeeping::GetMilliseconds() - start < 2000) {
IwxProcessEvents();
IwxDelayUs(200);
}
IwxDmaFree(g_iwx.ImlDma);
IwxFreeFwSecDma();
if (!g_iwx.AliveOk) {
KernelLogStream(ERROR, "WiFi") << "Firmware did not come alive"
<< (g_iwx.AliveIntr ? " (bad ALIVE response)" : " (timeout)");
// The firmware mirrors its boot progress into the prph-info page,
// so a stalled self-load can be told apart from one that ran but
// whose notification never reached us.
auto* info = (IwxPrphInfo*)g_iwx.PrphInfo.Virt;
KernelLogStream(INFO, "WiFi") << " boot_stage=" << base::hex
<< (uint64_t)info->boot_stage_mirror
<< " ipc_status=" << (uint64_t)info->ipc_status_mirror
<< " sleep_notif=" << (uint64_t)info->sleep_notif << base::dec;
KernelLogStream(INFO, "WiFi") << " fh_causes=" << base::hex
<< (uint64_t)IwxRead32(IWX_CSR_MSIX_FH_INT_CAUSES_AD)
<< " hw_causes=" << (uint64_t)IwxRead32(IWX_CSR_MSIX_HW_INT_CAUSES_AD)
<< " csr_int=" << (uint64_t)IwxRead32(IWX_CSR_INT)
<< " gp_cntrl=" << (uint64_t)IwxRead32(IWX_CSR_GP_CNTRL)
<< base::dec << (g_msix ? " [MSI-X]" : " [MSI]");
KernelLogStream(INFO, "WiFi") << " rx_used_widx="
<< (uint64_t)(*(volatile uint16_t*)g_iwx.RxQ.Stat.Virt & 0xfff)
<< " rx_cur=" << (uint64_t)g_iwx.RxQ.Cur;
return false;
}
KernelLogStream(OK, "WiFi") << "Firmware alive: " << g_iwx.Fw.Version;
return true;
}
// Point the firmware at the staged PNVM image and wait for its ack.
bool IwxLoadPnvm() {
if (g_iwx.SkuId.data[0] == 0 && g_iwx.SkuId.data[1] == 0
&& g_iwx.SkuId.data[2] == 0)
return true; // no PNVM needed
if (g_iwx.PnvmDma.Virt == nullptr && g_iwx.Fw.PnvmData) {
// Parse the PNVM TLV blob for our SKU; a missing match is not
// fatal, the firmware then runs with built-in defaults.
IwxPnvmParse(g_iwx.Fw.PnvmData, g_iwx.Fw.PnvmLen);
}
if (g_iwx.PnvmDma.Virt) {
auto* scratch = (IwxPrphScratch*)g_iwx.PrphScratch.Virt;
scratch->ctrl_cfg.pnvm_cfg.pnvm_base_addr = g_iwx.PnvmDma.Phys;
scratch->ctrl_cfg.pnvm_cfg.pnvm_size = g_iwx.PnvmSize;
asm volatile("" ::: "memory");
}
if (!IwxNicLock()) return false;
IwxWriteUmacPrph(IWX_UREG_DOORBELL_TO_ISR6, IWX_UREG_DOORBELL_TO_ISR6_PNVM);
IwxNicUnlock();
uint64_t start = Timekeeping::GetMilliseconds();
while (!(g_iwx.InitComplete & 0x2)
&& Timekeeping::GetMilliseconds() - start < 2000) {
IwxProcessEvents();
IwxDelayUs(200);
}
if (!(g_iwx.InitComplete & 0x2)) {
KernelLogStream(WARNING, "WiFi") << "No PNVM complete notification";
return false;
}
return true;
}
// =========================================================================
// Host command submission
// =========================================================================
int IwxLookupCmdVer(uint8_t group, uint8_t cmd) {
for (int i = 0; i < g_iwx.Fw.NumCmdVersions; i++) {
if (g_iwx.Fw.CmdVersions[i].group == group
&& g_iwx.Fw.CmdVersions[i].cmd == cmd)
return g_iwx.Fw.CmdVersions[i].cmd_ver;
}
return -1;
}
int IwxLookupNotifVer(uint8_t group, uint8_t cmd) {
for (int i = 0; i < g_iwx.Fw.NumCmdVersions; i++) {
if (g_iwx.Fw.CmdVersions[i].group == group
&& g_iwx.Fw.CmdVersions[i].cmd == cmd)
return g_iwx.Fw.CmdVersions[i].notif_ver;
}
return -1;
}
void IwxDumpFwError();
static uint32_t g_cmdTimeouts = 0; // consecutive unanswered commands
bool IwxSendCmd(IwxHostCmd& hcmd) {
if (g_iwx.State == IwxFwState::Error) return false;
IwxTxRing& ring = g_iwx.CmdQ;
g_iwx.CmdLock.Acquire();
uint32_t idx = ring.Cur;
uint32_t code = hcmd.Id;
// Firmware API >= 50 rejects group-0 commands; the legacy opcodes must
// be re-tagged into LONG_GROUP. Responses come back tagged the same
// way, so notification dispatch strips the group again.
if (IwxCmdGroupId(code) == 0)
code = IWX_WIDE_ID(IWX_LONG_GROUP, code);
uint32_t hdrLen = sizeof(IwxCmdHeaderWide);
uint32_t datasz = sizeof(IwxDeviceCmd) - hdrLen;
if (hcmd.Len > IWX_MAX_CMD_PAYLOAD_SIZE) {
g_iwx.CmdLock.Release();
KernelLogStream(ERROR, "WiFi") << "Firmware command too long ("
<< (uint64_t)hcmd.Len << " bytes)";
return false;
}
uint8_t* cmdBuf;
uint64_t cmdPhys;
if (hcmd.Len > datasz) {
cmdBuf = (uint8_t*)ring.Bounce.Virt;
cmdPhys = ring.Bounce.Phys;
} else {
cmdBuf = (uint8_t*)ring.Cmd.Virt + (uint64_t)idx * sizeof(IwxDeviceCmd);
cmdPhys = ring.Cmd.Phys + (uint64_t)idx * sizeof(IwxDeviceCmd);
}
memset(cmdBuf, 0, hdrLen + hcmd.Len);
auto* hdr = (IwxCmdHeaderWide*)cmdBuf;
hdr->opcode = IwxCmdOpcode(code);
hdr->group_id = IwxCmdGroupId(code);
hdr->qid = (uint8_t)ring.Qid;
hdr->idx = (uint8_t)idx;
hdr->length = (uint16_t)hcmd.Len;
hdr->version = 0;
if (hcmd.Len && hcmd.Data) memcpy(cmdBuf + hdrLen, hcmd.Data, hcmd.Len);
auto* desc = &((IwxTfhTfd*)ring.Desc.Virt)[idx];
memset(desc, 0, sizeof(*desc));
uint32_t total = hdrLen + hcmd.Len;
uint16_t firstLen = (uint16_t)(total < IWX_FIRST_TB_SIZE ? total : IWX_FIRST_TB_SIZE);
desc->tbs[0].tb_len = firstLen;
desc->tbs[0].addr = cmdPhys;
if (total > IWX_FIRST_TB_SIZE) {
desc->tbs[1].tb_len = (uint16_t)(total - IWX_FIRST_TB_SIZE);
desc->tbs[1].addr = cmdPhys + IWX_FIRST_TB_SIZE;
desc->num_tbs = 2;
} else {
desc->num_tbs = 1;
}
asm volatile("" ::: "memory");
g_iwx.CmdDone = false;
g_iwx.LastCmdId = code;
// Keep the payload so a firmware assert can show exactly what it
// choked on -- struct mismatches are invisible without the bytes.
g_iwx.LastCmdLen = hcmd.Len;
uint32_t keep = hcmd.Len < sizeof(g_iwx.LastCmdPayload)
? hcmd.Len : (uint32_t)sizeof(g_iwx.LastCmdPayload);
if (hcmd.Data && keep) memcpy(g_iwx.LastCmdPayload, hcmd.Data, keep);
g_iwx.CmdWantResp = hcmd.WantResp;
g_iwx.CmdRespLen = 0;
g_iwx.CmdIdx = idx;
ring.Queued++;
ring.Cur = (ring.Cur + 1) % IWX_TX_RING_COUNT;
ring.CurHw = (ring.CurHw + 1) % IWX_TFD_QUEUE_SIZE_MAX_GEN3;
IwxWrite32(IWX_HBUS_TARG_WRPTR, ((uint32_t)ring.Qid << 16) | ring.CurHw);
// Wait for the firmware's response/ack. Commands are serialized by
// CmdLock, so exactly one can be in flight and the completion is
// unambiguous.
// Two independent bounds. The wall clock is the intended one, but it
// is driven by the timer interrupt, so anything that leaves this loop
// running with interrupts disabled would spin forever and take the
// whole machine down with it -- the spin cap makes that impossible.
// Bail immediately if the firmware has asserted, because it will never
// answer this or any later command.
constexpr uint32_t MAX_SPINS = 20000; // ~2 s at 100 us
bool ok = false;
bool died = false;
uint64_t start = Timekeeping::GetMilliseconds();
for (uint32_t spins = 0; spins < MAX_SPINS; spins++) {
IwxProcessEvents();
if (g_iwx.CmdDone) { ok = true; break; }
if (g_iwx.State == IwxFwState::Error) { died = true; break; }
if (Timekeeping::GetMilliseconds() - start >= 1000) break;
IwxDelayUs(100);
}
if (!ok) {
KernelLogStream(WARNING, "WiFi") << "Command 0x" << base::hex
<< (uint64_t)code << base::dec
<< (died ? " abandoned: firmware has stopped responding"
: " timed out");
if (ring.Queued > 0) ring.Queued--;
if (!died) {
// Dump on the first silence: the firmware's error table names
// the command that asserted, and it is overwritten as later
// commands go unanswered.
if (++g_cmdTimeouts == 1) IwxDumpFwError();
// Repeated silence means it is wedged and every later command
// would burn the same timeout, so stop trying. A single late
// response is not worth disabling the adapter over.
if (g_cmdTimeouts >= 3) {
KernelLogStream(ERROR, "WiFi")
<< "Firmware stopped responding to host commands";
g_iwx.FwErrors++;
g_iwx.State = IwxFwState::Error;
}
}
} else {
g_cmdTimeouts = 0;
}
g_iwx.CmdWantResp = false;
g_iwx.CmdLock.Release();
return ok;
}
bool IwxSendCmdPdu(uint32_t id, const void* data, uint32_t len) {
IwxHostCmd cmd;
cmd.Id = id;
cmd.Data = data;
cmd.Len = len;
return IwxSendCmd(cmd);
}
bool IwxSendCmdStatus(uint32_t id, const void* data, uint32_t len,
uint32_t* statusOut) {
IwxHostCmd cmd;
cmd.Id = id;
cmd.Data = data;
cmd.Len = len;
cmd.WantResp = true;
if (!IwxSendCmd(cmd)) return false;
if (g_iwx.CmdRespLen < sizeof(IwxRxPacket) + sizeof(IwxCmdResponse))
return false;
auto* pkt = (IwxRxPacket*)g_iwx.CmdRespBuf;
auto* resp = (IwxCmdResponse*)pkt->data;
if (statusOut) *statusOut = resp->status;
return true;
}
// =========================================================================
// TX queue configuration (used by the connect path)
// =========================================================================
bool IwxEnableTxq(IwxTxRing& ring, int staId, int qid, int tid) {
IwxResetTxRing(ring);
ring.Qid = qid;
ring.Active = false;
int cmdVer = IwxLookupCmdVer(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD);
IwxHostCmd hcmd;
hcmd.WantResp = true;
// AX211 firmware 89 advertises v3 (DATA_PATH group). Older images
// expose no version at all and take the legacy SCD_QUEUE_CFG command.
IwxTxQueueCfgCmd cmdV0 = {};
IwxScdQueueCfgCmd cmdV3 = {};
if (cmdVer == 3) {
cmdV3.operation = IWX_SCD_QUEUE_ADD;
cmdV3.u.add.tfdq_dram_addr = ring.Desc.Phys;
cmdV3.u.add.bc_dram_addr = ring.BcTbl.Phys;
cmdV3.u.add.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT);
cmdV3.u.add.flags = 0;
cmdV3.u.add.sta_mask = 1u << staId;
cmdV3.u.add.tid = (uint8_t)tid;
hcmd.Id = IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD);
hcmd.Data = &cmdV3;
hcmd.Len = sizeof(cmdV3);
} else if (cmdVer <= 0) {
cmdV0.sta_id = (uint8_t)staId;
cmdV0.tid = (uint8_t)tid;
cmdV0.flags = IWX_TX_QUEUE_CFG_ENABLE_QUEUE;
cmdV0.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT);
cmdV0.byte_cnt_addr = ring.BcTbl.Phys;
cmdV0.tfdq_addr = ring.Desc.Phys;
hcmd.Id = IWX_SCD_QUEUE_CFG;
hcmd.Data = &cmdV0;
hcmd.Len = sizeof(cmdV0);
} else {
KernelLogStream(WARNING, "WiFi")
<< "Unsupported SCD_QUEUE_CONFIG version " << (uint64_t)cmdVer;
return false;
}
if (!IwxSendCmd(hcmd)) return false;
if (g_iwx.CmdRespLen < sizeof(IwxRxPacket) + sizeof(IwxTxQueueCfgRsp))
return false;
auto* pkt = (IwxRxPacket*)g_iwx.CmdRespBuf;
auto* resp = (IwxTxQueueCfgRsp*)pkt->data;
// On the v3 data-path API the firmware owns queue assignment: `qid` is
// only a hint and the response names the queue we actually got.
if (resp->queue_number != qid) {
KernelLogStream(INFO, "WiFi") << "Firmware assigned TX queue "
<< (uint64_t)resp->queue_number << " (asked for "
<< (uint64_t)qid << ")";
}
ring.Qid = resp->queue_number;
ring.Active = true;
return true;
}
void IwxDisableTxq(IwxTxRing& ring, int staId, int tid) {
if (!ring.Active) return;
ring.Active = false;
int cmdVer = IwxLookupCmdVer(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD);
if (cmdVer == 3) {
IwxScdQueueCfgCmd cmd = {};
cmd.operation = IWX_SCD_QUEUE_REMOVE;
cmd.u.remove.sta_mask = 1u << staId;
cmd.u.remove.tid = (uint32_t)tid;
IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SCD_QUEUE_CONFIG_CMD),
&cmd, sizeof(cmd));
} else {
IwxTxQueueCfgCmd cmd = {};
cmd.sta_id = (uint8_t)staId;
cmd.tid = (uint8_t)tid;
cmd.flags = 0; // clear ENABLE_QUEUE
cmd.cb_size = IWX_TFD_QUEUE_CB_SIZE(IWX_TX_RING_COUNT);
cmd.byte_cnt_addr = ring.BcTbl.Phys;
cmd.tfdq_addr = ring.Desc.Phys;
IwxSendCmdPdu(IWX_SCD_QUEUE_CFG, &cmd, sizeof(cmd));
}
IwxResetTxRing(ring);
}
// =========================================================================
// Frame transmission
// =========================================================================
// Lowest usable transmit antenna, as a rate_n_flags antenna field.
static uint32_t IwxTxAntBits() {
uint8_t ant = (uint8_t)((g_iwx.Fw.PhyConfig & IWX_FW_PHY_CFG_TX_CHAIN)
>> IWX_FW_PHY_CFG_TX_CHAIN_POS);
if (g_iwx.Nvm.ValidTxAnt) ant &= g_iwx.Nvm.ValidTxAnt;
if (!ant) ant = 1;
uint8_t lowest = (uint8_t)(ant & (uint8_t)(~ant + 1)); // isolate low bit
return (uint32_t)lowest << IWX_RATE_MCS_ANT_POS;
}
// The lowest basic rate for the current band, in whichever rate_n_flags
// encoding the firmware advertises: 1 Mbps CCK on 2.4 GHz, 6 Mbps OFDM on
// 5 GHz. Management frames go out at this rate because rate control has
// no table for the station until it is associated.
static uint32_t IwxLowestRate() {
uint32_t ant = IwxTxAntBits();
// Firmware exposing TX_CMD notification version 7 or later (equally,
// command version 9+) uses the "version 2" rate layout.
int notifVer = IwxLookupNotifVer(IWX_LONG_GROUP, IWX_TX_CMD);
int cmdVer = IwxLookupCmdVer(IWX_LONG_GROUP, IWX_TX_CMD);
bool v2 = notifVer > 6 || cmdVer >= 9;
if (v2) {
uint32_t mod = g_iwx.Is5GHz ? IWX_RATE_MCS_MOD_LEGACY_OFDM
: IWX_RATE_MCS_MOD_CCK;
return ant | mod | IWX_RATE_MCS_CHAN_WIDTH_20 | 0u; // index 0
}
uint32_t plcp = g_iwx.Is5GHz ? IWX_RATE_6M_PLCP : IWX_RATE_1M_PLCP;
uint32_t cck = g_iwx.Is5GHz ? 0 : IWX_RATE_MCS_CCK_MSK_V1;
return ant | cck | plcp;
}
// Recover a queue whose completions stopped arriving. Without this a
// single lost TX response would permanently consume a slot and, after
// StageSlots of them, wedge the queue.
static uint64_t g_txStallMs = 0;
static bool IwxTxQueueHasRoom(IwxTxRing& ring) {
if (ring.Queued < ring.StageSlots) {
g_txStallMs = 0;
return true;
}
uint64_t now = Timekeeping::GetMilliseconds();
if (g_txStallMs == 0) {
g_txStallMs = now;
return false;
}
if (now - g_txStallMs < 2000) return false;
KernelLogStream(WARNING, "WiFi")
<< "TX queue " << (uint64_t)ring.Qid
<< " stopped completing; resetting its outstanding count";
ring.Queued = 0;
g_txStallMs = 0;
return true;
}
bool IwxTxFrame(IwxTxRing& ring, const uint8_t* hdr, uint32_t hdrLen,
const uint8_t* payload, uint32_t payloadLen,
bool encrypt, bool fixedRate) {
if (g_iwx.State != IwxFwState::Running) return false;
if (!ring.Active || ring.StageSlots == 0) return false;
if (!hdr || hdrLen < IEEE80211_HDR_LEN || hdrLen > 32) return false;
// The 802.11 header is padded to a dword boundary before the payload;
// TX_CMD_OFFLD_PAD tells the firmware to skip those bytes.
uint32_t padded = (hdrLen + 3) & ~3u;
uint32_t head = (uint32_t)(sizeof(IwxCmdHeader) + sizeof(IwxTxCmdGen3)) + padded;
if (head + payloadLen > 4096) return false;
g_iwx.TxLock.Acquire();
if (!IwxTxQueueHasRoom(ring)) {
g_iwx.TxLock.Release();
return false;
}
uint32_t idx = ring.Cur;
uint32_t slot = idx % ring.StageSlots;
uint8_t* buf = ring.Stage[slot];
uint64_t phys = ring.StagePhys[slot];
memset(buf, 0, head);
auto* ch = (IwxCmdHeader*)buf;
ch->code = IWX_TX_CMD;
ch->flags = 0; // TX_CMD stays in the legacy group
ch->idx = (uint8_t)idx;
ch->qid = (uint8_t)ring.Qid;
auto* tx = (IwxTxCmdGen3*)(buf + sizeof(IwxCmdHeader));
tx->len = (uint16_t)(hdrLen + payloadLen);
uint16_t flags = 0;
if (!encrypt) flags |= IWX_TX_FLAGS_ENCRYPT_DIS;
if (fixedRate) {
flags |= IWX_TX_FLAGS_CMD_RATE;
tx->rate_n_flags = IwxLowestRate();
}
tx->flags = flags;
uint32_t offload = ((hdrLen / 2) & IWX_TX_CMD_OFFLD_MH_MASK)
<< IWX_TX_CMD_OFFLD_MH_SIZE_POS;
if (hdrLen % 4) offload |= IWX_TX_CMD_OFFLD_PAD;
tx->offload_assist = offload;
uint8_t* body = buf + sizeof(IwxCmdHeader) + sizeof(IwxTxCmdGen3);
memcpy(body, hdr, hdrLen);
if (payloadLen) memcpy(body + padded, payload, payloadLen);
auto* desc = &((IwxTfhTfd*)ring.Desc.Virt)[idx];
memset(desc, 0, sizeof(*desc));
desc->tbs[0].tb_len = (uint16_t)IWX_FIRST_TB_SIZE;
desc->tbs[0].addr = phys;
desc->tbs[1].tb_len = (uint16_t)(head - IWX_FIRST_TB_SIZE);
desc->tbs[1].addr = phys + IWX_FIRST_TB_SIZE;
uint16_t numTbs = 2;
if (payloadLen) {
desc->tbs[2].tb_len = (uint16_t)payloadLen;
desc->tbs[2].addr = phys + head;
numTbs = 3;
}
desc->num_tbs = numTbs;
// Byte-count table: AX210 wants the frame length in bytes plus the
// number of extra 64-byte chunks the firmware must fetch for the TFD.
uint32_t filled = (uint32_t)(sizeof(uint16_t) + numTbs * sizeof(IwxTfhTb));
uint32_t chunks = ((filled + 63) / 64) - 1;
auto* bc = (IwxGen3BcTblEntry*)ring.BcTbl.Virt;
bc[idx].tfd_offset = (uint16_t)((hdrLen + payloadLen) | (chunks << 14));
asm volatile("" ::: "memory");
ring.Queued++;
ring.Cur = (ring.Cur + 1) % IWX_TX_RING_COUNT;
ring.CurHw = (ring.CurHw + 1) % IWX_TFD_QUEUE_SIZE_MAX_GEN3;
IwxWrite32(IWX_HBUS_TARG_WRPTR, ((uint32_t)ring.Qid << 16) | ring.CurHw);
g_iwx.TxLock.Release();
return true;
}
void IwxTxComplete(int qid, int idx, uint32_t status) {
(void)idx;
IwxTxRing* ring = nullptr;
if (g_iwx.MgmtQ.Active && qid == g_iwx.MgmtQ.Qid) ring = &g_iwx.MgmtQ;
if (!ring) return;
g_iwx.TxLock.Acquire();
if (ring->Queued > 0) ring->Queued--;
g_iwx.TxLock.Release();
if (status == IWX_TX_STATUS_SUCCESS || status == IWX_TX_STATUS_DIRECT_DONE)
g_iwx.TxPackets++;
else
g_iwx.TxFailures++;
}
// =========================================================================
// Hardware key installation
// =========================================================================
// Keys go in and out through SEC_KEY_CMD: the MLD firmware does not
// implement the legacy ADD_STA_KEY, like the rest of the legacy station
// API. Values mirror the Linux trace: PTK as {sta_mask 1, key_id 0,
// flags CIPHER}, GTK as {sta_mask 1, key_id N, flags CIPHER|MCAST}.
static uint32_t SecKeyFlags(uint8_t cipher, uint32_t keyLen, bool pairwise) {
uint32_t flags;
switch (cipher) {
case RSN_CIPHER_CCMP:
case RSN_CIPHER_CCMP_256:
flags = IWX_SEC_KEY_FLAG_CIPHER_CCMP;
break;
case RSN_CIPHER_GCMP:
case RSN_CIPHER_GCMP_256:
flags = IWX_SEC_KEY_FLAG_CIPHER_GCMP;
break;
default:
return 0;
}
if (keyLen == 32) flags |= IWX_SEC_KEY_FLAG_KEY_SIZE;
if (!pairwise) flags |= IWX_SEC_KEY_FLAG_MCAST_KEY;
return flags;
}
bool IwxSetKey(const uint8_t* key, uint32_t keyLen, uint8_t keyIdx,
bool pairwise, uint8_t cipher, const uint8_t* rsc) {
if (!key || (keyLen != 16 && keyLen != 32)) return false;
uint32_t flags = SecKeyFlags(cipher, keyLen, pairwise);
if (!flags) {
KernelLogStream(WARNING, "WiFi")
<< "Cannot install a key for cipher " << (uint64_t)cipher;
return false;
}
IwxSecKeyCmd cmd = {};
cmd.action = IWX_FW_CTXT_ACTION_ADD;
cmd.sta_mask = 1u << IWX_STATION_ID;
cmd.key_id = keyIdx;
cmd.key_flags = flags;
memcpy(cmd.key, key, keyLen);
// The EAPOL RSC carries the AP's packet number for the group key,
// lowest byte first; it becomes the initial receive counter.
if (rsc) {
uint64_t pn = 0;
for (int i = 5; i >= 0; i--) pn = (pn << 8) | rsc[i];
cmd.rx_seq = pn;
}
return IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SEC_KEY_CMD),
&cmd, sizeof(cmd));
}
bool IwxRemoveKey(uint8_t keyIdx, bool pairwise, uint8_t cipher,
uint32_t keyLen) {
uint32_t flags = SecKeyFlags(cipher, keyLen, pairwise);
if (!flags) return false;
IwxSecKeyCmd cmd = {};
cmd.action = IWX_FW_CTXT_ACTION_REMOVE;
cmd.sta_mask = 1u << IWX_STATION_ID;
cmd.key_id = keyIdx;
cmd.key_flags = flags;
return IwxSendCmdPdu(IWX_WIDE_ID(IWX_DATA_PATH_GROUP, IWX_SEC_KEY_CMD),
&cmd, sizeof(cmd));
}
// =========================================================================
// RX / notification processing
// =========================================================================
static bool IwxRxPacketValid(const IwxRxPacket* pkt) {
int qid = pkt->hdr.qid & ~0x80;
int idx = pkt->hdr.idx;
uint32_t code = ((uint32_t)pkt->hdr.flags << 8) | pkt->hdr.code;
return !(qid == 0 && idx == 0 && code == 0)
&& pkt->len_n_flags != IWX_FH_RSCSR_FRAME_INVALID;
}
// Capture a command response for the in-flight synchronous command.
static void IwxCaptureCmdResp(const IwxRxPacket* pkt) {
uint32_t len = sizeof(pkt->len_n_flags) + IwxRxPacketLen(pkt);
if (len > sizeof(g_iwx.CmdRespBuf)) len = sizeof(g_iwx.CmdRespBuf);
memcpy(g_iwx.CmdRespBuf, pkt, len);
g_iwx.CmdRespLen = len;
}
static void IwxHandleAlive(const IwxRxPacket* pkt) {
uint32_t payload = IwxRxPacketPayloadLen(pkt);
g_iwx.AliveOk = false;
if (payload >= sizeof(IwxAliveRespV6)) {
auto* r = (const IwxAliveRespV6*)pkt->data;
g_iwx.SkuId = r->sku_id;
g_iwx.UmacErrorTable = r->umac_data.dbg_ptrs.error_info_addr;
g_iwx.LmacErrorTable = r->lmac_data[0].dbg_ptrs.error_event_table_ptr;
if (r->status == IWX_ALIVE_STATUS_OK) g_iwx.AliveOk = true;
} else if (payload >= sizeof(IwxAliveRespV5)) {
auto* r = (const IwxAliveRespV5*)pkt->data;
g_iwx.SkuId = r->sku_id;
g_iwx.UmacErrorTable = r->umac_data.dbg_ptrs.error_info_addr;
g_iwx.LmacErrorTable = r->lmac_data[0].dbg_ptrs.error_event_table_ptr;
if (r->status == IWX_ALIVE_STATUS_OK) g_iwx.AliveOk = true;
} else {
KernelLogStream(WARNING, "WiFi") << "Unexpected ALIVE payload size "
<< (uint64_t)payload;
}
g_iwx.AliveIntr = true;
}
// Read dwords out of device-internal SRAM (needs the MAC awake).
static bool IwxReadMem(uint32_t addr, uint32_t* out, int dwords) {
if (!IwxNicLock()) return false;
IwxWrite32(IWX_HBUS_TARG_MEM_RADDR, addr);
asm volatile("" ::: "memory");
for (int i = 0; i < dwords; i++)
out[i] = IwxRead32(IWX_HBUS_TARG_MEM_RDAT);
IwxNicUnlock();
return true;
}
// Dump the firmware's own error log after an assert. error_id identifies
// the assert and cmd_header names the last command the UMAC accepted --
// between them they pin down which host command the firmware rejected.
void IwxDumpFwError() {
KernelLogStream(ERROR, "WiFi-FW") << "Firmware assert; last command sent: 0x"
<< base::hex << (uint64_t)g_iwx.LastCmdId << base::dec
<< " (" << (uint64_t)g_iwx.LastCmdLen << " byte payload)";
// Dump the payload: a struct that does not match the firmware's
// expected layout is otherwise impossible to spot from the log.
{
uint32_t n = g_iwx.LastCmdLen;
if (n > sizeof(g_iwx.LastCmdPayload)) n = sizeof(g_iwx.LastCmdPayload);
for (uint32_t off = 0; off < n; off += 32) {
auto line = KernelLogStream(INFO, "WiFi-FW");
line << " +" << (uint64_t)off << ": " << base::hex;
for (uint32_t i = off; i < n && i < off + 32; i++) {
if (g_iwx.LastCmdPayload[i] < 0x10) line << "0";
line << (uint64_t)g_iwx.LastCmdPayload[i];
}
line << base::dec;
}
}
uint32_t base_ = g_iwx.UmacErrorTable;
if (base_ < 0x400000) {
KernelLogStream(WARNING, "WiFi-FW") << " no valid UMAC error log pointer (0x"
<< base::hex << (uint64_t)base_ << base::dec << ")";
return;
}
IwxUmacErrorEventTable t = {};
if (!IwxReadMem(base_, (uint32_t*)&t, sizeof(t) / sizeof(uint32_t))) {
KernelLogStream(WARNING, "WiFi-FW") << " could not read the error log";
return;
}
KernelLogStream(ERROR, "WiFi-FW") << " UMAC error 0x" << base::hex
<< (uint64_t)t.error_id << " last host cmd 0x" << (uint64_t)t.cmd_header
<< base::dec << " (valid=" << (uint64_t)t.valid << ")";
KernelLogStream(INFO, "WiFi-FW") << " data 0x" << base::hex
<< (uint64_t)t.data1 << " 0x" << (uint64_t)t.data2
<< " 0x" << (uint64_t)t.data3
<< " pc 0x" << (uint64_t)t.frame_pointer
<< " sp 0x" << (uint64_t)t.stack_pointer << base::dec;
KernelLogStream(INFO, "WiFi-FW") << " umac ver " << (uint64_t)t.umac_major
<< "." << (uint64_t)t.umac_minor << " blink 0x" << base::hex
<< (uint64_t)t.blink1 << "/0x" << (uint64_t)t.blink2
<< " ilink 0x" << (uint64_t)t.ilink1 << "/0x" << (uint64_t)t.ilink2
<< base::dec;
}
// Process one firmware packet out of an RX buffer.
static void IwxRxPkt(uint8_t* buf) {
const size_t minsz = sizeof(uint32_t) + sizeof(IwxCmdHeader);
auto* pkt = (IwxRxPacket*)buf;
if (!IwxRxPacketValid(pkt)) return;
uint32_t len = sizeof(pkt->len_n_flags) + IwxRxPacketLen(pkt);
if (len < minsz || len > IWX_RBUF_SIZE) return;
int qid = pkt->hdr.qid;
uint32_t code = ((uint32_t)pkt->hdr.flags << 8) | pkt->hdr.code;
// Legacy commands were re-tagged into LONG_GROUP on the way out; undo
// that so the dispatcher sees the original opcode.
if (IwxCmdGroupId(code) == IWX_LONG_GROUP)
code = IwxCmdOpcode(code);
switch (code) {
case IWX_ALIVE:
IwxHandleAlive(pkt);
break;
case IWX_INIT_COMPLETE_NOTIF:
g_iwx.InitComplete |= 0x1;
break;
case IWX_WIDE_ID(IWX_REGULATORY_AND_NVM_GROUP, IWX_PNVM_INIT_COMPLETE):
g_iwx.InitComplete |= 0x2;
break;
case IWX_TX_CMD: {
// TX completion for a frame we queued on a data/mgmt queue.
uint32_t status = 0;
if (IwxRxPacketPayloadLen(pkt) >= sizeof(IwxTxResp)) {
auto* r = (const IwxTxResp*)pkt->data;
status = r->status & IWX_TX_STATUS_MSK;
}
IwxTxComplete(qid & ~0x80, pkt->hdr.idx, status);
break;
}
case IWX_REPLY_ERROR: {
if (IwxRxPacketPayloadLen(pkt) >= 8) {
uint32_t errType = *(const uint32_t*)pkt->data;
uint8_t cmdId = pkt->data[4];
KernelLogStream(ERROR, "WiFi") << "Firmware error 0x"
<< base::hex << (uint64_t)errType << ", cmd 0x"
<< (uint64_t)cmdId << base::dec;
}
g_iwx.FwErrors++;
break;
}
default:
// Everything else (scan results, NVM/MCC responses, per-command
// acks, statistics) is handled by the MVM layer.
IwxHandleNotification(pkt, buf, IWX_RBUF_SIZE);
break;
}
// Capture the payload for a pending WANT_RESP command before signalling
// completion. Bit 7 of qid marks firmware-originated notifications,
// which never complete a command, and only the command queue carries
// command acks at all -- a data-queue packet with a colliding index
// must not release the waiter.
if (!(qid & 0x80) && (qid & ~0x80) == IWX_DQA_CMD_QUEUE) {
if (g_iwx.CmdWantResp && pkt->hdr.idx == g_iwx.CmdIdx
&& !(pkt->hdr.flags & IWX_CMD_FAILED_MSK))
IwxCaptureCmdResp(pkt);
if (pkt->hdr.idx == g_iwx.CmdIdx) {
if (g_iwx.CmdQ.Queued > 0) g_iwx.CmdQ.Queued--;
g_iwx.CmdDone = true;
}
}
}
// Drain the used (completion) ring: for each completed RBD, process the
// firmware packet it points at and hand the buffer back.
static void IwxNotifIntr() {
if (!g_iwx.RxQ.Stat.Virt) return;
uint16_t hw = *(volatile uint16_t*)g_iwx.RxQ.Stat.Virt & 0xfff;
hw &= (IWX_RX_MQ_RING_COUNT - 1);
auto* used = (IwxRxCompletionDesc*)g_iwx.RxQ.UsedDescs.Virt;
uint32_t processed = 0;
while (g_iwx.RxQ.Cur != hw && processed < IWX_RX_MQ_RING_COUNT) {
uint16_t rbid = used[g_iwx.RxQ.Cur].rbid & (IWX_RX_MQ_RING_COUNT - 1);
uint8_t* buf = g_iwx.RxQ.Buf[rbid];
if (buf) {
IwxRxPkt(buf);
g_iwx.RxPackets++;
}
g_iwx.RxQ.Cur = (g_iwx.RxQ.Cur + 1) % IWX_RX_MQ_RING_COUNT;
processed++;
}
if (processed == 0) return;
// Tell the firmware how far we have consumed. The write index must be
// 8-aligned; the hardware misbehaves otherwise (documented quirk).
uint16_t widx = (hw == 0) ? (uint16_t)(IWX_RX_MQ_RING_COUNT - 1)
: (uint16_t)(hw - 1);
IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, widx & ~7u);
}
void IwxProcessEvents() {
if (!g_iwx.Mmio) return;
// Never nest. With interrupts enabled during command waits this is a
// genuine multi-core race, so it has to be an atomic test-and-set.
if (g_iwx.InProcessEvents.test_and_set(std::memory_order_acquire)) return;
if (g_msix) {
uint32_t fh = IwxRead32(IWX_CSR_MSIX_FH_INT_CAUSES_AD);
uint32_t hwc = IwxRead32(IWX_CSR_MSIX_HW_INT_CAUSES_AD);
IwxWrite32(IWX_CSR_MSIX_FH_INT_CAUSES_AD, fh);
IwxWrite32(IWX_CSR_MSIX_HW_INT_CAUSES_AD, hwc);
fh &= g_fhMask;
hwc &= g_hwMask;
if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_ALIVE) {
// The firmware has configured the RFH; publish RX buffers.
IwxUpdateRxDescs();
IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, 8);
}
if (fh & (IWX_MSIX_FH_INT_CAUSES_Q0 | IWX_MSIX_FH_INT_CAUSES_Q1))
IwxNotifIntr();
if ((fh & IWX_MSIX_FH_INT_CAUSES_FH_ERR)
|| (hwc & IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR)
|| (hwc & IWX_MSIX_HW_INT_CAUSES_REG_SW_ERR_V2)) {
KernelLogStream(ERROR, "WiFi") << "Fatal firmware error";
g_iwx.State = IwxFwState::Error;
g_iwx.FwErrors++;
IwxDumpFwError();
}
if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_HW_ERR) {
KernelLogStream(ERROR, "WiFi") << "Hardware error";
g_iwx.State = IwxFwState::Error;
g_iwx.FwErrors++;
}
if (hwc & IWX_MSIX_HW_INT_CAUSES_REG_RF_KILL) {
if (IwxCheckRfKill()) {
KernelLogStream(WARNING, "WiFi") << "Radio disabled by RF-kill";
g_iwx.State = IwxFwState::RfKill;
}
}
IwxWrite32(IWX_CSR_MSIX_AUTOMASK_ST_AD, 1u << 0);
} else {
uint32_t r1 = IwxRead32(IWX_CSR_INT);
if (r1 != 0 && r1 != 0xffffffff && (r1 & 0xfffffff0) != 0xa5a5a5a0) {
uint32_t r2 = IwxRead32(IWX_CSR_FH_INT_STATUS);
IwxWrite32(IWX_CSR_INT, r1);
IwxWrite32(IWX_CSR_FH_INT_STATUS, r2);
if (r1 & (1u << 0)) { // ALIVE
IwxUpdateRxDescs();
IwxWrite32(IWX_RFH_Q0_FRBDCB_WIDX_TRG, 8);
}
if (r1 & (1u << 25)) { // SW_ERR
KernelLogStream(ERROR, "WiFi") << "Fatal firmware error";
g_iwx.State = IwxFwState::Error;
g_iwx.FwErrors++;
IwxDumpFwError();
}
if (r1 & ((1u << 31) | (1u << 3) | (1u << 28))) // FH_RX/SW_RX/periodic
IwxNotifIntr();
}
}
// The status word is updated by DMA and the interrupt may already have
// been consumed by another poller, so always sweep the ring.
IwxNotifIntr();
g_iwx.WorkPending = false;
g_iwx.InProcessEvents.clear(std::memory_order_release);
}
// =========================================================================
// MSI-X / MSI setup
// =========================================================================
static bool SetupMsix(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSIX);
if (cap == 0) return false;
Pci::DisableInterruptDelivery(bus, dev, func);
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
uint32_t tableOff = Pci::LegacyRead32(bus, dev, func, cap + 4);
uint8_t tableBir = (uint8_t)(tableOff & 0x7);
tableOff &= ~0x7u;
if (tableBir != 0) {
KernelLogStream(INFO, "WiFi")
<< "MSI-X table lives in BAR " << (uint64_t)tableBir
<< " (only BAR0 supported); falling back to MSI";
return false;
}
// Program entry 0: all causes are mapped to vector 0 by IwxConfMsixHw.
volatile uint32_t* entry = (volatile uint32_t*)(g_iwx.Mmio + tableOff);
entry[0] = MSI_ADDR_BASE; // message address (BSP, physical, fixed)
entry[1] = 0; // upper address
entry[2] = MSI_VECTOR; // message data
entry[3] = 0; // vector control: unmasked
msgCtrl |= (1 << 15); // MSI-X enable
msgCtrl &= ~(1 << 14); // function not masked
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "WiFi") << "MSI-X enabled: vector "
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")";
return true;
}
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) return false;
Pci::DisableInterruptDelivery(bus, dev, func);
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64 = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~0x70; // single message
msgCtrl |= (1 << 0); // MSI enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "WiFi") << "MSI enabled: vector "
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")";
return true;
}
// =========================================================================
// Probe
// =========================================================================
bool IwxProbe(const Pci::PciDevice& dev) {
if (g_iwx.State != IwxFwState::Absent) return false;
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
if (mmioPhys == 0) {
KernelLogStream(WARNING, "WiFi") << "Device has no memory BAR0";
return false;
}
// The CSR/MSI-X window is 16 KiB on these parts; map a bit more so the
// MSI-X table (which lives inside BAR0) is always covered.
constexpr uint64_t MmioSize = 0x8000;
for (uint64_t off = 0; off < MmioSize; off += 0x1000)
Memory::VMM::g_paging->MapMMIO(mmioPhys + off, Memory::HHDM(mmioPhys + off));
g_iwx.Mmio = (volatile uint8_t*)Memory::HHDM(mmioPhys);
g_iwx.Bus = dev.Bus;
g_iwx.Dev = dev.Device;
g_iwx.Func = dev.Function;
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
// LTR_CONFIG may only be sent when the PCIe link actually advertises
// Latency Tolerance Reporting (Device Control 2, bit 10); firmware
// rejects it otherwise.
constexpr uint8_t PCI_CAP_PCIE = 0x10;
constexpr uint8_t PCIE_DCSR2 = 0x28;
uint8_t pcieCap = Pci::FindCapability(dev.Bus, dev.Device, dev.Function,
PCI_CAP_PCIE);
if (pcieCap) {
uint32_t dcsr2 = Pci::LegacyRead32(dev.Bus, dev.Device, dev.Function,
(uint8_t)(pcieCap + PCIE_DCSR2));
g_iwx.LtrEnabled = (dcsr2 & (1u << 10)) != 0;
}
// Disable the PCI retry timeout (0x41) so Tx retries cannot interfere
// with deep CPU C-states.
uint32_t reg40 = Pci::LegacyRead32(dev.Bus, dev.Device, dev.Function, 0x40);
Pci::LegacyWrite32(dev.Bus, dev.Device, dev.Function, 0x40, reg40 & ~0xff00u);
IwxWrite32(IWX_CSR_INT_MASK, 0);
IwxWrite32(IWX_CSR_INT, ~0u);
IwxWrite32(IWX_CSR_FH_INT_STATUS, ~0u);
g_iwx.HwRev = IwxRead32(IWX_CSR_HW_REV);
g_iwx.HwRfId = IwxRead32(IWX_CSR_HW_RF_ID);
uint32_t macType = IWX_CSR_HW_REV_TYPE(g_iwx.HwRev);
uint32_t rfType = IWX_CSR_HW_RFID_TYPE(g_iwx.HwRfId);
KernelLogStream(OK, "WiFi") << "Intel Wi-Fi device " << base::hex
<< (uint64_t)dev.DeviceId << " at PCI " << (uint64_t)dev.Bus << ":"
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function
<< " hw_rev=" << (uint64_t)g_iwx.HwRev
<< " mac_type=" << (uint64_t)macType
<< " rf_type=" << (uint64_t)rfType << base::dec;
if (rfType != IWX_CFG_RF_TYPE_GF) {
KernelLogStream(WARNING, "WiFi")
<< "Unsupported RF type (only GF / AX211 firmware is bundled)";
g_iwx.Mmio = nullptr;
return false;
}
g_msix = SetupMsix(dev.Bus, dev.Device, dev.Function);
if (!g_msix && !SetupMsi(dev.Bus, dev.Device, dev.Function)) {
KernelLogStream(WARNING, "WiFi")
<< "No MSI-X/MSI available; the device cannot be driven by polling alone";
g_iwx.Mmio = nullptr;
return false;
}
// Boot-time DMA that persists for the device's lifetime.
if (!IwxDmaAlloc(g_iwx.CtxtInfo, sizeof(IwxContextInfoGen3))
|| !IwxDmaAlloc(g_iwx.PrphScratch, sizeof(IwxPrphScratch))
|| !IwxDmaAlloc(g_iwx.PrphInfo, 4096)
|| !IwxAllocRxRing()
|| !IwxAllocTxRing(g_iwx.CmdQ, IWX_DQA_CMD_QUEUE)
|| !IwxAllocTxRing(g_iwx.MgmtQ, IWX_DQA_MGMT_QUEUE,
IWX_TX_STAGE_SLOTS)) {
KernelLogStream(ERROR, "WiFi") << "Could not allocate device DMA memory";
IwxFreeRxRing();
IwxFreeTxRing(g_iwx.CmdQ);
IwxFreeTxRing(g_iwx.MgmtQ);
IwxDmaFree(g_iwx.CtxtInfo);
IwxDmaFree(g_iwx.PrphScratch);
IwxDmaFree(g_iwx.PrphInfo);
g_iwx.Mmio = nullptr;
return false;
}
g_iwx.State = IwxFwState::Detected;
KernelLogStream(INFO, "WiFi")
<< "Transport ready; deferring firmware load until the ramdisk is mounted";
return true;
}
}
Reference in New Issue View Git Blame Copy Permalink