daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
daa17a325fead415ef6635c4a54a8ec47586d45e
MontaukOS/kernel/src/Drivers/USB/Xhci.cpp
T

1942 lines
82 KiB
C++
Raw Blame History

/*
* Xhci.cpp
* xHCI (USB 3.x) Host Controller driver
* Copyright (c) 2025 Daniel Hammer
*/
#include "Xhci.hpp"
#include <atomic>
#include "UsbDevice.hpp"
#include "HidKeyboard.hpp"
#include "HidMouse.hpp"
#include "MassStorage.hpp"
#include "Radio/RtlSdr.hpp"
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/SmpBoot.hpp>
#include <Timekeeping/ApicTimer.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
static void BusyWaitMs(uint64_t ms) {
uint64_t flags;
asm volatile("pushfq; pop %0" : "=r"(flags));
if (flags & (1 << 9)) {
// Interrupts enabled — use timer-based delay
uint64_t start = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - start < ms) {
asm volatile("pause" ::: "memory");
}
} else {
// Interrupts disabled (e.g. timer tick context) — use I/O port delay
// Each outb to port 0x80 takes ~1µs on x86
for (uint64_t i = 0; i < ms * 1000; i++) {
asm volatile("outb %%al, $0x80" ::: "memory");
}
}
}
namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
// Static state
// -------------------------------------------------------------------------
static bool g_initialized = false;
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static volatile bool g_hotplugPending[MAX_PORTS] = {};
static volatile bool g_deferredWorkPending = false;
static std::atomic<bool> g_hotplugProcessing{false};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
static uint8_t g_capLength = 0;
static volatile uint8_t* g_opBase = nullptr;
static volatile uint8_t* g_rtBase = nullptr;
static volatile uint8_t* g_dbBase = nullptr;
// Controller parameters
static uint32_t g_maxSlots = 0;
static uint32_t g_maxPorts = 0;
// DCBAA (Device Context Base Address Array) -- not static: accessed by UsbDevice.cpp
uint64_t* g_dcbaa = nullptr;
static uint64_t g_dcbaaPhys = 0;
// Command ring
static TRB* g_cmdRing = nullptr;
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// Event ring
static TRB* g_evtRing = nullptr;
static uint64_t g_evtRingPhys = 0;
static uint32_t g_evtRingDequeue = 0;
static bool g_evtRingCCS = true;
// Event Ring Segment Table
static ERSTEntry* g_erst = nullptr;
static uint64_t g_erstPhys = 0;
// Command completion tracking
static volatile bool g_cmdCompleted = false;
static volatile uint32_t g_cmdCompletionCode = 0;
volatile uint32_t g_cmdCompletionSlotId = 0; // not static: accessed by UsbDevice.cpp
// Transfer completion tracking (for EP0 control transfers during init)
static volatile bool g_xferCompleted = false;
static volatile uint32_t g_xferCompletionCode = 0;
// Synchronous bulk transfer tracking. Only one storage-style blocking bulk
// transfer is active at a time.
static volatile bool g_syncBulkActive = false;
static volatile bool g_syncBulkCompleted = false;
static volatile uint8_t g_syncBulkSlotId = 0;
static volatile uint8_t g_syncBulkEpDci = 0;
static volatile uint32_t g_syncBulkLength = 0;
static volatile uint32_t g_syncBulkResidual = 0;
static volatile uint32_t g_syncBulkCompletionCode = 0;
// Per-device info
static UsbDeviceInfo g_devices[MAX_SLOTS + 1] = {};
// True while PollEvents() is draining the event ring. Submitters that are
// reached from inside an event callback (e.g. an HCI reply sent from a
// Bluetooth event handler) see this and must NOT wait for completion --
// PollEvents is non-reentrant, so the wait would never observe the
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from the MSI handler (BSP) AND from
// syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the
// app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set
// on a volatile is a cross-core TOCTOU: two cores both pass the check,
// both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync,
// and completions get double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
// CPU index of the core currently draining the event ring in PollEvents
// (-1 = none). ControlTransfer fire-and-forgets ONLY for true same-core
// nesting (a callback invoked from THIS core's PollEvents); a different
// core merely polling must not make a process-context transfer skip its
// wait -- that returned CC_SUCCESS before the device filled the buffer
// (observed as garbled RTL-SDR register reads while the BT firmware
// download was polling on another core).
static std::atomic<int> g_pollOwnerCpu{-1};
// Serialises non-nested (waiting) control transfers so only one EP0
// transfer is ever in flight; the completion signal (g_xferCompleted) is a
// single global, so two concurrent waiters would otherwise consume each
// other's completion. A Mutex (interrupts stay enabled) is correct here:
// ControlTransfer is process-context only and may hold this across its
// multi-millisecond poll/wait.
static kcp::Mutex g_controlXferLock;
static int CurrentCpuIndex() {
auto* cpu = Smp::GetCurrentCpuData();
return cpu ? cpu->cpuIndex : -1;
}
// Interrupt transfer data buffers (per slot)
static uint8_t* g_interruptDataBuf[MAX_SLOTS + 1] = {};
static uint64_t g_interruptDataBufPhys[MAX_SLOTS + 1] = {};
// Bulk IN transfer data buffers (per slot)
static uint8_t* g_bulkInDataBuf[MAX_SLOTS + 1] = {};
static uint64_t g_bulkInDataBufPhys[MAX_SLOTS + 1] = {};
// Multi-buffer bulk-IN streaming pool (per slot). When PoolCount>0 the slot
// keeps that many bulk-IN transfers outstanding at all times: as each one
// completes the event handler hands its buffer to the callback and instantly
// re-arms the SAME buffer at the ring tail, so the endpoint is never without
// a place to DMA. This closes the gap that single-outstanding bulk IN leaves
// between completion and re-arm, during which the device FIFO overflows
// (the RTL-SDR ~88% sample-drop at 2.048 Msps). PoolCount==0 => the legacy
// single-buffer path above (used by Bluetooth ACL), unchanged.
static constexpr uint32_t BULK_IN_POOL_MAX = 16;
static uint8_t* g_bulkInPool[MAX_SLOTS + 1][BULK_IN_POOL_MAX] = {};
static uint64_t g_bulkInPoolPhys[MAX_SLOTS + 1][BULK_IN_POOL_MAX] = {};
static uint32_t g_bulkInPoolCount[MAX_SLOTS + 1] = {}; // outstanding URBs (0=off)
static uint32_t g_bulkInPoolHead[MAX_SLOTS + 1] = {}; // next buffer to complete
static uint32_t g_bulkInPoolXferLen[MAX_SLOTS + 1] = {}; // bytes per transfer
// Transfer callbacks for non-HID class drivers (per slot)
static TransferCallback g_transferCallbacks[MAX_SLOTS + 1] = {};
// Scratchpad buffer array
static uint64_t* g_scratchpadBufs = nullptr;
// -------------------------------------------------------------------------
// Register access helpers
// -------------------------------------------------------------------------
static void WriteOp(uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(g_opBase + reg) = value;
}
static uint32_t ReadOp(uint32_t reg) {
return *(volatile uint32_t*)(g_opBase + reg);
}
static void WriteRt(uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(g_rtBase + reg) = value;
}
static uint32_t ReadRt(uint32_t reg) {
return *(volatile uint32_t*)(g_rtBase + reg);
}
static uint32_t ReadCap(uint32_t reg) {
return *(volatile uint32_t*)(g_mmioBase + reg);
}
static void WriteDoorbell(uint32_t index, uint32_t value) {
*(volatile uint32_t*)(g_dbBase + index * 4) = value;
}
// -------------------------------------------------------------------------
// DMA buffer allocation
// -------------------------------------------------------------------------
static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
void* virt = Memory::g_pfa->AllocateZeroed();
outPhysAddr = Memory::SubHHDM(virt);
return (uint8_t*)virt;
}
// -------------------------------------------------------------------------
// Transfer ring advance helpers (handle Link TRB wrap)
// -------------------------------------------------------------------------
// Advance EP0 ring enqueue pointer, activating Link TRB when reached
static void AdvanceEP0Ring(UsbDeviceInfo& dev) {
dev.EP0RingEnqueue++;
if (dev.EP0RingEnqueue >= XFER_RING_SIZE - 1) {
// Reached Link TRB — set its cycle bit and wrap
TRB& link = dev.EP0Ring[XFER_RING_SIZE - 1];
if (dev.EP0RingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.EP0RingCCS = !dev.EP0RingCCS;
dev.EP0RingEnqueue = 0;
}
}
// Advance interrupt ring enqueue pointer, activating Link TRB when reached
static void AdvanceInterruptRing(UsbDeviceInfo& dev) {
dev.InterruptRingEnqueue++;
if (dev.InterruptRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.InterruptRing[XFER_RING_SIZE - 1];
if (dev.InterruptRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.InterruptRingCCS = !dev.InterruptRingCCS;
dev.InterruptRingEnqueue = 0;
}
}
// Advance bulk IN ring enqueue pointer, activating Link TRB when reached
static void AdvanceBulkInRing(UsbDeviceInfo& dev) {
dev.BulkInRingEnqueue++;
if (dev.BulkInRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.BulkInRing[XFER_RING_SIZE - 1];
if (dev.BulkInRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.BulkInRingCCS = !dev.BulkInRingCCS;
dev.BulkInRingEnqueue = 0;
}
}
// Advance bulk OUT ring enqueue pointer, activating Link TRB when reached
static void AdvanceBulkOutRing(UsbDeviceInfo& dev) {
dev.BulkOutRingEnqueue++;
if (dev.BulkOutRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.BulkOutRing[XFER_RING_SIZE - 1];
if (dev.BulkOutRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.BulkOutRingCCS = !dev.BulkOutRingCCS;
dev.BulkOutRingEnqueue = 0;
}
}
// -------------------------------------------------------------------------
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
//
// Per xHCI spec section 7.1: walk the extended capability list starting at
// xECP (HCCPARAMS1[31:16]) to find Cap ID 0x01 (USB Legacy Support). Set
// the OS-Owned semaphore, wait for BIOS to clear its semaphore, and then
// disable all USB-Legacy SMI sources.
//
// Without this, BIOS SMI handlers that emulate PS/2 from USB devices stay
// active and steal i8042 events, so the built-in keyboard/trackpad input
// does not reach the OS until USB activity wakes the SMI handler.
// -------------------------------------------------------------------------
static constexpr uint8_t XHCI_EXTCAP_LEGACY = 0x01;
static constexpr uint32_t USBLEGSUP_BIOS_OWNED = (1u << 16);
static constexpr uint32_t USBLEGSUP_OS_OWNED = (1u << 24);
static constexpr uint32_t USBLEGCTLSTS_SMI_ENABLES =
(1u << 0) // USB SMI Enable
| (1u << 4) // SMI on Host System Error Enable
| (1u << 13) // SMI on OS Ownership Enable
| (1u << 14) // SMI on PCI Command Enable
| (1u << 15); // SMI on BAR Enable
static constexpr uint32_t USBLEGCTLSTS_W1C_STATUS =
(1u << 20) // SMI on OS Ownership Change
| (1u << 21) // SMI on PCI Command
| (1u << 22); // SMI on BAR
static void PerformBiosHandoff() {
uint32_t hccparams1 = ReadCap(CAP_HCCPARAMS1);
uint32_t xecp = (hccparams1 >> 16) & 0xFFFF;
if (xecp == 0) {
KernelLogStream(INFO, "xHCI") << "No extended capabilities (xECP=0)";
return;
}
volatile uint32_t* extCap = (volatile uint32_t*)(g_mmioBase + xecp * 4);
for (int i = 0; i < 64; i++) {
uint32_t capReg = *extCap;
uint8_t capId = capReg & 0xFF;
uint8_t nextOffset = (capReg >> 8) & 0xFF;
if (capId == XHCI_EXTCAP_LEGACY) {
KernelLogStream(INFO, "xHCI") << "USB Legacy Support found, requesting BIOS handoff";
// Assert OS-Owned. Preserve the BIOS-Owned bit so BIOS sees the
// request and can finish cleanly before clearing its own bit.
*extCap = capReg | USBLEGSUP_OS_OWNED;
uint64_t deadline = Timekeeping::GetMilliseconds() + 1000;
bool released = false;
while (Timekeeping::GetMilliseconds() < deadline) {
if (!(*extCap & USBLEGSUP_BIOS_OWNED)) {
released = true;
break;
}
BusyWaitMs(10);
}
if (!released) {
KernelLogStream(WARNING, "xHCI")
<< "BIOS did not release controller in 1s, forcing handoff";
uint32_t v = *extCap;
v &= ~USBLEGSUP_BIOS_OWNED;
v |= USBLEGSUP_OS_OWNED;
*extCap = v;
} else {
KernelLogStream(OK, "xHCI") << "BIOS released xHCI ownership";
}
// Disable all USB-Legacy SMI sources and clear any pending
// W1C status bits. Preserves RsvdP bits via read-modify-write.
volatile uint32_t* ctlsts = extCap + 1;
uint32_t v = *ctlsts;
v &= ~USBLEGCTLSTS_SMI_ENABLES;
v |= USBLEGCTLSTS_W1C_STATUS;
*ctlsts = v;
return;
}
if (nextOffset == 0) return;
extCap += nextOffset;
}
}
// -------------------------------------------------------------------------
// MSI setup (same pattern as E1000E)
// -------------------------------------------------------------------------
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
// Write Message Address (cap+4): BSP APIC ID 0, physical destination, fixed delivery
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
// Write Message Data (vector number, edge-triggered, fixed delivery)
if (is64bit) {
// 64-bit: Upper Address at cap+8, Data at cap+12
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
// 32-bit: Data at cap+8
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
// Enable MSI: set bit 0 (MSI Enable), clear bits 6:4 (single message)
msgCtrl &= ~(0x70); // Clear Multiple Message Enable (bits 6:4)
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
// Disable legacy INTx in PCI command register
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
// Register the interrupt handler for MSI vector
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "xHCI") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR
<< " (IRQ slot " << (uint64_t)MSI_IRQ << ")" << (is64bit ? " [64-bit]" : " [32-bit]");
return true;
}
// True when the caller is running inside PollEvents() (e.g. an HCI reply
// sent from a Bluetooth event handler). Such callers must fire-and-forget,
// not wait, since a nested PollEvents is a no-op.
bool InPollContext() {
return g_pollActive.load(std::memory_order_relaxed);
}
// -------------------------------------------------------------------------
// PollEvents - process event ring
// -------------------------------------------------------------------------
void PollEvents() {
// PollEvents runs both from the synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and from the xHCI MSI handler. On a
// single core the IRQ can preempt a poll loop mid-drain; if both advance
// g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the
// cycle-bit check can start matching stale entries forever -> the boot
// freezes (observed wedging the Bluetooth firmware download at ~635 KB,
// where the dying device floods the event ring). Guard against re-entry:
// the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and
// the active poll loop drains these events itself. g_pollActive is also
// read by InPollContext() so command submitters (ControlTransfer) can
// tell they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return; // another context is draining; it will reap our events
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound the work per call so a flooding/wedged device can never spin
// here forever; the outer wall-clock timeouts then fire instead of
// freezing. 4x the ring size is far above any legitimate burst.
constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < MAX_EVENTS_PER_CALL) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
bool evtCycle = (evt.Control & TRB_CYCLE_BIT) != 0;
if (evtCycle != g_evtRingCCS) {
break; // No more events
}
uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
lastType = trbType;
switch (trbType) {
case TRB_COMMAND_COMPLETION: {
uint32_t completionCode = (evt.Status >> 24) & 0xFF;
uint32_t slotId = (evt.Control >> 24) & 0xFF;
g_cmdCompletionCode = completionCode;
g_cmdCompletionSlotId = slotId;
g_cmdCompleted = true;
break;
}
case TRB_PORT_STATUS_CHANGE: {
portEvts++;
uint32_t portId = (evt.Parameter0 >> 24) & 0xFF;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE);
// Clear change bits (write-1-to-clear)
WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1] = true;
g_deferredWorkPending = true;
}
break;
}
case TRB_TRANSFER_EVENT: {
uint32_t completionCode = (evt.Status >> 24) & 0xFF;
uint32_t slotId = (evt.Control >> 24) & 0xFF;
uint32_t epDci = (evt.Control >> 16) & 0x1F;
uint32_t residual = evt.Status & 0x00FFFFFF;
xferEvts++;
lastSlot = slotId; lastEp = epDci; lastCC = completionCode;
if (epDci == 1) {
// EP0 (DCI 1) - control transfer completion
g_xferCompletionCode = completionCode;
g_xferCompleted = true;
} else if (slotId > 0 && slotId <= MAX_SLOTS && g_devices[slotId].Active) {
UsbDeviceInfo& dev = g_devices[slotId];
if (g_syncBulkActive &&
slotId == g_syncBulkSlotId &&
epDci == g_syncBulkEpDci) {
g_syncBulkResidual = residual;
g_syncBulkCompletionCode = completionCode;
g_syncBulkCompleted = true;
break;
}
if (completionCode == CC_SUCCESS || completionCode == CC_SHORT_PACKET) {
// Compute actual transfer length from residual
// Check if this is a bulk IN endpoint completion
uint8_t bulkInDci = dev.BulkInEpNum ? (dev.BulkInEpNum * 2 + 1) : 0;
uint8_t bulkOutDci = dev.BulkOutEpNum ? (dev.BulkOutEpNum * 2) : 0;
uint8_t intDci = dev.InterruptEpNum ? (dev.InterruptEpNum * 2 + 1) : 0;
if (epDci == bulkInDci && g_transferCallbacks[slotId]
&& g_bulkInPoolCount[slotId] > 0) {
// Multi-buffer streaming: hand back the rotating
// pool buffer and immediately re-arm it. The
// callback consumes the data synchronously (copies
// it out) before returning, so re-queuing the same
// buffer is safe -- it will not be DMA'd into again
// until the other PoolCount-1 transfers ahead of it
// complete (~PoolCount ms of slack).
uint32_t i = g_bulkInPoolHead[slotId];
uint32_t reqLen = g_bulkInPoolXferLen[slotId];
uint32_t len = (residual < reqLen) ? (reqLen - residual) : 0;
g_transferCallbacks[slotId](slotId, epDci,
g_bulkInPool[slotId][i], len, completionCode);
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], reqLen);
g_bulkInPoolHead[slotId] =
(i + 1) % g_bulkInPoolCount[slotId];
} else if (epDci == bulkInDci && g_transferCallbacks[slotId]) {
// Bulk IN — dispatch via registered callback.
// len = actually-transferred bytes (requested -
// residual). A 0-byte / ZLP completion has
// residual == requested, so len MUST be 0: the old
// code left len at the full max-packet and handed
// the callback a slice of STALE DMA buffer, which
// the BT driver then counted as a bogus ACL packet
// (the constant rx flood) and pushed into its RX
// ring, crowding out the real Config Response.
uint32_t reqLen = dev.BulkInMaxPacket;
uint16_t len = (residual < reqLen)
? (uint16_t)(reqLen - residual) : 0;
g_transferCallbacks[slotId](slotId, epDci,
g_bulkInDataBuf[slotId], len, completionCode);
} else if (epDci == bulkOutDci && g_transferCallbacks[slotId]) {
// Bulk OUT completion — notify callback
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
} else if (epDci == intDci) {
// Interrupt IN — HID or callback dispatch
uint16_t len = dev.InterruptMaxPacket;
if (residual < len) len = dev.InterruptMaxPacket - (uint16_t)residual;
if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
} else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
}
// Re-queue for next HID report
QueueInterruptTransfer(slotId);
} else if (g_transferCallbacks[slotId]) {
// Callback is responsible for re-queuing
g_transferCallbacks[slotId](slotId, epDci,
g_interruptDataBuf[slotId], len, completionCode);
}
} else if (g_transferCallbacks[slotId]) {
// Unknown endpoint — try callback
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
}
} else {
KernelLogStream(WARNING, "xHCI") << "Transfer error on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci
<< " cc=" << (uint64_t)completionCode;
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
}
}
}
break;
}
default:
break;
}
// Advance dequeue pointer
g_evtRingDequeue++;
if (g_evtRingDequeue >= EVT_RING_SIZE) {
g_evtRingDequeue = 0;
g_evtRingCCS = !g_evtRingCCS;
}
processed++;
}
// Update ERDP to tell the controller we have processed events
// Bit 3 (EHB - Event Handler Busy) must be set to clear it
uint64_t erdp = g_evtRingPhys + (uint64_t)g_evtRingDequeue * sizeof(TRB);
erdp |= (1 << 3); // Set EHB to clear it
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded -- surface the dominant
// event source (rate-limited) so a wedge is diagnosable, not silent.
if (processed >= MAX_EVENTS_PER_CALL) {
static uint32_t stormLogs = 0;
if (stormLogs < 8) {
stormLogs++;
KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed
<< "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts
<< " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot
<< " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")";
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Clear IMAN.IP and ensure IE stays enabled
WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE);
PollEvents();
}
// -------------------------------------------------------------------------
// SendCommand - send a command TRB on the command ring
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
slot.Parameter1 = trb.Parameter1;
slot.Status = trb.Status;
// Set the type and cycle bit in control
uint32_t control = trb.Control & ~TRB_CYCLE_BIT;
if (g_cmdRingCCS) {
control |= TRB_CYCLE_BIT;
}
slot.Control = control;
// Advance enqueue pointer
g_cmdRingEnqueue++;
if (g_cmdRingEnqueue >= CMD_RING_SIZE - 1) {
// We've reached the Link TRB - toggle its cycle bit and wrap
TRB& link = g_cmdRing[CMD_RING_SIZE - 1];
// Update the link TRB cycle bit to match current CCS
if (g_cmdRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
g_cmdRingCCS = !g_cmdRingCCS;
g_cmdRingEnqueue = 0;
}
// Clear completion flag and ring the host controller doorbell
g_cmdCompleted = false;
WriteDoorbell(0, 0);
// Poll until command completes. Wall-clock bounded so a storm/wedge
// can't stretch this into a multi-second freeze; normal commands
// complete in well under a millisecond.
uint64_t cmdStart = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
return g_cmdCompletionCode;
}
// Small delay
for (int j = 0; j < 100; j++) {
asm volatile("" ::: "memory");
}
}
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
// -------------------------------------------------------------------------
// ControlTransfer - perform a control transfer on EP0
// -------------------------------------------------------------------------
uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, bool dirIn) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
return 0xFF;
}
// True nesting = called from a callback inside THIS core's PollEvents;
// such a call must fire-and-forget (a nested PollEvents is a no-op, so
// it could never observe its own completion). A different core merely
// polling is NOT nesting: serialise behind g_controlXferLock and wait
// normally so the global completion signal is unambiguous.
bool trueNested = g_pollActive.load(std::memory_order_relaxed) &&
g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
if (!trueNested) g_controlXferLock.Acquire();
UsbDeviceInfo& dev = g_devices[slotId];
// --- Setup Stage TRB ---
TRB& setup = dev.EP0Ring[dev.EP0RingEnqueue];
setup.Parameter0 = (uint32_t)bmRequestType | ((uint32_t)bRequest << 8) | ((uint32_t)wValue << 16);
setup.Parameter1 = (uint32_t)wIndex | ((uint32_t)wLength << 16);
setup.Status = 8; // Setup packet is always 8 bytes
uint32_t setupControl = (TRB_SETUP_STAGE << TRB_TYPE_SHIFT) | TRB_IDT;
if (wLength > 0) {
setupControl |= dirIn ? TRB_TRT_IN : TRB_TRT_OUT;
} else {
setupControl |= TRB_TRT_NODATA;
}
if (dev.EP0RingCCS) {
setupControl |= TRB_CYCLE_BIT;
}
setup.Control = setupControl;
// Advance EP0 enqueue (handles Link TRB wrap)
AdvanceEP0Ring(dev);
// --- Data Stage TRB (if wLength > 0) ---
if (wLength > 0 && data != nullptr) {
uint64_t dataPhys = Memory::SubHHDM(data);
TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
dataTrb.Status = wLength;
uint32_t dataControl = (TRB_DATA_STAGE << TRB_TYPE_SHIFT);
if (dirIn) {
dataControl |= TRB_DIR_IN;
}
if (dev.EP0RingCCS) {
dataControl |= TRB_CYCLE_BIT;
}
dataTrb.Control = dataControl;
AdvanceEP0Ring(dev);
}
// --- Status Stage TRB ---
TRB& status = dev.EP0Ring[dev.EP0RingEnqueue];
status.Parameter0 = 0;
status.Parameter1 = 0;
status.Status = 0;
// Status stage direction is opposite of data stage
uint32_t statusControl = (TRB_STATUS_STAGE << TRB_TYPE_SHIFT) | TRB_IOC;
if (wLength > 0 && !dirIn) {
statusControl |= TRB_DIR_IN;
} else if (wLength == 0) {
statusControl |= TRB_DIR_IN;
}
if (dev.EP0RingCCS) {
statusControl |= TRB_CYCLE_BIT;
}
status.Control = statusControl;
AdvanceEP0Ring(dev);
// Ring doorbell for this slot, target EP0 (DCI 1)
g_xferCompleted = false;
WriteDoorbell(slotId, 1);
// If we are nested inside THIS core's PollEvents (e.g. an HCI reply sent
// from a Bluetooth event handler), we cannot wait for completion here:
// the reentrancy guard makes a nested PollEvents a no-op, so the
// completion would never be observed and the timeout+recovery path would
// corrupt the EP0 ring. The transfer is submitted (doorbell rung); let
// the active PollEvents reap its completion. Fire-and-forget (no lock
// was taken for the nested case).
if (trueNested) {
return CC_SUCCESS;
}
// Poll until transfer completes. Wall-clock bounded (not iteration
// bounded) so a wedged device fails in ~2s and reports its cc, instead
// of the per-iteration cost ballooning under an event storm into a
// multi-second freeze with no output.
uint64_t xferStart = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
PollEvents();
if (g_xferCompleted) {
g_controlXferLock.Release();
return g_xferCompletionCode;
}
for (int j = 0; j < 100; j++) {
asm volatile("" ::: "memory");
}
}
// Callers already log their own error with request context; the xHCI
// layer just returns 0xFF and recovers EP0 so the next transfer works.
// Recover EP0 ring: Stop Endpoint, then Set TR Dequeue Pointer
// so that subsequent control transfers on this slot still work.
TRB stopTrb = {};
stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0) in bits 20:16
SendCommand(stopTrb);
// Reset the enqueue pointer to the start and set the dequeue pointer
// to match so the ring is back in sync.
uint64_t newDeq = dev.EP0RingPhys
+ (uint64_t)dev.EP0RingEnqueue * sizeof(TRB);
if (dev.EP0RingCCS) {
newDeq |= 1; // DCS bit
}
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0)
SendCommand(deqTrb);
g_controlXferLock.Release();
return 0xFF;
}
// -------------------------------------------------------------------------
// QueueInterruptTransfer
// -------------------------------------------------------------------------
void QueueInterruptTransfer(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
return;
}
UsbDeviceInfo& dev = g_devices[slotId];
// Allocate interrupt data buffer if not yet allocated
if (g_interruptDataBuf[slotId] == nullptr) {
g_interruptDataBuf[slotId] = AllocateDmaBuffer(g_interruptDataBufPhys[slotId]);
}
// Build a Normal TRB on the interrupt ring
TRB& trb = dev.InterruptRing[dev.InterruptRingEnqueue];
trb.Parameter0 = (uint32_t)(g_interruptDataBufPhys[slotId] & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(g_interruptDataBufPhys[slotId] >> 32);
trb.Status = dev.InterruptMaxPacket;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.InterruptRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
// Advance enqueue (handles Link TRB wrap)
AdvanceInterruptRing(dev);
// Ring doorbell: target = (InterruptEpNum * 2 + 1) for IN endpoint DCI
uint8_t target = dev.InterruptEpNum * 2 + 1;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// ResetInterruptEndpoint - clear a halted interrupt IN endpoint and re-arm
// -------------------------------------------------------------------------
// A USB transaction error (cc=4) halts the endpoint; the host must issue
// Reset Endpoint + Set TR Dequeue before it will accept transfers again.
// Used by the Bluetooth firmware-download path, where a glitch on the event
// pipe near the end of a large upload otherwise kills event reception for
// good (no Command Complete / bootup events).
void ResetInterruptEndpoint(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
uint8_t dci = dev.InterruptEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(resetTrb);
uint64_t newDeq = dev.InterruptRingPhys
+ (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
if (dev.InterruptRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
// Re-arm reception.
QueueInterruptTransfer(slotId);
}
// -------------------------------------------------------------------------
// ResetBulkInEndpoint - clear a halted bulk IN endpoint (does not re-arm)
// -------------------------------------------------------------------------
void ResetBulkInEndpoint(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (dev.BulkInEpNum == 0 || !dev.BulkInRing) return;
uint8_t dci = dev.BulkInEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(resetTrb);
uint64_t newDeq = dev.BulkInRingPhys
+ (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB);
if (dev.BulkInRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
}
// -------------------------------------------------------------------------
// QueueBulkInTransfer
// -------------------------------------------------------------------------
void QueueBulkInTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return;
// If caller provides nullptr, use the per-slot bulk IN DMA buffer
if (data == nullptr) {
if (g_bulkInDataBuf[slotId] == nullptr) {
g_bulkInDataBuf[slotId] = AllocateDmaBuffer(g_bulkInDataBufPhys[slotId]);
}
data = g_bulkInDataBuf[slotId];
dataPhys = g_bulkInDataBufPhys[slotId];
}
TRB& trb = dev.BulkInRing[dev.BulkInRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.BulkInRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
AdvanceBulkInRing(dev);
// Ring doorbell: DCI for bulk IN = EpNum * 2 + 1
uint8_t target = dev.BulkInEpNum * 2 + 1;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// Multi-buffer bulk-IN streaming (see g_bulkInPool* declarations)
// -------------------------------------------------------------------------
// (Re)queue one transfer per pool buffer; resets the completion rotation.
// Safe to call from process context: the prime loop (a few TRB writes)
// finishes in microseconds, far below the >=tens-of-us minimum USB transfer
// time, so no completion can race the loop -- the same timing the legacy
// single-buffer start relies on.
void PrimeBulkInStream(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
uint32_t n = g_bulkInPoolCount[slotId];
if (n == 0) return;
g_bulkInPoolHead[slotId] = 0;
uint32_t len = g_bulkInPoolXferLen[slotId];
for (uint32_t i = 0; i < n; i++)
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], len);
}
void StartBulkInStream(uint8_t slotId, uint32_t xferLen, uint32_t numBuffers) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return;
// Already armed: priming again would put a second transfer on the ring
// for every pool buffer and desync the completion rotation (callbacks
// would be handed the wrong buffer). Callers must Stop first.
if (g_bulkInPoolCount[slotId] != 0) return;
if (numBuffers < 1) numBuffers = 1;
if (numBuffers > BULK_IN_POOL_MAX) numBuffers = BULK_IN_POOL_MAX;
// One page per buffer; a single TRB must not cross a 64 KiB boundary, so
// clamp the transfer to a page (AllocateDmaBuffer hands out one page).
if (xferLen == 0 || xferLen > 4096) xferLen = 4096;
// Lazily allocate the pool; buffers persist across stop/start (same
// keep-forever pattern as g_bulkInDataBuf).
for (uint32_t i = 0; i < numBuffers; i++) {
if (g_bulkInPool[slotId][i] == nullptr) {
g_bulkInPool[slotId][i] =
AllocateDmaBuffer(g_bulkInPoolPhys[slotId][i]);
if (g_bulkInPool[slotId][i] == nullptr) {
// Out of DMA pages: fall back to however many we got.
if (i == 0) return;
numBuffers = i;
break;
}
}
}
g_bulkInPoolXferLen[slotId] = xferLen;
// Prime the transfers BEFORE publishing PoolCount so the event handler
// only switches to the rotating path once every buffer is queued.
g_bulkInPoolHead[slotId] = 0;
for (uint32_t i = 0; i < numBuffers; i++)
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], xferLen);
g_bulkInPoolCount[slotId] = numBuffers;
}
void StopBulkInStream(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS) return;
// Disarm the rotation; any late completion now takes the (no-op for SDR)
// legacy path and is not re-armed. Buffers are retained for reuse.
bool wasArmed = g_bulkInPoolCount[slotId] != 0;
g_bulkInPoolCount[slotId] = 0;
if (!wasArmed) return;
// Flush the up-to-PoolCount TRBs still pending on the ring: Stop
// Endpoint, then Set TR Dequeue to the enqueue pointer. Without this a
// later Start would stack a fresh pool behind the stale TRBs and
// overflow the 32-entry transfer ring (16 stale + 16 new > 31 usable),
// wrapping the enqueue pointer onto still-pending TRBs.
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.Active || dev.BulkInEpNum == 0 || !dev.BulkInRing) return;
uint8_t dci = dev.BulkInEpNum * 2 + 1;
TRB stopTrb = {};
stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(stopTrb);
uint64_t newDeq = dev.BulkInRingPhys
+ (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB);
if (dev.BulkInRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
}
// -------------------------------------------------------------------------
// QueueBulkOutTransfer
// -------------------------------------------------------------------------
void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkOutRing || dev.BulkOutEpNum == 0) return;
TRB& trb = dev.BulkOutRing[dev.BulkOutRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
if (dev.BulkOutRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
AdvanceBulkOutRing(dev);
// Ring doorbell: DCI for bulk OUT = EpNum * 2
uint8_t target = dev.BulkOutEpNum * 2;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// BulkTransfer - blocking bulk IN/OUT transfer
// -------------------------------------------------------------------------
uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength) {
if (actualLength) *actualLength = 0;
if (length == 0) return CC_SUCCESS;
if (length > 0x10000) return 0xFF;
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return 0xFF;
if (data == nullptr) return 0xFF;
if (g_syncBulkActive) return 0xFF;
UsbDeviceInfo& dev = g_devices[slotId];
if (dataPhys == 0) dataPhys = Memory::SubHHDM(data);
uint8_t target = 0;
if (dirIn) {
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return 0xFF;
TRB& trb = dev.BulkInRing[dev.BulkInRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
trb.Control = control;
AdvanceBulkInRing(dev);
target = dev.BulkInEpNum * 2 + 1;
} else {
if (!dev.BulkOutRing || dev.BulkOutEpNum == 0) return 0xFF;
TRB& trb = dev.BulkOutRing[dev.BulkOutRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
trb.Control = control;
AdvanceBulkOutRing(dev);
target = dev.BulkOutEpNum * 2;
}
g_syncBulkSlotId = slotId;
g_syncBulkEpDci = target;
g_syncBulkLength = length;
g_syncBulkResidual = length;
g_syncBulkCompletionCode = 0;
g_syncBulkCompleted = false;
g_syncBulkActive = true;
WriteDoorbell(slotId, target);
uint64_t start = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - start < 5000) {
PollEvents();
if (g_syncBulkCompleted) {
uint32_t cc = g_syncBulkCompletionCode;
uint32_t residual = g_syncBulkResidual;
g_syncBulkActive = false;
if (actualLength) {
*actualLength = (residual < g_syncBulkLength)
? (g_syncBulkLength - residual)
: 0;
}
return cc;
}
for (int j = 0; j < 100; j++) {
asm volatile("" ::: "memory");
}
}
g_syncBulkActive = false;
KernelLogStream(WARNING, "xHCI") << "Bulk transfer timeout on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)target;
return 0xFF;
}
// -------------------------------------------------------------------------
// RegisterTransferCallback
// -------------------------------------------------------------------------
void RegisterTransferCallback(uint8_t slotId, TransferCallback cb) {
if (slotId > 0 && slotId <= MAX_SLOTS) {
g_transferCallbacks[slotId] = cb;
}
}
// -------------------------------------------------------------------------
// RingDoorbell
// -------------------------------------------------------------------------
void RingDoorbell(uint8_t slotId, uint8_t target) {
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// GetDevice
// -------------------------------------------------------------------------
UsbDeviceInfo* GetDevice(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS) {
return nullptr;
}
return &g_devices[slotId];
}
// -------------------------------------------------------------------------
// IsInitialized
// -------------------------------------------------------------------------
bool IsInitialized() {
return g_initialized;
}
bool HasDeferredWork() {
return g_initialized && g_deferredWorkPending;
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
if (Radio::IsRtlSdr(dev.VendorId, dev.ProductId)) {
Radio::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
MassStorage::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
HidKeyboard::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
HidMouse::UnregisterDevice(slotId);
}
}
static void DisableSlot(uint8_t slotId) {
TRB trb = {};
trb.Control = (TRB_DISABLE_SLOT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24);
uint32_t cc = SendCommand(trb);
if (cc != CC_SUCCESS) {
KernelLogStream(WARNING, "xHCI") << "Disable Slot failed for slot "
<< base::dec << (uint64_t)slotId << " cc=" << (uint64_t)cc;
}
}
// -------------------------------------------------------------------------
// ProcessDeferredWork - handle hot-plug outside interrupt context.
// Called from the BSP idle path so class-driver probing can wait on timers.
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
bool expected = false;
if (!g_hotplugProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
g_deferredWorkPending = false;
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port]) continue;
g_hotplugPending[port] = false;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (portsc & PORTSC_CCS) {
// Device connected — check if already assigned to a slot
bool alreadyActive = false;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
alreadyActive = true;
break;
}
}
if (alreadyActive) continue;
if (portsc & PORTSC_PED) {
// Already enabled — enumerate after recovery delay
uint32_t speed = (portsc >> 10) & 0xF;
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
} else {
// Need port reset first
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (ps & PORTSC_PRC) {
resetDone = true;
break;
}
for (int j = 0; j < 100; j++) {
asm volatile("" ::: "memory");
}
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Hot-plug: port "
<< base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
// Clear change bits
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
// Post-reset recovery delay (USB spec requires >= 10ms)
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
}
} else {
// Device disconnected — deactivate its slot
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
UnregisterClassDriver(s, g_devices[s]);
g_devices[s].Active = false;
g_transferCallbacks[s] = nullptr;
DisableSlot(s);
g_dcbaa[s] = 0;
g_devices[s] = {};
KernelLogStream(INFO, "xHCI") << "Hot-unplug: slot "
<< base::dec << (uint64_t)s << " (port "
<< (uint64_t)(port + 1) << ") deactivated";
break;
}
}
}
}
g_hotplugProcessing.store(false, std::memory_order_release);
}
// -------------------------------------------------------------------------
// Probe (called by PCI driver matching framework)
// -------------------------------------------------------------------------
bool Probe(const Pci::PciDevice& dev) {
if (g_initialized) return false;
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)dev.Bus << ":"
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
// Read BAR0 and map MMIO region
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
constexpr uint64_t MmioSize = 0x10000;
for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
// Parse capability registers
g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
<< ", CapLength: " << (uint64_t)g_capLength;
uint32_t hcsParams1 = ReadCap(CAP_HCSPARAMS1);
g_maxSlots = hcsParams1 & 0xFF;
g_maxPorts = (hcsParams1 >> 24) & 0xFF;
uint32_t hcsParams2 = ReadCap(CAP_HCSPARAMS2);
uint32_t scratchpadBufsHi = (hcsParams2 >> 21) & 0x1F;
uint32_t scratchpadBufsLo = (hcsParams2 >> 27) & 0x1F;
uint32_t maxScratchpadBufs = (scratchpadBufsHi << 5) | scratchpadBufsLo;
uint32_t dbOff = ReadCap(CAP_DBOFF) & ~0x3u;
uint32_t rtsOff = ReadCap(CAP_RTSOFF) & ~0x1Fu;
g_opBase = g_mmioBase + g_capLength;
g_rtBase = g_mmioBase + rtsOff;
g_dbBase = g_mmioBase + dbOff;
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
<< ", MaxPorts: " << (uint64_t)g_maxPorts
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
if (g_maxSlots > MAX_SLOTS) g_maxSlots = MAX_SLOTS;
if (g_maxPorts > MAX_PORTS) g_maxPorts = MAX_PORTS;
// Tell BIOS to release the controller and disable USB-Legacy SMIs.
// Must run before we halt/reset so BIOS gets a chance to shut down
// its own use of the controller cleanly.
PerformBiosHandoff();
// Halt controller
uint32_t usbcmd = ReadOp(OP_USBCMD);
usbcmd &= ~USBCMD_RS;
WriteOp(OP_USBCMD, usbcmd);
for (uint32_t i = 0; i < 100000; i++) {
if (ReadOp(OP_USBSTS) & USBSTS_HCH) break;
for (int j = 0; j < 10; j++) asm volatile("" ::: "memory");
}
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
KernelLogStream(WARNING, "xHCI") << "Controller failed to halt";
}
KernelLogStream(OK, "xHCI") << "Controller halted";
// Reset controller
WriteOp(OP_USBCMD, USBCMD_HCRST);
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBCMD) & USBCMD_HCRST)) break;
for (int j = 0; j < 10; j++) asm volatile("" ::: "memory");
}
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_CNR)) break;
for (int j = 0; j < 10; j++) asm volatile("" ::: "memory");
}
if (ReadOp(OP_USBSTS) & USBSTS_CNR) {
KernelLogStream(WARNING, "xHCI") << "Controller not ready after reset";
}
KernelLogStream(OK, "xHCI") << "Controller reset complete";
// Program CONFIG
WriteOp(OP_CONFIG, g_maxSlots);
// Allocate DCBAA
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
WriteOp(OP_DCBAAP + 4, (uint32_t)(g_dcbaaPhys >> 32));
KernelLogStream(OK, "xHCI") << "DCBAA at phys " << base::hex << g_dcbaaPhys;
// Scratchpad buffers
if (maxScratchpadBufs > 0) {
uint64_t spArrayPhys;
g_scratchpadBufs = (uint64_t*)AllocateDmaBuffer(spArrayPhys);
for (uint32_t i = 0; i < maxScratchpadBufs; i++) {
uint64_t bufPhys;
AllocateDmaBuffer(bufPhys);
g_scratchpadBufs[i] = bufPhys;
}
g_dcbaa[0] = spArrayPhys;
KernelLogStream(OK, "xHCI") << "Allocated " << base::dec << (uint64_t)maxScratchpadBufs << " scratchpad buffers";
}
// Command ring
g_cmdRing = (TRB*)AllocateDmaBuffer(g_cmdRingPhys);
TRB& linkTrb = g_cmdRing[CMD_RING_SIZE - 1];
linkTrb.Parameter0 = (uint32_t)(g_cmdRingPhys & 0xFFFFFFFF);
linkTrb.Parameter1 = (uint32_t)(g_cmdRingPhys >> 32);
linkTrb.Status = 0;
linkTrb.Control = (TRB_LINK << TRB_TYPE_SHIFT) | TRB_ENT;
uint64_t crcr = g_cmdRingPhys | TRB_CYCLE_BIT;
WriteOp(OP_CRCR, (uint32_t)(crcr & 0xFFFFFFFF));
WriteOp(OP_CRCR + 4, (uint32_t)(crcr >> 32));
g_cmdRingCCS = true;
g_cmdRingEnqueue = 0;
KernelLogStream(OK, "xHCI") << "Command ring at phys " << base::hex << g_cmdRingPhys;
// Event ring + ERST
g_evtRing = (TRB*)AllocateDmaBuffer(g_evtRingPhys);
g_erst = (ERSTEntry*)AllocateDmaBuffer(g_erstPhys);
g_erst[0].RingSegmentBase = g_evtRingPhys;
g_erst[0].RingSegmentSize = EVT_RING_SIZE;
g_erst[0].Reserved = 0;
WriteRt(IR0_ERSTSZ, 1);
WriteRt(IR0_ERDP, (uint32_t)(g_evtRingPhys & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(g_evtRingPhys >> 32));
WriteRt(IR0_ERSTBA, (uint32_t)(g_erstPhys & 0xFFFFFFFF));
WriteRt(IR0_ERSTBA + 4, (uint32_t)(g_erstPhys >> 32));
g_evtRingCCS = true;
g_evtRingDequeue = 0;
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
// MSI setup
if (!SetupMsi(dev.Bus, dev.Device, dev.Function)) {
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
}
// Enable interrupter 0
WriteRt(IR0_IMAN, IMAN_IE);
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
// Start controller
WriteOp(OP_USBCMD, USBCMD_RS | USBCMD_INTE | USBCMD_HSEE);
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) break;
for (int j = 0; j < 10; j++) asm volatile("" ::: "memory");
}
KernelLogStream(OK, "xHCI") << "Controller started";
g_initialized = true;
// Power on all ports
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_PP)) {
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE, PORTSC_PP);
}
}
BusyWaitMs(20);
KernelLogStream(OK, "xHCI") << "All ports powered";
// Port scanning
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_CCS)) continue;
KernelLogStream(INFO, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": device connected, PORTSC=" << base::hex << (uint64_t)portsc;
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (ps & PORTSC_PRC) { resetDone = true; break; }
for (int j = 0; j < 100; j++) asm volatile("" ::: "memory");
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
const char* speedStr = "Unknown";
switch (speed) {
case SPEED_FULL: speedStr = "Full (12 Mbps)"; break;
case SPEED_LOW: speedStr = "Low (1.5 Mbps)"; break;
case SPEED_HIGH: speedStr = "High (480 Mbps)"; break;
case SPEED_SUPER: speedStr = "Super (5 Gbps)"; break;
}
KernelLogStream(OK, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": reset complete, speed=" << speedStr;
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
}
g_bootScanComplete = true;
KernelLogStream(OK, "xHCI") << "Initialization complete";
return true;
}
// -------------------------------------------------------------------------
// Initialize (standalone fallback path)
// -------------------------------------------------------------------------
void Initialize() {
KernelLogStream(INFO, "xHCI") << "Scanning for xHCI controller...";
// -----------------------------------------------------------------
// Step 1: Find xHCI controller on PCI bus
// -----------------------------------------------------------------
auto& devices = Pci::GetDevices();
const Pci::PciDevice* foundDev = nullptr;
for (uint64_t i = 0; i < devices.size(); i++) {
if (devices[i].ClassCode == PCI_CLASS_SERIAL &&
devices[i].SubClass == PCI_SUBCLASS_USB &&
devices[i].ProgIf == PCI_PROGIF_XHCI) {
foundDev = &devices[i];
break;
}
}
if (foundDev == nullptr) {
KernelLogStream(WARNING, "xHCI") << "No xHCI controller found";
return;
}
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)foundDev->Bus << ":"
<< (uint64_t)foundDev->Device << "." << (uint64_t)foundDev->Function;
// -----------------------------------------------------------------
// Step 2: Read BAR0 and map MMIO region
// -----------------------------------------------------------------
uint64_t mmioPhys = Pci::ReadBar0(foundDev->Bus, foundDev->Device, foundDev->Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
// Map 64KB (16 pages) of MMIO space
constexpr uint64_t MmioSize = 0x10000;
for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
// -----------------------------------------------------------------
// Step 3: Enable PCI bus master and memory space
// -----------------------------------------------------------------
Pci::EnableBusMaster(foundDev->Bus, foundDev->Device, foundDev->Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
// -----------------------------------------------------------------
// Step 4: Parse capability registers
// -----------------------------------------------------------------
g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
<< ", CapLength: " << (uint64_t)g_capLength;
uint32_t hcsParams1 = ReadCap(CAP_HCSPARAMS1);
g_maxSlots = hcsParams1 & 0xFF;
g_maxPorts = (hcsParams1 >> 24) & 0xFF;
uint32_t hcsParams2 = ReadCap(CAP_HCSPARAMS2);
uint32_t scratchpadBufsHi = (hcsParams2 >> 21) & 0x1F;
uint32_t scratchpadBufsLo = (hcsParams2 >> 27) & 0x1F;
uint32_t maxScratchpadBufs = (scratchpadBufsHi << 5) | scratchpadBufsLo;
uint32_t dbOff = ReadCap(CAP_DBOFF) & ~0x3u;
uint32_t rtsOff = ReadCap(CAP_RTSOFF) & ~0x1Fu;
g_opBase = g_mmioBase + g_capLength;
g_rtBase = g_mmioBase + rtsOff;
g_dbBase = g_mmioBase + dbOff;
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
<< ", MaxPorts: " << (uint64_t)g_maxPorts
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
// Cap slots to our maximum
if (g_maxSlots > MAX_SLOTS) {
g_maxSlots = MAX_SLOTS;
}
if (g_maxPorts > MAX_PORTS) {
g_maxPorts = MAX_PORTS;
}
// -----------------------------------------------------------------
// Step 4.5: BIOS-to-OS handoff (USB Legacy Support)
// -----------------------------------------------------------------
PerformBiosHandoff();
// -----------------------------------------------------------------
// Step 5: Halt controller
// -----------------------------------------------------------------
uint32_t usbcmd = ReadOp(OP_USBCMD);
usbcmd &= ~USBCMD_RS;
WriteOp(OP_USBCMD, usbcmd);
// Wait for HCH (Halted) to be set
for (uint32_t i = 0; i < 100000; i++) {
if (ReadOp(OP_USBSTS) & USBSTS_HCH) {
break;
}
for (int j = 0; j < 10; j++) {
asm volatile("" ::: "memory");
}
}
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
KernelLogStream(WARNING, "xHCI") << "Controller failed to halt";
}
KernelLogStream(OK, "xHCI") << "Controller halted";
// -----------------------------------------------------------------
// Step 6: Reset controller
// -----------------------------------------------------------------
WriteOp(OP_USBCMD, USBCMD_HCRST);
// Wait for HCRST to clear
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBCMD) & USBCMD_HCRST)) {
break;
}
for (int j = 0; j < 10; j++) {
asm volatile("" ::: "memory");
}
}
// Wait for CNR (Controller Not Ready) to clear
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_CNR)) {
break;
}
for (int j = 0; j < 10; j++) {
asm volatile("" ::: "memory");
}
}
if (ReadOp(OP_USBSTS) & USBSTS_CNR) {
KernelLogStream(WARNING, "xHCI") << "Controller not ready after reset";
}
KernelLogStream(OK, "xHCI") << "Controller reset complete";
// -----------------------------------------------------------------
// Step 7: Program CONFIG register (MaxSlotsEn)
// -----------------------------------------------------------------
WriteOp(OP_CONFIG, g_maxSlots);
// -----------------------------------------------------------------
// Step 8: Allocate DCBAA
// -----------------------------------------------------------------
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
// Write DCBAAP (64-bit, split into two 32-bit writes)
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
WriteOp(OP_DCBAAP + 4, (uint32_t)(g_dcbaaPhys >> 32));
KernelLogStream(OK, "xHCI") << "DCBAA at phys " << base::hex << g_dcbaaPhys;
// -----------------------------------------------------------------
// Step 9: Scratchpad buffers
// -----------------------------------------------------------------
if (maxScratchpadBufs > 0) {
uint64_t spArrayPhys;
g_scratchpadBufs = (uint64_t*)AllocateDmaBuffer(spArrayPhys);
for (uint32_t i = 0; i < maxScratchpadBufs; i++) {
uint64_t bufPhys;
AllocateDmaBuffer(bufPhys);
g_scratchpadBufs[i] = bufPhys;
}
// DCBAA[0] = physical address of the scratchpad buffer array
g_dcbaa[0] = spArrayPhys;
KernelLogStream(OK, "xHCI") << "Allocated " << base::dec << (uint64_t)maxScratchpadBufs << " scratchpad buffers";
}
// -----------------------------------------------------------------
// Step 10: Command ring
// -----------------------------------------------------------------
g_cmdRing = (TRB*)AllocateDmaBuffer(g_cmdRingPhys);
// Set up Link TRB at the last position
TRB& linkTrb = g_cmdRing[CMD_RING_SIZE - 1];
linkTrb.Parameter0 = (uint32_t)(g_cmdRingPhys & 0xFFFFFFFF);
linkTrb.Parameter1 = (uint32_t)(g_cmdRingPhys >> 32);
linkTrb.Status = 0;
linkTrb.Control = (TRB_LINK << TRB_TYPE_SHIFT) | TRB_ENT;
// Toggle Cycle bit is bit 1 in the Link TRB control - use TRB_ENT which is bit 1
// Write CRCR = command ring physical address | cycle bit 1
uint64_t crcr = g_cmdRingPhys | TRB_CYCLE_BIT;
WriteOp(OP_CRCR, (uint32_t)(crcr & 0xFFFFFFFF));
WriteOp(OP_CRCR + 4, (uint32_t)(crcr >> 32));
g_cmdRingCCS = true;
g_cmdRingEnqueue = 0;
KernelLogStream(OK, "xHCI") << "Command ring at phys " << base::hex << g_cmdRingPhys;
// -----------------------------------------------------------------
// Step 11: Event ring + ERST
// -----------------------------------------------------------------
g_evtRing = (TRB*)AllocateDmaBuffer(g_evtRingPhys);
g_erst = (ERSTEntry*)AllocateDmaBuffer(g_erstPhys);
// Set up ERST entry 0
g_erst[0].RingSegmentBase = g_evtRingPhys;
g_erst[0].RingSegmentSize = EVT_RING_SIZE;
g_erst[0].Reserved = 0;
// Write interrupter 0 registers
// Order: ERSTSZ → ERDP → ERSTBA (ERSTBA triggers hardware read of ERST)
WriteRt(IR0_ERSTSZ, 1);
WriteRt(IR0_ERDP, (uint32_t)(g_evtRingPhys & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(g_evtRingPhys >> 32));
// Write ERSTBA last (triggers hardware to read the ERST)
WriteRt(IR0_ERSTBA, (uint32_t)(g_erstPhys & 0xFFFFFFFF));
WriteRt(IR0_ERSTBA + 4, (uint32_t)(g_erstPhys >> 32));
g_evtRingCCS = true;
g_evtRingDequeue = 0;
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
// -----------------------------------------------------------------
// Step 12: MSI setup
// -----------------------------------------------------------------
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function)) {
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
}
// -----------------------------------------------------------------
// Step 13: Enable interrupter 0
// -----------------------------------------------------------------
WriteRt(IR0_IMAN, IMAN_IE);
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
// -----------------------------------------------------------------
// Step 14: Start controller
// -----------------------------------------------------------------
WriteOp(OP_USBCMD, USBCMD_RS | USBCMD_INTE | USBCMD_HSEE);
// Wait for controller to start (HCH should clear)
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
break;
}
for (int j = 0; j < 10; j++) {
asm volatile("" ::: "memory");
}
}
KernelLogStream(OK, "xHCI") << "Controller started";
g_initialized = true;
// -----------------------------------------------------------------
// Step 14.5: Power on all ports
// -----------------------------------------------------------------
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_PP)) {
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE, PORTSC_PP);
}
}
// Wait for port power to stabilize (~20ms)
BusyWaitMs(20);
KernelLogStream(OK, "xHCI") << "All ports powered";
// -----------------------------------------------------------------
// Step 15: Port scanning
// -----------------------------------------------------------------
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
// Check if device is connected (CCS)
if (!(portsc & PORTSC_CCS)) {
continue;
}
KernelLogStream(INFO, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": device connected, PORTSC=" << base::hex << (uint64_t)portsc;
// Reset the port: preserve power, clear change bits, set port reset
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
// Wait for Port Reset Change (PRC) to be set
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (ps & PORTSC_PRC) {
resetDone = true;
break;
}
for (int j = 0; j < 100; j++) {
asm volatile("" ::: "memory");
}
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
// Re-read PORTSC after reset
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
// Clear change bits
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
const char* speedStr = "Unknown";
switch (speed) {
case SPEED_FULL: speedStr = "Full (12 Mbps)"; break;
case SPEED_LOW: speedStr = "Low (1.5 Mbps)"; break;
case SPEED_HIGH: speedStr = "High (480 Mbps)"; break;
case SPEED_SUPER: speedStr = "Super (5 Gbps)"; break;
}
KernelLogStream(OK, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": reset complete, speed=" << speedStr;
// Post-reset recovery delay (USB spec requires >= 10ms)
BusyWaitMs(10);
// Enumerate the device (port IDs are 1-based)
UsbDevice::EnumerateDevice(port + 1, speed);
}
g_bootScanComplete = true;
KernelLogStream(OK, "xHCI") << "Initialization complete";
}
};
Reference in New Issue View Git Blame Copy Permalink