daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kernel concurrency, interrupt context, and user VA safety

Browse Source
This commit is contained in:
daniel committed 2026-08-01 15:43:38 +02:00
1 parent f222bf1f04
commit b1c55073c7
71 files changed
+2654 -1092
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernel/src/Drivers/Audio/IntelHda.cpp
+126 -29
View File
@@ -15,6 +15,7 @@
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Libraries/Memory.hpp>
#include <atomic>
namespace Drivers::Audio::IntelHda {
@@ -26,10 +27,33 @@ namespace Drivers::Audio::IntelHda {
static bool g_initialized = false;
static kcp::Spinlock g_codecLock;
// Serializes stream lifecycle, DMA write-pointer updates, and IRQ-side
// stream inspection. The IRQ releases it before calling the mixer to keep
// the lock order consistently Mixer -> HDA stream.
// Serializes stream lifecycle and DMA write-pointer updates. Keep logging
// out of this lock: Spinlock disables local interrupts, while the terminal
// logger may wait for a lock owner that needs the local timer to run.
static kcp::Spinlock g_streamLock;
// The hard IRQ only acknowledges and masks stream completion sources.
// Mixing/refilling is substantially too expensive for interrupt context.
static std::atomic<bool> g_streamWorkPending{false};
static uint32_t g_streamInterruptMask = 0;
// Runtime diagnostics are emitted by ProcessDeferredWork() in idle
// context. Some HDA entry points run below the mixer's interrupt-disabling
// spinlock, so even logging after releasing g_streamLock is not safe.
enum DeferredDiagnostic : uint32_t {
DIAG_STREAM_OPENED = 1u << 0,
DIAG_STREAM_CLOSED = 1u << 1,
DIAG_HEADPHONES = 1u << 2,
DIAG_SPEAKERS = 1u << 3,
};
static std::atomic<uint32_t> g_diagnosticsPending{0};
static std::atomic<uint32_t> g_openSampleRate{0};
static std::atomic<uint32_t> g_openBitsPerSample{0};
static std::atomic<uint32_t> g_openChannels{0};
static std::atomic<uint32_t> g_codecTimeoutsPending{0};
static std::atomic<uint32_t> g_codecTimeoutReports{0};
static std::atomic<uint32_t> g_lastTimeoutCodec{0};
static std::atomic<uint32_t> g_lastTimeoutNid{0};
static std::atomic<uint32_t> g_lastTimeoutVerb{0};
static volatile uint8_t* g_mmioBase = nullptr;
static uint8_t g_bus, g_dev, g_func;
@@ -213,9 +237,10 @@ namespace Drivers::Audio::IntelHda {
uint32_t response = 0;
if (!ReadResponse(&response, nullptr)) {
g_codecLock.Release();
KernelLogStream(WARNING, "HDA") << "Verb timeout: codec=" << base::dec
<< (uint64_t)codec << " nid=" << (uint64_t)nid
<< " verb=" << base::hex << (uint64_t)verb;
g_lastTimeoutCodec.store(codec, std::memory_order_relaxed);
g_lastTimeoutNid.store(nid, std::memory_order_relaxed);
g_lastTimeoutVerb.store(verb, std::memory_order_relaxed);
g_codecTimeoutsPending.fetch_add(1, std::memory_order_release);
return 0;
}
g_codecLock.Release();
@@ -640,13 +665,15 @@ namespace Drivers::Audio::IntelHda {
DisablePin(g_speakerNid);
EnablePin(g_hpNid);
g_pinNid = g_hpNid;
KernelLogStream(INFO, "HDA") << "Switched to headphone output";
g_diagnosticsPending.fetch_or(DIAG_HEADPHONES,
std::memory_order_release);
} else {
// Mute HP, enable speaker
DisablePin(g_hpNid);
EnablePin(g_speakerNid);
g_pinNid = g_speakerNid;
KernelLogStream(INFO, "HDA") << "Switched to speaker output";
g_diagnosticsPending.fetch_or(DIAG_SPEAKERS,
std::memory_order_release);
}
// Re-apply volume on the now-active pin
SetOutputVolume(g_volume);
@@ -810,7 +837,7 @@ namespace Drivers::Audio::IntelHda {
// MSI setup
// =========================================================================
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
@@ -819,6 +846,8 @@ namespace Drivers::Audio::IntelHda {
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
@@ -851,21 +880,21 @@ namespace Drivers::Audio::IntelHda {
// Interrupt handler
// =========================================================================
static void HandleInterrupt(uint8_t /*irq*/) {
static void HandleInterrupt(uint8_t /*irq*/, bool /*fromUser*/) {
uint32_t intsts = Read32(REG_INTSTS);
bool bufferCompleted = false;
// Handle stream interrupts (bits 0-29 correspond to stream descriptors)
g_streamLock.Acquire();
if (g_stream.Active) {
uint8_t si = g_stream.StreamIndex;
if (intsts & (1u << si)) {
uint8_t sts = ReadSD8(si, SD_STS);
if (sts & SD_STS_BCIS) bufferCompleted = true;
WriteSD8(si, SD_STS, sts);
}
// Handle stream interrupts (bits 0-29 correspond to stream
// descriptors) without taking g_streamLock. Inspecting only the
// hardware status registers is race-safe against stream lifecycle and
// avoids spinning in an IRQ if another CPU is opening/closing audio.
uint32_t streamBits = intsts & 0x3FFFFFFFu;
for (uint8_t si = 0; si < 30; si++) {
if (!(streamBits & (1u << si))) continue;
uint8_t sts = ReadSD8(si, SD_STS);
if (sts & SD_STS_BCIS) bufferCompleted = true;
WriteSD8(si, SD_STS, sts);
}
g_streamLock.Release();
// Handle RIRB interrupt (controller interrupt enable bit 30)
// Do NOT advance g_rirbReadPtr here — ReadResponse() owns it.
@@ -876,10 +905,12 @@ namespace Drivers::Audio::IntelHda {
Write8(REG_RIRBSTS, rirbSts);
}
// Notify the mixer so it can refill the DMA ring with the next mix
// window. Done after clearing status bits so re-entry can't latch.
// A mixer pass can resample up to 4096 frames across every active
// stream and must never run inside this high-priority IRQ. Mask stream
// completion delivery and queue a process-safe bottom half instead.
if (bufferCompleted) {
Mixer::OnHdaBufferComplete();
Write32(REG_INTCTL, Read32(REG_INTCTL) & ~g_streamInterruptMask);
g_streamWorkPending.store(true, std::memory_order_release);
}
}
@@ -987,7 +1018,13 @@ namespace Drivers::Audio::IntelHda {
uint8_t irqLine = Pci::LegacyRead8(g_bus, g_dev, g_func, (uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "HDA") << "Falling back to legacy IRQ " << base::dec << (uint64_t)irqLine;
Pci::DisableInterruptDelivery(g_bus, g_dev, g_func);
uint16_t command = Pci::LegacyRead16(g_bus, g_dev, g_func,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(g_bus, g_dev, g_func,
(uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE);
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
} else {
KernelLogStream(WARNING, "HDA") << "No interrupt available, polling only";
}
@@ -997,8 +1034,9 @@ namespace Drivers::Audio::IntelHda {
uint32_t intctl = INTCTL_GIE | INTCTL_CIE;
// Enable interrupt for all output streams
for (uint8_t i = 0; i < g_numOutputStreams; i++) {
intctl |= (1u << (g_numInputStreams + i));
g_streamInterruptMask |= (1u << (g_numInputStreams + i));
}
intctl |= g_streamInterruptMask;
Write32(REG_INTCTL, intctl);
// Discover codecs
@@ -1028,6 +1066,63 @@ namespace Drivers::Audio::IntelHda {
return g_initialized;
}
bool HasDeferredWork() {
return (g_initialized &&
g_streamWorkPending.load(std::memory_order_acquire)) ||
g_diagnosticsPending.load(std::memory_order_acquire) != 0 ||
g_codecTimeoutsPending.load(std::memory_order_acquire) != 0;
}
void ProcessDeferredWork() {
if (g_initialized &&
g_streamWorkPending.exchange(false, std::memory_order_acq_rel)) {
Mixer::OnHdaBufferComplete();
// Clear-before-unmask closes the handoff race: if another completion
// arrived while masked its SD_STS/INTSTS state is still asserted and
// enabling the stream bit produces a fresh MSI/INTx interrupt.
Write32(REG_INTCTL, Read32(REG_INTCTL) | g_streamInterruptMask |
INTCTL_GIE | INTCTL_CIE);
}
uint32_t diagnostics =
g_diagnosticsPending.exchange(0, std::memory_order_acq_rel);
if (diagnostics & DIAG_STREAM_OPENED) {
KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec
<< (uint64_t)g_openSampleRate.load(std::memory_order_relaxed)
<< "Hz "
<< (uint64_t)g_openBitsPerSample.load(std::memory_order_relaxed)
<< "-bit "
<< (uint64_t)g_openChannels.load(std::memory_order_relaxed)
<< "ch";
}
if (diagnostics & DIAG_STREAM_CLOSED)
KernelLogStream(OK, "HDA") << "Stream closed";
if (diagnostics & DIAG_HEADPHONES)
KernelLogStream(INFO, "HDA") << "Switched to headphone output";
if (diagnostics & DIAG_SPEAKERS)
KernelLogStream(INFO, "HDA") << "Switched to speaker output";
uint32_t timeouts =
g_codecTimeoutsPending.exchange(0, std::memory_order_acq_rel);
if (timeouts != 0) {
uint32_t report =
g_codecTimeoutReports.fetch_add(1, std::memory_order_relaxed);
if (report < 8) {
KernelLogStream(WARNING, "HDA") << base::dec
<< (uint64_t)timeouts << " codec verb timeout(s), last: codec="
<< (uint64_t)g_lastTimeoutCodec.load(std::memory_order_relaxed)
<< " nid="
<< (uint64_t)g_lastTimeoutNid.load(std::memory_order_relaxed)
<< " verb=" << base::hex
<< (uint64_t)g_lastTimeoutVerb.load(std::memory_order_relaxed);
} else if (report == 8) {
KernelLogStream(WARNING, "HDA")
<< "Further codec verb timeout reports suppressed";
}
}
}
uint32_t GetCodecVendorId() {
return g_codecVendorId;
}
@@ -1075,11 +1170,12 @@ namespace Drivers::Audio::IntelHda {
// Start the stream
StartStream(streamIndex);
KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec
<< (uint64_t)sampleRate << "Hz " << (uint64_t)bitsPerSample << "-bit "
<< (uint64_t)channels << "ch";
g_streamLock.Release();
g_openSampleRate.store(sampleRate, std::memory_order_relaxed);
g_openBitsPerSample.store(bitsPerSample, std::memory_order_relaxed);
g_openChannels.store(channels, std::memory_order_relaxed);
g_diagnosticsPending.fetch_or(DIAG_STREAM_OPENED,
std::memory_order_release);
return 0; // Handle 0
}
@@ -1099,8 +1195,9 @@ namespace Drivers::Audio::IntelHda {
g_stream.Active = false;
KernelLogStream(OK, "HDA") << "Stream closed";
g_streamLock.Release();
g_diagnosticsPending.fetch_or(DIAG_STREAM_CLOSED,
std::memory_order_release);
}
uint32_t GetWriteSpace(int handle) {
kernel/src/Drivers/Audio/IntelHda.hpp
+2
View File
@@ -259,6 +259,8 @@ namespace Drivers::Audio::IntelHda {
bool Probe(const Pci::PciDevice& dev);
bool IsInitialized();
bool HasDeferredWork();
void ProcessDeferredWork();
// Returns the codec vendor/device ID (vendor in upper 16 bits, device in lower 16).
// Returns 0 if no codec was found.
kernel/src/Drivers/Audio/Mixer.cpp
+9 -5
View File
@@ -122,7 +122,11 @@ namespace Drivers::Audio::Mixer {
static void FreeRing(int16_t* ring) {
if (!ring) return;
Memory::g_pfa->ReallocConsecutive(ring, 0);
// The ring was allocated as one INPUT_RING_PAGES contiguous span.
// ReallocConsecutive(ptr, 0) is not a sized free operation and used
// to compute a zero-byte allocation at the end of the PFA pool, copy
// a page there, then free only the first page of this ring.
Memory::g_pfa->Free(ring, INPUT_RING_PAGES);
}
// Caller holds g_lock. HDA setup is non-blocking and serialized by the
@@ -380,10 +384,11 @@ namespace Drivers::Audio::Mixer {
if (!ring) return -1;
retry_after_switch:
uint64_t switchWake = Sched::ObserveObjectWake((void*)&g_switchingOutput);
g_lock.Acquire();
if (g_switchingOutput.load(std::memory_order_acquire)) {
g_lock.Release();
Sched::BlockOnObject((void*)&g_switchingOutput, 1000);
Sched::BlockOnObjectSince((void*)&g_switchingOutput, 1000, switchWake);
goto retry_after_switch;
}
@@ -821,9 +826,8 @@ namespace Drivers::Audio::Mixer {
}
void OnHdaBufferComplete() {
// Called from the HDA buffer-completion interrupt. Mixer state and
// HDA register access are both serialized through g_lock (which
// disables interrupts on acquire), so this is safe to call from IRQ.
// Called from IntelHda's idle-context bottom half. A pump may resample
// thousands of frames and is deliberately kept out of the hard IRQ.
g_lock.Acquire();
if (g_output == Output::Hda) Pump();
g_lock.Release();
kernel/src/Drivers/Graphics/IntelGPU.cpp
+273 -180
View File
@@ -10,14 +10,14 @@
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Io/IoPort.hpp>
#include <Graphics/Framebuffer.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Sched/Scheduler.hpp>
#include <Libraries/Memory.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
@@ -36,7 +36,6 @@ namespace Drivers::Graphics::IntelGPU {
// GTT state
static volatile void* g_gttBase = nullptr; // Virtual address of GTT entries
static uint64_t g_gttEntryCount = 0; // Number of GTT entries
static uint64_t g_scratchPagePhys = 0; // Physical address of scratch page
static uint8_t g_gpuGen = 0; // Cached generation number
// Framebuffer state
@@ -54,12 +53,15 @@ namespace Drivers::Graphics::IntelGPU {
static uint64_t g_fbGttOffsetA = 0; // GGTT offset of buffer 0 (firmware FB)
static uint32_t* g_buf1Virt = nullptr; // buffer 1 kernel VA (HHDM, WC)
static uint64_t g_buf1Phys = 0;
static uint64_t g_buf1Pages = 0; // allocation/mapping bound
static uint64_t g_buf1GttOffset = 0; // GGTT offset of buffer 1 (in bytes)
static bool g_flipSupported = false;
static int g_frontBuffer = 0;
static int g_flipOwnerPid = -1;
static volatile uint64_t g_vblankCount = 0;
static bool g_vblankIrqReady = false;
// Serializes surface programming, flip-owner teardown, and timing changes.
// Without this, a process exit or modeset on another CPU can rewrite
// DSPASURF/PIPE state in the middle of a desktop flip transaction.
static kcp::Mutex g_flipLock;
// Modesetting/display-management state. The firmware-trained port and
// link are deliberately retained; the driver owns the active transcoder,
@@ -216,6 +218,21 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// This driver intentionally uses register polling for display latches and
// does not own the GPU's many generation-specific interrupt sources. Mask
// PCI delivery before touching display state: otherwise firmware-left MSI
// enables can route an unhandled GT interrupt and create an interrupt
// storm even though Montauk never requested one.
static void DisableInterruptDelivery() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
if (Pci::DisableInterruptDelivery(bus, dev, func)) {
KernelLogStream(WARNING, "IntelGPU")
<< "Disabled pre-enabled GPU MSI/MSI-X; using bounded latch polling";
}
}
// =========================================================================
// BAR0 MMIO Mapping
// =========================================================================
@@ -357,12 +374,20 @@ namespace Drivers::Graphics::IntelGPU {
return false;
}
// Ensure pitch is at least width * 4 (BGRX8888)
// Ensure pitch is at least width * 4 (BGRX8888), and reject geometry
// whose size arithmetic could wrap before it reaches the allocator.
if (g_fbPitch == 0) {
g_fbPitch = g_fbWidth * 4;
KernelLogStream(WARNING, "IntelGPU") << "Stride not available, assuming "
<< base::dec << g_fbPitch << " bytes";
}
if (g_fbWidth > (~0ULL / 4)
|| g_fbPitch < g_fbWidth * 4
|| g_fbHeight > (~0ULL / g_fbPitch)) {
KernelLogStream(ERROR, "IntelGPU")
<< "Invalid or overflowing framebuffer geometry";
return false;
}
g_fbSize = g_fbHeight * g_fbPitch;
@@ -989,17 +1014,13 @@ namespace Drivers::Graphics::IntelGPU {
<< ", " << base::dec << g_gttEntryCount << " entries"
<< (g_gpuGen >= 8 ? " (64-bit PTEs)" : " (32-bit PTEs)");
// Allocate a scratch page (zeroed) for future use
void* scratchPageVirt = Memory::g_pfa->AllocateZeroed();
g_scratchPagePhys = Memory::SubHHDM(scratchPageVirt);
// Do NOT clear the entire GTT here. The firmware has active GTT mappings
// that the display engine is currently scanning out from. Clearing them
// would cause the display to go black (or worse) before we remap.
// Instead, we only write the entries we need in SetupFramebuffer().
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec << g_gttEntryCount
<< " entries, scratch page at " << base::hex << g_scratchPagePhys;
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec
<< g_gttEntryCount << " entries";
return true;
}
@@ -1108,18 +1129,47 @@ namespace Drivers::Graphics::IntelGPU {
// Initialization maps this window once and preserves the firmware's active
// scanout PTEs. Page flipping only proceeds after those entries validate.
static void MicroDelay(int us) {
// Simple busy-wait; us is approximate
for (volatile int i = 0; i < us * 100; i++) {
asm volatile("pause");
}
}
static uint64_t ReadGgttPte(uint64_t index) {
if (g_gpuGen >= 8) return ((volatile uint64_t*)g_ggtt)[index];
return ((volatile uint32_t*)g_ggtt)[index];
}
static int ReadLiveBufferIndex() {
if (!g_mmioBase) return -1;
uint32_t live = ReadReg(DSPASURFLIVE) & ~0xFFFu;
if (live == (uint32_t)g_fbGttOffsetA) return 0;
if (live == (uint32_t)g_buf1GttOffset) return 1;
return -1;
}
// Polling-only latch wait. This is safe during both ordinary syscalls and
// ExitProcess teardown because it never enters the scheduler. Keeping
// flips independent of GPU MSI is deliberate: MSI would route every
// firmware-enabled GPU source, not just the one source this driver owns.
// Three observed frames are the normal timeout. Laptop panel self-refresh
// can transiently stop the scanline counter, so also enforce a wall-clock
// bound: one million uncached MMIO reads can otherwise hold the desktop in
// SYS_FBFLIP long enough to look permanently frozen (cursor included).
static bool WaitForLiveBuffer(int index) {
uint32_t expected = (uint32_t)((index == 0) ? g_fbGttOffsetA
: g_buf1GttOffset);
uint32_t previous = ReadReg(PIPE_DSL) & 0x1FFFu;
uint64_t started = Timekeeping::GetMilliseconds();
int frameWraps = 0;
for (int i = 0; i < 200000; i++) {
uint32_t live = ReadReg(DSPASURFLIVE);
if (live == 0xFFFFFFFFu) return false;
if ((live & ~0xFFFu) == expected) return true;
uint32_t current = ReadReg(PIPE_DSL) & 0x1FFFu;
if (current < previous && ++frameWraps >= 3) return false;
previous = current;
if ((i & 0x3FF) == 0 &&
Timekeeping::GetMilliseconds() - started >= 100) return false;
asm volatile("pause");
}
return false;
}
static void WriteGgttPte(uint64_t index, uint64_t physAddr) {
if (g_gpuGen >= 8) {
((volatile uint64_t*)g_ggtt)[index] = MakeGttPte64(physAddr);
@@ -1168,14 +1218,37 @@ namespace Drivers::Graphics::IntelGPU {
static bool AllocateBackBuffer() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, pages);
if (pages == 0 || pages > 0x7FFFFFFFULL) {
KernelLogStream(WARNING, "IntelGPU")
<< "Invalid second scanout buffer page count " << base::dec << pages;
return false;
}
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, (int)pages);
if (!virt) {
KernelLogStream(WARNING, "IntelGPU") << "Failed to allocate " << base::dec
<< pages << " contiguous pages for the second scanout buffer";
return false;
}
g_buf1Phys = Memory::SubHHDM(virt);
uint64_t phys = Memory::SubHHDM(virt);
if ((pages - 1) > (UINT64_MAX - phys) / 0x1000ULL) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer physical range overflows";
return false;
}
uint64_t lastPhys = phys + (pages - 1) * 0x1000ULL;
if (!GgttCanAddress(phys, g_gpuGen) || !GgttCanAddress(lastPhys, g_gpuGen)) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer lies outside the GPU physical-address width ("
<< base::hex << phys << ".." << lastPhys << ")";
return false;
}
g_buf1Phys = phys;
g_buf1Virt = (uint32_t*)virt;
g_buf1Pages = pages;
// Display scanout does not snoop the CPU cache: remap the buffer
// write-combining before touching it, same as the firmware FB.
@@ -1192,23 +1265,50 @@ namespace Drivers::Graphics::IntelGPU {
static bool FindAndMapGgttRange() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
constexpr uint64_t guard = 16;
uint64_t need = pages + 2 * guard;
uint64_t avoidFirst = g_fbGttOffsetA >> 12;
uint64_t avoidLast = (g_fbGttOffsetA + g_fbSize - 1) >> 12;
// Montauk owns the GPU after firmware handoff. Reserve a deterministic
// high-aperture range for its second scanout buffer instead of trying
// to infer firmware ownership from scratch-PTE patterns. Keep guards
// around the allocation and never overlap the live firmware surface.
uint64_t base = g_ggttEntries / 2 + guard;
if (base <= avoidLast + guard) base = avoidLast + guard + 1;
base = (base + guard - 1) & ~(guard - 1);
if (base < guard || base + pages + guard > g_ggttEntries
|| !(base + pages - 1 < avoidFirst || base > avoidLast)) {
KernelLogStream(WARNING, "IntelGPU") << "No safe owned GGTT range for "
// Firmware normally fills unused GGTT entries with one scratch-page
// PTE. Verify that invariant at the tail, then claim only a complete
// scratch-backed (or invalid) run. Never overwrite arbitrary valid
// mappings merely because they happen to sit in the upper half.
if (g_ggttEntries < 32 || need > g_ggttEntries / 2) return false;
uint64_t scratchPte = ReadGgttPte(g_ggttEntries - 1);
for (uint64_t probe = 2; probe <= 32; probe++) {
if (ReadGgttPte(g_ggttEntries - probe) != scratchPte) {
KernelLogStream(WARNING, "IntelGPU")
<< "GGTT tail is owned/non-uniform; page flip unavailable";
return false;
}
}
uint64_t runStart = 0;
uint64_t run = 0;
bool found = false;
for (uint64_t idx = g_ggttEntries / 2; idx < g_ggttEntries; idx++) {
if (idx >= avoidFirst && idx <= avoidLast) {
run = 0;
continue;
}
uint64_t pte = ReadGgttPte(idx);
if ((pte & 1) && pte != scratchPte) {
run = 0;
continue;
}
if (run == 0) runStart = idx;
if (++run >= need) {
found = true;
break;
}
}
if (!found) {
KernelLogStream(WARNING, "IntelGPU") << "No verified free GGTT run for "
<< base::dec << pages << " back-buffer pages";
return false;
}
uint64_t base = runStart + guard;
uint64_t replacedFirst = ReadGgttPte(base);
uint64_t replacedLast = ReadGgttPte(base + pages - 1);
for (uint64_t i = 0; i < pages; i++) {
@@ -1235,94 +1335,38 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// =========================================================================
// Page flipping: vblank interrupt (Gen 11+)
// =========================================================================
static void HandleInterrupt(uint8_t /*irq*/) {
// Gen 11+ flow: park the master enable, ack the pipe IIR, re-enable
uint32_t master = ReadReg(GFX_MSTR_INTR);
if (!(master & GFX_MSTR_INTR_DISPLAY)) {
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
return;
}
WriteReg(GFX_MSTR_INTR, 0);
uint32_t iir = ReadReg(DE_PIPE_A_IIR);
if (iir) WriteReg(DE_PIPE_A_IIR, iir); // write-1-to-clear
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
if (iir & DE_PIPE_VBLANK) {
g_vblankCount = g_vblankCount + 1;
Sched::WakeObjectWaiters((void*)&g_vblankCount);
}
}
static bool SetupMsi() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "IntelGPU") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // one message
msgCtrl |= (1 << 0); // MSI enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "IntelGPU") << "MSI enabled: vector " << base::dec
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"
<< (is64bit ? " [64-bit]" : " [32-bit]");
return true;
}
static void EnableVblankIrq() {
WriteReg(DE_PIPE_A_IIR, 0xFFFFFFFFu); // clear stale events
WriteReg(DE_PIPE_A_IMR, ~DE_PIPE_VBLANK); // unmask vblank only
WriteReg(DE_PIPE_A_IER, DE_PIPE_VBLANK);
WriteReg(DISPLAY_INT_CTL, DISPLAY_IRQ_ENABLE);
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
(void)ReadReg(GFX_MSTR_INTR);
}
static void SetupPageFlip() {
if (!MapCorrectGgtt()) return;
if (!VerifyFirmwareScanoutPtes()) return;
if (!AllocateBackBuffer()) return;
if (!FindAndMapGgttRange()) return;
// Vblank interrupts: Gen 11+ register layout only. Older generations
// still flip, with DSPASURFLIVE polling standing in for the IRQ.
if (g_gpuGen >= 11 && SetupMsi()) {
EnableVblankIrq();
g_vblankIrqReady = true;
}
g_flipSupported = true;
g_frontBuffer = 0;
KernelLogStream(OK, "IntelGPU") << "Page flip ready: 2 buffers, vblank IRQ "
<< (g_vblankIrqReady ? "on" : "off (SURFLIVE polling)");
int live = ReadLiveBufferIndex();
g_frontBuffer = (live >= 0) ? live : 0;
KernelLogStream(OK, "IntelGPU")
<< "Page flip ready: 2 buffers, bounded SURFLIVE polling";
}
static bool FirmwareFramebufferMatchesGpu() {
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
bool match = g_fbWidth == fwWidth
&& g_fbHeight == fwHeight
&& g_fbPitch == fwPitch;
if (match) return true;
// Userspace obtains its drawing dimensions from the firmware
// framebuffer API. A differently-sized GPU allocation would make the
// second mapping overrun (or alias only part of) the backing buffer.
KernelLogStream(ERROR, "IntelGPU")
<< "GPU dimensions differ from firmware; unsafe framebuffer setup skipped";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
return false;
}
// =========================================================================
@@ -1388,6 +1432,7 @@ namespace Drivers::Graphics::IntelGPU {
}
g_gpuGen = g_gpuInfo.gen;
DisableInterruptDelivery();
if (!MapMmio()) {
KernelLogStream(ERROR, "IntelGPU") << "Failed to map MMIO region";
@@ -1400,6 +1445,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return false;
}
if (!FirmwareFramebufferMatchesGpu()) return false;
InitializeDisplayManagement();
if (!InitializeGtt()) {
@@ -1417,18 +1463,6 @@ namespace Drivers::Graphics::IntelGPU {
SetupPageFlip();
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1446,6 +1480,7 @@ namespace Drivers::Graphics::IntelGPU {
if (!DetectGpu()) {
return;
}
DisableInterruptDelivery();
// Step 2: Map BAR0 MMIO region
if (!MapMmio()) {
@@ -1461,6 +1496,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return;
}
if (!FirmwareFramebufferMatchesGpu()) return;
InitializeDisplayManagement();
// Step 5: Initialize GTT
@@ -1480,21 +1516,6 @@ namespace Drivers::Graphics::IntelGPU {
g_initialized = true;
// Diagnostic: compare GPU-detected values with firmware/Limine values
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight
<< " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1641,7 +1662,6 @@ namespace Drivers::Graphics::IntelGPU {
if (g_edidValid) out->capabilities |= montauk::abi::DISPLAY_CAP_EDID;
if (g_pwmSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_BRIGHTNESS;
if (g_flipSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_PAGE_FLIP;
if (g_vblankIrqReady) out->capabilities |= montauk::abi::DISPLAY_CAP_VBLANK_IRQ;
for (int i = 0; i < g_modeCount; i++) {
if (i != g_currentMode
&& (g_modes[i].flags & montauk::abi::DISPLAY_MODE_DRIVER_VALID)) {
@@ -1685,6 +1705,8 @@ namespace Drivers::Graphics::IntelGPU {
return -2;
if (modeIndex == g_currentMode) return modeIndex;
g_flipLock.Acquire();
// Link rate and framebuffer geometry remain fixed. Only timings with
// the same active area and pixel clock are safe without retraining the
// firmware-established HDMI/DP/eDP link.
@@ -1693,10 +1715,13 @@ namespace Drivers::Graphics::IntelGPU {
if (requested.hdisplay != current.hdisplay
|| requested.vdisplay != current.vdisplay
|| requested.pixelClock == 0
|| requested.pixelClock != current.pixelClock)
|| requested.pixelClock != current.pixelClock) {
g_flipLock.Release();
return -2;
}
if (!ApplyModeTiming(requested)) {
g_flipLock.Release();
KernelLogStream(ERROR, "IntelGPU") << "Modeset transaction failed; previous mode restored";
return -3;
}
@@ -1704,6 +1729,7 @@ namespace Drivers::Graphics::IntelGPU {
g_modes[g_currentMode].flags &= ~montauk::abi::DISPLAY_MODE_CURRENT;
g_currentMode = modeIndex;
g_modes[g_currentMode].flags |= montauk::abi::DISPLAY_MODE_CURRENT;
g_flipLock.Release();
KernelLogStream(OK, "IntelGPU") << "Mode set to " << base::dec
<< (uint64_t)requested.hdisplay << "x" << (uint64_t)requested.vdisplay
<< " @ " << (uint64_t)(g_modes[modeIndex].refreshMilliHz / 1000) << "Hz";
@@ -1741,59 +1767,131 @@ namespace Drivers::Graphics::IntelGPU {
return 0;
}
uint64_t GetBufferPageCount(int index) {
if (index == 0) return (g_fbSize + 0xFFF) >> 12;
if (index == 1 && g_flipSupported) return g_buf1Pages;
return 0;
}
int GetFrontBuffer() {
return g_frontBuffer;
g_flipLock.Acquire();
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
return live;
}
uint64_t GetVblankCount() {
return g_vblankCount;
int AcquireFlip(int pid) {
if (!g_flipSupported || pid < 0) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
int live = ReadLiveBufferIndex();
if (live < 0) {
// Recover from an unrecognised/pending firmware surface by putting
// the legacy buffer back synchronously before userspace draws.
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
if (WaitForLiveBuffer(0)) live = 0;
}
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
KernelLogStream(DEBUG, "IntelGPU") << "Flip acquired by pid "
<< base::dec << (uint64_t)pid << " (live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live) << ")";
return live;
}
bool WaitVblank(uint64_t timeoutMs) {
if (!g_vblankIrqReady) return false;
uint64_t start = g_vblankCount;
// The vblank IRQ fires every frame, so a wakeup lost to the race
// between reading the counter and blocking only costs one frame;
// the timeout is a backstop for a stalled pipe.
Sched::BlockOnObject((void*)&g_vblankCount, timeoutMs ? timeoutMs : 50);
return g_vblankCount != start;
}
int64_t Flip(int index, bool waitVsync, int pid) {
if (!g_flipSupported || index < 0 || index > 1 || pid < 0) return -1;
int64_t Flip(int index, bool waitVsync) {
if (!g_flipSupported || index < 0 || index > 1) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
// Scanout memory is exposed through WC mappings. Drain userspace's
// completed frame before programming the display-plane surface.
asm volatile("sfence" ::: "memory");
uint64_t gttOff = (index == 0) ? g_fbGttOffsetA : g_buf1GttOffset;
WriteReg(DSPASURF, (uint32_t)gttOff);
(void)ReadReg(DSPASURF);
g_frontBuffer = index;
if (waitVsync) {
if (g_vblankIrqReady) {
WaitVblank(50);
}
// Confirm the latch: DSPASURFLIVE tracks the surface actually
// being scanned out. With a working IRQ this succeeds on the
// first read; without one it is the vsync wait itself.
for (int i = 0; i < 3000; i++) {
if ((ReadReg(DSPASURFLIVE) & ~0xFFFu) == (uint32_t)gttOff) break;
MicroDelay(10);
// DSPASURFLIVE is authoritative and avoids taking ownership of
// the GPU's shared interrupt domain merely to wait for vblank.
if (!WaitForLiveBuffer(index)) {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmed = ReadReg(DSPASURF);
uint32_t liveSurface = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
KernelLogStream(WARNING, "IntelGPU") << "Flip latch timeout: requested="
<< base::dec << (uint64_t)index << " live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live)
<< " DSPASURF=" << base::hex << (uint64_t)programmed
<< " DSPASURFLIVE=" << (uint64_t)liveSurface;
return -1;
}
g_frontBuffer = index;
} else {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
}
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
return index;
}
void SetFlipOwner(int pid) {
g_flipOwnerPid = pid;
}
void OnProcessExit(int pid) {
if (!g_flipSupported || pid != g_flipOwnerPid) return;
if (!g_flipSupported) return;
g_flipLock.Acquire();
if (pid != g_flipOwnerPid) {
g_flipLock.Release();
return;
}
g_flipOwnerPid = -1;
if (g_frontBuffer != 0) Flip(0, false);
uint32_t programmedBefore = ReadReg(DSPASURF);
uint32_t liveBefore = ReadReg(DSPASURFLIVE);
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
bool liveAtZero = WaitForLiveBuffer(0);
bool restored = liveAtZero;
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmedAfter = ReadReg(DSPASURF);
uint32_t liveAfter = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
KernelLogStream(restored ? INFO : ERROR, "IntelGPU")
<< "Flip owner pid " << base::dec << (uint64_t)pid
<< " exited; restore buffer 0 " << (restored ? "latched" : "timed out")
<< " (before surf/live=" << base::hex << (uint64_t)programmedBefore
<< "/" << (uint64_t)liveBefore << ", after="
<< (uint64_t)programmedAfter << "/"
<< (uint64_t)liveAfter << ")";
}
void PanicRestoreScanout() {
if (!g_flipSupported || !g_mmioBase || g_frontBuffer == 0) return;
// g_frontBuffer is only a software cache and can be stale precisely
// when the GPU or kernel is failing. Always request buffer 0 so the
// panic console cannot be left painting an invisible surface.
if (!g_flipSupported || !g_mmioBase) return;
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
g_frontBuffer = 0;
@@ -1809,6 +1907,7 @@ namespace Drivers::Graphics::IntelGPU {
// command register, so MMIO writes to the GPU are silently dropped
// until we re-enable these bits.
Pci::EnableBusMaster(g_gpuInfo.pciBus, g_gpuInfo.pciDevice, g_gpuInfo.pciFunction);
DisableInterruptDelivery();
// Verify PCI memory space is accessible by reading back command register
uint16_t pciCmd = Pci::LegacyRead16(g_gpuInfo.pciBus, g_gpuInfo.pciDevice,
@@ -1875,9 +1974,8 @@ namespace Drivers::Graphics::IntelGPU {
// 5. Reprogram display plane to point at our GTT-mapped framebuffer
ProgramDisplayPlane();
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries, bring
// MSI + vblank interrupts back (the PCI config reset cleared MSI),
// and re-show whichever buffer was front when we suspended.
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries and
// re-show whichever buffer was front when we suspended.
if (g_flipSupported) {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
uint64_t baseIdx = g_buf1GttOffset >> 12;
@@ -1886,11 +1984,6 @@ namespace Drivers::Graphics::IntelGPU {
}
(void)ReadGgttPte(baseIdx + pages - 1);
if (g_vblankIrqReady) {
SetupMsi();
EnableVblankIrq();
}
if (g_frontBuffer != 0) {
WriteReg(DSPASURF, (uint32_t)g_buf1GttOffset);
(void)ReadReg(DSPASURF);
kernel/src/Drivers/Graphics/IntelGPU.hpp
+23 -36
View File
@@ -170,6 +170,7 @@ namespace Drivers::Graphics::IntelGPU {
static constexpr uint32_t VBLANK_A = 0x60010;
static constexpr uint32_t VSYNC_A = 0x60014;
static constexpr uint32_t PIPEASRC = 0x6001C;
static constexpr uint32_t PIPE_DSL = 0x70000; // current pipe A scanline
// --- Display timing registers (Pipe B) ---
static constexpr uint32_t HTOTAL_B = 0x61000;
@@ -231,30 +232,6 @@ namespace Drivers::Graphics::IntelGPU {
// engine is currently scanning out; DSPASURF writes latch at vblank) ---
static constexpr uint32_t DSPASURFLIVE = 0x701AC;
// --- Display engine interrupts (Gen 8+ pipe block, Gen 11+ master) ---
// Per-pipe interrupt registers, pipe A instance. Vblank is bit 0.
static constexpr uint32_t DE_PIPE_A_ISR = 0x44400;
static constexpr uint32_t DE_PIPE_A_IMR = 0x44404;
static constexpr uint32_t DE_PIPE_A_IIR = 0x44408;
static constexpr uint32_t DE_PIPE_A_IER = 0x4440C;
static constexpr uint32_t DE_PIPE_VBLANK = (1u << 0);
// Gen 11+ display interrupt master control (0x44200 was the Gen 8-10
// master IRQ register; on Gen 11+ it controls the display half only)
static constexpr uint32_t DISPLAY_INT_CTL = 0x44200;
static constexpr uint32_t DISPLAY_IRQ_ENABLE = (1u << 31);
static constexpr uint32_t DISPLAY_IRQ_PIPE_A = (1u << 16);
// Gen 11+ top-level graphics master interrupt
static constexpr uint32_t GFX_MSTR_INTR = 0x190010;
static constexpr uint32_t GFX_MSTR_INTR_ENABLE = (1u << 31);
static constexpr uint32_t GFX_MSTR_INTR_DISPLAY = (1u << 16);
// --- MSI (vblank interrupt delivery) ---
static constexpr uint8_t MSI_IRQ = 28; // IRQ slot 28 = vector 60
static constexpr uint32_t MSI_VECTOR = 60;
static constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// --- Output connectors ---
static constexpr uint32_t ADPA = 0x61100; // Analog Display Port (VGA/CRT)
static constexpr uint32_t DVOB = 0x61140; // DVO-B
@@ -347,6 +324,7 @@ namespace Drivers::Graphics::IntelGPU {
// Gen 8+ uses 64-bit GTT PTEs
static constexpr uint64_t GTT_PTE64_VALID = (1ULL << 0);
static constexpr uint64_t GTT_PTE64_ADDRESS_MASK = 0x000FFFFFFFFFF000ULL;
// Helper: Build a Gen 6/7 GTT PTE from a physical address
static inline uint32_t MakeGttPte32(uint64_t physAddr) {
@@ -358,7 +336,15 @@ namespace Drivers::Graphics::IntelGPU {
// Helper: Build a Gen 8+ GTT PTE from a physical address
static inline uint64_t MakeGttPte64(uint64_t physAddr) {
return (physAddr & ~0xFFFULL) | GTT_PTE64_VALID;
return (physAddr & GTT_PTE64_ADDRESS_MASK) | GTT_PTE64_VALID;
}
// Gen 6/7 PTEs carry 39 physical-address bits; Gen 8+ PTEs carry 52.
// Never silently truncate a DMA address into a different RAM page.
static inline bool GgttCanAddress(uint64_t physAddr, int gpuGen) {
if (physAddr & 0xFFFULL) return false;
if (gpuGen >= 8) return (physAddr & ~0x000FFFFFFFFFFFFFULL) == 0;
return (physAddr & ~0x0000007FFFFFFFFFULL) == 0;
}
// =========================================================================
@@ -525,25 +511,26 @@ namespace Drivers::Graphics::IntelGPU {
// Physical base of scanout buffer 0 (firmware FB) or 1 (kernel-allocated)
uint64_t GetBufferPhys(int index);
// Number of physically allocated pages backing the requested buffer.
uint64_t GetBufferPageCount(int index);
// Index of the buffer currently programmed for scanout
int GetFrontBuffer();
// Claim page flipping for a process and return the buffer the display
// engine is actually scanning out (0 or 1). Returns -1 if the live
// surface cannot be identified. This lets a newly-active fullscreen
// client resynchronise its private back-buffer state after a handoff.
int AcquireFlip(int pid);
// Point the display plane at the given buffer. The hardware latches the
// new surface address at the next vblank (tear-free). When waitVsync is
// set, blocks until the flip has been latched. Returns the new front
// buffer index, or -1 if flipping is unavailable.
int64_t Flip(int index, bool waitVsync);
int64_t Flip(int index, bool waitVsync, int pid);
// Block until the next vblank (Gen 11+ with working IRQ only).
// Returns false on timeout or when no vblank interrupt is available.
bool WaitVblank(uint64_t timeoutMs);
// Monotonic vblank counter (0 when no vblank IRQ)
uint64_t GetVblankCount();
// Track which process performs flips so scanout can be restored to
// buffer 0 when it exits (called from the flip syscall / scheduler).
void SetFlipOwner(int pid);
// The flip operation atomically claims ownership for pid while changing
// scanout. OnProcessExit restores buffer 0 for the last flip owner.
void OnProcessExit(int pid);
// Panic-safe: force scanout back to buffer 0 so panic output is visible.
kernel/src/Drivers/Net/E1000.cpp
+89 -41
View File
@@ -14,6 +14,8 @@
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
@@ -44,6 +46,11 @@ namespace Drivers::Net::E1000 {
// Current descriptor indices
static uint32_t g_rxTail = 0;
static uint32_t g_txTail = 0;
static kcp::Spinlock g_txLock;
static std::atomic<bool> g_rxWorkPending{false};
static std::atomic<bool> g_rxProcessing{false};
static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO;
// Statistics
static uint64_t g_rxPacketCount = 0;
@@ -238,52 +245,31 @@ namespace Drivers::Net::E1000 {
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Read and clear interrupt cause
uint32_t icr = ReadReg(REG_ICR);
if (icr & ICR_LSC) {
uint32_t status = ReadReg(REG_STATUS);
bool linkUp = (status & (1 << 1)) != 0;
KernelLogStream(INFO, "E1000") << "Link status change: " << (linkUp ? "UP" : "DOWN");
// ICR is clear-on-read; no further action is required. Do not log
// here: the terminal mutex is process-context-only and an IRQ that
// preempts its owner would otherwise deadlock the CPU.
(void)ReadReg(REG_STATUS);
}
// Both receive-timer and descriptor-threshold causes mean completed
// RX descriptors may be waiting. ICR is clear-on-read, so ignoring
// RXDMT0 can strand a lone packet until unrelated traffic arrives.
if (icr & (ICR_RXT0 | ICR_RXDMT0)) {
// Process received packets
while (true) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) {
break; // No more packets
}
uint16_t length = desc.Length;
g_rxPacketCount++;
// Dispatch to the network stack callback
if (g_rxCallback != nullptr) {
g_rxCallback(g_rxBuffers[nextIdx], length);
}
// Reset descriptor for reuse
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
}
// The callback enters Ethernet/TCP/UDP and IPC waitset notification,
// which takes process-context mutexes. Running it in this hard IRQ can
// self-deadlock if the interrupt preempts a holder on the same CPU.
// Mask RX causes and let an idle-context bottom half drain the ring.
if (icr & RX_INTERRUPT_MASK) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
if (icr & (ICR_TXDW | ICR_TXQE)) {
// TX completion - nothing to do for now
}
// TX completion interrupts are intentionally disabled. Descriptor DD
// is checked by SendPacket before reuse, so an interrupt per packet
// only creates a high-vector MSI load that can starve the LAPIC timer.
}
// -------------------------------------------------------------------------
@@ -314,6 +300,11 @@ namespace Drivers::Net::E1000 {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
uint16_t pciCommand = Pci::LegacyRead16(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE);
KernelLogStream(OK, "E1000") << "Bus mastering enabled";
// Read interrupt line from PCI config
@@ -351,7 +342,7 @@ namespace Drivers::Net::E1000 {
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
g_initialized = true;
@@ -403,6 +394,11 @@ namespace Drivers::Net::E1000 {
// Enable bus mastering and memory space
Pci::EnableBusMaster(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function);
Pci::DisableInterruptDelivery(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function);
uint16_t pciCommand = Pci::LegacyRead16(e1000Dev->Bus, e1000Dev->Device,
e1000Dev->Function, (uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function,
(uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE);
KernelLogStream(OK, "E1000") << "Bus mastering enabled";
@@ -457,8 +453,8 @@ namespace Drivers::Net::E1000 {
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
// Enable interrupts: RX, TX, Link Status Change
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
// Enable RX and link-change only. TX completion is polled by descriptor.
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
g_initialized = true;
@@ -473,10 +469,15 @@ namespace Drivers::Net::E1000 {
return false;
}
// Check if the current TX descriptor is available
// RX callbacks can send replies directly from interrupt context while
// userspace sends on another CPU. Serialize the shared tail/descriptor.
g_txLock.Acquire();
// Check if the current TX descriptor is available. Returning false is
// sufficient; logging here would be unsafe for the IRQ caller.
TxDescriptor& desc = g_txDescs[g_txTail];
if (!(desc.Status & TXSTA_DD)) {
KernelLogStream(WARNING, "E1000") << "TX ring full";
g_txLock.Release();
return false;
}
@@ -494,6 +495,7 @@ namespace Drivers::Net::E1000 {
WriteReg(REG_TDT, g_txTail);
g_txPacketCount++;
g_txLock.Release();
return true;
}
@@ -522,4 +524,50 @@ namespace Drivers::Net::E1000 {
g_rxCallback = callback;
}
bool HasDeferredWork() {
return g_initialized && g_rxWorkPending.load(std::memory_order_acquire);
}
void ProcessDeferredWork() {
if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return;
bool expected = false;
if (!g_rxProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) return;
g_rxWorkPending.store(false, std::memory_order_release);
// Bound one bottom-half pass to one ring revolution. Hardware may
// refill descriptors as quickly as we return them under a packet
// flood; an unbounded drain could otherwise keep an idle context in
// the network stack indefinitely. The post-pass check below queues
// another fair slice when completed descriptors remain.
uint32_t processed = 0;
while (processed < RX_DESC_COUNT) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) break;
uint16_t length = desc.Length;
g_rxPacketCount++;
if (g_rxCallback != nullptr) g_rxCallback(g_rxBuffers[nextIdx], length);
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
processed++;
}
// Re-arm RX only after every completed descriptor was returned. Check
// once more after unmasking to close the packet-arrival race window.
WriteReg(REG_IMS, RX_INTERRUPT_MASK);
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
if (g_rxDescs[nextIdx].Status & RXSTA_DD) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
g_rxProcessing.store(false, std::memory_order_release);
}
};
kernel/src/Drivers/Net/E1000.hpp
+5
View File
@@ -128,4 +128,9 @@ namespace Drivers::Net::E1000 {
// Register a callback for received packets
void SetRxCallback(RxCallback callback);
// Drain packets queued by the hard IRQ from idle/process context. The
// Ethernet/IP/IPC stack is not safe to execute inside an interrupt.
void ProcessDeferredWork();
bool HasDeferredWork();
};
kernel/src/Drivers/Net/E1000E.cpp
+66 -52
View File
@@ -14,6 +14,8 @@
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
@@ -104,6 +106,11 @@ namespace Drivers::Net::E1000E {
// Current descriptor indices
static uint32_t g_rxTail = 0;
static uint32_t g_txTail = 0;
static kcp::Spinlock g_txLock;
static std::atomic<bool> g_rxWorkPending{false};
static std::atomic<bool> g_rxProcessing{false};
static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO;
// Statistics
static uint64_t g_rxPacketCount = 0;
@@ -410,7 +417,7 @@ namespace Drivers::Net::E1000E {
// MSI setup
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq); // forward declaration
static void HandleInterrupt(uint8_t irq, bool fromUser); // forward declaration
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
@@ -419,6 +426,8 @@ namespace Drivers::Net::E1000E {
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "E1000E") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -461,7 +470,7 @@ namespace Drivers::Net::E1000E {
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
uint32_t icr = ReadReg(REG_ICR);
@@ -472,42 +481,21 @@ namespace Drivers::Net::E1000E {
}
if (icr & ICR_LSC) {
uint32_t status = ReadReg(REG_STATUS);
bool linkUp = (status & (1 << 1)) != 0;
KernelLogStream(INFO, "E1000E") << "Link status change: " << (linkUp ? "UP" : "DOWN");
// Acknowledge/observe the new state without printing from IRQ
// context; KernelLogStream's terminal mutex is not IRQ-safe.
(void)ReadReg(REG_STATUS);
}
// Both receive-timer and descriptor-threshold causes mean completed
// RX descriptors may be waiting. ICR is clear-on-read, so ignoring
// RXDMT0 can strand a lone packet until unrelated traffic arrives.
if (icr & (ICR_RXT0 | ICR_RXDMT0)) {
while (true) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) {
break;
}
uint16_t length = desc.Length;
g_rxPacketCount++;
if (g_rxCallback != nullptr) {
g_rxCallback(g_rxBuffers[nextIdx], length);
}
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
}
// Defer the full network/IPC stack. It takes process-context mutexes
// and cannot safely run nested under MSI/legacy interrupt delivery.
if (icr & RX_INTERRUPT_MASK) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
if (icr & (ICR_TXDW | ICR_TXQE)) {
// TX completion — nothing to do for now
}
// TX completion interrupts are intentionally disabled. Descriptor DD
// is checked by SendPacket before reuse, so an interrupt per packet
// only creates a high-vector MSI load that can starve the LAPIC timer.
}
// -------------------------------------------------------------------------
@@ -596,12 +584,17 @@ namespace Drivers::Net::E1000E {
ConfigureInterruptModeration();
if (SetupMsi(bus, device, function)) {
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
} else if (g_irqLine != 0xFF) {
KernelLogStream(INFO, "E1000E") << "Falling back to legacy IRQ " << base::dec << (uint64_t)g_irqLine;
Pci::DisableInterruptDelivery(bus, device, function);
uint16_t command = Pci::LegacyRead16(bus, device, function,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(bus, device, function,
(uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE);
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
} else {
KernelLogStream(WARNING, "E1000E") << "No MSI or legacy IRQ available, using polling mode";
g_pollingMode = true;
@@ -673,18 +666,12 @@ namespace Drivers::Net::E1000E {
// Polling: process received packets (with reentrancy guard)
// -------------------------------------------------------------------------
static bool g_polling = false;
static void PollRx() {
// Guard against reentrancy: RX callback can trigger ARP reply →
// Ethernet::Send → SendPacket → PollRx(). Two concurrent callers
// modifying g_rxTail / RDT would corrupt the descriptor ring.
if (g_polling) {
return;
}
g_polling = true;
while (true) {
// Never let a continuously refilled DMA ring monopolize the idle
// bottom half. Process at most one revolution; ProcessDeferredWork's
// final descriptor check schedules another pass if traffic remains.
uint32_t processed = 0;
while (processed < RX_DESC_COUNT) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
@@ -705,9 +692,9 @@ namespace Drivers::Net::E1000E {
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
processed++;
}
g_polling = false;
}
bool SendPacket(const uint8_t* data, uint16_t length) {
@@ -715,9 +702,13 @@ namespace Drivers::Net::E1000E {
return false;
}
// RX processing can send replies from MSI/timer context while a
// userspace sender runs on another CPU. Protect the shared TX ring.
g_txLock.Acquire();
TxDescriptor& desc = g_txDescs[g_txTail];
if (!(desc.Status & TXSTA_DD)) {
KernelLogStream(WARNING, "E1000E") << "TX ring full";
g_txLock.Release();
return false;
}
@@ -732,6 +723,7 @@ namespace Drivers::Net::E1000E {
WriteReg(REG_TDT, g_txTail);
g_txPacketCount++;
g_txLock.Release();
return true;
}
@@ -765,10 +757,32 @@ namespace Drivers::Net::E1000E {
}
void Poll() {
if (!g_initialized) {
return;
}
if (g_initialized) g_rxWorkPending.store(true, std::memory_order_release);
}
bool HasDeferredWork() {
return g_initialized && g_rxWorkPending.load(std::memory_order_acquire);
}
void ProcessDeferredWork() {
if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return;
bool expected = false;
if (!g_rxProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) return;
g_rxWorkPending.store(false, std::memory_order_release);
PollRx();
if (!g_pollingMode) {
WriteReg(REG_IMS, RX_INTERRUPT_MASK);
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
if (g_rxDescs[nextIdx].Status & RXSTA_DD) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
}
g_rxProcessing.store(false, std::memory_order_release);
}
};
kernel/src/Drivers/Net/E1000E.hpp
+5 -1
View File
@@ -165,7 +165,11 @@ namespace Drivers::Net::E1000E {
// Register a callback for received packets
void SetRxCallback(RxCallback callback);
// Poll for received packets (used when legacy IRQ is unavailable)
// Request an RX poll (used when hardware interrupts are unavailable).
void Poll();
// Drain queued RX descriptors outside MSI/timer interrupt context.
void ProcessDeferredWork();
bool HasDeferredWork();
};
kernel/src/Drivers/PS2/Keyboard.cpp
+1 -1
View File
@@ -150,7 +150,7 @@ namespace Drivers::PS2::Keyboard {
Kt::KernelLogStream(Kt::OK, "PS2/KB") << "Keyboard driver initialized";
}
void HandleIRQ(uint8_t irq) {
void HandleIRQ(uint8_t irq, bool) {
(void)irq;
uint8_t scancode = Io::In8(DataPort);
kernel/src/Drivers/PS2/Keyboard.hpp
+2 -2
View File
@@ -39,7 +39,7 @@ namespace Drivers::PS2::Keyboard {
void Initialize();
// Interrupt handler -- called from IRQ dispatch (EOI is sent automatically)
void HandleIRQ(uint8_t irq);
void HandleIRQ(uint8_t irq, bool fromUser);
// Public interface for consuming key events
bool IsKeyAvailable();
@@ -52,4 +52,4 @@ namespace Drivers::PS2::Keyboard {
// Inject a key event from an external source (e.g., USB HID keyboard)
void InjectKeyEvent(const KeyEvent& event);
};
};
kernel/src/Drivers/PS2/Mouse.cpp
+1 -1
View File
@@ -109,7 +109,7 @@ namespace Drivers::PS2::Mouse {
Kt::KernelLogStream(Kt::OK, "PS2/Mouse") << "Mouse driver initialized";
}
void HandleIRQ(uint8_t irq) {
void HandleIRQ(uint8_t irq, bool) {
(void)irq;
uint8_t data = Io::In8(DataPort);
kernel/src/Drivers/PS2/Mouse.hpp
+2 -2
View File
@@ -24,7 +24,7 @@ namespace Drivers::PS2::Mouse {
void Initialize();
// Interrupt handler -- called from IRQ dispatch (EOI is sent automatically)
void HandleIRQ(uint8_t irq);
void HandleIRQ(uint8_t irq, bool fromUser);
// Public interface
MouseState GetMouseState();
@@ -42,4 +42,4 @@ namespace Drivers::PS2::Mouse {
// Inject a mouse report from an external source (e.g., USB HID mouse)
void InjectMouseReport(uint8_t buttons, int8_t deltaX, int8_t deltaY, int8_t scroll);
};
};
kernel/src/Drivers/Storage/Ahci.cpp
+7 -80
View File
@@ -13,8 +13,6 @@
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
using namespace Kt;
@@ -230,10 +228,9 @@ namespace Drivers::Storage::Ahci {
WritePortReg(port, PORT_SERR, 0xFFFFFFFF);
WritePortReg(port, PORT_IS, 0xFFFFFFFF);
// Enable interrupts for this port
WritePortReg(port, PORT_IE,
PORT_IS_DHRS | PORT_IS_PSS | PORT_IS_DSS |
PORT_IS_SDBS | PORT_IS_TFES);
// Commands are completed by polling PxCI/PxIS. Interrupt delivery is
// unnecessary and can starve the LAPIC timer under sustained I/O.
WritePortReg(port, PORT_IE, 0);
// Power on and spin up if needed
uint32_t cmd = ReadPortReg(port, PORT_CMD);
@@ -533,65 +530,6 @@ namespace Drivers::Storage::Ahci {
return true;
}
// -------------------------------------------------------------------------
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
(void)irq;
uint32_t is = ReadReg(REG_IS);
if (is == 0) return;
// Acknowledge each port's interrupt
for (int i = 0; i < MAX_PORTS; i++) {
if (is & (1u << i)) {
uint32_t portIs = ReadPortReg(i, PORT_IS);
WritePortReg(i, PORT_IS, portIs);
}
}
// Clear global interrupt status
WriteReg(REG_IS, is);
}
// -------------------------------------------------------------------------
// MSI setup
// -------------------------------------------------------------------------
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "AHCI") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // Single message
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "AHCI") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR;
return true;
}
// -------------------------------------------------------------------------
// HBA reset
// -------------------------------------------------------------------------
@@ -659,6 +597,7 @@ namespace Drivers::Storage::Ahci {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
// BIOS/OS handoff
PerformBiosHandoff();
@@ -690,22 +629,10 @@ namespace Drivers::Storage::Ahci {
g_portsImplemented = ReadReg(REG_PI);
KernelLogStream(INFO, "AHCI") << "Ports implemented: " << base::hex << (uint64_t)g_portsImplemented;
// Set up MSI (or fall back to legacy IRQ)
bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function);
if (!hasMsi) {
uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "AHCI") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine;
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
}
}
// Enable global interrupts
// Keep controller interrupts disabled; every command path polls its
// completion and consumes/clears PxIS directly.
uint32_t ghc = ReadReg(REG_GHC);
ghc |= GHC_IE;
ghc &= ~GHC_IE;
WriteReg(REG_GHC, ghc);
// Initialize each implemented port
kernel/src/Drivers/Storage/Ahci.hpp
-5
View File
@@ -181,11 +181,6 @@ namespace Drivers::Storage::Ahci {
constexpr int MAX_PRDT_ENTRIES = 8; // Max PRDT entries per command
constexpr int SECTOR_SIZE = 512;
// MSI configuration
constexpr uint8_t MSI_IRQ = 25; // IRQ slot 25 = vector 57
constexpr uint32_t MSI_VECTOR = 57;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// =========================================================================
// Port info
// =========================================================================
kernel/src/Drivers/Storage/Nvme.cpp
+7 -64
View File
@@ -13,8 +13,6 @@
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
@@ -511,9 +509,9 @@ namespace Drivers::Storage::Nvme {
cmd.Prp1 = g_ioCqPhys;
// CDW10: bits 31:16 = queue size (0-based), bits 15:0 = queue ID
cmd.Cdw10 = ((uint32_t)(g_ioCqDepth - 1) << 16) | 1;
// CDW11: bit 0 = physically contiguous, bit 1 = interrupts enabled
// bits 31:16 = interrupt vector
cmd.Cdw11 = (1u << 0) | (1u << 1) | (0u << 16);
// Physically contiguous, with completion interrupts disabled. The
// queue is consumed synchronously by WaitCompletion().
cmd.Cdw11 = (1u << 0);
CqEntry cqe;
if (!AdminCommand(cmd, cqe)) {
@@ -549,54 +547,6 @@ namespace Drivers::Storage::Nvme {
return true;
}
// -------------------------------------------------------------------------
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
(void)irq;
// NVMe uses polling-based completion in this driver.
// The interrupt handler just acknowledges the interrupt.
// Completions are consumed in the polling loops above.
}
// -------------------------------------------------------------------------
// MSI setup
// -------------------------------------------------------------------------
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "NVMe") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // Single message
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "NVMe") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR;
return true;
}
// -------------------------------------------------------------------------
// Probe (PCI driver entry point)
// -------------------------------------------------------------------------
@@ -629,6 +579,7 @@ namespace Drivers::Storage::Nvme {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
// Read capabilities
uint64_t cap = ReadReg64(REG_CAP);
@@ -653,17 +604,9 @@ namespace Drivers::Storage::Nvme {
return false;
}
// Step 2: Set up MSI
bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function);
if (!hasMsi) {
uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "NVMe") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine;
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
}
}
// Step 2: keep PCI interrupt delivery disabled. Admin and I/O queues
// are polled synchronously, so completion interrupts add latency and
// can starve the lower-vector scheduler timer under sustained I/O.
// Step 3: Set up admin queues
if (!SetupAdminQueues()) {
kernel/src/Drivers/Storage/Nvme.hpp
-8
View File
@@ -158,14 +158,6 @@ namespace Drivers::Storage::Nvme {
char Model[41];
};
// =========================================================================
// MSI configuration (use a different IRQ slot than AHCI)
// =========================================================================
constexpr uint8_t MSI_IRQ = 26; // IRQ slot 26 = vector 58
constexpr uint32_t MSI_VECTOR = 58;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// =========================================================================
// Public API
// =========================================================================
kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp
+6 -1
View File
@@ -47,7 +47,12 @@ namespace Drivers::USB::Bluetooth::Avrcp {
// AVRCP notification events
constexpr uint8_t EVT_VOLUME_CHANGED = 0x0D;
static kcp::Spinlock g_notifyLock;
// AVRCP callbacks are dispatched from xHCI's deferred PollEvents path, not
// from the hard IRQ. Sending the INTERIM/CHANGED response can descend into
// the HCI and xHCI transmit rings, so keep interrupts enabled while this
// state is serialized. An IRQ-disabling spinlock here can stall the local
// timer behind an unrelated transmit/recovery operation.
static kcp::Mutex g_notifyLock;
static uint16_t g_volumeNotifyCid = 0;
static uint8_t g_volumeNotifyTransaction = 0;
kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp
+2 -1
View File
@@ -406,12 +406,13 @@ namespace Drivers::USB::Bluetooth {
// system this briefly pauses userspace, matching the old synchronous
// behavior minus the boot-path stall.
auto* cpu = Smp::GetCurrentCpuData();
bool wasReserved = cpu && cpu->reservedForKernelWork;
if (cpu) cpu->reservedForKernelWork = true;
Hci::SetFwTrace(true); // bounded per-completion event-pipe trace
CompleteInit();
Hci::DumpFwTrace(); // flush remaining records (process context)
Hci::SetFwTrace(false);
if (cpu) cpu->reservedForKernelWork = false;
if (cpu) cpu->reservedForKernelWork = wasReserved;
}
// =========================================================================
kernel/src/Drivers/USB/Bluetooth/Hci.cpp
+14 -8
View File
@@ -59,10 +59,9 @@ namespace Drivers::USB::Bluetooth::Hci {
//
// The trace is recorded into a lock-free ring and printed LATER by
// DumpFwTrace() from process context. It must NEVER KernelLogStream from
// TransferCallback: that runs nested under PollEvents in xHCI MSI (IRQ)
// context, and the terminal lock is a non-IRQ-disabling Mutex -- an IRQ
// logger interrupting a same-core holder spins forever (observed: hard
// boot freeze right after "Intel version raw", 2026-07-06).
// TransferCallback: it runs nested under PollEvents while the event-ring
// ownership guard is held. Logging there can recursively pump or contend
// with unrelated subsystem locks, so keep it a bounded data-only callback.
static std::atomic<bool> g_fwTrace{false};
static std::atomic<uint32_t> g_fwTraceCount{0};
static std::atomic<uint32_t> g_intInCompletions{0};
@@ -148,8 +147,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// guards, so one core can be sending a media packet while another sends a
// signaling reply; without this lock the two race on g_aclTxSlot and on
// the bulk OUT TRB ring's enqueue/cycle state, garbling packets on air.
// IRQ-disabling spinlock so the holder also can't be preempted mid-send.
static kcp::Spinlock g_aclTxLock;
// These paths run only from deferred/process context. Keep interrupts
// enabled: endpoint recovery also takes this lock and can wait for bounded
// xHCI commands, which must never happen below an IRQ-disabling spinlock.
static kcp::Mutex g_aclTxLock;
// HCI command DMA buffer (separate from ACL to avoid conflicts)
static uint8_t* g_cmdDmaBuf = nullptr;
@@ -559,8 +560,8 @@ namespace Drivers::USB::Bluetooth::Hci {
// Bounded bring-up trace: one RING RECORD per interrupt-IN
// completion while the firmware phase runs; DumpFwTrace() prints
// them later from process context. No logging here -- this runs
// in xHCI MSI (IRQ) context (see the deadlock note at the ring).
// them later from top-level process/idle context. No logging here:
// this callback is nested under the xHCI event-ring owner.
if (g_fwTrace.load(std::memory_order_relaxed)) {
uint32_t n = g_fwTraceCount.fetch_add(1, std::memory_order_relaxed);
if (n < FW_TRACE_CAP) {
@@ -2074,6 +2075,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// all TRBs queued behind it. Free those DMA slots and remove only
// those never-delivered packets from the controller-credit count;
// earlier successful OUT transfers still await their NOCP events.
// Serialize the dequeue rewrite with SendAcl/firmware bulk enqueue.
// Without this, another CPU can publish a TRB between Reset Endpoint
// and Set TR Dequeue and have recovery silently discard or split it.
g_aclTxLock.Acquire();
uint32_t discarded = AclTxInFlight();
KernelLogStream(WARNING, "BT-HCI")
<< "Recovering HCI ACL transmit pipe; discarding "
@@ -2086,6 +2091,7 @@ namespace Drivers::USB::Bluetooth::Hci {
- (int32_t)discarded;
if (remaining < 0)
g_aclPendingCount.store(0, std::memory_order_release);
g_aclTxLock.Release();
}
// Asynchronous HCI events are processed at top level. Besides avoiding
kernel/src/Drivers/USB/HidMouse.cpp
+30 -10
View File
@@ -172,18 +172,23 @@ namespace Drivers::USB::HidMouse {
// -------------------------------------------------------------------------
static int32_t ExtractSigned(const uint8_t* data, uint16_t bitOffset, uint8_t bitSize) {
int32_t value = 0;
uint32_t value = 0;
for (uint8_t i = 0; i < bitSize; i++) {
uint16_t byteIdx = (bitOffset + i) / 8;
uint8_t bitIdx = (bitOffset + i) % 8;
if (data[byteIdx] & (1 << bitIdx))
value |= (1 << i);
value |= (1u << i);
}
// Sign extend
if (bitSize < 32 && (value & (1 << (bitSize - 1)))) {
value |= ~((1 << bitSize) - 1);
if (bitSize < 32 && (value & (1u << (bitSize - 1)))) {
value |= ~((1u << bitSize) - 1);
}
return value;
return (int32_t)value;
}
static bool FieldFits(uint32_t bitOffset, uint32_t bitSize, uint32_t reportBits) {
return bitSize > 0 && bitSize <= 32 && bitOffset <= reportBits
&& bitSize <= reportBits - bitOffset;
}
// -------------------------------------------------------------------------
@@ -222,18 +227,30 @@ namespace Drivers::USB::HidMouse {
// Skip report ID byte if present
uint16_t byteOffset = g_Format.hasReportId ? 8 : 0;
if (g_Format.hasReportId && data[0] != g_Format.reportId) return;
uint32_t reportBits = (uint32_t)length * 8;
uint32_t buttonBits = g_Format.buttonCount < 8 ? g_Format.buttonCount : 8;
uint32_t buttonOffset = byteOffset + g_Format.buttonBitOffset;
uint32_t xOffset = byteOffset + g_Format.xBitOffset;
uint32_t yOffset = byteOffset + g_Format.yBitOffset;
if ((buttonBits != 0 && !FieldFits(buttonOffset, buttonBits, reportBits))
|| !FieldFits(xOffset, g_Format.xBitSize, reportBits)
|| !FieldFits(yOffset, g_Format.yBitSize, reportBits)) {
return;
}
// Extract buttons
uint8_t buttons = 0;
for (uint8_t i = 0; i < g_Format.buttonCount && i < 8; i++) {
uint16_t bit = byteOffset + g_Format.buttonBitOffset + i;
for (uint8_t i = 0; i < buttonBits; i++) {
uint16_t bit = (uint16_t)(buttonOffset + i);
if (data[bit / 8] & (1 << (bit % 8)))
buttons |= (1 << i);
}
// Extract X, Y
int32_t rawX = ExtractSigned(data, byteOffset + g_Format.xBitOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, byteOffset + g_Format.yBitOffset, g_Format.yBitSize);
int32_t rawX = ExtractSigned(data, (uint16_t)xOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, (uint16_t)yOffset, g_Format.yBitSize);
// Clamp to int8_t range for InjectMouseReport
if (rawX > 127) rawX = 127;
@@ -244,7 +261,10 @@ namespace Drivers::USB::HidMouse {
// Extract scroll wheel
int8_t scroll = 0;
if (g_Format.scrollBitSize > 0) {
int32_t rawScroll = ExtractSigned(data, byteOffset + g_Format.scrollBitOffset, g_Format.scrollBitSize);
uint32_t scrollOffset = byteOffset + g_Format.scrollBitOffset;
if (!FieldFits(scrollOffset, g_Format.scrollBitSize, reportBits)) return;
int32_t rawScroll = ExtractSigned(data, (uint16_t)scrollOffset,
g_Format.scrollBitSize);
if (rawScroll > 127) rawScroll = 127;
if (rawScroll < -128) rawScroll = -128;
scroll = (int8_t)rawScroll;
kernel/src/Drivers/USB/Xhci.cpp
+200 -53
View File
@@ -54,9 +54,30 @@ namespace Drivers::USB::Xhci {
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static volatile bool g_hotplugPending[MAX_PORTS] = {};
static volatile bool g_deferredWorkPending = false;
static std::atomic<bool> g_hotplugPending[MAX_PORTS] = {};
static std::atomic<bool> g_deferredWorkPending{false};
static std::atomic<bool> g_diagnosticWorkPending{false};
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
// diagnostic payload atomically and print it from ProcessDeferredWork().
static std::atomic<uint32_t> g_transferErrorsPending{0};
static std::atomic<uint32_t> g_transferErrorReports{0};
static std::atomic<uint32_t> g_lastErrorSlot{0};
static std::atomic<uint32_t> g_lastErrorEp{0};
static std::atomic<uint32_t> g_lastErrorCc{0};
static std::atomic<uint32_t> g_stormsPending{0};
static std::atomic<uint32_t> g_stormReports{0};
static std::atomic<uint32_t> g_lastStormProcessed{0};
static std::atomic<uint32_t> g_lastStormPortEvents{0};
static std::atomic<uint32_t> g_lastStormTransferEvents{0};
static std::atomic<uint32_t> g_lastStormType{0};
static std::atomic<uint32_t> g_lastStormSlot{0};
static std::atomic<uint32_t> g_lastStormEp{0};
static std::atomic<uint32_t> g_lastStormCc{0};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
@@ -78,6 +99,12 @@ namespace Drivers::USB::Xhci {
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// The controller exposes one command ring and one command-completion
// mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer
// recovery can originate on different CPUs, so the whole submit/wait
// transaction must be single-owner. This is a process/idle-context lock;
// SendCommand is never called from a transfer-event callback.
static kcp::Mutex g_commandLock;
// Event ring
static TRB* g_evtRing = nullptr;
@@ -125,12 +152,12 @@ namespace Drivers::USB::Xhci {
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from the MSI handler (BSP) AND from
// syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the
// app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set
// on a volatile is a cross-core TOCTOU: two cores both pass the check,
// both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync,
// and completions get double-delivered (duplicate ACL packets) or lost
// volatile bool: PollEvents runs from syscall-context pumps (A2DP
// WriteAudio/StartSource on whatever core the app runs on) and from idle
// cores (ProcessDeferredWork). A check-then-set on a volatile is a
// cross-core TOCTOU: two cores both pass the check, both drain the shared
// event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get
// double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
@@ -293,7 +320,7 @@ namespace Drivers::USB::Xhci {
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
@@ -389,10 +416,13 @@ namespace Drivers::USB::Xhci {
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -448,33 +478,37 @@ namespace Drivers::USB::Xhci {
// PollEvents - process event ring
// -------------------------------------------------------------------------
void PollEvents() {
// PollEvents runs both from the synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and from the xHCI MSI handler. On a
// single core the IRQ can preempt a poll loop mid-drain; if both advance
// g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the
// cycle-bit check can start matching stale entries forever -> the boot
// freezes (observed wedging the Bluetooth firmware download at ~635 KB,
// where the dying device floods the event ring). Guard against re-entry:
// the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and
// the active poll loop drains these events itself. g_pollActive is also
// read by InPollContext() so command submitters (ControlTransfer) can
// tell they are nested and must fire-and-forget instead of waiting.
enum class PollResult : uint8_t {
Drained,
Busy,
BudgetExhausted,
};
static PollResult PollEventsBounded(uint32_t maxEvents) {
// PollEvents runs from synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and idle bottom halves on different
// CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring
// tracking desyncs and the cycle-bit check can start matching stale
// entries forever -> the boot freezes (observed wedging the Bluetooth
// firmware download at ~635 KB, where the dying device floods the event
// ring). Guard against cross-core re-entry. g_pollActive is also read by
// InPollContext() so command submitters reached from a callback can tell
// they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return; // another context is draining; it will reap our events
return PollResult::Busy; // another context is draining
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound the work per call so a flooding/wedged device can never spin
// here forever; the outer wall-clock timeouts then fire instead of
// freezing. 4x the ring size is far above any legitimate burst.
constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4;
// Bound each pass so a flooding/wedged device cannot monopolize its
// caller. Synchronous poll loops use a generous batch; the timer uses
// a much smaller recovery slice after the hard IRQ masks the source.
if (maxEvents == 0) maxEvents = 1;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < MAX_EVENTS_PER_CALL) {
while (processed < maxEvents) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
@@ -506,8 +540,8 @@ namespace Drivers::USB::Xhci {
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1] = true;
g_deferredWorkPending = true;
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
break;
}
@@ -616,9 +650,16 @@ namespace Drivers::USB::Xhci {
nullptr, 0, completionCode);
}
} else {
KernelLogStream(WARNING, "xHCI") << "Transfer error on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci
<< " cc=" << (uint64_t)completionCode;
// Never print from this path: PollEvents may be
// running in the MSI handler. Defer a rate-limited
// diagnostic to idle/process context instead.
if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) {
g_lastErrorSlot.store(slotId, std::memory_order_relaxed);
g_lastErrorEp.store(epDci, std::memory_order_relaxed);
g_lastErrorCc.store(completionCode, std::memory_order_relaxed);
g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
@@ -651,37 +692,70 @@ namespace Drivers::USB::Xhci {
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded -- surface the dominant
// event source (rate-limited) so a wedge is diagnosable, not silent.
if (processed >= MAX_EVENTS_PER_CALL) {
static uint32_t stormLogs = 0;
if (stormLogs < 8) {
stormLogs++;
KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed
<< "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts
<< " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot
<< " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")";
// A full batch means the ring is being flooded. Record the dominant
// source for a deferred, rate-limited process-context diagnostic.
bool exhausted = processed >= maxEvents;
if (exhausted) {
if (g_stormReports.load(std::memory_order_relaxed) < 8) {
g_lastStormProcessed.store(processed, std::memory_order_relaxed);
g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed);
g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed);
g_lastStormType.store(lastType, std::memory_order_relaxed);
g_lastStormSlot.store(lastSlot, std::memory_order_relaxed);
g_lastStormEp.store(lastEp, std::memory_order_relaxed);
g_lastStormCc.store(lastCC, std::memory_order_relaxed);
g_stormsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
return exhausted ? PollResult::BudgetExhausted : PollResult::Drained;
}
void PollEvents() {
PollResult result = PollEventsBounded(EVT_RING_SIZE * 4);
if (result == PollResult::Drained &&
g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
// The hard IRQ leaves the interrupter masked. A synchronous
// process-context poll may drain the ring before an idle bottom
// half sees it, so finish the handoff here as well. Verify once
// more while masked: on another CPU, the IRQ may have acknowledged
// IMAN.IP for an event posted just after our first empty check. In
// that race, blindly enabling IE would strand the event because
// the IRQ already cleared its pending bit.
PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4);
if (verify != PollResult::Drained) {
g_irqDrainPending.store(true, std::memory_order_release);
return;
}
// If hardware queues an event after this masked verification,
// IMAN.IP remains asserted and enabling IE generates a fresh MSI.
WriteRt(IR0_IMAN, IMAN_IE);
}
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Clear IMAN.IP and ensure IE stays enabled
WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE);
PollEvents();
// Keep the hard IRQ strictly bounded. PollEvents dispatches HID,
// Bluetooth, and SDR callbacks; those callbacks can take subsystem
// locks, wake scheduler objects, re-arm endpoints, and copy sizeable
// buffers. Running them here lets a USB burst monopolize this
// higher-priority vector ahead of the LAPIC timer and can deadlock if
// an interrupted context owns one of those locks. Acknowledge and mask
// the source, then let an idle/process-context poll drain the ring.
WriteRt(IR0_IMAN, IMAN_IP);
g_irqDrainPending.store(true, std::memory_order_release);
}
// -------------------------------------------------------------------------
@@ -689,6 +763,8 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
g_commandLock.Acquire();
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
@@ -728,7 +804,9 @@ namespace Drivers::USB::Xhci {
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
return g_cmdCompletionCode;
uint32_t result = g_cmdCompletionCode;
g_commandLock.Release();
return result;
}
// Small delay
for (int j = 0; j < 100; j++) {
@@ -736,6 +814,9 @@ namespace Drivers::USB::Xhci {
}
}
// Do not hold the command-ring lock while taking the terminal mutex.
// A later caller may attempt recovery even if this command timed out.
g_commandLock.Release();
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
@@ -1304,7 +1385,55 @@ namespace Drivers::USB::Xhci {
}
bool HasDeferredWork() {
return g_initialized && g_deferredWorkPending;
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
void ServiceDeferredIrqEvents(uint32_t budget) {
if (!g_initialized ||
!g_irqDrainPending.load(std::memory_order_acquire)) return;
PollResult result = PollEventsBounded(budget);
if (result != PollResult::Drained) return;
// Clear before re-enabling. If hardware queued an event after the ring
// appeared empty, IMAN.IP remains asserted and enabling IE generates a
// fresh MSI; no event can be lost in this handoff.
if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
WriteRt(IR0_IMAN, IMAN_IE);
}
}
static void PrintDeferredDiagnostics() {
// Clear the wakeup before consuming counters. A producer racing after
// this store sets it again, so deferred work cannot be lost.
g_diagnosticWorkPending.store(false, std::memory_order_release);
uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel);
if (errors != 0 &&
g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors
<< " transfer error(s); last slot="
<< (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed);
}
uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel);
if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms
<< " event storm(s); last batch="
<< (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed)
<< " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed)
<< " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed)
<< " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed)
<< " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")";
}
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
@@ -1339,7 +1468,19 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return;
if (!g_initialized) return;
// The hard IRQ only acknowledges and masks the interrupter. Drain a
// bounded slice here, outside interrupt context, so callbacks may use
// their normal locks without blocking the timer or scheduler. A busy
// process that is synchronously waiting for USB calls PollEvents()
// itself, which performs the same unmask handoff when the ring is dry.
ServiceDeferredIrqEvents(64);
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
g_deferredWorkPending.load(std::memory_order_acquire);
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
@@ -1348,11 +1489,17 @@ namespace Drivers::USB::Xhci {
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
g_deferredWorkPending = false;
if (haveDiagnostics) PrintDeferredDiagnostics();
if (!haveHotplug) {
g_hotplugProcessing.store(false, std::memory_order_release);
return;
}
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port]) continue;
g_hotplugPending[port] = false;
if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
kernel/src/Drivers/USB/Xhci.hpp
+11 -4
View File
@@ -20,9 +20,11 @@ namespace Drivers::USB::Xhci {
constexpr uint32_t EVT_RING_SIZE = 64;
constexpr uint32_t XFER_RING_SIZE = 32;
// MSI configuration (E1000E uses IRQ 24/vector 56, we use 25/57)
constexpr uint8_t MSI_IRQ = 25;
constexpr uint32_t MSI_VECTOR = 57;
// MSI configuration. Keep xHCI distinct from AHCI's IRQ 25/vector 57;
// the IRQ dispatcher has one handler per slot, so sharing would replace
// one driver's handler and eventually strand USB or storage completions.
constexpr uint8_t MSI_IRQ = 29;
constexpr uint32_t MSI_VECTOR = 61;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// PCI class/subclass/progif for xHCI
@@ -299,6 +301,10 @@ namespace Drivers::USB::Xhci {
// Deferred hot-plug processing (call outside interrupt context)
void ProcessDeferredWork();
// Drain a bounded event batch after the hard IRQ masked the interrupter.
// Call only from process/idle context; this dispatches class callbacks.
void ServiceDeferredIrqEvents(uint32_t budget);
// Send a command on the command ring, wait for completion.
// Returns completion code.
uint32_t SendCommand(const TRB& trb);
@@ -367,7 +373,8 @@ namespace Drivers::USB::Xhci {
// Access device info
UsbDeviceInfo* GetDevice(uint8_t slotId);
// Poll event ring (called from interrupt handler or during init)
// Poll the event ring from process/idle context. The hard interrupt only
// acknowledges and masks the source, then queues deferred draining.
void PollEvents();
};