daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kernel concurrency, interrupt context, and user VA safety

Browse Source
This commit is contained in:
daniel committed 2026-08-01 15:43:38 +02:00
1 parent f222bf1f04
commit b1c55073c7
71 files changed
+2654 -1092
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernel/src/Drivers/Graphics/IntelGPU.cpp
+273 -180
View File
@@ -10,14 +10,14 @@
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Io/IoPort.hpp>
#include <Graphics/Framebuffer.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Sched/Scheduler.hpp>
#include <Libraries/Memory.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
@@ -36,7 +36,6 @@ namespace Drivers::Graphics::IntelGPU {
// GTT state
static volatile void* g_gttBase = nullptr; // Virtual address of GTT entries
static uint64_t g_gttEntryCount = 0; // Number of GTT entries
static uint64_t g_scratchPagePhys = 0; // Physical address of scratch page
static uint8_t g_gpuGen = 0; // Cached generation number
// Framebuffer state
@@ -54,12 +53,15 @@ namespace Drivers::Graphics::IntelGPU {
static uint64_t g_fbGttOffsetA = 0; // GGTT offset of buffer 0 (firmware FB)
static uint32_t* g_buf1Virt = nullptr; // buffer 1 kernel VA (HHDM, WC)
static uint64_t g_buf1Phys = 0;
static uint64_t g_buf1Pages = 0; // allocation/mapping bound
static uint64_t g_buf1GttOffset = 0; // GGTT offset of buffer 1 (in bytes)
static bool g_flipSupported = false;
static int g_frontBuffer = 0;
static int g_flipOwnerPid = -1;
static volatile uint64_t g_vblankCount = 0;
static bool g_vblankIrqReady = false;
// Serializes surface programming, flip-owner teardown, and timing changes.
// Without this, a process exit or modeset on another CPU can rewrite
// DSPASURF/PIPE state in the middle of a desktop flip transaction.
static kcp::Mutex g_flipLock;
// Modesetting/display-management state. The firmware-trained port and
// link are deliberately retained; the driver owns the active transcoder,
@@ -216,6 +218,21 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// This driver intentionally uses register polling for display latches and
// does not own the GPU's many generation-specific interrupt sources. Mask
// PCI delivery before touching display state: otherwise firmware-left MSI
// enables can route an unhandled GT interrupt and create an interrupt
// storm even though Montauk never requested one.
static void DisableInterruptDelivery() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
if (Pci::DisableInterruptDelivery(bus, dev, func)) {
KernelLogStream(WARNING, "IntelGPU")
<< "Disabled pre-enabled GPU MSI/MSI-X; using bounded latch polling";
}
}
// =========================================================================
// BAR0 MMIO Mapping
// =========================================================================
@@ -357,12 +374,20 @@ namespace Drivers::Graphics::IntelGPU {
return false;
}
// Ensure pitch is at least width * 4 (BGRX8888)
// Ensure pitch is at least width * 4 (BGRX8888), and reject geometry
// whose size arithmetic could wrap before it reaches the allocator.
if (g_fbPitch == 0) {
g_fbPitch = g_fbWidth * 4;
KernelLogStream(WARNING, "IntelGPU") << "Stride not available, assuming "
<< base::dec << g_fbPitch << " bytes";
}
if (g_fbWidth > (~0ULL / 4)
|| g_fbPitch < g_fbWidth * 4
|| g_fbHeight > (~0ULL / g_fbPitch)) {
KernelLogStream(ERROR, "IntelGPU")
<< "Invalid or overflowing framebuffer geometry";
return false;
}
g_fbSize = g_fbHeight * g_fbPitch;
@@ -989,17 +1014,13 @@ namespace Drivers::Graphics::IntelGPU {
<< ", " << base::dec << g_gttEntryCount << " entries"
<< (g_gpuGen >= 8 ? " (64-bit PTEs)" : " (32-bit PTEs)");
// Allocate a scratch page (zeroed) for future use
void* scratchPageVirt = Memory::g_pfa->AllocateZeroed();
g_scratchPagePhys = Memory::SubHHDM(scratchPageVirt);
// Do NOT clear the entire GTT here. The firmware has active GTT mappings
// that the display engine is currently scanning out from. Clearing them
// would cause the display to go black (or worse) before we remap.
// Instead, we only write the entries we need in SetupFramebuffer().
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec << g_gttEntryCount
<< " entries, scratch page at " << base::hex << g_scratchPagePhys;
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec
<< g_gttEntryCount << " entries";
return true;
}
@@ -1108,18 +1129,47 @@ namespace Drivers::Graphics::IntelGPU {
// Initialization maps this window once and preserves the firmware's active
// scanout PTEs. Page flipping only proceeds after those entries validate.
static void MicroDelay(int us) {
// Simple busy-wait; us is approximate
for (volatile int i = 0; i < us * 100; i++) {
asm volatile("pause");
}
}
static uint64_t ReadGgttPte(uint64_t index) {
if (g_gpuGen >= 8) return ((volatile uint64_t*)g_ggtt)[index];
return ((volatile uint32_t*)g_ggtt)[index];
}
static int ReadLiveBufferIndex() {
if (!g_mmioBase) return -1;
uint32_t live = ReadReg(DSPASURFLIVE) & ~0xFFFu;
if (live == (uint32_t)g_fbGttOffsetA) return 0;
if (live == (uint32_t)g_buf1GttOffset) return 1;
return -1;
}
// Polling-only latch wait. This is safe during both ordinary syscalls and
// ExitProcess teardown because it never enters the scheduler. Keeping
// flips independent of GPU MSI is deliberate: MSI would route every
// firmware-enabled GPU source, not just the one source this driver owns.
// Three observed frames are the normal timeout. Laptop panel self-refresh
// can transiently stop the scanline counter, so also enforce a wall-clock
// bound: one million uncached MMIO reads can otherwise hold the desktop in
// SYS_FBFLIP long enough to look permanently frozen (cursor included).
static bool WaitForLiveBuffer(int index) {
uint32_t expected = (uint32_t)((index == 0) ? g_fbGttOffsetA
: g_buf1GttOffset);
uint32_t previous = ReadReg(PIPE_DSL) & 0x1FFFu;
uint64_t started = Timekeeping::GetMilliseconds();
int frameWraps = 0;
for (int i = 0; i < 200000; i++) {
uint32_t live = ReadReg(DSPASURFLIVE);
if (live == 0xFFFFFFFFu) return false;
if ((live & ~0xFFFu) == expected) return true;
uint32_t current = ReadReg(PIPE_DSL) & 0x1FFFu;
if (current < previous && ++frameWraps >= 3) return false;
previous = current;
if ((i & 0x3FF) == 0 &&
Timekeeping::GetMilliseconds() - started >= 100) return false;
asm volatile("pause");
}
return false;
}
static void WriteGgttPte(uint64_t index, uint64_t physAddr) {
if (g_gpuGen >= 8) {
((volatile uint64_t*)g_ggtt)[index] = MakeGttPte64(physAddr);
@@ -1168,14 +1218,37 @@ namespace Drivers::Graphics::IntelGPU {
static bool AllocateBackBuffer() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, pages);
if (pages == 0 || pages > 0x7FFFFFFFULL) {
KernelLogStream(WARNING, "IntelGPU")
<< "Invalid second scanout buffer page count " << base::dec << pages;
return false;
}
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, (int)pages);
if (!virt) {
KernelLogStream(WARNING, "IntelGPU") << "Failed to allocate " << base::dec
<< pages << " contiguous pages for the second scanout buffer";
return false;
}
g_buf1Phys = Memory::SubHHDM(virt);
uint64_t phys = Memory::SubHHDM(virt);
if ((pages - 1) > (UINT64_MAX - phys) / 0x1000ULL) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer physical range overflows";
return false;
}
uint64_t lastPhys = phys + (pages - 1) * 0x1000ULL;
if (!GgttCanAddress(phys, g_gpuGen) || !GgttCanAddress(lastPhys, g_gpuGen)) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer lies outside the GPU physical-address width ("
<< base::hex << phys << ".." << lastPhys << ")";
return false;
}
g_buf1Phys = phys;
g_buf1Virt = (uint32_t*)virt;
g_buf1Pages = pages;
// Display scanout does not snoop the CPU cache: remap the buffer
// write-combining before touching it, same as the firmware FB.
@@ -1192,23 +1265,50 @@ namespace Drivers::Graphics::IntelGPU {
static bool FindAndMapGgttRange() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
constexpr uint64_t guard = 16;
uint64_t need = pages + 2 * guard;
uint64_t avoidFirst = g_fbGttOffsetA >> 12;
uint64_t avoidLast = (g_fbGttOffsetA + g_fbSize - 1) >> 12;
// Montauk owns the GPU after firmware handoff. Reserve a deterministic
// high-aperture range for its second scanout buffer instead of trying
// to infer firmware ownership from scratch-PTE patterns. Keep guards
// around the allocation and never overlap the live firmware surface.
uint64_t base = g_ggttEntries / 2 + guard;
if (base <= avoidLast + guard) base = avoidLast + guard + 1;
base = (base + guard - 1) & ~(guard - 1);
if (base < guard || base + pages + guard > g_ggttEntries
|| !(base + pages - 1 < avoidFirst || base > avoidLast)) {
KernelLogStream(WARNING, "IntelGPU") << "No safe owned GGTT range for "
// Firmware normally fills unused GGTT entries with one scratch-page
// PTE. Verify that invariant at the tail, then claim only a complete
// scratch-backed (or invalid) run. Never overwrite arbitrary valid
// mappings merely because they happen to sit in the upper half.
if (g_ggttEntries < 32 || need > g_ggttEntries / 2) return false;
uint64_t scratchPte = ReadGgttPte(g_ggttEntries - 1);
for (uint64_t probe = 2; probe <= 32; probe++) {
if (ReadGgttPte(g_ggttEntries - probe) != scratchPte) {
KernelLogStream(WARNING, "IntelGPU")
<< "GGTT tail is owned/non-uniform; page flip unavailable";
return false;
}
}
uint64_t runStart = 0;
uint64_t run = 0;
bool found = false;
for (uint64_t idx = g_ggttEntries / 2; idx < g_ggttEntries; idx++) {
if (idx >= avoidFirst && idx <= avoidLast) {
run = 0;
continue;
}
uint64_t pte = ReadGgttPte(idx);
if ((pte & 1) && pte != scratchPte) {
run = 0;
continue;
}
if (run == 0) runStart = idx;
if (++run >= need) {
found = true;
break;
}
}
if (!found) {
KernelLogStream(WARNING, "IntelGPU") << "No verified free GGTT run for "
<< base::dec << pages << " back-buffer pages";
return false;
}
uint64_t base = runStart + guard;
uint64_t replacedFirst = ReadGgttPte(base);
uint64_t replacedLast = ReadGgttPte(base + pages - 1);
for (uint64_t i = 0; i < pages; i++) {
@@ -1235,94 +1335,38 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// =========================================================================
// Page flipping: vblank interrupt (Gen 11+)
// =========================================================================
static void HandleInterrupt(uint8_t /*irq*/) {
// Gen 11+ flow: park the master enable, ack the pipe IIR, re-enable
uint32_t master = ReadReg(GFX_MSTR_INTR);
if (!(master & GFX_MSTR_INTR_DISPLAY)) {
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
return;
}
WriteReg(GFX_MSTR_INTR, 0);
uint32_t iir = ReadReg(DE_PIPE_A_IIR);
if (iir) WriteReg(DE_PIPE_A_IIR, iir); // write-1-to-clear
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
if (iir & DE_PIPE_VBLANK) {
g_vblankCount = g_vblankCount + 1;
Sched::WakeObjectWaiters((void*)&g_vblankCount);
}
}
static bool SetupMsi() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "IntelGPU") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // one message
msgCtrl |= (1 << 0); // MSI enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "IntelGPU") << "MSI enabled: vector " << base::dec
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"
<< (is64bit ? " [64-bit]" : " [32-bit]");
return true;
}
static void EnableVblankIrq() {
WriteReg(DE_PIPE_A_IIR, 0xFFFFFFFFu); // clear stale events
WriteReg(DE_PIPE_A_IMR, ~DE_PIPE_VBLANK); // unmask vblank only
WriteReg(DE_PIPE_A_IER, DE_PIPE_VBLANK);
WriteReg(DISPLAY_INT_CTL, DISPLAY_IRQ_ENABLE);
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
(void)ReadReg(GFX_MSTR_INTR);
}
static void SetupPageFlip() {
if (!MapCorrectGgtt()) return;
if (!VerifyFirmwareScanoutPtes()) return;
if (!AllocateBackBuffer()) return;
if (!FindAndMapGgttRange()) return;
// Vblank interrupts: Gen 11+ register layout only. Older generations
// still flip, with DSPASURFLIVE polling standing in for the IRQ.
if (g_gpuGen >= 11 && SetupMsi()) {
EnableVblankIrq();
g_vblankIrqReady = true;
}
g_flipSupported = true;
g_frontBuffer = 0;
KernelLogStream(OK, "IntelGPU") << "Page flip ready: 2 buffers, vblank IRQ "
<< (g_vblankIrqReady ? "on" : "off (SURFLIVE polling)");
int live = ReadLiveBufferIndex();
g_frontBuffer = (live >= 0) ? live : 0;
KernelLogStream(OK, "IntelGPU")
<< "Page flip ready: 2 buffers, bounded SURFLIVE polling";
}
static bool FirmwareFramebufferMatchesGpu() {
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
bool match = g_fbWidth == fwWidth
&& g_fbHeight == fwHeight
&& g_fbPitch == fwPitch;
if (match) return true;
// Userspace obtains its drawing dimensions from the firmware
// framebuffer API. A differently-sized GPU allocation would make the
// second mapping overrun (or alias only part of) the backing buffer.
KernelLogStream(ERROR, "IntelGPU")
<< "GPU dimensions differ from firmware; unsafe framebuffer setup skipped";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
return false;
}
// =========================================================================
@@ -1388,6 +1432,7 @@ namespace Drivers::Graphics::IntelGPU {
}
g_gpuGen = g_gpuInfo.gen;
DisableInterruptDelivery();
if (!MapMmio()) {
KernelLogStream(ERROR, "IntelGPU") << "Failed to map MMIO region";
@@ -1400,6 +1445,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return false;
}
if (!FirmwareFramebufferMatchesGpu()) return false;
InitializeDisplayManagement();
if (!InitializeGtt()) {
@@ -1417,18 +1463,6 @@ namespace Drivers::Graphics::IntelGPU {
SetupPageFlip();
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1446,6 +1480,7 @@ namespace Drivers::Graphics::IntelGPU {
if (!DetectGpu()) {
return;
}
DisableInterruptDelivery();
// Step 2: Map BAR0 MMIO region
if (!MapMmio()) {
@@ -1461,6 +1496,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return;
}
if (!FirmwareFramebufferMatchesGpu()) return;
InitializeDisplayManagement();
// Step 5: Initialize GTT
@@ -1480,21 +1516,6 @@ namespace Drivers::Graphics::IntelGPU {
g_initialized = true;
// Diagnostic: compare GPU-detected values with firmware/Limine values
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight
<< " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1641,7 +1662,6 @@ namespace Drivers::Graphics::IntelGPU {
if (g_edidValid) out->capabilities |= montauk::abi::DISPLAY_CAP_EDID;
if (g_pwmSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_BRIGHTNESS;
if (g_flipSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_PAGE_FLIP;
if (g_vblankIrqReady) out->capabilities |= montauk::abi::DISPLAY_CAP_VBLANK_IRQ;
for (int i = 0; i < g_modeCount; i++) {
if (i != g_currentMode
&& (g_modes[i].flags & montauk::abi::DISPLAY_MODE_DRIVER_VALID)) {
@@ -1685,6 +1705,8 @@ namespace Drivers::Graphics::IntelGPU {
return -2;
if (modeIndex == g_currentMode) return modeIndex;
g_flipLock.Acquire();
// Link rate and framebuffer geometry remain fixed. Only timings with
// the same active area and pixel clock are safe without retraining the
// firmware-established HDMI/DP/eDP link.
@@ -1693,10 +1715,13 @@ namespace Drivers::Graphics::IntelGPU {
if (requested.hdisplay != current.hdisplay
|| requested.vdisplay != current.vdisplay
|| requested.pixelClock == 0
|| requested.pixelClock != current.pixelClock)
|| requested.pixelClock != current.pixelClock) {
g_flipLock.Release();
return -2;
}
if (!ApplyModeTiming(requested)) {
g_flipLock.Release();
KernelLogStream(ERROR, "IntelGPU") << "Modeset transaction failed; previous mode restored";
return -3;
}
@@ -1704,6 +1729,7 @@ namespace Drivers::Graphics::IntelGPU {
g_modes[g_currentMode].flags &= ~montauk::abi::DISPLAY_MODE_CURRENT;
g_currentMode = modeIndex;
g_modes[g_currentMode].flags |= montauk::abi::DISPLAY_MODE_CURRENT;
g_flipLock.Release();
KernelLogStream(OK, "IntelGPU") << "Mode set to " << base::dec
<< (uint64_t)requested.hdisplay << "x" << (uint64_t)requested.vdisplay
<< " @ " << (uint64_t)(g_modes[modeIndex].refreshMilliHz / 1000) << "Hz";
@@ -1741,59 +1767,131 @@ namespace Drivers::Graphics::IntelGPU {
return 0;
}
uint64_t GetBufferPageCount(int index) {
if (index == 0) return (g_fbSize + 0xFFF) >> 12;
if (index == 1 && g_flipSupported) return g_buf1Pages;
return 0;
}
int GetFrontBuffer() {
return g_frontBuffer;
g_flipLock.Acquire();
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
return live;
}
uint64_t GetVblankCount() {
return g_vblankCount;
int AcquireFlip(int pid) {
if (!g_flipSupported || pid < 0) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
int live = ReadLiveBufferIndex();
if (live < 0) {
// Recover from an unrecognised/pending firmware surface by putting
// the legacy buffer back synchronously before userspace draws.
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
if (WaitForLiveBuffer(0)) live = 0;
}
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
KernelLogStream(DEBUG, "IntelGPU") << "Flip acquired by pid "
<< base::dec << (uint64_t)pid << " (live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live) << ")";
return live;
}
bool WaitVblank(uint64_t timeoutMs) {
if (!g_vblankIrqReady) return false;
uint64_t start = g_vblankCount;
// The vblank IRQ fires every frame, so a wakeup lost to the race
// between reading the counter and blocking only costs one frame;
// the timeout is a backstop for a stalled pipe.
Sched::BlockOnObject((void*)&g_vblankCount, timeoutMs ? timeoutMs : 50);
return g_vblankCount != start;
}
int64_t Flip(int index, bool waitVsync, int pid) {
if (!g_flipSupported || index < 0 || index > 1 || pid < 0) return -1;
int64_t Flip(int index, bool waitVsync) {
if (!g_flipSupported || index < 0 || index > 1) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
// Scanout memory is exposed through WC mappings. Drain userspace's
// completed frame before programming the display-plane surface.
asm volatile("sfence" ::: "memory");
uint64_t gttOff = (index == 0) ? g_fbGttOffsetA : g_buf1GttOffset;
WriteReg(DSPASURF, (uint32_t)gttOff);
(void)ReadReg(DSPASURF);
g_frontBuffer = index;
if (waitVsync) {
if (g_vblankIrqReady) {
WaitVblank(50);
}
// Confirm the latch: DSPASURFLIVE tracks the surface actually
// being scanned out. With a working IRQ this succeeds on the
// first read; without one it is the vsync wait itself.
for (int i = 0; i < 3000; i++) {
if ((ReadReg(DSPASURFLIVE) & ~0xFFFu) == (uint32_t)gttOff) break;
MicroDelay(10);
// DSPASURFLIVE is authoritative and avoids taking ownership of
// the GPU's shared interrupt domain merely to wait for vblank.
if (!WaitForLiveBuffer(index)) {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmed = ReadReg(DSPASURF);
uint32_t liveSurface = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
KernelLogStream(WARNING, "IntelGPU") << "Flip latch timeout: requested="
<< base::dec << (uint64_t)index << " live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live)
<< " DSPASURF=" << base::hex << (uint64_t)programmed
<< " DSPASURFLIVE=" << (uint64_t)liveSurface;
return -1;
}
g_frontBuffer = index;
} else {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
}
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
return index;
}
void SetFlipOwner(int pid) {
g_flipOwnerPid = pid;
}
void OnProcessExit(int pid) {
if (!g_flipSupported || pid != g_flipOwnerPid) return;
if (!g_flipSupported) return;
g_flipLock.Acquire();
if (pid != g_flipOwnerPid) {
g_flipLock.Release();
return;
}
g_flipOwnerPid = -1;
if (g_frontBuffer != 0) Flip(0, false);
uint32_t programmedBefore = ReadReg(DSPASURF);
uint32_t liveBefore = ReadReg(DSPASURFLIVE);
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
bool liveAtZero = WaitForLiveBuffer(0);
bool restored = liveAtZero;
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmedAfter = ReadReg(DSPASURF);
uint32_t liveAfter = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
KernelLogStream(restored ? INFO : ERROR, "IntelGPU")
<< "Flip owner pid " << base::dec << (uint64_t)pid
<< " exited; restore buffer 0 " << (restored ? "latched" : "timed out")
<< " (before surf/live=" << base::hex << (uint64_t)programmedBefore
<< "/" << (uint64_t)liveBefore << ", after="
<< (uint64_t)programmedAfter << "/"
<< (uint64_t)liveAfter << ")";
}
void PanicRestoreScanout() {
if (!g_flipSupported || !g_mmioBase || g_frontBuffer == 0) return;
// g_frontBuffer is only a software cache and can be stale precisely
// when the GPU or kernel is failing. Always request buffer 0 so the
// panic console cannot be left painting an invisible surface.
if (!g_flipSupported || !g_mmioBase) return;
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
g_frontBuffer = 0;
@@ -1809,6 +1907,7 @@ namespace Drivers::Graphics::IntelGPU {
// command register, so MMIO writes to the GPU are silently dropped
// until we re-enable these bits.
Pci::EnableBusMaster(g_gpuInfo.pciBus, g_gpuInfo.pciDevice, g_gpuInfo.pciFunction);
DisableInterruptDelivery();
// Verify PCI memory space is accessible by reading back command register
uint16_t pciCmd = Pci::LegacyRead16(g_gpuInfo.pciBus, g_gpuInfo.pciDevice,
@@ -1875,9 +1974,8 @@ namespace Drivers::Graphics::IntelGPU {
// 5. Reprogram display plane to point at our GTT-mapped framebuffer
ProgramDisplayPlane();
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries, bring
// MSI + vblank interrupts back (the PCI config reset cleared MSI),
// and re-show whichever buffer was front when we suspended.
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries and
// re-show whichever buffer was front when we suspended.
if (g_flipSupported) {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
uint64_t baseIdx = g_buf1GttOffset >> 12;
@@ -1886,11 +1984,6 @@ namespace Drivers::Graphics::IntelGPU {
}
(void)ReadGgttPte(baseIdx + pages - 1);
if (g_vblankIrqReady) {
SetupMsi();
EnableVblankIrq();
}
if (g_frontBuffer != 0) {
WriteReg(DSPASURF, (uint32_t)g_buf1GttOffset);
(void)ReadReg(DSPASURF);