feat: vblank IRQ and double-buffered page flip for Intel GPU
Add tear-free scanout to the Intel GPU driver: a second kernel-allocated scanout buffer, DSPASURF flips latched at vblank, and a vblank interrupt delivered over MSI (Gen 11+ master/display/pipe IRQ chain) with a monotonic vblank counter and WaitVblank(). Expose it as SYS_FBFLIP (150): index selects the front buffer, -1 queries support, flags bit0 waits for the flip to latch. fb_map() now maps buffer 1 right after buffer 0 when flipping is available, and gui::Framebuffer draws to the off-screen buffer and flips with vsync, falling back to the direct copy when unsupported. Scanout is restored to buffer 0 when the flip-owning process exits and on panic, so the terminal and panic box never land on the invisible buffer. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
danielandClaude Fable 5 committed
1 parent
500020ce47
commit
182520a585
14 files changed
+549
-3
No files matched your search
@@ -15,6 +15,9 @@
|
||||
#include <Memory/PageFrameAllocator.hpp>
|
||||
#include <Io/IoPort.hpp>
|
||||
#include <Graphics/Framebuffer.hpp>
|
||||
#include <Hal/Apic/Interrupts.hpp>
|
||||
#include <Sched/Scheduler.hpp>
|
||||
#include <Libraries/Memory.hpp>
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
@@ -45,6 +48,19 @@ namespace Drivers::Graphics::IntelGPU {
|
||||
static uint64_t g_fbSize = 0; // Total framebuffer size in bytes
|
||||
static uint64_t g_fbGttOffset = 0; // GTT offset where FB starts (in bytes)
|
||||
|
||||
// Page flip state
|
||||
static volatile void* g_ggtt = nullptr; // GGTT at the gen-correct BAR0 offset
|
||||
static uint64_t g_ggttEntries = 0;
|
||||
static uint64_t g_fbGttOffsetA = 0; // GGTT offset of buffer 0 (firmware FB)
|
||||
static uint32_t* g_buf1Virt = nullptr; // buffer 1 kernel VA (HHDM, WC)
|
||||
static uint64_t g_buf1Phys = 0;
|
||||
static uint64_t g_buf1GttOffset = 0; // GGTT offset of buffer 1 (in bytes)
|
||||
static bool g_flipSupported = false;
|
||||
static int g_frontBuffer = 0;
|
||||
static int g_flipOwnerPid = -1;
|
||||
static volatile uint64_t g_vblankCount = 0;
|
||||
static bool g_vblankIrqReady = false;
|
||||
|
||||
// =========================================================================
|
||||
// Register access helpers
|
||||
// =========================================================================
|
||||
@@ -481,6 +497,284 @@ namespace Drivers::Graphics::IntelGPU {
|
||||
<< ", stride=" << base::dec << g_fbPitch;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Page flipping: GGTT at the generation-correct offset
|
||||
// =========================================================================
|
||||
//
|
||||
// The GTT window lives in the upper half of GTTMMADR (BAR0):
|
||||
// Gen 6-7: 4 MB BAR, GTT at +2 MB, 32-bit entries
|
||||
// Gen 8+: 16 MB BAR, GGTT at +8 MB, 64-bit entries
|
||||
// The legacy init path above writes at +2 MB for all generations; on
|
||||
// Gen 8+ those writes land in register space, and scanout keeps working
|
||||
// only because the firmware's real GGTT entries (stored in stolen RAM,
|
||||
// which survives S3 in self-refresh) are never actually touched. Page
|
||||
// flipping needs entries the display engine really reads, so it uses the
|
||||
// correct window and refuses to run unless the firmware's own scanout
|
||||
// PTEs are visible there (valid bits set). On any failure page flipping
|
||||
// stays off and the driver behaves exactly as before.
|
||||
|
||||
static void MicroDelay(int us) {
|
||||
// Simple busy-wait; us is approximate
|
||||
for (volatile int i = 0; i < us * 100; i++) {
|
||||
asm volatile("pause");
|
||||
}
|
||||
}
|
||||
|
||||
static uint64_t ReadGgttPte(uint64_t index) {
|
||||
if (g_gpuGen >= 8) return ((volatile uint64_t*)g_ggtt)[index];
|
||||
return ((volatile uint32_t*)g_ggtt)[index];
|
||||
}
|
||||
|
||||
static void WriteGgttPte(uint64_t index, uint64_t physAddr) {
|
||||
if (g_gpuGen >= 8) {
|
||||
((volatile uint64_t*)g_ggtt)[index] = MakeGttPte64(physAddr);
|
||||
} else {
|
||||
((volatile uint32_t*)g_ggtt)[index] = MakeGttPte32(physAddr);
|
||||
}
|
||||
}
|
||||
|
||||
static bool MapCorrectGgtt() {
|
||||
if (g_gpuGen < 8) {
|
||||
// The legacy mapping at BAR0 + 2 MB is already the real GTT
|
||||
g_ggtt = g_gttBase;
|
||||
g_ggttEntries = g_gttEntryCount;
|
||||
return g_ggtt != nullptr;
|
||||
}
|
||||
|
||||
// Gen 8+: GGC.GGMS moved to bits 7:6 (0 = none, 1/2/3 = 2/4/8 MB)
|
||||
uint16_t gmchCtl = Pci::LegacyRead16(g_gpuInfo.pciBus, g_gpuInfo.pciDevice,
|
||||
g_gpuInfo.pciFunction, (uint8_t)PCI_REG_GMCH_CTL);
|
||||
uint8_t ggms = (gmchCtl >> 6) & 0x3;
|
||||
if (ggms == 0) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "GGC reports no GGTT (GGC="
|
||||
<< base::hex << (uint64_t)gmchCtl << "), page flip unavailable";
|
||||
return false;
|
||||
}
|
||||
uint64_t ggttBytes = (1ull << ggms) * 1024 * 1024;
|
||||
uint64_t ggttPhys = g_gpuInfo.mmioPhys + 8 * 1024 * 1024;
|
||||
|
||||
for (uint64_t off = 0; off < ggttBytes; off += 0x1000) {
|
||||
Memory::VMM::g_paging->MapMMIO(ggttPhys + off, Memory::HHDM(ggttPhys + off));
|
||||
}
|
||||
|
||||
g_ggtt = (volatile void*)Memory::HHDM(ggttPhys);
|
||||
g_ggttEntries = ggttBytes / sizeof(uint64_t);
|
||||
|
||||
KernelLogStream(INFO, "IntelGPU") << "GGTT window at BAR0+8MB (phys "
|
||||
<< base::hex << ggttPhys << "), " << base::dec << (ggttBytes / 1024)
|
||||
<< " KB, " << g_ggttEntries << " entries";
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool VerifyFirmwareScanoutPtes() {
|
||||
// The active scanout surface must be backed by valid PTEs in the
|
||||
// window we mapped; if not, our idea of where the GGTT lives is wrong
|
||||
// and writing to it could corrupt arbitrary state.
|
||||
g_fbGttOffsetA = ReadReg(DSPASURF) & ~0xFFFull;
|
||||
uint64_t firstIdx = g_fbGttOffsetA >> 12;
|
||||
uint64_t lastIdx = (g_fbGttOffsetA + g_fbSize - 1) >> 12;
|
||||
|
||||
if (lastIdx >= g_ggttEntries) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "Scanout range exceeds GGTT ("
|
||||
<< base::dec << lastIdx << " >= " << g_ggttEntries << ")";
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t checks[3] = { firstIdx, (firstIdx + lastIdx) / 2, lastIdx };
|
||||
for (int i = 0; i < 3; i++) {
|
||||
uint64_t pte = ReadGgttPte(checks[i]);
|
||||
if (!(pte & 1)) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "Scanout PTE " << base::dec
|
||||
<< checks[i] << " invalid (" << base::hex << pte
|
||||
<< ") - GGTT window not where expected, page flip unavailable";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
KernelLogStream(OK, "IntelGPU") << "Firmware scanout PTEs verified (surface at GGTT+"
|
||||
<< base::hex << g_fbGttOffsetA << ", PTE[" << base::dec << firstIdx
|
||||
<< "]=" << base::hex << ReadGgttPte(firstIdx) << ")";
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool AllocateBackBuffer() {
|
||||
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
|
||||
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, pages);
|
||||
if (!virt) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "Failed to allocate " << base::dec
|
||||
<< pages << " contiguous pages for the second scanout buffer";
|
||||
return false;
|
||||
}
|
||||
g_buf1Phys = Memory::SubHHDM(virt);
|
||||
g_buf1Virt = (uint32_t*)virt;
|
||||
|
||||
// Display scanout does not snoop the CPU cache: remap the buffer
|
||||
// write-combining before touching it, same as the firmware FB.
|
||||
for (uint64_t i = 0; i < pages; i++) {
|
||||
Memory::VMM::g_paging->MapWC(g_buf1Phys + i * 0x1000,
|
||||
Memory::HHDM(g_buf1Phys + i * 0x1000));
|
||||
}
|
||||
Memory::VMM::FlushTLB();
|
||||
|
||||
memset(virt, 0, pages * 0x1000);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool FindAndMapGgttRange() {
|
||||
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
|
||||
constexpr uint64_t guard = 16;
|
||||
uint64_t need = pages + 2 * guard;
|
||||
|
||||
// Firmware does not leave unused GGTT entries invalid: it points the
|
||||
// ENTIRE table at a scratch page, so free entries are valid but all
|
||||
// hold one identical PTE value (HW-confirmed on Raptor Lake, where a
|
||||
// scan for invalid entries found none in 1M entries). Sample the tail
|
||||
// of the table to learn the scratch value; a non-uniform tail means
|
||||
// the layout is not what we expect, so bail.
|
||||
uint64_t scratchPte = ReadGgttPte(g_ggttEntries - 1);
|
||||
for (uint64_t probe = 2; probe <= 32; probe++) {
|
||||
if (ReadGgttPte(g_ggttEntries - probe) != scratchPte) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "GGTT tail not uniform (entry -"
|
||||
<< base::dec << probe << " != " << base::hex << scratchPte
|
||||
<< "), page flip unavailable";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Search the upper half for a run of scratch-backed (or invalid)
|
||||
// entries. Never overlap the active scanout range.
|
||||
uint64_t avoidFirst = g_fbGttOffsetA >> 12;
|
||||
uint64_t avoidLast = (g_fbGttOffsetA + g_fbSize - 1) >> 12;
|
||||
|
||||
uint64_t runStart = 0, run = 0;
|
||||
bool found = false;
|
||||
for (uint64_t idx = g_ggttEntries / 2; idx < g_ggttEntries; idx++) {
|
||||
if (idx >= avoidFirst && idx <= avoidLast) { run = 0; continue; }
|
||||
uint64_t pte = ReadGgttPte(idx);
|
||||
if ((pte & 1) && pte != scratchPte) { run = 0; continue; }
|
||||
if (run == 0) runStart = idx;
|
||||
if (++run >= need) { found = true; break; }
|
||||
}
|
||||
if (!found) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "No free GGTT run of " << base::dec
|
||||
<< need << " entries in upper half, page flip unavailable";
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t base = runStart + guard;
|
||||
for (uint64_t i = 0; i < pages; i++) {
|
||||
WriteGgttPte(base + i, g_buf1Phys + i * 0x1000);
|
||||
}
|
||||
|
||||
// Read back first and last entry to confirm the writes stuck
|
||||
uint64_t expectFirst = (g_gpuGen >= 8) ? MakeGttPte64(g_buf1Phys)
|
||||
: MakeGttPte32(g_buf1Phys);
|
||||
uint64_t lastPhys = g_buf1Phys + (pages - 1) * 0x1000;
|
||||
uint64_t expectLast = (g_gpuGen >= 8) ? MakeGttPte64(lastPhys)
|
||||
: MakeGttPte32(lastPhys);
|
||||
if (ReadGgttPte(base) != expectFirst || ReadGgttPte(base + pages - 1) != expectLast) {
|
||||
KernelLogStream(WARNING, "IntelGPU") << "GGTT read-back mismatch at entry "
|
||||
<< base::dec << base << ", page flip unavailable";
|
||||
return false;
|
||||
}
|
||||
|
||||
g_buf1GttOffset = base << 12;
|
||||
KernelLogStream(OK, "IntelGPU") << "Buffer 1 mapped at GGTT+" << base::hex
|
||||
<< g_buf1GttOffset << " (" << base::dec << pages << " pages, phys "
|
||||
<< base::hex << g_buf1Phys << ")";
|
||||
return true;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Page flipping: vblank interrupt (Gen 11+)
|
||||
// =========================================================================
|
||||
|
||||
static void HandleInterrupt(uint8_t /*irq*/) {
|
||||
// Gen 11+ flow: park the master enable, ack the pipe IIR, re-enable
|
||||
uint32_t master = ReadReg(GFX_MSTR_INTR);
|
||||
if (!(master & GFX_MSTR_INTR_DISPLAY)) {
|
||||
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
|
||||
return;
|
||||
}
|
||||
WriteReg(GFX_MSTR_INTR, 0);
|
||||
|
||||
uint32_t iir = ReadReg(DE_PIPE_A_IIR);
|
||||
if (iir) WriteReg(DE_PIPE_A_IIR, iir); // write-1-to-clear
|
||||
|
||||
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
|
||||
|
||||
if (iir & DE_PIPE_VBLANK) {
|
||||
g_vblankCount = g_vblankCount + 1;
|
||||
Sched::WakeObjectWaiters((void*)&g_vblankCount);
|
||||
}
|
||||
}
|
||||
|
||||
static bool SetupMsi() {
|
||||
uint8_t bus = g_gpuInfo.pciBus;
|
||||
uint8_t dev = g_gpuInfo.pciDevice;
|
||||
uint8_t func = g_gpuInfo.pciFunction;
|
||||
|
||||
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
|
||||
if (cap == 0) {
|
||||
KernelLogStream(INFO, "IntelGPU") << "MSI capability not found";
|
||||
return false;
|
||||
}
|
||||
|
||||
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
|
||||
bool is64bit = (msgCtrl & (1 << 7)) != 0;
|
||||
|
||||
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
|
||||
if (is64bit) {
|
||||
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
|
||||
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
|
||||
} else {
|
||||
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
|
||||
}
|
||||
|
||||
msgCtrl &= ~(0x70); // one message
|
||||
msgCtrl |= (1 << 0); // MSI enable
|
||||
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
|
||||
|
||||
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
|
||||
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
|
||||
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
|
||||
|
||||
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
|
||||
|
||||
KernelLogStream(OK, "IntelGPU") << "MSI enabled: vector " << base::dec
|
||||
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"
|
||||
<< (is64bit ? " [64-bit]" : " [32-bit]");
|
||||
return true;
|
||||
}
|
||||
|
||||
static void EnableVblankIrq() {
|
||||
WriteReg(DE_PIPE_A_IIR, 0xFFFFFFFFu); // clear stale events
|
||||
WriteReg(DE_PIPE_A_IMR, ~DE_PIPE_VBLANK); // unmask vblank only
|
||||
WriteReg(DE_PIPE_A_IER, DE_PIPE_VBLANK);
|
||||
WriteReg(DISPLAY_INT_CTL, DISPLAY_IRQ_ENABLE);
|
||||
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
|
||||
(void)ReadReg(GFX_MSTR_INTR);
|
||||
}
|
||||
|
||||
static void SetupPageFlip() {
|
||||
if (!MapCorrectGgtt()) return;
|
||||
if (!VerifyFirmwareScanoutPtes()) return;
|
||||
if (!AllocateBackBuffer()) return;
|
||||
if (!FindAndMapGgttRange()) return;
|
||||
|
||||
// Vblank interrupts: Gen 11+ register layout only. Older generations
|
||||
// still flip, with DSPASURFLIVE polling standing in for the IRQ.
|
||||
if (g_gpuGen >= 11 && SetupMsi()) {
|
||||
EnableVblankIrq();
|
||||
g_vblankIrqReady = true;
|
||||
}
|
||||
|
||||
g_flipSupported = true;
|
||||
g_frontBuffer = 0;
|
||||
KernelLogStream(OK, "IntelGPU") << "Page flip ready: 2 buffers, vblank IRQ "
|
||||
<< (g_vblankIrqReady ? "on" : "off (SURFLIVE polling)");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Public API
|
||||
// =========================================================================
|
||||
@@ -570,6 +864,8 @@ namespace Drivers::Graphics::IntelGPU {
|
||||
ProgramDisplayPlane();
|
||||
g_initialized = true;
|
||||
|
||||
SetupPageFlip();
|
||||
|
||||
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
|
||||
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
|
||||
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
|
||||
@@ -680,6 +976,78 @@ namespace Drivers::Graphics::IntelGPU {
|
||||
return g_fbPitch;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Page flip public API
|
||||
// =========================================================================
|
||||
|
||||
bool FlipSupported() {
|
||||
return g_flipSupported;
|
||||
}
|
||||
|
||||
uint64_t GetBufferPhys(int index) {
|
||||
if (index == 0) return g_fbPhysBase;
|
||||
if (index == 1 && g_flipSupported) return g_buf1Phys;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int GetFrontBuffer() {
|
||||
return g_frontBuffer;
|
||||
}
|
||||
|
||||
uint64_t GetVblankCount() {
|
||||
return g_vblankCount;
|
||||
}
|
||||
|
||||
bool WaitVblank(uint64_t timeoutMs) {
|
||||
if (!g_vblankIrqReady) return false;
|
||||
uint64_t start = g_vblankCount;
|
||||
// The vblank IRQ fires every frame, so a wakeup lost to the race
|
||||
// between reading the counter and blocking only costs one frame;
|
||||
// the timeout is a backstop for a stalled pipe.
|
||||
Sched::BlockOnObject((void*)&g_vblankCount, timeoutMs ? timeoutMs : 50);
|
||||
return g_vblankCount != start;
|
||||
}
|
||||
|
||||
int64_t Flip(int index, bool waitVsync) {
|
||||
if (!g_flipSupported || index < 0 || index > 1) return -1;
|
||||
|
||||
uint64_t gttOff = (index == 0) ? g_fbGttOffsetA : g_buf1GttOffset;
|
||||
WriteReg(DSPASURF, (uint32_t)gttOff);
|
||||
(void)ReadReg(DSPASURF);
|
||||
g_frontBuffer = index;
|
||||
|
||||
if (waitVsync) {
|
||||
if (g_vblankIrqReady) {
|
||||
WaitVblank(50);
|
||||
}
|
||||
// Confirm the latch: DSPASURFLIVE tracks the surface actually
|
||||
// being scanned out. With a working IRQ this succeeds on the
|
||||
// first read; without one it is the vsync wait itself.
|
||||
for (int i = 0; i < 3000; i++) {
|
||||
if ((ReadReg(DSPASURFLIVE) & ~0xFFFu) == (uint32_t)gttOff) break;
|
||||
MicroDelay(10);
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
void SetFlipOwner(int pid) {
|
||||
g_flipOwnerPid = pid;
|
||||
}
|
||||
|
||||
void OnProcessExit(int pid) {
|
||||
if (!g_flipSupported || pid != g_flipOwnerPid) return;
|
||||
g_flipOwnerPid = -1;
|
||||
if (g_frontBuffer != 0) Flip(0, false);
|
||||
}
|
||||
|
||||
void PanicRestoreScanout() {
|
||||
if (!g_flipSupported || !g_mmioBase || g_frontBuffer == 0) return;
|
||||
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
|
||||
(void)ReadReg(DSPASURF);
|
||||
g_frontBuffer = 0;
|
||||
}
|
||||
|
||||
void Reinitialize() {
|
||||
if (!g_initialized || !g_mmioBase) return;
|
||||
|
||||
@@ -760,6 +1128,31 @@ namespace Drivers::Graphics::IntelGPU {
|
||||
// 5. Reprogram display plane to point at our GTT-mapped framebuffer
|
||||
ProgramDisplayPlane();
|
||||
|
||||
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries, bring
|
||||
// MSI + vblank interrupts back (the PCI config reset cleared MSI),
|
||||
// and re-show whichever buffer was front when we suspended.
|
||||
if (g_flipSupported) {
|
||||
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
|
||||
uint64_t baseIdx = g_buf1GttOffset >> 12;
|
||||
for (uint64_t i = 0; i < pages; i++) {
|
||||
WriteGgttPte(baseIdx + i, g_buf1Phys + i * 0x1000);
|
||||
}
|
||||
(void)ReadGgttPte(baseIdx + pages - 1);
|
||||
|
||||
if (g_vblankIrqReady) {
|
||||
SetupMsi();
|
||||
EnableVblankIrq();
|
||||
}
|
||||
|
||||
if (g_frontBuffer != 0) {
|
||||
WriteReg(DSPASURF, (uint32_t)g_buf1GttOffset);
|
||||
(void)ReadReg(DSPASURF);
|
||||
}
|
||||
|
||||
KernelLogStream(DEBUG, "IntelGPU") << "Page flip state restored (front="
|
||||
<< base::dec << (uint64_t)g_frontBuffer << ")";
|
||||
}
|
||||
|
||||
KernelLogStream(OK, "IntelGPU") << "Display restored after S3 resume";
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user