From b1c55073c7d352d4759e48cc08e636960178d445 Mon Sep 17 00:00:00 2001 From: Daniel Hammer Date: Sat, 1 Aug 2026 15:43:38 +0200 Subject: [PATCH] fix: kernel concurrency, interrupt context, and user VA safety --- kernel/src/ACPI/AcpiEvents.cpp | 106 ++- kernel/src/ACPI/FADT.cpp | 23 + kernel/src/Api/BluetoothSyscall.hpp | 6 + kernel/src/Api/BuildNo.hpp | 2 +- kernel/src/Api/Common.hpp | 47 +- kernel/src/Api/Graphics.hpp | 48 +- kernel/src/Api/Heap.hpp | 67 +- kernel/src/Api/IoRedir.hpp | 12 +- kernel/src/Api/Keyboard.hpp | 21 +- kernel/src/Api/LibSyscall.hpp | 58 +- kernel/src/Api/Power.hpp | 13 + kernel/src/Api/Storage.hpp | 6 + kernel/src/Api/Terminal.hpp | 6 +- kernel/src/Common/Panic.cpp | 7 + kernel/src/CppLib/Spinlock.cpp | 21 +- kernel/src/CppLib/Spinlock.hpp | 10 +- kernel/src/Drivers/Audio/IntelHda.cpp | 155 +++- kernel/src/Drivers/Audio/IntelHda.hpp | 2 + kernel/src/Drivers/Audio/Mixer.cpp | 14 +- kernel/src/Drivers/Graphics/IntelGPU.cpp | 453 +++++++----- kernel/src/Drivers/Graphics/IntelGPU.hpp | 59 +- kernel/src/Drivers/Net/E1000.cpp | 130 ++-- kernel/src/Drivers/Net/E1000.hpp | 5 + kernel/src/Drivers/Net/E1000E.cpp | 118 +-- kernel/src/Drivers/Net/E1000E.hpp | 6 +- kernel/src/Drivers/PS2/Keyboard.cpp | 2 +- kernel/src/Drivers/PS2/Keyboard.hpp | 4 +- kernel/src/Drivers/PS2/Mouse.cpp | 2 +- kernel/src/Drivers/PS2/Mouse.hpp | 4 +- kernel/src/Drivers/Storage/Ahci.cpp | 87 +-- kernel/src/Drivers/Storage/Ahci.hpp | 5 - kernel/src/Drivers/Storage/Nvme.cpp | 71 +- kernel/src/Drivers/Storage/Nvme.hpp | 8 - kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp | 7 +- .../src/Drivers/USB/Bluetooth/Bluetooth.cpp | 3 +- kernel/src/Drivers/USB/Bluetooth/Hci.cpp | 22 +- kernel/src/Drivers/USB/HidMouse.cpp | 40 +- kernel/src/Drivers/USB/Xhci.cpp | 253 +++++-- kernel/src/Drivers/USB/Xhci.hpp | 15 +- kernel/src/Hal/Apic/Apic.cpp | 21 +- kernel/src/Hal/Apic/Apic.hpp | 3 +- kernel/src/Hal/Apic/Interrupts.cpp | 5 +- kernel/src/Hal/Apic/Interrupts.hpp | 6 +- kernel/src/Hal/Apic/IsrStubs.asm | 12 +- kernel/src/Hal/CpuPower.cpp | 20 +- kernel/src/Hal/CpuPower.hpp | 4 + kernel/src/Hal/IDT.cpp | 13 +- kernel/src/Hal/SmpBoot.cpp | 3 + kernel/src/Hal/SmpBoot.hpp | 16 + kernel/src/Hal/SyscallEntry.asm | 10 + kernel/src/Ipc/Ipc.cpp | 685 ++++++++++++------ kernel/src/Ipc/Ipc.hpp | 24 +- kernel/src/Memory/PageFrameAllocator.cpp | 28 +- kernel/src/Memory/Paging.cpp | 60 +- kernel/src/Net/Icmp.cpp | 7 - kernel/src/Net/Tcp.cpp | 24 +- kernel/src/Pci/Pci.cpp | 92 ++- kernel/src/Pci/Pci.hpp | 7 +- kernel/src/Sched/Scheduler.cpp | 433 +++++++++-- kernel/src/Sched/Scheduler.hpp | 42 +- kernel/src/Terminal/Terminal.cpp | 9 + kernel/src/Terminal/Terminal.hpp | 6 +- kernel/src/Timekeeping/ApicTimer.cpp | 82 ++- kernel/src/Timekeeping/ApicTimer.hpp | 3 + programs/include/gui/framebuffer.hpp | 84 ++- programs/include/montauk/syscall.h | 1 + programs/man/framebuffer.2 | 5 +- programs/src/display/main.cpp | 8 +- template/sysroot/include/Api/Syscall.hpp | 1 + template/sysroot/include/gui/framebuffer.hpp | 109 ++- template/sysroot/include/montauk/syscall.h | 5 + 71 files changed, 2654 insertions(+), 1092 deletions(-) diff --git a/kernel/src/ACPI/AcpiEvents.cpp b/kernel/src/ACPI/AcpiEvents.cpp index edd3cf8..bf09ffe 100644 --- a/kernel/src/ACPI/AcpiEvents.cpp +++ b/kernel/src/ACPI/AcpiEvents.cpp @@ -24,6 +24,10 @@ namespace Hal { static uint32_t g_pm1aEventBlock = 0; static uint32_t g_pm1bEventBlock = 0; static uint8_t g_pm1EventLength = 0; + static uint32_t g_gpe0Block = 0; + static uint32_t g_gpe1Block = 0; + static uint8_t g_gpe0Length = 0; + static uint8_t g_gpe1Length = 0; static uint16_t g_sciIrq = 0; static bool g_initialized = false; @@ -39,6 +43,18 @@ namespace Hal { return sts; } + static uint16_t ReadPM1Enable() { + uint16_t enableOffset = g_pm1EventLength / 2; + uint16_t enable = 0; + if (enableOffset == 0) return 0; + + if (g_pm1aEventBlock != 0) + enable |= Io::In16((uint16_t)(g_pm1aEventBlock + enableOffset)); + if (g_pm1bEventBlock != 0) + enable |= Io::In16((uint16_t)(g_pm1bEventBlock + enableOffset)); + return enable; + } + static void ClearPM1StatusBits(uint16_t bits) { // Write-1-to-clear semantics if (g_pm1aEventBlock != 0) @@ -59,23 +75,67 @@ namespace Hal { } } + // The kernel does not currently evaluate GPE control methods. Leaving + // firmware-enabled GPEs armed is therefore unsafe: an unacknowledged + // level-triggered SCI can continuously retrigger and starve the BSP. + static bool IsValidGpeIoBlock(uint32_t base, uint8_t length) { + return base != 0 && length >= 2 && (length & 1) == 0 && + base <= 0xFFFF && length <= 0x10000U - base; + } + + static void DisableAndClearGpeBlock(uint32_t base, uint8_t length) { + if (!IsValidGpeIoBlock(base, length)) return; + + uint16_t statusBytes = length / 2; + for (uint16_t i = 0; i < statusBytes; i++) + Io::Out8(0, (uint16_t)(base + statusBytes + i)); + for (uint16_t i = 0; i < statusBytes; i++) + Io::Out8(0xFF, (uint16_t)(base + i)); + } + + static void QuiesceActiveGpeBlock(uint32_t base, uint8_t length) { + if (!IsValidGpeIoBlock(base, length)) return; + + uint16_t statusBytes = length / 2; + for (uint16_t i = 0; i < statusBytes; i++) { + uint16_t statusPort = (uint16_t)(base + i); + uint16_t enablePort = (uint16_t)(base + statusBytes + i); + uint8_t enabled = Io::In8(enablePort); + uint8_t active = Io::In8(statusPort) & enabled; + if (active == 0) continue; + + // Disable the unsupported source before acknowledging it so a + // level source cannot immediately assert the SCI again. + Io::Out8(enabled & (uint8_t)~active, enablePort); + Io::Out8(active, statusPort); + } + } + + static void DisableAndClearGpes() { + DisableAndClearGpeBlock(g_gpe0Block, g_gpe0Length); + DisableAndClearGpeBlock(g_gpe1Block, g_gpe1Length); + } + + static void QuiesceActiveGpes() { + QuiesceActiveGpeBlock(g_gpe0Block, g_gpe0Length); + QuiesceActiveGpeBlock(g_gpe1Block, g_gpe1Length); + } + // ============================================================================ // SCI Interrupt Handler // ============================================================================ - static void SciHandler(uint8_t irq) { + static void SciHandler(uint8_t /*irq*/, bool) { uint16_t sts = ReadPM1Status(); + uint16_t activeFixedEvents = sts & ReadPM1Enable(); + if (activeFixedEvents != 0) + ClearPM1StatusBits(activeFixedEvents); - if (sts & AcpiSleep::PM1_PWRBTN_STS) { - ClearPM1StatusBits(AcpiSleep::PM1_PWRBTN_STS); - KernelLogStream(INFO, "ACPI") << "Power button pressed"; - } + // Firmware may re-enable a GPE after initialization. Since no GPE + // AML methods are dispatched yet, disable and acknowledge every + // active source rather than allowing an SCI interrupt storm. + QuiesceActiveGpes(); - if (sts & AcpiSleep::PM1_SLPBTN_STS) { - ClearPM1StatusBits(AcpiSleep::PM1_SLPBTN_STS); - KernelLogStream(INFO, "ACPI") << "Sleep button pressed"; - } - - // Note: EOI is sent by HalIrqDispatch after this handler returns. + // Note: HalIrqDispatch sends the LAPIC EOI before invoking handlers. } // ============================================================================ @@ -93,6 +153,10 @@ namespace Hal { g_pm1aEventBlock = fadt.PM1aEventBlock; g_pm1bEventBlock = fadt.PM1bEventBlock; g_pm1EventLength = fadt.PM1EventLength; + g_gpe0Block = fadt.GPE0Block; + g_gpe1Block = fadt.GPE1Block; + g_gpe0Length = fadt.GPE0Length; + g_gpe1Length = fadt.GPE1Length; g_sciIrq = fadt.SCI_Interrupt; if (g_pm1aEventBlock == 0) { @@ -100,6 +164,19 @@ namespace Hal { return; } + if (g_gpe0Length != 0 && !IsValidGpeIoBlock(g_gpe0Block, g_gpe0Length)) { + KernelLogStream(WARNING, "ACPI") + << "GPE0 block is not usable SystemIO (base=" << base::hex + << (uint64_t)g_gpe0Block << ", length=" << base::dec + << (uint64_t)g_gpe0Length << ")"; + } + if (g_gpe1Length != 0 && !IsValidGpeIoBlock(g_gpe1Block, g_gpe1Length)) { + KernelLogStream(WARNING, "ACPI") + << "GPE1 block is not usable SystemIO (base=" << base::hex + << (uint64_t)g_gpe1Block << ", length=" << base::dec + << (uint64_t)g_gpe1Length << ")"; + } + // Clear any pending status bits ClearPM1StatusBits( AcpiSleep::PM1_PWRBTN_STS | AcpiSleep::PM1_SLPBTN_STS | @@ -110,6 +187,10 @@ namespace Hal { // Enable power button event SetPM1Events(AcpiSleep::PM1_PWRBTN_EN); + // Disable unsupported firmware GPE sources before the level SCI is + // routed. Clear stale status only after the enables are down. + DisableAndClearGpes(); + // Route SCI to an IRQ vector. The SCI is level-triggered, // active-low (ACPI spec requirement). The MADT may have an // Interrupt Source Override for this IRQ that already sets @@ -145,6 +226,9 @@ namespace Hal { // Drop the suspend-time RTC wake enable and restore only the // fixed events we actually want during normal runtime. SetPM1Events(AcpiSleep::PM1_PWRBTN_EN); + + // Firmware and _WAK may have restored GPE enables while asleep. + DisableAndClearGpes(); } }; diff --git a/kernel/src/ACPI/FADT.cpp b/kernel/src/ACPI/FADT.cpp index dcaa3de..97dd57e 100644 --- a/kernel/src/ACPI/FADT.cpp +++ b/kernel/src/ACPI/FADT.cpp @@ -21,6 +21,21 @@ namespace Hal { return true; } + static uint32_t ExtendedSystemIoAddress(const Table* fadt, + const uint8_t gas[12], + size_t fieldOffset) { + // Generic Address Structure: byte 0 is AddressSpaceId and bytes + // 4..11 are the little-endian address. This event code uses x86 + // port I/O, so accept only SystemIO GAS entries that fit the + // architectural 16-bit port space. + if (fadt->Header.Length < fieldOffset + 12 || gas[0] != 1) return 0; + + uint64_t address = 0; + for (int i = 0; i < 8; i++) + address |= (uint64_t)gas[4 + i] << (i * 8); + return address <= 0xFFFF ? (uint32_t)address : 0; + } + static Table* FindFADTInXSDT(ACPI::CommonSDTHeader* xsdt) { uint32_t entryCount = (xsdt->Length - sizeof(ACPI::CommonSDTHeader)) / 8; uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(ACPI::CommonSDTHeader)); @@ -73,6 +88,14 @@ namespace Hal { result.PMTimerBlock = fadt->PMTimerBlock; result.GPE0Block = fadt->GPE0Block; result.GPE1Block = fadt->GPE1Block; + if (result.GPE0Block == 0) { + result.GPE0Block = ExtendedSystemIoAddress( + fadt, fadt->X_GPE0Block, offsetof(Table, X_GPE0Block)); + } + if (result.GPE1Block == 0) { + result.GPE1Block = ExtendedSystemIoAddress( + fadt, fadt->X_GPE1Block, offsetof(Table, X_GPE1Block)); + } result.SMI_CommandPort = fadt->SMI_CommandPort; result.SCI_Interrupt = fadt->SCI_Interrupt; result.PM1EventLength = fadt->PM1EventLength; diff --git a/kernel/src/Api/BluetoothSyscall.hpp b/kernel/src/Api/BluetoothSyscall.hpp index 79f2e37..bd7a385 100644 --- a/kernel/src/Api/BluetoothSyscall.hpp +++ b/kernel/src/Api/BluetoothSyscall.hpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include "Syscall.hpp" @@ -53,6 +56,9 @@ namespace montauk::abi { if (!bdAddr) return -1; if (!Drivers::USB::Bluetooth::IsInitialized()) return -1; + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() + << " entered Bluetooth disconnect stage"; return (int64_t)Drivers::USB::Bluetooth::Disconnect(bdAddr); } diff --git a/kernel/src/Api/BuildNo.hpp b/kernel/src/Api/BuildNo.hpp index 50f8da1..084f2b1 100644 --- a/kernel/src/Api/BuildNo.hpp +++ b/kernel/src/Api/BuildNo.hpp @@ -12,4 +12,4 @@ #pragma once -#define MONTAUK_BUILD_NUMBER 33 +#define MONTAUK_BUILD_NUMBER 37 diff --git a/kernel/src/Api/Common.hpp b/kernel/src/Api/Common.hpp index fb0dab3..a8a619d 100644 --- a/kernel/src/Api/Common.hpp +++ b/kernel/src/Api/Common.hpp @@ -14,59 +14,47 @@ namespace montauk::abi { } static bool ResolveProcessHandle(Sched::Process* proc, int handle, Ipc::HandleType expectedType, - Ipc::Object*& outObject, uint32_t* outRights = nullptr) { + Ipc::HandleSnapshot& snapshot) { if (proc == nullptr || handle < 0) return false; int slot = Ipc::SlotForPid(proc->pid); if (slot < 0) return false; - Ipc::HandleType type = Ipc::HandleType::None; - Ipc::Object* object = nullptr; - uint32_t rights = 0; - if (!Ipc::SnapshotHandleForSlot(slot, handle, type, object, rights)) return false; - if (type != expectedType || object == nullptr) return false; - - outObject = object; - if (outRights != nullptr) *outRights = rights; - return true; + if (!snapshot.Capture(slot, handle)) return false; + return snapshot.type == expectedType && snapshot.object != nullptr; } - static Ipc::Stream* GetRedirOutStream(Sched::Process* proc) { + static Ipc::Stream* GetRedirOutStream(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) { proc = GetRedirTarget(proc); if (proc == nullptr) return nullptr; - Ipc::Object* object = nullptr; - if (!ResolveProcessHandle(proc, proc->ioOutHandle, Ipc::HandleType::Stream, object)) return nullptr; - return (Ipc::Stream*)object; + if (!ResolveProcessHandle(proc, proc->ioOutHandle, Ipc::HandleType::Stream, snapshot)) return nullptr; + return (Ipc::Stream*)snapshot.object; } - static Ipc::Stream* GetRedirInStream(Sched::Process* proc) { + static Ipc::Stream* GetRedirInStream(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) { proc = GetRedirTarget(proc); if (proc == nullptr) return nullptr; - Ipc::Object* object = nullptr; - if (!ResolveProcessHandle(proc, proc->ioInHandle, Ipc::HandleType::Stream, object)) return nullptr; - return (Ipc::Stream*)object; + if (!ResolveProcessHandle(proc, proc->ioInHandle, Ipc::HandleType::Stream, snapshot)) return nullptr; + return (Ipc::Stream*)snapshot.object; } - static Ipc::Mailbox* GetRedirKeyMailbox(Sched::Process* proc) { + static Ipc::Mailbox* GetRedirKeyMailbox(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) { proc = GetRedirTarget(proc); if (proc == nullptr) return nullptr; - Ipc::Object* object = nullptr; - if (!ResolveProcessHandle(proc, proc->ioKeyHandle, Ipc::HandleType::Mailbox, object)) return nullptr; - return (Ipc::Mailbox*)object; + if (!ResolveProcessHandle(proc, proc->ioKeyHandle, Ipc::HandleType::Mailbox, snapshot)) return nullptr; + return (Ipc::Mailbox*)snapshot.object; } static int DuplicateHandleBetweenSlots(int srcSlot, int handle, int dstSlot) { if (srcSlot < 0 || dstSlot < 0 || handle < 0) return -1; - Ipc::HandleType type = Ipc::HandleType::None; - Ipc::Object* object = nullptr; - uint32_t rights = 0; - if (!Ipc::SnapshotHandleForSlot(srcSlot, handle, type, object, rights)) return -1; - if ((rights & Ipc::RightDup) == 0) return -1; - return Ipc::InstallHandleForSlot(dstSlot, object, type, rights); + Ipc::HandleSnapshot snapshot; + if (!snapshot.Capture(srcSlot, handle)) return -1; + if ((snapshot.rights & Ipc::RightDup) == 0) return -1; + return Ipc::InstallHandleForSlot(dstSlot, snapshot.object, snapshot.type, snapshot.rights); } static bool ConfigureRedirWaitsetForSlot(int slot, Sched::Process* proc) { @@ -100,12 +88,13 @@ namespace montauk::abi { int total = 0; while (total < len) { + uint64_t observedWake = Sched::ObserveObjectWake(stream); int written = Ipc::StreamWrite(stream, data + total, len - total, false); if (written < 0) { return (total > 0) ? total : -1; } if (written == 0) { - Sched::BlockOnObject(stream, 0); + Sched::BlockOnObjectSince(stream, 0, observedWake); continue; } total += written; diff --git a/kernel/src/Api/Graphics.hpp b/kernel/src/Api/Graphics.hpp index fe1522f..4a04cee 100644 --- a/kernel/src/Api/Graphics.hpp +++ b/kernel/src/Api/Graphics.hpp @@ -36,8 +36,17 @@ namespace montauk::abi { if (fbBase == nullptr) return 0; uint64_t fbPhys = Memory::SubHHDM((uint64_t)fbBase); - uint64_t fbSize = Graphics::Framebuffer::GetHeight() - * Graphics::Framebuffer::GetPitch(); + uint64_t width = Graphics::Framebuffer::GetWidth(); + uint64_t height = Graphics::Framebuffer::GetHeight(); + uint64_t pitch = Graphics::Framebuffer::GetPitch(); + if (width == 0 || height == 0 || pitch == 0 + || width > (~0ULL / 4) || pitch < width * 4 + || height > (~0ULL / pitch)) { + Kt::KernelLogStream(Kt::ERROR, "FbMap") + << "Invalid or overflowing framebuffer geometry"; + return 0; + } + uint64_t fbSize = height * pitch; uint64_t numPages = (fbSize + 0xFFF) / 0x1000; Kt::KernelLogStream(Kt::INFO, "FbMap") << "fbPhys=" << kcp::hex << fbPhys @@ -47,8 +56,28 @@ namespace montauk::abi { << "x" << Graphics::Framebuffer::GetHeight() << " pitch=" << Graphics::Framebuffer::GetPitch() << ")"; - // Map at a fixed user VA - constexpr uint64_t userVa = 0x50000000ULL; + // Map in the dedicated framebuffer arena, away from the heap, + // shared libraries, reusable IPC surfaces, and the user stack. + constexpr uint64_t userVa = Sched::UserFramebufferBase; + uint64_t mappedBytes = numPages * 0x1000ULL; + bool flipSupported = Drivers::Graphics::IntelGPU::FlipSupported(); + uint64_t buffers = flipSupported ? 2 : 1; + if (mappedBytes > (Sched::UserFramebufferLimit - userVa) / buffers) { + Kt::KernelLogStream(Kt::ERROR, "FbMap") + << "Framebuffer mappings exceed reserved user VA range"; + return 0; + } + + // Never expose more of buffer 1 than the GPU driver allocated. This + // independently contains a future geometry-contract regression before + // any partial framebuffer mapping is installed in the process. + if (flipSupported + && (Drivers::Graphics::IntelGPU::GetBufferPhys(1) == 0 + || Drivers::Graphics::IntelGPU::GetBufferPageCount(1) < numPages)) { + Kt::KernelLogStream(Kt::ERROR, "FbMap") + << "Page-flip buffer is smaller than firmware framebuffer"; + return 0; + } for (uint64_t i = 0; i < numPages; i++) { if (!Memory::VMM::Paging::MapUserInWC( @@ -63,7 +92,7 @@ namespace montauk::abi { // When page flipping is available, map the second scanout buffer // immediately after the first so a double-buffered client can reach // both. Single-buffer clients never touch the extra pages. - if (Drivers::Graphics::IntelGPU::FlipSupported()) { + if (flipSupported) { uint64_t buf1Phys = Drivers::Graphics::IntelGPU::GetBufferPhys(1); for (uint64_t i = 0; i < numPages; i++) { if (!Memory::VMM::Paging::MapUserInWC( @@ -82,6 +111,7 @@ namespace montauk::abi { // SYS_FBFLIP: point scanout at buffer `index` (0 or 1). The hardware // latches the new surface at vblank, so the flip is tear-free. // index == -1 queries support without side effects. + // index == -2 acquires flip ownership and returns the live front buffer. static int64_t Sys_FbFlip(uint64_t index, uint64_t flags) { namespace GPU = Drivers::Graphics::IntelGPU; @@ -91,9 +121,11 @@ namespace montauk::abi { if (!GPU::FlipSupported()) return -1; auto* proc = Sched::GetCurrentProcessPtr(); - if (proc != nullptr) GPU::SetFlipOwner(proc->pid); - - return GPU::Flip((int)index, (flags & 1) != 0); + if ((int64_t)index == -2) { + return proc != nullptr ? GPU::AcquireFlip(proc->pid) : -1; + } + if (proc == nullptr) return -1; + return GPU::Flip((int)index, (flags & 1) != 0, proc->pid); } static int64_t Sys_DisplayInfo(DisplayInfo* out) { diff --git a/kernel/src/Api/Heap.hpp b/kernel/src/Api/Heap.hpp index 80843ef..5fa6476 100644 --- a/kernel/src/Api/Heap.hpp +++ b/kernel/src/Api/Heap.hpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include namespace montauk::abi { @@ -22,6 +24,7 @@ namespace montauk::abi { inline HeapAlloc g_heapAllocs[Sched::MaxProcesses][MaxHeapAllocs] = {}; inline int g_heapAllocCount[Sched::MaxProcesses] = {}; + inline kcp::Mutex g_heapLocks[Sched::MaxProcesses]; // Get the process table slot index for the current process inline int GetCurrentSlot() { @@ -38,20 +41,15 @@ namespace montauk::abi { int slot = GetCurrentSlot(); if (slot < 0) return 0; - // Guard against overflow before rounding - static constexpr uint64_t USER_SPACE_END = 0x0000800000000000ULL; + // Guard against overflow before rounding. if (size > 0xFFFFFFFFFFFF0000ULL) return 0; // Round up to page boundary size = (size + 0xFFF) & ~0xFFFULL; if (size == 0) size = 0x1000; - uint64_t userVa = proc->heapNext; - - // Ensure allocation stays within user address space - if (userVa + size < userVa || userVa + size > USER_SPACE_END) return 0; - uint64_t numPages = size / 0x1000; + g_heapLocks[slot].Acquire(); if (g_heapAllocCount[slot] >= MaxHeapAllocs) { // Out of allocation records, not out of memory. Log it: a // silent 0 here surfaced as bogus downstream errors (BFD @@ -60,6 +58,16 @@ namespace montauk::abi { << "pid " << proc->pid << " (" << proc->name << ") hit MaxHeapAllocs (" << (uint64_t)MaxHeapAllocs << "), SYS_ALLOC refused"; + g_heapLocks[slot].Release(); + return 0; + } + + uint64_t userVa = 0; + if (!Sched::ReserveUserHeapRange(slot, size, userVa)) { + Kt::KernelLogStream(Kt::ERROR, "Heap") + << "pid " << proc->pid << " (" << proc->name + << ") exhausted bounded user heap"; + g_heapLocks[slot].Release(); return 0; } @@ -68,38 +76,25 @@ namespace montauk::abi { for (uint64_t i = 0; i < numPages; i++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { - for (uint64_t j = 0; j < mappedPages; j++) { - uint64_t pageVa = userVa + j * 0x1000; - uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa); - if (physAddr != 0) { - Memory::g_pfa->Free((void*)Memory::HHDM(physAddr)); - } - Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa); - } + Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages); + g_heapLocks[slot].Release(); return 0; } uint64_t physAddr = Memory::SubHHDM((uint64_t)page); if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, userVa + i * 0x1000)) { Memory::g_pfa->Free(page); - for (uint64_t j = 0; j < mappedPages; j++) { - uint64_t pageVa = userVa + j * 0x1000; - uint64_t mappedPhys = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa); - if (mappedPhys != 0) { - Memory::g_pfa->Free((void*)Memory::HHDM(mappedPhys)); - } - Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa); - } + Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages); + g_heapLocks[slot].Release(); return 0; } mappedPages++; } - proc->heapNext += size; - // Track the allocation so Sys_Free can release it Sched::g_allocatedPages[slot] += numPages; g_heapAllocs[slot][g_heapAllocCount[slot]++] = { userVa, numPages }; + g_heapLocks[slot].Release(); return userVa; } @@ -107,8 +102,10 @@ namespace montauk::abi { // The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup. inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) { if (slot < 0 || slot >= Sched::MaxProcesses) return; + g_heapLocks[slot].Acquire(); g_heapAllocCount[slot] = 0; Sched::g_allocatedPages[slot] = 0; + g_heapLocks[slot].Release(); } inline void Sys_Free(uint64_t addr) { @@ -118,6 +115,8 @@ namespace montauk::abi { int slot = GetCurrentSlot(); if (slot < 0) return; + g_heapLocks[slot].Acquire(); + // Find the allocation record matching this address int idx = -1; for (int i = 0; i < g_heapAllocCount[slot]; i++) { @@ -126,25 +125,23 @@ namespace montauk::abi { break; } } - if (idx < 0) return; // Unknown address — ignore + if (idx < 0) { + g_heapLocks[slot].Release(); + return; // Unknown address — ignore + } uint64_t va = g_heapAllocs[slot][idx].va; uint64_t numPages = g_heapAllocs[slot][idx].numPages; - // Free physical pages in bulk and unmap virtual addresses - for (uint64_t i = 0; i < numPages; i++) { - uint64_t pageVa = va + i * 0x1000; - uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa); - if (physAddr != 0) { - Memory::g_pfa->Free((void*)Memory::HHDM(physAddr)); - } - Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa); - } + // Unmap and invalidate sibling CPUs before recycling the frames. A + // stale user TLB entry can otherwise corrupt the frame's next owner. + Ipc::UnmapAndFreeUserRange(proc->pml4Phys, va, numPages); Sched::g_allocatedPages[slot] -= numPages; // Remove tracking entry by swapping with the last element g_heapAllocs[slot][idx] = g_heapAllocs[slot][g_heapAllocCount[slot] - 1]; g_heapAllocCount[slot]--; + g_heapLocks[slot].Release(); } }; diff --git a/kernel/src/Api/IoRedir.hpp b/kernel/src/Api/IoRedir.hpp index f9c042e..f07fa09 100644 --- a/kernel/src/Api/IoRedir.hpp +++ b/kernel/src/Api/IoRedir.hpp @@ -85,14 +85,16 @@ namespace montauk::abi { static int Sys_ChildIoRead(int childPid, char* buf, int maxLen) { auto* child = Sched::GetProcessByPid(childPid); - Ipc::Stream* stream = GetRedirOutStream(child); + Ipc::HandleSnapshot snapshot; + Ipc::Stream* stream = GetRedirOutStream(child, snapshot); if (child == nullptr || !child->redirected || stream == nullptr) return -1; return Ipc::StreamRead(stream, (uint8_t*)buf, maxLen, true); } static int Sys_ChildIoWrite(int childPid, const char* data, int len) { auto* child = Sched::GetProcessByPid(childPid); - Ipc::Stream* stream = GetRedirInStream(child); + Ipc::HandleSnapshot snapshot; + Ipc::Stream* stream = GetRedirInStream(child, snapshot); if (child == nullptr || !child->redirected || stream == nullptr) return -1; return WriteAllToStream(stream, (const uint8_t*)data, len); } @@ -100,14 +102,16 @@ namespace montauk::abi { static int Sys_ChildIoWriteKey(int childPid, const KeyEvent* key) { if (key == nullptr) return -1; auto* child = Sched::GetProcessByPid(childPid); - Ipc::Mailbox* mailbox = GetRedirKeyMailbox(child); + Ipc::HandleSnapshot snapshot; + Ipc::Mailbox* mailbox = GetRedirKeyMailbox(child, snapshot); if (child == nullptr || !child->redirected || mailbox == nullptr) return -1; for (;;) { + uint64_t observedWake = Sched::ObserveObjectWake(mailbox); int rc = Ipc::MailboxSend(mailbox, 0, key, sizeof(KeyEvent)); if (rc < 0) return -1; if (rc == 0) { - Sched::BlockOnObject(mailbox, 0); + Sched::BlockOnObjectSince(mailbox, 0, observedWake); continue; } return 0; diff --git a/kernel/src/Api/Keyboard.hpp b/kernel/src/Api/Keyboard.hpp index 21a78d0..c0e964b 100644 --- a/kernel/src/Api/Keyboard.hpp +++ b/kernel/src/Api/Keyboard.hpp @@ -15,7 +15,8 @@ namespace montauk::abi { static bool Sys_IsKeyAvailable() { auto* proc = Sched::GetCurrentProcessPtr(); if (proc && proc->redirected) { - Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc); + Ipc::HandleSnapshot snapshot; + Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, snapshot); if (mailbox != nullptr) return Ipc::MailboxHasMessage(mailbox); } return Drivers::PS2::Keyboard::IsKeyAvailable(); @@ -25,9 +26,11 @@ namespace montauk::abi { if (outEvent == nullptr) return; auto* proc = Sched::GetCurrentProcessPtr(); if (proc && proc->redirected) { - Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc); + Ipc::HandleSnapshot snapshot; + Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, snapshot); if (mailbox != nullptr) { for (;;) { + uint64_t observedWake = Sched::ObserveObjectWake(mailbox); uint16_t len = sizeof(KeyEvent); int rc = Ipc::MailboxRecv(mailbox, nullptr, outEvent, &len, true); if (rc > 0) return; @@ -35,7 +38,7 @@ namespace montauk::abi { memset(outEvent, 0, sizeof(KeyEvent)); return; } - Sched::BlockOnObject(mailbox, 0); + Sched::BlockOnObjectSince(mailbox, 0, observedWake); } } } @@ -51,10 +54,14 @@ namespace montauk::abi { static char Sys_GetChar() { auto* proc = Sched::GetCurrentProcessPtr(); if (proc && proc->redirected) { - Ipc::Stream* input = GetRedirInStream(proc); - Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc); + Ipc::HandleSnapshot inputSnapshot; + Ipc::HandleSnapshot mailboxSnapshot; + Ipc::Stream* input = GetRedirInStream(proc, inputSnapshot); + Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, mailboxSnapshot); if (input != nullptr || mailbox != nullptr) { for (;;) { + uint64_t inputWake = input ? Sched::ObserveObjectWake(input) : 0; + uint64_t mailboxWake = mailbox ? Sched::ObserveObjectWake(mailbox) : 0; if (input != nullptr) { uint8_t c = 0; int rc = Ipc::StreamRead(input, &c, 1, true); @@ -79,9 +86,9 @@ namespace montauk::abi { return 0; } } else if (input != nullptr) { - Sched::BlockOnObject(input, 0); + Sched::BlockOnObjectSince(input, 0, inputWake); } else if (mailbox != nullptr) { - Sched::BlockOnObject(mailbox, 0); + Sched::BlockOnObjectSince(mailbox, 0, mailboxWake); } else { return 0; } diff --git a/kernel/src/Api/LibSyscall.hpp b/kernel/src/Api/LibSyscall.hpp index a5896c9..c779926 100644 --- a/kernel/src/Api/LibSyscall.hpp +++ b/kernel/src/Api/LibSyscall.hpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include namespace montauk::abi { @@ -34,16 +36,19 @@ namespace montauk::abi { // Per-process library table inline LibEntry g_libTable[MaxProcesses][MaxLibsPerProcess] = {}; + inline kcp::Mutex g_libLocks[MaxProcesses]; // Initialize library table for a process slot inline void InitLibTable(int slot) { if (slot < 0 || slot >= MaxProcesses) return; + g_libLocks[slot].Acquire(); for (int i = 0; i < MaxLibsPerProcess; i++) { g_libTable[slot][i].inUse = false; g_libTable[slot][i].refcount = 0; g_libTable[slot][i].loadBias = 0; g_libTable[slot][i].path[0] = '\0'; } + g_libLocks[slot].Release(); } // Load a shared library into the current process's address space. @@ -61,6 +66,8 @@ namespace montauk::abi { size_t pathLen = Lib::strlen(path); if (pathLen == 0 || pathLen >= sizeof(LibEntry::path)) return 0; + g_libLocks[slot].Acquire(); + // Find a free slot or reuse an existing slot with the same path int libSlot = -1; for (int i = 0; i < MaxLibsPerProcess; i++) { @@ -68,6 +75,7 @@ namespace montauk::abi { if (Lib::strncmp(g_libTable[slot][i].path, path, sizeof(LibEntry::path)) == 0) { // Same library already loaded - just increment refcount g_libTable[slot][i].refcount++; + g_libLocks[slot].Release(); return (uint64_t)(i + 1); // Handle = slot + 1 (0 = invalid) } } else if (libSlot < 0) { @@ -77,6 +85,7 @@ namespace montauk::abi { if (libSlot < 0) { Kt::KernelLogStream(Kt::ERROR, "Lib") << "No free library slots"; + g_libLocks[slot].Release(); return 0; } @@ -84,6 +93,7 @@ namespace montauk::abi { uint64_t loadBias = Sched::ElfLoadLib(path, proc->pml4Phys, libSlot); if (loadBias == 0) { Kt::KernelLogStream(Kt::ERROR, "Lib") << "Failed to load library: " << path; + g_libLocks[slot].Release(); return 0; } @@ -93,6 +103,8 @@ namespace montauk::abi { g_libTable[slot][libSlot].loadBias = loadBias; Lib::strncpy(g_libTable[slot][libSlot].path, path, sizeof(LibEntry::path)); + g_libLocks[slot].Release(); + Kt::KernelLogStream(Kt::OK, "Lib") << "Loaded library: " << path << " with load bias " << kcp::hex << loadBias << kcp::dec; return (uint64_t)(libSlot + 1); // Handle = slot + 1 (0 = invalid) @@ -107,7 +119,11 @@ namespace montauk::abi { int libSlot = (int)handle - 1; if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return -1; - if (!g_libTable[slot][libSlot].inUse) return -1; + g_libLocks[slot].Acquire(); + if (!g_libTable[slot][libSlot].inUse) { + g_libLocks[slot].Release(); + return -1; + } g_libTable[slot][libSlot].refcount--; if (g_libTable[slot][libSlot].refcount == 0) { @@ -117,14 +133,8 @@ namespace montauk::abi { auto* proc = Sched::GetCurrentProcessPtr(); if (proc != nullptr) { - // Unmap all pages in the library region - for (uint64_t va = libBase; va < libEnd; va += 0x1000) { - uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, va); - if (physAddr != 0) { - Memory::g_pfa->Free((void*)Memory::HHDM(physAddr)); - Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, va); - } - } + Ipc::UnmapAndFreeUserRange(proc->pml4Phys, libBase, + (libEnd - libBase) / 0x1000ULL); } g_libTable[slot][libSlot].inUse = false; @@ -132,6 +142,7 @@ namespace montauk::abi { g_libTable[slot][libSlot].path[0] = '\0'; } + g_libLocks[slot].Release(); return 0; } @@ -146,9 +157,14 @@ namespace montauk::abi { int libSlot = (int)handle - 1; if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return 0; - if (!g_libTable[slot][libSlot].inUse) return 0; + g_libLocks[slot].Acquire(); + if (!g_libTable[slot][libSlot].inUse) { + g_libLocks[slot].Release(); + return 0; + } uint64_t loadBias = g_libTable[slot][libSlot].loadBias; + g_libLocks[slot].Release(); return loadBias + symbolOffset; } @@ -160,9 +176,15 @@ namespace montauk::abi { int libSlot = (int)handle - 1; if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return 0; - if (!g_libTable[slot][libSlot].inUse) return 0; + g_libLocks[slot].Acquire(); + if (!g_libTable[slot][libSlot].inUse) { + g_libLocks[slot].Release(); + return 0; + } - return g_libTable[slot][libSlot].loadBias; + uint64_t loadBias = g_libTable[slot][libSlot].loadBias; + g_libLocks[slot].Release(); + return loadBias; } // Cleanup library table for a process slot @@ -172,25 +194,23 @@ namespace montauk::abi { auto* proc = Sched::GetProcessSlot(slot); if (proc == nullptr) return; + g_libLocks[slot].Acquire(); + // Unmap all libraries for this process for (int i = 0; i < MaxLibsPerProcess; i++) { if (g_libTable[slot][i].inUse) { uint64_t libBase = GetLibSlotBase(i); uint64_t libEnd = libBase + Sched::LIB_MAX_SIZE; - for (uint64_t va = libBase; va < libEnd; va += 0x1000) { - uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, va); - if (physAddr != 0) { - Memory::g_pfa->Free((void*)Memory::HHDM(physAddr)); - Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, va); - } - } + Ipc::UnmapAndFreeUserRange(proc->pml4Phys, libBase, + (libEnd - libBase) / 0x1000ULL); g_libTable[slot][i].inUse = false; g_libTable[slot][i].refcount = 0; g_libTable[slot][i].loadBias = 0; } } + g_libLocks[slot].Release(); } } diff --git a/kernel/src/Api/Power.hpp b/kernel/src/Api/Power.hpp index bcdc3de..39b63d6 100644 --- a/kernel/src/Api/Power.hpp +++ b/kernel/src/Api/Power.hpp @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include "Syscall.hpp" @@ -31,13 +34,21 @@ namespace montauk::abi { if (action == POWER_REQ_QUERY) { int pending = g_pendingPowerAction; g_pendingPowerAction = POWER_REQ_QUERY; + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() + << " consumed graceful power request " << (uint64_t)pending; return (int64_t)pending; } g_pendingPowerAction = action; + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() + << " posted graceful power request " << (uint64_t)action; return 0; } static void Sys_Reset() { + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() << " entered final reboot stage"; if (Efi::g_ResetSystem) { /* Switch to kernel PML4 which has identity-mapped UEFI runtime regions */ Memory::VMM::LoadCR3(Memory::VMM::g_paging->PML4); @@ -51,6 +62,8 @@ namespace montauk::abi { } static void Sys_Shutdown() { + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() << " entered final shutdown stage"; /* Primary: ACPI S5 shutdown via PM1 control registers */ if (Hal::AcpiShutdown::IsAvailable()) { Hal::AcpiShutdown::Shutdown(); diff --git a/kernel/src/Api/Storage.hpp b/kernel/src/Api/Storage.hpp index 0852af6..9ec194c 100644 --- a/kernel/src/Api/Storage.hpp +++ b/kernel/src/Api/Storage.hpp @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include "Syscall.hpp" @@ -112,6 +115,9 @@ namespace montauk::abi { // Flush all block-device write caches and cleanly unmount disk-backed // volumes ahead of power-off. Returns the number of volumes unmounted. static int64_t Sys_FsSync() { + Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec + << (uint64_t)Sched::GetCurrentPid() + << " entered filesystem sync stage"; return (int64_t)Fs::FsProbe::SyncAndUnmountAll(); } diff --git a/kernel/src/Api/Terminal.hpp b/kernel/src/Api/Terminal.hpp index cfe70db..5f85543 100644 --- a/kernel/src/Api/Terminal.hpp +++ b/kernel/src/Api/Terminal.hpp @@ -15,7 +15,8 @@ namespace montauk::abi { static void Sys_Print(const char* text) { auto* proc = Sched::GetCurrentProcessPtr(); if (proc && proc->redirected) { - Ipc::Stream* stream = GetRedirOutStream(proc); + Ipc::HandleSnapshot snapshot; + Ipc::Stream* stream = GetRedirOutStream(proc, snapshot); if (stream != nullptr) { int len = 0; while (text[len]) len++; @@ -33,7 +34,8 @@ namespace montauk::abi { static void Sys_Putchar(char c) { auto* proc = Sched::GetCurrentProcessPtr(); if (proc && proc->redirected) { - Ipc::Stream* stream = GetRedirOutStream(proc); + Ipc::HandleSnapshot snapshot; + Ipc::Stream* stream = GetRedirOutStream(proc, snapshot); if (stream != nullptr) { uint8_t byte = (uint8_t)c; WriteAllToStream(stream, &byte, 1); diff --git a/kernel/src/Common/Panic.cpp b/kernel/src/Common/Panic.cpp index bee58a4..fd2e0d5 100644 --- a/kernel/src/Common/Panic.cpp +++ b/kernel/src/Common/Panic.cpp @@ -70,6 +70,7 @@ void Panic(const char *meditationString, System::PanicFrame* frame) { // showing the other buffer, the panic box would land on an invisible // surface. Safe no-op when the GPU driver is inactive. Drivers::Graphics::IntelGPU::PanicRestoreScanout(); + Kt::EnablePanicOutput(); // Header kerr << BOXUI_ANSI_RED_BG << BOXUI_ANSI_WHITE_FG << BOXUI_ANSI_BOLD << "\n"; @@ -105,5 +106,11 @@ void Panic(const char *meditationString, System::PanicFrame* frame) { PrintHorizontalEdge(BOXUI_BL, BOXUI_BR); kerr << BOXUI_ANSI_RESET; +#if defined (__x86_64__) + // The framebuffer is write-combining. Drain the final panic text before + // halting all forward progress so it cannot remain in a CPU write buffer. + asm volatile("sfence" ::: "memory"); +#endif + Halt(); } diff --git a/kernel/src/CppLib/Spinlock.cpp b/kernel/src/CppLib/Spinlock.cpp index 104624b..e311a0f 100644 --- a/kernel/src/CppLib/Spinlock.cpp +++ b/kernel/src/CppLib/Spinlock.cpp @@ -7,18 +7,29 @@ #include "Spinlock.hpp" namespace kcp { - void Spinlock::Acquire() { + uint64_t Spinlock::AcquireIrqSave() { uint64_t flags; asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory"); while (atomic_flag.test_and_set(std::memory_order_acquire)) { asm volatile("pause"); } - savedFlags = flags; + return flags; } - void Spinlock::Release() { - uint64_t flags = savedFlags; + void Spinlock::ReleaseIrqRestore(uint64_t flags) { atomic_flag.clear(std::memory_order_release); asm volatile("push %0; popfq" :: "r"(flags) : "memory"); } -}; \ No newline at end of file + + void Spinlock::ReleaseIrqNoRestore() { + atomic_flag.clear(std::memory_order_release); + } + + void Spinlock::Acquire() { + savedFlags = AcquireIrqSave(); + } + + void Spinlock::Release() { + ReleaseIrqRestore(savedFlags); + } +}; diff --git a/kernel/src/CppLib/Spinlock.hpp b/kernel/src/CppLib/Spinlock.hpp index d1e8871..6ef80f9 100644 --- a/kernel/src/CppLib/Spinlock.hpp +++ b/kernel/src/CppLib/Spinlock.hpp @@ -14,6 +14,14 @@ namespace kcp { public: void Acquire(); void Release(); + + // Explicit IRQ-save form for locks whose ownership is deliberately + // handed across a context switch. The saved flags live on the caller's + // stack (or in scheduler-owned per-context storage), not in the shared + // lock object, so the resumed context restores its own interrupt state. + uint64_t AcquireIrqSave(); + void ReleaseIrqRestore(uint64_t flags); + void ReleaseIrqNoRestore(); }; // Non-interrupt-disabling mutex for subsystems that are only called @@ -31,4 +39,4 @@ namespace kcp { flag.clear(std::memory_order_release); } }; -}; \ No newline at end of file +}; diff --git a/kernel/src/Drivers/Audio/IntelHda.cpp b/kernel/src/Drivers/Audio/IntelHda.cpp index c2bb112..7eaedf5 100644 --- a/kernel/src/Drivers/Audio/IntelHda.cpp +++ b/kernel/src/Drivers/Audio/IntelHda.cpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace Drivers::Audio::IntelHda { @@ -26,10 +27,33 @@ namespace Drivers::Audio::IntelHda { static bool g_initialized = false; static kcp::Spinlock g_codecLock; - // Serializes stream lifecycle, DMA write-pointer updates, and IRQ-side - // stream inspection. The IRQ releases it before calling the mixer to keep - // the lock order consistently Mixer -> HDA stream. + // Serializes stream lifecycle and DMA write-pointer updates. Keep logging + // out of this lock: Spinlock disables local interrupts, while the terminal + // logger may wait for a lock owner that needs the local timer to run. static kcp::Spinlock g_streamLock; + // The hard IRQ only acknowledges and masks stream completion sources. + // Mixing/refilling is substantially too expensive for interrupt context. + static std::atomic g_streamWorkPending{false}; + static uint32_t g_streamInterruptMask = 0; + + // Runtime diagnostics are emitted by ProcessDeferredWork() in idle + // context. Some HDA entry points run below the mixer's interrupt-disabling + // spinlock, so even logging after releasing g_streamLock is not safe. + enum DeferredDiagnostic : uint32_t { + DIAG_STREAM_OPENED = 1u << 0, + DIAG_STREAM_CLOSED = 1u << 1, + DIAG_HEADPHONES = 1u << 2, + DIAG_SPEAKERS = 1u << 3, + }; + static std::atomic g_diagnosticsPending{0}; + static std::atomic g_openSampleRate{0}; + static std::atomic g_openBitsPerSample{0}; + static std::atomic g_openChannels{0}; + static std::atomic g_codecTimeoutsPending{0}; + static std::atomic g_codecTimeoutReports{0}; + static std::atomic g_lastTimeoutCodec{0}; + static std::atomic g_lastTimeoutNid{0}; + static std::atomic g_lastTimeoutVerb{0}; static volatile uint8_t* g_mmioBase = nullptr; static uint8_t g_bus, g_dev, g_func; @@ -213,9 +237,10 @@ namespace Drivers::Audio::IntelHda { uint32_t response = 0; if (!ReadResponse(&response, nullptr)) { g_codecLock.Release(); - KernelLogStream(WARNING, "HDA") << "Verb timeout: codec=" << base::dec - << (uint64_t)codec << " nid=" << (uint64_t)nid - << " verb=" << base::hex << (uint64_t)verb; + g_lastTimeoutCodec.store(codec, std::memory_order_relaxed); + g_lastTimeoutNid.store(nid, std::memory_order_relaxed); + g_lastTimeoutVerb.store(verb, std::memory_order_relaxed); + g_codecTimeoutsPending.fetch_add(1, std::memory_order_release); return 0; } g_codecLock.Release(); @@ -640,13 +665,15 @@ namespace Drivers::Audio::IntelHda { DisablePin(g_speakerNid); EnablePin(g_hpNid); g_pinNid = g_hpNid; - KernelLogStream(INFO, "HDA") << "Switched to headphone output"; + g_diagnosticsPending.fetch_or(DIAG_HEADPHONES, + std::memory_order_release); } else { // Mute HP, enable speaker DisablePin(g_hpNid); EnablePin(g_speakerNid); g_pinNid = g_speakerNid; - KernelLogStream(INFO, "HDA") << "Switched to speaker output"; + g_diagnosticsPending.fetch_or(DIAG_SPEAKERS, + std::memory_order_release); } // Re-apply volume on the now-active pin SetOutputVolume(g_volume); @@ -810,7 +837,7 @@ namespace Drivers::Audio::IntelHda { // MSI setup // ========================================================================= - static void HandleInterrupt(uint8_t irq); + static void HandleInterrupt(uint8_t irq, bool fromUser); static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); @@ -819,6 +846,8 @@ namespace Drivers::Audio::IntelHda { return false; } + Pci::DisableInterruptDelivery(bus, dev, func); + uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); bool is64bit = (msgCtrl & (1 << 7)) != 0; @@ -851,21 +880,21 @@ namespace Drivers::Audio::IntelHda { // Interrupt handler // ========================================================================= - static void HandleInterrupt(uint8_t /*irq*/) { + static void HandleInterrupt(uint8_t /*irq*/, bool /*fromUser*/) { uint32_t intsts = Read32(REG_INTSTS); bool bufferCompleted = false; - // Handle stream interrupts (bits 0-29 correspond to stream descriptors) - g_streamLock.Acquire(); - if (g_stream.Active) { - uint8_t si = g_stream.StreamIndex; - if (intsts & (1u << si)) { - uint8_t sts = ReadSD8(si, SD_STS); - if (sts & SD_STS_BCIS) bufferCompleted = true; - WriteSD8(si, SD_STS, sts); - } + // Handle stream interrupts (bits 0-29 correspond to stream + // descriptors) without taking g_streamLock. Inspecting only the + // hardware status registers is race-safe against stream lifecycle and + // avoids spinning in an IRQ if another CPU is opening/closing audio. + uint32_t streamBits = intsts & 0x3FFFFFFFu; + for (uint8_t si = 0; si < 30; si++) { + if (!(streamBits & (1u << si))) continue; + uint8_t sts = ReadSD8(si, SD_STS); + if (sts & SD_STS_BCIS) bufferCompleted = true; + WriteSD8(si, SD_STS, sts); } - g_streamLock.Release(); // Handle RIRB interrupt (controller interrupt enable bit 30) // Do NOT advance g_rirbReadPtr here — ReadResponse() owns it. @@ -876,10 +905,12 @@ namespace Drivers::Audio::IntelHda { Write8(REG_RIRBSTS, rirbSts); } - // Notify the mixer so it can refill the DMA ring with the next mix - // window. Done after clearing status bits so re-entry can't latch. + // A mixer pass can resample up to 4096 frames across every active + // stream and must never run inside this high-priority IRQ. Mask stream + // completion delivery and queue a process-safe bottom half instead. if (bufferCompleted) { - Mixer::OnHdaBufferComplete(); + Write32(REG_INTCTL, Read32(REG_INTCTL) & ~g_streamInterruptMask); + g_streamWorkPending.store(true, std::memory_order_release); } } @@ -987,7 +1018,13 @@ namespace Drivers::Audio::IntelHda { uint8_t irqLine = Pci::LegacyRead8(g_bus, g_dev, g_func, (uint8_t)Pci::PCI_REG_INTERRUPT); if (irqLine != 0xFF) { KernelLogStream(INFO, "HDA") << "Falling back to legacy IRQ " << base::dec << (uint64_t)irqLine; + Pci::DisableInterruptDelivery(g_bus, g_dev, g_func); + uint16_t command = Pci::LegacyRead16(g_bus, g_dev, g_func, + (uint8_t)Pci::PCI_REG_COMMAND); + Pci::LegacyWrite16(g_bus, g_dev, g_func, + (uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE); Hal::RegisterIrqHandler(irqLine, HandleInterrupt); + Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine)); } else { KernelLogStream(WARNING, "HDA") << "No interrupt available, polling only"; } @@ -997,8 +1034,9 @@ namespace Drivers::Audio::IntelHda { uint32_t intctl = INTCTL_GIE | INTCTL_CIE; // Enable interrupt for all output streams for (uint8_t i = 0; i < g_numOutputStreams; i++) { - intctl |= (1u << (g_numInputStreams + i)); + g_streamInterruptMask |= (1u << (g_numInputStreams + i)); } + intctl |= g_streamInterruptMask; Write32(REG_INTCTL, intctl); // Discover codecs @@ -1028,6 +1066,63 @@ namespace Drivers::Audio::IntelHda { return g_initialized; } + bool HasDeferredWork() { + return (g_initialized && + g_streamWorkPending.load(std::memory_order_acquire)) || + g_diagnosticsPending.load(std::memory_order_acquire) != 0 || + g_codecTimeoutsPending.load(std::memory_order_acquire) != 0; + } + + void ProcessDeferredWork() { + if (g_initialized && + g_streamWorkPending.exchange(false, std::memory_order_acq_rel)) { + Mixer::OnHdaBufferComplete(); + + // Clear-before-unmask closes the handoff race: if another completion + // arrived while masked its SD_STS/INTSTS state is still asserted and + // enabling the stream bit produces a fresh MSI/INTx interrupt. + Write32(REG_INTCTL, Read32(REG_INTCTL) | g_streamInterruptMask | + INTCTL_GIE | INTCTL_CIE); + } + + uint32_t diagnostics = + g_diagnosticsPending.exchange(0, std::memory_order_acq_rel); + if (diagnostics & DIAG_STREAM_OPENED) { + KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec + << (uint64_t)g_openSampleRate.load(std::memory_order_relaxed) + << "Hz " + << (uint64_t)g_openBitsPerSample.load(std::memory_order_relaxed) + << "-bit " + << (uint64_t)g_openChannels.load(std::memory_order_relaxed) + << "ch"; + } + if (diagnostics & DIAG_STREAM_CLOSED) + KernelLogStream(OK, "HDA") << "Stream closed"; + if (diagnostics & DIAG_HEADPHONES) + KernelLogStream(INFO, "HDA") << "Switched to headphone output"; + if (diagnostics & DIAG_SPEAKERS) + KernelLogStream(INFO, "HDA") << "Switched to speaker output"; + + uint32_t timeouts = + g_codecTimeoutsPending.exchange(0, std::memory_order_acq_rel); + if (timeouts != 0) { + uint32_t report = + g_codecTimeoutReports.fetch_add(1, std::memory_order_relaxed); + if (report < 8) { + KernelLogStream(WARNING, "HDA") << base::dec + << (uint64_t)timeouts << " codec verb timeout(s), last: codec=" + << (uint64_t)g_lastTimeoutCodec.load(std::memory_order_relaxed) + << " nid=" + << (uint64_t)g_lastTimeoutNid.load(std::memory_order_relaxed) + << " verb=" << base::hex + << (uint64_t)g_lastTimeoutVerb.load(std::memory_order_relaxed); + } else if (report == 8) { + KernelLogStream(WARNING, "HDA") + << "Further codec verb timeout reports suppressed"; + } + } + } + uint32_t GetCodecVendorId() { return g_codecVendorId; } @@ -1075,11 +1170,12 @@ namespace Drivers::Audio::IntelHda { // Start the stream StartStream(streamIndex); - KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec - << (uint64_t)sampleRate << "Hz " << (uint64_t)bitsPerSample << "-bit " - << (uint64_t)channels << "ch"; - g_streamLock.Release(); + g_openSampleRate.store(sampleRate, std::memory_order_relaxed); + g_openBitsPerSample.store(bitsPerSample, std::memory_order_relaxed); + g_openChannels.store(channels, std::memory_order_relaxed); + g_diagnosticsPending.fetch_or(DIAG_STREAM_OPENED, + std::memory_order_release); return 0; // Handle 0 } @@ -1099,8 +1195,9 @@ namespace Drivers::Audio::IntelHda { g_stream.Active = false; - KernelLogStream(OK, "HDA") << "Stream closed"; g_streamLock.Release(); + g_diagnosticsPending.fetch_or(DIAG_STREAM_CLOSED, + std::memory_order_release); } uint32_t GetWriteSpace(int handle) { diff --git a/kernel/src/Drivers/Audio/IntelHda.hpp b/kernel/src/Drivers/Audio/IntelHda.hpp index 05f2496..8117d3d 100644 --- a/kernel/src/Drivers/Audio/IntelHda.hpp +++ b/kernel/src/Drivers/Audio/IntelHda.hpp @@ -259,6 +259,8 @@ namespace Drivers::Audio::IntelHda { bool Probe(const Pci::PciDevice& dev); bool IsInitialized(); + bool HasDeferredWork(); + void ProcessDeferredWork(); // Returns the codec vendor/device ID (vendor in upper 16 bits, device in lower 16). // Returns 0 if no codec was found. diff --git a/kernel/src/Drivers/Audio/Mixer.cpp b/kernel/src/Drivers/Audio/Mixer.cpp index 85cbda8..1222c47 100644 --- a/kernel/src/Drivers/Audio/Mixer.cpp +++ b/kernel/src/Drivers/Audio/Mixer.cpp @@ -122,7 +122,11 @@ namespace Drivers::Audio::Mixer { static void FreeRing(int16_t* ring) { if (!ring) return; - Memory::g_pfa->ReallocConsecutive(ring, 0); + // The ring was allocated as one INPUT_RING_PAGES contiguous span. + // ReallocConsecutive(ptr, 0) is not a sized free operation and used + // to compute a zero-byte allocation at the end of the PFA pool, copy + // a page there, then free only the first page of this ring. + Memory::g_pfa->Free(ring, INPUT_RING_PAGES); } // Caller holds g_lock. HDA setup is non-blocking and serialized by the @@ -380,10 +384,11 @@ namespace Drivers::Audio::Mixer { if (!ring) return -1; retry_after_switch: + uint64_t switchWake = Sched::ObserveObjectWake((void*)&g_switchingOutput); g_lock.Acquire(); if (g_switchingOutput.load(std::memory_order_acquire)) { g_lock.Release(); - Sched::BlockOnObject((void*)&g_switchingOutput, 1000); + Sched::BlockOnObjectSince((void*)&g_switchingOutput, 1000, switchWake); goto retry_after_switch; } @@ -821,9 +826,8 @@ namespace Drivers::Audio::Mixer { } void OnHdaBufferComplete() { - // Called from the HDA buffer-completion interrupt. Mixer state and - // HDA register access are both serialized through g_lock (which - // disables interrupts on acquire), so this is safe to call from IRQ. + // Called from IntelHda's idle-context bottom half. A pump may resample + // thousands of frames and is deliberately kept out of the hard IRQ. g_lock.Acquire(); if (g_output == Output::Hda) Pump(); g_lock.Release(); diff --git a/kernel/src/Drivers/Graphics/IntelGPU.cpp b/kernel/src/Drivers/Graphics/IntelGPU.cpp index 4a00fdc..e5a8e7b 100644 --- a/kernel/src/Drivers/Graphics/IntelGPU.cpp +++ b/kernel/src/Drivers/Graphics/IntelGPU.cpp @@ -10,14 +10,14 @@ #include #include #include +#include #include #include #include #include #include -#include -#include #include +#include using namespace Kt; @@ -36,7 +36,6 @@ namespace Drivers::Graphics::IntelGPU { // GTT state static volatile void* g_gttBase = nullptr; // Virtual address of GTT entries static uint64_t g_gttEntryCount = 0; // Number of GTT entries - static uint64_t g_scratchPagePhys = 0; // Physical address of scratch page static uint8_t g_gpuGen = 0; // Cached generation number // Framebuffer state @@ -54,12 +53,15 @@ namespace Drivers::Graphics::IntelGPU { static uint64_t g_fbGttOffsetA = 0; // GGTT offset of buffer 0 (firmware FB) static uint32_t* g_buf1Virt = nullptr; // buffer 1 kernel VA (HHDM, WC) static uint64_t g_buf1Phys = 0; + static uint64_t g_buf1Pages = 0; // allocation/mapping bound static uint64_t g_buf1GttOffset = 0; // GGTT offset of buffer 1 (in bytes) static bool g_flipSupported = false; static int g_frontBuffer = 0; static int g_flipOwnerPid = -1; - static volatile uint64_t g_vblankCount = 0; - static bool g_vblankIrqReady = false; + // Serializes surface programming, flip-owner teardown, and timing changes. + // Without this, a process exit or modeset on another CPU can rewrite + // DSPASURF/PIPE state in the middle of a desktop flip transaction. + static kcp::Mutex g_flipLock; // Modesetting/display-management state. The firmware-trained port and // link are deliberately retained; the driver owns the active transcoder, @@ -216,6 +218,21 @@ namespace Drivers::Graphics::IntelGPU { return true; } + // This driver intentionally uses register polling for display latches and + // does not own the GPU's many generation-specific interrupt sources. Mask + // PCI delivery before touching display state: otherwise firmware-left MSI + // enables can route an unhandled GT interrupt and create an interrupt + // storm even though Montauk never requested one. + static void DisableInterruptDelivery() { + uint8_t bus = g_gpuInfo.pciBus; + uint8_t dev = g_gpuInfo.pciDevice; + uint8_t func = g_gpuInfo.pciFunction; + if (Pci::DisableInterruptDelivery(bus, dev, func)) { + KernelLogStream(WARNING, "IntelGPU") + << "Disabled pre-enabled GPU MSI/MSI-X; using bounded latch polling"; + } + } + // ========================================================================= // BAR0 MMIO Mapping // ========================================================================= @@ -357,12 +374,20 @@ namespace Drivers::Graphics::IntelGPU { return false; } - // Ensure pitch is at least width * 4 (BGRX8888) + // Ensure pitch is at least width * 4 (BGRX8888), and reject geometry + // whose size arithmetic could wrap before it reaches the allocator. if (g_fbPitch == 0) { g_fbPitch = g_fbWidth * 4; KernelLogStream(WARNING, "IntelGPU") << "Stride not available, assuming " << base::dec << g_fbPitch << " bytes"; } + if (g_fbWidth > (~0ULL / 4) + || g_fbPitch < g_fbWidth * 4 + || g_fbHeight > (~0ULL / g_fbPitch)) { + KernelLogStream(ERROR, "IntelGPU") + << "Invalid or overflowing framebuffer geometry"; + return false; + } g_fbSize = g_fbHeight * g_fbPitch; @@ -989,17 +1014,13 @@ namespace Drivers::Graphics::IntelGPU { << ", " << base::dec << g_gttEntryCount << " entries" << (g_gpuGen >= 8 ? " (64-bit PTEs)" : " (32-bit PTEs)"); - // Allocate a scratch page (zeroed) for future use - void* scratchPageVirt = Memory::g_pfa->AllocateZeroed(); - g_scratchPagePhys = Memory::SubHHDM(scratchPageVirt); - // Do NOT clear the entire GTT here. The firmware has active GTT mappings // that the display engine is currently scanning out from. Clearing them // would cause the display to go black (or worse) before we remap. // Instead, we only write the entries we need in SetupFramebuffer(). - KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec << g_gttEntryCount - << " entries, scratch page at " << base::hex << g_scratchPagePhys; + KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec + << g_gttEntryCount << " entries"; return true; } @@ -1108,18 +1129,47 @@ namespace Drivers::Graphics::IntelGPU { // Initialization maps this window once and preserves the firmware's active // scanout PTEs. Page flipping only proceeds after those entries validate. - static void MicroDelay(int us) { - // Simple busy-wait; us is approximate - for (volatile int i = 0; i < us * 100; i++) { - asm volatile("pause"); - } - } - static uint64_t ReadGgttPte(uint64_t index) { if (g_gpuGen >= 8) return ((volatile uint64_t*)g_ggtt)[index]; return ((volatile uint32_t*)g_ggtt)[index]; } + static int ReadLiveBufferIndex() { + if (!g_mmioBase) return -1; + uint32_t live = ReadReg(DSPASURFLIVE) & ~0xFFFu; + if (live == (uint32_t)g_fbGttOffsetA) return 0; + if (live == (uint32_t)g_buf1GttOffset) return 1; + return -1; + } + + // Polling-only latch wait. This is safe during both ordinary syscalls and + // ExitProcess teardown because it never enters the scheduler. Keeping + // flips independent of GPU MSI is deliberate: MSI would route every + // firmware-enabled GPU source, not just the one source this driver owns. + // Three observed frames are the normal timeout. Laptop panel self-refresh + // can transiently stop the scanline counter, so also enforce a wall-clock + // bound: one million uncached MMIO reads can otherwise hold the desktop in + // SYS_FBFLIP long enough to look permanently frozen (cursor included). + static bool WaitForLiveBuffer(int index) { + uint32_t expected = (uint32_t)((index == 0) ? g_fbGttOffsetA + : g_buf1GttOffset); + uint32_t previous = ReadReg(PIPE_DSL) & 0x1FFFu; + uint64_t started = Timekeeping::GetMilliseconds(); + int frameWraps = 0; + for (int i = 0; i < 200000; i++) { + uint32_t live = ReadReg(DSPASURFLIVE); + if (live == 0xFFFFFFFFu) return false; + if ((live & ~0xFFFu) == expected) return true; + uint32_t current = ReadReg(PIPE_DSL) & 0x1FFFu; + if (current < previous && ++frameWraps >= 3) return false; + previous = current; + if ((i & 0x3FF) == 0 && + Timekeeping::GetMilliseconds() - started >= 100) return false; + asm volatile("pause"); + } + return false; + } + static void WriteGgttPte(uint64_t index, uint64_t physAddr) { if (g_gpuGen >= 8) { ((volatile uint64_t*)g_ggtt)[index] = MakeGttPte64(physAddr); @@ -1168,14 +1218,37 @@ namespace Drivers::Graphics::IntelGPU { static bool AllocateBackBuffer() { uint64_t pages = (g_fbSize + 0xFFF) >> 12; - void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, pages); + if (pages == 0 || pages > 0x7FFFFFFFULL) { + KernelLogStream(WARNING, "IntelGPU") + << "Invalid second scanout buffer page count " << base::dec << pages; + return false; + } + + void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, (int)pages); if (!virt) { KernelLogStream(WARNING, "IntelGPU") << "Failed to allocate " << base::dec << pages << " contiguous pages for the second scanout buffer"; return false; } - g_buf1Phys = Memory::SubHHDM(virt); + uint64_t phys = Memory::SubHHDM(virt); + if ((pages - 1) > (UINT64_MAX - phys) / 0x1000ULL) { + Memory::g_pfa->Free(virt, (int)pages); + KernelLogStream(WARNING, "IntelGPU") + << "Second scanout buffer physical range overflows"; + return false; + } + uint64_t lastPhys = phys + (pages - 1) * 0x1000ULL; + if (!GgttCanAddress(phys, g_gpuGen) || !GgttCanAddress(lastPhys, g_gpuGen)) { + Memory::g_pfa->Free(virt, (int)pages); + KernelLogStream(WARNING, "IntelGPU") + << "Second scanout buffer lies outside the GPU physical-address width (" + << base::hex << phys << ".." << lastPhys << ")"; + return false; + } + + g_buf1Phys = phys; g_buf1Virt = (uint32_t*)virt; + g_buf1Pages = pages; // Display scanout does not snoop the CPU cache: remap the buffer // write-combining before touching it, same as the firmware FB. @@ -1192,23 +1265,50 @@ namespace Drivers::Graphics::IntelGPU { static bool FindAndMapGgttRange() { uint64_t pages = (g_fbSize + 0xFFF) >> 12; constexpr uint64_t guard = 16; + uint64_t need = pages + 2 * guard; uint64_t avoidFirst = g_fbGttOffsetA >> 12; uint64_t avoidLast = (g_fbGttOffsetA + g_fbSize - 1) >> 12; - // Montauk owns the GPU after firmware handoff. Reserve a deterministic - // high-aperture range for its second scanout buffer instead of trying - // to infer firmware ownership from scratch-PTE patterns. Keep guards - // around the allocation and never overlap the live firmware surface. - uint64_t base = g_ggttEntries / 2 + guard; - if (base <= avoidLast + guard) base = avoidLast + guard + 1; - base = (base + guard - 1) & ~(guard - 1); - if (base < guard || base + pages + guard > g_ggttEntries - || !(base + pages - 1 < avoidFirst || base > avoidLast)) { - KernelLogStream(WARNING, "IntelGPU") << "No safe owned GGTT range for " + // Firmware normally fills unused GGTT entries with one scratch-page + // PTE. Verify that invariant at the tail, then claim only a complete + // scratch-backed (or invalid) run. Never overwrite arbitrary valid + // mappings merely because they happen to sit in the upper half. + if (g_ggttEntries < 32 || need > g_ggttEntries / 2) return false; + uint64_t scratchPte = ReadGgttPte(g_ggttEntries - 1); + for (uint64_t probe = 2; probe <= 32; probe++) { + if (ReadGgttPte(g_ggttEntries - probe) != scratchPte) { + KernelLogStream(WARNING, "IntelGPU") + << "GGTT tail is owned/non-uniform; page flip unavailable"; + return false; + } + } + + uint64_t runStart = 0; + uint64_t run = 0; + bool found = false; + for (uint64_t idx = g_ggttEntries / 2; idx < g_ggttEntries; idx++) { + if (idx >= avoidFirst && idx <= avoidLast) { + run = 0; + continue; + } + uint64_t pte = ReadGgttPte(idx); + if ((pte & 1) && pte != scratchPte) { + run = 0; + continue; + } + if (run == 0) runStart = idx; + if (++run >= need) { + found = true; + break; + } + } + if (!found) { + KernelLogStream(WARNING, "IntelGPU") << "No verified free GGTT run for " << base::dec << pages << " back-buffer pages"; return false; } + uint64_t base = runStart + guard; uint64_t replacedFirst = ReadGgttPte(base); uint64_t replacedLast = ReadGgttPte(base + pages - 1); for (uint64_t i = 0; i < pages; i++) { @@ -1235,94 +1335,38 @@ namespace Drivers::Graphics::IntelGPU { return true; } - // ========================================================================= - // Page flipping: vblank interrupt (Gen 11+) - // ========================================================================= - - static void HandleInterrupt(uint8_t /*irq*/) { - // Gen 11+ flow: park the master enable, ack the pipe IIR, re-enable - uint32_t master = ReadReg(GFX_MSTR_INTR); - if (!(master & GFX_MSTR_INTR_DISPLAY)) { - WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE); - return; - } - WriteReg(GFX_MSTR_INTR, 0); - - uint32_t iir = ReadReg(DE_PIPE_A_IIR); - if (iir) WriteReg(DE_PIPE_A_IIR, iir); // write-1-to-clear - - WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE); - - if (iir & DE_PIPE_VBLANK) { - g_vblankCount = g_vblankCount + 1; - Sched::WakeObjectWaiters((void*)&g_vblankCount); - } - } - - static bool SetupMsi() { - uint8_t bus = g_gpuInfo.pciBus; - uint8_t dev = g_gpuInfo.pciDevice; - uint8_t func = g_gpuInfo.pciFunction; - - uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); - if (cap == 0) { - KernelLogStream(INFO, "IntelGPU") << "MSI capability not found"; - return false; - } - - uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); - bool is64bit = (msgCtrl & (1 << 7)) != 0; - - Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE); - if (is64bit) { - Pci::LegacyWrite32(bus, dev, func, cap + 8, 0); - Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR); - } else { - Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR); - } - - msgCtrl &= ~(0x70); // one message - msgCtrl |= (1 << 0); // MSI enable - Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl); - - uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND); - pciCmd |= Pci::PCI_CMD_INTX_DISABLE; - Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd); - - Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt); - - KernelLogStream(OK, "IntelGPU") << "MSI enabled: vector " << base::dec - << (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")" - << (is64bit ? " [64-bit]" : " [32-bit]"); - return true; - } - - static void EnableVblankIrq() { - WriteReg(DE_PIPE_A_IIR, 0xFFFFFFFFu); // clear stale events - WriteReg(DE_PIPE_A_IMR, ~DE_PIPE_VBLANK); // unmask vblank only - WriteReg(DE_PIPE_A_IER, DE_PIPE_VBLANK); - WriteReg(DISPLAY_INT_CTL, DISPLAY_IRQ_ENABLE); - WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE); - (void)ReadReg(GFX_MSTR_INTR); - } - static void SetupPageFlip() { if (!MapCorrectGgtt()) return; if (!VerifyFirmwareScanoutPtes()) return; if (!AllocateBackBuffer()) return; if (!FindAndMapGgttRange()) return; - // Vblank interrupts: Gen 11+ register layout only. Older generations - // still flip, with DSPASURFLIVE polling standing in for the IRQ. - if (g_gpuGen >= 11 && SetupMsi()) { - EnableVblankIrq(); - g_vblankIrqReady = true; - } - g_flipSupported = true; - g_frontBuffer = 0; - KernelLogStream(OK, "IntelGPU") << "Page flip ready: 2 buffers, vblank IRQ " - << (g_vblankIrqReady ? "on" : "off (SURFLIVE polling)"); + int live = ReadLiveBufferIndex(); + g_frontBuffer = (live >= 0) ? live : 0; + KernelLogStream(OK, "IntelGPU") + << "Page flip ready: 2 buffers, bounded SURFLIVE polling"; + } + + static bool FirmwareFramebufferMatchesGpu() { + uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth(); + uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight(); + uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch(); + bool match = g_fbWidth == fwWidth + && g_fbHeight == fwHeight + && g_fbPitch == fwPitch; + if (match) return true; + + // Userspace obtains its drawing dimensions from the firmware + // framebuffer API. A differently-sized GPU allocation would make the + // second mapping overrun (or alias only part of) the backing buffer. + KernelLogStream(ERROR, "IntelGPU") + << "GPU dimensions differ from firmware; unsafe framebuffer setup skipped"; + KernelLogStream(WARNING, "IntelGPU") << " GPU: " + << base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch; + KernelLogStream(WARNING, "IntelGPU") << " Firmware: " + << base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch; + return false; } // ========================================================================= @@ -1388,6 +1432,7 @@ namespace Drivers::Graphics::IntelGPU { } g_gpuGen = g_gpuInfo.gen; + DisableInterruptDelivery(); if (!MapMmio()) { KernelLogStream(ERROR, "IntelGPU") << "Failed to map MMIO region"; @@ -1400,6 +1445,7 @@ namespace Drivers::Graphics::IntelGPU { KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state"; return false; } + if (!FirmwareFramebufferMatchesGpu()) return false; InitializeDisplayManagement(); if (!InitializeGtt()) { @@ -1417,18 +1463,6 @@ namespace Drivers::Graphics::IntelGPU { SetupPageFlip(); - uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth(); - uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight(); - uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch(); - - if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) { - KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!"; - KernelLogStream(WARNING, "IntelGPU") << " GPU: " - << base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch; - KernelLogStream(WARNING, "IntelGPU") << " Firmware: " - << base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch; - } - KernelLogStream(OK, "IntelGPU") << "Initialization complete: " << base::dec << g_fbWidth << "x" << g_fbHeight << " @ " << base::hex << (uint64_t)g_fbBase; @@ -1446,6 +1480,7 @@ namespace Drivers::Graphics::IntelGPU { if (!DetectGpu()) { return; } + DisableInterruptDelivery(); // Step 2: Map BAR0 MMIO region if (!MapMmio()) { @@ -1461,6 +1496,7 @@ namespace Drivers::Graphics::IntelGPU { KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state"; return; } + if (!FirmwareFramebufferMatchesGpu()) return; InitializeDisplayManagement(); // Step 5: Initialize GTT @@ -1480,21 +1516,6 @@ namespace Drivers::Graphics::IntelGPU { g_initialized = true; - // Diagnostic: compare GPU-detected values with firmware/Limine values - uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth(); - uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight(); - uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch(); - - if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) { - KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!"; - KernelLogStream(WARNING, "IntelGPU") << " GPU: " - << base::dec << g_fbWidth << "x" << g_fbHeight - << " pitch=" << g_fbPitch; - KernelLogStream(WARNING, "IntelGPU") << " Firmware: " - << base::dec << fwWidth << "x" << fwHeight - << " pitch=" << fwPitch; - } - KernelLogStream(OK, "IntelGPU") << "Initialization complete: " << base::dec << g_fbWidth << "x" << g_fbHeight << " @ " << base::hex << (uint64_t)g_fbBase; @@ -1641,7 +1662,6 @@ namespace Drivers::Graphics::IntelGPU { if (g_edidValid) out->capabilities |= montauk::abi::DISPLAY_CAP_EDID; if (g_pwmSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_BRIGHTNESS; if (g_flipSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_PAGE_FLIP; - if (g_vblankIrqReady) out->capabilities |= montauk::abi::DISPLAY_CAP_VBLANK_IRQ; for (int i = 0; i < g_modeCount; i++) { if (i != g_currentMode && (g_modes[i].flags & montauk::abi::DISPLAY_MODE_DRIVER_VALID)) { @@ -1685,6 +1705,8 @@ namespace Drivers::Graphics::IntelGPU { return -2; if (modeIndex == g_currentMode) return modeIndex; + g_flipLock.Acquire(); + // Link rate and framebuffer geometry remain fixed. Only timings with // the same active area and pixel clock are safe without retraining the // firmware-established HDMI/DP/eDP link. @@ -1693,10 +1715,13 @@ namespace Drivers::Graphics::IntelGPU { if (requested.hdisplay != current.hdisplay || requested.vdisplay != current.vdisplay || requested.pixelClock == 0 - || requested.pixelClock != current.pixelClock) + || requested.pixelClock != current.pixelClock) { + g_flipLock.Release(); return -2; + } if (!ApplyModeTiming(requested)) { + g_flipLock.Release(); KernelLogStream(ERROR, "IntelGPU") << "Modeset transaction failed; previous mode restored"; return -3; } @@ -1704,6 +1729,7 @@ namespace Drivers::Graphics::IntelGPU { g_modes[g_currentMode].flags &= ~montauk::abi::DISPLAY_MODE_CURRENT; g_currentMode = modeIndex; g_modes[g_currentMode].flags |= montauk::abi::DISPLAY_MODE_CURRENT; + g_flipLock.Release(); KernelLogStream(OK, "IntelGPU") << "Mode set to " << base::dec << (uint64_t)requested.hdisplay << "x" << (uint64_t)requested.vdisplay << " @ " << (uint64_t)(g_modes[modeIndex].refreshMilliHz / 1000) << "Hz"; @@ -1741,59 +1767,131 @@ namespace Drivers::Graphics::IntelGPU { return 0; } + uint64_t GetBufferPageCount(int index) { + if (index == 0) return (g_fbSize + 0xFFF) >> 12; + if (index == 1 && g_flipSupported) return g_buf1Pages; + return 0; + } + int GetFrontBuffer() { - return g_frontBuffer; + g_flipLock.Acquire(); + int live = ReadLiveBufferIndex(); + if (live >= 0) g_frontBuffer = live; + g_flipLock.Release(); + return live; } - uint64_t GetVblankCount() { - return g_vblankCount; + int AcquireFlip(int pid) { + if (!g_flipSupported || pid < 0) return -1; + + g_flipLock.Acquire(); + int oldOwner = g_flipOwnerPid; + g_flipOwnerPid = pid; + int live = ReadLiveBufferIndex(); + if (live < 0) { + // Recover from an unrecognised/pending firmware surface by putting + // the legacy buffer back synchronously before userspace draws. + WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA); + (void)ReadReg(DSPASURF); + if (WaitForLiveBuffer(0)) live = 0; + } + if (live >= 0) g_frontBuffer = live; + g_flipLock.Release(); + if (oldOwner != pid) { + KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec + << (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner) + << " -> " << (uint64_t)pid; + } + KernelLogStream(DEBUG, "IntelGPU") << "Flip acquired by pid " + << base::dec << (uint64_t)pid << " (live=" + << (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live) << ")"; + return live; } - bool WaitVblank(uint64_t timeoutMs) { - if (!g_vblankIrqReady) return false; - uint64_t start = g_vblankCount; - // The vblank IRQ fires every frame, so a wakeup lost to the race - // between reading the counter and blocking only costs one frame; - // the timeout is a backstop for a stalled pipe. - Sched::BlockOnObject((void*)&g_vblankCount, timeoutMs ? timeoutMs : 50); - return g_vblankCount != start; - } + int64_t Flip(int index, bool waitVsync, int pid) { + if (!g_flipSupported || index < 0 || index > 1 || pid < 0) return -1; - int64_t Flip(int index, bool waitVsync) { - if (!g_flipSupported || index < 0 || index > 1) return -1; + g_flipLock.Acquire(); + int oldOwner = g_flipOwnerPid; + g_flipOwnerPid = pid; + // Scanout memory is exposed through WC mappings. Drain userspace's + // completed frame before programming the display-plane surface. + asm volatile("sfence" ::: "memory"); uint64_t gttOff = (index == 0) ? g_fbGttOffsetA : g_buf1GttOffset; WriteReg(DSPASURF, (uint32_t)gttOff); (void)ReadReg(DSPASURF); - g_frontBuffer = index; if (waitVsync) { - if (g_vblankIrqReady) { - WaitVblank(50); - } - // Confirm the latch: DSPASURFLIVE tracks the surface actually - // being scanned out. With a working IRQ this succeeds on the - // first read; without one it is the vsync wait itself. - for (int i = 0; i < 3000; i++) { - if ((ReadReg(DSPASURFLIVE) & ~0xFFFu) == (uint32_t)gttOff) break; - MicroDelay(10); + // DSPASURFLIVE is authoritative and avoids taking ownership of + // the GPU's shared interrupt domain merely to wait for vblank. + if (!WaitForLiveBuffer(index)) { + int live = ReadLiveBufferIndex(); + if (live >= 0) g_frontBuffer = live; + uint32_t programmed = ReadReg(DSPASURF); + uint32_t liveSurface = ReadReg(DSPASURFLIVE); + g_flipLock.Release(); + if (oldOwner != pid) { + KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec + << (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner) + << " -> " << (uint64_t)pid; + } + KernelLogStream(WARNING, "IntelGPU") << "Flip latch timeout: requested=" + << base::dec << (uint64_t)index << " live=" + << (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live) + << " DSPASURF=" << base::hex << (uint64_t)programmed + << " DSPASURFLIVE=" << (uint64_t)liveSurface; + return -1; } + g_frontBuffer = index; + } else { + int live = ReadLiveBufferIndex(); + if (live >= 0) g_frontBuffer = live; + } + g_flipLock.Release(); + if (oldOwner != pid) { + KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec + << (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner) + << " -> " << (uint64_t)pid; } return index; } - void SetFlipOwner(int pid) { - g_flipOwnerPid = pid; - } - void OnProcessExit(int pid) { - if (!g_flipSupported || pid != g_flipOwnerPid) return; + if (!g_flipSupported) return; + g_flipLock.Acquire(); + if (pid != g_flipOwnerPid) { + g_flipLock.Release(); + return; + } g_flipOwnerPid = -1; - if (g_frontBuffer != 0) Flip(0, false); + + uint32_t programmedBefore = ReadReg(DSPASURF); + uint32_t liveBefore = ReadReg(DSPASURFLIVE); + WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA); + (void)ReadReg(DSPASURF); + bool liveAtZero = WaitForLiveBuffer(0); + bool restored = liveAtZero; + int live = ReadLiveBufferIndex(); + if (live >= 0) g_frontBuffer = live; + uint32_t programmedAfter = ReadReg(DSPASURF); + uint32_t liveAfter = ReadReg(DSPASURFLIVE); + g_flipLock.Release(); + + KernelLogStream(restored ? INFO : ERROR, "IntelGPU") + << "Flip owner pid " << base::dec << (uint64_t)pid + << " exited; restore buffer 0 " << (restored ? "latched" : "timed out") + << " (before surf/live=" << base::hex << (uint64_t)programmedBefore + << "/" << (uint64_t)liveBefore << ", after=" + << (uint64_t)programmedAfter << "/" + << (uint64_t)liveAfter << ")"; } void PanicRestoreScanout() { - if (!g_flipSupported || !g_mmioBase || g_frontBuffer == 0) return; + // g_frontBuffer is only a software cache and can be stale precisely + // when the GPU or kernel is failing. Always request buffer 0 so the + // panic console cannot be left painting an invisible surface. + if (!g_flipSupported || !g_mmioBase) return; WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA); (void)ReadReg(DSPASURF); g_frontBuffer = 0; @@ -1809,6 +1907,7 @@ namespace Drivers::Graphics::IntelGPU { // command register, so MMIO writes to the GPU are silently dropped // until we re-enable these bits. Pci::EnableBusMaster(g_gpuInfo.pciBus, g_gpuInfo.pciDevice, g_gpuInfo.pciFunction); + DisableInterruptDelivery(); // Verify PCI memory space is accessible by reading back command register uint16_t pciCmd = Pci::LegacyRead16(g_gpuInfo.pciBus, g_gpuInfo.pciDevice, @@ -1875,9 +1974,8 @@ namespace Drivers::Graphics::IntelGPU { // 5. Reprogram display plane to point at our GTT-mapped framebuffer ProgramDisplayPlane(); - // 6. Restore page-flip state: rewrite buffer 1's GGTT entries, bring - // MSI + vblank interrupts back (the PCI config reset cleared MSI), - // and re-show whichever buffer was front when we suspended. + // 6. Restore page-flip state: rewrite buffer 1's GGTT entries and + // re-show whichever buffer was front when we suspended. if (g_flipSupported) { uint64_t pages = (g_fbSize + 0xFFF) >> 12; uint64_t baseIdx = g_buf1GttOffset >> 12; @@ -1886,11 +1984,6 @@ namespace Drivers::Graphics::IntelGPU { } (void)ReadGgttPte(baseIdx + pages - 1); - if (g_vblankIrqReady) { - SetupMsi(); - EnableVblankIrq(); - } - if (g_frontBuffer != 0) { WriteReg(DSPASURF, (uint32_t)g_buf1GttOffset); (void)ReadReg(DSPASURF); diff --git a/kernel/src/Drivers/Graphics/IntelGPU.hpp b/kernel/src/Drivers/Graphics/IntelGPU.hpp index 5d9c7d8..e7765fd 100644 --- a/kernel/src/Drivers/Graphics/IntelGPU.hpp +++ b/kernel/src/Drivers/Graphics/IntelGPU.hpp @@ -170,6 +170,7 @@ namespace Drivers::Graphics::IntelGPU { static constexpr uint32_t VBLANK_A = 0x60010; static constexpr uint32_t VSYNC_A = 0x60014; static constexpr uint32_t PIPEASRC = 0x6001C; + static constexpr uint32_t PIPE_DSL = 0x70000; // current pipe A scanline // --- Display timing registers (Pipe B) --- static constexpr uint32_t HTOTAL_B = 0x61000; @@ -231,30 +232,6 @@ namespace Drivers::Graphics::IntelGPU { // engine is currently scanning out; DSPASURF writes latch at vblank) --- static constexpr uint32_t DSPASURFLIVE = 0x701AC; - // --- Display engine interrupts (Gen 8+ pipe block, Gen 11+ master) --- - // Per-pipe interrupt registers, pipe A instance. Vblank is bit 0. - static constexpr uint32_t DE_PIPE_A_ISR = 0x44400; - static constexpr uint32_t DE_PIPE_A_IMR = 0x44404; - static constexpr uint32_t DE_PIPE_A_IIR = 0x44408; - static constexpr uint32_t DE_PIPE_A_IER = 0x4440C; - static constexpr uint32_t DE_PIPE_VBLANK = (1u << 0); - - // Gen 11+ display interrupt master control (0x44200 was the Gen 8-10 - // master IRQ register; on Gen 11+ it controls the display half only) - static constexpr uint32_t DISPLAY_INT_CTL = 0x44200; - static constexpr uint32_t DISPLAY_IRQ_ENABLE = (1u << 31); - static constexpr uint32_t DISPLAY_IRQ_PIPE_A = (1u << 16); - - // Gen 11+ top-level graphics master interrupt - static constexpr uint32_t GFX_MSTR_INTR = 0x190010; - static constexpr uint32_t GFX_MSTR_INTR_ENABLE = (1u << 31); - static constexpr uint32_t GFX_MSTR_INTR_DISPLAY = (1u << 16); - - // --- MSI (vblank interrupt delivery) --- - static constexpr uint8_t MSI_IRQ = 28; // IRQ slot 28 = vector 60 - static constexpr uint32_t MSI_VECTOR = 60; - static constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000; - // --- Output connectors --- static constexpr uint32_t ADPA = 0x61100; // Analog Display Port (VGA/CRT) static constexpr uint32_t DVOB = 0x61140; // DVO-B @@ -347,6 +324,7 @@ namespace Drivers::Graphics::IntelGPU { // Gen 8+ uses 64-bit GTT PTEs static constexpr uint64_t GTT_PTE64_VALID = (1ULL << 0); + static constexpr uint64_t GTT_PTE64_ADDRESS_MASK = 0x000FFFFFFFFFF000ULL; // Helper: Build a Gen 6/7 GTT PTE from a physical address static inline uint32_t MakeGttPte32(uint64_t physAddr) { @@ -358,7 +336,15 @@ namespace Drivers::Graphics::IntelGPU { // Helper: Build a Gen 8+ GTT PTE from a physical address static inline uint64_t MakeGttPte64(uint64_t physAddr) { - return (physAddr & ~0xFFFULL) | GTT_PTE64_VALID; + return (physAddr & GTT_PTE64_ADDRESS_MASK) | GTT_PTE64_VALID; + } + + // Gen 6/7 PTEs carry 39 physical-address bits; Gen 8+ PTEs carry 52. + // Never silently truncate a DMA address into a different RAM page. + static inline bool GgttCanAddress(uint64_t physAddr, int gpuGen) { + if (physAddr & 0xFFFULL) return false; + if (gpuGen >= 8) return (physAddr & ~0x000FFFFFFFFFFFFFULL) == 0; + return (physAddr & ~0x0000007FFFFFFFFFULL) == 0; } // ========================================================================= @@ -525,25 +511,26 @@ namespace Drivers::Graphics::IntelGPU { // Physical base of scanout buffer 0 (firmware FB) or 1 (kernel-allocated) uint64_t GetBufferPhys(int index); + // Number of physically allocated pages backing the requested buffer. + uint64_t GetBufferPageCount(int index); + // Index of the buffer currently programmed for scanout int GetFrontBuffer(); + // Claim page flipping for a process and return the buffer the display + // engine is actually scanning out (0 or 1). Returns -1 if the live + // surface cannot be identified. This lets a newly-active fullscreen + // client resynchronise its private back-buffer state after a handoff. + int AcquireFlip(int pid); + // Point the display plane at the given buffer. The hardware latches the // new surface address at the next vblank (tear-free). When waitVsync is // set, blocks until the flip has been latched. Returns the new front // buffer index, or -1 if flipping is unavailable. - int64_t Flip(int index, bool waitVsync); + int64_t Flip(int index, bool waitVsync, int pid); - // Block until the next vblank (Gen 11+ with working IRQ only). - // Returns false on timeout or when no vblank interrupt is available. - bool WaitVblank(uint64_t timeoutMs); - - // Monotonic vblank counter (0 when no vblank IRQ) - uint64_t GetVblankCount(); - - // Track which process performs flips so scanout can be restored to - // buffer 0 when it exits (called from the flip syscall / scheduler). - void SetFlipOwner(int pid); + // The flip operation atomically claims ownership for pid while changing + // scanout. OnProcessExit restores buffer 0 for the last flip owner. void OnProcessExit(int pid); // Panic-safe: force scanout back to buffer 0 so panic output is visible. diff --git a/kernel/src/Drivers/Net/E1000.cpp b/kernel/src/Drivers/Net/E1000.cpp index 5e778d5..2b987d1 100644 --- a/kernel/src/Drivers/Net/E1000.cpp +++ b/kernel/src/Drivers/Net/E1000.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include using namespace Kt; @@ -44,6 +46,11 @@ namespace Drivers::Net::E1000 { // Current descriptor indices static uint32_t g_rxTail = 0; static uint32_t g_txTail = 0; + static kcp::Spinlock g_txLock; + static std::atomic g_rxWorkPending{false}; + static std::atomic g_rxProcessing{false}; + + static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO; // Statistics static uint64_t g_rxPacketCount = 0; @@ -238,52 +245,31 @@ namespace Drivers::Net::E1000 { // Interrupt handler // ------------------------------------------------------------------------- - static void HandleInterrupt(uint8_t irq) { + static void HandleInterrupt(uint8_t irq, bool) { (void)irq; // Read and clear interrupt cause uint32_t icr = ReadReg(REG_ICR); if (icr & ICR_LSC) { - uint32_t status = ReadReg(REG_STATUS); - bool linkUp = (status & (1 << 1)) != 0; - KernelLogStream(INFO, "E1000") << "Link status change: " << (linkUp ? "UP" : "DOWN"); + // ICR is clear-on-read; no further action is required. Do not log + // here: the terminal mutex is process-context-only and an IRQ that + // preempts its owner would otherwise deadlock the CPU. + (void)ReadReg(REG_STATUS); } - // Both receive-timer and descriptor-threshold causes mean completed - // RX descriptors may be waiting. ICR is clear-on-read, so ignoring - // RXDMT0 can strand a lone packet until unrelated traffic arrives. - if (icr & (ICR_RXT0 | ICR_RXDMT0)) { - // Process received packets - while (true) { - uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; - RxDescriptor& desc = g_rxDescs[nextIdx]; - - if (!(desc.Status & RXSTA_DD)) { - break; // No more packets - } - - uint16_t length = desc.Length; - g_rxPacketCount++; - - // Dispatch to the network stack callback - if (g_rxCallback != nullptr) { - g_rxCallback(g_rxBuffers[nextIdx], length); - } - - // Reset descriptor for reuse - desc.Status = 0; - desc.Length = 0; - desc.Errors = 0; - - g_rxTail = nextIdx; - WriteReg(REG_RDT, g_rxTail); - } + // The callback enters Ethernet/TCP/UDP and IPC waitset notification, + // which takes process-context mutexes. Running it in this hard IRQ can + // self-deadlock if the interrupt preempts a holder on the same CPU. + // Mask RX causes and let an idle-context bottom half drain the ring. + if (icr & RX_INTERRUPT_MASK) { + WriteReg(REG_IMC, RX_INTERRUPT_MASK); + g_rxWorkPending.store(true, std::memory_order_release); } - if (icr & (ICR_TXDW | ICR_TXQE)) { - // TX completion - nothing to do for now - } + // TX completion interrupts are intentionally disabled. Descriptor DD + // is checked by SendPacket before reuse, so an interrupt per packet + // only creates a high-vector MSI load that can starve the LAPIC timer. } // ------------------------------------------------------------------------- @@ -314,6 +300,11 @@ namespace Drivers::Net::E1000 { // Enable bus mastering and memory space Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function); + Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function); + uint16_t pciCommand = Pci::LegacyRead16(dev.Bus, dev.Device, dev.Function, + (uint8_t)Pci::PCI_REG_COMMAND); + Pci::LegacyWrite16(dev.Bus, dev.Device, dev.Function, + (uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE); KernelLogStream(OK, "E1000") << "Bus mastering enabled"; // Read interrupt line from PCI config @@ -351,7 +342,7 @@ namespace Drivers::Net::E1000 { Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt); Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine)); - WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0); + WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC); g_initialized = true; @@ -403,6 +394,11 @@ namespace Drivers::Net::E1000 { // Enable bus mastering and memory space Pci::EnableBusMaster(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function); + Pci::DisableInterruptDelivery(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function); + uint16_t pciCommand = Pci::LegacyRead16(e1000Dev->Bus, e1000Dev->Device, + e1000Dev->Function, (uint8_t)Pci::PCI_REG_COMMAND); + Pci::LegacyWrite16(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function, + (uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE); KernelLogStream(OK, "E1000") << "Bus mastering enabled"; @@ -457,8 +453,8 @@ namespace Drivers::Net::E1000 { Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt); Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine)); - // Enable interrupts: RX, TX, Link Status Change - WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0); + // Enable RX and link-change only. TX completion is polled by descriptor. + WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC); g_initialized = true; @@ -473,10 +469,15 @@ namespace Drivers::Net::E1000 { return false; } - // Check if the current TX descriptor is available + // RX callbacks can send replies directly from interrupt context while + // userspace sends on another CPU. Serialize the shared tail/descriptor. + g_txLock.Acquire(); + + // Check if the current TX descriptor is available. Returning false is + // sufficient; logging here would be unsafe for the IRQ caller. TxDescriptor& desc = g_txDescs[g_txTail]; if (!(desc.Status & TXSTA_DD)) { - KernelLogStream(WARNING, "E1000") << "TX ring full"; + g_txLock.Release(); return false; } @@ -494,6 +495,7 @@ namespace Drivers::Net::E1000 { WriteReg(REG_TDT, g_txTail); g_txPacketCount++; + g_txLock.Release(); return true; } @@ -522,4 +524,50 @@ namespace Drivers::Net::E1000 { g_rxCallback = callback; } + bool HasDeferredWork() { + return g_initialized && g_rxWorkPending.load(std::memory_order_acquire); + } + + void ProcessDeferredWork() { + if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return; + + bool expected = false; + if (!g_rxProcessing.compare_exchange_strong(expected, true, + std::memory_order_acquire, std::memory_order_relaxed)) return; + + g_rxWorkPending.store(false, std::memory_order_release); + // Bound one bottom-half pass to one ring revolution. Hardware may + // refill descriptors as quickly as we return them under a packet + // flood; an unbounded drain could otherwise keep an idle context in + // the network stack indefinitely. The post-pass check below queues + // another fair slice when completed descriptors remain. + uint32_t processed = 0; + while (processed < RX_DESC_COUNT) { + uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; + RxDescriptor& desc = g_rxDescs[nextIdx]; + if (!(desc.Status & RXSTA_DD)) break; + + uint16_t length = desc.Length; + g_rxPacketCount++; + if (g_rxCallback != nullptr) g_rxCallback(g_rxBuffers[nextIdx], length); + + desc.Status = 0; + desc.Length = 0; + desc.Errors = 0; + g_rxTail = nextIdx; + WriteReg(REG_RDT, g_rxTail); + processed++; + } + + // Re-arm RX only after every completed descriptor was returned. Check + // once more after unmasking to close the packet-arrival race window. + WriteReg(REG_IMS, RX_INTERRUPT_MASK); + uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; + if (g_rxDescs[nextIdx].Status & RXSTA_DD) { + WriteReg(REG_IMC, RX_INTERRUPT_MASK); + g_rxWorkPending.store(true, std::memory_order_release); + } + g_rxProcessing.store(false, std::memory_order_release); + } + }; diff --git a/kernel/src/Drivers/Net/E1000.hpp b/kernel/src/Drivers/Net/E1000.hpp index 3e21332..f6b260d 100644 --- a/kernel/src/Drivers/Net/E1000.hpp +++ b/kernel/src/Drivers/Net/E1000.hpp @@ -128,4 +128,9 @@ namespace Drivers::Net::E1000 { // Register a callback for received packets void SetRxCallback(RxCallback callback); + // Drain packets queued by the hard IRQ from idle/process context. The + // Ethernet/IP/IPC stack is not safe to execute inside an interrupt. + void ProcessDeferredWork(); + bool HasDeferredWork(); + }; diff --git a/kernel/src/Drivers/Net/E1000E.cpp b/kernel/src/Drivers/Net/E1000E.cpp index dc95896..085c94f 100644 --- a/kernel/src/Drivers/Net/E1000E.cpp +++ b/kernel/src/Drivers/Net/E1000E.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include using namespace Kt; @@ -104,6 +106,11 @@ namespace Drivers::Net::E1000E { // Current descriptor indices static uint32_t g_rxTail = 0; static uint32_t g_txTail = 0; + static kcp::Spinlock g_txLock; + static std::atomic g_rxWorkPending{false}; + static std::atomic g_rxProcessing{false}; + + static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO; // Statistics static uint64_t g_rxPacketCount = 0; @@ -410,7 +417,7 @@ namespace Drivers::Net::E1000E { // MSI setup // ------------------------------------------------------------------------- - static void HandleInterrupt(uint8_t irq); // forward declaration + static void HandleInterrupt(uint8_t irq, bool fromUser); // forward declaration static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); @@ -419,6 +426,8 @@ namespace Drivers::Net::E1000E { return false; } + Pci::DisableInterruptDelivery(bus, dev, func); + KernelLogStream(INFO, "E1000E") << "MSI capability at offset " << base::hex << (uint64_t)cap; // Read Message Control (cap+2) @@ -461,7 +470,7 @@ namespace Drivers::Net::E1000E { // Interrupt handler // ------------------------------------------------------------------------- - static void HandleInterrupt(uint8_t irq) { + static void HandleInterrupt(uint8_t irq, bool) { (void)irq; uint32_t icr = ReadReg(REG_ICR); @@ -472,42 +481,21 @@ namespace Drivers::Net::E1000E { } if (icr & ICR_LSC) { - uint32_t status = ReadReg(REG_STATUS); - bool linkUp = (status & (1 << 1)) != 0; - KernelLogStream(INFO, "E1000E") << "Link status change: " << (linkUp ? "UP" : "DOWN"); + // Acknowledge/observe the new state without printing from IRQ + // context; KernelLogStream's terminal mutex is not IRQ-safe. + (void)ReadReg(REG_STATUS); } - // Both receive-timer and descriptor-threshold causes mean completed - // RX descriptors may be waiting. ICR is clear-on-read, so ignoring - // RXDMT0 can strand a lone packet until unrelated traffic arrives. - if (icr & (ICR_RXT0 | ICR_RXDMT0)) { - while (true) { - uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; - RxDescriptor& desc = g_rxDescs[nextIdx]; - - if (!(desc.Status & RXSTA_DD)) { - break; - } - - uint16_t length = desc.Length; - g_rxPacketCount++; - - if (g_rxCallback != nullptr) { - g_rxCallback(g_rxBuffers[nextIdx], length); - } - - desc.Status = 0; - desc.Length = 0; - desc.Errors = 0; - - g_rxTail = nextIdx; - WriteReg(REG_RDT, g_rxTail); - } + // Defer the full network/IPC stack. It takes process-context mutexes + // and cannot safely run nested under MSI/legacy interrupt delivery. + if (icr & RX_INTERRUPT_MASK) { + WriteReg(REG_IMC, RX_INTERRUPT_MASK); + g_rxWorkPending.store(true, std::memory_order_release); } - if (icr & (ICR_TXDW | ICR_TXQE)) { - // TX completion — nothing to do for now - } + // TX completion interrupts are intentionally disabled. Descriptor DD + // is checked by SendPacket before reuse, so an interrupt per packet + // only creates a high-vector MSI load that can starve the LAPIC timer. } // ------------------------------------------------------------------------- @@ -596,12 +584,17 @@ namespace Drivers::Net::E1000E { ConfigureInterruptModeration(); if (SetupMsi(bus, device, function)) { - WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0); + WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC); } else if (g_irqLine != 0xFF) { KernelLogStream(INFO, "E1000E") << "Falling back to legacy IRQ " << base::dec << (uint64_t)g_irqLine; + Pci::DisableInterruptDelivery(bus, device, function); + uint16_t command = Pci::LegacyRead16(bus, device, function, + (uint8_t)Pci::PCI_REG_COMMAND); + Pci::LegacyWrite16(bus, device, function, + (uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE); Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt); Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine)); - WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0); + WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC); } else { KernelLogStream(WARNING, "E1000E") << "No MSI or legacy IRQ available, using polling mode"; g_pollingMode = true; @@ -673,18 +666,12 @@ namespace Drivers::Net::E1000E { // Polling: process received packets (with reentrancy guard) // ------------------------------------------------------------------------- - static bool g_polling = false; - static void PollRx() { - // Guard against reentrancy: RX callback can trigger ARP reply → - // Ethernet::Send → SendPacket → PollRx(). Two concurrent callers - // modifying g_rxTail / RDT would corrupt the descriptor ring. - if (g_polling) { - return; - } - g_polling = true; - - while (true) { + // Never let a continuously refilled DMA ring monopolize the idle + // bottom half. Process at most one revolution; ProcessDeferredWork's + // final descriptor check schedules another pass if traffic remains. + uint32_t processed = 0; + while (processed < RX_DESC_COUNT) { uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; RxDescriptor& desc = g_rxDescs[nextIdx]; @@ -705,9 +692,9 @@ namespace Drivers::Net::E1000E { g_rxTail = nextIdx; WriteReg(REG_RDT, g_rxTail); + processed++; } - g_polling = false; } bool SendPacket(const uint8_t* data, uint16_t length) { @@ -715,9 +702,13 @@ namespace Drivers::Net::E1000E { return false; } + // RX processing can send replies from MSI/timer context while a + // userspace sender runs on another CPU. Protect the shared TX ring. + g_txLock.Acquire(); + TxDescriptor& desc = g_txDescs[g_txTail]; if (!(desc.Status & TXSTA_DD)) { - KernelLogStream(WARNING, "E1000E") << "TX ring full"; + g_txLock.Release(); return false; } @@ -732,6 +723,7 @@ namespace Drivers::Net::E1000E { WriteReg(REG_TDT, g_txTail); g_txPacketCount++; + g_txLock.Release(); return true; } @@ -765,10 +757,32 @@ namespace Drivers::Net::E1000E { } void Poll() { - if (!g_initialized) { - return; - } + if (g_initialized) g_rxWorkPending.store(true, std::memory_order_release); + } + + bool HasDeferredWork() { + return g_initialized && g_rxWorkPending.load(std::memory_order_acquire); + } + + void ProcessDeferredWork() { + if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return; + + bool expected = false; + if (!g_rxProcessing.compare_exchange_strong(expected, true, + std::memory_order_acquire, std::memory_order_relaxed)) return; + + g_rxWorkPending.store(false, std::memory_order_release); PollRx(); + + if (!g_pollingMode) { + WriteReg(REG_IMS, RX_INTERRUPT_MASK); + uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT; + if (g_rxDescs[nextIdx].Status & RXSTA_DD) { + WriteReg(REG_IMC, RX_INTERRUPT_MASK); + g_rxWorkPending.store(true, std::memory_order_release); + } + } + g_rxProcessing.store(false, std::memory_order_release); } }; diff --git a/kernel/src/Drivers/Net/E1000E.hpp b/kernel/src/Drivers/Net/E1000E.hpp index 9275d1c..9e36e63 100644 --- a/kernel/src/Drivers/Net/E1000E.hpp +++ b/kernel/src/Drivers/Net/E1000E.hpp @@ -165,7 +165,11 @@ namespace Drivers::Net::E1000E { // Register a callback for received packets void SetRxCallback(RxCallback callback); - // Poll for received packets (used when legacy IRQ is unavailable) + // Request an RX poll (used when hardware interrupts are unavailable). void Poll(); + // Drain queued RX descriptors outside MSI/timer interrupt context. + void ProcessDeferredWork(); + bool HasDeferredWork(); + }; diff --git a/kernel/src/Drivers/PS2/Keyboard.cpp b/kernel/src/Drivers/PS2/Keyboard.cpp index 6e4745d..94d1b7f 100644 --- a/kernel/src/Drivers/PS2/Keyboard.cpp +++ b/kernel/src/Drivers/PS2/Keyboard.cpp @@ -150,7 +150,7 @@ namespace Drivers::PS2::Keyboard { Kt::KernelLogStream(Kt::OK, "PS2/KB") << "Keyboard driver initialized"; } - void HandleIRQ(uint8_t irq) { + void HandleIRQ(uint8_t irq, bool) { (void)irq; uint8_t scancode = Io::In8(DataPort); diff --git a/kernel/src/Drivers/PS2/Keyboard.hpp b/kernel/src/Drivers/PS2/Keyboard.hpp index 2695241..f3970a5 100644 --- a/kernel/src/Drivers/PS2/Keyboard.hpp +++ b/kernel/src/Drivers/PS2/Keyboard.hpp @@ -39,7 +39,7 @@ namespace Drivers::PS2::Keyboard { void Initialize(); // Interrupt handler -- called from IRQ dispatch (EOI is sent automatically) - void HandleIRQ(uint8_t irq); + void HandleIRQ(uint8_t irq, bool fromUser); // Public interface for consuming key events bool IsKeyAvailable(); @@ -52,4 +52,4 @@ namespace Drivers::PS2::Keyboard { // Inject a key event from an external source (e.g., USB HID keyboard) void InjectKeyEvent(const KeyEvent& event); -}; \ No newline at end of file +}; diff --git a/kernel/src/Drivers/PS2/Mouse.cpp b/kernel/src/Drivers/PS2/Mouse.cpp index 6662c48..131c513 100644 --- a/kernel/src/Drivers/PS2/Mouse.cpp +++ b/kernel/src/Drivers/PS2/Mouse.cpp @@ -109,7 +109,7 @@ namespace Drivers::PS2::Mouse { Kt::KernelLogStream(Kt::OK, "PS2/Mouse") << "Mouse driver initialized"; } - void HandleIRQ(uint8_t irq) { + void HandleIRQ(uint8_t irq, bool) { (void)irq; uint8_t data = Io::In8(DataPort); diff --git a/kernel/src/Drivers/PS2/Mouse.hpp b/kernel/src/Drivers/PS2/Mouse.hpp index 00565a1..00471fb 100644 --- a/kernel/src/Drivers/PS2/Mouse.hpp +++ b/kernel/src/Drivers/PS2/Mouse.hpp @@ -24,7 +24,7 @@ namespace Drivers::PS2::Mouse { void Initialize(); // Interrupt handler -- called from IRQ dispatch (EOI is sent automatically) - void HandleIRQ(uint8_t irq); + void HandleIRQ(uint8_t irq, bool fromUser); // Public interface MouseState GetMouseState(); @@ -42,4 +42,4 @@ namespace Drivers::PS2::Mouse { // Inject a mouse report from an external source (e.g., USB HID mouse) void InjectMouseReport(uint8_t buttons, int8_t deltaX, int8_t deltaY, int8_t scroll); -}; \ No newline at end of file +}; diff --git a/kernel/src/Drivers/Storage/Ahci.cpp b/kernel/src/Drivers/Storage/Ahci.cpp index 6c0591d..b51954a 100644 --- a/kernel/src/Drivers/Storage/Ahci.cpp +++ b/kernel/src/Drivers/Storage/Ahci.cpp @@ -13,8 +13,6 @@ #include #include #include -#include -#include using namespace Kt; @@ -230,10 +228,9 @@ namespace Drivers::Storage::Ahci { WritePortReg(port, PORT_SERR, 0xFFFFFFFF); WritePortReg(port, PORT_IS, 0xFFFFFFFF); - // Enable interrupts for this port - WritePortReg(port, PORT_IE, - PORT_IS_DHRS | PORT_IS_PSS | PORT_IS_DSS | - PORT_IS_SDBS | PORT_IS_TFES); + // Commands are completed by polling PxCI/PxIS. Interrupt delivery is + // unnecessary and can starve the LAPIC timer under sustained I/O. + WritePortReg(port, PORT_IE, 0); // Power on and spin up if needed uint32_t cmd = ReadPortReg(port, PORT_CMD); @@ -533,65 +530,6 @@ namespace Drivers::Storage::Ahci { return true; } - // ------------------------------------------------------------------------- - // Interrupt handler - // ------------------------------------------------------------------------- - - static void HandleInterrupt(uint8_t irq) { - (void)irq; - - uint32_t is = ReadReg(REG_IS); - if (is == 0) return; - - // Acknowledge each port's interrupt - for (int i = 0; i < MAX_PORTS; i++) { - if (is & (1u << i)) { - uint32_t portIs = ReadPortReg(i, PORT_IS); - WritePortReg(i, PORT_IS, portIs); - } - } - - // Clear global interrupt status - WriteReg(REG_IS, is); - } - - // ------------------------------------------------------------------------- - // MSI setup - // ------------------------------------------------------------------------- - - static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { - uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); - if (cap == 0) { - KernelLogStream(INFO, "AHCI") << "MSI capability not found"; - return false; - } - - uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); - bool is64bit = (msgCtrl & (1 << 7)) != 0; - - Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE); - - if (is64bit) { - Pci::LegacyWrite32(bus, dev, func, cap + 8, 0); - Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR); - } else { - Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR); - } - - msgCtrl &= ~(0x70); // Single message - msgCtrl |= (1 << 0); // MSI Enable - Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl); - - uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND); - pciCmd |= Pci::PCI_CMD_INTX_DISABLE; - Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd); - - Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt); - - KernelLogStream(OK, "AHCI") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR; - return true; - } - // ------------------------------------------------------------------------- // HBA reset // ------------------------------------------------------------------------- @@ -659,6 +597,7 @@ namespace Drivers::Storage::Ahci { // Enable bus mastering and memory space Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function); + Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function); // BIOS/OS handoff PerformBiosHandoff(); @@ -690,22 +629,10 @@ namespace Drivers::Storage::Ahci { g_portsImplemented = ReadReg(REG_PI); KernelLogStream(INFO, "AHCI") << "Ports implemented: " << base::hex << (uint64_t)g_portsImplemented; - // Set up MSI (or fall back to legacy IRQ) - bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function); - - if (!hasMsi) { - uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function, - (uint8_t)Pci::PCI_REG_INTERRUPT); - if (irqLine != 0xFF) { - KernelLogStream(INFO, "AHCI") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine; - Hal::RegisterIrqHandler(irqLine, HandleInterrupt); - Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine)); - } - } - - // Enable global interrupts + // Keep controller interrupts disabled; every command path polls its + // completion and consumes/clears PxIS directly. uint32_t ghc = ReadReg(REG_GHC); - ghc |= GHC_IE; + ghc &= ~GHC_IE; WriteReg(REG_GHC, ghc); // Initialize each implemented port diff --git a/kernel/src/Drivers/Storage/Ahci.hpp b/kernel/src/Drivers/Storage/Ahci.hpp index a147de9..0aab4b6 100644 --- a/kernel/src/Drivers/Storage/Ahci.hpp +++ b/kernel/src/Drivers/Storage/Ahci.hpp @@ -181,11 +181,6 @@ namespace Drivers::Storage::Ahci { constexpr int MAX_PRDT_ENTRIES = 8; // Max PRDT entries per command constexpr int SECTOR_SIZE = 512; - // MSI configuration - constexpr uint8_t MSI_IRQ = 25; // IRQ slot 25 = vector 57 - constexpr uint32_t MSI_VECTOR = 57; - constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000; - // ========================================================================= // Port info // ========================================================================= diff --git a/kernel/src/Drivers/Storage/Nvme.cpp b/kernel/src/Drivers/Storage/Nvme.cpp index ccf3934..2fef021 100644 --- a/kernel/src/Drivers/Storage/Nvme.cpp +++ b/kernel/src/Drivers/Storage/Nvme.cpp @@ -13,8 +13,6 @@ #include #include #include -#include -#include #include using namespace Kt; @@ -511,9 +509,9 @@ namespace Drivers::Storage::Nvme { cmd.Prp1 = g_ioCqPhys; // CDW10: bits 31:16 = queue size (0-based), bits 15:0 = queue ID cmd.Cdw10 = ((uint32_t)(g_ioCqDepth - 1) << 16) | 1; - // CDW11: bit 0 = physically contiguous, bit 1 = interrupts enabled - // bits 31:16 = interrupt vector - cmd.Cdw11 = (1u << 0) | (1u << 1) | (0u << 16); + // Physically contiguous, with completion interrupts disabled. The + // queue is consumed synchronously by WaitCompletion(). + cmd.Cdw11 = (1u << 0); CqEntry cqe; if (!AdminCommand(cmd, cqe)) { @@ -549,54 +547,6 @@ namespace Drivers::Storage::Nvme { return true; } - // ------------------------------------------------------------------------- - // Interrupt handler - // ------------------------------------------------------------------------- - - static void HandleInterrupt(uint8_t irq) { - (void)irq; - // NVMe uses polling-based completion in this driver. - // The interrupt handler just acknowledges the interrupt. - // Completions are consumed in the polling loops above. - } - - // ------------------------------------------------------------------------- - // MSI setup - // ------------------------------------------------------------------------- - - static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { - uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); - if (cap == 0) { - KernelLogStream(INFO, "NVMe") << "MSI capability not found"; - return false; - } - - uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2); - bool is64bit = (msgCtrl & (1 << 7)) != 0; - - Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE); - - if (is64bit) { - Pci::LegacyWrite32(bus, dev, func, cap + 8, 0); - Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR); - } else { - Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR); - } - - msgCtrl &= ~(0x70); // Single message - msgCtrl |= (1 << 0); // MSI Enable - Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl); - - uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND); - pciCmd |= Pci::PCI_CMD_INTX_DISABLE; - Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd); - - Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt); - - KernelLogStream(OK, "NVMe") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR; - return true; - } - // ------------------------------------------------------------------------- // Probe (PCI driver entry point) // ------------------------------------------------------------------------- @@ -629,6 +579,7 @@ namespace Drivers::Storage::Nvme { // Enable bus mastering and memory space Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function); + Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function); // Read capabilities uint64_t cap = ReadReg64(REG_CAP); @@ -653,17 +604,9 @@ namespace Drivers::Storage::Nvme { return false; } - // Step 2: Set up MSI - bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function); - if (!hasMsi) { - uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function, - (uint8_t)Pci::PCI_REG_INTERRUPT); - if (irqLine != 0xFF) { - KernelLogStream(INFO, "NVMe") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine; - Hal::RegisterIrqHandler(irqLine, HandleInterrupt); - Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine)); - } - } + // Step 2: keep PCI interrupt delivery disabled. Admin and I/O queues + // are polled synchronously, so completion interrupts add latency and + // can starve the lower-vector scheduler timer under sustained I/O. // Step 3: Set up admin queues if (!SetupAdminQueues()) { diff --git a/kernel/src/Drivers/Storage/Nvme.hpp b/kernel/src/Drivers/Storage/Nvme.hpp index 102a9e3..adc3acf 100644 --- a/kernel/src/Drivers/Storage/Nvme.hpp +++ b/kernel/src/Drivers/Storage/Nvme.hpp @@ -158,14 +158,6 @@ namespace Drivers::Storage::Nvme { char Model[41]; }; - // ========================================================================= - // MSI configuration (use a different IRQ slot than AHCI) - // ========================================================================= - - constexpr uint8_t MSI_IRQ = 26; // IRQ slot 26 = vector 58 - constexpr uint32_t MSI_VECTOR = 58; - constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000; - // ========================================================================= // Public API // ========================================================================= diff --git a/kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp b/kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp index e4dd459..6e54d0f 100644 --- a/kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp +++ b/kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp @@ -47,7 +47,12 @@ namespace Drivers::USB::Bluetooth::Avrcp { // AVRCP notification events constexpr uint8_t EVT_VOLUME_CHANGED = 0x0D; - static kcp::Spinlock g_notifyLock; + // AVRCP callbacks are dispatched from xHCI's deferred PollEvents path, not + // from the hard IRQ. Sending the INTERIM/CHANGED response can descend into + // the HCI and xHCI transmit rings, so keep interrupts enabled while this + // state is serialized. An IRQ-disabling spinlock here can stall the local + // timer behind an unrelated transmit/recovery operation. + static kcp::Mutex g_notifyLock; static uint16_t g_volumeNotifyCid = 0; static uint8_t g_volumeNotifyTransaction = 0; diff --git a/kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp b/kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp index 1bbf639..aa0fe64 100644 --- a/kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp +++ b/kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp @@ -406,12 +406,13 @@ namespace Drivers::USB::Bluetooth { // system this briefly pauses userspace, matching the old synchronous // behavior minus the boot-path stall. auto* cpu = Smp::GetCurrentCpuData(); + bool wasReserved = cpu && cpu->reservedForKernelWork; if (cpu) cpu->reservedForKernelWork = true; Hci::SetFwTrace(true); // bounded per-completion event-pipe trace CompleteInit(); Hci::DumpFwTrace(); // flush remaining records (process context) Hci::SetFwTrace(false); - if (cpu) cpu->reservedForKernelWork = false; + if (cpu) cpu->reservedForKernelWork = wasReserved; } // ========================================================================= diff --git a/kernel/src/Drivers/USB/Bluetooth/Hci.cpp b/kernel/src/Drivers/USB/Bluetooth/Hci.cpp index 40870c3..f2ce045 100644 --- a/kernel/src/Drivers/USB/Bluetooth/Hci.cpp +++ b/kernel/src/Drivers/USB/Bluetooth/Hci.cpp @@ -59,10 +59,9 @@ namespace Drivers::USB::Bluetooth::Hci { // // The trace is recorded into a lock-free ring and printed LATER by // DumpFwTrace() from process context. It must NEVER KernelLogStream from - // TransferCallback: that runs nested under PollEvents in xHCI MSI (IRQ) - // context, and the terminal lock is a non-IRQ-disabling Mutex -- an IRQ - // logger interrupting a same-core holder spins forever (observed: hard - // boot freeze right after "Intel version raw", 2026-07-06). + // TransferCallback: it runs nested under PollEvents while the event-ring + // ownership guard is held. Logging there can recursively pump or contend + // with unrelated subsystem locks, so keep it a bounded data-only callback. static std::atomic g_fwTrace{false}; static std::atomic g_fwTraceCount{0}; static std::atomic g_intInCompletions{0}; @@ -148,8 +147,10 @@ namespace Drivers::USB::Bluetooth::Hci { // guards, so one core can be sending a media packet while another sends a // signaling reply; without this lock the two race on g_aclTxSlot and on // the bulk OUT TRB ring's enqueue/cycle state, garbling packets on air. - // IRQ-disabling spinlock so the holder also can't be preempted mid-send. - static kcp::Spinlock g_aclTxLock; + // These paths run only from deferred/process context. Keep interrupts + // enabled: endpoint recovery also takes this lock and can wait for bounded + // xHCI commands, which must never happen below an IRQ-disabling spinlock. + static kcp::Mutex g_aclTxLock; // HCI command DMA buffer (separate from ACL to avoid conflicts) static uint8_t* g_cmdDmaBuf = nullptr; @@ -559,8 +560,8 @@ namespace Drivers::USB::Bluetooth::Hci { // Bounded bring-up trace: one RING RECORD per interrupt-IN // completion while the firmware phase runs; DumpFwTrace() prints - // them later from process context. No logging here -- this runs - // in xHCI MSI (IRQ) context (see the deadlock note at the ring). + // them later from top-level process/idle context. No logging here: + // this callback is nested under the xHCI event-ring owner. if (g_fwTrace.load(std::memory_order_relaxed)) { uint32_t n = g_fwTraceCount.fetch_add(1, std::memory_order_relaxed); if (n < FW_TRACE_CAP) { @@ -2074,6 +2075,10 @@ namespace Drivers::USB::Bluetooth::Hci { // all TRBs queued behind it. Free those DMA slots and remove only // those never-delivered packets from the controller-credit count; // earlier successful OUT transfers still await their NOCP events. + // Serialize the dequeue rewrite with SendAcl/firmware bulk enqueue. + // Without this, another CPU can publish a TRB between Reset Endpoint + // and Set TR Dequeue and have recovery silently discard or split it. + g_aclTxLock.Acquire(); uint32_t discarded = AclTxInFlight(); KernelLogStream(WARNING, "BT-HCI") << "Recovering HCI ACL transmit pipe; discarding " @@ -2086,6 +2091,7 @@ namespace Drivers::USB::Bluetooth::Hci { - (int32_t)discarded; if (remaining < 0) g_aclPendingCount.store(0, std::memory_order_release); + g_aclTxLock.Release(); } // Asynchronous HCI events are processed at top level. Besides avoiding diff --git a/kernel/src/Drivers/USB/HidMouse.cpp b/kernel/src/Drivers/USB/HidMouse.cpp index 42d7517..2d42147 100644 --- a/kernel/src/Drivers/USB/HidMouse.cpp +++ b/kernel/src/Drivers/USB/HidMouse.cpp @@ -172,18 +172,23 @@ namespace Drivers::USB::HidMouse { // ------------------------------------------------------------------------- static int32_t ExtractSigned(const uint8_t* data, uint16_t bitOffset, uint8_t bitSize) { - int32_t value = 0; + uint32_t value = 0; for (uint8_t i = 0; i < bitSize; i++) { uint16_t byteIdx = (bitOffset + i) / 8; uint8_t bitIdx = (bitOffset + i) % 8; if (data[byteIdx] & (1 << bitIdx)) - value |= (1 << i); + value |= (1u << i); } // Sign extend - if (bitSize < 32 && (value & (1 << (bitSize - 1)))) { - value |= ~((1 << bitSize) - 1); + if (bitSize < 32 && (value & (1u << (bitSize - 1)))) { + value |= ~((1u << bitSize) - 1); } - return value; + return (int32_t)value; + } + + static bool FieldFits(uint32_t bitOffset, uint32_t bitSize, uint32_t reportBits) { + return bitSize > 0 && bitSize <= 32 && bitOffset <= reportBits + && bitSize <= reportBits - bitOffset; } // ------------------------------------------------------------------------- @@ -222,18 +227,30 @@ namespace Drivers::USB::HidMouse { // Skip report ID byte if present uint16_t byteOffset = g_Format.hasReportId ? 8 : 0; + if (g_Format.hasReportId && data[0] != g_Format.reportId) return; + uint32_t reportBits = (uint32_t)length * 8; + + uint32_t buttonBits = g_Format.buttonCount < 8 ? g_Format.buttonCount : 8; + uint32_t buttonOffset = byteOffset + g_Format.buttonBitOffset; + uint32_t xOffset = byteOffset + g_Format.xBitOffset; + uint32_t yOffset = byteOffset + g_Format.yBitOffset; + if ((buttonBits != 0 && !FieldFits(buttonOffset, buttonBits, reportBits)) + || !FieldFits(xOffset, g_Format.xBitSize, reportBits) + || !FieldFits(yOffset, g_Format.yBitSize, reportBits)) { + return; + } // Extract buttons uint8_t buttons = 0; - for (uint8_t i = 0; i < g_Format.buttonCount && i < 8; i++) { - uint16_t bit = byteOffset + g_Format.buttonBitOffset + i; + for (uint8_t i = 0; i < buttonBits; i++) { + uint16_t bit = (uint16_t)(buttonOffset + i); if (data[bit / 8] & (1 << (bit % 8))) buttons |= (1 << i); } // Extract X, Y - int32_t rawX = ExtractSigned(data, byteOffset + g_Format.xBitOffset, g_Format.xBitSize); - int32_t rawY = ExtractSigned(data, byteOffset + g_Format.yBitOffset, g_Format.yBitSize); + int32_t rawX = ExtractSigned(data, (uint16_t)xOffset, g_Format.xBitSize); + int32_t rawY = ExtractSigned(data, (uint16_t)yOffset, g_Format.yBitSize); // Clamp to int8_t range for InjectMouseReport if (rawX > 127) rawX = 127; @@ -244,7 +261,10 @@ namespace Drivers::USB::HidMouse { // Extract scroll wheel int8_t scroll = 0; if (g_Format.scrollBitSize > 0) { - int32_t rawScroll = ExtractSigned(data, byteOffset + g_Format.scrollBitOffset, g_Format.scrollBitSize); + uint32_t scrollOffset = byteOffset + g_Format.scrollBitOffset; + if (!FieldFits(scrollOffset, g_Format.scrollBitSize, reportBits)) return; + int32_t rawScroll = ExtractSigned(data, (uint16_t)scrollOffset, + g_Format.scrollBitSize); if (rawScroll > 127) rawScroll = 127; if (rawScroll < -128) rawScroll = -128; scroll = (int8_t)rawScroll; diff --git a/kernel/src/Drivers/USB/Xhci.cpp b/kernel/src/Drivers/USB/Xhci.cpp index 647bdc6..3a8fef9 100644 --- a/kernel/src/Drivers/USB/Xhci.cpp +++ b/kernel/src/Drivers/USB/Xhci.cpp @@ -54,9 +54,30 @@ namespace Drivers::USB::Xhci { static bool g_bootScanComplete = false; // true after initial port scan finishes // Hot-plug deferred work - static volatile bool g_hotplugPending[MAX_PORTS] = {}; - static volatile bool g_deferredWorkPending = false; + static std::atomic g_hotplugPending[MAX_PORTS] = {}; + static std::atomic g_deferredWorkPending{false}; + static std::atomic g_diagnosticWorkPending{false}; static std::atomic g_hotplugProcessing{false}; + static std::atomic g_irqDrainPending{false}; + + // PollEvents also runs directly from the MSI handler. KernelLogStream uses + // a process-context mutex, so printing there can deadlock if the interrupt + // preempts a CPU that already owns the terminal lock. Capture the small + // diagnostic payload atomically and print it from ProcessDeferredWork(). + static std::atomic g_transferErrorsPending{0}; + static std::atomic g_transferErrorReports{0}; + static std::atomic g_lastErrorSlot{0}; + static std::atomic g_lastErrorEp{0}; + static std::atomic g_lastErrorCc{0}; + static std::atomic g_stormsPending{0}; + static std::atomic g_stormReports{0}; + static std::atomic g_lastStormProcessed{0}; + static std::atomic g_lastStormPortEvents{0}; + static std::atomic g_lastStormTransferEvents{0}; + static std::atomic g_lastStormType{0}; + static std::atomic g_lastStormSlot{0}; + static std::atomic g_lastStormEp{0}; + static std::atomic g_lastStormCc{0}; // MMIO region pointers static volatile uint8_t* g_mmioBase = nullptr; @@ -78,6 +99,12 @@ namespace Drivers::USB::Xhci { static uint64_t g_cmdRingPhys = 0; static uint32_t g_cmdRingEnqueue = 0; static bool g_cmdRingCCS = true; + // The controller exposes one command ring and one command-completion + // mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer + // recovery can originate on different CPUs, so the whole submit/wait + // transaction must be single-owner. This is a process/idle-context lock; + // SendCommand is never called from a transfer-event callback. + static kcp::Mutex g_commandLock; // Event ring static TRB* g_evtRing = nullptr; @@ -125,12 +152,12 @@ namespace Drivers::USB::Xhci { // completion. Used by InPollContext() / ControlTransfer(). // // MUST be a real atomic claimed with compare-exchange, not a plain - // volatile bool: PollEvents runs from the MSI handler (BSP) AND from - // syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the - // app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set - // on a volatile is a cross-core TOCTOU: two cores both pass the check, - // both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync, - // and completions get double-delivered (duplicate ACL packets) or lost + // volatile bool: PollEvents runs from syscall-context pumps (A2DP + // WriteAudio/StartSource on whatever core the app runs on) and from idle + // cores (ProcessDeferredWork). A check-then-set on a volatile is a + // cross-core TOCTOU: two cores both pass the check, both drain the shared + // event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get + // double-delivered (duplicate ACL packets) or lost // (missed AVDTP responses) -- both observed on HW once audio streaming // started pumping from syscall context. static std::atomic g_pollActive{false}; @@ -293,7 +320,7 @@ namespace Drivers::USB::Xhci { // Forward declarations // ------------------------------------------------------------------------- - static void HandleInterrupt(uint8_t irq); + static void HandleInterrupt(uint8_t irq, bool fromUser); // ------------------------------------------------------------------------- // BIOS-to-OS handoff (USB Legacy Support) @@ -389,10 +416,13 @@ namespace Drivers::USB::Xhci { static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) { uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI); if (cap == 0) { + Pci::DisableInterruptDelivery(bus, dev, func); KernelLogStream(INFO, "xHCI") << "MSI capability not found"; return false; } + Pci::DisableInterruptDelivery(bus, dev, func); + KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap; // Read Message Control (cap+2) @@ -448,33 +478,37 @@ namespace Drivers::USB::Xhci { // PollEvents - process event ring // ------------------------------------------------------------------------- - void PollEvents() { - // PollEvents runs both from the synchronous poll loops (ControlTransfer, - // SendCommand, firmware download) and from the xHCI MSI handler. On a - // single core the IRQ can preempt a poll loop mid-drain; if both advance - // g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the - // cycle-bit check can start matching stale entries forever -> the boot - // freezes (observed wedging the Bluetooth firmware download at ~635 KB, - // where the dying device floods the event ring). Guard against re-entry: - // the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and - // the active poll loop drains these events itself. g_pollActive is also - // read by InPollContext() so command submitters (ControlTransfer) can - // tell they are nested and must fire-and-forget instead of waiting. + enum class PollResult : uint8_t { + Drained, + Busy, + BudgetExhausted, + }; + + static PollResult PollEventsBounded(uint32_t maxEvents) { + // PollEvents runs from synchronous poll loops (ControlTransfer, + // SendCommand, firmware download) and idle bottom halves on different + // CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring + // tracking desyncs and the cycle-bit check can start matching stale + // entries forever -> the boot freezes (observed wedging the Bluetooth + // firmware download at ~635 KB, where the dying device floods the event + // ring). Guard against cross-core re-entry. g_pollActive is also read by + // InPollContext() so command submitters reached from a callback can tell + // they are nested and must fire-and-forget instead of waiting. bool expected = false; if (!g_pollActive.compare_exchange_strong(expected, true, std::memory_order_acquire)) { - return; // another context is draining; it will reap our events + return PollResult::Busy; // another context is draining } g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed); - // Bound the work per call so a flooding/wedged device can never spin - // here forever; the outer wall-clock timeouts then fire instead of - // freezing. 4x the ring size is far above any legitimate burst. - constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4; + // Bound each pass so a flooding/wedged device cannot monopolize its + // caller. Synchronous poll loops use a generous batch; the timer uses + // a much smaller recovery slice after the hard IRQ masks the source. + if (maxEvents == 0) maxEvents = 1; uint32_t processed = 0, portEvts = 0, xferEvts = 0; uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0; - while (processed < MAX_EVENTS_PER_CALL) { + while (processed < maxEvents) { TRB& evt = g_evtRing[g_evtRingDequeue]; // Check if the cycle bit matches our expected cycle state @@ -506,8 +540,8 @@ namespace Drivers::USB::Xhci { // Defer enumeration to ProcessDeferredWork outside interrupt context. if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) { - g_hotplugPending[portId - 1] = true; - g_deferredWorkPending = true; + g_hotplugPending[portId - 1].store(true, std::memory_order_release); + g_deferredWorkPending.store(true, std::memory_order_release); } break; } @@ -616,9 +650,16 @@ namespace Drivers::USB::Xhci { nullptr, 0, completionCode); } } else { - KernelLogStream(WARNING, "xHCI") << "Transfer error on slot " - << base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci - << " cc=" << (uint64_t)completionCode; + // Never print from this path: PollEvents may be + // running in the MSI handler. Defer a rate-limited + // diagnostic to idle/process context instead. + if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) { + g_lastErrorSlot.store(slotId, std::memory_order_relaxed); + g_lastErrorEp.store(epDci, std::memory_order_relaxed); + g_lastErrorCc.store(completionCode, std::memory_order_relaxed); + g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed); + g_diagnosticWorkPending.store(true, std::memory_order_release); + } // Notify callback of errors too if (g_transferCallbacks[slotId]) { @@ -651,37 +692,70 @@ namespace Drivers::USB::Xhci { WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF)); WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32)); - // A full batch means the ring is being flooded -- surface the dominant - // event source (rate-limited) so a wedge is diagnosable, not silent. - if (processed >= MAX_EVENTS_PER_CALL) { - static uint32_t stormLogs = 0; - if (stormLogs < 8) { - stormLogs++; - KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed - << "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts - << " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot - << " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")"; + // A full batch means the ring is being flooded. Record the dominant + // source for a deferred, rate-limited process-context diagnostic. + bool exhausted = processed >= maxEvents; + if (exhausted) { + if (g_stormReports.load(std::memory_order_relaxed) < 8) { + g_lastStormProcessed.store(processed, std::memory_order_relaxed); + g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed); + g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed); + g_lastStormType.store(lastType, std::memory_order_relaxed); + g_lastStormSlot.store(lastSlot, std::memory_order_relaxed); + g_lastStormEp.store(lastEp, std::memory_order_relaxed); + g_lastStormCc.store(lastCC, std::memory_order_relaxed); + g_stormsPending.fetch_add(1, std::memory_order_relaxed); + g_diagnosticWorkPending.store(true, std::memory_order_release); } } g_pollOwnerCpu.store(-1, std::memory_order_relaxed); g_pollActive.store(false, std::memory_order_release); + return exhausted ? PollResult::BudgetExhausted : PollResult::Drained; + } + + void PollEvents() { + PollResult result = PollEventsBounded(EVT_RING_SIZE * 4); + if (result == PollResult::Drained && + g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) { + // The hard IRQ leaves the interrupter masked. A synchronous + // process-context poll may drain the ring before an idle bottom + // half sees it, so finish the handoff here as well. Verify once + // more while masked: on another CPU, the IRQ may have acknowledged + // IMAN.IP for an event posted just after our first empty check. In + // that race, blindly enabling IE would strand the event because + // the IRQ already cleared its pending bit. + PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4); + if (verify != PollResult::Drained) { + g_irqDrainPending.store(true, std::memory_order_release); + return; + } + + // If hardware queues an event after this masked verification, + // IMAN.IP remains asserted and enabling IE generates a fresh MSI. + WriteRt(IR0_IMAN, IMAN_IE); + } } // ------------------------------------------------------------------------- // HandleInterrupt // ------------------------------------------------------------------------- - static void HandleInterrupt(uint8_t irq) { + static void HandleInterrupt(uint8_t irq, bool) { (void)irq; // Clear USBSTS.EINT only (don't accidentally clear other W1C bits) WriteOp(OP_USBSTS, USBSTS_EINT); - // Clear IMAN.IP and ensure IE stays enabled - WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE); - - PollEvents(); + // Keep the hard IRQ strictly bounded. PollEvents dispatches HID, + // Bluetooth, and SDR callbacks; those callbacks can take subsystem + // locks, wake scheduler objects, re-arm endpoints, and copy sizeable + // buffers. Running them here lets a USB burst monopolize this + // higher-priority vector ahead of the LAPIC timer and can deadlock if + // an interrupted context owns one of those locks. Acknowledge and mask + // the source, then let an idle/process-context poll drain the ring. + WriteRt(IR0_IMAN, IMAN_IP); + g_irqDrainPending.store(true, std::memory_order_release); } // ------------------------------------------------------------------------- @@ -689,6 +763,8 @@ namespace Drivers::USB::Xhci { // ------------------------------------------------------------------------- uint32_t SendCommand(const TRB& trb) { + g_commandLock.Acquire(); + // Place TRB at current enqueue position TRB& slot = g_cmdRing[g_cmdRingEnqueue]; slot.Parameter0 = trb.Parameter0; @@ -728,7 +804,9 @@ namespace Drivers::USB::Xhci { while (Timekeeping::GetMilliseconds() - cmdStart < 2000) { PollEvents(); if (g_cmdCompleted) { - return g_cmdCompletionCode; + uint32_t result = g_cmdCompletionCode; + g_commandLock.Release(); + return result; } // Small delay for (int j = 0; j < 100; j++) { @@ -736,6 +814,9 @@ namespace Drivers::USB::Xhci { } } + // Do not hold the command-ring lock while taking the terminal mutex. + // A later caller may attempt recovery even if this command timed out. + g_commandLock.Release(); KernelLogStream(WARNING, "xHCI") << "Command timeout"; return 0xFF; } @@ -1304,7 +1385,55 @@ namespace Drivers::USB::Xhci { } bool HasDeferredWork() { - return g_initialized && g_deferredWorkPending; + return g_initialized && + (g_irqDrainPending.load(std::memory_order_acquire) || + g_deferredWorkPending.load(std::memory_order_acquire) || + g_diagnosticWorkPending.load(std::memory_order_acquire)); + } + + void ServiceDeferredIrqEvents(uint32_t budget) { + if (!g_initialized || + !g_irqDrainPending.load(std::memory_order_acquire)) return; + + PollResult result = PollEventsBounded(budget); + if (result != PollResult::Drained) return; + + // Clear before re-enabling. If hardware queued an event after the ring + // appeared empty, IMAN.IP remains asserted and enabling IE generates a + // fresh MSI; no event can be lost in this handoff. + if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) { + WriteRt(IR0_IMAN, IMAN_IE); + } + } + + static void PrintDeferredDiagnostics() { + // Clear the wakeup before consuming counters. A producer racing after + // this store sets it again, so deferred work cannot be lost. + g_diagnosticWorkPending.store(false, std::memory_order_release); + + uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel); + if (errors != 0 && + g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) { + KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors + << " transfer error(s); last slot=" + << (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed) + << " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed) + << " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed); + } + + uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel); + if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) { + KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms + << " event storm(s); last batch=" + << (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed) + << " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed) + << " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed) + << " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed) + << " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed) + << " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed) + << " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")"; + } + } static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) { @@ -1339,7 +1468,19 @@ namespace Drivers::USB::Xhci { // ------------------------------------------------------------------------- void ProcessDeferredWork() { - if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return; + if (!g_initialized) return; + + // The hard IRQ only acknowledges and masks the interrupter. Drain a + // bounded slice here, outside interrupt context, so callbacks may use + // their normal locks without blocking the timer or scheduler. A busy + // process that is synchronously waiting for USB calls PollEvents() + // itself, which performs the same unmask handoff when the ring is dry. + ServiceDeferredIrqEvents(64); + + bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire); + bool haveHotplug = g_bootScanComplete && + g_deferredWorkPending.load(std::memory_order_acquire); + if (!haveDiagnostics && !haveHotplug) return; // CAS claim: any idle core may call this, but only one runs at a time. // AcquireTransport in MassStorage handles the SCSI-in-flight case itself. @@ -1348,11 +1489,17 @@ namespace Drivers::USB::Xhci { std::memory_order_acquire, std::memory_order_relaxed)) { return; } - g_deferredWorkPending = false; + if (haveDiagnostics) PrintDeferredDiagnostics(); + + if (!haveHotplug) { + g_hotplugProcessing.store(false, std::memory_order_release); + return; + } + + g_deferredWorkPending.store(false, std::memory_order_release); for (uint32_t port = 0; port < g_maxPorts; port++) { - if (!g_hotplugPending[port]) continue; - g_hotplugPending[port] = false; + if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue; uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE); diff --git a/kernel/src/Drivers/USB/Xhci.hpp b/kernel/src/Drivers/USB/Xhci.hpp index 6ab2bba..0e6e5af 100644 --- a/kernel/src/Drivers/USB/Xhci.hpp +++ b/kernel/src/Drivers/USB/Xhci.hpp @@ -20,9 +20,11 @@ namespace Drivers::USB::Xhci { constexpr uint32_t EVT_RING_SIZE = 64; constexpr uint32_t XFER_RING_SIZE = 32; - // MSI configuration (E1000E uses IRQ 24/vector 56, we use 25/57) - constexpr uint8_t MSI_IRQ = 25; - constexpr uint32_t MSI_VECTOR = 57; + // MSI configuration. Keep xHCI distinct from AHCI's IRQ 25/vector 57; + // the IRQ dispatcher has one handler per slot, so sharing would replace + // one driver's handler and eventually strand USB or storage completions. + constexpr uint8_t MSI_IRQ = 29; + constexpr uint32_t MSI_VECTOR = 61; constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000; // PCI class/subclass/progif for xHCI @@ -299,6 +301,10 @@ namespace Drivers::USB::Xhci { // Deferred hot-plug processing (call outside interrupt context) void ProcessDeferredWork(); + // Drain a bounded event batch after the hard IRQ masked the interrupter. + // Call only from process/idle context; this dispatches class callbacks. + void ServiceDeferredIrqEvents(uint32_t budget); + // Send a command on the command ring, wait for completion. // Returns completion code. uint32_t SendCommand(const TRB& trb); @@ -367,7 +373,8 @@ namespace Drivers::USB::Xhci { // Access device info UsbDeviceInfo* GetDevice(uint8_t slotId); - // Poll event ring (called from interrupt handler or during init) + // Poll the event ring from process/idle context. The hard interrupt only + // acknowledges and masks the source, then queues deferred draining. void PollEvents(); }; diff --git a/kernel/src/Hal/Apic/Apic.cpp b/kernel/src/Hal/Apic/Apic.cpp index 2f1e2cc..b09d9ed 100644 --- a/kernel/src/Hal/Apic/Apic.cpp +++ b/kernel/src/Hal/Apic/Apic.cpp @@ -16,6 +16,10 @@ namespace Hal { static volatile uint32_t* g_apicBase = nullptr; static constexpr uint32_t ICR_DELIVERY_STATUS = (1 << 12); static constexpr uint32_t ICR_LEVEL_ASSERT = (1 << 14); + // Fixed-IPIs normally clear delivery status within a few MMIO reads. + // Keep the exceptional path short because scheduler kicks can originate + // in an input IRQ where a long MMIO poll would itself stall the BSP. + static constexpr uint32_t ICR_WAIT_SPINS = 10000; static inline uint64_t ReadMSR(uint32_t msr) { uint32_t lo, hi; @@ -115,19 +119,20 @@ namespace Hal { WriteRegister(REG_EOI, 0); } - void SendFixedIpi(uint32_t apicId, uint8_t vector) { - if (g_apicBase == nullptr) return; - - while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) { + static bool WaitForIcrIdle() { + for (uint32_t i = 0; i < ICR_WAIT_SPINS; i++) { + if (!(ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS)) return true; asm volatile("pause"); } + return false; + } + + bool SendFixedIpi(uint32_t apicId, uint8_t vector) { + if (g_apicBase == nullptr || !WaitForIcrIdle()) return false; WriteRegister(REG_ICR_HIGH, apicId << 24); WriteRegister(REG_ICR_LOW, (uint32_t)vector | ICR_LEVEL_ASSERT); - - while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) { - asm volatile("pause"); - } + return WaitForIcrIdle(); } uint32_t GetId() { diff --git a/kernel/src/Hal/Apic/Apic.hpp b/kernel/src/Hal/Apic/Apic.hpp index 438ed78..4e944bc 100644 --- a/kernel/src/Hal/Apic/Apic.hpp +++ b/kernel/src/Hal/Apic/Apic.hpp @@ -44,7 +44,8 @@ namespace Hal { void Reinitialize(); void SendEOI(); - void SendFixedIpi(uint32_t apicId, uint8_t vector); + // Returns false if the local APIC delivery-status bit fails to clear. + bool SendFixedIpi(uint32_t apicId, uint8_t vector); uint32_t GetId(); uint32_t ReadRegister(uint32_t reg); diff --git a/kernel/src/Hal/Apic/Interrupts.cpp b/kernel/src/Hal/Apic/Interrupts.cpp index e67f2f9..7806645 100644 --- a/kernel/src/Hal/Apic/Interrupts.cpp +++ b/kernel/src/Hal/Apic/Interrupts.cpp @@ -38,10 +38,11 @@ namespace Hal { << " IRQ stubs (vectors " << (uint64_t)IRQ_VECTOR_BASE << "-" << (uint64_t)(IRQ_VECTOR_BASE + IRQ_COUNT - 1) << ")"; } + }; // C linkage dispatch function called from assembly stubs -extern "C" void HalIrqDispatch(uint64_t irqNumber) { +extern "C" void HalIrqDispatch(uint64_t irqNumber, uint64_t savedCs) { // Send EOI BEFORE calling the handler. The handler may context-switch // (via Tick -> Schedule -> SchedContextSwitch) and never return here. // If EOI is deferred until after the handler, the LAPIC timer vector @@ -52,6 +53,6 @@ extern "C" void HalIrqDispatch(uint64_t irqNumber) { Hal::LocalApic::SendEOI(); if (irqNumber < Hal::IRQ_COUNT && Hal::g_irqHandlers[irqNumber] != nullptr) { - Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber); + Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber, (savedCs & 3) == 3); } } diff --git a/kernel/src/Hal/Apic/Interrupts.hpp b/kernel/src/Hal/Apic/Interrupts.hpp index 95f84e5..983ec39 100644 --- a/kernel/src/Hal/Apic/Interrupts.hpp +++ b/kernel/src/Hal/Apic/Interrupts.hpp @@ -9,7 +9,10 @@ namespace Hal { // IRQ handler function type. The parameter is the IRQ number (0-47). - using IrqHandler = void(*)(uint8_t irq); + // `fromUser` is derived directly from the saved interrupt-frame CS. Do not + // cache it in CpuData: an IRQ handler may context-switch and later resume + // on another CPU, making per-CPU saved-origin state stale. + using IrqHandler = void(*)(uint8_t irq, bool fromUser); // Number of IRQ slots supported (0-23: legacy ISA via IOAPIC, // 24-45: MSI, 46-47: kernel IPIs) @@ -37,4 +40,5 @@ namespace Hal { // Install IRQ stubs into the IDT and set up the dispatch table void InitializeIrqHandlers(); + }; diff --git a/kernel/src/Hal/Apic/IsrStubs.asm b/kernel/src/Hal/Apic/IsrStubs.asm index 71a8b66..badf5d1 100644 --- a/kernel/src/Hal/Apic/IsrStubs.asm +++ b/kernel/src/Hal/Apic/IsrStubs.asm @@ -50,11 +50,19 @@ IrqCommon: ; Pass IRQ number as first argument (rdi) mov rdi, rax + ; Pass the saved CS as the second argument. Fifteen saved GPRs occupy + ; 120 bytes, and CS is eight bytes after the interrupt-frame RIP. + mov rsi, [rsp + 128] - ; Align stack to 16 bytes before call (we pushed 15 registers x 8 = 120 bytes - ; + return address 8 = 128, which is 16-byte aligned, so we're good) + ; An IRQ that interrupted ring 0 can arrive at any kernel stack depth, so + ; the saved frame is not guaranteed to satisfy the SysV call alignment. + ; RBX is already saved above and is callee-saved, making it a safe place to + ; retain the exact frame pointer while aligning down for the C++ call. + mov rbx, rsp + and rsp, -16 cld call HalIrqDispatch + mov rsp, rbx pop r15 pop r14 diff --git a/kernel/src/Hal/CpuPower.cpp b/kernel/src/Hal/CpuPower.cpp index d1cc975..4220541 100644 --- a/kernel/src/Hal/CpuPower.cpp +++ b/kernel/src/Hal/CpuPower.cpp @@ -70,6 +70,8 @@ namespace Hal { static constexpr uint32_t GovernorIntervalMs = 500; static constexpr uint8_t PassiveStepDown = 2; // ratio units (~200 MHz) static constexpr uint8_t HotStepDown = 8; // ratio units (~800 MHz) + static constexpr uint32_t ThermalLogThrottle = 0x100; + static constexpr uint32_t ThermalLogRecovered = 0x200; // ============================================================ // Detected features and shared policy state @@ -106,6 +108,7 @@ namespace Hal { static bool g_throttling = false; static std::atomic g_lastTempC{0}; + static std::atomic g_pendingThermalLog{0}; static std::atomic g_effMHz{0}; static uint64_t g_lastGovernorMs = 0; static uint64_t g_lastAperf = 0; @@ -339,10 +342,23 @@ namespace Hal { uint32_t epoch = g_policyEpoch.fetch_add(1, std::memory_order_acq_rel) + 1; ApplyHwpRequestOnThisCpu(epoch); // BSP applies immediately; APs on tick - if (g_throttling && !wasThrottling) { + // Keep policy transitions separate from terminal output so this + // maintenance step never enters process-context logging locks. + if (g_throttling && !wasThrottling) + g_pendingThermalLog.store(ThermalLogThrottle | temp, + std::memory_order_release); + else if (!g_throttling && wasThrottling) + g_pendingThermalLog.store(ThermalLogRecovered | temp, + std::memory_order_release); + } + + void ServiceDeferredDiagnostics() { + uint32_t pending = g_pendingThermalLog.exchange(0, std::memory_order_acq_rel); + uint8_t temp = (uint8_t)(pending & 0xFF); + if (pending & ThermalLogThrottle) { KernelLogStream(WARNING, "CpuPower") << "Package at " << base::dec << (uint64_t)temp << "C - thermal throttle engaged"; - } else if (!g_throttling && wasThrottling) { + } else if (pending & ThermalLogRecovered) { KernelLogStream(OK, "CpuPower") << "Package cooled to " << base::dec << (uint64_t)temp << "C - thermal throttle released"; } diff --git a/kernel/src/Hal/CpuPower.hpp b/kernel/src/Hal/CpuPower.hpp index 757f5c3..15face5 100644 --- a/kernel/src/Hal/CpuPower.hpp +++ b/kernel/src/Hal/CpuPower.hpp @@ -43,6 +43,10 @@ namespace Hal { // be called from every maintenance pass. void ThermalTick(uint64_t nowMs); + // Print thermal-transition diagnostics from idle/process context. + // ThermalTick only records transitions for this function to report. + void ServiceDeferredDiagnostics(); + // Redo the BSP MSR setup after S3 wake (HWP enable and POWER_CTL do // not survive suspend). void ReapplyAfterWake(); diff --git a/kernel/src/Hal/IDT.cpp b/kernel/src/Hal/IDT.cpp index 04d4da2..9a31d64 100644 --- a/kernel/src/Hal/IDT.cpp +++ b/kernel/src/Hal/IDT.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace Hal { @@ -93,6 +94,14 @@ namespace Hal { // If the fault originated in user-mode (ring 3), kill the process // instead of panicking the entire system. if (fromUser && Sched::GetCurrentPid() >= 0) { + // Interrupt gates arrive with IF clear. Full process teardown can + // wait on device completions, sibling CPUs, and wall-clock-bounded + // recovery paths, so leaving IF clear here can stop the BSP clock + // and its local device IRQs indefinitely. GS is already the kernel + // per-CPU base, and timer/IPI scheduling refuses to switch away + // from a ring-0 frame, so nested hardware IRQs are safe now. + asm volatile("sti" ::: "memory"); + auto* proc = Sched::GetCurrentProcessPtr(); auto* regs = GetExceptionRegs(i, frame); Kt::KernelLogStream(Kt::ERROR, "Exception") @@ -183,6 +192,7 @@ namespace Hal { { bool fromUser = (frame->CS & 3) == 3; if (fromUser) asm volatile("swapgs"); + auto* cpu = Smp::TryGetCurrentCpuData(); uint64_t cr2; asm volatile("mov %%cr2, %0" : "=r"(cr2)); @@ -190,7 +200,8 @@ namespace Hal { // Bit 0 of the error code: 0 = non-present page. Covers both user // pushes past the mapped stack and kernel accesses to not-yet-grown // user stack buffers passed into syscalls. - if ((errorCode & 1) == 0 && Sched::GetCurrentPid() >= 0 + if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0 + && Sched::GetCurrentPid() >= 0 && Sched::TryGrowUserStack(cr2)) { if (fromUser) asm volatile("swapgs"); return; diff --git a/kernel/src/Hal/SmpBoot.cpp b/kernel/src/Hal/SmpBoot.cpp index b1a529d..910faab 100644 --- a/kernel/src/Hal/SmpBoot.cpp +++ b/kernel/src/Hal/SmpBoot.cpp @@ -218,6 +218,9 @@ namespace Smp { for (;;) { // Pick up thermal-governor frequency changes decided by the BSP. Hal::CpuPower::ApplyPolicyIfChanged(); + // Any idle core may run bounded USB/NIC bottom halves. Preserve + // the AP's ACPI/MWAIT idle selection after servicing them. + Timekeeping::ServiceDeferredWork(); Hal::CpuIdle::Wait(10, cpu->hasMwait, &s_idleMonitor); } } diff --git a/kernel/src/Hal/SmpBoot.hpp b/kernel/src/Hal/SmpBoot.hpp index 7be1d59..82208f4 100644 --- a/kernel/src/Hal/SmpBoot.hpp +++ b/kernel/src/Hal/SmpBoot.hpp @@ -49,6 +49,11 @@ namespace Smp { // Other CPUs schedule normally while this is held. volatile bool reservedForKernelWork = false; + // Set by the BSP timer when an IRQ bottom half needs process-safe + // execution even though runnable user processes keep this CPU busy. + // Schedule() consumes it by yielding once to the idle kernel context. + bool kernelWorkPending = false; + // Per-CPU GDT and TSS (APs use these; BSP uses globals) Hal::BasicGDT cpuGdt __attribute__((aligned(16))); Hal::TSS64 cpuTss __attribute__((aligned(16))); @@ -61,6 +66,17 @@ namespace Smp { return ptr; } + // Exception paths may run before InitBsp() has installed a GS base. Do + // not read gs:0 in that state: address zero would fault recursively and + // turn the original, diagnosable exception into a triple fault. + inline CpuData* TryGetCurrentCpuData() { + uint32_t low; + uint32_t high; + asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(0xC0000101u)); + uint64_t base = ((uint64_t)high << 32) | low; + return (CpuData*)base; + } + // Get CPU data by index CpuData* GetCpuData(int index); diff --git a/kernel/src/Hal/SyscallEntry.asm b/kernel/src/Hal/SyscallEntry.asm index 01dad9c..0020140 100644 --- a/kernel/src/Hal/SyscallEntry.asm +++ b/kernel/src/Hal/SyscallEntry.asm @@ -8,6 +8,7 @@ section .text extern SyscallDispatch +extern SchedExitIfKilled ; ==================================================================== ; Per-CPU data offsets (must match CpuData in SmpBoot.hpp) @@ -52,6 +53,15 @@ SyscallEntry: mov rdi, rsp ; arg1 = pointer to SyscallFrame call SyscallDispatch ; returns int64_t in rax + ; A remote kill observed while this syscall was executing must not tear + ; the process down from the timer IRQ on top of held syscall locks. This + ; safe point runs after dispatch returned and all such locks are released. + sub rsp, 16 + mov [rsp], rax + call SchedExitIfKilled + mov rax, [rsp] + add rsp, 16 + cli ; disable interrupts for sysret pop r15 diff --git a/kernel/src/Ipc/Ipc.cpp b/kernel/src/Ipc/Ipc.cpp index f88007c..cdaf2ee 100644 --- a/kernel/src/Ipc/Ipc.cpp +++ b/kernel/src/Ipc/Ipc.cpp @@ -35,6 +35,8 @@ namespace Ipc { static constexpr uint32_t UdpRingSize = 4096; static constexpr int MaxSurfaces = 32; static constexpr int MaxSurfacePages = 8192; + static_assert((uint64_t)MaxSurfacePages * 0x1000ULL <= Sched::UserSurfaceSlotSize); + static_assert(MaxSurfaceMapsPerProcess == Sched::UserSurfaceSlots); static constexpr int MaxProcessObjects = 512; static constexpr int MaxWaitsets = 32; static constexpr int MaxWaitsetEntries = 16; @@ -114,6 +116,8 @@ namespace Ipc { struct WaitsetEntry { bool used; + bool retiring; + uint32_t readers; HandleType type; Object* object; uint32_t rights; @@ -121,6 +125,7 @@ namespace Ipc { }; struct Waitset : Object { + kcp::Spinlock entriesLock; WaitsetEntry entries[MaxWaitsetEntries]; }; @@ -148,7 +153,9 @@ namespace Ipc { static HandleEntry g_handleTables[Sched::MaxProcesses][MaxHandlesPerProcess] = {}; static uint64_t g_handleBitmaps[Sched::MaxProcesses][2] = {}; // 128-bit bitmap per process + static kcp::Mutex g_handleTableLocks[Sched::MaxProcesses]; static SurfaceMap g_surfaceMaps[Sched::MaxProcesses][MaxSurfaceMapsPerProcess] = {}; + static kcp::Mutex g_surfaceMapLocks[Sched::MaxProcesses]; static Stream g_streams[MaxStreams] = {}; static Mailbox g_mailboxes[MaxMailboxes] = {}; @@ -171,6 +178,13 @@ namespace Ipc { static void ReleaseRawObject(Object* object); + // MUST be a Mutex, never a Spinlock. ShootdownUserRange holds this while + // waiting for remote CPUs to acknowledge the shootdown IPI, so a CPU that + // is queued behind the holder has to stay interruptible long enough to + // service that IPI itself. An interrupt-disabling Spinlock here deadlocks + // every CPU contending for the lock, and the bounded-retry logic below + // then reports it as a "target failed to acknowledge" Panic -- which reads + // like a hardware fault rather than a lock-type regression. static kcp::Mutex g_tlbShootdownLock; static volatile uint64_t g_tlbShootdownSeq = 0; static volatile uint64_t g_tlbShootdownPml4 = 0; @@ -201,7 +215,7 @@ namespace Ipc { } } - static void TlbShootdownIpiHandler(uint8_t) { + static void TlbShootdownIpiHandler(uint8_t, bool) { Smp::CpuData* cpu = Smp::GetCurrentCpuData(); uint64_t seq = g_tlbShootdownSeq; uint64_t pml4 = g_tlbShootdownPml4; @@ -218,7 +232,7 @@ namespace Ipc { } } - static void ShootdownUserRange(uint64_t pml4Phys, uint64_t startVa, uint32_t pages) { + void ShootdownUserRange(uint64_t pml4Phys, uint64_t startVa, uint32_t pages) { if (pml4Phys == 0 || pages == 0) return; bool targets[Smp::MaxCPUs] = {}; @@ -251,20 +265,63 @@ namespace Ipc { } targets[i] = true; - Hal::LocalApic::SendFixedIpi(cpu->lapicId, + (void)Hal::LocalApic::SendFixedIpi(cpu->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_TLB_SHOOTDOWN); } for (int i = 0; i < Smp::GetCpuCount(); i++) { if (!targets[i]) continue; + uint32_t spins = 0; + uint32_t retries = 0; while (g_tlbShootdownDone[i] != seq) { asm volatile("pause"); + if (++spins < 1000000) continue; + + // Delivery normally completes in a handful of cycles. Retry a + // bounded number of times in case the first IPI was lost while + // the target changed interrupt state. Continuing without an + // acknowledgement would let the caller free frames still + // reachable through a remote stale TLB entry, so fail loudly + // instead of either corrupting memory or spinning forever. + spins = 0; + if (++retries > 4) { + Panic("TLB shootdown target failed to acknowledge", nullptr); + } + Smp::CpuData* cpu = Smp::GetCpuData(i); + if (cpu != nullptr && cpu->started) { + (void)Hal::LocalApic::SendFixedIpi(cpu->lapicId, + Hal::IRQ_VECTOR_BASE + Hal::IRQ_TLB_SHOOTDOWN); + } } } g_tlbShootdownLock.Release(); } + void UnmapAndFreeUserRange(uint64_t pml4Phys, uint64_t startVa, uint64_t pages) { + static constexpr uint32_t PagesPerChunk = 64; + uint64_t physPages[PagesPerChunk]; + + for (uint64_t base = 0; base < pages; base += PagesPerChunk) { + uint32_t count = (uint32_t)((pages - base > PagesPerChunk) + ? PagesPerChunk : pages - base); + + for (uint32_t i = 0; i < count; i++) { + uint64_t pageVa = startVa + (base + i) * 0x1000ULL; + physPages[i] = Memory::VMM::Paging::GetPhysAddr(pml4Phys, pageVa); + Memory::VMM::Paging::UnmapUserIn(pml4Phys, pageVa); + } + + ShootdownUserRange(pml4Phys, startVa + base * 0x1000ULL, count); + + for (uint32_t i = 0; i < count; i++) { + if (physPages[i] != 0) { + Memory::g_pfa->Free((void*)Memory::HHDM(physPages[i])); + } + } + } + } + static void InitObject(Object& object, HandleType type) { object.type = type; object.active = true; @@ -382,13 +439,34 @@ namespace Ipc { static void DestroyWaitset(Waitset* waitset) { for (int i = 0; i < MaxWaitsetEntries; i++) { - if (!waitset->entries[i].used) continue; - ReleaseRawObject(waitset->entries[i].object); - waitset->entries[i].used = false; - waitset->entries[i].object = nullptr; - waitset->entries[i].rights = 0; - waitset->entries[i].signals = 0; - waitset->entries[i].type = HandleType::None; + Object* target = nullptr; + + waitset->entriesLock.Acquire(); + WaitsetEntry& entry = waitset->entries[i]; + entry.used = false; + entry.retiring = entry.object != nullptr; + waitset->entriesLock.Release(); + + // IRQ-side notification readers never retain/release the target; + // the entry's permanent reference remains valid until all of them + // have left this slot. + for (;;) { + waitset->entriesLock.Acquire(); + if (entry.readers == 0) { + target = entry.object; + entry.object = nullptr; + entry.rights = 0; + entry.signals = 0; + entry.type = HandleType::None; + entry.retiring = false; + waitset->entriesLock.Release(); + break; + } + waitset->entriesLock.Release(); + asm volatile("pause"); + } + + ReleaseRawObject(target); } } @@ -522,31 +600,46 @@ namespace Ipc { return span; } - static void RetainForHandle(Object* object, HandleType type, uint32_t rights) { - if (object == nullptr) return; + static bool RetainForHandle(Object* object, HandleType type, uint32_t rights) { + if (object == nullptr) return false; switch (type) { case HandleType::Stream: { auto* stream = (Stream*)object; stream->lock.Acquire(); + if (!stream->active || stream->destroying) { + stream->lock.Release(); + return false; + } stream->refs++; if (rights & RightRead) stream->readerRefs++; if (rights & RightWrite) stream->writerRefs++; stream->lock.Release(); - break; + return true; } case HandleType::Mailbox: { auto* mailbox = (Mailbox*)object; mailbox->lock.Acquire(); + if (!mailbox->active || mailbox->destroying) { + mailbox->lock.Release(); + return false; + } mailbox->refs++; if (rights & RightSend) mailbox->senderRefs++; if (rights & RightRecv) mailbox->receiverRefs++; mailbox->lock.Release(); - break; + return true; + } + default: { + object->lock.Acquire(); + if (!object->active || object->destroying) { + object->lock.Release(); + return false; + } + object->refs++; + object->lock.Release(); + return true; } - default: - RetainRawObject(object); - break; } } @@ -618,6 +711,7 @@ namespace Ipc { int InstallHandleForSlot(int slot, Object* object, HandleType type, uint32_t rights) { if (slot < 0 || slot >= Sched::MaxProcesses || object == nullptr) return -1; + g_handleTableLocks[slot].Acquire(); uint64_t& bm0 = g_handleBitmaps[slot][0]; uint64_t& bm1 = g_handleBitmaps[slot][1]; // Find first zero bit in bitmap (free slot) @@ -625,43 +719,67 @@ namespace Ipc { uint64_t bits1 = ~bm1; if (bits0) { int i = __builtin_ctzll(bits0); + if (!RetainForHandle(object, type, rights)) { + g_handleTableLocks[slot].Release(); + return -1; + } bm0 |= (1ULL << i); g_handleTables[slot][i].used = true; g_handleTables[slot][i].rights = rights; g_handleTables[slot][i].type = type; g_handleTables[slot][i].object = object; - RetainForHandle(object, type, rights); + g_handleTableLocks[slot].Release(); return i; } else if (bits1) { int i = 64 + __builtin_ctzll(bits1); + if (!RetainForHandle(object, type, rights)) { + g_handleTableLocks[slot].Release(); + return -1; + } bm1 |= (1ULL << (i - 64)); g_handleTables[slot][i].used = true; g_handleTables[slot][i].rights = rights; g_handleTables[slot][i].type = type; g_handleTables[slot][i].object = object; - RetainForHandle(object, type, rights); + g_handleTableLocks[slot].Release(); return i; } + g_handleTableLocks[slot].Release(); return -1; } - bool SnapshotHandleForSlot(int slot, int handle, HandleType& type, Object*& object, uint32_t& rights) { + bool HandleSnapshot::Capture(int slot, int handle) { + if (object != nullptr) return false; if (slot < 0 || slot >= Sched::MaxProcesses) return false; if (handle < 0 || handle >= MaxHandlesPerProcess) return false; + g_handleTableLocks[slot].Acquire(); const HandleEntry& entry = g_handleTables[slot][handle]; - if (!entry.used || entry.object == nullptr) return false; + if (!entry.used || entry.object == nullptr) { + g_handleTableLocks[slot].Release(); + return false; + } type = entry.type; object = entry.object; rights = entry.rights; + RetainRawObject(object); + g_handleTableLocks[slot].Release(); return true; } + HandleSnapshot::~HandleSnapshot() { + ReleaseRawObject(object); + } + int CloseHandleForSlot(int slot, int handle) { if (slot < 0 || slot >= Sched::MaxProcesses) return -1; if (handle < 0 || handle >= MaxHandlesPerProcess) return -1; - if (!g_handleTables[slot][handle].used) return -1; + g_handleTableLocks[slot].Acquire(); + if (!g_handleTables[slot][handle].used) { + g_handleTableLocks[slot].Release(); + return -1; + } HandleEntry entry = g_handleTables[slot][handle]; g_handleTables[slot][handle].used = false; @@ -673,6 +791,7 @@ namespace Ipc { if (handle < 64) g_handleBitmaps[slot][0] &= ~(1ULL << handle); else g_handleBitmaps[slot][1] &= ~(1ULL << (handle - 64)); + g_handleTableLocks[slot].Release(); ReleaseForHandle(entry.object, entry.type, entry.rights); return 0; } @@ -682,13 +801,11 @@ namespace Ipc { } int DupHandle(int handle) { - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; + HandleSnapshot snapshot; int slot = CurrentSlot(); - if (!SnapshotHandleForSlot(slot, handle, type, object, rights)) return -1; - if ((rights & RightDup) == 0) return -1; - return InstallHandleForSlot(slot, object, type, rights); + if (!snapshot.Capture(slot, handle)) return -1; + if ((snapshot.rights & RightDup) == 0) return -1; + return InstallHandleForSlot(slot, snapshot.object, snapshot.type, snapshot.rights); } Stream* CreateStream(uint32_t capacity) { @@ -850,24 +967,20 @@ namespace Ipc { if (out == nullptr) return -1; if (maxLen > 0 && !montauk::abi::UserMemory::Range((uint64_t)out, (uint64_t)maxLen, true)) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return -1; - if (type != HandleType::Stream || (rights & RightRead) == 0) return -1; - return StreamRead((Stream*)object, out, maxLen, true); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return -1; + if (snapshot.type != HandleType::Stream || (snapshot.rights & RightRead) == 0) return -1; + return StreamRead((Stream*)snapshot.object, out, maxLen, true); } int StreamWriteHandle(int handle, const uint8_t* data, int len) { if (data == nullptr) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, (uint64_t)len, false)) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return -1; - if (type != HandleType::Stream || (rights & RightWrite) == 0) return -1; - return StreamWrite((Stream*)object, data, len, true); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return -1; + if (snapshot.type != HandleType::Stream || (snapshot.rights & RightWrite) == 0) return -1; + return StreamWrite((Stream*)snapshot.object, data, len, true); } bool StreamHasData(Stream* stream) { @@ -955,37 +1068,37 @@ namespace Ipc { if (mailbox == nullptr) return -1; if (len > MaxMailboxMessageBytes) return -1; - HandleType attachmentType = HandleType::None; - Object* attachmentObject = nullptr; - uint32_t attachmentRights = 0; + HandleSnapshot attachment; if (attachHandle >= 0) { if (senderSlot < 0) return -1; - if (!SnapshotHandleForSlot(senderSlot, attachHandle, attachmentType, attachmentObject, attachmentRights)) { + if (!attachment.Capture(senderSlot, attachHandle)) { return -1; } - if ((attachmentRights & RightDup) == 0 || attachmentObject == nullptr) return -1; - RetainRawObject(attachmentObject); + if ((attachment.rights & RightDup) == 0 || attachment.object == nullptr) return -1; + // The queued message owns a separate reference after this syscall's + // scoped handle snapshot is released. + RetainRawObject(attachment.object); } mailbox->lock.Acquire(); if (mailbox->receiverRefs == 0) { mailbox->lock.Release(); - if (attachmentObject != nullptr) ReleaseRawObject(attachmentObject); + if (attachment.object != nullptr) ReleaseRawObject(attachment.object); return -1; } if (mailbox->count >= MaxMailboxMessages) { mailbox->lock.Release(); - if (attachmentObject != nullptr) ReleaseRawObject(attachmentObject); + if (attachment.object != nullptr) ReleaseRawObject(attachment.object); return 0; } MailboxMessage& msg = mailbox->messages[mailbox->head]; msg.type = msgType; msg.size = len; - msg.attachmentType = (uint8_t)attachmentType; - msg.hasAttachment = attachmentObject != nullptr ? 1 : 0; - msg.attachmentRights = attachmentRights; - msg.attachmentObject = attachmentObject; + msg.attachmentType = (uint8_t)attachment.type; + msg.hasAttachment = attachment.object != nullptr ? 1 : 0; + msg.attachmentRights = attachment.rights; + msg.attachmentObject = attachment.object; if (len > 0 && data != nullptr) { memcpy(msg.data, data, len); } @@ -1052,19 +1165,35 @@ namespace Ipc { return closed ? -1 : 0; } - MailboxMessage& msg = mailbox->messages[mailbox->tail]; - int attachedHandle = -1; - if (msg.hasAttachment) { + MailboxMessage& queued = mailbox->messages[mailbox->tail]; + if (queued.hasAttachment) { if (receiverSlot < 0 || outAttachHandle == nullptr) { mailbox->lock.Release(); return -1; } + } + // Remove the message before installing an attached handle. Installation + // takes the receiver's handle-table lock and then the attached object's + // lock; doing that while holding mailbox->lock would invert the global + // table->object order used by HandleSnapshot::Capture. + MailboxMessage msg = queued; + mailbox->tail = (mailbox->tail + 1) % MaxMailboxMessages; + mailbox->count--; + queued.attachmentType = (uint8_t)HandleType::None; + queued.hasAttachment = 0; + queued.attachmentRights = 0; + queued.attachmentObject = nullptr; + mailbox->lock.Release(); + + int attachedHandle = -1; + if (msg.hasAttachment) { attachedHandle = InstallHandleForSlot(receiverSlot, msg.attachmentObject, (HandleType)msg.attachmentType, msg.attachmentRights); if (attachedHandle < 0) { - mailbox->lock.Release(); + ReleaseRawObject(msg.attachmentObject); + NotifyObjectChanged((Object*)mailbox); return -1; } } @@ -1078,18 +1207,8 @@ namespace Ipc { *inOutLen = copyLen; if (outAttachHandle != nullptr) *outAttachHandle = attachedHandle; - mailbox->tail = (mailbox->tail + 1) % MaxMailboxMessages; - mailbox->count--; - Object* attachedObject = msg.attachmentObject; - bool hadAttachment = msg.hasAttachment != 0; - msg.attachmentType = (uint8_t)HandleType::None; - msg.hasAttachment = 0; - msg.attachmentRights = 0; - msg.attachmentObject = nullptr; - mailbox->lock.Release(); - - if (hadAttachment && attachedObject != nullptr) { - ReleaseRawObject(attachedObject); + if (msg.hasAttachment && msg.attachmentObject != nullptr) { + ReleaseRawObject(msg.attachmentObject); } NotifyObjectChanged((Object*)mailbox); return (int)copyLen; @@ -1102,13 +1221,11 @@ namespace Ipc { int MailboxSendHandle(int handle, uint32_t msgType, const void* data, uint16_t len, int attachHandle) { if (len > 0 && (data == nullptr || !montauk::abi::UserMemory::Range((uint64_t)data, len, false))) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; + HandleSnapshot snapshot; int slot = CurrentSlot(); - if (!SnapshotHandleForSlot(slot, handle, type, object, rights)) return -1; - if (type != HandleType::Mailbox || (rights & RightSend) == 0) return -1; - return MailboxSendInternal((Mailbox*)object, slot, msgType, data, len, attachHandle); + if (!snapshot.Capture(slot, handle)) return -1; + if (snapshot.type != HandleType::Mailbox || (snapshot.rights & RightSend) == 0) return -1; + return MailboxSendInternal((Mailbox*)snapshot.object, slot, msgType, data, len, attachHandle); } int MailboxRecvHandle(int handle, uint32_t* msgType, void* data, uint16_t* inOutLen, int* outAttachHandle) { @@ -1119,13 +1236,11 @@ namespace Ipc { if (requestedLen > 0 && (data == nullptr || !montauk::abi::UserMemory::Range((uint64_t)data, requestedLen, true))) return -1; if (outAttachHandle != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)outAttachHandle)) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; + HandleSnapshot snapshot; int slot = CurrentSlot(); - if (!SnapshotHandleForSlot(slot, handle, type, object, rights)) return -1; - if (type != HandleType::Mailbox || (rights & RightRecv) == 0) return -1; - return MailboxRecvInternal((Mailbox*)object, slot, msgType, data, inOutLen, outAttachHandle, true); + if (!snapshot.Capture(slot, handle)) return -1; + if (snapshot.type != HandleType::Mailbox || (snapshot.rights & RightRecv) == 0) return -1; + return MailboxRecvInternal((Mailbox*)snapshot.object, slot, msgType, data, inOutLen, outAttachHandle, true); } bool MailboxHasMessage(Mailbox* mailbox) { @@ -1181,33 +1296,27 @@ namespace Ipc { if (buffer == nullptr) return -1; if (size > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, size, true)) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return -1; - if (type != HandleType::File || (rights & RightRead) == 0) return -1; - return Fs::Vfs::ReadBackendFile(((File*)object)->backend, buffer, offset, size); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return -1; + if (snapshot.type != HandleType::File || (snapshot.rights & RightRead) == 0) return -1; + return Fs::Vfs::ReadBackendFile(((File*)snapshot.object)->backend, buffer, offset, size); } int FileWriteHandle(int handle, const uint8_t* buffer, uint64_t offset, uint64_t size) { if (buffer == nullptr) return -1; if (size > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, size, false)) return -1; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return -1; - if (type != HandleType::File || (rights & RightWrite) == 0) return -1; - return Fs::Vfs::WriteBackendFile(((File*)object)->backend, buffer, offset, size); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return -1; + if (snapshot.type != HandleType::File || (snapshot.rights & RightWrite) == 0) return -1; + return Fs::Vfs::WriteBackendFile(((File*)snapshot.object)->backend, buffer, offset, size); } uint64_t FileGetSizeHandle(int handle) { - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return 0; - if (type != HandleType::File || (rights & RightRead) == 0) return 0; - return Fs::Vfs::GetBackendFileSize(((File*)object)->backend); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return 0; + if (snapshot.type != HandleType::File || (snapshot.rights & RightRead) == 0) return 0; + return Fs::Vfs::GetBackendFileSize(((File*)snapshot.object)->backend); } static Socket* AllocateSocketObject(int type) { @@ -1229,12 +1338,12 @@ namespace Ipc { return nullptr; } - static bool SnapshotSocketHandle(int handle, Socket*& outSocket, uint32_t& outRights) { - HandleType type = HandleType::None; - Object* object = nullptr; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, outRights)) return false; - if (type != HandleType::Socket || object == nullptr) return false; - outSocket = (Socket*)object; + static bool SnapshotSocketHandle(int handle, HandleSnapshot& snapshot, + Socket*& outSocket, uint32_t& outRights) { + if (!snapshot.Capture(CurrentSlot(), handle)) return false; + if (snapshot.type != HandleType::Socket || snapshot.object == nullptr) return false; + outSocket = (Socket*)snapshot.object; + outRights = snapshot.rights; return true; } @@ -1259,9 +1368,10 @@ namespace Ipc { } int SocketConnectHandle(int handle, uint32_t ip, uint16_t port) { + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); @@ -1296,9 +1406,10 @@ namespace Ipc { } int SocketBindHandle(int handle, uint16_t port) { + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0) return -1; if (socket->socketType == SocketTypeTcp) { @@ -1344,9 +1455,10 @@ namespace Ipc { } int SocketListenHandle(int handle) { + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); @@ -1372,9 +1484,10 @@ namespace Ipc { int SocketAcceptHandle(int handle) { int slot = CurrentSlot(); + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); @@ -1413,9 +1526,10 @@ namespace Ipc { if (data == nullptr) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, len, false)) return -1; + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightWrite) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); @@ -1430,9 +1544,10 @@ namespace Ipc { if (buffer == nullptr) return -1; if (maxLen > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, maxLen, true)) return -1; + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightRead) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); @@ -1451,9 +1566,10 @@ namespace Ipc { if (len > 1472) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, len, false)) return -1; + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightWrite) == 0 || socket->socketType != SocketTypeUdp) return -1; socket->socketLock.Acquire(); @@ -1484,9 +1600,10 @@ namespace Ipc { if (srcIp != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)srcIp)) return -1; if (srcPort != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)srcPort)) return -1; + HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; - if (!SnapshotSocketHandle(handle, socket, rights)) return -1; + if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightRead) == 0 || socket->socketType != SocketTypeUdp) return -1; socket->socketLock.Acquire(); @@ -1596,7 +1713,11 @@ namespace Ipc { uint64_t GetSurfaceSize(const Surface* surface) { if (surface == nullptr) return 0; - return surface->sizeBytes; + auto* mutableSurface = const_cast(surface); + mutableSurface->lock.Acquire(); + uint64_t size = mutableSurface->sizeBytes; + mutableSurface->lock.Release(); + return size; } static void RollbackSurfaceGrowMaps(Surface* surface, uint32_t newPages) { @@ -1607,6 +1728,7 @@ namespace Ipc { uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; + g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; @@ -1619,6 +1741,7 @@ namespace Ipc { } ShootdownUserRange(pml4, startVa, rollbackPages); } + g_surfaceMapLocks[s].Release(); } } @@ -1630,6 +1753,8 @@ namespace Ipc { uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; + bool failed = false; + g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; @@ -1638,10 +1763,16 @@ namespace Ipc { for (uint32_t p = m.numPages; p < newPages; p++) { uint64_t va = m.va + (uint64_t)p * 0x1000ULL; if (!Memory::VMM::Paging::MapUserIn(pml4, scratch[p], va)) { - RollbackSurfaceGrowMaps(surface, newPages); - return false; + failed = true; + break; } } + if (failed) break; + } + g_surfaceMapLocks[s].Release(); + if (failed) { + RollbackSurfaceGrowMaps(surface, newPages); + return false; } } @@ -1731,6 +1862,7 @@ namespace Ipc { uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; + g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; @@ -1762,6 +1894,7 @@ namespace Ipc { ShootdownUserRange(pml4, baseVa, flushPages); m.numPages = newPages; } + g_surfaceMapLocks[s].Release(); } surface->lock.Release(); @@ -1780,37 +1913,50 @@ namespace Ipc { auto* proc = Sched::GetCurrentProcessPtr(); if (proc == nullptr) return 0; - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return 0; - if (type != HandleType::Surface || (rights & RightMap) == 0) return 0; + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return 0; + if (snapshot.type != HandleType::Surface || (snapshot.rights & RightMap) == 0) return 0; uint64_t va = 0; - if (MapSurfaceForPid((Surface*)object, proc->pid, proc->pml4Phys, proc->heapNext, va) < 0) { + if (MapSurfaceForPid((Surface*)snapshot.object, proc->pid, proc->pml4Phys, proc->heapNext, va) < 0) { return 0; } return va; } int ResizeSurfaceHandle(int handle, uint64_t newSize) { - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(CurrentSlot(), handle, type, object, rights)) return -1; - if (type != HandleType::Surface || (rights & RightManage) == 0) return -1; - return ResizeSurface((Surface*)object, newSize); + HandleSnapshot snapshot; + if (!snapshot.Capture(CurrentSlot(), handle)) return -1; + if (snapshot.type != HandleType::Surface || (snapshot.rights & RightManage) == 0) return -1; + return ResizeSurface((Surface*)snapshot.object, newSize); } int CopySurface(Surface* dst, Surface* src) { if (dst == nullptr || src == nullptr) return -1; - if (dst->numPages != src->numPages) return -1; + if (dst == src) return 0; + + Surface* first = ((uintptr_t)dst < (uintptr_t)src) ? dst : src; + Surface* second = (first == dst) ? src : dst; + first->lock.Acquire(); + second->lock.Acquire(); + if (dst->numPages != src->numPages) { + second->lock.Release(); + first->lock.Release(); + return -1; + } for (uint32_t i = 0; i < src->numPages; i++) { + if (src->physPages[i] == 0 || dst->physPages[i] == 0) { + second->lock.Release(); + first->lock.Release(); + return -1; + } void* srcPage = (void*)Memory::HHDM(src->physPages[i]); void* dstPage = (void*)Memory::HHDM(dst->physPages[i]); memcpy(dstPage, srcPage, 0x1000); } + second->lock.Release(); + first->lock.Release(); NotifyObjectChanged((Object*)dst); return 0; } @@ -1821,10 +1967,19 @@ namespace Ipc { if (dst == nullptr || src == nullptr) return -1; if (dstWidth <= 0 || dstHeight <= 0 || srcWidth <= 0 || srcHeight <= 0) return -1; + Surface* first = ((uintptr_t)dst < (uintptr_t)src) ? dst : src; + Surface* second = (first == dst) ? src : dst; + first->lock.Acquire(); + if (second != first) second->lock.Acquire(); + uint64_t dstPixels = (uint64_t)dstWidth * (uint64_t)dstHeight; uint64_t srcPixels = (uint64_t)srcWidth * (uint64_t)srcHeight; - if (dstPixels * sizeof(uint32_t) > dst->sizeBytes) return -1; - if (srcPixels * sizeof(uint32_t) > src->sizeBytes) return -1; + if (dstPixels * sizeof(uint32_t) > dst->sizeBytes || + srcPixels * sizeof(uint32_t) > src->sizeBytes) { + if (second != first) second->lock.Release(); + first->lock.Release(); + return -1; + } for (uint64_t i = 0; i < dstPixels; i++) { uint32_t* p = SurfacePixelPtr(dst, i); @@ -1845,6 +2000,8 @@ namespace Ipc { } } + if (second != first) second->lock.Release(); + first->lock.Release(); NotifyObjectChanged((Object*)dst); return 0; } @@ -1853,6 +2010,11 @@ namespace Ipc { uint64_t& outVa, bool reserveGrowthRange) { if (surface == nullptr) return -1; + // Kept in the interface for existing callers. Surface VA now comes + // from reusable fixed slots and no longer consumes the process heap. + (void)heapNext; + (void)reserveGrowthRange; + int slot = SlotForPid(pid); if (slot < 0) return -1; @@ -1866,10 +2028,12 @@ namespace Ipc { // from Object), so the refcount bump is inlined below to avoid // self-deadlocking on the non-reentrant mutex. surface->lock.Acquire(); + g_surfaceMapLocks[slot].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { if (g_surfaceMaps[slot][i].used && g_surfaceMaps[slot][i].surface == surface) { outVa = g_surfaceMaps[slot][i].va; + g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return 0; } @@ -1883,18 +2047,21 @@ namespace Ipc { } } if (mapIdx < 0) { + g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return -1; } uint32_t numPages = surface->numPages; - uint64_t baseVa = heapNext; + uint64_t baseVa = Sched::UserSurfaceBase + + (uint64_t)mapIdx * Sched::UserSurfaceSlotSize; for (uint32_t i = 0; i < numPages; i++) { if (!Memory::VMM::Paging::MapUserIn(pml4Phys, surface->physPages[i], baseVa + (uint64_t)i * 0x1000ULL)) { for (uint32_t j = 0; j < i; j++) { Memory::VMM::Paging::UnmapUserIn(pml4Phys, baseVa + (uint64_t)j * 0x1000ULL); } + g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return -1; } @@ -1906,15 +2073,8 @@ namespace Ipc { g_surfaceMaps[slot][mapIdx].numPages = numPages; surface->refs++; // inlined RetainRawObject — we already hold surface->lock - // Generic IPC surfaces can grow in place through ResizeSurface(), so - // their mapping must reserve the maximum VA range that a later grow - // can populate. Window-server surfaces are replaced rather than grown - // in place, so callers can opt out and avoid burning 32 MiB of process - // VA on every window map/remap during a long desktop session. - uint32_t reservedPages = reserveGrowthRange ? MaxSurfacePages : numPages; - if (reservedPages < numPages) reservedPages = numPages; - heapNext += (uint64_t)reservedPages * 0x1000ULL; outVa = baseVa; + g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return 0; } @@ -1925,41 +2085,52 @@ namespace Ipc { int slot = SlotForPid(pid); if (slot < 0) return -1; + surface->lock.Acquire(); + g_surfaceMapLocks[slot].Acquire(); int unmapped = 0; for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { if (!g_surfaceMaps[slot][i].used || g_surfaceMaps[slot][i].surface != surface) continue; - for (uint32_t p = 0; p < g_surfaceMaps[slot][i].numPages; p++) { - Memory::VMM::Paging::UnmapUserIn(pml4Phys, g_surfaceMaps[slot][i].va + (uint64_t)p * 0x1000ULL); + uint64_t baseVa = g_surfaceMaps[slot][i].va; + uint32_t numPages = g_surfaceMaps[slot][i].numPages; + for (uint32_t p = 0; p < numPages; p++) { + Memory::VMM::Paging::UnmapUserIn(pml4Phys, + baseVa + (uint64_t)p * 0x1000ULL); } + + // UnmapUserIn invalidates only the calling CPU. A sibling thread + // in this same process may still have one of these PTEs cached; + // releasing the mapping reference can then destroy the surface + // and recycle its frames while that sibling writes through its + // stale TLB entry. Quiesce every CPU using this PML4 first. + ShootdownUserRange(pml4Phys, baseVa, numPages); + g_surfaceMaps[slot][i].used = false; g_surfaceMaps[slot][i].surface = nullptr; g_surfaceMaps[slot][i].va = 0; g_surfaceMaps[slot][i].numPages = 0; - ReleaseRawObject((Object*)surface); unmapped++; } + g_surfaceMapLocks[slot].Release(); + surface->lock.Release(); + for (int i = 0; i < unmapped; i++) ReleaseRawObject((Object*)surface); + return unmapped > 0 ? 0 : -1; } - ProcessObject* GetProcessObject(int pid) { + int OpenProcessHandle(int pid) { g_processPoolLock.Acquire(); for (int i = 0; i < MaxProcessObjects; i++) { if (!g_processObjects[i].active) continue; if (g_processObjects[i].pid == pid) { + int handle = InstallHandleForSlot(CurrentSlot(), (Object*)&g_processObjects[i], + HandleType::Process, RightWait | RightDup); g_processPoolLock.Release(); - return &g_processObjects[i]; + return handle; } } g_processPoolLock.Release(); - return nullptr; - } - - int OpenProcessHandle(int pid) { - ProcessObject* process = GetProcessObject(pid); - if (process == nullptr) return -1; - return InstallHandleForSlot(CurrentSlot(), (Object*)process, HandleType::Process, - RightWait | RightDup); + return -1; } void ProcessStartedInSlot(int slot, int pid) { @@ -2098,21 +2269,37 @@ namespace Ipc { } uint32_t GetHandleSignalsForSlot(int slot, int handle) { - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(slot, handle, type, object, rights)) return SignalNone; - return CurrentSignalsForSnapshot(type, object, rights); + HandleSnapshot snapshot; + if (!snapshot.Capture(slot, handle)) return SignalNone; + return CurrentSignalsForSnapshot(snapshot.type, snapshot.object, snapshot.rights); } static bool WaitsetCheckReady(Waitset* waitset, WaitsetReady* outReady) { if (waitset == nullptr) return false; for (int i = 0; i < MaxWaitsetEntries; i++) { - if (!waitset->entries[i].used || waitset->entries[i].object == nullptr) continue; - uint32_t current = CurrentSignalsForSnapshot(waitset->entries[i].type, - waitset->entries[i].object, - waitset->entries[i].rights); - uint32_t readySignals = current & waitset->entries[i].signals; + HandleType type = HandleType::None; + Object* object = nullptr; + uint32_t rights = 0; + uint32_t wanted = 0; + + waitset->entriesLock.Acquire(); + WaitsetEntry& entry = waitset->entries[i]; + if (entry.used && !entry.retiring && entry.object != nullptr) { + entry.readers++; + type = entry.type; + object = entry.object; + rights = entry.rights; + wanted = entry.signals; + } + waitset->entriesLock.Release(); + if (object == nullptr) continue; + + uint32_t current = CurrentSignalsForSnapshot(type, object, rights); + uint32_t readySignals = current & wanted; + + waitset->entriesLock.Acquire(); + entry.readers--; + waitset->entriesLock.Release(); if (readySignals == 0) continue; if (outReady != nullptr) { outReady->index = i; @@ -2123,6 +2310,21 @@ namespace Ipc { return false; } + static bool WaitsetContainsObject(Waitset* waitset, Object* object) { + if (waitset == nullptr || object == nullptr) return false; + bool found = false; + waitset->entriesLock.Acquire(); + for (int i = 0; i < MaxWaitsetEntries; i++) { + const WaitsetEntry& entry = waitset->entries[i]; + if (entry.used && !entry.retiring && entry.object == object) { + found = true; + break; + } + } + waitset->entriesLock.Release(); + return found; + } + void NotifyObjectChanged(Object* object) { if (object == nullptr) return; @@ -2130,8 +2332,12 @@ namespace Ipc { for (int i = 0; i < MaxWaitsets; i++) { if (!g_waitsets[i].active) continue; - WaitsetReady ready{}; - if (WaitsetCheckReady(&g_waitsets[i], &ready)) { + // This function is also called by the NIC receive IRQ. Do not + // evaluate readiness here: that takes target stream/mailbox + // mutexes and can self-deadlock if the IRQ preempted their owner. + // A pointer match is enough; the awakened waiter rechecks signals + // in process context before returning to userspace. + if (WaitsetContainsObject(&g_waitsets[i], object)) { Sched::WakeObjectWaiters(&g_waitsets[i]); } } @@ -2139,15 +2345,14 @@ namespace Ipc { uint32_t WaitOnHandle(int handle, uint32_t wantedSignals, uint64_t timeoutMs) { int slot = CurrentSlot(); - HandleType type = HandleType::None; - Object* object = nullptr; - uint32_t rights = 0; - if (!SnapshotHandleForSlot(slot, handle, type, object, rights)) return (uint32_t)-1; - if ((rights & RightWait) == 0) return (uint32_t)-1; + HandleSnapshot snapshot; + if (!snapshot.Capture(slot, handle)) return (uint32_t)-1; + if ((snapshot.rights & RightWait) == 0) return (uint32_t)-1; uint64_t start = Timekeeping::GetMilliseconds(); for (;;) { - uint32_t current = CurrentSignalsForSnapshot(type, object, rights); + uint64_t observedWake = Sched::ObserveObjectWake(snapshot.object); + uint32_t current = CurrentSignalsForSnapshot(snapshot.type, snapshot.object, snapshot.rights); uint32_t ready = current & wantedSignals; if (ready != 0) return ready; @@ -2155,9 +2360,9 @@ namespace Ipc { if (timeoutMs != ~0ULL) { uint64_t elapsed = Timekeeping::GetMilliseconds() - start; if (elapsed >= timeoutMs) return 0; - Sched::BlockOnObject(object, timeoutMs - elapsed); + Sched::BlockOnObjectSince(snapshot.object, timeoutMs - elapsed, observedWake); } else { - Sched::BlockOnObject(object, 0); + Sched::BlockOnObjectSince(snapshot.object, 0, observedWake); } } } @@ -2169,13 +2374,17 @@ namespace Ipc { for (int i = 0; i < MaxWaitsets; i++) { if (g_waitsets[i].active || g_waitsets[i].destroying) continue; InitObject(g_waitsets[i], HandleType::Waitset); + g_waitsets[i].entriesLock.Acquire(); for (int j = 0; j < MaxWaitsetEntries; j++) { g_waitsets[i].entries[j].used = false; + g_waitsets[i].entries[j].retiring = false; + g_waitsets[i].entries[j].readers = 0; g_waitsets[i].entries[j].type = HandleType::None; g_waitsets[i].entries[j].object = nullptr; g_waitsets[i].entries[j].rights = 0; g_waitsets[i].entries[j].signals = 0; } + g_waitsets[i].entriesLock.Release(); g_waitsetPoolLock.Release(); return InstallHandleForSlot(slot, (Object*)&g_waitsets[i], HandleType::Waitset, RightWait | RightManage | RightDup); @@ -2189,34 +2398,35 @@ namespace Ipc { } int WaitsetAddHandleForSlot(int slot, int waitsetHandle, int targetHandle, uint32_t signals) { - HandleType waitsetType = HandleType::None; - Object* waitsetObject = nullptr; - uint32_t waitsetRights = 0; - if (!SnapshotHandleForSlot(slot, waitsetHandle, waitsetType, waitsetObject, waitsetRights)) return -1; - if (waitsetType != HandleType::Waitset || (waitsetRights & RightManage) == 0) return -1; + HandleSnapshot waitsetSnapshot; + if (!waitsetSnapshot.Capture(slot, waitsetHandle)) return -1; + if (waitsetSnapshot.type != HandleType::Waitset || + (waitsetSnapshot.rights & RightManage) == 0) return -1; - HandleType targetType = HandleType::None; - Object* targetObject = nullptr; - uint32_t targetRights = 0; - if (!SnapshotHandleForSlot(slot, targetHandle, targetType, targetObject, targetRights)) return -1; - if ((targetRights & RightWait) == 0) return -1; - if (targetType == HandleType::Waitset) return -1; + HandleSnapshot targetSnapshot; + if (!targetSnapshot.Capture(slot, targetHandle)) return -1; + if ((targetSnapshot.rights & RightWait) == 0) return -1; + if (targetSnapshot.type == HandleType::Waitset) return -1; - auto* waitset = (Waitset*)waitsetObject; - waitset->lock.Acquire(); + auto* waitset = (Waitset*)waitsetSnapshot.object; + // Take the entry's permanent target reference before disabling IRQs + // for the short metadata update. + RetainRawObject(targetSnapshot.object); + waitset->entriesLock.Acquire(); for (int i = 0; i < MaxWaitsetEntries; i++) { - if (waitset->entries[i].used) continue; + if (waitset->entries[i].used || waitset->entries[i].retiring || + waitset->entries[i].object != nullptr) continue; waitset->entries[i].used = true; - waitset->entries[i].type = targetType; - waitset->entries[i].object = targetObject; - waitset->entries[i].rights = targetRights; + waitset->entries[i].type = targetSnapshot.type; + waitset->entries[i].object = targetSnapshot.object; + waitset->entries[i].rights = targetSnapshot.rights; waitset->entries[i].signals = signals; - RetainRawObject(targetObject); - waitset->lock.Release(); - NotifyObjectChanged(waitsetObject); + waitset->entriesLock.Release(); + NotifyObjectChanged(waitsetSnapshot.object); return i; } - waitset->lock.Release(); + waitset->entriesLock.Release(); + ReleaseRawObject(targetSnapshot.object); return -1; } @@ -2225,27 +2435,39 @@ namespace Ipc { } int WaitsetRemoveIndexForSlot(int slot, int waitsetHandle, int index) { - HandleType waitsetType = HandleType::None; - Object* waitsetObject = nullptr; - uint32_t waitsetRights = 0; - if (!SnapshotHandleForSlot(slot, waitsetHandle, waitsetType, waitsetObject, waitsetRights)) return -1; - if (waitsetType != HandleType::Waitset || (waitsetRights & RightManage) == 0) return -1; + HandleSnapshot snapshot; + if (!snapshot.Capture(slot, waitsetHandle)) return -1; + if (snapshot.type != HandleType::Waitset || (snapshot.rights & RightManage) == 0) return -1; if (index < 0 || index >= MaxWaitsetEntries) return -1; - auto* waitset = (Waitset*)waitsetObject; - waitset->lock.Acquire(); - if (!waitset->entries[index].used) { - waitset->lock.Release(); + auto* waitset = (Waitset*)snapshot.object; + waitset->entriesLock.Acquire(); + WaitsetEntry& entry = waitset->entries[index]; + if (!entry.used || entry.retiring || entry.object == nullptr) { + waitset->entriesLock.Release(); return -1; } - Object* targetObject = waitset->entries[index].object; - waitset->entries[index].used = false; - waitset->entries[index].type = HandleType::None; - waitset->entries[index].object = nullptr; - waitset->entries[index].rights = 0; - waitset->entries[index].signals = 0; - waitset->lock.Release(); + entry.used = false; + entry.retiring = true; + waitset->entriesLock.Release(); + + Object* targetObject = nullptr; + for (;;) { + waitset->entriesLock.Acquire(); + if (entry.readers == 0) { + targetObject = entry.object; + entry.type = HandleType::None; + entry.object = nullptr; + entry.rights = 0; + entry.signals = 0; + entry.retiring = false; + waitset->entriesLock.Release(); + break; + } + waitset->entriesLock.Release(); + asm volatile("pause"); + } ReleaseRawObject(targetObject); return 0; @@ -2258,16 +2480,15 @@ namespace Ipc { int WaitsetWaitHandle(int waitsetHandle, WaitsetReady* outReady, uint64_t timeoutMs) { int slot = CurrentSlot(); - HandleType waitsetType = HandleType::None; - Object* waitsetObject = nullptr; - uint32_t waitsetRights = 0; - if (!SnapshotHandleForSlot(slot, waitsetHandle, waitsetType, waitsetObject, waitsetRights)) return -1; - if (waitsetType != HandleType::Waitset || (waitsetRights & RightWait) == 0) return -1; + HandleSnapshot snapshot; + if (!snapshot.Capture(slot, waitsetHandle)) return -1; + if (snapshot.type != HandleType::Waitset || (snapshot.rights & RightWait) == 0) return -1; uint64_t start = Timekeeping::GetMilliseconds(); for (;;) { + uint64_t observedWake = Sched::ObserveObjectWake(snapshot.object); WaitsetReady ready = {-1, 0}; - if (WaitsetCheckReady((Waitset*)waitsetObject, &ready)) { + if (WaitsetCheckReady((Waitset*)snapshot.object, &ready)) { if (outReady != nullptr) *outReady = ready; return 1; } @@ -2276,9 +2497,9 @@ namespace Ipc { if (timeoutMs != ~0ULL) { uint64_t elapsed = Timekeeping::GetMilliseconds() - start; if (elapsed >= timeoutMs) return 0; - Sched::BlockOnObject(waitsetObject, timeoutMs - elapsed); + Sched::BlockOnObjectSince(snapshot.object, timeoutMs - elapsed, observedWake); } else { - Sched::BlockOnObject(waitsetObject, 0); + Sched::BlockOnObjectSince(snapshot.object, 0, observedWake); } } } @@ -2287,21 +2508,29 @@ namespace Ipc { if (slot < 0 || slot >= Sched::MaxProcesses) return; for (int h = 0; h < MaxHandlesPerProcess; h++) { - if (g_handleTables[slot][h].used) { - CloseHandleForSlot(slot, h); - } + // CloseHandleForSlot performs the synchronized used check. Reading + // the table directly here would race a sibling thread still + // unwinding during process teardown. + CloseHandleForSlot(slot, h); } for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { - if (!g_surfaceMaps[slot][i].used || g_surfaceMaps[slot][i].surface == nullptr) continue; - for (uint32_t p = 0; p < g_surfaceMaps[slot][i].numPages; p++) { - Memory::VMM::Paging::UnmapUserIn(pml4Phys, g_surfaceMaps[slot][i].va + (uint64_t)p * 0x1000ULL); + g_surfaceMapLocks[slot].Acquire(); + SurfaceMap& map = g_surfaceMaps[slot][i]; + if (!map.used || map.surface == nullptr) { + g_surfaceMapLocks[slot].Release(); + continue; } - Object* surfaceObject = (Object*)g_surfaceMaps[slot][i].surface; - g_surfaceMaps[slot][i].used = false; - g_surfaceMaps[slot][i].surface = nullptr; - g_surfaceMaps[slot][i].va = 0; - g_surfaceMaps[slot][i].numPages = 0; + + for (uint32_t p = 0; p < map.numPages; p++) { + Memory::VMM::Paging::UnmapUserIn(pml4Phys, map.va + (uint64_t)p * 0x1000ULL); + } + Object* surfaceObject = (Object*)map.surface; + map.used = false; + map.surface = nullptr; + map.va = 0; + map.numPages = 0; + g_surfaceMapLocks[slot].Release(); ReleaseRawObject(surfaceObject); } } diff --git a/kernel/src/Ipc/Ipc.hpp b/kernel/src/Ipc/Ipc.hpp index 87dba67..3316fe1 100644 --- a/kernel/src/Ipc/Ipc.hpp +++ b/kernel/src/Ipc/Ipc.hpp @@ -63,6 +63,23 @@ namespace Ipc { Object* object; }; + // Pins the object referenced by a handle for the lifetime of the snapshot. + // Process threads share one handle table, so a raw table lookup is not + // sufficient: another thread may close and destroy the object immediately + // after the lookup returns. + struct HandleSnapshot { + HandleType type = HandleType::None; + Object* object = nullptr; + uint32_t rights = 0; + + HandleSnapshot() = default; + ~HandleSnapshot(); + HandleSnapshot(const HandleSnapshot&) = delete; + HandleSnapshot& operator=(const HandleSnapshot&) = delete; + + bool Capture(int slot, int handle); + }; + struct WaitsetReady { int32_t index; uint32_t signals; @@ -78,7 +95,6 @@ namespace Ipc { int CloseHandle(int handle); int DupHandle(int handle); - bool SnapshotHandleForSlot(int slot, int handle, HandleType& type, Object*& object, uint32_t& rights); uint32_t GetHandleSignalsForSlot(int slot, int handle); Stream* CreateStream(uint32_t capacity = DefaultStreamCapacity); @@ -139,7 +155,6 @@ namespace Ipc { uint64_t& outVa, bool reserveGrowthRange = true); int UnmapSurfaceForPid(Surface* surface, int pid, uint64_t pml4Phys); - ProcessObject* GetProcessObject(int pid); int OpenProcessHandle(int pid); void ProcessStartedInSlot(int slot, int pid); void ProcessExitedInSlot(int slot, int pid); @@ -156,6 +171,11 @@ namespace Ipc { int WaitsetWaitHandle(int waitsetHandle, WaitsetReady* outReady, uint64_t timeoutMs); void NotifyObjectChanged(Object* object); + // Invalidate a user range on every CPU currently running the address + // space. Call this after removing PTEs and before releasing their frames. + void ShootdownUserRange(uint64_t pml4Phys, uint64_t startVa, uint32_t pages); + // Safely remove ordinary PFA-backed user mappings and release their frames. + void UnmapAndFreeUserRange(uint64_t pml4Phys, uint64_t startVa, uint64_t pages); void CleanupProcessSlot(int slot, int pid, uint64_t pml4Phys); } diff --git a/kernel/src/Memory/PageFrameAllocator.cpp b/kernel/src/Memory/PageFrameAllocator.cpp index 4359335..442d6e1 100644 --- a/kernel/src/Memory/PageFrameAllocator.cpp +++ b/kernel/src/Memory/PageFrameAllocator.cpp @@ -70,6 +70,15 @@ namespace Memory { } void* PageFrameAllocator::ReallocConsecutive(void* ptr, int n) { + // This primitive grows a single-page allocation into a contiguous + // span; it is not a sized-free API. Guard zero/negative requests so a + // caller can never carve a zero-byte block at the end of the free + // pool and memcpy beyond it. Multi-page owners must use Free(ptr, n). + if (n <= 0) { + if (ptr != nullptr) Free(ptr); + return nullptr; + } + Lock.Acquire(); // Search the free list for a single contiguous region >= n pages. @@ -118,6 +127,19 @@ namespace Memory { Lock.Acquire(); uint64_t addr = (uint64_t)ptr; + uint64_t poolStart = g_section.address; + uint64_t poolEnd = poolStart + g_section.size; + + // Reject malformed frees before they can splice arbitrary memory into + // the allocator's linked list. Avoid overflowing addr + size while + // validating the upper bound. + if ((addr & 0xFFFULL) || (size & 0xFFFULL) || addr < poolStart + || poolEnd < poolStart || addr > poolEnd || size > poolEnd - addr) { + Lock.Release(); + Kt::KernelLogStream(Kt::WARNING, "PFA") + << "Invalid free range at " << addr << " size " << size << ", ignoring"; + return; + } // Walk to find the sorted insertion point: prev < addr < current Page* prev = &head; @@ -126,9 +148,9 @@ namespace Memory { while (current != nullptr && (uint64_t)current < addr) { // Double-free check: addr falls within an existing free block if (addr < (uint64_t)current + current->size) { + Lock.Release(); Kt::KernelLogStream(Kt::WARNING, "PFA") << "Double-free detected at " << addr << ", ignoring"; - Lock.Release(); return; } prev = current; @@ -137,9 +159,9 @@ namespace Memory { // Double-free check: exact match with next block if (current != nullptr && (uint64_t)current == addr) { + Lock.Release(); Kt::KernelLogStream(Kt::WARNING, "PFA") << "Double-free detected at " << addr << ", ignoring"; - Lock.Release(); return; } @@ -148,9 +170,9 @@ namespace Memory { // freeing a multi-page span that overlaps the start of an existing // block would silently corrupt the free list. if (current != nullptr && addr + size > (uint64_t)current) { + Lock.Release(); Kt::KernelLogStream(Kt::WARNING, "PFA") << "Overlapping free at " << addr << " size " << size << ", ignoring"; - Lock.Release(); return; } diff --git a/kernel/src/Memory/Paging.cpp b/kernel/src/Memory/Paging.cpp index 9f68626..d09d2c4 100644 --- a/kernel/src/Memory/Paging.cpp +++ b/kernel/src/Memory/Paging.cpp @@ -7,6 +7,26 @@ namespace Memory::VMM { Paging* g_paging = nullptr; + static constexpr uint64_t LeafAddressMask = 0x000FFFFFFFFFF000ULL; + static constexpr uint64_t LeafPresent = 1ULL << 0; + static constexpr uint64_t LeafWritable = 1ULL << 1; + static constexpr uint64_t LeafUser = 1ULL << 2; + static constexpr uint64_t LeafWriteThrough = 1ULL << 3; + static constexpr uint64_t LeafCacheDisabled = 1ULL << 4; + + static inline void SetLeafPte(PageTableEntry* entry, uint64_t physicalAddress, + bool user, bool writeThrough, bool cacheDisabled) { + uint64_t flags = LeafPresent | LeafWritable; + if (user) flags |= LeafUser; + if (writeThrough) flags |= LeafWriteThrough; + if (cacheDisabled) flags |= LeafCacheDisabled; + + // Replace the complete PTE in one aligned store. Updating individual + // bitfields left old PWT/PCD/PAT/accessed state behind when a virtual + // address was reused for a mapping with a different cache policy. + *(volatile uint64_t*)entry = (physicalAddress & LeafAddressMask) | flags; + } + // Protects user page table modifications from concurrent SMP access static kcp::Mutex pagingLock; @@ -143,10 +163,7 @@ namespace Memory::VMM { PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, false, false, false); } void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { @@ -162,11 +179,7 @@ namespace Memory::VMM { PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - pageEntry->WriteThrough = true; // PWT=1, PCD=0 → PAT entry 1 = WC - - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, false, true, false); } void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { @@ -182,12 +195,7 @@ namespace Memory::VMM { PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - pageEntry->CacheDisabled = true; - pageEntry->WriteThrough = true; - - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, false, true, true); } void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { @@ -203,11 +211,7 @@ namespace Memory::VMM { PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - pageEntry->Supervisor = 1; // User-accessible - - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, true, false, false); } std::uint64_t Paging::CreateUserPML4() { @@ -262,10 +266,7 @@ namespace Memory::VMM { if (!pml1) { pagingLock.Release(); return false; } PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - pageEntry->Supervisor = 1; - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, true, false, false); pagingLock.Release(); return true; } @@ -305,11 +306,7 @@ namespace Memory::VMM { if (!pml1) { pagingLock.Release(); return false; } PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]); - pageEntry->Present = true; - pageEntry->Writable = true; - pageEntry->Supervisor = 1; - pageEntry->WriteThrough = true; // PWT=1, PCD=0 -> PAT entry 1 = WC - pageEntry->Address = physicalAddress >> 12; + SetLeafPte(pageEntry, physicalAddress, true, true, false); pagingLock.Release(); return true; } @@ -456,9 +453,11 @@ namespace Memory::VMM { PageTable* pml4Virt = (PageTable*)HHDM(pml4); PageTableEntry* pml4_entry = &pml4Virt->entries[virtualAddressObj.GetL4Index()]; + if (!pml4_entry->Present) return 0; PageTable* pml3 = (PageTable*)HHDM((pml4_entry->Address & kPhysAddrMask) << 12); PageTableEntry* pml3_entry = &pml3->entries[virtualAddressObj.GetL3Index()]; + if (!pml3_entry->Present) return 0; // 1GB large page at PML3 level if (pml3_entry->LargerPages) { @@ -468,6 +467,7 @@ namespace Memory::VMM { PageTable* pml2 = (PageTable*)HHDM((pml3_entry->Address & kPhysAddrMask) << 12); PageTableEntry* pml2_entry = &pml2->entries[virtualAddressObj.GetL2Index()]; + if (!pml2_entry->Present) return 0; // 2MB large page at PML2 level if (pml2_entry->LargerPages) { @@ -479,10 +479,12 @@ namespace Memory::VMM { if (use40BitL1 == true) { PageTableEntry40Bit* pml1_entry = (PageTableEntry40Bit*)&pml1->entries[virtualAddressObj.GetPageIndex()]; + if (!pml1_entry->Present) return 0; return (uint64_t)pml1_entry->Address << 12; } PageTableEntry* pml1_entry = &pml1->entries[virtualAddressObj.GetPageIndex()]; + if (!pml1_entry->Present) return 0; return (uint64_t)(pml1_entry->Address & kPhysAddrMask) << 12; } diff --git a/kernel/src/Net/Icmp.cpp b/kernel/src/Net/Icmp.cpp index 737a6dc..ce19ef4 100644 --- a/kernel/src/Net/Icmp.cpp +++ b/kernel/src/Net/Icmp.cpp @@ -62,13 +62,6 @@ namespace Net::Icmp { } if (hdr->Type == TYPE_ECHO_REQUEST && hdr->Code == 0) { - KernelLogStream(INFO, "Net") << "ICMP echo request from " - << base::dec - << (uint64_t)(srcIp & 0xFF) << "." - << (uint64_t)((srcIp >> 8) & 0xFF) << "." - << (uint64_t)((srcIp >> 16) & 0xFF) << "." - << (uint64_t)((srcIp >> 24) & 0xFF); - // Build echo reply -- same payload, different type uint8_t reply[1500]; if (length > sizeof(reply)) { diff --git a/kernel/src/Net/Tcp.cpp b/kernel/src/Net/Tcp.cpp index 1eb0754..7d1bab1 100644 --- a/kernel/src/Net/Tcp.cpp +++ b/kernel/src/Net/Tcp.cpp @@ -465,6 +465,7 @@ namespace Net::Tcp { // Block until a SYN arrives while (true) { + uint64_t listenerWake = Sched::ObserveObjectWake(listener); listener->Lock.Acquire(); if (listener->PendingCount > 0) { Connection::PendingSyn pending = listener->Pending[listener->PendingHead]; @@ -524,13 +525,14 @@ namespace Net::Tcp { // Wait for ACK to complete the handshake uint64_t deadline = Timekeeping::GetMilliseconds() + 5000; while (Timekeeping::GetMilliseconds() < deadline) { + uint64_t connWake = Sched::ObserveObjectWake(conn); if (conn->CurrentState == State::Established) { return conn; } uint64_t now = Timekeeping::GetMilliseconds(); uint64_t waitMs = (deadline > now) ? (deadline - now) : 0; if (waitMs == 0) break; - Sched::BlockOnObject(conn, waitMs); + Sched::BlockOnObjectSince(conn, waitMs, connWake); } // Timed out waiting for ACK @@ -540,7 +542,7 @@ namespace Net::Tcp { return nullptr; } listener->Lock.Release(); - Sched::BlockOnObject(listener, 0); + Sched::BlockOnObjectSince(listener, 0, listenerWake); } } @@ -589,13 +591,14 @@ namespace Net::Tcp { for (int attempt = 0; attempt < MAX_RETRANSMITS; attempt++) { uint64_t deadline = Timekeeping::GetMilliseconds() + 1000; while (Timekeeping::GetMilliseconds() < deadline) { + uint64_t connWake = Sched::ObserveObjectWake(conn); if (conn->CurrentState == State::Established) { return conn; } uint64_t now = Timekeeping::GetMilliseconds(); uint64_t waitMs = (deadline > now) ? (deadline - now) : 0; if (waitMs == 0) break; - Sched::BlockOnObject(conn, waitMs); + Sched::BlockOnObjectSince(conn, waitMs, connWake); } if (conn->CurrentState == State::SynSent) { @@ -636,6 +639,7 @@ namespace Net::Tcp { // writers so concurrent threads cannot overwrite its in-flight // retransmit buffer or allocate overlapping sequence numbers. while (true) { + uint64_t connWake = Sched::ObserveObjectWake(conn); uint64_t flags; asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory"); conn->Lock.Acquire(); @@ -652,7 +656,7 @@ namespace Net::Tcp { } conn->Lock.Release(); asm volatile("push %0; popfq" :: "r"(flags) : "memory"); - Sched::BlockOnObject(conn, 50); + Sched::BlockOnObjectSince(conn, 50, connWake); } auto finishSend = [&]() { @@ -712,6 +716,7 @@ namespace Net::Tcp { asm volatile("push %0; popfq" :: "r"(flags) : "memory"); while (true) { + uint64_t connWake = Sched::ObserveObjectWake(conn); asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory"); conn->Lock.Acquire(); @@ -775,7 +780,7 @@ namespace Net::Tcp { conn->Lock.Release(); asm volatile("push %0; popfq" :: "r"(flags) : "memory"); - Sched::BlockOnObject(conn, waitMs); + Sched::BlockOnObjectSince(conn, waitMs, connWake); } } @@ -790,6 +795,7 @@ namespace Net::Tcp { // Block until data is available or connection is closing while (true) { + uint64_t connWake = Sched::ObserveObjectWake(conn); uint64_t flags; asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory"); conn->Lock.Acquire(); @@ -829,7 +835,7 @@ namespace Net::Tcp { conn->Lock.Release(); asm volatile("push %0; popfq" :: "r"(flags) : "memory"); - Sched::BlockOnObject(conn, 0); + Sched::BlockOnObjectSince(conn, 0, connWake); } } @@ -905,11 +911,12 @@ namespace Net::Tcp { // Wait for close to complete for (int i = 0; i < 100; i++) { + uint64_t connWake = Sched::ObserveObjectWake(conn); if (conn->CurrentState == State::TimeWait || conn->CurrentState == State::Closed) { break; } - Sched::BlockOnObject(conn, 50); + Sched::BlockOnObjectSince(conn, 50, connWake); } Sched::WakeObjectWaiters(conn); Ipc::NotifyTcpConnectionChanged(conn); @@ -926,10 +933,11 @@ namespace Net::Tcp { // Wait for final ACK for (int i = 0; i < 100; i++) { + uint64_t connWake = Sched::ObserveObjectWake(conn); if (conn->CurrentState == State::Closed) { break; } - Sched::BlockOnObject(conn, 50); + Sched::BlockOnObjectSince(conn, 50, connWake); } Sched::WakeObjectWaiters(conn); Ipc::NotifyTcpConnectionChanged(conn); diff --git a/kernel/src/Pci/Pci.cpp b/kernel/src/Pci/Pci.cpp index 6cb116e..abb5a31 100644 --- a/kernel/src/Pci/Pci.cpp +++ b/kernel/src/Pci/Pci.cpp @@ -10,6 +10,7 @@ #include #include #include +#include using namespace Kt; @@ -19,6 +20,14 @@ namespace Pci { static constexpr uint16_t ConfigAddressPort = 0xCF8; static constexpr uint16_t ConfigDataPort = 0xCFC; + // CF8/CFC is one machine-wide address/data latch, not a per-CPU interface. + // Keep the address write and matching data access indivisible: otherwise a + // second CPU can replace CF8 between them and make us read or write an + // unrelated device's register. Besides bogus probing, a crossed runtime + // command/MSI write can disable interrupts or bus mastering underneath an + // active GPU, USB controller, NIC, or HDA controller. + static kcp::Spinlock g_legacyConfigLock; + // PCI config space register offsets static constexpr uint16_t RegVendorId = 0x00; static constexpr uint16_t RegDeviceId = 0x02; @@ -163,11 +172,26 @@ namespace Pci { | (offset & 0xFC); } - uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) { + static uint32_t LegacyRead32Unlocked(uint8_t bus, uint8_t device, + uint8_t function, uint8_t offset) { Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort); return Io::In32(ConfigDataPort); } + static void LegacyWrite32Unlocked(uint8_t bus, uint8_t device, + uint8_t function, uint8_t offset, + uint32_t value) { + Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort); + Io::Out32(value, ConfigDataPort); + } + + uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) { + g_legacyConfigLock.Acquire(); + uint32_t value = LegacyRead32Unlocked(bus, device, function, offset); + g_legacyConfigLock.Release(); + return value; + } + uint16_t LegacyRead16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) { uint32_t val = LegacyRead32(bus, device, function, offset & 0xFC); return (uint16_t)(val >> ((offset & 2) * 8)); @@ -179,30 +203,29 @@ namespace Pci { } void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) { - Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort); - Io::Out32(value, ConfigDataPort); + g_legacyConfigLock.Acquire(); + LegacyWrite32Unlocked(bus, device, function, offset, value); + g_legacyConfigLock.Release(); } void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) { - uint32_t addr = LegacyBuildAddress(bus, device, function, offset & 0xFC); - Io::Out32(addr, ConfigAddressPort); - uint32_t tmp = Io::In32(ConfigDataPort); + g_legacyConfigLock.Acquire(); + uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC); int shift = (offset & 2) * 8; tmp &= ~(0xFFFF << shift); tmp |= ((uint32_t)value << shift); - Io::Out32(addr, ConfigAddressPort); - Io::Out32(tmp, ConfigDataPort); + LegacyWrite32Unlocked(bus, device, function, offset & 0xFC, tmp); + g_legacyConfigLock.Release(); } void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) { - uint32_t addr = LegacyBuildAddress(bus, device, function, offset & 0xFC); - Io::Out32(addr, ConfigAddressPort); - uint32_t tmp = Io::In32(ConfigDataPort); + g_legacyConfigLock.Acquire(); + uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC); int shift = (offset & 3) * 8; tmp &= ~(0xFF << shift); tmp |= ((uint32_t)value << shift); - Io::Out32(addr, ConfigAddressPort); - Io::Out32(tmp, ConfigDataPort); + LegacyWrite32Unlocked(bus, device, function, offset & 0xFC, tmp); + g_legacyConfigLock.Release(); } // ------------------------------------------------------------------------- @@ -237,8 +260,17 @@ namespace Pci { // Read Capabilities Pointer (offset 0x34), mask to dword-aligned uint8_t offset = ReadConfig8(bus, device, function, 0x34) & 0xFC; - // Walk the linked list (cap_id @ +0, next_ptr @ +1) + // Walk the conventional 256-byte capability list (cap_id @ +0, + // next_ptr @ +1). Firmware owns these pointers, so reject offsets + // below the capability area and cycles instead of allowing a corrupt + // list to spin forever during device initialization or resume. + uint64_t visited = 0; while (offset != 0) { + if (offset < 0x40) return 0; + uint64_t bit = 1ULL << (offset >> 2); + if (visited & bit) return 0; + visited |= bit; + uint8_t id = ReadConfig8(bus, device, function, offset); if (id == capId) { return offset; @@ -249,6 +281,38 @@ namespace Pci { return 0; } + bool DisableInterruptDelivery(uint8_t bus, uint8_t device, uint8_t function) { + uint16_t command = LegacyRead16(bus, device, function, + (uint8_t)PCI_REG_COMMAND); + if (!(command & PCI_CMD_INTX_DISABLE)) { + LegacyWrite16(bus, device, function, (uint8_t)PCI_REG_COMMAND, + command | PCI_CMD_INTX_DISABLE); + } + + bool disabledMessageDelivery = false; + uint8_t msi = FindCapability(bus, device, function, PCI_CAP_MSI); + if (msi != 0) { + uint16_t control = LegacyRead16(bus, device, function, msi + 2); + if (control & 1) { + LegacyWrite16(bus, device, function, msi + 2, control & ~1u); + disabledMessageDelivery = true; + } + } + + uint8_t msix = FindCapability(bus, device, function, PCI_CAP_MSIX); + if (msix != 0) { + uint16_t control = LegacyRead16(bus, device, function, msix + 2); + if (control & (1u << 15)) { + // Function-mask before clearing MSI-X Enable so no vector can + // escape during the ownership transition. + uint16_t disabled = (control | (1u << 14)) & ~(1u << 15); + LegacyWrite16(bus, device, function, msix + 2, disabled); + disabledMessageDelivery = true; + } + } + return disabledMessageDelivery; + } + // ------------------------------------------------------------------------- // PCI class code names // ------------------------------------------------------------------------- diff --git a/kernel/src/Pci/Pci.hpp b/kernel/src/Pci/Pci.hpp index aa64915..4837df8 100644 --- a/kernel/src/Pci/Pci.hpp +++ b/kernel/src/Pci/Pci.hpp @@ -61,7 +61,8 @@ namespace Pci { void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value); // PCI capability IDs - constexpr uint8_t PCI_CAP_MSI = 0x05; + constexpr uint8_t PCI_CAP_MSI = 0x05; + constexpr uint8_t PCI_CAP_MSIX = 0x11; // Walk the PCI capability linked list for a given device. // Returns the config-space offset of the capability, or 0 if not found. @@ -96,6 +97,10 @@ namespace Pci { // Enable memory space access and bus mastering in PCI command register. void EnableBusMaster(uint8_t bus, uint8_t device, uint8_t function); + // Disable legacy INTx plus any enabled MSI/MSI-X capability. Returns true + // when message-signalled delivery had been enabled and was turned off. + bool DisableInterruptDelivery(uint8_t bus, uint8_t device, uint8_t function); + // ========================================================================= // PCI driver matching // ========================================================================= diff --git a/kernel/src/Sched/Scheduler.cpp b/kernel/src/Sched/Scheduler.cpp index 9d3d798..7248e94 100644 --- a/kernel/src/Sched/Scheduler.cpp +++ b/kernel/src/Sched/Scheduler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -71,21 +72,105 @@ namespace Sched { // It is held ACROSS context switches to prevent the race where // another CPU picks up a process whose RSP hasn't been saved yet. // The resumed process releases it. - static kcp::Spinlock schedLock; + // schedLock is a baton: the context that acquires it switches stacks and + // the context being resumed releases it. A Spinlock's ordinary shared + // savedFlags field cannot represent that -- it would restore the switching + // context's IF state into the resumed stack. Keep the flags with each saved + // process/idle context instead, so timer frames resume with IF=0 and syscall + // frames resume with their original IF=1 even after CPU migration. + class SchedulerSpinlock { + kcp::Spinlock lock; + uint64_t processFlags[MaxProcesses] = {}; + uint64_t idleFlags[Smp::MaxCPUs] = {}; + + public: + void Acquire() { + uint64_t flags = lock.AcquireIrqSave(); + auto* cpu = Smp::GetCurrentCpuData(); + if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) { + processFlags[cpu->currentSlot] = flags; + } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) { + idleFlags[cpu->cpuIndex] = flags; + } + } + + void Release() { + uint64_t flags = 0; // Fresh startup remains IF=0 until JumpToUserMode's IRETQ. + auto* cpu = Smp::GetCurrentCpuData(); + if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) { + flags = processFlags[cpu->currentSlot]; + } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) { + flags = idleFlags[cpu->cpuIndex]; + } + lock.ReleaseIrqRestore(flags); + } + + void ReleaseForStartup() { + lock.ReleaseIrqNoRestore(); + } + }; + + static SchedulerSpinlock schedLock; // Approximate count of Ready processes. Incremented/decremented // under schedLock. Idle CPUs check this to avoid scanning all 256 // process slots on every timer tick. - static volatile int readyCount = 0; + // Updated while schedLock is held, but sampled locklessly by idle CPUs and + // reschedule-IPI paths. Atomic loads make those wakeup decisions coherent + // across cores instead of relying on volatile's compiler-only semantics. + static std::atomic readyCount{0}; + + // Per-object wake generations close the readiness-check/enrollment gap. + // Each bucket packs [63:32] = a tag identifying the last object to signal + // it and [31:0] = a signal counter, published together in one atomic word. + // Publishing both halves atomically lets a waiter distinguish "exactly one + // signal landed here and it carried someone else's tag" (safe to keep + // blocking) from "something ambiguous happened" (must treat as a wake). + // Without the tag, an unrelated hot object sharing the bucket forces the + // waiter to re-run its condition on every signal. Tag collisions only cost + // a spurious wake; they can never lose one, which is the only direction + // that would reintroduce the lost-wakeup bug this table exists to prevent. + static constexpr uint32_t ObjectWakeBuckets = 1024; + static std::atomic objectWakeEpochs[ObjectWakeBuckets] = {}; + + static uint32_t ObjectWakeBucket(void* object) { + uint64_t value = (uint64_t)object; + return (uint32_t)(((value >> 4) ^ (value >> 12)) & (ObjectWakeBuckets - 1)); + } + + static uint32_t ObjectWakeTag(void* object) { + uint64_t value = (uint64_t)object; + value ^= value >> 32; + value *= 0x9E3779B97F4A7C15ULL; + return (uint32_t)(value >> 32); + } // The idle loop runs in the kernel PML4 static uint64_t GetKernelCR3() { return (uint64_t)Memory::VMM::g_paging->PML4; } - static void RescheduleIpiHandler(uint8_t) { + static void RescheduleIpiHandler(uint8_t, bool interruptedUser) { auto* cpu = Smp::GetCurrentCpuData(); - if (cpu == nullptr || cpu->currentSlot >= 0 || readyCount <= 0) { + if (cpu == nullptr) { + return; + } + + // Process teardown uses this IPI to promptly stop sibling threads on + // other CPUs. A user-mode sibling can exit immediately. If the IPI + // interrupted a syscall, merely expire its slice: teardown on that + // suspended kernel stack could reacquire a mutex it already owns. + if (cpu->currentSlot >= 0 && + processTable[cpu->currentSlot].killPending.load(std::memory_order_acquire)) { + if (interruptedUser) { + ExitProcess(); + return; + } + processTable[cpu->currentSlot].sliceRemaining = 0; + return; + } + + if (cpu->currentSlot >= 0 || readyCount <= 0 || cpu->reservedForKernelWork) { return; } @@ -101,10 +186,10 @@ namespace Sched { if (target == nullptr || !target->started) return false; if (target->cpuIndex == sourceCpuIndex) return false; if (target->currentSlot >= 0) return false; + if (target->reservedForKernelWork) return false; - Hal::LocalApic::SendFixedIpi(target->lapicId, + return Hal::LocalApic::SendFixedIpi(target->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE); - return true; }; if (tryKick(Smp::GetCpuData(0))) { @@ -163,7 +248,9 @@ namespace Sched { // The schedLock is held (acquired by the switching-from CPU's Schedule). static void ProcessStartup() { // Release the schedLock that the switching-from CPU held - schedLock.Release(); + // without consulting this brand-new slot's (intentionally empty) + // saved interrupt state. JumpToUserMode's IRETQ enables interrupts. + schedLock.ReleaseForStartup(); auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; @@ -526,10 +613,11 @@ namespace Sched { // Give the new thread its own TLS block when the image has one. // We are executing inside the process's address space, so the // freshly mapped pages and the template are directly addressable. - // The block is process-heap memory; it is reclaimed with the rest - // of the user half at process teardown (a thread that exits before - // its process leaks its block until then - acceptable for now). + // The block uses process-heap virtual space. Its physical pages are + // reclaimed when the sibling exits (or if creation fails). uint64_t threadFsBase = 0; + uint64_t threadTlsBase = 0; + uint64_t threadTlsPages = 0; Process& primary = processTable[primarySlot_]; if (primary.tlsMemSize > 0) { uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign; @@ -537,12 +625,15 @@ namespace Sched { uint64_t total = S + 16; uint64_t numPages = (total + 0xFFF) / 0x1000; - schedLock.Acquire(); - uint64_t base = primary.heapNext; - primary.heapNext += numPages * 0x1000; - schedLock.Release(); + uint64_t base = 0; + if (!ReserveUserHeapRange(primarySlot_, numPages * 0x1000, base)) { + Kt::KernelLogStream(Kt::ERROR, "Sched") + << "Thread TLS exceeds user heap range"; + return -1; + } bool ok = true; + uint64_t mappedPages = 0; for (uint64_t p = 0; p < numPages; p++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { @@ -552,11 +643,14 @@ namespace Sched { uint64_t physAddr = Memory::SubHHDM((uint64_t)page); if (!Memory::VMM::Paging::MapUserIn(sharedPml4, physAddr, base + p * 0x1000)) { + Memory::g_pfa->Free(page); ok = false; break; } + mappedPages++; } if (!ok) { + Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages); Kt::KernelLogStream(Kt::ERROR, "Sched") << "Thread TLS allocation failed"; return -1; @@ -567,6 +661,8 @@ namespace Sched { primary.tlsFileSize); *(uint64_t*)tp = tp; /* ABI self-pointer */ threadFsBase = tp; + threadTlsBase = base; + threadTlsPages = numPages; } // We do not write to the user stack from kernel mode (that would @@ -578,7 +674,11 @@ namespace Sched { // Allocate kernel stack. void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages); - if (stackMem == nullptr) return -1; + if (stackMem == nullptr) { + if (threadTlsPages != 0) + Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages); + return -1; + } memset(stackMem, 0, StackSize); uint8_t* kernelStackBase = (uint8_t*)stackMem; uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize; @@ -598,6 +698,8 @@ namespace Sched { if (slot < 0) { schedLock.Release(); Memory::g_pfa->Free(stackMem, StackPages); + if (threadTlsPages != 0) + Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages); return -1; } @@ -663,6 +765,54 @@ namespace Sched { return tid; } + bool ReserveUserHeapRange(int primarySlot_, uint64_t size, uint64_t& outVa) { + outVa = 0; + if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 || + (size & 0xFFFULL) != 0) { + return false; + } + + schedLock.Acquire(); + Process& primary = processTable[primarySlot_]; + uint64_t base = primary.heapNext; + bool validOwner = primary.primarySlot == primarySlot_ && + primary.state != ProcessState::Free && + primary.state != ProcessState::Terminated; + bool fits = validOwner && base >= UserHeapBase && + base <= UserHeapLimit && size <= UserHeapLimit - base; + if (fits) { + primary.heapNext = base + size; + outVa = base; + } + schedLock.Release(); + return fits; + } + + static void FreeSiblingThreadTls(Process& thr) { + int primarySlot_ = thr.primarySlot; + if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || + primarySlot_ == (int)(&thr - processTable) || thr.fsBase == 0) { + return; + } + + Process& primary = processTable[primarySlot_]; + if (primary.tlsMemSize == 0 || primary.pml4Phys == 0) { + thr.fsBase = 0; + return; + } + + uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign; + uint64_t blockSize = (primary.tlsMemSize + align - 1) & ~(align - 1); + if (blockSize > thr.fsBase || blockSize > UINT64_MAX - 16) { + Panic("Invalid sibling TLS metadata during thread teardown", nullptr); + } + + uint64_t base = thr.fsBase - blockSize; + uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000; + thr.fsBase = 0; + Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages); + } + // Switch away from a slot we have just marked non-runnable while holding // schedLock. Mirrors SwitchAwayFromBlockedCurrentLocked but does NOT // assume the caller will be resumed -- used by ExitCurrentThread. @@ -714,10 +864,20 @@ namespace Sched { __builtin_unreachable(); } - thr.killPending = false; + bool processExitKill = + thr.killPending.exchange(false, std::memory_order_acq_rel); thr.startPending = false; thr.exitCode = exitCode; + // TLS belongs only to this sibling. Reclaim it before publishing the + // Terminated state; otherwise a long-lived multithreaded GUI process + // leaks these pages until the entire process exits. When the primary + // is already coordinating full teardown, defer this to its final + // sibling sweep: the primary may be waiting from an IF=0 timer frame + // and therefore cannot acknowledge our TLB-shootdown IPI here. + if (!processExitKill) + FreeSiblingThreadTls(thr); + schedLock.Acquire(); thr.state = ProcessState::Terminated; thr.runningOnCpu = -1; @@ -847,10 +1007,16 @@ namespace Sched { // New processes release it in ProcessStartup(). // ==================================================================== - // Reclaim terminated process slots. Called from BSP's Tick only, - // NOT from every Schedule() call on every CPU. This avoids holding - // schedLock (with interrupts disabled) during PFA::Free on the - // hot scheduling path. + // BSP maintenance that may allocate/free memory or touch policy MSRs must + // run from the idle kernel context, never from the timer interrupt. The + // pending flag is set by process teardown on any CPU and periodically by + // the BSP timer. It remains set until an idle maintenance pass consumes it. + static std::atomic bspMaintenancePending{false}; + static uint64_t nextBspPolicyTick = 0; // BSP timer only + + // Reclaim terminated process slots from BSP idle context. This avoids + // holding schedLock (with interrupts disabled) during PFA::Free on the + // hot scheduling path or inside a hard interrupt. static void ReclaimTerminated() { schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { @@ -880,7 +1046,17 @@ namespace Sched { return readyCount > 0; } - void RunBspMaintenance() { + void RequestKernelWork() { + auto* cpu = Smp::GetCurrentCpuData(); + if (cpu != nullptr && cpu->currentSlot >= 0) { + cpu->kernelWorkPending = true; + } + } + + // Bounded, IRQ-safe portion of BSP maintenance. Keeping deadline expiry + // on every 1 ms BSP tick prevents timed input/object waits from depending + // on the BSP becoming idle under sustained desktop load. + static void WakeExpiredProcesses() { bool wokeProcesses = false; schedLock.Acquire(); @@ -902,12 +1078,19 @@ namespace Sched { if (wokeProcesses) { KickOneIdleCpu(0); } + } + + void RunBspMaintenance() { + // Clear before scanning. A producer that races this pass sets the flag + // again and therefore cannot have its request acknowledged by mistake. + if (!bspMaintenancePending.exchange(false, std::memory_order_acq_rel)) { + return; + } ReclaimTerminated(); // Thermal governor step (rate-limited internally to 500 ms). - // Runs here because BSP maintenance is reached from both the idle - // loop and the timer tick, so it keeps running under full load. + // The timer requests this idle-context pass even under full load. Hal::CpuPower::ThermalTick(Timekeeping::GetMilliseconds()); } @@ -933,21 +1116,41 @@ namespace Sched { schedLock.Acquire(); - // Find the next Ready process (round-robin from after current slot) + // Normally select the next Ready process. A pending bottom half gets + // one deliberate trip through the idle kernel context first; that + // context drains bounded work and immediately schedules user work + // again. This avoids both hard-IRQ callbacks and bottom-half starvation + // when there is always at least one runnable desktop process. int next = -1; + bool yieldToKernel = cpu->currentSlot >= 0 && cpu->kernelWorkPending; + if (yieldToKernel) cpu->kernelWorkPending = false; int start = (cpu->currentSlot >= 0) ? cpu->currentSlot + 1 : 0; - for (int i = 0; i < MaxProcesses; i++) { - int idx = (start + i) % MaxProcesses; - if (processTable[idx].state == ProcessState::Ready) { - next = idx; - break; + if (!yieldToKernel) { + for (int i = 0; i < MaxProcesses; i++) { + int idx = (start + i) % MaxProcesses; + if (processTable[idx].state == ProcessState::Ready) { + next = idx; + break; + } } } if (next < 0) { - // No ready processes. If we were running one, return to idle. + // No other process is ready. A normal time-slice expiry must keep + // the sole Running process on this CPU: bouncing it through idle + // can suspend a syscall while it owns a process-context Mutex, then + // let ServiceDeferredWork spin on that same mutex with the idle + // context reserved against scheduling. That is a permanent + // self-deadlock. Only an explicit, ring-3-safe bottom-half request + // is allowed to yield a running process to idle. if (cpu->currentSlot >= 0) { + if (!yieldToKernel) { + processTable[cpu->currentSlot].sliceRemaining = TimeSliceMs; + schedLock.Release(); + return; + } + int oldSlot = cpu->currentSlot; processTable[oldSlot].state = ProcessState::Ready; readyCount++; @@ -1019,7 +1222,7 @@ namespace Sched { schedLock.Release(); } - void Tick(uint32_t elapsedMs) { + void Tick(uint32_t elapsedMs, bool interruptedUser) { auto* cpu = Smp::GetCurrentCpuData(); // Pick up thermal-governor frequency changes (HWP requests are @@ -1027,9 +1230,21 @@ namespace Sched { // the governor bumped the policy epoch since our last tick. Hal::CpuPower::ApplyPolicyIfChanged(); - // BSP: wake sleeping processes and reclaim terminated slots + // BSP: expire timed waits in the bounded IRQ-safe pass. Queue heavier + // reclamation/policy work for the idle kernel context. if (cpu->cpuIndex == 0) { - RunBspMaintenance(); + WakeExpiredProcesses(); + + uint64_t now = Timekeeping::GetTicks(); + if (nextBspPolicyTick == 0 || now >= nextBspPolicyTick) { + nextBspPolicyTick = now + 500; + bspMaintenancePending.store(true, std::memory_order_release); + } + + if (interruptedUser && + bspMaintenancePending.load(std::memory_order_acquire)) { + RequestKernelWork(); + } } int slot = cpu->currentSlot; @@ -1052,12 +1267,21 @@ namespace Sched { processTable[slot].cpuTimeMs += elapsedMs; - // Check if another CPU requested this process be killed. - // We are on the CPU running it, so ExitProcess is safe here. + if (cpu->kernelWorkPending) { + processTable[slot].sliceRemaining = 0; + } + + // Check if another CPU requested this process be killed. Full teardown + // takes process-context mutexes. It is safe here only when the timer + // interrupted ring 3; if it interrupted a syscall, teardown could try + // to reacquire a lock already held by this same suspended kernel stack. + // In that case force a reschedule and finish at syscall-return instead. if (processTable[slot].killPending) { - processTable[slot].killPending = false; - ExitProcess(); - return; + if (interruptedUser) { + ExitProcess(); + return; + } + processTable[slot].sliceRemaining = 0; } if (processTable[slot].sliceRemaining > elapsedMs) { @@ -1066,7 +1290,13 @@ namespace Sched { processTable[slot].sliceRemaining = 0; } - if (processTable[slot].sliceRemaining == 0) { + // Never context-switch away from an arbitrary kernel stack. Syscalls + // routinely own non-IRQ-disabling subsystem mutexes; suspending that + // stack and dispatching a process that wants the same mutex leaves the + // new process spinning in ring 0, where the timer deliberately cannot + // preempt it to resume the owner. Preserve the expired slice and take + // the switch on the first timer interrupt after SYSRET reaches ring 3. + if (processTable[slot].sliceRemaining == 0 && interruptedUser) { Schedule(); } } @@ -1080,6 +1310,14 @@ namespace Sched { return processTable[primary].pid; } + void ExitIfKilled() { + auto* cpu = Smp::GetCurrentCpuData(); + int slot = cpu ? cpu->currentSlot : -1; + if (slot >= 0 && processTable[slot].killPending) { + ExitProcess(); + } + } + int GetCurrentTid() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; @@ -1173,8 +1411,17 @@ namespace Sched { // terminated immediately; siblings running on other CPUs are tagged // killPending and we spin until their tick handler routes them // through ExitCurrentThread. + // Several device syscalls have bounded waits up to five seconds. Give + // an interrupted kernel-mode sibling room to finish that recovery + // path before declaring it wedged. + static constexpr uint64_t SiblingDrainTimeoutMs = 10000; + static constexpr uint32_t SiblingDrainMaxPasses = 262144; + uint64_t siblingDrainDeadline = Timekeeping::GetTicks() + SiblingDrainTimeoutMs; + uint32_t siblingDrainPasses = 0; + for (;;) { bool anyRunning = false; + bool targetCpus[Smp::MaxCPUs] = {}; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (i == primarySlot_) continue; @@ -1184,7 +1431,10 @@ namespace Sched { continue; } if (st == ProcessState::Running) { - processTable[i].killPending = true; + processTable[i].killPending.store(true, std::memory_order_release); + int targetCpu = processTable[i].runningOnCpu; + if (targetCpu >= 0 && targetCpu < Smp::MaxCPUs) + targetCpus[targetCpu] = true; anyRunning = true; continue; } @@ -1211,6 +1461,30 @@ namespace Sched { } schedLock.Release(); if (!anyRunning) break; + + // Do not depend solely on a remote CPU's next periodic timer. + // Laptop firmware/IRQ pathologies can delay that tick, and the + // old unbounded wait then trapped the exiting CPU forever. + if ((siblingDrainPasses & 0x3FF) == 0) { + for (int i = 0; i < Smp::GetCpuCount(); i++) { + if (!targetCpus[i] || i == cpu->cpuIndex) continue; + Smp::CpuData* target = Smp::GetCpuData(i); + if (target != nullptr && target->started) { + (void)Hal::LocalApic::SendFixedIpi(target->lapicId, + Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE); + } + } + } + + siblingDrainPasses++; + if (siblingDrainPasses >= SiblingDrainMaxPasses || + Timekeeping::GetTicks() >= siblingDrainDeadline) { + // It is unsafe to free the shared address space while another + // CPU may still execute in it. A visible panic is preferable + // to silent lockup or use-after-free corruption. + Panic("Timed out draining sibling threads during process exit", nullptr); + } + // Wait for the sibling CPU(s) to observe killPending on their // next timer tick. Briefly busy-wait; this path is rare. for (int spin = 0; spin < 1024; spin++) { @@ -1224,6 +1498,10 @@ namespace Sched { if (i == primarySlot_) continue; if (processTable[i].primarySlot != primarySlot_) continue; if (processTable[i].state != ProcessState::Terminated) continue; + // Ready/blocked siblings terminated inline above never ran their + // ordinary ExitCurrentThread cleanup. Running siblings normally + // arrive here with fsBase already cleared, making this a no-op. + FreeSiblingThreadTls(processTable[i]); void* stackBase = (void*)processTable[i].stackBase; processTable[i].stackBase = 0; processTable[i].pml4Phys = 0; @@ -1280,6 +1558,7 @@ namespace Sched { proc.state = ProcessState::Terminated; proc.runningOnCpu = -1; proc.reapReady = true; + bspMaintenancePending.store(true, std::memory_order_release); // Publish the exit code before waking waiters so SYS_WAITPID // observes it the moment it unblocks. @@ -1429,6 +1708,8 @@ namespace Sched { } void BlockOnPid(int pid) { + ExitIfKilled(); + // If the target is already dead, return immediately if (!IsAlive(pid)) return; @@ -1464,11 +1745,14 @@ namespace Sched { processTable[slot].sleepUntilTick = 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); + ExitIfKilled(); } void BlockForSleep(uint64_t ms) { if (ms == 0) return; + ExitIfKilled(); + auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return; @@ -1481,11 +1765,14 @@ namespace Sched { processTable[slot].sleepUntilTick = Timekeeping::GetTicks() + ms; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); + ExitIfKilled(); } void BlockOnObject(void* object, uint64_t timeoutMs) { if (object == nullptr) return; + ExitIfKilled(); + auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return; @@ -1499,12 +1786,15 @@ namespace Sched { : 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); + ExitIfKilled(); } bool BlockOnObjectIf(void* object, uint64_t timeoutMs, bool (*shouldBlock)(void*), void* context) { if (object == nullptr || shouldBlock == nullptr) return false; + ExitIfKilled(); + auto* cpu = Smp::GetCurrentCpuData(); if (cpu == nullptr) return false; @@ -1526,12 +1816,58 @@ namespace Sched { : 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); + ExitIfKilled(); return true; } + uint64_t ObserveObjectWake(void* object) { + if (object == nullptr) return 0; + return objectWakeEpochs[ObjectWakeBucket(object)].load(std::memory_order_acquire); + } + + struct ObjectWakeObservation { + void* object; + uint64_t epoch; + }; + + static bool ObjectWakeIsUnchanged(void* opaque) { + auto* observation = (ObjectWakeObservation*)opaque; + uint64_t now = ObserveObjectWake(observation->object); + uint64_t was = observation->epoch; + if (now == was) return true; // nothing signalled this bucket at all + + // Exactly one signal landed here since the observation, and it carried + // a different object's tag, so it cannot have been ours: the caller may + // still block. Any larger or wrapped delta is ambiguous -- more than + // one object may have signalled -- and must be reported as a wake so a + // real edge is never dropped. + if ((uint32_t)now == (uint32_t)was + 1 && + (uint32_t)(now >> 32) != ObjectWakeTag(observation->object)) { + return true; + } + return false; + } + + bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake) { + ObjectWakeObservation observation{object, observedWake}; + return BlockOnObjectIf(object, timeoutMs, ObjectWakeIsUnchanged, &observation); + } + void WakeObjectWaiters(void* object) { if (object == nullptr) return; + // Publish this object's tag and the bumped counter as one word. A CAS + // loop (rather than fetch_add) keeps the pair consistent and, unlike a + // plain store, cannot drop a concurrent waker's increment on the same + // bucket -- a dropped increment would be an unobserved wake edge. + auto& bucket = objectWakeEpochs[ObjectWakeBucket(object)]; + uint64_t tagBits = (uint64_t)ObjectWakeTag(object) << 32; + uint64_t observed = bucket.load(std::memory_order_relaxed); + while (!bucket.compare_exchange_weak( + observed, tagBits | (uint32_t)((uint32_t)observed + 1), + std::memory_order_release, std::memory_order_relaxed)) { + } + bool wokeAny = false; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { @@ -1608,13 +1944,10 @@ namespace Sched { bool inheritRedir = crasher && crasher->redirected && crasher->ioOutHandle >= 0 && crasherSlot >= 0; - Ipc::Object* outObj = nullptr; - Ipc::HandleType outType = Ipc::HandleType::None; - uint32_t outRights = 0; + Ipc::HandleSnapshot outSnapshot; if (inheritRedir) { - if (!Ipc::SnapshotHandleForSlot(crasherSlot, crasher->ioOutHandle, - outType, outObj, outRights) || - outType != Ipc::HandleType::Stream || outObj == nullptr) { + if (!outSnapshot.Capture(crasherSlot, crasher->ioOutHandle) || + outSnapshot.type != Ipc::HandleType::Stream || outSnapshot.object == nullptr) { inheritRedir = false; } } @@ -1630,7 +1963,7 @@ namespace Sched { // so it outlives the crasher's imminent ExitProcess teardown. The // snapshot above and this install run back-to-back with no blocking // call between, while the crasher's handle still pins the stream. - int h = Ipc::InstallHandleForSlot(childSlot, outObj, Ipc::HandleType::Stream, + int h = Ipc::InstallHandleForSlot(childSlot, outSnapshot.object, Ipc::HandleType::Stream, Ipc::RightWrite | Ipc::RightWait | Ipc::RightDup); if (h >= 0) { child->ioOutHandle = h; @@ -1647,3 +1980,9 @@ namespace Sched { } } + +// Called by the syscall entry assembly after the syscall implementation has +// released all of its subsystem locks but before returning to ring 3. +extern "C" void SchedExitIfKilled() { + Sched::ExitIfKilled(); +} diff --git a/kernel/src/Sched/Scheduler.hpp b/kernel/src/Sched/Scheduler.hpp index d6dc66d..44b83ed 100644 --- a/kernel/src/Sched/Scheduler.hpp +++ b/kernel/src/Sched/Scheduler.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include namespace Sched { @@ -28,6 +29,24 @@ namespace Sched { // Heavy recursive programs (GCC's cc1plus) need megabytes of stack. static constexpr uint64_t UserStackMax = 64 * 1024 * 1024; // 64 MiB static constexpr uint64_t UserHeapBase = 0x40000000ULL; // User heap start VA + // Keep the bump-allocated heap below the fixed shared-library range. A + // failed bound check is recoverable (SYS_ALLOC returns null); allowing the + // heap to cross this boundary would silently replace library PTEs. + static constexpr uint64_t UserHeapLimit = 0x60000000ULL; + // Surface mappings use reusable, fixed-size per-process slots instead of + // consuming heap VA forever. 64 * 32 MiB occupies [0x7000000000, + // 0x7080000000), comfortably below the main user stack. + static constexpr uint64_t UserSurfaceBase = 0x7000000000ULL; + static constexpr uint64_t UserSurfaceSlotSize = 32ULL * 1024 * 1024; + static constexpr uint64_t UserSurfaceSlots = 64; + static constexpr uint64_t UserSurfaceEnd = + UserSurfaceBase + UserSurfaceSlots * UserSurfaceSlotSize; + // The two write-combined scanout mappings live in their own region. + static constexpr uint64_t UserFramebufferBase = 0x7100000000ULL; + static constexpr uint64_t UserFramebufferLimit = 0x7140000000ULL; + static_assert(UserHeapBase < UserHeapLimit); + static_assert(UserSurfaceEnd <= UserFramebufferBase); + static_assert(UserFramebufferLimit < UserStackTop - UserStackMax); static constexpr uint32_t UserReadDirSlots = 64; // rotating scratch pages for SYS_READDIR static constexpr uint64_t UserReadDirBase = UserHeapBase - (uint64_t)UserReadDirSlots * 0x1000ULL; @@ -74,7 +93,7 @@ namespace Sched { uint64_t tlsAlign = 0; int runningOnCpu; // CPU index running this process (-1 if not running) - bool killPending = false; // Set by Sys_Kill when target is running on another CPU + std::atomic killPending{false}; // Cross-CPU kill handoff bool reapReady = false; // Set once teardown is complete and BSP may free slot resources bool startPending = false; // Spawned but not yet made runnable @@ -124,9 +143,13 @@ namespace Sched { // True when there is runnable work somewhere in the process table. bool HasReadyProcesses(); + // Ask the current CPU to yield once to its idle kernel context on the next + // scheduler pass so deferred device work cannot starve under full load. + void RequestKernelWork(); + // Called from the APIC timer handler with the elapsed time for that CPU's // tick interval. The BSP runs at 1 ms; APs may use a coarser interval. - void Tick(uint32_t elapsedMs = 1); + void Tick(uint32_t elapsedMs = 1, bool interruptedUser = false); // Get the PID of the currently running process (-1 if idle). // For sibling threads this returns the primary slot's PID -- i.e. the @@ -164,6 +187,11 @@ namespace Sched { // out of the user heap. Returns the new TID, or -1 on failure. int SpawnThread(uint64_t entry, uint64_t arg, uint64_t userStackTop); + // Reserve a page-aligned range from a process's shared bump heap. The + // reservation is serialized across sibling threads and bounded below the + // fixed shared-library mappings. + bool ReserveUserHeapRange(int primarySlot, uint64_t size, uint64_t& outVa); + // Terminate the currently executing thread. If this is the main thread, // the entire process exits (equivalent to ExitProcess). [[noreturn]] void ExitCurrentThread(int exitCode); @@ -194,8 +222,14 @@ namespace Sched { bool BlockOnObjectIf(void* object, uint64_t timeoutMs, bool (*shouldBlock)(void*), void* context); - // BSP-only scheduler housekeeping: wake expired sleepers and reclaim - // terminated process resources. + // Take this token before testing an object's ready condition, then pass it + // to BlockOnObjectSince. A wake between the test and scheduler enrollment + // makes the latter return without blocking instead of losing the edge. + uint64_t ObserveObjectWake(void* object); + bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake); + + // BSP idle-context housekeeping: reclaim terminated process resources and + // run policy work previously queued by the timer or process teardown. void RunBspMaintenance(); // Return the earliest blocked sleep/object timeout deadline in ticks, diff --git a/kernel/src/Terminal/Terminal.cpp b/kernel/src/Terminal/Terminal.cpp index e147b57..76c2e94 100644 --- a/kernel/src/Terminal/Terminal.cpp +++ b/kernel/src/Terminal/Terminal.cpp @@ -241,6 +241,15 @@ namespace Kt { g_suppressKernelLog = true; } + void EnablePanicOutput() { + // Once the graphical desktop starts, ordinary console writes are + // suppressed to avoid painting over it. A kernel panic is different: + // leaving suppression enabled makes the halted system look like a + // frozen desktop. Do not acquire g_termLock here; panic may have + // interrupted its owner on this or another CPU. + g_suppressKernelLog = false; + } + int64_t ReadKernelLog(char* buf, uint64_t size) { if (buf == nullptr || size == 0) return 0; diff --git a/kernel/src/Terminal/Terminal.hpp b/kernel/src/Terminal/Terminal.hpp index 14752a5..4783484 100644 --- a/kernel/src/Terminal/Terminal.hpp +++ b/kernel/src/Terminal/Terminal.hpp @@ -117,6 +117,10 @@ namespace Kt extern bool g_suppressKernelLog; void SuppressKernelLog(); + // Panic output must bypass desktop-time console suppression. This is + // intentionally lock-free: a panic can occur while another CPU owns the + // terminal mutex, and the system is about to halt. + void EnablePanicOutput(); int64_t ReadKernelLog(char* buf, uint64_t size); class KernelLogStream { @@ -189,4 +193,4 @@ extern Kt::KernelErrorStream kerr; struct flanterm_context; namespace Kt { extern flanterm_context *ctx; -} \ No newline at end of file +} diff --git a/kernel/src/Timekeeping/ApicTimer.cpp b/kernel/src/Timekeeping/ApicTimer.cpp index c712d2d..ea7094f 100644 --- a/kernel/src/Timekeeping/ApicTimer.cpp +++ b/kernel/src/Timekeeping/ApicTimer.cpp @@ -10,12 +10,15 @@ #include #include #include +#include #include #include #include #include #include +#include #include +#include #include #include #include @@ -47,7 +50,7 @@ namespace Timekeeping { static std::atomic g_tickCount{0}; static uint32_t g_ticksPerMs = 0; - static bool g_schedEnabled = false; + static std::atomic g_schedEnabled{false}; static uint32_t CountForIntervalMs(uint32_t intervalMs) { return g_ticksPerMs * intervalMs; @@ -71,7 +74,7 @@ namespace Timekeeping { // Timer IRQ handler: BSP runs timekeeping + scheduler accounting; APs // run scheduler accounting only. - static void TimerHandler(uint8_t) { + static void TimerHandler(uint8_t, bool interruptedUser) { auto* cpu = Smp::GetCurrentCpuData(); uint32_t schedElapsedMs = (cpu->cpuIndex == 0) ? BSP_TICK_INTERVAL_MS : AP_TICK_INTERVAL_MS; @@ -82,10 +85,29 @@ namespace Timekeeping { Drivers::Net::E1000E::Poll(); } Drivers::USB::HidKeyboard::Tick(); + + // Bottom halves normally run from idle, but a continuously busy + // desktop may never enter idle. Force one bounded idle-context + // pass for USB immediately, and for network RX at most every + // 10 ms (important for E1000E's polling fallback). + bool usbWork = Drivers::USB::Xhci::HasDeferredWork(); + bool audioWork = Drivers::Audio::IntelHda::HasDeferredWork(); + bool netWork = Drivers::Net::E1000::HasDeferredWork() || + Drivers::Net::E1000E::HasDeferredWork(); + // A forced trip into the idle bottom-half context is safe only when + // this timer interrupted ring 3. If it interrupted a syscall, that + // suspended kernel stack may own one of the process-context mutexes + // used below; the reserved idle context would then spin on the lock + // while refusing to schedule its owner, deadlocking the CPU. + if (interruptedUser && + (usbWork || audioWork || + (netWork && (g_tickCount.load(std::memory_order_relaxed) % 10 == 0)))) { + Sched::RequestKernelWork(); + } } if (g_schedEnabled) { - Sched::Tick(schedElapsedMs); + Sched::Tick(schedElapsedMs, interruptedUser); } } @@ -188,7 +210,7 @@ namespace Timekeeping { } void EnableSchedulerTick() { - g_schedEnabled = true; + g_schedEnabled.store(true, std::memory_order_release); } void ApicTimerInitializeAP() { @@ -202,16 +224,36 @@ namespace Timekeeping { ProgramTimer(true, AP_TICK_INTERVAL_MS); } - void IdleOnce(bool hasMwait, volatile uint64_t* monitorAddr) { - (void)hasMwait; - (void)monitorAddr; + void ServiceDeferredWork() { auto* cpu = Smp::GetCurrentCpuData(); + if (cpu == nullptr || cpu->currentSlot >= 0) return; + cpu->kernelWorkPending = false; + + // These bottom halves deliberately run outside a process because they + // enter subsystems with process-context mutexes. Do not let a timer or + // reschedule IPI switch this idle stack to a process while such a lock + // remains held: the process could immediately wait on that same lock, + // leaving the suspended idle owner with no opportunity to release it. + // The scheduler and reschedule-IPI path both honor this reservation. + bool wasReserved = cpu->reservedForKernelWork; + cpu->reservedForKernelWork = true; // Drain USB hot-plug deferred work from any idle core, not just the BSP. if (Drivers::USB::Xhci::HasDeferredWork()) { Drivers::USB::Xhci::ProcessDeferredWork(); } + // NIC hard IRQs only acknowledge/mask and queue RX work. Dispatching + // Ethernet/TCP/UDP here keeps process-context IPC mutexes out of IRQs. + Drivers::Net::E1000::ProcessDeferredWork(); + Drivers::Net::E1000E::ProcessDeferredWork(); + + // HDA completion IRQs only acknowledge/mask. Resampling and DMA-ring + // refill are far too expensive for hard interrupt context. + if (cpu->cpuIndex == 0) { + Drivers::Audio::IntelHda::ProcessDeferredWork(); + } + // Complete any boot-deferred Bluetooth bring-up (Intel firmware // download) here instead of on the boot path: the download takes // seconds and used to stall kmain before the first process spawned. @@ -228,10 +270,35 @@ namespace Timekeeping { // the adapter is down. Drivers::USB::Bluetooth::ServiceEvents(); + // Thermal policy records transitions during BSP maintenance; print + // them from this explicitly non-interrupt idle path. + Hal::CpuPower::ServiceDeferredDiagnostics(); + + cpu->reservedForKernelWork = wasReserved; + } + + void IdleOnce(bool hasMwait, volatile uint64_t* monitorAddr) { + (void)hasMwait; + (void)monitorAddr; + auto* cpu = Smp::GetCurrentCpuData(); + + // HaltWithInterruptsDisabled() deliberately returns with IF clear so + // the next readiness check can lead into an atomic STI+HLT. Deferred + // work cannot inherit that state: xHCI/Bluetooth service contains + // wall-clock waits and depends on the BSP timer and device MSIs. If + // it runs with IF=0, GetMilliseconds() never advances on the BSP and + // the completion interrupt cannot arrive, producing a permanent idle- + // loop freeze. Keep IRQs enabled for all active idle-context work and + // disable them again only immediately before entering HLT below. + asm volatile("sti" ::: "memory"); + + ServiceDeferredWork(); + if (cpu == nullptr || cpu->cpuIndex != 0 || !g_schedEnabled || g_ticksPerMs == 0) { // Non-BSP or pre-scheduler fallback: plain HLT keeps the LAPIC // timer running in C1 on hardware where MWAIT promotes to deeper // states that freeze the timer (no ARAT in practice). + asm volatile("cli" ::: "memory"); Hal::HaltWithInterruptsDisabled(); return; } @@ -256,6 +323,7 @@ namespace Timekeeping { // HLT (vs MWAIT) is the second half: MWAIT EAX=0 is a *hint* and // the CPU may go deeper than C1; HLT generally stays in C1, which // is the deepest state guaranteed to keep the LAPIC timer alive. + asm volatile("cli" ::: "memory"); Hal::HaltWithInterruptsDisabled(); } diff --git a/kernel/src/Timekeeping/ApicTimer.hpp b/kernel/src/Timekeeping/ApicTimer.hpp index cbc246c..0b1a525 100644 --- a/kernel/src/Timekeeping/ApicTimer.hpp +++ b/kernel/src/Timekeeping/ApicTimer.hpp @@ -28,6 +28,9 @@ namespace Timekeeping { // Enable scheduler tick (called after scheduler is initialized) void EnableSchedulerTick(); + // Run bounded process-context bottom halves from an idle CPU. + void ServiceDeferredWork(); + // Enter one idle wait cycle for the current CPU. Uses HLT and relies on // the boot-time periodic LAPIC timer to wake -- one-shot wake doesn't // survive deep C-states on some laptop CPUs. diff --git a/programs/include/gui/framebuffer.hpp b/programs/include/gui/framebuffer.hpp index 5a0235d..afb94ec 100644 --- a/programs/include/gui/framebuffer.hpp +++ b/programs/include/gui/framebuffer.hpp @@ -16,6 +16,8 @@ namespace gui { class Framebuffer { uint32_t* hw_fb; uint32_t* hw_fb2; // second scanout buffer (page flip), nullptr if unsupported + uint32_t* hw_direct; // buffer known to be live after page-flip fallback + uint32_t* hw_mirror; // second possible live buffer when SURFLIVE is unavailable uint32_t* back_buf; int fb_width; int fb_height; @@ -66,18 +68,19 @@ class Framebuffer { char user[64] = {}; if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) { auto doc = montauk::config::load_user(user, "display"); - bool enabled = doc.get_bool("graphics.tear_free", true); + bool enabled = doc.get_bool("graphics.tear_free", false); doc.destroy(); return enabled; } auto doc = montauk::config::load("display"); - bool enabled = doc.get_bool("graphics.tear_free", true); + bool enabled = doc.get_bool("graphics.tear_free", false); doc.destroy(); return enabled; } public: - Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), back_buf(nullptr), + Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), hw_direct(nullptr), hw_mirror(nullptr), + back_buf(nullptr), fb_width(0), fb_height(0), fb_pitch(0), hw_next(1) { montauk::abi::FbInfo info; montauk::fb_info(&info); @@ -87,14 +90,39 @@ public: fb_pitch = (int)info.pitch; hw_fb = (uint32_t*)montauk::fb_map(); + hw_direct = hw_fb; back_buf = (uint32_t*)montauk::alloc((uint64_t)fb_height * fb_pitch); // Hardware page flipping: fb_map() maps the second scanout buffer - // directly after the first when the kernel supports flipping. - if (hw_fb && prefer_hardware_page_flip() - && montauk::fb_flip(-1, 0) == 1) { - uint64_t pages = ((uint64_t)fb_height * fb_pitch + 0xFFF) / 0x1000; - hw_fb2 = (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000); + // directly after the first when the kernel supports flipping. Acquire + // the scanout before choosing an off-screen buffer: another fullscreen + // process (notably login.elf) may have left either buffer live. + bool flip_supported = hw_fb && montauk::fb_flip(-1, 0) == 1; + uint64_t pages = ((uint64_t)fb_height * fb_pitch + 0xFFF) / 0x1000; + uint32_t* second = flip_supported + ? (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000) : nullptr; + if (flip_supported && prefer_hardware_page_flip()) { + int64_t front = montauk::fb_flip(-2, 0); + if (front == 0 || front == 1) { + hw_fb2 = second; + hw_next = (int)front ^ 1; + } else { + // If the live register cannot be trusted even at startup, + // avoid flipping and update both possible scanout surfaces. + hw_mirror = second; + } + } else if (flip_supported) { + // A direct-copy client must make buffer 0 live before it starts + // drawing there; the previous fullscreen client may have left + // buffer 1 scanning out. + if (montauk::fb_flip(0, 1) != 0) { + int64_t front = montauk::fb_flip(-2, 0); + if (front == 1) { + hw_direct = second; + } else if (front != 0) { + hw_mirror = second; + } + } } } @@ -280,7 +308,7 @@ public: // currently OFF screen, then ask the display engine to present it at // the next vblank (tear-free). Without it, fall back to copying // straight into the visible framebuffer. - uint32_t* dst_fb = hw_fb; + uint32_t* dst_fb = hw_direct; if (hw_fb2) dst_fb = (hw_next == 1) ? hw_fb2 : hw_fb; // Copy back buffer to hardware framebuffer, row by row (pitch may differ) @@ -289,13 +317,49 @@ public: uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch); uint32_t* dst = (uint32_t*)((uint8_t*)dst_fb + y * fb_pitch); montauk::memcpy(dst, src, row_bytes); + if (hw_mirror) { + uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch); + montauk::memcpy(mirror, src, row_bytes); + } } + if (hw_mirror) asm volatile("sfence" ::: "memory"); if (hw_fb2) { // Wait for the latch (vsync): the buffer we just left becomes // safe to draw into only once the new one is actually scanning. - if (montauk::fb_flip(hw_next, 1) == hw_next) { + asm volatile("sfence" ::: "memory"); + int64_t presented = montauk::fb_flip(hw_next, 1); + if (presented == hw_next) { hw_next ^= 1; + } else { + // Do not retry a faulty page-flip path forever: that turns a + // transient latch failure into an apparently frozen desktop. + // Keep drawing directly into whichever buffer hardware says + // is live for the rest of this process. If SURFLIVE itself is + // unavailable, request buffer 0 without another blocking wait. + int64_t front = montauk::fb_flip(-2, 0); + if (front == 0 || front == 1) { + hw_direct = (front == 1) ? hw_fb2 : hw_fb; + } else { + (void)montauk::fb_flip(0, 0); + hw_direct = hw_fb; + hw_mirror = hw_fb2; + } + hw_fb2 = nullptr; + + // The frame above was copied to the requested (possibly + // off-screen) buffer. Publish it once more to the live direct + // buffer so the cursor and desktop resume immediately. + for (int y = 0; y < fb_height; y++) { + uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch); + uint32_t* dst = (uint32_t*)((uint8_t*)hw_direct + y * fb_pitch); + montauk::memcpy(dst, src, row_bytes); + if (hw_mirror) { + uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch); + montauk::memcpy(mirror, src, row_bytes); + } + } + asm volatile("sfence" ::: "memory"); } } } diff --git a/programs/include/montauk/syscall.h b/programs/include/montauk/syscall.h index 2972ada..d19b054 100644 --- a/programs/include/montauk/syscall.h +++ b/programs/include/montauk/syscall.h @@ -339,6 +339,7 @@ namespace montauk { // the buffer to show; the hardware latches it at vblank (tear-free). // flags bit0 = block until the flip has been latched (vsync). // fb_flip(-1, 0) returns 1 when page flipping is available, 0 when not. + // fb_flip(-2, 0) acquires ownership and returns the live buffer (0 or 1). inline int64_t fb_flip(int64_t index, uint64_t flags) { return syscall2(montauk::abi::SYS_FBFLIP, (uint64_t)index, flags); } diff --git a/programs/man/framebuffer.2 b/programs/man/framebuffer.2 index 684f246..ec61014 100644 --- a/programs/man/framebuffer.2 +++ b/programs/man/framebuffer.2 @@ -21,8 +21,9 @@ than width * 4 due to alignment). bpp is always 32. .SS fb_map - Maps the physical framebuffer into the process address space at - a fixed virtual address (0x50000000) and returns that address. + Maps the physical framebuffer into a dedicated region of the process + address space and returns its address. Programs must use the returned + pointer rather than assuming a particular virtual address. uint32_t* pixels = (uint32_t*)montauk::fb_map(); diff --git a/programs/src/display/main.cpp b/programs/src/display/main.cpp index 4f35f05..c7517cd 100644 --- a/programs/src/display/main.cpp +++ b/programs/src/display/main.cpp @@ -55,8 +55,8 @@ static int g_mode_count = 0; static bool g_driver_available = false; static int g_selected_mode = -1; static int g_pending_brightness = -1; -static bool g_tear_free = true; -static bool g_saved_tear_free = true; +static bool g_tear_free = false; +static bool g_saved_tear_free = false; static int g_mode_scroll = 0; static char g_status[128] = {}; static uint64_t g_status_time = 0; @@ -164,11 +164,11 @@ static void load_graphics_preference() { char user[64] = {}; if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) { auto doc = montauk::config::load_user(user, "display"); - g_tear_free = doc.get_bool("graphics.tear_free", true); + g_tear_free = doc.get_bool("graphics.tear_free", false); doc.destroy(); } else { auto doc = montauk::config::load("display"); - g_tear_free = doc.get_bool("graphics.tear_free", true); + g_tear_free = doc.get_bool("graphics.tear_free", false); doc.destroy(); } g_saved_tear_free = g_tear_free; diff --git a/template/sysroot/include/Api/Syscall.hpp b/template/sysroot/include/Api/Syscall.hpp index a5c83d1..aa7edb2 100644 --- a/template/sysroot/include/Api/Syscall.hpp +++ b/template/sysroot/include/Api/Syscall.hpp @@ -192,6 +192,7 @@ namespace montauk::abi { static constexpr uint64_t SYS_SDR_READ = 146; // (handle, buf, len) -> bytes static constexpr uint64_t SYS_SDR_SETPARAM = 147; // (handle, param, value) static constexpr uint64_t SYS_SDR_GETPARAM = 148; // (handle, param) -> value + static constexpr uint64_t SYS_FBFLIP = 150; static constexpr uint64_t SYS_SETUNIXTIME = 153; static constexpr uint64_t SYS_DISPLAYINFO = 154; static constexpr uint64_t SYS_DISPLAYMODES = 155; diff --git a/template/sysroot/include/gui/framebuffer.hpp b/template/sysroot/include/gui/framebuffer.hpp index 303d657..afb94ec 100644 --- a/template/sysroot/include/gui/framebuffer.hpp +++ b/template/sysroot/include/gui/framebuffer.hpp @@ -8,16 +8,21 @@ #include #include #include +#include #include "gui/gui.hpp" namespace gui { class Framebuffer { uint32_t* hw_fb; + uint32_t* hw_fb2; // second scanout buffer (page flip), nullptr if unsupported + uint32_t* hw_direct; // buffer known to be live after page-flip fallback + uint32_t* hw_mirror; // second possible live buffer when SURFLIVE is unavailable uint32_t* back_buf; int fb_width; int fb_height; int fb_pitch; // in bytes + int hw_next; // scanout buffer the next flip() will present (page flip only) static inline void fill_pixels(uint32_t* dst, int count, uint32_t pixel) { if (!dst || count <= 0) return; @@ -59,8 +64,24 @@ class Framebuffer { return 0xFF000000 | (rr << 16) | (gg << 8) | bb; } + static bool prefer_hardware_page_flip() { + char user[64] = {}; + if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) { + auto doc = montauk::config::load_user(user, "display"); + bool enabled = doc.get_bool("graphics.tear_free", false); + doc.destroy(); + return enabled; + } + auto doc = montauk::config::load("display"); + bool enabled = doc.get_bool("graphics.tear_free", false); + doc.destroy(); + return enabled; + } + public: - Framebuffer() : hw_fb(nullptr), back_buf(nullptr), fb_width(0), fb_height(0), fb_pitch(0) { + Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), hw_direct(nullptr), hw_mirror(nullptr), + back_buf(nullptr), + fb_width(0), fb_height(0), fb_pitch(0), hw_next(1) { montauk::abi::FbInfo info; montauk::fb_info(&info); @@ -69,7 +90,40 @@ public: fb_pitch = (int)info.pitch; hw_fb = (uint32_t*)montauk::fb_map(); + hw_direct = hw_fb; back_buf = (uint32_t*)montauk::alloc((uint64_t)fb_height * fb_pitch); + + // Hardware page flipping: fb_map() maps the second scanout buffer + // directly after the first when the kernel supports flipping. Acquire + // the scanout before choosing an off-screen buffer: another fullscreen + // process (notably login.elf) may have left either buffer live. + bool flip_supported = hw_fb && montauk::fb_flip(-1, 0) == 1; + uint64_t pages = ((uint64_t)fb_height * fb_pitch + 0xFFF) / 0x1000; + uint32_t* second = flip_supported + ? (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000) : nullptr; + if (flip_supported && prefer_hardware_page_flip()) { + int64_t front = montauk::fb_flip(-2, 0); + if (front == 0 || front == 1) { + hw_fb2 = second; + hw_next = (int)front ^ 1; + } else { + // If the live register cannot be trusted even at startup, + // avoid flipping and update both possible scanout surfaces. + hw_mirror = second; + } + } else if (flip_supported) { + // A direct-copy client must make buffer 0 live before it starts + // drawing there; the previous fullscreen client may have left + // buffer 1 scanning out. + if (montauk::fb_flip(0, 1) != 0) { + int64_t front = montauk::fb_flip(-2, 0); + if (front == 1) { + hw_direct = second; + } else if (front != 0) { + hw_mirror = second; + } + } + } } int width() const { return fb_width; } @@ -250,12 +304,63 @@ public: inline void flip() { if (!hw_fb || !back_buf) return; + // With hardware page flipping, copy into the scanout buffer that is + // currently OFF screen, then ask the display engine to present it at + // the next vblank (tear-free). Without it, fall back to copying + // straight into the visible framebuffer. + uint32_t* dst_fb = hw_direct; + if (hw_fb2) dst_fb = (hw_next == 1) ? hw_fb2 : hw_fb; + // Copy back buffer to hardware framebuffer, row by row (pitch may differ) uint64_t row_bytes = (uint64_t)fb_width * sizeof(uint32_t); for (int y = 0; y < fb_height; y++) { uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch); - uint32_t* dst = (uint32_t*)((uint8_t*)hw_fb + y * fb_pitch); + uint32_t* dst = (uint32_t*)((uint8_t*)dst_fb + y * fb_pitch); montauk::memcpy(dst, src, row_bytes); + if (hw_mirror) { + uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch); + montauk::memcpy(mirror, src, row_bytes); + } + } + if (hw_mirror) asm volatile("sfence" ::: "memory"); + + if (hw_fb2) { + // Wait for the latch (vsync): the buffer we just left becomes + // safe to draw into only once the new one is actually scanning. + asm volatile("sfence" ::: "memory"); + int64_t presented = montauk::fb_flip(hw_next, 1); + if (presented == hw_next) { + hw_next ^= 1; + } else { + // Do not retry a faulty page-flip path forever: that turns a + // transient latch failure into an apparently frozen desktop. + // Keep drawing directly into whichever buffer hardware says + // is live for the rest of this process. If SURFLIVE itself is + // unavailable, request buffer 0 without another blocking wait. + int64_t front = montauk::fb_flip(-2, 0); + if (front == 0 || front == 1) { + hw_direct = (front == 1) ? hw_fb2 : hw_fb; + } else { + (void)montauk::fb_flip(0, 0); + hw_direct = hw_fb; + hw_mirror = hw_fb2; + } + hw_fb2 = nullptr; + + // The frame above was copied to the requested (possibly + // off-screen) buffer. Publish it once more to the live direct + // buffer so the cursor and desktop resume immediately. + for (int y = 0; y < fb_height; y++) { + uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch); + uint32_t* dst = (uint32_t*)((uint8_t*)hw_direct + y * fb_pitch); + montauk::memcpy(dst, src, row_bytes); + if (hw_mirror) { + uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch); + montauk::memcpy(mirror, src, row_bytes); + } + } + asm volatile("sfence" ::: "memory"); + } } } }; diff --git a/template/sysroot/include/montauk/syscall.h b/template/sysroot/include/montauk/syscall.h index 8dbe7ed..2663f53 100644 --- a/template/sysroot/include/montauk/syscall.h +++ b/template/sysroot/include/montauk/syscall.h @@ -326,6 +326,11 @@ namespace montauk { // Framebuffer inline void fb_info(montauk::abi::FbInfo* info) { syscall1(montauk::abi::SYS_FBINFO, (uint64_t)info); } inline void* fb_map() { return (void*)syscall0(montauk::abi::SYS_FBMAP); } + // Page flip between two scanout buffers. index -1 queries support; + // index -2 acquires ownership and returns the live front buffer. + inline int64_t fb_flip(int64_t index, uint64_t flags) { + return syscall2(montauk::abi::SYS_FBFLIP, (uint64_t)index, flags); + } inline int display_info(montauk::abi::DisplayInfo* out) { return (int)syscall1(montauk::abi::SYS_DISPLAYINFO, (uint64_t)out); }