daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kernel concurrency, interrupt context, and user VA safety

Browse Source
This commit is contained in:
daniel
2026-08-01 15:43:38 +02:00
parent f222bf1f04
commit b1c55073c7
71 changed files with 2654 additions and 1092 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/src/ACPI/AcpiEvents.cpp
+95 -11
View File
@@ -24,6 +24,10 @@ namespace Hal {
static uint32_t g_pm1aEventBlock = 0;
static uint32_t g_pm1bEventBlock = 0;
static uint8_t g_pm1EventLength = 0;
static uint32_t g_gpe0Block = 0;
static uint32_t g_gpe1Block = 0;
static uint8_t g_gpe0Length = 0;
static uint8_t g_gpe1Length = 0;
static uint16_t g_sciIrq = 0;
static bool g_initialized = false;
@@ -39,6 +43,18 @@ namespace Hal {
return sts;
}
static uint16_t ReadPM1Enable() {
uint16_t enableOffset = g_pm1EventLength / 2;
uint16_t enable = 0;
if (enableOffset == 0) return 0;
if (g_pm1aEventBlock != 0)
enable |= Io::In16((uint16_t)(g_pm1aEventBlock + enableOffset));
if (g_pm1bEventBlock != 0)
enable |= Io::In16((uint16_t)(g_pm1bEventBlock + enableOffset));
return enable;
}
static void ClearPM1StatusBits(uint16_t bits) {
// Write-1-to-clear semantics
if (g_pm1aEventBlock != 0)
@@ -59,23 +75,67 @@ namespace Hal {
}
}
// The kernel does not currently evaluate GPE control methods. Leaving
// firmware-enabled GPEs armed is therefore unsafe: an unacknowledged
// level-triggered SCI can continuously retrigger and starve the BSP.
static bool IsValidGpeIoBlock(uint32_t base, uint8_t length) {
return base != 0 && length >= 2 && (length & 1) == 0 &&
base <= 0xFFFF && length <= 0x10000U - base;
}
static void DisableAndClearGpeBlock(uint32_t base, uint8_t length) {
if (!IsValidGpeIoBlock(base, length)) return;
uint16_t statusBytes = length / 2;
for (uint16_t i = 0; i < statusBytes; i++)
Io::Out8(0, (uint16_t)(base + statusBytes + i));
for (uint16_t i = 0; i < statusBytes; i++)
Io::Out8(0xFF, (uint16_t)(base + i));
}
static void QuiesceActiveGpeBlock(uint32_t base, uint8_t length) {
if (!IsValidGpeIoBlock(base, length)) return;
uint16_t statusBytes = length / 2;
for (uint16_t i = 0; i < statusBytes; i++) {
uint16_t statusPort = (uint16_t)(base + i);
uint16_t enablePort = (uint16_t)(base + statusBytes + i);
uint8_t enabled = Io::In8(enablePort);
uint8_t active = Io::In8(statusPort) & enabled;
if (active == 0) continue;
// Disable the unsupported source before acknowledging it so a
// level source cannot immediately assert the SCI again.
Io::Out8(enabled & (uint8_t)~active, enablePort);
Io::Out8(active, statusPort);
}
}
static void DisableAndClearGpes() {
DisableAndClearGpeBlock(g_gpe0Block, g_gpe0Length);
DisableAndClearGpeBlock(g_gpe1Block, g_gpe1Length);
}
static void QuiesceActiveGpes() {
QuiesceActiveGpeBlock(g_gpe0Block, g_gpe0Length);
QuiesceActiveGpeBlock(g_gpe1Block, g_gpe1Length);
}
// ============================================================================
// SCI Interrupt Handler
// ============================================================================
static void SciHandler(uint8_t irq) {
static void SciHandler(uint8_t /*irq*/, bool) {
uint16_t sts = ReadPM1Status();
uint16_t activeFixedEvents = sts & ReadPM1Enable();
if (activeFixedEvents != 0)
ClearPM1StatusBits(activeFixedEvents);
if (sts & AcpiSleep::PM1_PWRBTN_STS) {
ClearPM1StatusBits(AcpiSleep::PM1_PWRBTN_STS);
KernelLogStream(INFO, "ACPI") << "Power button pressed";
}
// Firmware may re-enable a GPE after initialization. Since no GPE
// AML methods are dispatched yet, disable and acknowledge every
// active source rather than allowing an SCI interrupt storm.
QuiesceActiveGpes();
if (sts & AcpiSleep::PM1_SLPBTN_STS) {
ClearPM1StatusBits(AcpiSleep::PM1_SLPBTN_STS);
KernelLogStream(INFO, "ACPI") << "Sleep button pressed";
}
// Note: EOI is sent by HalIrqDispatch after this handler returns.
// Note: HalIrqDispatch sends the LAPIC EOI before invoking handlers.
}
// ============================================================================
@@ -93,6 +153,10 @@ namespace Hal {
g_pm1aEventBlock = fadt.PM1aEventBlock;
g_pm1bEventBlock = fadt.PM1bEventBlock;
g_pm1EventLength = fadt.PM1EventLength;
g_gpe0Block = fadt.GPE0Block;
g_gpe1Block = fadt.GPE1Block;
g_gpe0Length = fadt.GPE0Length;
g_gpe1Length = fadt.GPE1Length;
g_sciIrq = fadt.SCI_Interrupt;
if (g_pm1aEventBlock == 0) {
@@ -100,6 +164,19 @@ namespace Hal {
return;
}
if (g_gpe0Length != 0 && !IsValidGpeIoBlock(g_gpe0Block, g_gpe0Length)) {
KernelLogStream(WARNING, "ACPI")
<< "GPE0 block is not usable SystemIO (base=" << base::hex
<< (uint64_t)g_gpe0Block << ", length=" << base::dec
<< (uint64_t)g_gpe0Length << ")";
}
if (g_gpe1Length != 0 && !IsValidGpeIoBlock(g_gpe1Block, g_gpe1Length)) {
KernelLogStream(WARNING, "ACPI")
<< "GPE1 block is not usable SystemIO (base=" << base::hex
<< (uint64_t)g_gpe1Block << ", length=" << base::dec
<< (uint64_t)g_gpe1Length << ")";
}
// Clear any pending status bits
ClearPM1StatusBits(
AcpiSleep::PM1_PWRBTN_STS | AcpiSleep::PM1_SLPBTN_STS |
@@ -110,6 +187,10 @@ namespace Hal {
// Enable power button event
SetPM1Events(AcpiSleep::PM1_PWRBTN_EN);
// Disable unsupported firmware GPE sources before the level SCI is
// routed. Clear stale status only after the enables are down.
DisableAndClearGpes();
// Route SCI to an IRQ vector. The SCI is level-triggered,
// active-low (ACPI spec requirement). The MADT may have an
// Interrupt Source Override for this IRQ that already sets
@@ -145,6 +226,9 @@ namespace Hal {
// Drop the suspend-time RTC wake enable and restore only the
// fixed events we actually want during normal runtime.
SetPM1Events(AcpiSleep::PM1_PWRBTN_EN);
// Firmware and _WAK may have restored GPE enables while asleep.
DisableAndClearGpes();
}
};
kernel/src/ACPI/FADT.cpp
+23
View File
@@ -21,6 +21,21 @@ namespace Hal {
return true;
}
static uint32_t ExtendedSystemIoAddress(const Table* fadt,
const uint8_t gas[12],
size_t fieldOffset) {
// Generic Address Structure: byte 0 is AddressSpaceId and bytes
// 4..11 are the little-endian address. This event code uses x86
// port I/O, so accept only SystemIO GAS entries that fit the
// architectural 16-bit port space.
if (fadt->Header.Length < fieldOffset + 12 || gas[0] != 1) return 0;
uint64_t address = 0;
for (int i = 0; i < 8; i++)
address |= (uint64_t)gas[4 + i] << (i * 8);
return address <= 0xFFFF ? (uint32_t)address : 0;
}
static Table* FindFADTInXSDT(ACPI::CommonSDTHeader* xsdt) {
uint32_t entryCount = (xsdt->Length - sizeof(ACPI::CommonSDTHeader)) / 8;
uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(ACPI::CommonSDTHeader));
@@ -73,6 +88,14 @@ namespace Hal {
result.PMTimerBlock = fadt->PMTimerBlock;
result.GPE0Block = fadt->GPE0Block;
result.GPE1Block = fadt->GPE1Block;
if (result.GPE0Block == 0) {
result.GPE0Block = ExtendedSystemIoAddress(
fadt, fadt->X_GPE0Block, offsetof(Table, X_GPE0Block));
}
if (result.GPE1Block == 0) {
result.GPE1Block = ExtendedSystemIoAddress(
fadt, fadt->X_GPE1Block, offsetof(Table, X_GPE1Block));
}
result.SMI_CommandPort = fadt->SMI_CommandPort;
result.SCI_Interrupt = fadt->SCI_Interrupt;
result.PM1EventLength = fadt->PM1EventLength;
kernel/src/Api/BluetoothSyscall.hpp
+6
View File
@@ -9,6 +9,9 @@
#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
#include <Drivers/USB/Bluetooth/Hci.hpp>
#include <Libraries/Memory.hpp>
#include <Sched/Scheduler.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include "Syscall.hpp"
@@ -53,6 +56,9 @@ namespace montauk::abi {
if (!bdAddr) return -1;
if (!Drivers::USB::Bluetooth::IsInitialized()) return -1;
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid()
<< " entered Bluetooth disconnect stage";
return (int64_t)Drivers::USB::Bluetooth::Disconnect(bdAddr);
}
kernel/src/Api/BuildNo.hpp
+1 -1
View File
@@ -12,4 +12,4 @@
#pragma once
#define MONTAUK_BUILD_NUMBER 33
#define MONTAUK_BUILD_NUMBER 37
kernel/src/Api/Common.hpp
+18 -29
View File
@@ -14,59 +14,47 @@ namespace montauk::abi {
}
static bool ResolveProcessHandle(Sched::Process* proc, int handle, Ipc::HandleType expectedType,
Ipc::Object*& outObject, uint32_t* outRights = nullptr) {
Ipc::HandleSnapshot& snapshot) {
if (proc == nullptr || handle < 0) return false;
int slot = Ipc::SlotForPid(proc->pid);
if (slot < 0) return false;
Ipc::HandleType type = Ipc::HandleType::None;
Ipc::Object* object = nullptr;
uint32_t rights = 0;
if (!Ipc::SnapshotHandleForSlot(slot, handle, type, object, rights)) return false;
if (type != expectedType || object == nullptr) return false;
outObject = object;
if (outRights != nullptr) *outRights = rights;
return true;
if (!snapshot.Capture(slot, handle)) return false;
return snapshot.type == expectedType && snapshot.object != nullptr;
}
static Ipc::Stream* GetRedirOutStream(Sched::Process* proc) {
static Ipc::Stream* GetRedirOutStream(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) {
proc = GetRedirTarget(proc);
if (proc == nullptr) return nullptr;
Ipc::Object* object = nullptr;
if (!ResolveProcessHandle(proc, proc->ioOutHandle, Ipc::HandleType::Stream, object)) return nullptr;
return (Ipc::Stream*)object;
if (!ResolveProcessHandle(proc, proc->ioOutHandle, Ipc::HandleType::Stream, snapshot)) return nullptr;
return (Ipc::Stream*)snapshot.object;
}
static Ipc::Stream* GetRedirInStream(Sched::Process* proc) {
static Ipc::Stream* GetRedirInStream(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) {
proc = GetRedirTarget(proc);
if (proc == nullptr) return nullptr;
Ipc::Object* object = nullptr;
if (!ResolveProcessHandle(proc, proc->ioInHandle, Ipc::HandleType::Stream, object)) return nullptr;
return (Ipc::Stream*)object;
if (!ResolveProcessHandle(proc, proc->ioInHandle, Ipc::HandleType::Stream, snapshot)) return nullptr;
return (Ipc::Stream*)snapshot.object;
}
static Ipc::Mailbox* GetRedirKeyMailbox(Sched::Process* proc) {
static Ipc::Mailbox* GetRedirKeyMailbox(Sched::Process* proc, Ipc::HandleSnapshot& snapshot) {
proc = GetRedirTarget(proc);
if (proc == nullptr) return nullptr;
Ipc::Object* object = nullptr;
if (!ResolveProcessHandle(proc, proc->ioKeyHandle, Ipc::HandleType::Mailbox, object)) return nullptr;
return (Ipc::Mailbox*)object;
if (!ResolveProcessHandle(proc, proc->ioKeyHandle, Ipc::HandleType::Mailbox, snapshot)) return nullptr;
return (Ipc::Mailbox*)snapshot.object;
}
static int DuplicateHandleBetweenSlots(int srcSlot, int handle, int dstSlot) {
if (srcSlot < 0 || dstSlot < 0 || handle < 0) return -1;
Ipc::HandleType type = Ipc::HandleType::None;
Ipc::Object* object = nullptr;
uint32_t rights = 0;
if (!Ipc::SnapshotHandleForSlot(srcSlot, handle, type, object, rights)) return -1;
if ((rights & Ipc::RightDup) == 0) return -1;
return Ipc::InstallHandleForSlot(dstSlot, object, type, rights);
Ipc::HandleSnapshot snapshot;
if (!snapshot.Capture(srcSlot, handle)) return -1;
if ((snapshot.rights & Ipc::RightDup) == 0) return -1;
return Ipc::InstallHandleForSlot(dstSlot, snapshot.object, snapshot.type, snapshot.rights);
}
static bool ConfigureRedirWaitsetForSlot(int slot, Sched::Process* proc) {
@@ -100,12 +88,13 @@ namespace montauk::abi {
int total = 0;
while (total < len) {
uint64_t observedWake = Sched::ObserveObjectWake(stream);
int written = Ipc::StreamWrite(stream, data + total, len - total, false);
if (written < 0) {
return (total > 0) ? total : -1;
}
if (written == 0) {
Sched::BlockOnObject(stream, 0);
Sched::BlockOnObjectSince(stream, 0, observedWake);
continue;
}
total += written;
kernel/src/Api/Graphics.hpp
+40 -8
View File
@@ -36,8 +36,17 @@ namespace montauk::abi {
if (fbBase == nullptr) return 0;
uint64_t fbPhys = Memory::SubHHDM((uint64_t)fbBase);
uint64_t fbSize = Graphics::Framebuffer::GetHeight()
* Graphics::Framebuffer::GetPitch();
uint64_t width = Graphics::Framebuffer::GetWidth();
uint64_t height = Graphics::Framebuffer::GetHeight();
uint64_t pitch = Graphics::Framebuffer::GetPitch();
if (width == 0 || height == 0 || pitch == 0
|| width > (~0ULL / 4) || pitch < width * 4
|| height > (~0ULL / pitch)) {
Kt::KernelLogStream(Kt::ERROR, "FbMap")
<< "Invalid or overflowing framebuffer geometry";
return 0;
}
uint64_t fbSize = height * pitch;
uint64_t numPages = (fbSize + 0xFFF) / 0x1000;
Kt::KernelLogStream(Kt::INFO, "FbMap") << "fbPhys=" << kcp::hex << fbPhys
@@ -47,8 +56,28 @@ namespace montauk::abi {
<< "x" << Graphics::Framebuffer::GetHeight()
<< " pitch=" << Graphics::Framebuffer::GetPitch() << ")";
// Map at a fixed user VA
constexpr uint64_t userVa = 0x50000000ULL;
// Map in the dedicated framebuffer arena, away from the heap,
// shared libraries, reusable IPC surfaces, and the user stack.
constexpr uint64_t userVa = Sched::UserFramebufferBase;
uint64_t mappedBytes = numPages * 0x1000ULL;
bool flipSupported = Drivers::Graphics::IntelGPU::FlipSupported();
uint64_t buffers = flipSupported ? 2 : 1;
if (mappedBytes > (Sched::UserFramebufferLimit - userVa) / buffers) {
Kt::KernelLogStream(Kt::ERROR, "FbMap")
<< "Framebuffer mappings exceed reserved user VA range";
return 0;
}
// Never expose more of buffer 1 than the GPU driver allocated. This
// independently contains a future geometry-contract regression before
// any partial framebuffer mapping is installed in the process.
if (flipSupported
&& (Drivers::Graphics::IntelGPU::GetBufferPhys(1) == 0
|| Drivers::Graphics::IntelGPU::GetBufferPageCount(1) < numPages)) {
Kt::KernelLogStream(Kt::ERROR, "FbMap")
<< "Page-flip buffer is smaller than firmware framebuffer";
return 0;
}
for (uint64_t i = 0; i < numPages; i++) {
if (!Memory::VMM::Paging::MapUserInWC(
@@ -63,7 +92,7 @@ namespace montauk::abi {
// When page flipping is available, map the second scanout buffer
// immediately after the first so a double-buffered client can reach
// both. Single-buffer clients never touch the extra pages.
if (Drivers::Graphics::IntelGPU::FlipSupported()) {
if (flipSupported) {
uint64_t buf1Phys = Drivers::Graphics::IntelGPU::GetBufferPhys(1);
for (uint64_t i = 0; i < numPages; i++) {
if (!Memory::VMM::Paging::MapUserInWC(
@@ -82,6 +111,7 @@ namespace montauk::abi {
// SYS_FBFLIP: point scanout at buffer `index` (0 or 1). The hardware
// latches the new surface at vblank, so the flip is tear-free.
// index == -1 queries support without side effects.
// index == -2 acquires flip ownership and returns the live front buffer.
static int64_t Sys_FbFlip(uint64_t index, uint64_t flags) {
namespace GPU = Drivers::Graphics::IntelGPU;
@@ -91,9 +121,11 @@ namespace montauk::abi {
if (!GPU::FlipSupported()) return -1;
auto* proc = Sched::GetCurrentProcessPtr();
if (proc != nullptr) GPU::SetFlipOwner(proc->pid);
return GPU::Flip((int)index, (flags & 1) != 0);
if ((int64_t)index == -2) {
return proc != nullptr ? GPU::AcquireFlip(proc->pid) : -1;
}
if (proc == nullptr) return -1;
return GPU::Flip((int)index, (flags & 1) != 0, proc->pid);
}
static int64_t Sys_DisplayInfo(DisplayInfo* out) {
kernel/src/Api/Heap.hpp
+32 -35
View File
@@ -9,6 +9,8 @@
#include <Memory/Paging.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <CppLib/Spinlock.hpp>
#include <Ipc/Ipc.hpp>
namespace montauk::abi {
@@ -22,6 +24,7 @@ namespace montauk::abi {
inline HeapAlloc g_heapAllocs[Sched::MaxProcesses][MaxHeapAllocs] = {};
inline int g_heapAllocCount[Sched::MaxProcesses] = {};
inline kcp::Mutex g_heapLocks[Sched::MaxProcesses];
// Get the process table slot index for the current process
inline int GetCurrentSlot() {
@@ -38,20 +41,15 @@ namespace montauk::abi {
int slot = GetCurrentSlot();
if (slot < 0) return 0;
// Guard against overflow before rounding
static constexpr uint64_t USER_SPACE_END = 0x0000800000000000ULL;
// Guard against overflow before rounding.
if (size > 0xFFFFFFFFFFFF0000ULL) return 0;
// Round up to page boundary
size = (size + 0xFFF) & ~0xFFFULL;
if (size == 0) size = 0x1000;
uint64_t userVa = proc->heapNext;
// Ensure allocation stays within user address space
if (userVa + size < userVa || userVa + size > USER_SPACE_END) return 0;
uint64_t numPages = size / 0x1000;
g_heapLocks[slot].Acquire();
if (g_heapAllocCount[slot] >= MaxHeapAllocs) {
// Out of allocation records, not out of memory. Log it: a
// silent 0 here surfaced as bogus downstream errors (BFD
@@ -60,6 +58,16 @@ namespace montauk::abi {
<< "pid " << proc->pid << " (" << proc->name
<< ") hit MaxHeapAllocs (" << (uint64_t)MaxHeapAllocs
<< "), SYS_ALLOC refused";
g_heapLocks[slot].Release();
return 0;
}
uint64_t userVa = 0;
if (!Sched::ReserveUserHeapRange(slot, size, userVa)) {
Kt::KernelLogStream(Kt::ERROR, "Heap")
<< "pid " << proc->pid << " (" << proc->name
<< ") exhausted bounded user heap";
g_heapLocks[slot].Release();
return 0;
}
@@ -68,38 +76,25 @@ namespace montauk::abi {
for (uint64_t i = 0; i < numPages; i++) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
for (uint64_t j = 0; j < mappedPages; j++) {
uint64_t pageVa = userVa + j * 0x1000;
uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa);
if (physAddr != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(physAddr));
}
Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa);
}
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
g_heapLocks[slot].Release();
return 0;
}
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, userVa + i * 0x1000)) {
Memory::g_pfa->Free(page);
for (uint64_t j = 0; j < mappedPages; j++) {
uint64_t pageVa = userVa + j * 0x1000;
uint64_t mappedPhys = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa);
if (mappedPhys != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(mappedPhys));
}
Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa);
}
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
g_heapLocks[slot].Release();
return 0;
}
mappedPages++;
}
proc->heapNext += size;
// Track the allocation so Sys_Free can release it
Sched::g_allocatedPages[slot] += numPages;
g_heapAllocs[slot][g_heapAllocCount[slot]++] = { userVa, numPages };
g_heapLocks[slot].Release();
return userVa;
}
@@ -107,8 +102,10 @@ namespace montauk::abi {
// The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup.
inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) {
if (slot < 0 || slot >= Sched::MaxProcesses) return;
g_heapLocks[slot].Acquire();
g_heapAllocCount[slot] = 0;
Sched::g_allocatedPages[slot] = 0;
g_heapLocks[slot].Release();
}
inline void Sys_Free(uint64_t addr) {
@@ -118,6 +115,8 @@ namespace montauk::abi {
int slot = GetCurrentSlot();
if (slot < 0) return;
g_heapLocks[slot].Acquire();
// Find the allocation record matching this address
int idx = -1;
for (int i = 0; i < g_heapAllocCount[slot]; i++) {
@@ -126,25 +125,23 @@ namespace montauk::abi {
break;
}
}
if (idx < 0) return; // Unknown address — ignore
if (idx < 0) {
g_heapLocks[slot].Release();
return; // Unknown address — ignore
}
uint64_t va = g_heapAllocs[slot][idx].va;
uint64_t numPages = g_heapAllocs[slot][idx].numPages;
// Free physical pages in bulk and unmap virtual addresses
for (uint64_t i = 0; i < numPages; i++) {
uint64_t pageVa = va + i * 0x1000;
uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa);
if (physAddr != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(physAddr));
}
Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, pageVa);
}
// Unmap and invalidate sibling CPUs before recycling the frames. A
// stale user TLB entry can otherwise corrupt the frame's next owner.
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, va, numPages);
Sched::g_allocatedPages[slot] -= numPages;
// Remove tracking entry by swapping with the last element
g_heapAllocs[slot][idx] = g_heapAllocs[slot][g_heapAllocCount[slot] - 1];
g_heapAllocCount[slot]--;
g_heapLocks[slot].Release();
}
};
kernel/src/Api/IoRedir.hpp
+8 -4
View File
@@ -85,14 +85,16 @@ namespace montauk::abi {
static int Sys_ChildIoRead(int childPid, char* buf, int maxLen) {
auto* child = Sched::GetProcessByPid(childPid);
Ipc::Stream* stream = GetRedirOutStream(child);
Ipc::HandleSnapshot snapshot;
Ipc::Stream* stream = GetRedirOutStream(child, snapshot);
if (child == nullptr || !child->redirected || stream == nullptr) return -1;
return Ipc::StreamRead(stream, (uint8_t*)buf, maxLen, true);
}
static int Sys_ChildIoWrite(int childPid, const char* data, int len) {
auto* child = Sched::GetProcessByPid(childPid);
Ipc::Stream* stream = GetRedirInStream(child);
Ipc::HandleSnapshot snapshot;
Ipc::Stream* stream = GetRedirInStream(child, snapshot);
if (child == nullptr || !child->redirected || stream == nullptr) return -1;
return WriteAllToStream(stream, (const uint8_t*)data, len);
}
@@ -100,14 +102,16 @@ namespace montauk::abi {
static int Sys_ChildIoWriteKey(int childPid, const KeyEvent* key) {
if (key == nullptr) return -1;
auto* child = Sched::GetProcessByPid(childPid);
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(child);
Ipc::HandleSnapshot snapshot;
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(child, snapshot);
if (child == nullptr || !child->redirected || mailbox == nullptr) return -1;
for (;;) {
uint64_t observedWake = Sched::ObserveObjectWake(mailbox);
int rc = Ipc::MailboxSend(mailbox, 0, key, sizeof(KeyEvent));
if (rc < 0) return -1;
if (rc == 0) {
Sched::BlockOnObject(mailbox, 0);
Sched::BlockOnObjectSince(mailbox, 0, observedWake);
continue;
}
return 0;
kernel/src/Api/Keyboard.hpp
+14 -7
View File
@@ -15,7 +15,8 @@ namespace montauk::abi {
static bool Sys_IsKeyAvailable() {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc && proc->redirected) {
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc);
Ipc::HandleSnapshot snapshot;
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, snapshot);
if (mailbox != nullptr) return Ipc::MailboxHasMessage(mailbox);
}
return Drivers::PS2::Keyboard::IsKeyAvailable();
@@ -25,9 +26,11 @@ namespace montauk::abi {
if (outEvent == nullptr) return;
auto* proc = Sched::GetCurrentProcessPtr();
if (proc && proc->redirected) {
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc);
Ipc::HandleSnapshot snapshot;
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, snapshot);
if (mailbox != nullptr) {
for (;;) {
uint64_t observedWake = Sched::ObserveObjectWake(mailbox);
uint16_t len = sizeof(KeyEvent);
int rc = Ipc::MailboxRecv(mailbox, nullptr, outEvent, &len, true);
if (rc > 0) return;
@@ -35,7 +38,7 @@ namespace montauk::abi {
memset(outEvent, 0, sizeof(KeyEvent));
return;
}
Sched::BlockOnObject(mailbox, 0);
Sched::BlockOnObjectSince(mailbox, 0, observedWake);
}
}
}
@@ -51,10 +54,14 @@ namespace montauk::abi {
static char Sys_GetChar() {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc && proc->redirected) {
Ipc::Stream* input = GetRedirInStream(proc);
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc);
Ipc::HandleSnapshot inputSnapshot;
Ipc::HandleSnapshot mailboxSnapshot;
Ipc::Stream* input = GetRedirInStream(proc, inputSnapshot);
Ipc::Mailbox* mailbox = GetRedirKeyMailbox(proc, mailboxSnapshot);
if (input != nullptr || mailbox != nullptr) {
for (;;) {
uint64_t inputWake = input ? Sched::ObserveObjectWake(input) : 0;
uint64_t mailboxWake = mailbox ? Sched::ObserveObjectWake(mailbox) : 0;
if (input != nullptr) {
uint8_t c = 0;
int rc = Ipc::StreamRead(input, &c, 1, true);
@@ -79,9 +86,9 @@ namespace montauk::abi {
return 0;
}
} else if (input != nullptr) {
Sched::BlockOnObject(input, 0);
Sched::BlockOnObjectSince(input, 0, inputWake);
} else if (mailbox != nullptr) {
Sched::BlockOnObject(mailbox, 0);
Sched::BlockOnObjectSince(mailbox, 0, mailboxWake);
} else {
return 0;
}
kernel/src/Api/LibSyscall.hpp
+39 -19
View File
@@ -13,6 +13,8 @@
#include <Terminal/Terminal.hpp>
#include <Libraries/Memory.hpp>
#include <Libraries/String.hpp>
#include <CppLib/Spinlock.hpp>
#include <Ipc/Ipc.hpp>
namespace montauk::abi {
@@ -34,16 +36,19 @@ namespace montauk::abi {
// Per-process library table
inline LibEntry g_libTable[MaxProcesses][MaxLibsPerProcess] = {};
inline kcp::Mutex g_libLocks[MaxProcesses];
// Initialize library table for a process slot
inline void InitLibTable(int slot) {
if (slot < 0 || slot >= MaxProcesses) return;
g_libLocks[slot].Acquire();
for (int i = 0; i < MaxLibsPerProcess; i++) {
g_libTable[slot][i].inUse = false;
g_libTable[slot][i].refcount = 0;
g_libTable[slot][i].loadBias = 0;
g_libTable[slot][i].path[0] = '\0';
}
g_libLocks[slot].Release();
}
// Load a shared library into the current process's address space.
@@ -61,6 +66,8 @@ namespace montauk::abi {
size_t pathLen = Lib::strlen(path);
if (pathLen == 0 || pathLen >= sizeof(LibEntry::path)) return 0;
g_libLocks[slot].Acquire();
// Find a free slot or reuse an existing slot with the same path
int libSlot = -1;
for (int i = 0; i < MaxLibsPerProcess; i++) {
@@ -68,6 +75,7 @@ namespace montauk::abi {
if (Lib::strncmp(g_libTable[slot][i].path, path, sizeof(LibEntry::path)) == 0) {
// Same library already loaded - just increment refcount
g_libTable[slot][i].refcount++;
g_libLocks[slot].Release();
return (uint64_t)(i + 1); // Handle = slot + 1 (0 = invalid)
}
} else if (libSlot < 0) {
@@ -77,6 +85,7 @@ namespace montauk::abi {
if (libSlot < 0) {
Kt::KernelLogStream(Kt::ERROR, "Lib") << "No free library slots";
g_libLocks[slot].Release();
return 0;
}
@@ -84,6 +93,7 @@ namespace montauk::abi {
uint64_t loadBias = Sched::ElfLoadLib(path, proc->pml4Phys, libSlot);
if (loadBias == 0) {
Kt::KernelLogStream(Kt::ERROR, "Lib") << "Failed to load library: " << path;
g_libLocks[slot].Release();
return 0;
}
@@ -93,6 +103,8 @@ namespace montauk::abi {
g_libTable[slot][libSlot].loadBias = loadBias;
Lib::strncpy(g_libTable[slot][libSlot].path, path, sizeof(LibEntry::path));
g_libLocks[slot].Release();
Kt::KernelLogStream(Kt::OK, "Lib") << "Loaded library: " << path << " with load bias " << kcp::hex << loadBias << kcp::dec;
return (uint64_t)(libSlot + 1); // Handle = slot + 1 (0 = invalid)
@@ -107,7 +119,11 @@ namespace montauk::abi {
int libSlot = (int)handle - 1;
if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return -1;
if (!g_libTable[slot][libSlot].inUse) return -1;
g_libLocks[slot].Acquire();
if (!g_libTable[slot][libSlot].inUse) {
g_libLocks[slot].Release();
return -1;
}
g_libTable[slot][libSlot].refcount--;
if (g_libTable[slot][libSlot].refcount == 0) {
@@ -117,14 +133,8 @@ namespace montauk::abi {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc != nullptr) {
// Unmap all pages in the library region
for (uint64_t va = libBase; va < libEnd; va += 0x1000) {
uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, va);
if (physAddr != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(physAddr));
Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, va);
}
}
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, libBase,
(libEnd - libBase) / 0x1000ULL);
}
g_libTable[slot][libSlot].inUse = false;
@@ -132,6 +142,7 @@ namespace montauk::abi {
g_libTable[slot][libSlot].path[0] = '\0';
}
g_libLocks[slot].Release();
return 0;
}
@@ -146,9 +157,14 @@ namespace montauk::abi {
int libSlot = (int)handle - 1;
if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return 0;
if (!g_libTable[slot][libSlot].inUse) return 0;
g_libLocks[slot].Acquire();
if (!g_libTable[slot][libSlot].inUse) {
g_libLocks[slot].Release();
return 0;
}
uint64_t loadBias = g_libTable[slot][libSlot].loadBias;
g_libLocks[slot].Release();
return loadBias + symbolOffset;
}
@@ -160,9 +176,15 @@ namespace montauk::abi {
int libSlot = (int)handle - 1;
if (libSlot < 0 || libSlot >= MaxLibsPerProcess) return 0;
if (!g_libTable[slot][libSlot].inUse) return 0;
g_libLocks[slot].Acquire();
if (!g_libTable[slot][libSlot].inUse) {
g_libLocks[slot].Release();
return 0;
}
return g_libTable[slot][libSlot].loadBias;
uint64_t loadBias = g_libTable[slot][libSlot].loadBias;
g_libLocks[slot].Release();
return loadBias;
}
// Cleanup library table for a process slot
@@ -172,25 +194,23 @@ namespace montauk::abi {
auto* proc = Sched::GetProcessSlot(slot);
if (proc == nullptr) return;
g_libLocks[slot].Acquire();
// Unmap all libraries for this process
for (int i = 0; i < MaxLibsPerProcess; i++) {
if (g_libTable[slot][i].inUse) {
uint64_t libBase = GetLibSlotBase(i);
uint64_t libEnd = libBase + Sched::LIB_MAX_SIZE;
for (uint64_t va = libBase; va < libEnd; va += 0x1000) {
uint64_t physAddr = Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, va);
if (physAddr != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(physAddr));
Memory::VMM::Paging::UnmapUserIn(proc->pml4Phys, va);
}
}
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, libBase,
(libEnd - libBase) / 0x1000ULL);
g_libTable[slot][i].inUse = false;
g_libTable[slot][i].refcount = 0;
g_libTable[slot][i].loadBias = 0;
}
}
g_libLocks[slot].Release();
}
}
kernel/src/Api/Power.hpp
+13
View File
@@ -14,6 +14,9 @@
#include <ACPI/AcpiSleep.hpp>
#include <ACPI/CpuIdle.hpp>
#include <Hal/CpuPower.hpp>
#include <Sched/Scheduler.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include "Syscall.hpp"
@@ -31,13 +34,21 @@ namespace montauk::abi {
if (action == POWER_REQ_QUERY) {
int pending = g_pendingPowerAction;
g_pendingPowerAction = POWER_REQ_QUERY;
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid()
<< " consumed graceful power request " << (uint64_t)pending;
return (int64_t)pending;
}
g_pendingPowerAction = action;
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid()
<< " posted graceful power request " << (uint64_t)action;
return 0;
}
static void Sys_Reset() {
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid() << " entered final reboot stage";
if (Efi::g_ResetSystem) {
/* Switch to kernel PML4 which has identity-mapped UEFI runtime regions */
Memory::VMM::LoadCR3(Memory::VMM::g_paging->PML4);
@@ -51,6 +62,8 @@ namespace montauk::abi {
}
static void Sys_Shutdown() {
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid() << " entered final shutdown stage";
/* Primary: ACPI S5 shutdown via PM1 control registers */
if (Hal::AcpiShutdown::IsAvailable()) {
Hal::AcpiShutdown::Shutdown();
kernel/src/Api/Storage.hpp
+6
View File
@@ -12,6 +12,9 @@
#include <Fs/Fat32.hpp>
#include <Fs/Ext2.hpp>
#include <Api/UserMemory.hpp>
#include <Sched/Scheduler.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include "Syscall.hpp"
@@ -112,6 +115,9 @@ namespace montauk::abi {
// Flush all block-device write caches and cleanly unmount disk-backed
// volumes ahead of power-off. Returns the number of volumes unmounted.
static int64_t Sys_FsSync() {
Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
<< (uint64_t)Sched::GetCurrentPid()
<< " entered filesystem sync stage";
return (int64_t)Fs::FsProbe::SyncAndUnmountAll();
}
kernel/src/Api/Terminal.hpp
+4 -2
View File
@@ -15,7 +15,8 @@ namespace montauk::abi {
static void Sys_Print(const char* text) {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc && proc->redirected) {
Ipc::Stream* stream = GetRedirOutStream(proc);
Ipc::HandleSnapshot snapshot;
Ipc::Stream* stream = GetRedirOutStream(proc, snapshot);
if (stream != nullptr) {
int len = 0;
while (text[len]) len++;
@@ -33,7 +34,8 @@ namespace montauk::abi {
static void Sys_Putchar(char c) {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc && proc->redirected) {
Ipc::Stream* stream = GetRedirOutStream(proc);
Ipc::HandleSnapshot snapshot;
Ipc::Stream* stream = GetRedirOutStream(proc, snapshot);
if (stream != nullptr) {
uint8_t byte = (uint8_t)c;
WriteAllToStream(stream, &byte, 1);
kernel/src/Common/Panic.cpp
+7
View File
@@ -70,6 +70,7 @@ void Panic(const char *meditationString, System::PanicFrame* frame) {
// showing the other buffer, the panic box would land on an invisible
// surface. Safe no-op when the GPU driver is inactive.
Drivers::Graphics::IntelGPU::PanicRestoreScanout();
Kt::EnablePanicOutput();
// Header
kerr << BOXUI_ANSI_RED_BG << BOXUI_ANSI_WHITE_FG << BOXUI_ANSI_BOLD << "\n";
@@ -105,5 +106,11 @@ void Panic(const char *meditationString, System::PanicFrame* frame) {
PrintHorizontalEdge(BOXUI_BL, BOXUI_BR);
kerr << BOXUI_ANSI_RESET;
#if defined (__x86_64__)
// The framebuffer is write-combining. Drain the final panic text before
// halting all forward progress so it cannot remain in a CPU write buffer.
asm volatile("sfence" ::: "memory");
#endif
Halt();
}
kernel/src/CppLib/Spinlock.cpp
+15 -4
View File
@@ -7,18 +7,29 @@
#include "Spinlock.hpp"
namespace kcp {
void Spinlock::Acquire() {
uint64_t Spinlock::AcquireIrqSave() {
uint64_t flags;
asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory");
while (atomic_flag.test_and_set(std::memory_order_acquire)) {
asm volatile("pause");
}
savedFlags = flags;
return flags;
}
void Spinlock::Release() {
uint64_t flags = savedFlags;
void Spinlock::ReleaseIrqRestore(uint64_t flags) {
atomic_flag.clear(std::memory_order_release);
asm volatile("push %0; popfq" :: "r"(flags) : "memory");
}
void Spinlock::ReleaseIrqNoRestore() {
atomic_flag.clear(std::memory_order_release);
}
void Spinlock::Acquire() {
savedFlags = AcquireIrqSave();
}
void Spinlock::Release() {
ReleaseIrqRestore(savedFlags);
}
};
kernel/src/CppLib/Spinlock.hpp
+8
View File
@@ -14,6 +14,14 @@ namespace kcp {
public:
void Acquire();
void Release();
// Explicit IRQ-save form for locks whose ownership is deliberately
// handed across a context switch. The saved flags live on the caller's
// stack (or in scheduler-owned per-context storage), not in the shared
// lock object, so the resumed context restores its own interrupt state.
uint64_t AcquireIrqSave();
void ReleaseIrqRestore(uint64_t flags);
void ReleaseIrqNoRestore();
};
// Non-interrupt-disabling mutex for subsystems that are only called
kernel/src/Drivers/Audio/IntelHda.cpp
+123 -26
View File
@@ -15,6 +15,7 @@
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Libraries/Memory.hpp>
#include <atomic>
namespace Drivers::Audio::IntelHda {
@@ -26,10 +27,33 @@ namespace Drivers::Audio::IntelHda {
static bool g_initialized = false;
static kcp::Spinlock g_codecLock;
// Serializes stream lifecycle, DMA write-pointer updates, and IRQ-side
// stream inspection. The IRQ releases it before calling the mixer to keep
// the lock order consistently Mixer -> HDA stream.
// Serializes stream lifecycle and DMA write-pointer updates. Keep logging
// out of this lock: Spinlock disables local interrupts, while the terminal
// logger may wait for a lock owner that needs the local timer to run.
static kcp::Spinlock g_streamLock;
// The hard IRQ only acknowledges and masks stream completion sources.
// Mixing/refilling is substantially too expensive for interrupt context.
static std::atomic<bool> g_streamWorkPending{false};
static uint32_t g_streamInterruptMask = 0;
// Runtime diagnostics are emitted by ProcessDeferredWork() in idle
// context. Some HDA entry points run below the mixer's interrupt-disabling
// spinlock, so even logging after releasing g_streamLock is not safe.
enum DeferredDiagnostic : uint32_t {
DIAG_STREAM_OPENED = 1u << 0,
DIAG_STREAM_CLOSED = 1u << 1,
DIAG_HEADPHONES = 1u << 2,
DIAG_SPEAKERS = 1u << 3,
};
static std::atomic<uint32_t> g_diagnosticsPending{0};
static std::atomic<uint32_t> g_openSampleRate{0};
static std::atomic<uint32_t> g_openBitsPerSample{0};
static std::atomic<uint32_t> g_openChannels{0};
static std::atomic<uint32_t> g_codecTimeoutsPending{0};
static std::atomic<uint32_t> g_codecTimeoutReports{0};
static std::atomic<uint32_t> g_lastTimeoutCodec{0};
static std::atomic<uint32_t> g_lastTimeoutNid{0};
static std::atomic<uint32_t> g_lastTimeoutVerb{0};
static volatile uint8_t* g_mmioBase = nullptr;
static uint8_t g_bus, g_dev, g_func;
@@ -213,9 +237,10 @@ namespace Drivers::Audio::IntelHda {
uint32_t response = 0;
if (!ReadResponse(&response, nullptr)) {
g_codecLock.Release();
KernelLogStream(WARNING, "HDA") << "Verb timeout: codec=" << base::dec
<< (uint64_t)codec << " nid=" << (uint64_t)nid
<< " verb=" << base::hex << (uint64_t)verb;
g_lastTimeoutCodec.store(codec, std::memory_order_relaxed);
g_lastTimeoutNid.store(nid, std::memory_order_relaxed);
g_lastTimeoutVerb.store(verb, std::memory_order_relaxed);
g_codecTimeoutsPending.fetch_add(1, std::memory_order_release);
return 0;
}
g_codecLock.Release();
@@ -640,13 +665,15 @@ namespace Drivers::Audio::IntelHda {
DisablePin(g_speakerNid);
EnablePin(g_hpNid);
g_pinNid = g_hpNid;
KernelLogStream(INFO, "HDA") << "Switched to headphone output";
g_diagnosticsPending.fetch_or(DIAG_HEADPHONES,
std::memory_order_release);
} else {
// Mute HP, enable speaker
DisablePin(g_hpNid);
EnablePin(g_speakerNid);
g_pinNid = g_speakerNid;
KernelLogStream(INFO, "HDA") << "Switched to speaker output";
g_diagnosticsPending.fetch_or(DIAG_SPEAKERS,
std::memory_order_release);
}
// Re-apply volume on the now-active pin
SetOutputVolume(g_volume);
@@ -810,7 +837,7 @@ namespace Drivers::Audio::IntelHda {
// MSI setup
// =========================================================================
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
@@ -819,6 +846,8 @@ namespace Drivers::Audio::IntelHda {
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
@@ -851,21 +880,21 @@ namespace Drivers::Audio::IntelHda {
// Interrupt handler
// =========================================================================
static void HandleInterrupt(uint8_t /*irq*/) {
static void HandleInterrupt(uint8_t /*irq*/, bool /*fromUser*/) {
uint32_t intsts = Read32(REG_INTSTS);
bool bufferCompleted = false;
// Handle stream interrupts (bits 0-29 correspond to stream descriptors)
g_streamLock.Acquire();
if (g_stream.Active) {
uint8_t si = g_stream.StreamIndex;
if (intsts & (1u << si)) {
// Handle stream interrupts (bits 0-29 correspond to stream
// descriptors) without taking g_streamLock. Inspecting only the
// hardware status registers is race-safe against stream lifecycle and
// avoids spinning in an IRQ if another CPU is opening/closing audio.
uint32_t streamBits = intsts & 0x3FFFFFFFu;
for (uint8_t si = 0; si < 30; si++) {
if (!(streamBits & (1u << si))) continue;
uint8_t sts = ReadSD8(si, SD_STS);
if (sts & SD_STS_BCIS) bufferCompleted = true;
WriteSD8(si, SD_STS, sts);
}
}
g_streamLock.Release();
// Handle RIRB interrupt (controller interrupt enable bit 30)
// Do NOT advance g_rirbReadPtr here — ReadResponse() owns it.
@@ -876,10 +905,12 @@ namespace Drivers::Audio::IntelHda {
Write8(REG_RIRBSTS, rirbSts);
}
// Notify the mixer so it can refill the DMA ring with the next mix
// window. Done after clearing status bits so re-entry can't latch.
// A mixer pass can resample up to 4096 frames across every active
// stream and must never run inside this high-priority IRQ. Mask stream
// completion delivery and queue a process-safe bottom half instead.
if (bufferCompleted) {
Mixer::OnHdaBufferComplete();
Write32(REG_INTCTL, Read32(REG_INTCTL) & ~g_streamInterruptMask);
g_streamWorkPending.store(true, std::memory_order_release);
}
}
@@ -987,7 +1018,13 @@ namespace Drivers::Audio::IntelHda {
uint8_t irqLine = Pci::LegacyRead8(g_bus, g_dev, g_func, (uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "HDA") << "Falling back to legacy IRQ " << base::dec << (uint64_t)irqLine;
Pci::DisableInterruptDelivery(g_bus, g_dev, g_func);
uint16_t command = Pci::LegacyRead16(g_bus, g_dev, g_func,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(g_bus, g_dev, g_func,
(uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE);
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
} else {
KernelLogStream(WARNING, "HDA") << "No interrupt available, polling only";
}
@@ -997,8 +1034,9 @@ namespace Drivers::Audio::IntelHda {
uint32_t intctl = INTCTL_GIE | INTCTL_CIE;
// Enable interrupt for all output streams
for (uint8_t i = 0; i < g_numOutputStreams; i++) {
intctl |= (1u << (g_numInputStreams + i));
g_streamInterruptMask |= (1u << (g_numInputStreams + i));
}
intctl |= g_streamInterruptMask;
Write32(REG_INTCTL, intctl);
// Discover codecs
@@ -1028,6 +1066,63 @@ namespace Drivers::Audio::IntelHda {
return g_initialized;
}
bool HasDeferredWork() {
return (g_initialized &&
g_streamWorkPending.load(std::memory_order_acquire)) ||
g_diagnosticsPending.load(std::memory_order_acquire) != 0 ||
g_codecTimeoutsPending.load(std::memory_order_acquire) != 0;
}
void ProcessDeferredWork() {
if (g_initialized &&
g_streamWorkPending.exchange(false, std::memory_order_acq_rel)) {
Mixer::OnHdaBufferComplete();
// Clear-before-unmask closes the handoff race: if another completion
// arrived while masked its SD_STS/INTSTS state is still asserted and
// enabling the stream bit produces a fresh MSI/INTx interrupt.
Write32(REG_INTCTL, Read32(REG_INTCTL) | g_streamInterruptMask |
INTCTL_GIE | INTCTL_CIE);
}
uint32_t diagnostics =
g_diagnosticsPending.exchange(0, std::memory_order_acq_rel);
if (diagnostics & DIAG_STREAM_OPENED) {
KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec
<< (uint64_t)g_openSampleRate.load(std::memory_order_relaxed)
<< "Hz "
<< (uint64_t)g_openBitsPerSample.load(std::memory_order_relaxed)
<< "-bit "
<< (uint64_t)g_openChannels.load(std::memory_order_relaxed)
<< "ch";
}
if (diagnostics & DIAG_STREAM_CLOSED)
KernelLogStream(OK, "HDA") << "Stream closed";
if (diagnostics & DIAG_HEADPHONES)
KernelLogStream(INFO, "HDA") << "Switched to headphone output";
if (diagnostics & DIAG_SPEAKERS)
KernelLogStream(INFO, "HDA") << "Switched to speaker output";
uint32_t timeouts =
g_codecTimeoutsPending.exchange(0, std::memory_order_acq_rel);
if (timeouts != 0) {
uint32_t report =
g_codecTimeoutReports.fetch_add(1, std::memory_order_relaxed);
if (report < 8) {
KernelLogStream(WARNING, "HDA") << base::dec
<< (uint64_t)timeouts << " codec verb timeout(s), last: codec="
<< (uint64_t)g_lastTimeoutCodec.load(std::memory_order_relaxed)
<< " nid="
<< (uint64_t)g_lastTimeoutNid.load(std::memory_order_relaxed)
<< " verb=" << base::hex
<< (uint64_t)g_lastTimeoutVerb.load(std::memory_order_relaxed);
} else if (report == 8) {
KernelLogStream(WARNING, "HDA")
<< "Further codec verb timeout reports suppressed";
}
}
}
uint32_t GetCodecVendorId() {
return g_codecVendorId;
}
@@ -1075,11 +1170,12 @@ namespace Drivers::Audio::IntelHda {
// Start the stream
StartStream(streamIndex);
KernelLogStream(OK, "HDA") << "Stream opened: " << base::dec
<< (uint64_t)sampleRate << "Hz " << (uint64_t)bitsPerSample << "-bit "
<< (uint64_t)channels << "ch";
g_streamLock.Release();
g_openSampleRate.store(sampleRate, std::memory_order_relaxed);
g_openBitsPerSample.store(bitsPerSample, std::memory_order_relaxed);
g_openChannels.store(channels, std::memory_order_relaxed);
g_diagnosticsPending.fetch_or(DIAG_STREAM_OPENED,
std::memory_order_release);
return 0; // Handle 0
}
@@ -1099,8 +1195,9 @@ namespace Drivers::Audio::IntelHda {
g_stream.Active = false;
KernelLogStream(OK, "HDA") << "Stream closed";
g_streamLock.Release();
g_diagnosticsPending.fetch_or(DIAG_STREAM_CLOSED,
std::memory_order_release);
}
uint32_t GetWriteSpace(int handle) {
kernel/src/Drivers/Audio/IntelHda.hpp
+2
View File
@@ -259,6 +259,8 @@ namespace Drivers::Audio::IntelHda {
bool Probe(const Pci::PciDevice& dev);
bool IsInitialized();
bool HasDeferredWork();
void ProcessDeferredWork();
// Returns the codec vendor/device ID (vendor in upper 16 bits, device in lower 16).
// Returns 0 if no codec was found.
kernel/src/Drivers/Audio/Mixer.cpp
+9 -5
View File
@@ -122,7 +122,11 @@ namespace Drivers::Audio::Mixer {
static void FreeRing(int16_t* ring) {
if (!ring) return;
Memory::g_pfa->ReallocConsecutive(ring, 0);
// The ring was allocated as one INPUT_RING_PAGES contiguous span.
// ReallocConsecutive(ptr, 0) is not a sized free operation and used
// to compute a zero-byte allocation at the end of the PFA pool, copy
// a page there, then free only the first page of this ring.
Memory::g_pfa->Free(ring, INPUT_RING_PAGES);
}
// Caller holds g_lock. HDA setup is non-blocking and serialized by the
@@ -380,10 +384,11 @@ namespace Drivers::Audio::Mixer {
if (!ring) return -1;
retry_after_switch:
uint64_t switchWake = Sched::ObserveObjectWake((void*)&g_switchingOutput);
g_lock.Acquire();
if (g_switchingOutput.load(std::memory_order_acquire)) {
g_lock.Release();
Sched::BlockOnObject((void*)&g_switchingOutput, 1000);
Sched::BlockOnObjectSince((void*)&g_switchingOutput, 1000, switchWake);
goto retry_after_switch;
}
@@ -821,9 +826,8 @@ namespace Drivers::Audio::Mixer {
}
void OnHdaBufferComplete() {
// Called from the HDA buffer-completion interrupt. Mixer state and
// HDA register access are both serialized through g_lock (which
// disables interrupts on acquire), so this is safe to call from IRQ.
// Called from IntelHda's idle-context bottom half. A pump may resample
// thousands of frames and is deliberately kept out of the hard IRQ.
g_lock.Acquire();
if (g_output == Output::Hda) Pump();
g_lock.Release();
kernel/src/Drivers/Graphics/IntelGPU.cpp
+271 -178
View File
@@ -10,14 +10,14 @@
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Io/IoPort.hpp>
#include <Graphics/Framebuffer.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Sched/Scheduler.hpp>
#include <Libraries/Memory.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
@@ -36,7 +36,6 @@ namespace Drivers::Graphics::IntelGPU {
// GTT state
static volatile void* g_gttBase = nullptr; // Virtual address of GTT entries
static uint64_t g_gttEntryCount = 0; // Number of GTT entries
static uint64_t g_scratchPagePhys = 0; // Physical address of scratch page
static uint8_t g_gpuGen = 0; // Cached generation number
// Framebuffer state
@@ -54,12 +53,15 @@ namespace Drivers::Graphics::IntelGPU {
static uint64_t g_fbGttOffsetA = 0; // GGTT offset of buffer 0 (firmware FB)
static uint32_t* g_buf1Virt = nullptr; // buffer 1 kernel VA (HHDM, WC)
static uint64_t g_buf1Phys = 0;
static uint64_t g_buf1Pages = 0; // allocation/mapping bound
static uint64_t g_buf1GttOffset = 0; // GGTT offset of buffer 1 (in bytes)
static bool g_flipSupported = false;
static int g_frontBuffer = 0;
static int g_flipOwnerPid = -1;
static volatile uint64_t g_vblankCount = 0;
static bool g_vblankIrqReady = false;
// Serializes surface programming, flip-owner teardown, and timing changes.
// Without this, a process exit or modeset on another CPU can rewrite
// DSPASURF/PIPE state in the middle of a desktop flip transaction.
static kcp::Mutex g_flipLock;
// Modesetting/display-management state. The firmware-trained port and
// link are deliberately retained; the driver owns the active transcoder,
@@ -216,6 +218,21 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// This driver intentionally uses register polling for display latches and
// does not own the GPU's many generation-specific interrupt sources. Mask
// PCI delivery before touching display state: otherwise firmware-left MSI
// enables can route an unhandled GT interrupt and create an interrupt
// storm even though Montauk never requested one.
static void DisableInterruptDelivery() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
if (Pci::DisableInterruptDelivery(bus, dev, func)) {
KernelLogStream(WARNING, "IntelGPU")
<< "Disabled pre-enabled GPU MSI/MSI-X; using bounded latch polling";
}
}
// =========================================================================
// BAR0 MMIO Mapping
// =========================================================================
@@ -357,12 +374,20 @@ namespace Drivers::Graphics::IntelGPU {
return false;
}
// Ensure pitch is at least width * 4 (BGRX8888)
// Ensure pitch is at least width * 4 (BGRX8888), and reject geometry
// whose size arithmetic could wrap before it reaches the allocator.
if (g_fbPitch == 0) {
g_fbPitch = g_fbWidth * 4;
KernelLogStream(WARNING, "IntelGPU") << "Stride not available, assuming "
<< base::dec << g_fbPitch << " bytes";
}
if (g_fbWidth > (~0ULL / 4)
|| g_fbPitch < g_fbWidth * 4
|| g_fbHeight > (~0ULL / g_fbPitch)) {
KernelLogStream(ERROR, "IntelGPU")
<< "Invalid or overflowing framebuffer geometry";
return false;
}
g_fbSize = g_fbHeight * g_fbPitch;
@@ -989,17 +1014,13 @@ namespace Drivers::Graphics::IntelGPU {
<< ", " << base::dec << g_gttEntryCount << " entries"
<< (g_gpuGen >= 8 ? " (64-bit PTEs)" : " (32-bit PTEs)");
// Allocate a scratch page (zeroed) for future use
void* scratchPageVirt = Memory::g_pfa->AllocateZeroed();
g_scratchPagePhys = Memory::SubHHDM(scratchPageVirt);
// Do NOT clear the entire GTT here. The firmware has active GTT mappings
// that the display engine is currently scanning out from. Clearing them
// would cause the display to go black (or worse) before we remap.
// Instead, we only write the entries we need in SetupFramebuffer().
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec << g_gttEntryCount
<< " entries, scratch page at " << base::hex << g_scratchPagePhys;
KernelLogStream(OK, "IntelGPU") << "GTT ready: " << base::dec
<< g_gttEntryCount << " entries";
return true;
}
@@ -1108,18 +1129,47 @@ namespace Drivers::Graphics::IntelGPU {
// Initialization maps this window once and preserves the firmware's active
// scanout PTEs. Page flipping only proceeds after those entries validate.
static void MicroDelay(int us) {
// Simple busy-wait; us is approximate
for (volatile int i = 0; i < us * 100; i++) {
asm volatile("pause");
}
}
static uint64_t ReadGgttPte(uint64_t index) {
if (g_gpuGen >= 8) return ((volatile uint64_t*)g_ggtt)[index];
return ((volatile uint32_t*)g_ggtt)[index];
}
static int ReadLiveBufferIndex() {
if (!g_mmioBase) return -1;
uint32_t live = ReadReg(DSPASURFLIVE) & ~0xFFFu;
if (live == (uint32_t)g_fbGttOffsetA) return 0;
if (live == (uint32_t)g_buf1GttOffset) return 1;
return -1;
}
// Polling-only latch wait. This is safe during both ordinary syscalls and
// ExitProcess teardown because it never enters the scheduler. Keeping
// flips independent of GPU MSI is deliberate: MSI would route every
// firmware-enabled GPU source, not just the one source this driver owns.
// Three observed frames are the normal timeout. Laptop panel self-refresh
// can transiently stop the scanline counter, so also enforce a wall-clock
// bound: one million uncached MMIO reads can otherwise hold the desktop in
// SYS_FBFLIP long enough to look permanently frozen (cursor included).
static bool WaitForLiveBuffer(int index) {
uint32_t expected = (uint32_t)((index == 0) ? g_fbGttOffsetA
: g_buf1GttOffset);
uint32_t previous = ReadReg(PIPE_DSL) & 0x1FFFu;
uint64_t started = Timekeeping::GetMilliseconds();
int frameWraps = 0;
for (int i = 0; i < 200000; i++) {
uint32_t live = ReadReg(DSPASURFLIVE);
if (live == 0xFFFFFFFFu) return false;
if ((live & ~0xFFFu) == expected) return true;
uint32_t current = ReadReg(PIPE_DSL) & 0x1FFFu;
if (current < previous && ++frameWraps >= 3) return false;
previous = current;
if ((i & 0x3FF) == 0 &&
Timekeeping::GetMilliseconds() - started >= 100) return false;
asm volatile("pause");
}
return false;
}
static void WriteGgttPte(uint64_t index, uint64_t physAddr) {
if (g_gpuGen >= 8) {
((volatile uint64_t*)g_ggtt)[index] = MakeGttPte64(physAddr);
@@ -1168,14 +1218,37 @@ namespace Drivers::Graphics::IntelGPU {
static bool AllocateBackBuffer() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, pages);
if (pages == 0 || pages > 0x7FFFFFFFULL) {
KernelLogStream(WARNING, "IntelGPU")
<< "Invalid second scanout buffer page count " << base::dec << pages;
return false;
}
void* virt = Memory::g_pfa->ReallocConsecutive(nullptr, (int)pages);
if (!virt) {
KernelLogStream(WARNING, "IntelGPU") << "Failed to allocate " << base::dec
<< pages << " contiguous pages for the second scanout buffer";
return false;
}
g_buf1Phys = Memory::SubHHDM(virt);
uint64_t phys = Memory::SubHHDM(virt);
if ((pages - 1) > (UINT64_MAX - phys) / 0x1000ULL) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer physical range overflows";
return false;
}
uint64_t lastPhys = phys + (pages - 1) * 0x1000ULL;
if (!GgttCanAddress(phys, g_gpuGen) || !GgttCanAddress(lastPhys, g_gpuGen)) {
Memory::g_pfa->Free(virt, (int)pages);
KernelLogStream(WARNING, "IntelGPU")
<< "Second scanout buffer lies outside the GPU physical-address width ("
<< base::hex << phys << ".." << lastPhys << ")";
return false;
}
g_buf1Phys = phys;
g_buf1Virt = (uint32_t*)virt;
g_buf1Pages = pages;
// Display scanout does not snoop the CPU cache: remap the buffer
// write-combining before touching it, same as the firmware FB.
@@ -1192,23 +1265,50 @@ namespace Drivers::Graphics::IntelGPU {
static bool FindAndMapGgttRange() {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
constexpr uint64_t guard = 16;
uint64_t need = pages + 2 * guard;
uint64_t avoidFirst = g_fbGttOffsetA >> 12;
uint64_t avoidLast = (g_fbGttOffsetA + g_fbSize - 1) >> 12;
// Montauk owns the GPU after firmware handoff. Reserve a deterministic
// high-aperture range for its second scanout buffer instead of trying
// to infer firmware ownership from scratch-PTE patterns. Keep guards
// around the allocation and never overlap the live firmware surface.
uint64_t base = g_ggttEntries / 2 + guard;
if (base <= avoidLast + guard) base = avoidLast + guard + 1;
base = (base + guard - 1) & ~(guard - 1);
if (base < guard || base + pages + guard > g_ggttEntries
|| !(base + pages - 1 < avoidFirst || base > avoidLast)) {
KernelLogStream(WARNING, "IntelGPU") << "No safe owned GGTT range for "
// Firmware normally fills unused GGTT entries with one scratch-page
// PTE. Verify that invariant at the tail, then claim only a complete
// scratch-backed (or invalid) run. Never overwrite arbitrary valid
// mappings merely because they happen to sit in the upper half.
if (g_ggttEntries < 32 || need > g_ggttEntries / 2) return false;
uint64_t scratchPte = ReadGgttPte(g_ggttEntries - 1);
for (uint64_t probe = 2; probe <= 32; probe++) {
if (ReadGgttPte(g_ggttEntries - probe) != scratchPte) {
KernelLogStream(WARNING, "IntelGPU")
<< "GGTT tail is owned/non-uniform; page flip unavailable";
return false;
}
}
uint64_t runStart = 0;
uint64_t run = 0;
bool found = false;
for (uint64_t idx = g_ggttEntries / 2; idx < g_ggttEntries; idx++) {
if (idx >= avoidFirst && idx <= avoidLast) {
run = 0;
continue;
}
uint64_t pte = ReadGgttPte(idx);
if ((pte & 1) && pte != scratchPte) {
run = 0;
continue;
}
if (run == 0) runStart = idx;
if (++run >= need) {
found = true;
break;
}
}
if (!found) {
KernelLogStream(WARNING, "IntelGPU") << "No verified free GGTT run for "
<< base::dec << pages << " back-buffer pages";
return false;
}
uint64_t base = runStart + guard;
uint64_t replacedFirst = ReadGgttPte(base);
uint64_t replacedLast = ReadGgttPte(base + pages - 1);
for (uint64_t i = 0; i < pages; i++) {
@@ -1235,94 +1335,38 @@ namespace Drivers::Graphics::IntelGPU {
return true;
}
// =========================================================================
// Page flipping: vblank interrupt (Gen 11+)
// =========================================================================
static void HandleInterrupt(uint8_t /*irq*/) {
// Gen 11+ flow: park the master enable, ack the pipe IIR, re-enable
uint32_t master = ReadReg(GFX_MSTR_INTR);
if (!(master & GFX_MSTR_INTR_DISPLAY)) {
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
return;
}
WriteReg(GFX_MSTR_INTR, 0);
uint32_t iir = ReadReg(DE_PIPE_A_IIR);
if (iir) WriteReg(DE_PIPE_A_IIR, iir); // write-1-to-clear
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
if (iir & DE_PIPE_VBLANK) {
g_vblankCount = g_vblankCount + 1;
Sched::WakeObjectWaiters((void*)&g_vblankCount);
}
}
static bool SetupMsi() {
uint8_t bus = g_gpuInfo.pciBus;
uint8_t dev = g_gpuInfo.pciDevice;
uint8_t func = g_gpuInfo.pciFunction;
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "IntelGPU") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // one message
msgCtrl |= (1 << 0); // MSI enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "IntelGPU") << "MSI enabled: vector " << base::dec
<< (uint64_t)MSI_VECTOR << " (IRQ slot " << (uint64_t)MSI_IRQ << ")"
<< (is64bit ? " [64-bit]" : " [32-bit]");
return true;
}
static void EnableVblankIrq() {
WriteReg(DE_PIPE_A_IIR, 0xFFFFFFFFu); // clear stale events
WriteReg(DE_PIPE_A_IMR, ~DE_PIPE_VBLANK); // unmask vblank only
WriteReg(DE_PIPE_A_IER, DE_PIPE_VBLANK);
WriteReg(DISPLAY_INT_CTL, DISPLAY_IRQ_ENABLE);
WriteReg(GFX_MSTR_INTR, GFX_MSTR_INTR_ENABLE);
(void)ReadReg(GFX_MSTR_INTR);
}
static void SetupPageFlip() {
if (!MapCorrectGgtt()) return;
if (!VerifyFirmwareScanoutPtes()) return;
if (!AllocateBackBuffer()) return;
if (!FindAndMapGgttRange()) return;
// Vblank interrupts: Gen 11+ register layout only. Older generations
// still flip, with DSPASURFLIVE polling standing in for the IRQ.
if (g_gpuGen >= 11 && SetupMsi()) {
EnableVblankIrq();
g_vblankIrqReady = true;
g_flipSupported = true;
int live = ReadLiveBufferIndex();
g_frontBuffer = (live >= 0) ? live : 0;
KernelLogStream(OK, "IntelGPU")
<< "Page flip ready: 2 buffers, bounded SURFLIVE polling";
}
g_flipSupported = true;
g_frontBuffer = 0;
KernelLogStream(OK, "IntelGPU") << "Page flip ready: 2 buffers, vblank IRQ "
<< (g_vblankIrqReady ? "on" : "off (SURFLIVE polling)");
static bool FirmwareFramebufferMatchesGpu() {
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
bool match = g_fbWidth == fwWidth
&& g_fbHeight == fwHeight
&& g_fbPitch == fwPitch;
if (match) return true;
// Userspace obtains its drawing dimensions from the firmware
// framebuffer API. A differently-sized GPU allocation would make the
// second mapping overrun (or alias only part of) the backing buffer.
KernelLogStream(ERROR, "IntelGPU")
<< "GPU dimensions differ from firmware; unsafe framebuffer setup skipped";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
return false;
}
// =========================================================================
@@ -1388,6 +1432,7 @@ namespace Drivers::Graphics::IntelGPU {
}
g_gpuGen = g_gpuInfo.gen;
DisableInterruptDelivery();
if (!MapMmio()) {
KernelLogStream(ERROR, "IntelGPU") << "Failed to map MMIO region";
@@ -1400,6 +1445,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return false;
}
if (!FirmwareFramebufferMatchesGpu()) return false;
InitializeDisplayManagement();
if (!InitializeGtt()) {
@@ -1417,18 +1463,6 @@ namespace Drivers::Graphics::IntelGPU {
SetupPageFlip();
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight << " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight << " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1446,6 +1480,7 @@ namespace Drivers::Graphics::IntelGPU {
if (!DetectGpu()) {
return;
}
DisableInterruptDelivery();
// Step 2: Map BAR0 MMIO region
if (!MapMmio()) {
@@ -1461,6 +1496,7 @@ namespace Drivers::Graphics::IntelGPU {
KernelLogStream(ERROR, "IntelGPU") << "Failed to read display state";
return;
}
if (!FirmwareFramebufferMatchesGpu()) return;
InitializeDisplayManagement();
// Step 5: Initialize GTT
@@ -1480,21 +1516,6 @@ namespace Drivers::Graphics::IntelGPU {
g_initialized = true;
// Diagnostic: compare GPU-detected values with firmware/Limine values
uint64_t fwWidth = ::Graphics::Framebuffer::GetWidth();
uint64_t fwHeight = ::Graphics::Framebuffer::GetHeight();
uint64_t fwPitch = ::Graphics::Framebuffer::GetPitch();
if (g_fbWidth != fwWidth || g_fbHeight != fwHeight || g_fbPitch != fwPitch) {
KernelLogStream(WARNING, "IntelGPU") << "GPU dimensions differ from firmware!";
KernelLogStream(WARNING, "IntelGPU") << " GPU: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " pitch=" << g_fbPitch;
KernelLogStream(WARNING, "IntelGPU") << " Firmware: "
<< base::dec << fwWidth << "x" << fwHeight
<< " pitch=" << fwPitch;
}
KernelLogStream(OK, "IntelGPU") << "Initialization complete: "
<< base::dec << g_fbWidth << "x" << g_fbHeight
<< " @ " << base::hex << (uint64_t)g_fbBase;
@@ -1641,7 +1662,6 @@ namespace Drivers::Graphics::IntelGPU {
if (g_edidValid) out->capabilities |= montauk::abi::DISPLAY_CAP_EDID;
if (g_pwmSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_BRIGHTNESS;
if (g_flipSupported) out->capabilities |= montauk::abi::DISPLAY_CAP_PAGE_FLIP;
if (g_vblankIrqReady) out->capabilities |= montauk::abi::DISPLAY_CAP_VBLANK_IRQ;
for (int i = 0; i < g_modeCount; i++) {
if (i != g_currentMode
&& (g_modes[i].flags & montauk::abi::DISPLAY_MODE_DRIVER_VALID)) {
@@ -1685,6 +1705,8 @@ namespace Drivers::Graphics::IntelGPU {
return -2;
if (modeIndex == g_currentMode) return modeIndex;
g_flipLock.Acquire();
// Link rate and framebuffer geometry remain fixed. Only timings with
// the same active area and pixel clock are safe without retraining the
// firmware-established HDMI/DP/eDP link.
@@ -1693,10 +1715,13 @@ namespace Drivers::Graphics::IntelGPU {
if (requested.hdisplay != current.hdisplay
|| requested.vdisplay != current.vdisplay
|| requested.pixelClock == 0
|| requested.pixelClock != current.pixelClock)
|| requested.pixelClock != current.pixelClock) {
g_flipLock.Release();
return -2;
}
if (!ApplyModeTiming(requested)) {
g_flipLock.Release();
KernelLogStream(ERROR, "IntelGPU") << "Modeset transaction failed; previous mode restored";
return -3;
}
@@ -1704,6 +1729,7 @@ namespace Drivers::Graphics::IntelGPU {
g_modes[g_currentMode].flags &= ~montauk::abi::DISPLAY_MODE_CURRENT;
g_currentMode = modeIndex;
g_modes[g_currentMode].flags |= montauk::abi::DISPLAY_MODE_CURRENT;
g_flipLock.Release();
KernelLogStream(OK, "IntelGPU") << "Mode set to " << base::dec
<< (uint64_t)requested.hdisplay << "x" << (uint64_t)requested.vdisplay
<< " @ " << (uint64_t)(g_modes[modeIndex].refreshMilliHz / 1000) << "Hz";
@@ -1741,59 +1767,131 @@ namespace Drivers::Graphics::IntelGPU {
return 0;
}
uint64_t GetBufferPageCount(int index) {
if (index == 0) return (g_fbSize + 0xFFF) >> 12;
if (index == 1 && g_flipSupported) return g_buf1Pages;
return 0;
}
int GetFrontBuffer() {
return g_frontBuffer;
g_flipLock.Acquire();
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
return live;
}
uint64_t GetVblankCount() {
return g_vblankCount;
int AcquireFlip(int pid) {
if (!g_flipSupported || pid < 0) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
int live = ReadLiveBufferIndex();
if (live < 0) {
// Recover from an unrecognised/pending firmware surface by putting
// the legacy buffer back synchronously before userspace draws.
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
if (WaitForLiveBuffer(0)) live = 0;
}
if (live >= 0) g_frontBuffer = live;
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
KernelLogStream(DEBUG, "IntelGPU") << "Flip acquired by pid "
<< base::dec << (uint64_t)pid << " (live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live) << ")";
return live;
}
bool WaitVblank(uint64_t timeoutMs) {
if (!g_vblankIrqReady) return false;
uint64_t start = g_vblankCount;
// The vblank IRQ fires every frame, so a wakeup lost to the race
// between reading the counter and blocking only costs one frame;
// the timeout is a backstop for a stalled pipe.
Sched::BlockOnObject((void*)&g_vblankCount, timeoutMs ? timeoutMs : 50);
return g_vblankCount != start;
}
int64_t Flip(int index, bool waitVsync, int pid) {
if (!g_flipSupported || index < 0 || index > 1 || pid < 0) return -1;
int64_t Flip(int index, bool waitVsync) {
if (!g_flipSupported || index < 0 || index > 1) return -1;
g_flipLock.Acquire();
int oldOwner = g_flipOwnerPid;
g_flipOwnerPid = pid;
// Scanout memory is exposed through WC mappings. Drain userspace's
// completed frame before programming the display-plane surface.
asm volatile("sfence" ::: "memory");
uint64_t gttOff = (index == 0) ? g_fbGttOffsetA : g_buf1GttOffset;
WriteReg(DSPASURF, (uint32_t)gttOff);
(void)ReadReg(DSPASURF);
g_frontBuffer = index;
if (waitVsync) {
if (g_vblankIrqReady) {
WaitVblank(50);
// DSPASURFLIVE is authoritative and avoids taking ownership of
// the GPU's shared interrupt domain merely to wait for vblank.
if (!WaitForLiveBuffer(index)) {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmed = ReadReg(DSPASURF);
uint32_t liveSurface = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
// Confirm the latch: DSPASURFLIVE tracks the surface actually
// being scanned out. With a working IRQ this succeeds on the
// first read; without one it is the vsync wait itself.
for (int i = 0; i < 3000; i++) {
if ((ReadReg(DSPASURFLIVE) & ~0xFFFu) == (uint32_t)gttOff) break;
MicroDelay(10);
KernelLogStream(WARNING, "IntelGPU") << "Flip latch timeout: requested="
<< base::dec << (uint64_t)index << " live="
<< (uint64_t)(live < 0 ? 0xFFFFFFFFu : (uint32_t)live)
<< " DSPASURF=" << base::hex << (uint64_t)programmed
<< " DSPASURFLIVE=" << (uint64_t)liveSurface;
return -1;
}
g_frontBuffer = index;
} else {
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
}
g_flipLock.Release();
if (oldOwner != pid) {
KernelLogStream(DEBUG, "IntelGPU") << "Flip owner " << base::dec
<< (uint64_t)(oldOwner < 0 ? 0xFFFFFFFFu : (uint32_t)oldOwner)
<< " -> " << (uint64_t)pid;
}
return index;
}
void SetFlipOwner(int pid) {
g_flipOwnerPid = pid;
}
void OnProcessExit(int pid) {
if (!g_flipSupported || pid != g_flipOwnerPid) return;
if (!g_flipSupported) return;
g_flipLock.Acquire();
if (pid != g_flipOwnerPid) {
g_flipLock.Release();
return;
}
g_flipOwnerPid = -1;
if (g_frontBuffer != 0) Flip(0, false);
uint32_t programmedBefore = ReadReg(DSPASURF);
uint32_t liveBefore = ReadReg(DSPASURFLIVE);
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
bool liveAtZero = WaitForLiveBuffer(0);
bool restored = liveAtZero;
int live = ReadLiveBufferIndex();
if (live >= 0) g_frontBuffer = live;
uint32_t programmedAfter = ReadReg(DSPASURF);
uint32_t liveAfter = ReadReg(DSPASURFLIVE);
g_flipLock.Release();
KernelLogStream(restored ? INFO : ERROR, "IntelGPU")
<< "Flip owner pid " << base::dec << (uint64_t)pid
<< " exited; restore buffer 0 " << (restored ? "latched" : "timed out")
<< " (before surf/live=" << base::hex << (uint64_t)programmedBefore
<< "/" << (uint64_t)liveBefore << ", after="
<< (uint64_t)programmedAfter << "/"
<< (uint64_t)liveAfter << ")";
}
void PanicRestoreScanout() {
if (!g_flipSupported || !g_mmioBase || g_frontBuffer == 0) return;
// g_frontBuffer is only a software cache and can be stale precisely
// when the GPU or kernel is failing. Always request buffer 0 so the
// panic console cannot be left painting an invisible surface.
if (!g_flipSupported || !g_mmioBase) return;
WriteReg(DSPASURF, (uint32_t)g_fbGttOffsetA);
(void)ReadReg(DSPASURF);
g_frontBuffer = 0;
@@ -1809,6 +1907,7 @@ namespace Drivers::Graphics::IntelGPU {
// command register, so MMIO writes to the GPU are silently dropped
// until we re-enable these bits.
Pci::EnableBusMaster(g_gpuInfo.pciBus, g_gpuInfo.pciDevice, g_gpuInfo.pciFunction);
DisableInterruptDelivery();
// Verify PCI memory space is accessible by reading back command register
uint16_t pciCmd = Pci::LegacyRead16(g_gpuInfo.pciBus, g_gpuInfo.pciDevice,
@@ -1875,9 +1974,8 @@ namespace Drivers::Graphics::IntelGPU {
// 5. Reprogram display plane to point at our GTT-mapped framebuffer
ProgramDisplayPlane();
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries, bring
// MSI + vblank interrupts back (the PCI config reset cleared MSI),
// and re-show whichever buffer was front when we suspended.
// 6. Restore page-flip state: rewrite buffer 1's GGTT entries and
// re-show whichever buffer was front when we suspended.
if (g_flipSupported) {
uint64_t pages = (g_fbSize + 0xFFF) >> 12;
uint64_t baseIdx = g_buf1GttOffset >> 12;
@@ -1886,11 +1984,6 @@ namespace Drivers::Graphics::IntelGPU {
}
(void)ReadGgttPte(baseIdx + pages - 1);
if (g_vblankIrqReady) {
SetupMsi();
EnableVblankIrq();
}
if (g_frontBuffer != 0) {
WriteReg(DSPASURF, (uint32_t)g_buf1GttOffset);
(void)ReadReg(DSPASURF);
kernel/src/Drivers/Graphics/IntelGPU.hpp
+23 -36
View File
@@ -170,6 +170,7 @@ namespace Drivers::Graphics::IntelGPU {
static constexpr uint32_t VBLANK_A = 0x60010;
static constexpr uint32_t VSYNC_A = 0x60014;
static constexpr uint32_t PIPEASRC = 0x6001C;
static constexpr uint32_t PIPE_DSL = 0x70000; // current pipe A scanline
// --- Display timing registers (Pipe B) ---
static constexpr uint32_t HTOTAL_B = 0x61000;
@@ -231,30 +232,6 @@ namespace Drivers::Graphics::IntelGPU {
// engine is currently scanning out; DSPASURF writes latch at vblank) ---
static constexpr uint32_t DSPASURFLIVE = 0x701AC;
// --- Display engine interrupts (Gen 8+ pipe block, Gen 11+ master) ---
// Per-pipe interrupt registers, pipe A instance. Vblank is bit 0.
static constexpr uint32_t DE_PIPE_A_ISR = 0x44400;
static constexpr uint32_t DE_PIPE_A_IMR = 0x44404;
static constexpr uint32_t DE_PIPE_A_IIR = 0x44408;
static constexpr uint32_t DE_PIPE_A_IER = 0x4440C;
static constexpr uint32_t DE_PIPE_VBLANK = (1u << 0);
// Gen 11+ display interrupt master control (0x44200 was the Gen 8-10
// master IRQ register; on Gen 11+ it controls the display half only)
static constexpr uint32_t DISPLAY_INT_CTL = 0x44200;
static constexpr uint32_t DISPLAY_IRQ_ENABLE = (1u << 31);
static constexpr uint32_t DISPLAY_IRQ_PIPE_A = (1u << 16);
// Gen 11+ top-level graphics master interrupt
static constexpr uint32_t GFX_MSTR_INTR = 0x190010;
static constexpr uint32_t GFX_MSTR_INTR_ENABLE = (1u << 31);
static constexpr uint32_t GFX_MSTR_INTR_DISPLAY = (1u << 16);
// --- MSI (vblank interrupt delivery) ---
static constexpr uint8_t MSI_IRQ = 28; // IRQ slot 28 = vector 60
static constexpr uint32_t MSI_VECTOR = 60;
static constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// --- Output connectors ---
static constexpr uint32_t ADPA = 0x61100; // Analog Display Port (VGA/CRT)
static constexpr uint32_t DVOB = 0x61140; // DVO-B
@@ -347,6 +324,7 @@ namespace Drivers::Graphics::IntelGPU {
// Gen 8+ uses 64-bit GTT PTEs
static constexpr uint64_t GTT_PTE64_VALID = (1ULL << 0);
static constexpr uint64_t GTT_PTE64_ADDRESS_MASK = 0x000FFFFFFFFFF000ULL;
// Helper: Build a Gen 6/7 GTT PTE from a physical address
static inline uint32_t MakeGttPte32(uint64_t physAddr) {
@@ -358,7 +336,15 @@ namespace Drivers::Graphics::IntelGPU {
// Helper: Build a Gen 8+ GTT PTE from a physical address
static inline uint64_t MakeGttPte64(uint64_t physAddr) {
return (physAddr & ~0xFFFULL) | GTT_PTE64_VALID;
return (physAddr & GTT_PTE64_ADDRESS_MASK) | GTT_PTE64_VALID;
}
// Gen 6/7 PTEs carry 39 physical-address bits; Gen 8+ PTEs carry 52.
// Never silently truncate a DMA address into a different RAM page.
static inline bool GgttCanAddress(uint64_t physAddr, int gpuGen) {
if (physAddr & 0xFFFULL) return false;
if (gpuGen >= 8) return (physAddr & ~0x000FFFFFFFFFFFFFULL) == 0;
return (physAddr & ~0x0000007FFFFFFFFFULL) == 0;
}
// =========================================================================
@@ -525,25 +511,26 @@ namespace Drivers::Graphics::IntelGPU {
// Physical base of scanout buffer 0 (firmware FB) or 1 (kernel-allocated)
uint64_t GetBufferPhys(int index);
// Number of physically allocated pages backing the requested buffer.
uint64_t GetBufferPageCount(int index);
// Index of the buffer currently programmed for scanout
int GetFrontBuffer();
// Claim page flipping for a process and return the buffer the display
// engine is actually scanning out (0 or 1). Returns -1 if the live
// surface cannot be identified. This lets a newly-active fullscreen
// client resynchronise its private back-buffer state after a handoff.
int AcquireFlip(int pid);
// Point the display plane at the given buffer. The hardware latches the
// new surface address at the next vblank (tear-free). When waitVsync is
// set, blocks until the flip has been latched. Returns the new front
// buffer index, or -1 if flipping is unavailable.
int64_t Flip(int index, bool waitVsync);
int64_t Flip(int index, bool waitVsync, int pid);
// Block until the next vblank (Gen 11+ with working IRQ only).
// Returns false on timeout or when no vblank interrupt is available.
bool WaitVblank(uint64_t timeoutMs);
// Monotonic vblank counter (0 when no vblank IRQ)
uint64_t GetVblankCount();
// Track which process performs flips so scanout can be restored to
// buffer 0 when it exits (called from the flip syscall / scheduler).
void SetFlipOwner(int pid);
// The flip operation atomically claims ownership for pid while changing
// scanout. OnProcessExit restores buffer 0 for the last flip owner.
void OnProcessExit(int pid);
// Panic-safe: force scanout back to buffer 0 so panic output is visible.
kernel/src/Drivers/Net/E1000.cpp
+89 -41
View File
@@ -14,6 +14,8 @@
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
@@ -44,6 +46,11 @@ namespace Drivers::Net::E1000 {
// Current descriptor indices
static uint32_t g_rxTail = 0;
static uint32_t g_txTail = 0;
static kcp::Spinlock g_txLock;
static std::atomic<bool> g_rxWorkPending{false};
static std::atomic<bool> g_rxProcessing{false};
static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO;
// Statistics
static uint64_t g_rxPacketCount = 0;
@@ -238,52 +245,31 @@ namespace Drivers::Net::E1000 {
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Read and clear interrupt cause
uint32_t icr = ReadReg(REG_ICR);
if (icr & ICR_LSC) {
uint32_t status = ReadReg(REG_STATUS);
bool linkUp = (status & (1 << 1)) != 0;
KernelLogStream(INFO, "E1000") << "Link status change: " << (linkUp ? "UP" : "DOWN");
// ICR is clear-on-read; no further action is required. Do not log
// here: the terminal mutex is process-context-only and an IRQ that
// preempts its owner would otherwise deadlock the CPU.
(void)ReadReg(REG_STATUS);
}
// Both receive-timer and descriptor-threshold causes mean completed
// RX descriptors may be waiting. ICR is clear-on-read, so ignoring
// RXDMT0 can strand a lone packet until unrelated traffic arrives.
if (icr & (ICR_RXT0 | ICR_RXDMT0)) {
// Process received packets
while (true) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) {
break; // No more packets
// The callback enters Ethernet/TCP/UDP and IPC waitset notification,
// which takes process-context mutexes. Running it in this hard IRQ can
// self-deadlock if the interrupt preempts a holder on the same CPU.
// Mask RX causes and let an idle-context bottom half drain the ring.
if (icr & RX_INTERRUPT_MASK) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
uint16_t length = desc.Length;
g_rxPacketCount++;
// Dispatch to the network stack callback
if (g_rxCallback != nullptr) {
g_rxCallback(g_rxBuffers[nextIdx], length);
}
// Reset descriptor for reuse
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
}
}
if (icr & (ICR_TXDW | ICR_TXQE)) {
// TX completion - nothing to do for now
}
// TX completion interrupts are intentionally disabled. Descriptor DD
// is checked by SendPacket before reuse, so an interrupt per packet
// only creates a high-vector MSI load that can starve the LAPIC timer.
}
// -------------------------------------------------------------------------
@@ -314,6 +300,11 @@ namespace Drivers::Net::E1000 {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
uint16_t pciCommand = Pci::LegacyRead16(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE);
KernelLogStream(OK, "E1000") << "Bus mastering enabled";
// Read interrupt line from PCI config
@@ -351,7 +342,7 @@ namespace Drivers::Net::E1000 {
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
g_initialized = true;
@@ -403,6 +394,11 @@ namespace Drivers::Net::E1000 {
// Enable bus mastering and memory space
Pci::EnableBusMaster(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function);
Pci::DisableInterruptDelivery(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function);
uint16_t pciCommand = Pci::LegacyRead16(e1000Dev->Bus, e1000Dev->Device,
e1000Dev->Function, (uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(e1000Dev->Bus, e1000Dev->Device, e1000Dev->Function,
(uint8_t)Pci::PCI_REG_COMMAND, pciCommand & ~Pci::PCI_CMD_INTX_DISABLE);
KernelLogStream(OK, "E1000") << "Bus mastering enabled";
@@ -457,8 +453,8 @@ namespace Drivers::Net::E1000 {
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
// Enable interrupts: RX, TX, Link Status Change
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
// Enable RX and link-change only. TX completion is polled by descriptor.
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
g_initialized = true;
@@ -473,10 +469,15 @@ namespace Drivers::Net::E1000 {
return false;
}
// Check if the current TX descriptor is available
// RX callbacks can send replies directly from interrupt context while
// userspace sends on another CPU. Serialize the shared tail/descriptor.
g_txLock.Acquire();
// Check if the current TX descriptor is available. Returning false is
// sufficient; logging here would be unsafe for the IRQ caller.
TxDescriptor& desc = g_txDescs[g_txTail];
if (!(desc.Status & TXSTA_DD)) {
KernelLogStream(WARNING, "E1000") << "TX ring full";
g_txLock.Release();
return false;
}
@@ -494,6 +495,7 @@ namespace Drivers::Net::E1000 {
WriteReg(REG_TDT, g_txTail);
g_txPacketCount++;
g_txLock.Release();
return true;
}
@@ -522,4 +524,50 @@ namespace Drivers::Net::E1000 {
g_rxCallback = callback;
}
bool HasDeferredWork() {
return g_initialized && g_rxWorkPending.load(std::memory_order_acquire);
}
void ProcessDeferredWork() {
if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return;
bool expected = false;
if (!g_rxProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) return;
g_rxWorkPending.store(false, std::memory_order_release);
// Bound one bottom-half pass to one ring revolution. Hardware may
// refill descriptors as quickly as we return them under a packet
// flood; an unbounded drain could otherwise keep an idle context in
// the network stack indefinitely. The post-pass check below queues
// another fair slice when completed descriptors remain.
uint32_t processed = 0;
while (processed < RX_DESC_COUNT) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) break;
uint16_t length = desc.Length;
g_rxPacketCount++;
if (g_rxCallback != nullptr) g_rxCallback(g_rxBuffers[nextIdx], length);
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
processed++;
}
// Re-arm RX only after every completed descriptor was returned. Check
// once more after unmasking to close the packet-arrival race window.
WriteReg(REG_IMS, RX_INTERRUPT_MASK);
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
if (g_rxDescs[nextIdx].Status & RXSTA_DD) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
g_rxProcessing.store(false, std::memory_order_release);
}
};
kernel/src/Drivers/Net/E1000.hpp
+5
View File
@@ -128,4 +128,9 @@ namespace Drivers::Net::E1000 {
// Register a callback for received packets
void SetRxCallback(RxCallback callback);
// Drain packets queued by the hard IRQ from idle/process context. The
// Ethernet/IP/IPC stack is not safe to execute inside an interrupt.
void ProcessDeferredWork();
bool HasDeferredWork();
};
kernel/src/Drivers/Net/E1000E.cpp
+65 -51
View File
@@ -14,6 +14,8 @@
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
@@ -104,6 +106,11 @@ namespace Drivers::Net::E1000E {
// Current descriptor indices
static uint32_t g_rxTail = 0;
static uint32_t g_txTail = 0;
static kcp::Spinlock g_txLock;
static std::atomic<bool> g_rxWorkPending{false};
static std::atomic<bool> g_rxProcessing{false};
static constexpr uint32_t RX_INTERRUPT_MASK = ICR_RXT0 | ICR_RXDMT0 | ICR_RXO;
// Statistics
static uint64_t g_rxPacketCount = 0;
@@ -410,7 +417,7 @@ namespace Drivers::Net::E1000E {
// MSI setup
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq); // forward declaration
static void HandleInterrupt(uint8_t irq, bool fromUser); // forward declaration
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
@@ -419,6 +426,8 @@ namespace Drivers::Net::E1000E {
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "E1000E") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -461,7 +470,7 @@ namespace Drivers::Net::E1000E {
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
uint32_t icr = ReadReg(REG_ICR);
@@ -472,42 +481,21 @@ namespace Drivers::Net::E1000E {
}
if (icr & ICR_LSC) {
uint32_t status = ReadReg(REG_STATUS);
bool linkUp = (status & (1 << 1)) != 0;
KernelLogStream(INFO, "E1000E") << "Link status change: " << (linkUp ? "UP" : "DOWN");
// Acknowledge/observe the new state without printing from IRQ
// context; KernelLogStream's terminal mutex is not IRQ-safe.
(void)ReadReg(REG_STATUS);
}
// Both receive-timer and descriptor-threshold causes mean completed
// RX descriptors may be waiting. ICR is clear-on-read, so ignoring
// RXDMT0 can strand a lone packet until unrelated traffic arrives.
if (icr & (ICR_RXT0 | ICR_RXDMT0)) {
while (true) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
if (!(desc.Status & RXSTA_DD)) {
break;
// Defer the full network/IPC stack. It takes process-context mutexes
// and cannot safely run nested under MSI/legacy interrupt delivery.
if (icr & RX_INTERRUPT_MASK) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
uint16_t length = desc.Length;
g_rxPacketCount++;
if (g_rxCallback != nullptr) {
g_rxCallback(g_rxBuffers[nextIdx], length);
}
desc.Status = 0;
desc.Length = 0;
desc.Errors = 0;
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
}
}
if (icr & (ICR_TXDW | ICR_TXQE)) {
// TX completion — nothing to do for now
}
// TX completion interrupts are intentionally disabled. Descriptor DD
// is checked by SendPacket before reuse, so an interrupt per packet
// only creates a high-vector MSI load that can starve the LAPIC timer.
}
// -------------------------------------------------------------------------
@@ -596,12 +584,17 @@ namespace Drivers::Net::E1000E {
ConfigureInterruptModeration();
if (SetupMsi(bus, device, function)) {
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
} else if (g_irqLine != 0xFF) {
KernelLogStream(INFO, "E1000E") << "Falling back to legacy IRQ " << base::dec << (uint64_t)g_irqLine;
Pci::DisableInterruptDelivery(bus, device, function);
uint16_t command = Pci::LegacyRead16(bus, device, function,
(uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(bus, device, function,
(uint8_t)Pci::PCI_REG_COMMAND, command & ~Pci::PCI_CMD_INTX_DISABLE);
Hal::RegisterIrqHandler(g_irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(g_irqLine));
WriteReg(REG_IMS, ICR_RXT0 | ICR_TXDW | ICR_TXQE | ICR_LSC | ICR_RXDMT0);
WriteReg(REG_IMS, RX_INTERRUPT_MASK | ICR_LSC);
} else {
KernelLogStream(WARNING, "E1000E") << "No MSI or legacy IRQ available, using polling mode";
g_pollingMode = true;
@@ -673,18 +666,12 @@ namespace Drivers::Net::E1000E {
// Polling: process received packets (with reentrancy guard)
// -------------------------------------------------------------------------
static bool g_polling = false;
static void PollRx() {
// Guard against reentrancy: RX callback can trigger ARP reply →
// Ethernet::Send → SendPacket → PollRx(). Two concurrent callers
// modifying g_rxTail / RDT would corrupt the descriptor ring.
if (g_polling) {
return;
}
g_polling = true;
while (true) {
// Never let a continuously refilled DMA ring monopolize the idle
// bottom half. Process at most one revolution; ProcessDeferredWork's
// final descriptor check schedules another pass if traffic remains.
uint32_t processed = 0;
while (processed < RX_DESC_COUNT) {
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
RxDescriptor& desc = g_rxDescs[nextIdx];
@@ -705,9 +692,9 @@ namespace Drivers::Net::E1000E {
g_rxTail = nextIdx;
WriteReg(REG_RDT, g_rxTail);
processed++;
}
g_polling = false;
}
bool SendPacket(const uint8_t* data, uint16_t length) {
@@ -715,9 +702,13 @@ namespace Drivers::Net::E1000E {
return false;
}
// RX processing can send replies from MSI/timer context while a
// userspace sender runs on another CPU. Protect the shared TX ring.
g_txLock.Acquire();
TxDescriptor& desc = g_txDescs[g_txTail];
if (!(desc.Status & TXSTA_DD)) {
KernelLogStream(WARNING, "E1000E") << "TX ring full";
g_txLock.Release();
return false;
}
@@ -732,6 +723,7 @@ namespace Drivers::Net::E1000E {
WriteReg(REG_TDT, g_txTail);
g_txPacketCount++;
g_txLock.Release();
return true;
}
@@ -765,10 +757,32 @@ namespace Drivers::Net::E1000E {
}
void Poll() {
if (!g_initialized) {
return;
if (g_initialized) g_rxWorkPending.store(true, std::memory_order_release);
}
bool HasDeferredWork() {
return g_initialized && g_rxWorkPending.load(std::memory_order_acquire);
}
void ProcessDeferredWork() {
if (!g_initialized || !g_rxWorkPending.load(std::memory_order_acquire)) return;
bool expected = false;
if (!g_rxProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) return;
g_rxWorkPending.store(false, std::memory_order_release);
PollRx();
if (!g_pollingMode) {
WriteReg(REG_IMS, RX_INTERRUPT_MASK);
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
if (g_rxDescs[nextIdx].Status & RXSTA_DD) {
WriteReg(REG_IMC, RX_INTERRUPT_MASK);
g_rxWorkPending.store(true, std::memory_order_release);
}
}
g_rxProcessing.store(false, std::memory_order_release);
}
};
kernel/src/Drivers/Net/E1000E.hpp
+5 -1
View File
@@ -165,7 +165,11 @@ namespace Drivers::Net::E1000E {
// Register a callback for received packets
void SetRxCallback(RxCallback callback);
// Poll for received packets (used when legacy IRQ is unavailable)
// Request an RX poll (used when hardware interrupts are unavailable).
void Poll();
// Drain queued RX descriptors outside MSI/timer interrupt context.
void ProcessDeferredWork();
bool HasDeferredWork();
};
kernel/src/Drivers/PS2/Keyboard.cpp
+1 -1
View File
@@ -150,7 +150,7 @@ namespace Drivers::PS2::Keyboard {
Kt::KernelLogStream(Kt::OK, "PS2/KB") << "Keyboard driver initialized";
}
void HandleIRQ(uint8_t irq) {
void HandleIRQ(uint8_t irq, bool) {
(void)irq;
uint8_t scancode = Io::In8(DataPort);
kernel/src/Drivers/PS2/Keyboard.hpp
+1 -1
View File
@@ -39,7 +39,7 @@ namespace Drivers::PS2::Keyboard {
void Initialize();
// Interrupt handler -- called from IRQ dispatch (EOI is sent automatically)
void HandleIRQ(uint8_t irq);
void HandleIRQ(uint8_t irq, bool fromUser);
// Public interface for consuming key events
bool IsKeyAvailable();
kernel/src/Drivers/PS2/Mouse.cpp
+1 -1
View File
@@ -109,7 +109,7 @@ namespace Drivers::PS2::Mouse {
Kt::KernelLogStream(Kt::OK, "PS2/Mouse") << "Mouse driver initialized";
}
void HandleIRQ(uint8_t irq) {
void HandleIRQ(uint8_t irq, bool) {
(void)irq;
uint8_t data = Io::In8(DataPort);
kernel/src/Drivers/PS2/Mouse.hpp
+1 -1
View File
@@ -24,7 +24,7 @@ namespace Drivers::PS2::Mouse {
void Initialize();
// Interrupt handler -- called from IRQ dispatch (EOI is sent automatically)
void HandleIRQ(uint8_t irq);
void HandleIRQ(uint8_t irq, bool fromUser);
// Public interface
MouseState GetMouseState();
kernel/src/Drivers/Storage/Ahci.cpp
+7 -80
View File
@@ -13,8 +13,6 @@
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
using namespace Kt;
@@ -230,10 +228,9 @@ namespace Drivers::Storage::Ahci {
WritePortReg(port, PORT_SERR, 0xFFFFFFFF);
WritePortReg(port, PORT_IS, 0xFFFFFFFF);
// Enable interrupts for this port
WritePortReg(port, PORT_IE,
PORT_IS_DHRS | PORT_IS_PSS | PORT_IS_DSS |
PORT_IS_SDBS | PORT_IS_TFES);
// Commands are completed by polling PxCI/PxIS. Interrupt delivery is
// unnecessary and can starve the LAPIC timer under sustained I/O.
WritePortReg(port, PORT_IE, 0);
// Power on and spin up if needed
uint32_t cmd = ReadPortReg(port, PORT_CMD);
@@ -533,65 +530,6 @@ namespace Drivers::Storage::Ahci {
return true;
}
// -------------------------------------------------------------------------
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
(void)irq;
uint32_t is = ReadReg(REG_IS);
if (is == 0) return;
// Acknowledge each port's interrupt
for (int i = 0; i < MAX_PORTS; i++) {
if (is & (1u << i)) {
uint32_t portIs = ReadPortReg(i, PORT_IS);
WritePortReg(i, PORT_IS, portIs);
}
}
// Clear global interrupt status
WriteReg(REG_IS, is);
}
// -------------------------------------------------------------------------
// MSI setup
// -------------------------------------------------------------------------
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "AHCI") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // Single message
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "AHCI") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR;
return true;
}
// -------------------------------------------------------------------------
// HBA reset
// -------------------------------------------------------------------------
@@ -659,6 +597,7 @@ namespace Drivers::Storage::Ahci {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
// BIOS/OS handoff
PerformBiosHandoff();
@@ -690,22 +629,10 @@ namespace Drivers::Storage::Ahci {
g_portsImplemented = ReadReg(REG_PI);
KernelLogStream(INFO, "AHCI") << "Ports implemented: " << base::hex << (uint64_t)g_portsImplemented;
// Set up MSI (or fall back to legacy IRQ)
bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function);
if (!hasMsi) {
uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "AHCI") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine;
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
}
}
// Enable global interrupts
// Keep controller interrupts disabled; every command path polls its
// completion and consumes/clears PxIS directly.
uint32_t ghc = ReadReg(REG_GHC);
ghc |= GHC_IE;
ghc &= ~GHC_IE;
WriteReg(REG_GHC, ghc);
// Initialize each implemented port
kernel/src/Drivers/Storage/Ahci.hpp
-5
View File
@@ -181,11 +181,6 @@ namespace Drivers::Storage::Ahci {
constexpr int MAX_PRDT_ENTRIES = 8; // Max PRDT entries per command
constexpr int SECTOR_SIZE = 512;
// MSI configuration
constexpr uint8_t MSI_IRQ = 25; // IRQ slot 25 = vector 57
constexpr uint32_t MSI_VECTOR = 57;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// =========================================================================
// Port info
// =========================================================================
kernel/src/Drivers/Storage/Nvme.cpp
+7 -64
View File
@@ -13,8 +13,6 @@
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Apic/IoApic.hpp>
#include <Timekeeping/ApicTimer.hpp>
using namespace Kt;
@@ -511,9 +509,9 @@ namespace Drivers::Storage::Nvme {
cmd.Prp1 = g_ioCqPhys;
// CDW10: bits 31:16 = queue size (0-based), bits 15:0 = queue ID
cmd.Cdw10 = ((uint32_t)(g_ioCqDepth - 1) << 16) | 1;
// CDW11: bit 0 = physically contiguous, bit 1 = interrupts enabled
// bits 31:16 = interrupt vector
cmd.Cdw11 = (1u << 0) | (1u << 1) | (0u << 16);
// Physically contiguous, with completion interrupts disabled. The
// queue is consumed synchronously by WaitCompletion().
cmd.Cdw11 = (1u << 0);
CqEntry cqe;
if (!AdminCommand(cmd, cqe)) {
@@ -549,54 +547,6 @@ namespace Drivers::Storage::Nvme {
return true;
}
// -------------------------------------------------------------------------
// Interrupt handler
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
(void)irq;
// NVMe uses polling-based completion in this driver.
// The interrupt handler just acknowledges the interrupt.
// Completions are consumed in the polling loops above.
}
// -------------------------------------------------------------------------
// MSI setup
// -------------------------------------------------------------------------
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
KernelLogStream(INFO, "NVMe") << "MSI capability not found";
return false;
}
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
Pci::LegacyWrite32(bus, dev, func, cap + 4, MSI_ADDR_BASE);
if (is64bit) {
Pci::LegacyWrite32(bus, dev, func, cap + 8, 0);
Pci::LegacyWrite16(bus, dev, func, cap + 12, MSI_VECTOR);
} else {
Pci::LegacyWrite16(bus, dev, func, cap + 8, MSI_VECTOR);
}
msgCtrl &= ~(0x70); // Single message
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
Hal::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "NVMe") << "MSI enabled: vector " << base::dec << (uint64_t)MSI_VECTOR;
return true;
}
// -------------------------------------------------------------------------
// Probe (PCI driver entry point)
// -------------------------------------------------------------------------
@@ -629,6 +579,7 @@ namespace Drivers::Storage::Nvme {
// Enable bus mastering and memory space
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
Pci::DisableInterruptDelivery(dev.Bus, dev.Device, dev.Function);
// Read capabilities
uint64_t cap = ReadReg64(REG_CAP);
@@ -653,17 +604,9 @@ namespace Drivers::Storage::Nvme {
return false;
}
// Step 2: Set up MSI
bool hasMsi = SetupMsi(dev.Bus, dev.Device, dev.Function);
if (!hasMsi) {
uint8_t irqLine = Pci::LegacyRead8(dev.Bus, dev.Device, dev.Function,
(uint8_t)Pci::PCI_REG_INTERRUPT);
if (irqLine != 0xFF) {
KernelLogStream(INFO, "NVMe") << "Using legacy IRQ " << base::dec << (uint64_t)irqLine;
Hal::RegisterIrqHandler(irqLine, HandleInterrupt);
Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irqLine));
}
}
// Step 2: keep PCI interrupt delivery disabled. Admin and I/O queues
// are polled synchronously, so completion interrupts add latency and
// can starve the lower-vector scheduler timer under sustained I/O.
// Step 3: Set up admin queues
if (!SetupAdminQueues()) {
kernel/src/Drivers/Storage/Nvme.hpp
-8
View File
@@ -158,14 +158,6 @@ namespace Drivers::Storage::Nvme {
char Model[41];
};
// =========================================================================
// MSI configuration (use a different IRQ slot than AHCI)
// =========================================================================
constexpr uint8_t MSI_IRQ = 26; // IRQ slot 26 = vector 58
constexpr uint32_t MSI_VECTOR = 58;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// =========================================================================
// Public API
// =========================================================================
kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp
+6 -1
View File
@@ -47,7 +47,12 @@ namespace Drivers::USB::Bluetooth::Avrcp {
// AVRCP notification events
constexpr uint8_t EVT_VOLUME_CHANGED = 0x0D;
static kcp::Spinlock g_notifyLock;
// AVRCP callbacks are dispatched from xHCI's deferred PollEvents path, not
// from the hard IRQ. Sending the INTERIM/CHANGED response can descend into
// the HCI and xHCI transmit rings, so keep interrupts enabled while this
// state is serialized. An IRQ-disabling spinlock here can stall the local
// timer behind an unrelated transmit/recovery operation.
static kcp::Mutex g_notifyLock;
static uint16_t g_volumeNotifyCid = 0;
static uint8_t g_volumeNotifyTransaction = 0;
kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp
+2 -1
View File
@@ -406,12 +406,13 @@ namespace Drivers::USB::Bluetooth {
// system this briefly pauses userspace, matching the old synchronous
// behavior minus the boot-path stall.
auto* cpu = Smp::GetCurrentCpuData();
bool wasReserved = cpu && cpu->reservedForKernelWork;
if (cpu) cpu->reservedForKernelWork = true;
Hci::SetFwTrace(true); // bounded per-completion event-pipe trace
CompleteInit();
Hci::DumpFwTrace(); // flush remaining records (process context)
Hci::SetFwTrace(false);
if (cpu) cpu->reservedForKernelWork = false;
if (cpu) cpu->reservedForKernelWork = wasReserved;
}
// =========================================================================
kernel/src/Drivers/USB/Bluetooth/Hci.cpp
+14 -8
View File
@@ -59,10 +59,9 @@ namespace Drivers::USB::Bluetooth::Hci {
//
// The trace is recorded into a lock-free ring and printed LATER by
// DumpFwTrace() from process context. It must NEVER KernelLogStream from
// TransferCallback: that runs nested under PollEvents in xHCI MSI (IRQ)
// context, and the terminal lock is a non-IRQ-disabling Mutex -- an IRQ
// logger interrupting a same-core holder spins forever (observed: hard
// boot freeze right after "Intel version raw", 2026-07-06).
// TransferCallback: it runs nested under PollEvents while the event-ring
// ownership guard is held. Logging there can recursively pump or contend
// with unrelated subsystem locks, so keep it a bounded data-only callback.
static std::atomic<bool> g_fwTrace{false};
static std::atomic<uint32_t> g_fwTraceCount{0};
static std::atomic<uint32_t> g_intInCompletions{0};
@@ -148,8 +147,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// guards, so one core can be sending a media packet while another sends a
// signaling reply; without this lock the two race on g_aclTxSlot and on
// the bulk OUT TRB ring's enqueue/cycle state, garbling packets on air.
// IRQ-disabling spinlock so the holder also can't be preempted mid-send.
static kcp::Spinlock g_aclTxLock;
// These paths run only from deferred/process context. Keep interrupts
// enabled: endpoint recovery also takes this lock and can wait for bounded
// xHCI commands, which must never happen below an IRQ-disabling spinlock.
static kcp::Mutex g_aclTxLock;
// HCI command DMA buffer (separate from ACL to avoid conflicts)
static uint8_t* g_cmdDmaBuf = nullptr;
@@ -559,8 +560,8 @@ namespace Drivers::USB::Bluetooth::Hci {
// Bounded bring-up trace: one RING RECORD per interrupt-IN
// completion while the firmware phase runs; DumpFwTrace() prints
// them later from process context. No logging here -- this runs
// in xHCI MSI (IRQ) context (see the deadlock note at the ring).
// them later from top-level process/idle context. No logging here:
// this callback is nested under the xHCI event-ring owner.
if (g_fwTrace.load(std::memory_order_relaxed)) {
uint32_t n = g_fwTraceCount.fetch_add(1, std::memory_order_relaxed);
if (n < FW_TRACE_CAP) {
@@ -2074,6 +2075,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// all TRBs queued behind it. Free those DMA slots and remove only
// those never-delivered packets from the controller-credit count;
// earlier successful OUT transfers still await their NOCP events.
// Serialize the dequeue rewrite with SendAcl/firmware bulk enqueue.
// Without this, another CPU can publish a TRB between Reset Endpoint
// and Set TR Dequeue and have recovery silently discard or split it.
g_aclTxLock.Acquire();
uint32_t discarded = AclTxInFlight();
KernelLogStream(WARNING, "BT-HCI")
<< "Recovering HCI ACL transmit pipe; discarding "
@@ -2086,6 +2091,7 @@ namespace Drivers::USB::Bluetooth::Hci {
- (int32_t)discarded;
if (remaining < 0)
g_aclPendingCount.store(0, std::memory_order_release);
g_aclTxLock.Release();
}
// Asynchronous HCI events are processed at top level. Besides avoiding
kernel/src/Drivers/USB/HidMouse.cpp
+30 -10
View File
@@ -172,18 +172,23 @@ namespace Drivers::USB::HidMouse {
// -------------------------------------------------------------------------
static int32_t ExtractSigned(const uint8_t* data, uint16_t bitOffset, uint8_t bitSize) {
int32_t value = 0;
uint32_t value = 0;
for (uint8_t i = 0; i < bitSize; i++) {
uint16_t byteIdx = (bitOffset + i) / 8;
uint8_t bitIdx = (bitOffset + i) % 8;
if (data[byteIdx] & (1 << bitIdx))
value |= (1 << i);
value |= (1u << i);
}
// Sign extend
if (bitSize < 32 && (value & (1 << (bitSize - 1)))) {
value |= ~((1 << bitSize) - 1);
if (bitSize < 32 && (value & (1u << (bitSize - 1)))) {
value |= ~((1u << bitSize) - 1);
}
return value;
return (int32_t)value;
}
static bool FieldFits(uint32_t bitOffset, uint32_t bitSize, uint32_t reportBits) {
return bitSize > 0 && bitSize <= 32 && bitOffset <= reportBits
&& bitSize <= reportBits - bitOffset;
}
// -------------------------------------------------------------------------
@@ -222,18 +227,30 @@ namespace Drivers::USB::HidMouse {
// Skip report ID byte if present
uint16_t byteOffset = g_Format.hasReportId ? 8 : 0;
if (g_Format.hasReportId && data[0] != g_Format.reportId) return;
uint32_t reportBits = (uint32_t)length * 8;
uint32_t buttonBits = g_Format.buttonCount < 8 ? g_Format.buttonCount : 8;
uint32_t buttonOffset = byteOffset + g_Format.buttonBitOffset;
uint32_t xOffset = byteOffset + g_Format.xBitOffset;
uint32_t yOffset = byteOffset + g_Format.yBitOffset;
if ((buttonBits != 0 && !FieldFits(buttonOffset, buttonBits, reportBits))
|| !FieldFits(xOffset, g_Format.xBitSize, reportBits)
|| !FieldFits(yOffset, g_Format.yBitSize, reportBits)) {
return;
}
// Extract buttons
uint8_t buttons = 0;
for (uint8_t i = 0; i < g_Format.buttonCount && i < 8; i++) {
uint16_t bit = byteOffset + g_Format.buttonBitOffset + i;
for (uint8_t i = 0; i < buttonBits; i++) {
uint16_t bit = (uint16_t)(buttonOffset + i);
if (data[bit / 8] & (1 << (bit % 8)))
buttons |= (1 << i);
}
// Extract X, Y
int32_t rawX = ExtractSigned(data, byteOffset + g_Format.xBitOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, byteOffset + g_Format.yBitOffset, g_Format.yBitSize);
int32_t rawX = ExtractSigned(data, (uint16_t)xOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, (uint16_t)yOffset, g_Format.yBitSize);
// Clamp to int8_t range for InjectMouseReport
if (rawX > 127) rawX = 127;
@@ -244,7 +261,10 @@ namespace Drivers::USB::HidMouse {
// Extract scroll wheel
int8_t scroll = 0;
if (g_Format.scrollBitSize > 0) {
int32_t rawScroll = ExtractSigned(data, byteOffset + g_Format.scrollBitOffset, g_Format.scrollBitSize);
uint32_t scrollOffset = byteOffset + g_Format.scrollBitOffset;
if (!FieldFits(scrollOffset, g_Format.scrollBitSize, reportBits)) return;
int32_t rawScroll = ExtractSigned(data, (uint16_t)scrollOffset,
g_Format.scrollBitSize);
if (rawScroll > 127) rawScroll = 127;
if (rawScroll < -128) rawScroll = -128;
scroll = (int8_t)rawScroll;
kernel/src/Drivers/USB/Xhci.cpp
+200 -53
View File
@@ -54,9 +54,30 @@ namespace Drivers::USB::Xhci {
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static volatile bool g_hotplugPending[MAX_PORTS] = {};
static volatile bool g_deferredWorkPending = false;
static std::atomic<bool> g_hotplugPending[MAX_PORTS] = {};
static std::atomic<bool> g_deferredWorkPending{false};
static std::atomic<bool> g_diagnosticWorkPending{false};
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
// diagnostic payload atomically and print it from ProcessDeferredWork().
static std::atomic<uint32_t> g_transferErrorsPending{0};
static std::atomic<uint32_t> g_transferErrorReports{0};
static std::atomic<uint32_t> g_lastErrorSlot{0};
static std::atomic<uint32_t> g_lastErrorEp{0};
static std::atomic<uint32_t> g_lastErrorCc{0};
static std::atomic<uint32_t> g_stormsPending{0};
static std::atomic<uint32_t> g_stormReports{0};
static std::atomic<uint32_t> g_lastStormProcessed{0};
static std::atomic<uint32_t> g_lastStormPortEvents{0};
static std::atomic<uint32_t> g_lastStormTransferEvents{0};
static std::atomic<uint32_t> g_lastStormType{0};
static std::atomic<uint32_t> g_lastStormSlot{0};
static std::atomic<uint32_t> g_lastStormEp{0};
static std::atomic<uint32_t> g_lastStormCc{0};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
@@ -78,6 +99,12 @@ namespace Drivers::USB::Xhci {
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// The controller exposes one command ring and one command-completion
// mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer
// recovery can originate on different CPUs, so the whole submit/wait
// transaction must be single-owner. This is a process/idle-context lock;
// SendCommand is never called from a transfer-event callback.
static kcp::Mutex g_commandLock;
// Event ring
static TRB* g_evtRing = nullptr;
@@ -125,12 +152,12 @@ namespace Drivers::USB::Xhci {
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from the MSI handler (BSP) AND from
// syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the
// app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set
// on a volatile is a cross-core TOCTOU: two cores both pass the check,
// both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync,
// and completions get double-delivered (duplicate ACL packets) or lost
// volatile bool: PollEvents runs from syscall-context pumps (A2DP
// WriteAudio/StartSource on whatever core the app runs on) and from idle
// cores (ProcessDeferredWork). A check-then-set on a volatile is a
// cross-core TOCTOU: two cores both pass the check, both drain the shared
// event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get
// double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
@@ -293,7 +320,7 @@ namespace Drivers::USB::Xhci {
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
@@ -389,10 +416,13 @@ namespace Drivers::USB::Xhci {
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -448,33 +478,37 @@ namespace Drivers::USB::Xhci {
// PollEvents - process event ring
// -------------------------------------------------------------------------
void PollEvents() {
// PollEvents runs both from the synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and from the xHCI MSI handler. On a
// single core the IRQ can preempt a poll loop mid-drain; if both advance
// g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the
// cycle-bit check can start matching stale entries forever -> the boot
// freezes (observed wedging the Bluetooth firmware download at ~635 KB,
// where the dying device floods the event ring). Guard against re-entry:
// the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and
// the active poll loop drains these events itself. g_pollActive is also
// read by InPollContext() so command submitters (ControlTransfer) can
// tell they are nested and must fire-and-forget instead of waiting.
enum class PollResult : uint8_t {
Drained,
Busy,
BudgetExhausted,
};
static PollResult PollEventsBounded(uint32_t maxEvents) {
// PollEvents runs from synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and idle bottom halves on different
// CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring
// tracking desyncs and the cycle-bit check can start matching stale
// entries forever -> the boot freezes (observed wedging the Bluetooth
// firmware download at ~635 KB, where the dying device floods the event
// ring). Guard against cross-core re-entry. g_pollActive is also read by
// InPollContext() so command submitters reached from a callback can tell
// they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return; // another context is draining; it will reap our events
return PollResult::Busy; // another context is draining
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound the work per call so a flooding/wedged device can never spin
// here forever; the outer wall-clock timeouts then fire instead of
// freezing. 4x the ring size is far above any legitimate burst.
constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4;
// Bound each pass so a flooding/wedged device cannot monopolize its
// caller. Synchronous poll loops use a generous batch; the timer uses
// a much smaller recovery slice after the hard IRQ masks the source.
if (maxEvents == 0) maxEvents = 1;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < MAX_EVENTS_PER_CALL) {
while (processed < maxEvents) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
@@ -506,8 +540,8 @@ namespace Drivers::USB::Xhci {
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1] = true;
g_deferredWorkPending = true;
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
break;
}
@@ -616,9 +650,16 @@ namespace Drivers::USB::Xhci {
nullptr, 0, completionCode);
}
} else {
KernelLogStream(WARNING, "xHCI") << "Transfer error on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci
<< " cc=" << (uint64_t)completionCode;
// Never print from this path: PollEvents may be
// running in the MSI handler. Defer a rate-limited
// diagnostic to idle/process context instead.
if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) {
g_lastErrorSlot.store(slotId, std::memory_order_relaxed);
g_lastErrorEp.store(epDci, std::memory_order_relaxed);
g_lastErrorCc.store(completionCode, std::memory_order_relaxed);
g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
@@ -651,37 +692,70 @@ namespace Drivers::USB::Xhci {
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded -- surface the dominant
// event source (rate-limited) so a wedge is diagnosable, not silent.
if (processed >= MAX_EVENTS_PER_CALL) {
static uint32_t stormLogs = 0;
if (stormLogs < 8) {
stormLogs++;
KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed
<< "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts
<< " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot
<< " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")";
// A full batch means the ring is being flooded. Record the dominant
// source for a deferred, rate-limited process-context diagnostic.
bool exhausted = processed >= maxEvents;
if (exhausted) {
if (g_stormReports.load(std::memory_order_relaxed) < 8) {
g_lastStormProcessed.store(processed, std::memory_order_relaxed);
g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed);
g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed);
g_lastStormType.store(lastType, std::memory_order_relaxed);
g_lastStormSlot.store(lastSlot, std::memory_order_relaxed);
g_lastStormEp.store(lastEp, std::memory_order_relaxed);
g_lastStormCc.store(lastCC, std::memory_order_relaxed);
g_stormsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
return exhausted ? PollResult::BudgetExhausted : PollResult::Drained;
}
void PollEvents() {
PollResult result = PollEventsBounded(EVT_RING_SIZE * 4);
if (result == PollResult::Drained &&
g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
// The hard IRQ leaves the interrupter masked. A synchronous
// process-context poll may drain the ring before an idle bottom
// half sees it, so finish the handoff here as well. Verify once
// more while masked: on another CPU, the IRQ may have acknowledged
// IMAN.IP for an event posted just after our first empty check. In
// that race, blindly enabling IE would strand the event because
// the IRQ already cleared its pending bit.
PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4);
if (verify != PollResult::Drained) {
g_irqDrainPending.store(true, std::memory_order_release);
return;
}
// If hardware queues an event after this masked verification,
// IMAN.IP remains asserted and enabling IE generates a fresh MSI.
WriteRt(IR0_IMAN, IMAN_IE);
}
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Clear IMAN.IP and ensure IE stays enabled
WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE);
PollEvents();
// Keep the hard IRQ strictly bounded. PollEvents dispatches HID,
// Bluetooth, and SDR callbacks; those callbacks can take subsystem
// locks, wake scheduler objects, re-arm endpoints, and copy sizeable
// buffers. Running them here lets a USB burst monopolize this
// higher-priority vector ahead of the LAPIC timer and can deadlock if
// an interrupted context owns one of those locks. Acknowledge and mask
// the source, then let an idle/process-context poll drain the ring.
WriteRt(IR0_IMAN, IMAN_IP);
g_irqDrainPending.store(true, std::memory_order_release);
}
// -------------------------------------------------------------------------
@@ -689,6 +763,8 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
g_commandLock.Acquire();
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
@@ -728,7 +804,9 @@ namespace Drivers::USB::Xhci {
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
return g_cmdCompletionCode;
uint32_t result = g_cmdCompletionCode;
g_commandLock.Release();
return result;
}
// Small delay
for (int j = 0; j < 100; j++) {
@@ -736,6 +814,9 @@ namespace Drivers::USB::Xhci {
}
}
// Do not hold the command-ring lock while taking the terminal mutex.
// A later caller may attempt recovery even if this command timed out.
g_commandLock.Release();
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
@@ -1304,7 +1385,55 @@ namespace Drivers::USB::Xhci {
}
bool HasDeferredWork() {
return g_initialized && g_deferredWorkPending;
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
void ServiceDeferredIrqEvents(uint32_t budget) {
if (!g_initialized ||
!g_irqDrainPending.load(std::memory_order_acquire)) return;
PollResult result = PollEventsBounded(budget);
if (result != PollResult::Drained) return;
// Clear before re-enabling. If hardware queued an event after the ring
// appeared empty, IMAN.IP remains asserted and enabling IE generates a
// fresh MSI; no event can be lost in this handoff.
if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
WriteRt(IR0_IMAN, IMAN_IE);
}
}
static void PrintDeferredDiagnostics() {
// Clear the wakeup before consuming counters. A producer racing after
// this store sets it again, so deferred work cannot be lost.
g_diagnosticWorkPending.store(false, std::memory_order_release);
uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel);
if (errors != 0 &&
g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors
<< " transfer error(s); last slot="
<< (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed);
}
uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel);
if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms
<< " event storm(s); last batch="
<< (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed)
<< " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed)
<< " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed)
<< " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed)
<< " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")";
}
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
@@ -1339,7 +1468,19 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return;
if (!g_initialized) return;
// The hard IRQ only acknowledges and masks the interrupter. Drain a
// bounded slice here, outside interrupt context, so callbacks may use
// their normal locks without blocking the timer or scheduler. A busy
// process that is synchronously waiting for USB calls PollEvents()
// itself, which performs the same unmask handoff when the ring is dry.
ServiceDeferredIrqEvents(64);
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
g_deferredWorkPending.load(std::memory_order_acquire);
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
@@ -1348,11 +1489,17 @@ namespace Drivers::USB::Xhci {
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
g_deferredWorkPending = false;
if (haveDiagnostics) PrintDeferredDiagnostics();
if (!haveHotplug) {
g_hotplugProcessing.store(false, std::memory_order_release);
return;
}
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port]) continue;
g_hotplugPending[port] = false;
if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
kernel/src/Drivers/USB/Xhci.hpp
+11 -4
View File
@@ -20,9 +20,11 @@ namespace Drivers::USB::Xhci {
constexpr uint32_t EVT_RING_SIZE = 64;
constexpr uint32_t XFER_RING_SIZE = 32;
// MSI configuration (E1000E uses IRQ 24/vector 56, we use 25/57)
constexpr uint8_t MSI_IRQ = 25;
constexpr uint32_t MSI_VECTOR = 57;
// MSI configuration. Keep xHCI distinct from AHCI's IRQ 25/vector 57;
// the IRQ dispatcher has one handler per slot, so sharing would replace
// one driver's handler and eventually strand USB or storage completions.
constexpr uint8_t MSI_IRQ = 29;
constexpr uint32_t MSI_VECTOR = 61;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// PCI class/subclass/progif for xHCI
@@ -299,6 +301,10 @@ namespace Drivers::USB::Xhci {
// Deferred hot-plug processing (call outside interrupt context)
void ProcessDeferredWork();
// Drain a bounded event batch after the hard IRQ masked the interrupter.
// Call only from process/idle context; this dispatches class callbacks.
void ServiceDeferredIrqEvents(uint32_t budget);
// Send a command on the command ring, wait for completion.
// Returns completion code.
uint32_t SendCommand(const TRB& trb);
@@ -367,7 +373,8 @@ namespace Drivers::USB::Xhci {
// Access device info
UsbDeviceInfo* GetDevice(uint8_t slotId);
// Poll event ring (called from interrupt handler or during init)
// Poll the event ring from process/idle context. The hard interrupt only
// acknowledges and masks the source, then queues deferred draining.
void PollEvents();
};
kernel/src/Hal/Apic/Apic.cpp
+13 -8
View File
@@ -16,6 +16,10 @@ namespace Hal {
static volatile uint32_t* g_apicBase = nullptr;
static constexpr uint32_t ICR_DELIVERY_STATUS = (1 << 12);
static constexpr uint32_t ICR_LEVEL_ASSERT = (1 << 14);
// Fixed-IPIs normally clear delivery status within a few MMIO reads.
// Keep the exceptional path short because scheduler kicks can originate
// in an input IRQ where a long MMIO poll would itself stall the BSP.
static constexpr uint32_t ICR_WAIT_SPINS = 10000;
static inline uint64_t ReadMSR(uint32_t msr) {
uint32_t lo, hi;
@@ -115,19 +119,20 @@ namespace Hal {
WriteRegister(REG_EOI, 0);
}
void SendFixedIpi(uint32_t apicId, uint8_t vector) {
if (g_apicBase == nullptr) return;
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
static bool WaitForIcrIdle() {
for (uint32_t i = 0; i < ICR_WAIT_SPINS; i++) {
if (!(ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS)) return true;
asm volatile("pause");
}
return false;
}
bool SendFixedIpi(uint32_t apicId, uint8_t vector) {
if (g_apicBase == nullptr || !WaitForIcrIdle()) return false;
WriteRegister(REG_ICR_HIGH, apicId << 24);
WriteRegister(REG_ICR_LOW, (uint32_t)vector | ICR_LEVEL_ASSERT);
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
asm volatile("pause");
}
return WaitForIcrIdle();
}
uint32_t GetId() {
kernel/src/Hal/Apic/Apic.hpp
+2 -1
View File
@@ -44,7 +44,8 @@ namespace Hal {
void Reinitialize();
void SendEOI();
void SendFixedIpi(uint32_t apicId, uint8_t vector);
// Returns false if the local APIC delivery-status bit fails to clear.
bool SendFixedIpi(uint32_t apicId, uint8_t vector);
uint32_t GetId();
uint32_t ReadRegister(uint32_t reg);
kernel/src/Hal/Apic/Interrupts.cpp
+3 -2
View File
@@ -38,10 +38,11 @@ namespace Hal {
<< " IRQ stubs (vectors " << (uint64_t)IRQ_VECTOR_BASE << "-"
<< (uint64_t)(IRQ_VECTOR_BASE + IRQ_COUNT - 1) << ")";
}
};
// C linkage dispatch function called from assembly stubs
extern "C" void HalIrqDispatch(uint64_t irqNumber) {
extern "C" void HalIrqDispatch(uint64_t irqNumber, uint64_t savedCs) {
// Send EOI BEFORE calling the handler. The handler may context-switch
// (via Tick -> Schedule -> SchedContextSwitch) and never return here.
// If EOI is deferred until after the handler, the LAPIC timer vector
@@ -52,6 +53,6 @@ extern "C" void HalIrqDispatch(uint64_t irqNumber) {
Hal::LocalApic::SendEOI();
if (irqNumber < Hal::IRQ_COUNT && Hal::g_irqHandlers[irqNumber] != nullptr) {
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber);
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber, (savedCs & 3) == 3);
}
}
kernel/src/Hal/Apic/Interrupts.hpp
+5 -1
View File
@@ -9,7 +9,10 @@
namespace Hal {
// IRQ handler function type. The parameter is the IRQ number (0-47).
using IrqHandler = void(*)(uint8_t irq);
// `fromUser` is derived directly from the saved interrupt-frame CS. Do not
// cache it in CpuData: an IRQ handler may context-switch and later resume
// on another CPU, making per-CPU saved-origin state stale.
using IrqHandler = void(*)(uint8_t irq, bool fromUser);
// Number of IRQ slots supported (0-23: legacy ISA via IOAPIC,
// 24-45: MSI, 46-47: kernel IPIs)
@@ -37,4 +40,5 @@ namespace Hal {
// Install IRQ stubs into the IDT and set up the dispatch table
void InitializeIrqHandlers();
};
kernel/src/Hal/Apic/IsrStubs.asm
+10 -2
View File
@@ -50,11 +50,19 @@ IrqCommon:
; Pass IRQ number as first argument (rdi)
mov rdi, rax
; Pass the saved CS as the second argument. Fifteen saved GPRs occupy
; 120 bytes, and CS is eight bytes after the interrupt-frame RIP.
mov rsi, [rsp + 128]
; Align stack to 16 bytes before call (we pushed 15 registers x 8 = 120 bytes
; + return address 8 = 128, which is 16-byte aligned, so we're good)
; An IRQ that interrupted ring 0 can arrive at any kernel stack depth, so
; the saved frame is not guaranteed to satisfy the SysV call alignment.
; RBX is already saved above and is callee-saved, making it a safe place to
; retain the exact frame pointer while aligning down for the C++ call.
mov rbx, rsp
and rsp, -16
cld
call HalIrqDispatch
mov rsp, rbx
pop r15
pop r14
kernel/src/Hal/CpuPower.cpp
+18 -2
View File
@@ -70,6 +70,8 @@ namespace Hal {
static constexpr uint32_t GovernorIntervalMs = 500;
static constexpr uint8_t PassiveStepDown = 2; // ratio units (~200 MHz)
static constexpr uint8_t HotStepDown = 8; // ratio units (~800 MHz)
static constexpr uint32_t ThermalLogThrottle = 0x100;
static constexpr uint32_t ThermalLogRecovered = 0x200;
// ============================================================
// Detected features and shared policy state
@@ -106,6 +108,7 @@ namespace Hal {
static bool g_throttling = false;
static std::atomic<uint8_t> g_lastTempC{0};
static std::atomic<uint32_t> g_pendingThermalLog{0};
static std::atomic<uint32_t> g_effMHz{0};
static uint64_t g_lastGovernorMs = 0;
static uint64_t g_lastAperf = 0;
@@ -339,10 +342,23 @@ namespace Hal {
uint32_t epoch = g_policyEpoch.fetch_add(1, std::memory_order_acq_rel) + 1;
ApplyHwpRequestOnThisCpu(epoch); // BSP applies immediately; APs on tick
if (g_throttling && !wasThrottling) {
// Keep policy transitions separate from terminal output so this
// maintenance step never enters process-context logging locks.
if (g_throttling && !wasThrottling)
g_pendingThermalLog.store(ThermalLogThrottle | temp,
std::memory_order_release);
else if (!g_throttling && wasThrottling)
g_pendingThermalLog.store(ThermalLogRecovered | temp,
std::memory_order_release);
}
void ServiceDeferredDiagnostics() {
uint32_t pending = g_pendingThermalLog.exchange(0, std::memory_order_acq_rel);
uint8_t temp = (uint8_t)(pending & 0xFF);
if (pending & ThermalLogThrottle) {
KernelLogStream(WARNING, "CpuPower") << "Package at " << base::dec
<< (uint64_t)temp << "C - thermal throttle engaged";
} else if (!g_throttling && wasThrottling) {
} else if (pending & ThermalLogRecovered) {
KernelLogStream(OK, "CpuPower") << "Package cooled to " << base::dec
<< (uint64_t)temp << "C - thermal throttle released";
}
kernel/src/Hal/CpuPower.hpp
+4
View File
@@ -43,6 +43,10 @@ namespace Hal {
// be called from every maintenance pass.
void ThermalTick(uint64_t nowMs);
// Print thermal-transition diagnostics from idle/process context.
// ThermalTick only records transitions for this function to report.
void ServiceDeferredDiagnostics();
// Redo the BSP MSR setup after S3 wake (HWP enable and POWER_CTL do
// not survive suspend).
void ReapplyAfterWake();
kernel/src/Hal/IDT.cpp
+12 -1
View File
@@ -13,6 +13,7 @@
#include <Memory/PageFrameAllocator.hpp>
#include <Sched/Scheduler.hpp>
#include <Sched/CrashReport.hpp>
#include <Hal/SmpBoot.hpp>
#include <Timekeeping/ApicTimer.hpp>
namespace Hal {
@@ -93,6 +94,14 @@ namespace Hal {
// If the fault originated in user-mode (ring 3), kill the process
// instead of panicking the entire system.
if (fromUser && Sched::GetCurrentPid() >= 0) {
// Interrupt gates arrive with IF clear. Full process teardown can
// wait on device completions, sibling CPUs, and wall-clock-bounded
// recovery paths, so leaving IF clear here can stop the BSP clock
// and its local device IRQs indefinitely. GS is already the kernel
// per-CPU base, and timer/IPI scheduling refuses to switch away
// from a ring-0 frame, so nested hardware IRQs are safe now.
asm volatile("sti" ::: "memory");
auto* proc = Sched::GetCurrentProcessPtr();
auto* regs = GetExceptionRegs(i, frame);
Kt::KernelLogStream(Kt::ERROR, "Exception")
@@ -183,6 +192,7 @@ namespace Hal {
{
bool fromUser = (frame->CS & 3) == 3;
if (fromUser) asm volatile("swapgs");
auto* cpu = Smp::TryGetCurrentCpuData();
uint64_t cr2;
asm volatile("mov %%cr2, %0" : "=r"(cr2));
@@ -190,7 +200,8 @@ namespace Hal {
// Bit 0 of the error code: 0 = non-present page. Covers both user
// pushes past the mapped stack and kernel accesses to not-yet-grown
// user stack buffers passed into syscalls.
if ((errorCode & 1) == 0 && Sched::GetCurrentPid() >= 0
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
&& Sched::GetCurrentPid() >= 0
&& Sched::TryGrowUserStack(cr2)) {
if (fromUser) asm volatile("swapgs");
return;
kernel/src/Hal/SmpBoot.cpp
+3
View File
@@ -218,6 +218,9 @@ namespace Smp {
for (;;) {
// Pick up thermal-governor frequency changes decided by the BSP.
Hal::CpuPower::ApplyPolicyIfChanged();
// Any idle core may run bounded USB/NIC bottom halves. Preserve
// the AP's ACPI/MWAIT idle selection after servicing them.
Timekeeping::ServiceDeferredWork();
Hal::CpuIdle::Wait(10, cpu->hasMwait, &s_idleMonitor);
}
}
kernel/src/Hal/SmpBoot.hpp
+16
View File
@@ -49,6 +49,11 @@ namespace Smp {
// Other CPUs schedule normally while this is held.
volatile bool reservedForKernelWork = false;
// Set by the BSP timer when an IRQ bottom half needs process-safe
// execution even though runnable user processes keep this CPU busy.
// Schedule() consumes it by yielding once to the idle kernel context.
bool kernelWorkPending = false;
// Per-CPU GDT and TSS (APs use these; BSP uses globals)
Hal::BasicGDT cpuGdt __attribute__((aligned(16)));
Hal::TSS64 cpuTss __attribute__((aligned(16)));
@@ -61,6 +66,17 @@ namespace Smp {
return ptr;
}
// Exception paths may run before InitBsp() has installed a GS base. Do
// not read gs:0 in that state: address zero would fault recursively and
// turn the original, diagnosable exception into a triple fault.
inline CpuData* TryGetCurrentCpuData() {
uint32_t low;
uint32_t high;
asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(0xC0000101u));
uint64_t base = ((uint64_t)high << 32) | low;
return (CpuData*)base;
}
// Get CPU data by index
CpuData* GetCpuData(int index);
kernel/src/Hal/SyscallEntry.asm
+10
View File
@@ -8,6 +8,7 @@
section .text
extern SyscallDispatch
extern SchedExitIfKilled
; ====================================================================
; Per-CPU data offsets (must match CpuData in SmpBoot.hpp)
@@ -52,6 +53,15 @@ SyscallEntry:
mov rdi, rsp ; arg1 = pointer to SyscallFrame
call SyscallDispatch ; returns int64_t in rax
; A remote kill observed while this syscall was executing must not tear
; the process down from the timer IRQ on top of held syscall locks. This
; safe point runs after dispatch returned and all such locks are released.
sub rsp, 16
mov [rsp], rax
call SchedExitIfKilled
mov rax, [rsp]
add rsp, 16
cli ; disable interrupts for sysret
pop r15
kernel/src/Ipc/Ipc.cpp
+456 -227
View File
File diff suppressed because it is too large Load Diff
kernel/src/Ipc/Ipc.hpp
+22 -2
View File
@@ -63,6 +63,23 @@ namespace Ipc {
Object* object;
};
// Pins the object referenced by a handle for the lifetime of the snapshot.
// Process threads share one handle table, so a raw table lookup is not
// sufficient: another thread may close and destroy the object immediately
// after the lookup returns.
struct HandleSnapshot {
HandleType type = HandleType::None;
Object* object = nullptr;
uint32_t rights = 0;
HandleSnapshot() = default;
~HandleSnapshot();
HandleSnapshot(const HandleSnapshot&) = delete;
HandleSnapshot& operator=(const HandleSnapshot&) = delete;
bool Capture(int slot, int handle);
};
struct WaitsetReady {
int32_t index;
uint32_t signals;
@@ -78,7 +95,6 @@ namespace Ipc {
int CloseHandle(int handle);
int DupHandle(int handle);
bool SnapshotHandleForSlot(int slot, int handle, HandleType& type, Object*& object, uint32_t& rights);
uint32_t GetHandleSignalsForSlot(int slot, int handle);
Stream* CreateStream(uint32_t capacity = DefaultStreamCapacity);
@@ -139,7 +155,6 @@ namespace Ipc {
uint64_t& outVa, bool reserveGrowthRange = true);
int UnmapSurfaceForPid(Surface* surface, int pid, uint64_t pml4Phys);
ProcessObject* GetProcessObject(int pid);
int OpenProcessHandle(int pid);
void ProcessStartedInSlot(int slot, int pid);
void ProcessExitedInSlot(int slot, int pid);
@@ -156,6 +171,11 @@ namespace Ipc {
int WaitsetWaitHandle(int waitsetHandle, WaitsetReady* outReady, uint64_t timeoutMs);
void NotifyObjectChanged(Object* object);
// Invalidate a user range on every CPU currently running the address
// space. Call this after removing PTEs and before releasing their frames.
void ShootdownUserRange(uint64_t pml4Phys, uint64_t startVa, uint32_t pages);
// Safely remove ordinary PFA-backed user mappings and release their frames.
void UnmapAndFreeUserRange(uint64_t pml4Phys, uint64_t startVa, uint64_t pages);
void CleanupProcessSlot(int slot, int pid, uint64_t pml4Phys);
}
kernel/src/Memory/PageFrameAllocator.cpp
+25 -3
View File
@@ -70,6 +70,15 @@ namespace Memory {
}
void* PageFrameAllocator::ReallocConsecutive(void* ptr, int n) {
// This primitive grows a single-page allocation into a contiguous
// span; it is not a sized-free API. Guard zero/negative requests so a
// caller can never carve a zero-byte block at the end of the free
// pool and memcpy beyond it. Multi-page owners must use Free(ptr, n).
if (n <= 0) {
if (ptr != nullptr) Free(ptr);
return nullptr;
}
Lock.Acquire();
// Search the free list for a single contiguous region >= n pages.
@@ -118,6 +127,19 @@ namespace Memory {
Lock.Acquire();
uint64_t addr = (uint64_t)ptr;
uint64_t poolStart = g_section.address;
uint64_t poolEnd = poolStart + g_section.size;
// Reject malformed frees before they can splice arbitrary memory into
// the allocator's linked list. Avoid overflowing addr + size while
// validating the upper bound.
if ((addr & 0xFFFULL) || (size & 0xFFFULL) || addr < poolStart
|| poolEnd < poolStart || addr > poolEnd || size > poolEnd - addr) {
Lock.Release();
Kt::KernelLogStream(Kt::WARNING, "PFA")
<< "Invalid free range at " << addr << " size " << size << ", ignoring";
return;
}
// Walk to find the sorted insertion point: prev < addr < current
Page* prev = &head;
@@ -126,9 +148,9 @@ namespace Memory {
while (current != nullptr && (uint64_t)current < addr) {
// Double-free check: addr falls within an existing free block
if (addr < (uint64_t)current + current->size) {
Lock.Release();
Kt::KernelLogStream(Kt::WARNING, "PFA")
<< "Double-free detected at " << addr << ", ignoring";
Lock.Release();
return;
}
prev = current;
@@ -137,9 +159,9 @@ namespace Memory {
// Double-free check: exact match with next block
if (current != nullptr && (uint64_t)current == addr) {
Lock.Release();
Kt::KernelLogStream(Kt::WARNING, "PFA")
<< "Double-free detected at " << addr << ", ignoring";
Lock.Release();
return;
}
@@ -148,9 +170,9 @@ namespace Memory {
// freeing a multi-page span that overlaps the start of an existing
// block would silently corrupt the free list.
if (current != nullptr && addr + size > (uint64_t)current) {
Lock.Release();
Kt::KernelLogStream(Kt::WARNING, "PFA")
<< "Overlapping free at " << addr << " size " << size << ", ignoring";
Lock.Release();
return;
}
kernel/src/Memory/Paging.cpp
+31 -29
View File
@@ -7,6 +7,26 @@
namespace Memory::VMM {
Paging* g_paging = nullptr;
static constexpr uint64_t LeafAddressMask = 0x000FFFFFFFFFF000ULL;
static constexpr uint64_t LeafPresent = 1ULL << 0;
static constexpr uint64_t LeafWritable = 1ULL << 1;
static constexpr uint64_t LeafUser = 1ULL << 2;
static constexpr uint64_t LeafWriteThrough = 1ULL << 3;
static constexpr uint64_t LeafCacheDisabled = 1ULL << 4;
static inline void SetLeafPte(PageTableEntry* entry, uint64_t physicalAddress,
bool user, bool writeThrough, bool cacheDisabled) {
uint64_t flags = LeafPresent | LeafWritable;
if (user) flags |= LeafUser;
if (writeThrough) flags |= LeafWriteThrough;
if (cacheDisabled) flags |= LeafCacheDisabled;
// Replace the complete PTE in one aligned store. Updating individual
// bitfields left old PWT/PCD/PAT/accessed state behind when a virtual
// address was reused for a mapping with a different cache policy.
*(volatile uint64_t*)entry = (physicalAddress & LeafAddressMask) | flags;
}
// Protects user page table modifications from concurrent SMP access
static kcp::Mutex pagingLock;
@@ -143,10 +163,7 @@ namespace Memory::VMM {
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, false, false, false);
}
void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
@@ -162,11 +179,7 @@ namespace Memory::VMM {
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->WriteThrough = true; // PWT=1, PCD=0 → PAT entry 1 = WC
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, false, true, false);
}
void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
@@ -182,12 +195,7 @@ namespace Memory::VMM {
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->CacheDisabled = true;
pageEntry->WriteThrough = true;
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, false, true, true);
}
void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
@@ -203,11 +211,7 @@ namespace Memory::VMM {
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&PML1->entries[virtualAddressObj.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->Supervisor = 1; // User-accessible
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, true, false, false);
}
std::uint64_t Paging::CreateUserPML4() {
@@ -262,10 +266,7 @@ namespace Memory::VMM {
if (!pml1) { pagingLock.Release(); return false; }
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->Supervisor = 1;
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, true, false, false);
pagingLock.Release();
return true;
}
@@ -305,11 +306,7 @@ namespace Memory::VMM {
if (!pml1) { pagingLock.Release(); return false; }
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
pageEntry->Present = true;
pageEntry->Writable = true;
pageEntry->Supervisor = 1;
pageEntry->WriteThrough = true; // PWT=1, PCD=0 -> PAT entry 1 = WC
pageEntry->Address = physicalAddress >> 12;
SetLeafPte(pageEntry, physicalAddress, true, true, false);
pagingLock.Release();
return true;
}
@@ -456,9 +453,11 @@ namespace Memory::VMM {
PageTable* pml4Virt = (PageTable*)HHDM(pml4);
PageTableEntry* pml4_entry = &pml4Virt->entries[virtualAddressObj.GetL4Index()];
if (!pml4_entry->Present) return 0;
PageTable* pml3 = (PageTable*)HHDM((pml4_entry->Address & kPhysAddrMask) << 12);
PageTableEntry* pml3_entry = &pml3->entries[virtualAddressObj.GetL3Index()];
if (!pml3_entry->Present) return 0;
// 1GB large page at PML3 level
if (pml3_entry->LargerPages) {
@@ -468,6 +467,7 @@ namespace Memory::VMM {
PageTable* pml2 = (PageTable*)HHDM((pml3_entry->Address & kPhysAddrMask) << 12);
PageTableEntry* pml2_entry = &pml2->entries[virtualAddressObj.GetL2Index()];
if (!pml2_entry->Present) return 0;
// 2MB large page at PML2 level
if (pml2_entry->LargerPages) {
@@ -479,10 +479,12 @@ namespace Memory::VMM {
if (use40BitL1 == true) {
PageTableEntry40Bit* pml1_entry = (PageTableEntry40Bit*)&pml1->entries[virtualAddressObj.GetPageIndex()];
if (!pml1_entry->Present) return 0;
return (uint64_t)pml1_entry->Address << 12;
}
PageTableEntry* pml1_entry = &pml1->entries[virtualAddressObj.GetPageIndex()];
if (!pml1_entry->Present) return 0;
return (uint64_t)(pml1_entry->Address & kPhysAddrMask) << 12;
}
kernel/src/Net/Icmp.cpp
-7
View File
@@ -62,13 +62,6 @@ namespace Net::Icmp {
}
if (hdr->Type == TYPE_ECHO_REQUEST && hdr->Code == 0) {
KernelLogStream(INFO, "Net") << "ICMP echo request from "
<< base::dec
<< (uint64_t)(srcIp & 0xFF) << "."
<< (uint64_t)((srcIp >> 8) & 0xFF) << "."
<< (uint64_t)((srcIp >> 16) & 0xFF) << "."
<< (uint64_t)((srcIp >> 24) & 0xFF);
// Build echo reply -- same payload, different type
uint8_t reply[1500];
if (length > sizeof(reply)) {
kernel/src/Net/Tcp.cpp
+16 -8
View File
@@ -465,6 +465,7 @@ namespace Net::Tcp {
// Block until a SYN arrives
while (true) {
uint64_t listenerWake = Sched::ObserveObjectWake(listener);
listener->Lock.Acquire();
if (listener->PendingCount > 0) {
Connection::PendingSyn pending = listener->Pending[listener->PendingHead];
@@ -524,13 +525,14 @@ namespace Net::Tcp {
// Wait for ACK to complete the handshake
uint64_t deadline = Timekeeping::GetMilliseconds() + 5000;
while (Timekeeping::GetMilliseconds() < deadline) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
if (conn->CurrentState == State::Established) {
return conn;
}
uint64_t now = Timekeeping::GetMilliseconds();
uint64_t waitMs = (deadline > now) ? (deadline - now) : 0;
if (waitMs == 0) break;
Sched::BlockOnObject(conn, waitMs);
Sched::BlockOnObjectSince(conn, waitMs, connWake);
}
// Timed out waiting for ACK
@@ -540,7 +542,7 @@ namespace Net::Tcp {
return nullptr;
}
listener->Lock.Release();
Sched::BlockOnObject(listener, 0);
Sched::BlockOnObjectSince(listener, 0, listenerWake);
}
}
@@ -589,13 +591,14 @@ namespace Net::Tcp {
for (int attempt = 0; attempt < MAX_RETRANSMITS; attempt++) {
uint64_t deadline = Timekeeping::GetMilliseconds() + 1000;
while (Timekeeping::GetMilliseconds() < deadline) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
if (conn->CurrentState == State::Established) {
return conn;
}
uint64_t now = Timekeeping::GetMilliseconds();
uint64_t waitMs = (deadline > now) ? (deadline - now) : 0;
if (waitMs == 0) break;
Sched::BlockOnObject(conn, waitMs);
Sched::BlockOnObjectSince(conn, waitMs, connWake);
}
if (conn->CurrentState == State::SynSent) {
@@ -636,6 +639,7 @@ namespace Net::Tcp {
// writers so concurrent threads cannot overwrite its in-flight
// retransmit buffer or allocate overlapping sequence numbers.
while (true) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
uint64_t flags;
asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory");
conn->Lock.Acquire();
@@ -652,7 +656,7 @@ namespace Net::Tcp {
}
conn->Lock.Release();
asm volatile("push %0; popfq" :: "r"(flags) : "memory");
Sched::BlockOnObject(conn, 50);
Sched::BlockOnObjectSince(conn, 50, connWake);
}
auto finishSend = [&]() {
@@ -712,6 +716,7 @@ namespace Net::Tcp {
asm volatile("push %0; popfq" :: "r"(flags) : "memory");
while (true) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory");
conn->Lock.Acquire();
@@ -775,7 +780,7 @@ namespace Net::Tcp {
conn->Lock.Release();
asm volatile("push %0; popfq" :: "r"(flags) : "memory");
Sched::BlockOnObject(conn, waitMs);
Sched::BlockOnObjectSince(conn, waitMs, connWake);
}
}
@@ -790,6 +795,7 @@ namespace Net::Tcp {
// Block until data is available or connection is closing
while (true) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
uint64_t flags;
asm volatile("pushfq; pop %0; cli" : "=r"(flags) :: "memory");
conn->Lock.Acquire();
@@ -829,7 +835,7 @@ namespace Net::Tcp {
conn->Lock.Release();
asm volatile("push %0; popfq" :: "r"(flags) : "memory");
Sched::BlockOnObject(conn, 0);
Sched::BlockOnObjectSince(conn, 0, connWake);
}
}
@@ -905,11 +911,12 @@ namespace Net::Tcp {
// Wait for close to complete
for (int i = 0; i < 100; i++) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
if (conn->CurrentState == State::TimeWait ||
conn->CurrentState == State::Closed) {
break;
}
Sched::BlockOnObject(conn, 50);
Sched::BlockOnObjectSince(conn, 50, connWake);
}
Sched::WakeObjectWaiters(conn);
Ipc::NotifyTcpConnectionChanged(conn);
@@ -926,10 +933,11 @@ namespace Net::Tcp {
// Wait for final ACK
for (int i = 0; i < 100; i++) {
uint64_t connWake = Sched::ObserveObjectWake(conn);
if (conn->CurrentState == State::Closed) {
break;
}
Sched::BlockOnObject(conn, 50);
Sched::BlockOnObjectSince(conn, 50, connWake);
}
Sched::WakeObjectWaiters(conn);
Ipc::NotifyTcpConnectionChanged(conn);
kernel/src/Pci/Pci.cpp
+78 -14
View File
@@ -10,6 +10,7 @@
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Io/IoPort.hpp>
#include <CppLib/Spinlock.hpp>
using namespace Kt;
@@ -19,6 +20,14 @@ namespace Pci {
static constexpr uint16_t ConfigAddressPort = 0xCF8;
static constexpr uint16_t ConfigDataPort = 0xCFC;
// CF8/CFC is one machine-wide address/data latch, not a per-CPU interface.
// Keep the address write and matching data access indivisible: otherwise a
// second CPU can replace CF8 between them and make us read or write an
// unrelated device's register. Besides bogus probing, a crossed runtime
// command/MSI write can disable interrupts or bus mastering underneath an
// active GPU, USB controller, NIC, or HDA controller.
static kcp::Spinlock g_legacyConfigLock;
// PCI config space register offsets
static constexpr uint16_t RegVendorId = 0x00;
static constexpr uint16_t RegDeviceId = 0x02;
@@ -163,11 +172,26 @@ namespace Pci {
| (offset & 0xFC);
}
uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
static uint32_t LegacyRead32Unlocked(uint8_t bus, uint8_t device,
uint8_t function, uint8_t offset) {
Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort);
return Io::In32(ConfigDataPort);
}
static void LegacyWrite32Unlocked(uint8_t bus, uint8_t device,
uint8_t function, uint8_t offset,
uint32_t value) {
Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort);
Io::Out32(value, ConfigDataPort);
}
uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
g_legacyConfigLock.Acquire();
uint32_t value = LegacyRead32Unlocked(bus, device, function, offset);
g_legacyConfigLock.Release();
return value;
}
uint16_t LegacyRead16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
uint32_t val = LegacyRead32(bus, device, function, offset & 0xFC);
return (uint16_t)(val >> ((offset & 2) * 8));
@@ -179,30 +203,29 @@ namespace Pci {
}
void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
Io::Out32(LegacyBuildAddress(bus, device, function, offset), ConfigAddressPort);
Io::Out32(value, ConfigDataPort);
g_legacyConfigLock.Acquire();
LegacyWrite32Unlocked(bus, device, function, offset, value);
g_legacyConfigLock.Release();
}
void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) {
uint32_t addr = LegacyBuildAddress(bus, device, function, offset & 0xFC);
Io::Out32(addr, ConfigAddressPort);
uint32_t tmp = Io::In32(ConfigDataPort);
g_legacyConfigLock.Acquire();
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
int shift = (offset & 2) * 8;
tmp &= ~(0xFFFF << shift);
tmp |= ((uint32_t)value << shift);
Io::Out32(addr, ConfigAddressPort);
Io::Out32(tmp, ConfigDataPort);
LegacyWrite32Unlocked(bus, device, function, offset & 0xFC, tmp);
g_legacyConfigLock.Release();
}
void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) {
uint32_t addr = LegacyBuildAddress(bus, device, function, offset & 0xFC);
Io::Out32(addr, ConfigAddressPort);
uint32_t tmp = Io::In32(ConfigDataPort);
g_legacyConfigLock.Acquire();
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
int shift = (offset & 3) * 8;
tmp &= ~(0xFF << shift);
tmp |= ((uint32_t)value << shift);
Io::Out32(addr, ConfigAddressPort);
Io::Out32(tmp, ConfigDataPort);
LegacyWrite32Unlocked(bus, device, function, offset & 0xFC, tmp);
g_legacyConfigLock.Release();
}
// -------------------------------------------------------------------------
@@ -237,8 +260,17 @@ namespace Pci {
// Read Capabilities Pointer (offset 0x34), mask to dword-aligned
uint8_t offset = ReadConfig8(bus, device, function, 0x34) & 0xFC;
// Walk the linked list (cap_id @ +0, next_ptr @ +1)
// Walk the conventional 256-byte capability list (cap_id @ +0,
// next_ptr @ +1). Firmware owns these pointers, so reject offsets
// below the capability area and cycles instead of allowing a corrupt
// list to spin forever during device initialization or resume.
uint64_t visited = 0;
while (offset != 0) {
if (offset < 0x40) return 0;
uint64_t bit = 1ULL << (offset >> 2);
if (visited & bit) return 0;
visited |= bit;
uint8_t id = ReadConfig8(bus, device, function, offset);
if (id == capId) {
return offset;
@@ -249,6 +281,38 @@ namespace Pci {
return 0;
}
bool DisableInterruptDelivery(uint8_t bus, uint8_t device, uint8_t function) {
uint16_t command = LegacyRead16(bus, device, function,
(uint8_t)PCI_REG_COMMAND);
if (!(command & PCI_CMD_INTX_DISABLE)) {
LegacyWrite16(bus, device, function, (uint8_t)PCI_REG_COMMAND,
command | PCI_CMD_INTX_DISABLE);
}
bool disabledMessageDelivery = false;
uint8_t msi = FindCapability(bus, device, function, PCI_CAP_MSI);
if (msi != 0) {
uint16_t control = LegacyRead16(bus, device, function, msi + 2);
if (control & 1) {
LegacyWrite16(bus, device, function, msi + 2, control & ~1u);
disabledMessageDelivery = true;
}
}
uint8_t msix = FindCapability(bus, device, function, PCI_CAP_MSIX);
if (msix != 0) {
uint16_t control = LegacyRead16(bus, device, function, msix + 2);
if (control & (1u << 15)) {
// Function-mask before clearing MSI-X Enable so no vector can
// escape during the ownership transition.
uint16_t disabled = (control | (1u << 14)) & ~(1u << 15);
LegacyWrite16(bus, device, function, msix + 2, disabled);
disabledMessageDelivery = true;
}
}
return disabledMessageDelivery;
}
// -------------------------------------------------------------------------
// PCI class code names
// -------------------------------------------------------------------------
kernel/src/Pci/Pci.hpp
+5
View File
@@ -62,6 +62,7 @@ namespace Pci {
// PCI capability IDs
constexpr uint8_t PCI_CAP_MSI = 0x05;
constexpr uint8_t PCI_CAP_MSIX = 0x11;
// Walk the PCI capability linked list for a given device.
// Returns the config-space offset of the capability, or 0 if not found.
@@ -96,6 +97,10 @@ namespace Pci {
// Enable memory space access and bus mastering in PCI command register.
void EnableBusMaster(uint8_t bus, uint8_t device, uint8_t function);
// Disable legacy INTx plus any enabled MSI/MSI-X capability. Returns true
// when message-signalled delivery had been enabled and was turned off.
bool DisableInterruptDelivery(uint8_t bus, uint8_t device, uint8_t function);
// =========================================================================
// PCI driver matching
// =========================================================================
kernel/src/Sched/Scheduler.cpp
+379 -40
View File
@@ -11,6 +11,7 @@
#include <Memory/HHDM.hpp>
#include <Libraries/Memory.hpp>
#include <Libraries/String.hpp>
#include <Common/Panic.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
@@ -71,21 +72,105 @@ namespace Sched {
// It is held ACROSS context switches to prevent the race where
// another CPU picks up a process whose RSP hasn't been saved yet.
// The resumed process releases it.
static kcp::Spinlock schedLock;
// schedLock is a baton: the context that acquires it switches stacks and
// the context being resumed releases it. A Spinlock's ordinary shared
// savedFlags field cannot represent that -- it would restore the switching
// context's IF state into the resumed stack. Keep the flags with each saved
// process/idle context instead, so timer frames resume with IF=0 and syscall
// frames resume with their original IF=1 even after CPU migration.
class SchedulerSpinlock {
kcp::Spinlock lock;
uint64_t processFlags[MaxProcesses] = {};
uint64_t idleFlags[Smp::MaxCPUs] = {};
public:
void Acquire() {
uint64_t flags = lock.AcquireIrqSave();
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
processFlags[cpu->currentSlot] = flags;
} else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
idleFlags[cpu->cpuIndex] = flags;
}
}
void Release() {
uint64_t flags = 0; // Fresh startup remains IF=0 until JumpToUserMode's IRETQ.
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
flags = processFlags[cpu->currentSlot];
} else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
flags = idleFlags[cpu->cpuIndex];
}
lock.ReleaseIrqRestore(flags);
}
void ReleaseForStartup() {
lock.ReleaseIrqNoRestore();
}
};
static SchedulerSpinlock schedLock;
// Approximate count of Ready processes. Incremented/decremented
// under schedLock. Idle CPUs check this to avoid scanning all 256
// process slots on every timer tick.
static volatile int readyCount = 0;
// Updated while schedLock is held, but sampled locklessly by idle CPUs and
// reschedule-IPI paths. Atomic loads make those wakeup decisions coherent
// across cores instead of relying on volatile's compiler-only semantics.
static std::atomic<int> readyCount{0};
// Per-object wake generations close the readiness-check/enrollment gap.
// Each bucket packs [63:32] = a tag identifying the last object to signal
// it and [31:0] = a signal counter, published together in one atomic word.
// Publishing both halves atomically lets a waiter distinguish "exactly one
// signal landed here and it carried someone else's tag" (safe to keep
// blocking) from "something ambiguous happened" (must treat as a wake).
// Without the tag, an unrelated hot object sharing the bucket forces the
// waiter to re-run its condition on every signal. Tag collisions only cost
// a spurious wake; they can never lose one, which is the only direction
// that would reintroduce the lost-wakeup bug this table exists to prevent.
static constexpr uint32_t ObjectWakeBuckets = 1024;
static std::atomic<uint64_t> objectWakeEpochs[ObjectWakeBuckets] = {};
static uint32_t ObjectWakeBucket(void* object) {
uint64_t value = (uint64_t)object;
return (uint32_t)(((value >> 4) ^ (value >> 12)) & (ObjectWakeBuckets - 1));
}
static uint32_t ObjectWakeTag(void* object) {
uint64_t value = (uint64_t)object;
value ^= value >> 32;
value *= 0x9E3779B97F4A7C15ULL;
return (uint32_t)(value >> 32);
}
// The idle loop runs in the kernel PML4
static uint64_t GetKernelCR3() {
return (uint64_t)Memory::VMM::g_paging->PML4;
}
static void RescheduleIpiHandler(uint8_t) {
static void RescheduleIpiHandler(uint8_t, bool interruptedUser) {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr || cpu->currentSlot >= 0 || readyCount <= 0) {
if (cpu == nullptr) {
return;
}
// Process teardown uses this IPI to promptly stop sibling threads on
// other CPUs. A user-mode sibling can exit immediately. If the IPI
// interrupted a syscall, merely expire its slice: teardown on that
// suspended kernel stack could reacquire a mutex it already owns.
if (cpu->currentSlot >= 0 &&
processTable[cpu->currentSlot].killPending.load(std::memory_order_acquire)) {
if (interruptedUser) {
ExitProcess();
return;
}
processTable[cpu->currentSlot].sliceRemaining = 0;
return;
}
if (cpu->currentSlot >= 0 || readyCount <= 0 || cpu->reservedForKernelWork) {
return;
}
@@ -101,10 +186,10 @@ namespace Sched {
if (target == nullptr || !target->started) return false;
if (target->cpuIndex == sourceCpuIndex) return false;
if (target->currentSlot >= 0) return false;
if (target->reservedForKernelWork) return false;
Hal::LocalApic::SendFixedIpi(target->lapicId,
return Hal::LocalApic::SendFixedIpi(target->lapicId,
Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
return true;
};
if (tryKick(Smp::GetCpuData(0))) {
@@ -163,7 +248,9 @@ namespace Sched {
// The schedLock is held (acquired by the switching-from CPU's Schedule).
static void ProcessStartup() {
// Release the schedLock that the switching-from CPU held
schedLock.Release();
// without consulting this brand-new slot's (intentionally empty)
// saved interrupt state. JumpToUserMode's IRETQ enables interrupts.
schedLock.ReleaseForStartup();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
@@ -526,10 +613,11 @@ namespace Sched {
// Give the new thread its own TLS block when the image has one.
// We are executing inside the process's address space, so the
// freshly mapped pages and the template are directly addressable.
// The block is process-heap memory; it is reclaimed with the rest
// of the user half at process teardown (a thread that exits before
// its process leaks its block until then - acceptable for now).
// The block uses process-heap virtual space. Its physical pages are
// reclaimed when the sibling exits (or if creation fails).
uint64_t threadFsBase = 0;
uint64_t threadTlsBase = 0;
uint64_t threadTlsPages = 0;
Process& primary = processTable[primarySlot_];
if (primary.tlsMemSize > 0) {
uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign;
@@ -537,12 +625,15 @@ namespace Sched {
uint64_t total = S + 16;
uint64_t numPages = (total + 0xFFF) / 0x1000;
schedLock.Acquire();
uint64_t base = primary.heapNext;
primary.heapNext += numPages * 0x1000;
schedLock.Release();
uint64_t base = 0;
if (!ReserveUserHeapRange(primarySlot_, numPages * 0x1000, base)) {
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Thread TLS exceeds user heap range";
return -1;
}
bool ok = true;
uint64_t mappedPages = 0;
for (uint64_t p = 0; p < numPages; p++) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
@@ -552,11 +643,14 @@ namespace Sched {
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(sharedPml4, physAddr,
base + p * 0x1000)) {
Memory::g_pfa->Free(page);
ok = false;
break;
}
mappedPages++;
}
if (!ok) {
Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Thread TLS allocation failed";
return -1;
@@ -567,6 +661,8 @@ namespace Sched {
primary.tlsFileSize);
*(uint64_t*)tp = tp; /* ABI self-pointer */
threadFsBase = tp;
threadTlsBase = base;
threadTlsPages = numPages;
}
// We do not write to the user stack from kernel mode (that would
@@ -578,7 +674,11 @@ namespace Sched {
// Allocate kernel stack.
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
if (stackMem == nullptr) return -1;
if (stackMem == nullptr) {
if (threadTlsPages != 0)
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
return -1;
}
memset(stackMem, 0, StackSize);
uint8_t* kernelStackBase = (uint8_t*)stackMem;
uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize;
@@ -598,6 +698,8 @@ namespace Sched {
if (slot < 0) {
schedLock.Release();
Memory::g_pfa->Free(stackMem, StackPages);
if (threadTlsPages != 0)
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
return -1;
}
@@ -663,6 +765,54 @@ namespace Sched {
return tid;
}
bool ReserveUserHeapRange(int primarySlot_, uint64_t size, uint64_t& outVa) {
outVa = 0;
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
(size & 0xFFFULL) != 0) {
return false;
}
schedLock.Acquire();
Process& primary = processTable[primarySlot_];
uint64_t base = primary.heapNext;
bool validOwner = primary.primarySlot == primarySlot_ &&
primary.state != ProcessState::Free &&
primary.state != ProcessState::Terminated;
bool fits = validOwner && base >= UserHeapBase &&
base <= UserHeapLimit && size <= UserHeapLimit - base;
if (fits) {
primary.heapNext = base + size;
outVa = base;
}
schedLock.Release();
return fits;
}
static void FreeSiblingThreadTls(Process& thr) {
int primarySlot_ = thr.primarySlot;
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
primarySlot_ == (int)(&thr - processTable) || thr.fsBase == 0) {
return;
}
Process& primary = processTable[primarySlot_];
if (primary.tlsMemSize == 0 || primary.pml4Phys == 0) {
thr.fsBase = 0;
return;
}
uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign;
uint64_t blockSize = (primary.tlsMemSize + align - 1) & ~(align - 1);
if (blockSize > thr.fsBase || blockSize > UINT64_MAX - 16) {
Panic("Invalid sibling TLS metadata during thread teardown", nullptr);
}
uint64_t base = thr.fsBase - blockSize;
uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000;
thr.fsBase = 0;
Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
}
// Switch away from a slot we have just marked non-runnable while holding
// schedLock. Mirrors SwitchAwayFromBlockedCurrentLocked but does NOT
// assume the caller will be resumed -- used by ExitCurrentThread.
@@ -714,10 +864,20 @@ namespace Sched {
__builtin_unreachable();
}
thr.killPending = false;
bool processExitKill =
thr.killPending.exchange(false, std::memory_order_acq_rel);
thr.startPending = false;
thr.exitCode = exitCode;
// TLS belongs only to this sibling. Reclaim it before publishing the
// Terminated state; otherwise a long-lived multithreaded GUI process
// leaks these pages until the entire process exits. When the primary
// is already coordinating full teardown, defer this to its final
// sibling sweep: the primary may be waiting from an IF=0 timer frame
// and therefore cannot acknowledge our TLB-shootdown IPI here.
if (!processExitKill)
FreeSiblingThreadTls(thr);
schedLock.Acquire();
thr.state = ProcessState::Terminated;
thr.runningOnCpu = -1;
@@ -847,10 +1007,16 @@ namespace Sched {
// New processes release it in ProcessStartup().
// ====================================================================
// Reclaim terminated process slots. Called from BSP's Tick only,
// NOT from every Schedule() call on every CPU. This avoids holding
// schedLock (with interrupts disabled) during PFA::Free on the
// hot scheduling path.
// BSP maintenance that may allocate/free memory or touch policy MSRs must
// run from the idle kernel context, never from the timer interrupt. The
// pending flag is set by process teardown on any CPU and periodically by
// the BSP timer. It remains set until an idle maintenance pass consumes it.
static std::atomic<bool> bspMaintenancePending{false};
static uint64_t nextBspPolicyTick = 0; // BSP timer only
// Reclaim terminated process slots from BSP idle context. This avoids
// holding schedLock (with interrupts disabled) during PFA::Free on the
// hot scheduling path or inside a hard interrupt.
static void ReclaimTerminated() {
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
@@ -880,7 +1046,17 @@ namespace Sched {
return readyCount > 0;
}
void RunBspMaintenance() {
void RequestKernelWork() {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0) {
cpu->kernelWorkPending = true;
}
}
// Bounded, IRQ-safe portion of BSP maintenance. Keeping deadline expiry
// on every 1 ms BSP tick prevents timed input/object waits from depending
// on the BSP becoming idle under sustained desktop load.
static void WakeExpiredProcesses() {
bool wokeProcesses = false;
schedLock.Acquire();
@@ -902,12 +1078,19 @@ namespace Sched {
if (wokeProcesses) {
KickOneIdleCpu(0);
}
}
void RunBspMaintenance() {
// Clear before scanning. A producer that races this pass sets the flag
// again and therefore cannot have its request acknowledged by mistake.
if (!bspMaintenancePending.exchange(false, std::memory_order_acq_rel)) {
return;
}
ReclaimTerminated();
// Thermal governor step (rate-limited internally to 500 ms).
// Runs here because BSP maintenance is reached from both the idle
// loop and the timer tick, so it keeps running under full load.
// The timer requests this idle-context pass even under full load.
Hal::CpuPower::ThermalTick(Timekeeping::GetMilliseconds());
}
@@ -933,10 +1116,17 @@ namespace Sched {
schedLock.Acquire();
// Find the next Ready process (round-robin from after current slot)
// Normally select the next Ready process. A pending bottom half gets
// one deliberate trip through the idle kernel context first; that
// context drains bounded work and immediately schedules user work
// again. This avoids both hard-IRQ callbacks and bottom-half starvation
// when there is always at least one runnable desktop process.
int next = -1;
bool yieldToKernel = cpu->currentSlot >= 0 && cpu->kernelWorkPending;
if (yieldToKernel) cpu->kernelWorkPending = false;
int start = (cpu->currentSlot >= 0) ? cpu->currentSlot + 1 : 0;
if (!yieldToKernel) {
for (int i = 0; i < MaxProcesses; i++) {
int idx = (start + i) % MaxProcesses;
if (processTable[idx].state == ProcessState::Ready) {
@@ -944,10 +1134,23 @@ namespace Sched {
break;
}
}
}
if (next < 0) {
// No ready processes. If we were running one, return to idle.
// No other process is ready. A normal time-slice expiry must keep
// the sole Running process on this CPU: bouncing it through idle
// can suspend a syscall while it owns a process-context Mutex, then
// let ServiceDeferredWork spin on that same mutex with the idle
// context reserved against scheduling. That is a permanent
// self-deadlock. Only an explicit, ring-3-safe bottom-half request
// is allowed to yield a running process to idle.
if (cpu->currentSlot >= 0) {
if (!yieldToKernel) {
processTable[cpu->currentSlot].sliceRemaining = TimeSliceMs;
schedLock.Release();
return;
}
int oldSlot = cpu->currentSlot;
processTable[oldSlot].state = ProcessState::Ready;
readyCount++;
@@ -1019,7 +1222,7 @@ namespace Sched {
schedLock.Release();
}
void Tick(uint32_t elapsedMs) {
void Tick(uint32_t elapsedMs, bool interruptedUser) {
auto* cpu = Smp::GetCurrentCpuData();
// Pick up thermal-governor frequency changes (HWP requests are
@@ -1027,9 +1230,21 @@ namespace Sched {
// the governor bumped the policy epoch since our last tick.
Hal::CpuPower::ApplyPolicyIfChanged();
// BSP: wake sleeping processes and reclaim terminated slots
// BSP: expire timed waits in the bounded IRQ-safe pass. Queue heavier
// reclamation/policy work for the idle kernel context.
if (cpu->cpuIndex == 0) {
RunBspMaintenance();
WakeExpiredProcesses();
uint64_t now = Timekeeping::GetTicks();
if (nextBspPolicyTick == 0 || now >= nextBspPolicyTick) {
nextBspPolicyTick = now + 500;
bspMaintenancePending.store(true, std::memory_order_release);
}
if (interruptedUser &&
bspMaintenancePending.load(std::memory_order_acquire)) {
RequestKernelWork();
}
}
int slot = cpu->currentSlot;
@@ -1052,13 +1267,22 @@ namespace Sched {
processTable[slot].cpuTimeMs += elapsedMs;
// Check if another CPU requested this process be killed.
// We are on the CPU running it, so ExitProcess is safe here.
if (cpu->kernelWorkPending) {
processTable[slot].sliceRemaining = 0;
}
// Check if another CPU requested this process be killed. Full teardown
// takes process-context mutexes. It is safe here only when the timer
// interrupted ring 3; if it interrupted a syscall, teardown could try
// to reacquire a lock already held by this same suspended kernel stack.
// In that case force a reschedule and finish at syscall-return instead.
if (processTable[slot].killPending) {
processTable[slot].killPending = false;
if (interruptedUser) {
ExitProcess();
return;
}
processTable[slot].sliceRemaining = 0;
}
if (processTable[slot].sliceRemaining > elapsedMs) {
processTable[slot].sliceRemaining -= elapsedMs;
@@ -1066,7 +1290,13 @@ namespace Sched {
processTable[slot].sliceRemaining = 0;
}
if (processTable[slot].sliceRemaining == 0) {
// Never context-switch away from an arbitrary kernel stack. Syscalls
// routinely own non-IRQ-disabling subsystem mutexes; suspending that
// stack and dispatching a process that wants the same mutex leaves the
// new process spinning in ring 0, where the timer deliberately cannot
// preempt it to resume the owner. Preserve the expired slice and take
// the switch on the first timer interrupt after SYSRET reaches ring 3.
if (processTable[slot].sliceRemaining == 0 && interruptedUser) {
Schedule();
}
}
@@ -1080,6 +1310,14 @@ namespace Sched {
return processTable[primary].pid;
}
void ExitIfKilled() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu ? cpu->currentSlot : -1;
if (slot >= 0 && processTable[slot].killPending) {
ExitProcess();
}
}
int GetCurrentTid() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
@@ -1173,8 +1411,17 @@ namespace Sched {
// terminated immediately; siblings running on other CPUs are tagged
// killPending and we spin until their tick handler routes them
// through ExitCurrentThread.
// Several device syscalls have bounded waits up to five seconds. Give
// an interrupted kernel-mode sibling room to finish that recovery
// path before declaring it wedged.
static constexpr uint64_t SiblingDrainTimeoutMs = 10000;
static constexpr uint32_t SiblingDrainMaxPasses = 262144;
uint64_t siblingDrainDeadline = Timekeeping::GetTicks() + SiblingDrainTimeoutMs;
uint32_t siblingDrainPasses = 0;
for (;;) {
bool anyRunning = false;
bool targetCpus[Smp::MaxCPUs] = {};
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (i == primarySlot_) continue;
@@ -1184,7 +1431,10 @@ namespace Sched {
continue;
}
if (st == ProcessState::Running) {
processTable[i].killPending = true;
processTable[i].killPending.store(true, std::memory_order_release);
int targetCpu = processTable[i].runningOnCpu;
if (targetCpu >= 0 && targetCpu < Smp::MaxCPUs)
targetCpus[targetCpu] = true;
anyRunning = true;
continue;
}
@@ -1211,6 +1461,30 @@ namespace Sched {
}
schedLock.Release();
if (!anyRunning) break;
// Do not depend solely on a remote CPU's next periodic timer.
// Laptop firmware/IRQ pathologies can delay that tick, and the
// old unbounded wait then trapped the exiting CPU forever.
if ((siblingDrainPasses & 0x3FF) == 0) {
for (int i = 0; i < Smp::GetCpuCount(); i++) {
if (!targetCpus[i] || i == cpu->cpuIndex) continue;
Smp::CpuData* target = Smp::GetCpuData(i);
if (target != nullptr && target->started) {
(void)Hal::LocalApic::SendFixedIpi(target->lapicId,
Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
}
}
}
siblingDrainPasses++;
if (siblingDrainPasses >= SiblingDrainMaxPasses ||
Timekeeping::GetTicks() >= siblingDrainDeadline) {
// It is unsafe to free the shared address space while another
// CPU may still execute in it. A visible panic is preferable
// to silent lockup or use-after-free corruption.
Panic("Timed out draining sibling threads during process exit", nullptr);
}
// Wait for the sibling CPU(s) to observe killPending on their
// next timer tick. Briefly busy-wait; this path is rare.
for (int spin = 0; spin < 1024; spin++) {
@@ -1224,6 +1498,10 @@ namespace Sched {
if (i == primarySlot_) continue;
if (processTable[i].primarySlot != primarySlot_) continue;
if (processTable[i].state != ProcessState::Terminated) continue;
// Ready/blocked siblings terminated inline above never ran their
// ordinary ExitCurrentThread cleanup. Running siblings normally
// arrive here with fsBase already cleared, making this a no-op.
FreeSiblingThreadTls(processTable[i]);
void* stackBase = (void*)processTable[i].stackBase;
processTable[i].stackBase = 0;
processTable[i].pml4Phys = 0;
@@ -1280,6 +1558,7 @@ namespace Sched {
proc.state = ProcessState::Terminated;
proc.runningOnCpu = -1;
proc.reapReady = true;
bspMaintenancePending.store(true, std::memory_order_release);
// Publish the exit code before waking waiters so SYS_WAITPID
// observes it the moment it unblocks.
@@ -1429,6 +1708,8 @@ namespace Sched {
}
void BlockOnPid(int pid) {
ExitIfKilled();
// If the target is already dead, return immediately
if (!IsAlive(pid)) return;
@@ -1464,11 +1745,14 @@ namespace Sched {
processTable[slot].sleepUntilTick = 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
void BlockForSleep(uint64_t ms) {
if (ms == 0) return;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return;
@@ -1481,11 +1765,14 @@ namespace Sched {
processTable[slot].sleepUntilTick = Timekeeping::GetTicks() + ms;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
void BlockOnObject(void* object, uint64_t timeoutMs) {
if (object == nullptr) return;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return;
@@ -1499,12 +1786,15 @@ namespace Sched {
: 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
bool BlockOnObjectIf(void* object, uint64_t timeoutMs,
bool (*shouldBlock)(void*), void* context) {
if (object == nullptr || shouldBlock == nullptr) return false;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr) return false;
@@ -1526,12 +1816,58 @@ namespace Sched {
: 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
return true;
}
uint64_t ObserveObjectWake(void* object) {
if (object == nullptr) return 0;
return objectWakeEpochs[ObjectWakeBucket(object)].load(std::memory_order_acquire);
}
struct ObjectWakeObservation {
void* object;
uint64_t epoch;
};
static bool ObjectWakeIsUnchanged(void* opaque) {
auto* observation = (ObjectWakeObservation*)opaque;
uint64_t now = ObserveObjectWake(observation->object);
uint64_t was = observation->epoch;
if (now == was) return true; // nothing signalled this bucket at all
// Exactly one signal landed here since the observation, and it carried
// a different object's tag, so it cannot have been ours: the caller may
// still block. Any larger or wrapped delta is ambiguous -- more than
// one object may have signalled -- and must be reported as a wake so a
// real edge is never dropped.
if ((uint32_t)now == (uint32_t)was + 1 &&
(uint32_t)(now >> 32) != ObjectWakeTag(observation->object)) {
return true;
}
return false;
}
bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake) {
ObjectWakeObservation observation{object, observedWake};
return BlockOnObjectIf(object, timeoutMs, ObjectWakeIsUnchanged, &observation);
}
void WakeObjectWaiters(void* object) {
if (object == nullptr) return;
// Publish this object's tag and the bumped counter as one word. A CAS
// loop (rather than fetch_add) keeps the pair consistent and, unlike a
// plain store, cannot drop a concurrent waker's increment on the same
// bucket -- a dropped increment would be an unobserved wake edge.
auto& bucket = objectWakeEpochs[ObjectWakeBucket(object)];
uint64_t tagBits = (uint64_t)ObjectWakeTag(object) << 32;
uint64_t observed = bucket.load(std::memory_order_relaxed);
while (!bucket.compare_exchange_weak(
observed, tagBits | (uint32_t)((uint32_t)observed + 1),
std::memory_order_release, std::memory_order_relaxed)) {
}
bool wokeAny = false;
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
@@ -1608,13 +1944,10 @@ namespace Sched {
bool inheritRedir = crasher && crasher->redirected &&
crasher->ioOutHandle >= 0 && crasherSlot >= 0;
Ipc::Object* outObj = nullptr;
Ipc::HandleType outType = Ipc::HandleType::None;
uint32_t outRights = 0;
Ipc::HandleSnapshot outSnapshot;
if (inheritRedir) {
if (!Ipc::SnapshotHandleForSlot(crasherSlot, crasher->ioOutHandle,
outType, outObj, outRights) ||
outType != Ipc::HandleType::Stream || outObj == nullptr) {
if (!outSnapshot.Capture(crasherSlot, crasher->ioOutHandle) ||
outSnapshot.type != Ipc::HandleType::Stream || outSnapshot.object == nullptr) {
inheritRedir = false;
}
}
@@ -1630,7 +1963,7 @@ namespace Sched {
// so it outlives the crasher's imminent ExitProcess teardown. The
// snapshot above and this install run back-to-back with no blocking
// call between, while the crasher's handle still pins the stream.
int h = Ipc::InstallHandleForSlot(childSlot, outObj, Ipc::HandleType::Stream,
int h = Ipc::InstallHandleForSlot(childSlot, outSnapshot.object, Ipc::HandleType::Stream,
Ipc::RightWrite | Ipc::RightWait | Ipc::RightDup);
if (h >= 0) {
child->ioOutHandle = h;
@@ -1647,3 +1980,9 @@ namespace Sched {
}
}
// Called by the syscall entry assembly after the syscall implementation has
// released all of its subsystem locks but before returning to ring 3.
extern "C" void SchedExitIfKilled() {
Sched::ExitIfKilled();
}
kernel/src/Sched/Scheduler.hpp
+38 -4
View File
@@ -6,6 +6,7 @@
#pragma once
#include <cstdint>
#include <atomic>
#include <Api/Syscall.hpp>
namespace Sched {
@@ -28,6 +29,24 @@ namespace Sched {
// Heavy recursive programs (GCC's cc1plus) need megabytes of stack.
static constexpr uint64_t UserStackMax = 64 * 1024 * 1024; // 64 MiB
static constexpr uint64_t UserHeapBase = 0x40000000ULL; // User heap start VA
// Keep the bump-allocated heap below the fixed shared-library range. A
// failed bound check is recoverable (SYS_ALLOC returns null); allowing the
// heap to cross this boundary would silently replace library PTEs.
static constexpr uint64_t UserHeapLimit = 0x60000000ULL;
// Surface mappings use reusable, fixed-size per-process slots instead of
// consuming heap VA forever. 64 * 32 MiB occupies [0x7000000000,
// 0x7080000000), comfortably below the main user stack.
static constexpr uint64_t UserSurfaceBase = 0x7000000000ULL;
static constexpr uint64_t UserSurfaceSlotSize = 32ULL * 1024 * 1024;
static constexpr uint64_t UserSurfaceSlots = 64;
static constexpr uint64_t UserSurfaceEnd =
UserSurfaceBase + UserSurfaceSlots * UserSurfaceSlotSize;
// The two write-combined scanout mappings live in their own region.
static constexpr uint64_t UserFramebufferBase = 0x7100000000ULL;
static constexpr uint64_t UserFramebufferLimit = 0x7140000000ULL;
static_assert(UserHeapBase < UserHeapLimit);
static_assert(UserSurfaceEnd <= UserFramebufferBase);
static_assert(UserFramebufferLimit < UserStackTop - UserStackMax);
static constexpr uint32_t UserReadDirSlots = 64; // rotating scratch pages for SYS_READDIR
static constexpr uint64_t UserReadDirBase =
UserHeapBase - (uint64_t)UserReadDirSlots * 0x1000ULL;
@@ -74,7 +93,7 @@ namespace Sched {
uint64_t tlsAlign = 0;
int runningOnCpu; // CPU index running this process (-1 if not running)
bool killPending = false; // Set by Sys_Kill when target is running on another CPU
std::atomic<bool> killPending{false}; // Cross-CPU kill handoff
bool reapReady = false; // Set once teardown is complete and BSP may free slot resources
bool startPending = false; // Spawned but not yet made runnable
@@ -124,9 +143,13 @@ namespace Sched {
// True when there is runnable work somewhere in the process table.
bool HasReadyProcesses();
// Ask the current CPU to yield once to its idle kernel context on the next
// scheduler pass so deferred device work cannot starve under full load.
void RequestKernelWork();
// Called from the APIC timer handler with the elapsed time for that CPU's
// tick interval. The BSP runs at 1 ms; APs may use a coarser interval.
void Tick(uint32_t elapsedMs = 1);
void Tick(uint32_t elapsedMs = 1, bool interruptedUser = false);
// Get the PID of the currently running process (-1 if idle).
// For sibling threads this returns the primary slot's PID -- i.e. the
@@ -164,6 +187,11 @@ namespace Sched {
// out of the user heap. Returns the new TID, or -1 on failure.
int SpawnThread(uint64_t entry, uint64_t arg, uint64_t userStackTop);
// Reserve a page-aligned range from a process's shared bump heap. The
// reservation is serialized across sibling threads and bounded below the
// fixed shared-library mappings.
bool ReserveUserHeapRange(int primarySlot, uint64_t size, uint64_t& outVa);
// Terminate the currently executing thread. If this is the main thread,
// the entire process exits (equivalent to ExitProcess).
[[noreturn]] void ExitCurrentThread(int exitCode);
@@ -194,8 +222,14 @@ namespace Sched {
bool BlockOnObjectIf(void* object, uint64_t timeoutMs,
bool (*shouldBlock)(void*), void* context);
// BSP-only scheduler housekeeping: wake expired sleepers and reclaim
// terminated process resources.
// Take this token before testing an object's ready condition, then pass it
// to BlockOnObjectSince. A wake between the test and scheduler enrollment
// makes the latter return without blocking instead of losing the edge.
uint64_t ObserveObjectWake(void* object);
bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake);
// BSP idle-context housekeeping: reclaim terminated process resources and
// run policy work previously queued by the timer or process teardown.
void RunBspMaintenance();
// Return the earliest blocked sleep/object timeout deadline in ticks,
kernel/src/Terminal/Terminal.cpp
+9
View File
@@ -241,6 +241,15 @@ namespace Kt {
g_suppressKernelLog = true;
}
void EnablePanicOutput() {
// Once the graphical desktop starts, ordinary console writes are
// suppressed to avoid painting over it. A kernel panic is different:
// leaving suppression enabled makes the halted system look like a
// frozen desktop. Do not acquire g_termLock here; panic may have
// interrupted its owner on this or another CPU.
g_suppressKernelLog = false;
}
int64_t ReadKernelLog(char* buf, uint64_t size) {
if (buf == nullptr || size == 0) return 0;
kernel/src/Terminal/Terminal.hpp
+4
View File
@@ -117,6 +117,10 @@ namespace Kt
extern bool g_suppressKernelLog;
void SuppressKernelLog();
// Panic output must bypass desktop-time console suppression. This is
// intentionally lock-free: a panic can occur while another CPU owns the
// terminal mutex, and the system is about to halt.
void EnablePanicOutput();
int64_t ReadKernelLog(char* buf, uint64_t size);
class KernelLogStream {
kernel/src/Timekeeping/ApicTimer.cpp
+75 -7
View File
@@ -10,12 +10,15 @@
#include <Hal/Apic/Apic.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/Cpu.hpp>
#include <Hal/CpuPower.hpp>
#include <Hal/SmpBoot.hpp>
#include <Io/IoPort.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Sched/Scheduler.hpp>
#include <Drivers/Net/E1000.hpp>
#include <Drivers/Net/E1000E.hpp>
#include <Drivers/Audio/IntelHda.hpp>
#include <Drivers/USB/Xhci.hpp>
#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
#include <Drivers/USB/HidKeyboard.hpp>
@@ -47,7 +50,7 @@ namespace Timekeeping {
static std::atomic<uint64_t> g_tickCount{0};
static uint32_t g_ticksPerMs = 0;
static bool g_schedEnabled = false;
static std::atomic<bool> g_schedEnabled{false};
static uint32_t CountForIntervalMs(uint32_t intervalMs) {
return g_ticksPerMs * intervalMs;
@@ -71,7 +74,7 @@ namespace Timekeeping {
// Timer IRQ handler: BSP runs timekeeping + scheduler accounting; APs
// run scheduler accounting only.
static void TimerHandler(uint8_t) {
static void TimerHandler(uint8_t, bool interruptedUser) {
auto* cpu = Smp::GetCurrentCpuData();
uint32_t schedElapsedMs = (cpu->cpuIndex == 0) ? BSP_TICK_INTERVAL_MS : AP_TICK_INTERVAL_MS;
@@ -82,10 +85,29 @@ namespace Timekeeping {
Drivers::Net::E1000E::Poll();
}
Drivers::USB::HidKeyboard::Tick();
// Bottom halves normally run from idle, but a continuously busy
// desktop may never enter idle. Force one bounded idle-context
// pass for USB immediately, and for network RX at most every
// 10 ms (important for E1000E's polling fallback).
bool usbWork = Drivers::USB::Xhci::HasDeferredWork();
bool audioWork = Drivers::Audio::IntelHda::HasDeferredWork();
bool netWork = Drivers::Net::E1000::HasDeferredWork() ||
Drivers::Net::E1000E::HasDeferredWork();
// A forced trip into the idle bottom-half context is safe only when
// this timer interrupted ring 3. If it interrupted a syscall, that
// suspended kernel stack may own one of the process-context mutexes
// used below; the reserved idle context would then spin on the lock
// while refusing to schedule its owner, deadlocking the CPU.
if (interruptedUser &&
(usbWork || audioWork ||
(netWork && (g_tickCount.load(std::memory_order_relaxed) % 10 == 0)))) {
Sched::RequestKernelWork();
}
}
if (g_schedEnabled) {
Sched::Tick(schedElapsedMs);
Sched::Tick(schedElapsedMs, interruptedUser);
}
}
@@ -188,7 +210,7 @@ namespace Timekeeping {
}
void EnableSchedulerTick() {
g_schedEnabled = true;
g_schedEnabled.store(true, std::memory_order_release);
}
void ApicTimerInitializeAP() {
@@ -202,16 +224,36 @@ namespace Timekeeping {
ProgramTimer(true, AP_TICK_INTERVAL_MS);
}
void IdleOnce(bool hasMwait, volatile uint64_t* monitorAddr) {
(void)hasMwait;
(void)monitorAddr;
void ServiceDeferredWork() {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr || cpu->currentSlot >= 0) return;
cpu->kernelWorkPending = false;
// These bottom halves deliberately run outside a process because they
// enter subsystems with process-context mutexes. Do not let a timer or
// reschedule IPI switch this idle stack to a process while such a lock
// remains held: the process could immediately wait on that same lock,
// leaving the suspended idle owner with no opportunity to release it.
// The scheduler and reschedule-IPI path both honor this reservation.
bool wasReserved = cpu->reservedForKernelWork;
cpu->reservedForKernelWork = true;
// Drain USB hot-plug deferred work from any idle core, not just the BSP.
if (Drivers::USB::Xhci::HasDeferredWork()) {
Drivers::USB::Xhci::ProcessDeferredWork();
}
// NIC hard IRQs only acknowledge/mask and queue RX work. Dispatching
// Ethernet/TCP/UDP here keeps process-context IPC mutexes out of IRQs.
Drivers::Net::E1000::ProcessDeferredWork();
Drivers::Net::E1000E::ProcessDeferredWork();
// HDA completion IRQs only acknowledge/mask. Resampling and DMA-ring
// refill are far too expensive for hard interrupt context.
if (cpu->cpuIndex == 0) {
Drivers::Audio::IntelHda::ProcessDeferredWork();
}
// Complete any boot-deferred Bluetooth bring-up (Intel firmware
// download) here instead of on the boot path: the download takes
// seconds and used to stall kmain before the first process spawned.
@@ -228,10 +270,35 @@ namespace Timekeeping {
// the adapter is down.
Drivers::USB::Bluetooth::ServiceEvents();
// Thermal policy records transitions during BSP maintenance; print
// them from this explicitly non-interrupt idle path.
Hal::CpuPower::ServiceDeferredDiagnostics();
cpu->reservedForKernelWork = wasReserved;
}
void IdleOnce(bool hasMwait, volatile uint64_t* monitorAddr) {
(void)hasMwait;
(void)monitorAddr;
auto* cpu = Smp::GetCurrentCpuData();
// HaltWithInterruptsDisabled() deliberately returns with IF clear so
// the next readiness check can lead into an atomic STI+HLT. Deferred
// work cannot inherit that state: xHCI/Bluetooth service contains
// wall-clock waits and depends on the BSP timer and device MSIs. If
// it runs with IF=0, GetMilliseconds() never advances on the BSP and
// the completion interrupt cannot arrive, producing a permanent idle-
// loop freeze. Keep IRQs enabled for all active idle-context work and
// disable them again only immediately before entering HLT below.
asm volatile("sti" ::: "memory");
ServiceDeferredWork();
if (cpu == nullptr || cpu->cpuIndex != 0 || !g_schedEnabled || g_ticksPerMs == 0) {
// Non-BSP or pre-scheduler fallback: plain HLT keeps the LAPIC
// timer running in C1 on hardware where MWAIT promotes to deeper
// states that freeze the timer (no ARAT in practice).
asm volatile("cli" ::: "memory");
Hal::HaltWithInterruptsDisabled();
return;
}
@@ -256,6 +323,7 @@ namespace Timekeeping {
// HLT (vs MWAIT) is the second half: MWAIT EAX=0 is a *hint* and
// the CPU may go deeper than C1; HLT generally stays in C1, which
// is the deepest state guaranteed to keep the LAPIC timer alive.
asm volatile("cli" ::: "memory");
Hal::HaltWithInterruptsDisabled();
}
kernel/src/Timekeeping/ApicTimer.hpp
+3
View File
@@ -28,6 +28,9 @@ namespace Timekeeping {
// Enable scheduler tick (called after scheduler is initialized)
void EnableSchedulerTick();
// Run bounded process-context bottom halves from an idle CPU.
void ServiceDeferredWork();
// Enter one idle wait cycle for the current CPU. Uses HLT and relies on
// the boot-time periodic LAPIC timer to wake -- one-shot wake doesn't
// survive deep C-states on some laptop CPUs.
programs/include/gui/framebuffer.hpp
+73 -9
View File
@@ -16,6 +16,8 @@ namespace gui {
class Framebuffer {
uint32_t* hw_fb;
uint32_t* hw_fb2; // second scanout buffer (page flip), nullptr if unsupported
uint32_t* hw_direct; // buffer known to be live after page-flip fallback
uint32_t* hw_mirror; // second possible live buffer when SURFLIVE is unavailable
uint32_t* back_buf;
int fb_width;
int fb_height;
@@ -66,18 +68,19 @@ class Framebuffer {
char user[64] = {};
if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) {
auto doc = montauk::config::load_user(user, "display");
bool enabled = doc.get_bool("graphics.tear_free", true);
bool enabled = doc.get_bool("graphics.tear_free", false);
doc.destroy();
return enabled;
}
auto doc = montauk::config::load("display");
bool enabled = doc.get_bool("graphics.tear_free", true);
bool enabled = doc.get_bool("graphics.tear_free", false);
doc.destroy();
return enabled;
}
public:
Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), back_buf(nullptr),
Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), hw_direct(nullptr), hw_mirror(nullptr),
back_buf(nullptr),
fb_width(0), fb_height(0), fb_pitch(0), hw_next(1) {
montauk::abi::FbInfo info;
montauk::fb_info(&info);
@@ -87,14 +90,39 @@ public:
fb_pitch = (int)info.pitch;
hw_fb = (uint32_t*)montauk::fb_map();
hw_direct = hw_fb;
back_buf = (uint32_t*)montauk::alloc((uint64_t)fb_height * fb_pitch);
// Hardware page flipping: fb_map() maps the second scanout buffer
// directly after the first when the kernel supports flipping.
if (hw_fb && prefer_hardware_page_flip()
&& montauk::fb_flip(-1, 0) == 1) {
// directly after the first when the kernel supports flipping. Acquire
// the scanout before choosing an off-screen buffer: another fullscreen
// process (notably login.elf) may have left either buffer live.
bool flip_supported = hw_fb && montauk::fb_flip(-1, 0) == 1;
uint64_t pages = ((uint64_t)fb_height * fb_pitch + 0xFFF) / 0x1000;
hw_fb2 = (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000);
uint32_t* second = flip_supported
? (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000) : nullptr;
if (flip_supported && prefer_hardware_page_flip()) {
int64_t front = montauk::fb_flip(-2, 0);
if (front == 0 || front == 1) {
hw_fb2 = second;
hw_next = (int)front ^ 1;
} else {
// If the live register cannot be trusted even at startup,
// avoid flipping and update both possible scanout surfaces.
hw_mirror = second;
}
} else if (flip_supported) {
// A direct-copy client must make buffer 0 live before it starts
// drawing there; the previous fullscreen client may have left
// buffer 1 scanning out.
if (montauk::fb_flip(0, 1) != 0) {
int64_t front = montauk::fb_flip(-2, 0);
if (front == 1) {
hw_direct = second;
} else if (front != 0) {
hw_mirror = second;
}
}
}
}
@@ -280,7 +308,7 @@ public:
// currently OFF screen, then ask the display engine to present it at
// the next vblank (tear-free). Without it, fall back to copying
// straight into the visible framebuffer.
uint32_t* dst_fb = hw_fb;
uint32_t* dst_fb = hw_direct;
if (hw_fb2) dst_fb = (hw_next == 1) ? hw_fb2 : hw_fb;
// Copy back buffer to hardware framebuffer, row by row (pitch may differ)
@@ -289,13 +317,49 @@ public:
uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch);
uint32_t* dst = (uint32_t*)((uint8_t*)dst_fb + y * fb_pitch);
montauk::memcpy(dst, src, row_bytes);
if (hw_mirror) {
uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch);
montauk::memcpy(mirror, src, row_bytes);
}
}
if (hw_mirror) asm volatile("sfence" ::: "memory");
if (hw_fb2) {
// Wait for the latch (vsync): the buffer we just left becomes
// safe to draw into only once the new one is actually scanning.
if (montauk::fb_flip(hw_next, 1) == hw_next) {
asm volatile("sfence" ::: "memory");
int64_t presented = montauk::fb_flip(hw_next, 1);
if (presented == hw_next) {
hw_next ^= 1;
} else {
// Do not retry a faulty page-flip path forever: that turns a
// transient latch failure into an apparently frozen desktop.
// Keep drawing directly into whichever buffer hardware says
// is live for the rest of this process. If SURFLIVE itself is
// unavailable, request buffer 0 without another blocking wait.
int64_t front = montauk::fb_flip(-2, 0);
if (front == 0 || front == 1) {
hw_direct = (front == 1) ? hw_fb2 : hw_fb;
} else {
(void)montauk::fb_flip(0, 0);
hw_direct = hw_fb;
hw_mirror = hw_fb2;
}
hw_fb2 = nullptr;
// The frame above was copied to the requested (possibly
// off-screen) buffer. Publish it once more to the live direct
// buffer so the cursor and desktop resume immediately.
for (int y = 0; y < fb_height; y++) {
uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch);
uint32_t* dst = (uint32_t*)((uint8_t*)hw_direct + y * fb_pitch);
montauk::memcpy(dst, src, row_bytes);
if (hw_mirror) {
uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch);
montauk::memcpy(mirror, src, row_bytes);
}
}
asm volatile("sfence" ::: "memory");
}
}
}
programs/include/montauk/syscall.h
+1
View File
@@ -339,6 +339,7 @@ namespace montauk {
// the buffer to show; the hardware latches it at vblank (tear-free).
// flags bit0 = block until the flip has been latched (vsync).
// fb_flip(-1, 0) returns 1 when page flipping is available, 0 when not.
// fb_flip(-2, 0) acquires ownership and returns the live buffer (0 or 1).
inline int64_t fb_flip(int64_t index, uint64_t flags) {
return syscall2(montauk::abi::SYS_FBFLIP, (uint64_t)index, flags);
}
programs/man/framebuffer.2
+3 -2
View File
@@ -21,8 +21,9 @@
than width * 4 due to alignment). bpp is always 32.
.SS fb_map
Maps the physical framebuffer into the process address space at
a fixed virtual address (0x50000000) and returns that address.
Maps the physical framebuffer into a dedicated region of the process
address space and returns its address. Programs must use the returned
pointer rather than assuming a particular virtual address.
uint32_t* pixels = (uint32_t*)montauk::fb_map();
programs/src/display/main.cpp
+4 -4
View File
@@ -55,8 +55,8 @@ static int g_mode_count = 0;
static bool g_driver_available = false;
static int g_selected_mode = -1;
static int g_pending_brightness = -1;
static bool g_tear_free = true;
static bool g_saved_tear_free = true;
static bool g_tear_free = false;
static bool g_saved_tear_free = false;
static int g_mode_scroll = 0;
static char g_status[128] = {};
static uint64_t g_status_time = 0;
@@ -164,11 +164,11 @@ static void load_graphics_preference() {
char user[64] = {};
if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) {
auto doc = montauk::config::load_user(user, "display");
g_tear_free = doc.get_bool("graphics.tear_free", true);
g_tear_free = doc.get_bool("graphics.tear_free", false);
doc.destroy();
} else {
auto doc = montauk::config::load("display");
g_tear_free = doc.get_bool("graphics.tear_free", true);
g_tear_free = doc.get_bool("graphics.tear_free", false);
doc.destroy();
}
g_saved_tear_free = g_tear_free;
template/sysroot/include/Api/Syscall.hpp
+1
View File
@@ -192,6 +192,7 @@ namespace montauk::abi {
static constexpr uint64_t SYS_SDR_READ = 146; // (handle, buf, len) -> bytes
static constexpr uint64_t SYS_SDR_SETPARAM = 147; // (handle, param, value)
static constexpr uint64_t SYS_SDR_GETPARAM = 148; // (handle, param) -> value
static constexpr uint64_t SYS_FBFLIP = 150;
static constexpr uint64_t SYS_SETUNIXTIME = 153;
static constexpr uint64_t SYS_DISPLAYINFO = 154;
static constexpr uint64_t SYS_DISPLAYMODES = 155;
template/sysroot/include/gui/framebuffer.hpp
+107 -2
View File
@@ -8,16 +8,21 @@
#include <cstdint>
#include <montauk/syscall.h>
#include <montauk/string.h>
#include <montauk/config.h>
#include "gui/gui.hpp"
namespace gui {
class Framebuffer {
uint32_t* hw_fb;
uint32_t* hw_fb2; // second scanout buffer (page flip), nullptr if unsupported
uint32_t* hw_direct; // buffer known to be live after page-flip fallback
uint32_t* hw_mirror; // second possible live buffer when SURFLIVE is unavailable
uint32_t* back_buf;
int fb_width;
int fb_height;
int fb_pitch; // in bytes
int hw_next; // scanout buffer the next flip() will present (page flip only)
static inline void fill_pixels(uint32_t* dst, int count, uint32_t pixel) {
if (!dst || count <= 0) return;
@@ -59,8 +64,24 @@ class Framebuffer {
return 0xFF000000 | (rr << 16) | (gg << 8) | bb;
}
static bool prefer_hardware_page_flip() {
char user[64] = {};
if (montauk::getuser(user, sizeof(user)) > 0 && user[0]) {
auto doc = montauk::config::load_user(user, "display");
bool enabled = doc.get_bool("graphics.tear_free", false);
doc.destroy();
return enabled;
}
auto doc = montauk::config::load("display");
bool enabled = doc.get_bool("graphics.tear_free", false);
doc.destroy();
return enabled;
}
public:
Framebuffer() : hw_fb(nullptr), back_buf(nullptr), fb_width(0), fb_height(0), fb_pitch(0) {
Framebuffer() : hw_fb(nullptr), hw_fb2(nullptr), hw_direct(nullptr), hw_mirror(nullptr),
back_buf(nullptr),
fb_width(0), fb_height(0), fb_pitch(0), hw_next(1) {
montauk::abi::FbInfo info;
montauk::fb_info(&info);
@@ -69,7 +90,40 @@ public:
fb_pitch = (int)info.pitch;
hw_fb = (uint32_t*)montauk::fb_map();
hw_direct = hw_fb;
back_buf = (uint32_t*)montauk::alloc((uint64_t)fb_height * fb_pitch);
// Hardware page flipping: fb_map() maps the second scanout buffer
// directly after the first when the kernel supports flipping. Acquire
// the scanout before choosing an off-screen buffer: another fullscreen
// process (notably login.elf) may have left either buffer live.
bool flip_supported = hw_fb && montauk::fb_flip(-1, 0) == 1;
uint64_t pages = ((uint64_t)fb_height * fb_pitch + 0xFFF) / 0x1000;
uint32_t* second = flip_supported
? (uint32_t*)((uint8_t*)hw_fb + pages * 0x1000) : nullptr;
if (flip_supported && prefer_hardware_page_flip()) {
int64_t front = montauk::fb_flip(-2, 0);
if (front == 0 || front == 1) {
hw_fb2 = second;
hw_next = (int)front ^ 1;
} else {
// If the live register cannot be trusted even at startup,
// avoid flipping and update both possible scanout surfaces.
hw_mirror = second;
}
} else if (flip_supported) {
// A direct-copy client must make buffer 0 live before it starts
// drawing there; the previous fullscreen client may have left
// buffer 1 scanning out.
if (montauk::fb_flip(0, 1) != 0) {
int64_t front = montauk::fb_flip(-2, 0);
if (front == 1) {
hw_direct = second;
} else if (front != 0) {
hw_mirror = second;
}
}
}
}
int width() const { return fb_width; }
@@ -250,12 +304,63 @@ public:
inline void flip() {
if (!hw_fb || !back_buf) return;
// With hardware page flipping, copy into the scanout buffer that is
// currently OFF screen, then ask the display engine to present it at
// the next vblank (tear-free). Without it, fall back to copying
// straight into the visible framebuffer.
uint32_t* dst_fb = hw_direct;
if (hw_fb2) dst_fb = (hw_next == 1) ? hw_fb2 : hw_fb;
// Copy back buffer to hardware framebuffer, row by row (pitch may differ)
uint64_t row_bytes = (uint64_t)fb_width * sizeof(uint32_t);
for (int y = 0; y < fb_height; y++) {
uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch);
uint32_t* dst = (uint32_t*)((uint8_t*)hw_fb + y * fb_pitch);
uint32_t* dst = (uint32_t*)((uint8_t*)dst_fb + y * fb_pitch);
montauk::memcpy(dst, src, row_bytes);
if (hw_mirror) {
uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch);
montauk::memcpy(mirror, src, row_bytes);
}
}
if (hw_mirror) asm volatile("sfence" ::: "memory");
if (hw_fb2) {
// Wait for the latch (vsync): the buffer we just left becomes
// safe to draw into only once the new one is actually scanning.
asm volatile("sfence" ::: "memory");
int64_t presented = montauk::fb_flip(hw_next, 1);
if (presented == hw_next) {
hw_next ^= 1;
} else {
// Do not retry a faulty page-flip path forever: that turns a
// transient latch failure into an apparently frozen desktop.
// Keep drawing directly into whichever buffer hardware says
// is live for the rest of this process. If SURFLIVE itself is
// unavailable, request buffer 0 without another blocking wait.
int64_t front = montauk::fb_flip(-2, 0);
if (front == 0 || front == 1) {
hw_direct = (front == 1) ? hw_fb2 : hw_fb;
} else {
(void)montauk::fb_flip(0, 0);
hw_direct = hw_fb;
hw_mirror = hw_fb2;
}
hw_fb2 = nullptr;
// The frame above was copied to the requested (possibly
// off-screen) buffer. Publish it once more to the live direct
// buffer so the cursor and desktop resume immediately.
for (int y = 0; y < fb_height; y++) {
uint32_t* src = (uint32_t*)((uint8_t*)back_buf + y * fb_pitch);
uint32_t* dst = (uint32_t*)((uint8_t*)hw_direct + y * fb_pitch);
montauk::memcpy(dst, src, row_bytes);
if (hw_mirror) {
uint32_t* mirror = (uint32_t*)((uint8_t*)hw_mirror + y * fb_pitch);
montauk::memcpy(mirror, src, row_bytes);
}
}
asm volatile("sfence" ::: "memory");
}
}
}
};
template/sysroot/include/montauk/syscall.h
+5
View File
@@ -326,6 +326,11 @@ namespace montauk {
// Framebuffer
inline void fb_info(montauk::abi::FbInfo* info) { syscall1(montauk::abi::SYS_FBINFO, (uint64_t)info); }
inline void* fb_map() { return (void*)syscall0(montauk::abi::SYS_FBMAP); }
// Page flip between two scanout buffers. index -1 queries support;
// index -2 acquires ownership and returns the live front buffer.
inline int64_t fb_flip(int64_t index, uint64_t flags) {
return syscall2(montauk::abi::SYS_FBFLIP, (uint64_t)index, flags);
}
inline int display_info(montauk::abi::DisplayInfo* out) {
return (int)syscall1(montauk::abi::SYS_DISPLAYINFO, (uint64_t)out);
}