fix: kernel concurrency, interrupt context, and user VA safety

This commit is contained in:
daniel committed 2026-08-01 15:43:38 +02:00
1 parent f222bf1f04
commit b1c55073c7
71 files changed
+2654 -1092

No files matched your search

+13 -8
View File
@@ -16,6 +16,10 @@ namespace Hal {
static volatile uint32_t* g_apicBase = nullptr;
static constexpr uint32_t ICR_DELIVERY_STATUS = (1 << 12);
static constexpr uint32_t ICR_LEVEL_ASSERT = (1 << 14);
// Fixed-IPIs normally clear delivery status within a few MMIO reads.
// Keep the exceptional path short because scheduler kicks can originate
// in an input IRQ where a long MMIO poll would itself stall the BSP.
static constexpr uint32_t ICR_WAIT_SPINS = 10000;
static inline uint64_t ReadMSR(uint32_t msr) {
uint32_t lo, hi;
@@ -115,19 +119,20 @@ namespace Hal {
WriteRegister(REG_EOI, 0);
}
void SendFixedIpi(uint32_t apicId, uint8_t vector) {
if (g_apicBase == nullptr) return;
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
static bool WaitForIcrIdle() {
for (uint32_t i = 0; i < ICR_WAIT_SPINS; i++) {
if (!(ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS)) return true;
asm volatile("pause");
}
return false;
}
bool SendFixedIpi(uint32_t apicId, uint8_t vector) {
if (g_apicBase == nullptr || !WaitForIcrIdle()) return false;
WriteRegister(REG_ICR_HIGH, apicId << 24);
WriteRegister(REG_ICR_LOW, (uint32_t)vector | ICR_LEVEL_ASSERT);
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
asm volatile("pause");
}
return WaitForIcrIdle();
}
uint32_t GetId() {
+2 -1
View File
@@ -44,7 +44,8 @@ namespace Hal {
void Reinitialize();
void SendEOI();
void SendFixedIpi(uint32_t apicId, uint8_t vector);
// Returns false if the local APIC delivery-status bit fails to clear.
bool SendFixedIpi(uint32_t apicId, uint8_t vector);
uint32_t GetId();
uint32_t ReadRegister(uint32_t reg);
+3 -2
View File
@@ -38,10 +38,11 @@ namespace Hal {
<< " IRQ stubs (vectors " << (uint64_t)IRQ_VECTOR_BASE << "-"
<< (uint64_t)(IRQ_VECTOR_BASE + IRQ_COUNT - 1) << ")";
}
};
// C linkage dispatch function called from assembly stubs
extern "C" void HalIrqDispatch(uint64_t irqNumber) {
extern "C" void HalIrqDispatch(uint64_t irqNumber, uint64_t savedCs) {
// Send EOI BEFORE calling the handler. The handler may context-switch
// (via Tick -> Schedule -> SchedContextSwitch) and never return here.
// If EOI is deferred until after the handler, the LAPIC timer vector
@@ -52,6 +53,6 @@ extern "C" void HalIrqDispatch(uint64_t irqNumber) {
Hal::LocalApic::SendEOI();
if (irqNumber < Hal::IRQ_COUNT && Hal::g_irqHandlers[irqNumber] != nullptr) {
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber);
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber, (savedCs & 3) == 3);
}
}
+5 -1
View File
@@ -9,7 +9,10 @@
namespace Hal {
// IRQ handler function type. The parameter is the IRQ number (0-47).
using IrqHandler = void(*)(uint8_t irq);
// `fromUser` is derived directly from the saved interrupt-frame CS. Do not
// cache it in CpuData: an IRQ handler may context-switch and later resume
// on another CPU, making per-CPU saved-origin state stale.
using IrqHandler = void(*)(uint8_t irq, bool fromUser);
// Number of IRQ slots supported (0-23: legacy ISA via IOAPIC,
// 24-45: MSI, 46-47: kernel IPIs)
@@ -37,4 +40,5 @@ namespace Hal {
// Install IRQ stubs into the IDT and set up the dispatch table
void InitializeIrqHandlers();
};
+10 -2
View File
@@ -50,11 +50,19 @@ IrqCommon:
; Pass IRQ number as first argument (rdi)
mov rdi, rax
; Pass the saved CS as the second argument. Fifteen saved GPRs occupy
; 120 bytes, and CS is eight bytes after the interrupt-frame RIP.
mov rsi, [rsp + 128]
; Align stack to 16 bytes before call (we pushed 15 registers x 8 = 120 bytes
; + return address 8 = 128, which is 16-byte aligned, so we're good)
; An IRQ that interrupted ring 0 can arrive at any kernel stack depth, so
; the saved frame is not guaranteed to satisfy the SysV call alignment.
; RBX is already saved above and is callee-saved, making it a safe place to
; retain the exact frame pointer while aligning down for the C++ call.
mov rbx, rsp
and rsp, -16
cld
call HalIrqDispatch
mov rsp, rbx
pop r15
pop r14
+18 -2
View File
@@ -70,6 +70,8 @@ namespace Hal {
static constexpr uint32_t GovernorIntervalMs = 500;
static constexpr uint8_t PassiveStepDown = 2; // ratio units (~200 MHz)
static constexpr uint8_t HotStepDown = 8; // ratio units (~800 MHz)
static constexpr uint32_t ThermalLogThrottle = 0x100;
static constexpr uint32_t ThermalLogRecovered = 0x200;
// ============================================================
// Detected features and shared policy state
@@ -106,6 +108,7 @@ namespace Hal {
static bool g_throttling = false;
static std::atomic<uint8_t> g_lastTempC{0};
static std::atomic<uint32_t> g_pendingThermalLog{0};
static std::atomic<uint32_t> g_effMHz{0};
static uint64_t g_lastGovernorMs = 0;
static uint64_t g_lastAperf = 0;
@@ -339,10 +342,23 @@ namespace Hal {
uint32_t epoch = g_policyEpoch.fetch_add(1, std::memory_order_acq_rel) + 1;
ApplyHwpRequestOnThisCpu(epoch); // BSP applies immediately; APs on tick
if (g_throttling && !wasThrottling) {
// Keep policy transitions separate from terminal output so this
// maintenance step never enters process-context logging locks.
if (g_throttling && !wasThrottling)
g_pendingThermalLog.store(ThermalLogThrottle | temp,
std::memory_order_release);
else if (!g_throttling && wasThrottling)
g_pendingThermalLog.store(ThermalLogRecovered | temp,
std::memory_order_release);
}
void ServiceDeferredDiagnostics() {
uint32_t pending = g_pendingThermalLog.exchange(0, std::memory_order_acq_rel);
uint8_t temp = (uint8_t)(pending & 0xFF);
if (pending & ThermalLogThrottle) {
KernelLogStream(WARNING, "CpuPower") << "Package at " << base::dec
<< (uint64_t)temp << "C - thermal throttle engaged";
} else if (!g_throttling && wasThrottling) {
} else if (pending & ThermalLogRecovered) {
KernelLogStream(OK, "CpuPower") << "Package cooled to " << base::dec
<< (uint64_t)temp << "C - thermal throttle released";
}
+4
View File
@@ -43,6 +43,10 @@ namespace Hal {
// be called from every maintenance pass.
void ThermalTick(uint64_t nowMs);
// Print thermal-transition diagnostics from idle/process context.
// ThermalTick only records transitions for this function to report.
void ServiceDeferredDiagnostics();
// Redo the BSP MSR setup after S3 wake (HWP enable and POWER_CTL do
// not survive suspend).
void ReapplyAfterWake();
+12 -1
View File
@@ -13,6 +13,7 @@
#include <Memory/PageFrameAllocator.hpp>
#include <Sched/Scheduler.hpp>
#include <Sched/CrashReport.hpp>
#include <Hal/SmpBoot.hpp>
#include <Timekeeping/ApicTimer.hpp>
namespace Hal {
@@ -93,6 +94,14 @@ namespace Hal {
// If the fault originated in user-mode (ring 3), kill the process
// instead of panicking the entire system.
if (fromUser && Sched::GetCurrentPid() >= 0) {
// Interrupt gates arrive with IF clear. Full process teardown can
// wait on device completions, sibling CPUs, and wall-clock-bounded
// recovery paths, so leaving IF clear here can stop the BSP clock
// and its local device IRQs indefinitely. GS is already the kernel
// per-CPU base, and timer/IPI scheduling refuses to switch away
// from a ring-0 frame, so nested hardware IRQs are safe now.
asm volatile("sti" ::: "memory");
auto* proc = Sched::GetCurrentProcessPtr();
auto* regs = GetExceptionRegs(i, frame);
Kt::KernelLogStream(Kt::ERROR, "Exception")
@@ -183,6 +192,7 @@ namespace Hal {
{
bool fromUser = (frame->CS & 3) == 3;
if (fromUser) asm volatile("swapgs");
auto* cpu = Smp::TryGetCurrentCpuData();
uint64_t cr2;
asm volatile("mov %%cr2, %0" : "=r"(cr2));
@@ -190,7 +200,8 @@ namespace Hal {
// Bit 0 of the error code: 0 = non-present page. Covers both user
// pushes past the mapped stack and kernel accesses to not-yet-grown
// user stack buffers passed into syscalls.
if ((errorCode & 1) == 0 && Sched::GetCurrentPid() >= 0
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
&& Sched::GetCurrentPid() >= 0
&& Sched::TryGrowUserStack(cr2)) {
if (fromUser) asm volatile("swapgs");
return;
+3
View File
@@ -218,6 +218,9 @@ namespace Smp {
for (;;) {
// Pick up thermal-governor frequency changes decided by the BSP.
Hal::CpuPower::ApplyPolicyIfChanged();
// Any idle core may run bounded USB/NIC bottom halves. Preserve
// the AP's ACPI/MWAIT idle selection after servicing them.
Timekeeping::ServiceDeferredWork();
Hal::CpuIdle::Wait(10, cpu->hasMwait, &s_idleMonitor);
}
}
+16
View File
@@ -49,6 +49,11 @@ namespace Smp {
// Other CPUs schedule normally while this is held.
volatile bool reservedForKernelWork = false;
// Set by the BSP timer when an IRQ bottom half needs process-safe
// execution even though runnable user processes keep this CPU busy.
// Schedule() consumes it by yielding once to the idle kernel context.
bool kernelWorkPending = false;
// Per-CPU GDT and TSS (APs use these; BSP uses globals)
Hal::BasicGDT cpuGdt __attribute__((aligned(16)));
Hal::TSS64 cpuTss __attribute__((aligned(16)));
@@ -61,6 +66,17 @@ namespace Smp {
return ptr;
}
// Exception paths may run before InitBsp() has installed a GS base. Do
// not read gs:0 in that state: address zero would fault recursively and
// turn the original, diagnosable exception into a triple fault.
inline CpuData* TryGetCurrentCpuData() {
uint32_t low;
uint32_t high;
asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(0xC0000101u));
uint64_t base = ((uint64_t)high << 32) | low;
return (CpuData*)base;
}
// Get CPU data by index
CpuData* GetCpuData(int index);
+10
View File
@@ -8,6 +8,7 @@
section .text
extern SyscallDispatch
extern SchedExitIfKilled
; ====================================================================
; Per-CPU data offsets (must match CpuData in SmpBoot.hpp)
@@ -52,6 +53,15 @@ SyscallEntry:
mov rdi, rsp ; arg1 = pointer to SyscallFrame
call SyscallDispatch ; returns int64_t in rax
; A remote kill observed while this syscall was executing must not tear
; the process down from the timer IRQ on top of held syscall locks. This
; safe point runs after dispatch returned and all such locks are released.
sub rsp, 16
mov [rsp], rax
call SchedExitIfKilled
mov rax, [rsp]
add rsp, 16
cli ; disable interrupts for sysret
pop r15