fix: kernel concurrency, interrupt context, and user VA safety
This commit is contained in:
1 parent
f222bf1f04
commit
b1c55073c7
71 files changed
+2654
-1092
No files matched your search
@@ -16,6 +16,10 @@ namespace Hal {
|
||||
static volatile uint32_t* g_apicBase = nullptr;
|
||||
static constexpr uint32_t ICR_DELIVERY_STATUS = (1 << 12);
|
||||
static constexpr uint32_t ICR_LEVEL_ASSERT = (1 << 14);
|
||||
// Fixed-IPIs normally clear delivery status within a few MMIO reads.
|
||||
// Keep the exceptional path short because scheduler kicks can originate
|
||||
// in an input IRQ where a long MMIO poll would itself stall the BSP.
|
||||
static constexpr uint32_t ICR_WAIT_SPINS = 10000;
|
||||
|
||||
static inline uint64_t ReadMSR(uint32_t msr) {
|
||||
uint32_t lo, hi;
|
||||
@@ -115,19 +119,20 @@ namespace Hal {
|
||||
WriteRegister(REG_EOI, 0);
|
||||
}
|
||||
|
||||
void SendFixedIpi(uint32_t apicId, uint8_t vector) {
|
||||
if (g_apicBase == nullptr) return;
|
||||
|
||||
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
|
||||
static bool WaitForIcrIdle() {
|
||||
for (uint32_t i = 0; i < ICR_WAIT_SPINS; i++) {
|
||||
if (!(ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS)) return true;
|
||||
asm volatile("pause");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SendFixedIpi(uint32_t apicId, uint8_t vector) {
|
||||
if (g_apicBase == nullptr || !WaitForIcrIdle()) return false;
|
||||
|
||||
WriteRegister(REG_ICR_HIGH, apicId << 24);
|
||||
WriteRegister(REG_ICR_LOW, (uint32_t)vector | ICR_LEVEL_ASSERT);
|
||||
|
||||
while (ReadRegister(REG_ICR_LOW) & ICR_DELIVERY_STATUS) {
|
||||
asm volatile("pause");
|
||||
}
|
||||
return WaitForIcrIdle();
|
||||
}
|
||||
|
||||
uint32_t GetId() {
|
||||
|
||||
@@ -44,7 +44,8 @@ namespace Hal {
|
||||
void Reinitialize();
|
||||
|
||||
void SendEOI();
|
||||
void SendFixedIpi(uint32_t apicId, uint8_t vector);
|
||||
// Returns false if the local APIC delivery-status bit fails to clear.
|
||||
bool SendFixedIpi(uint32_t apicId, uint8_t vector);
|
||||
uint32_t GetId();
|
||||
|
||||
uint32_t ReadRegister(uint32_t reg);
|
||||
|
||||
@@ -38,10 +38,11 @@ namespace Hal {
|
||||
<< " IRQ stubs (vectors " << (uint64_t)IRQ_VECTOR_BASE << "-"
|
||||
<< (uint64_t)(IRQ_VECTOR_BASE + IRQ_COUNT - 1) << ")";
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
// C linkage dispatch function called from assembly stubs
|
||||
extern "C" void HalIrqDispatch(uint64_t irqNumber) {
|
||||
extern "C" void HalIrqDispatch(uint64_t irqNumber, uint64_t savedCs) {
|
||||
// Send EOI BEFORE calling the handler. The handler may context-switch
|
||||
// (via Tick -> Schedule -> SchedContextSwitch) and never return here.
|
||||
// If EOI is deferred until after the handler, the LAPIC timer vector
|
||||
@@ -52,6 +53,6 @@ extern "C" void HalIrqDispatch(uint64_t irqNumber) {
|
||||
Hal::LocalApic::SendEOI();
|
||||
|
||||
if (irqNumber < Hal::IRQ_COUNT && Hal::g_irqHandlers[irqNumber] != nullptr) {
|
||||
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber);
|
||||
Hal::g_irqHandlers[irqNumber]((uint8_t)irqNumber, (savedCs & 3) == 3);
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,10 @@
|
||||
|
||||
namespace Hal {
|
||||
// IRQ handler function type. The parameter is the IRQ number (0-47).
|
||||
using IrqHandler = void(*)(uint8_t irq);
|
||||
// `fromUser` is derived directly from the saved interrupt-frame CS. Do not
|
||||
// cache it in CpuData: an IRQ handler may context-switch and later resume
|
||||
// on another CPU, making per-CPU saved-origin state stale.
|
||||
using IrqHandler = void(*)(uint8_t irq, bool fromUser);
|
||||
|
||||
// Number of IRQ slots supported (0-23: legacy ISA via IOAPIC,
|
||||
// 24-45: MSI, 46-47: kernel IPIs)
|
||||
@@ -37,4 +40,5 @@ namespace Hal {
|
||||
|
||||
// Install IRQ stubs into the IDT and set up the dispatch table
|
||||
void InitializeIrqHandlers();
|
||||
|
||||
};
|
||||
@@ -50,11 +50,19 @@ IrqCommon:
|
||||
|
||||
; Pass IRQ number as first argument (rdi)
|
||||
mov rdi, rax
|
||||
; Pass the saved CS as the second argument. Fifteen saved GPRs occupy
|
||||
; 120 bytes, and CS is eight bytes after the interrupt-frame RIP.
|
||||
mov rsi, [rsp + 128]
|
||||
|
||||
; Align stack to 16 bytes before call (we pushed 15 registers x 8 = 120 bytes
|
||||
; + return address 8 = 128, which is 16-byte aligned, so we're good)
|
||||
; An IRQ that interrupted ring 0 can arrive at any kernel stack depth, so
|
||||
; the saved frame is not guaranteed to satisfy the SysV call alignment.
|
||||
; RBX is already saved above and is callee-saved, making it a safe place to
|
||||
; retain the exact frame pointer while aligning down for the C++ call.
|
||||
mov rbx, rsp
|
||||
and rsp, -16
|
||||
cld
|
||||
call HalIrqDispatch
|
||||
mov rsp, rbx
|
||||
|
||||
pop r15
|
||||
pop r14
|
||||
|
||||
@@ -70,6 +70,8 @@ namespace Hal {
|
||||
static constexpr uint32_t GovernorIntervalMs = 500;
|
||||
static constexpr uint8_t PassiveStepDown = 2; // ratio units (~200 MHz)
|
||||
static constexpr uint8_t HotStepDown = 8; // ratio units (~800 MHz)
|
||||
static constexpr uint32_t ThermalLogThrottle = 0x100;
|
||||
static constexpr uint32_t ThermalLogRecovered = 0x200;
|
||||
|
||||
// ============================================================
|
||||
// Detected features and shared policy state
|
||||
@@ -106,6 +108,7 @@ namespace Hal {
|
||||
|
||||
static bool g_throttling = false;
|
||||
static std::atomic<uint8_t> g_lastTempC{0};
|
||||
static std::atomic<uint32_t> g_pendingThermalLog{0};
|
||||
static std::atomic<uint32_t> g_effMHz{0};
|
||||
static uint64_t g_lastGovernorMs = 0;
|
||||
static uint64_t g_lastAperf = 0;
|
||||
@@ -339,10 +342,23 @@ namespace Hal {
|
||||
uint32_t epoch = g_policyEpoch.fetch_add(1, std::memory_order_acq_rel) + 1;
|
||||
ApplyHwpRequestOnThisCpu(epoch); // BSP applies immediately; APs on tick
|
||||
|
||||
if (g_throttling && !wasThrottling) {
|
||||
// Keep policy transitions separate from terminal output so this
|
||||
// maintenance step never enters process-context logging locks.
|
||||
if (g_throttling && !wasThrottling)
|
||||
g_pendingThermalLog.store(ThermalLogThrottle | temp,
|
||||
std::memory_order_release);
|
||||
else if (!g_throttling && wasThrottling)
|
||||
g_pendingThermalLog.store(ThermalLogRecovered | temp,
|
||||
std::memory_order_release);
|
||||
}
|
||||
|
||||
void ServiceDeferredDiagnostics() {
|
||||
uint32_t pending = g_pendingThermalLog.exchange(0, std::memory_order_acq_rel);
|
||||
uint8_t temp = (uint8_t)(pending & 0xFF);
|
||||
if (pending & ThermalLogThrottle) {
|
||||
KernelLogStream(WARNING, "CpuPower") << "Package at " << base::dec
|
||||
<< (uint64_t)temp << "C - thermal throttle engaged";
|
||||
} else if (!g_throttling && wasThrottling) {
|
||||
} else if (pending & ThermalLogRecovered) {
|
||||
KernelLogStream(OK, "CpuPower") << "Package cooled to " << base::dec
|
||||
<< (uint64_t)temp << "C - thermal throttle released";
|
||||
}
|
||||
|
||||
@@ -43,6 +43,10 @@ namespace Hal {
|
||||
// be called from every maintenance pass.
|
||||
void ThermalTick(uint64_t nowMs);
|
||||
|
||||
// Print thermal-transition diagnostics from idle/process context.
|
||||
// ThermalTick only records transitions for this function to report.
|
||||
void ServiceDeferredDiagnostics();
|
||||
|
||||
// Redo the BSP MSR setup after S3 wake (HWP enable and POWER_CTL do
|
||||
// not survive suspend).
|
||||
void ReapplyAfterWake();
|
||||
|
||||
+12
-1
@@ -13,6 +13,7 @@
|
||||
#include <Memory/PageFrameAllocator.hpp>
|
||||
#include <Sched/Scheduler.hpp>
|
||||
#include <Sched/CrashReport.hpp>
|
||||
#include <Hal/SmpBoot.hpp>
|
||||
#include <Timekeeping/ApicTimer.hpp>
|
||||
|
||||
namespace Hal {
|
||||
@@ -93,6 +94,14 @@ namespace Hal {
|
||||
// If the fault originated in user-mode (ring 3), kill the process
|
||||
// instead of panicking the entire system.
|
||||
if (fromUser && Sched::GetCurrentPid() >= 0) {
|
||||
// Interrupt gates arrive with IF clear. Full process teardown can
|
||||
// wait on device completions, sibling CPUs, and wall-clock-bounded
|
||||
// recovery paths, so leaving IF clear here can stop the BSP clock
|
||||
// and its local device IRQs indefinitely. GS is already the kernel
|
||||
// per-CPU base, and timer/IPI scheduling refuses to switch away
|
||||
// from a ring-0 frame, so nested hardware IRQs are safe now.
|
||||
asm volatile("sti" ::: "memory");
|
||||
|
||||
auto* proc = Sched::GetCurrentProcessPtr();
|
||||
auto* regs = GetExceptionRegs(i, frame);
|
||||
Kt::KernelLogStream(Kt::ERROR, "Exception")
|
||||
@@ -183,6 +192,7 @@ namespace Hal {
|
||||
{
|
||||
bool fromUser = (frame->CS & 3) == 3;
|
||||
if (fromUser) asm volatile("swapgs");
|
||||
auto* cpu = Smp::TryGetCurrentCpuData();
|
||||
|
||||
uint64_t cr2;
|
||||
asm volatile("mov %%cr2, %0" : "=r"(cr2));
|
||||
@@ -190,7 +200,8 @@ namespace Hal {
|
||||
// Bit 0 of the error code: 0 = non-present page. Covers both user
|
||||
// pushes past the mapped stack and kernel accesses to not-yet-grown
|
||||
// user stack buffers passed into syscalls.
|
||||
if ((errorCode & 1) == 0 && Sched::GetCurrentPid() >= 0
|
||||
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
|
||||
&& Sched::GetCurrentPid() >= 0
|
||||
&& Sched::TryGrowUserStack(cr2)) {
|
||||
if (fromUser) asm volatile("swapgs");
|
||||
return;
|
||||
|
||||
@@ -218,6 +218,9 @@ namespace Smp {
|
||||
for (;;) {
|
||||
// Pick up thermal-governor frequency changes decided by the BSP.
|
||||
Hal::CpuPower::ApplyPolicyIfChanged();
|
||||
// Any idle core may run bounded USB/NIC bottom halves. Preserve
|
||||
// the AP's ACPI/MWAIT idle selection after servicing them.
|
||||
Timekeeping::ServiceDeferredWork();
|
||||
Hal::CpuIdle::Wait(10, cpu->hasMwait, &s_idleMonitor);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,11 @@ namespace Smp {
|
||||
// Other CPUs schedule normally while this is held.
|
||||
volatile bool reservedForKernelWork = false;
|
||||
|
||||
// Set by the BSP timer when an IRQ bottom half needs process-safe
|
||||
// execution even though runnable user processes keep this CPU busy.
|
||||
// Schedule() consumes it by yielding once to the idle kernel context.
|
||||
bool kernelWorkPending = false;
|
||||
|
||||
// Per-CPU GDT and TSS (APs use these; BSP uses globals)
|
||||
Hal::BasicGDT cpuGdt __attribute__((aligned(16)));
|
||||
Hal::TSS64 cpuTss __attribute__((aligned(16)));
|
||||
@@ -61,6 +66,17 @@ namespace Smp {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
// Exception paths may run before InitBsp() has installed a GS base. Do
|
||||
// not read gs:0 in that state: address zero would fault recursively and
|
||||
// turn the original, diagnosable exception into a triple fault.
|
||||
inline CpuData* TryGetCurrentCpuData() {
|
||||
uint32_t low;
|
||||
uint32_t high;
|
||||
asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(0xC0000101u));
|
||||
uint64_t base = ((uint64_t)high << 32) | low;
|
||||
return (CpuData*)base;
|
||||
}
|
||||
|
||||
// Get CPU data by index
|
||||
CpuData* GetCpuData(int index);
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
section .text
|
||||
|
||||
extern SyscallDispatch
|
||||
extern SchedExitIfKilled
|
||||
|
||||
; ====================================================================
|
||||
; Per-CPU data offsets (must match CpuData in SmpBoot.hpp)
|
||||
@@ -52,6 +53,15 @@ SyscallEntry:
|
||||
mov rdi, rsp ; arg1 = pointer to SyscallFrame
|
||||
call SyscallDispatch ; returns int64_t in rax
|
||||
|
||||
; A remote kill observed while this syscall was executing must not tear
|
||||
; the process down from the timer IRQ on top of held syscall locks. This
|
||||
; safe point runs after dispatch returned and all such locks are released.
|
||||
sub rsp, 16
|
||||
mov [rsp], rax
|
||||
call SchedExitIfKilled
|
||||
mov rax, [rsp]
|
||||
add rsp, 16
|
||||
|
||||
cli ; disable interrupts for sysret
|
||||
|
||||
pop r15
|
||||
|
||||
Reference in new issue
Block a user