daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
278b20825815ec04a22fd1c2fcfedbd38e116e6d
MontaukOS/kernel/src/Sched/Scheduler.cpp
T

2411 lines
95 KiB
C++
Raw Blame History

/*
* Scheduler.cpp
* Preemptive process scheduler with SMP support
* Copyright (c) 2025-2026 Daniel Hammer
*/
#include <Memory/UserRange.hpp>
#include "Scheduler.hpp"
#include "ElfLoader.hpp"
#include "Consent.hpp"
#include <Fs/ProtectedPaths.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Memory/Paging.hpp>
#include <Memory/HHDM.hpp>
#include <Libraries/Memory.hpp>
#include <Libraries/String.hpp>
#include <Common/Panic.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Hal/Apic/Apic.hpp>
#include <Hal/Apic/Interrupts.hpp>
#include <Hal/GDT.hpp>
#include <Hal/SmpBoot.hpp>
#include <Hal/CpuPower.hpp>
#include <Timekeeping/ApicTimer.hpp>
#include <Api/WinServer.hpp>
#include <Api/Heap.hpp>
#include <Api/LibSyscall.hpp>
#include <Drivers/Audio/Mixer.hpp>
#include <Drivers/Graphics/IntelGPU.hpp>
#include <Ipc/Ipc.hpp>
#include <Drivers/USB/UserUsb.hpp>
// Assembly: context switch with CR3 and FPU state parameters
extern "C" void SchedContextSwitch(uint64_t* oldRsp, uint64_t newRsp, uint64_t newCR3,
uint8_t* oldFpuArea, uint8_t* newFpuArea);
// Load a user thread's FS base (TLS thread pointer) before dispatching it.
// Cached per CPU: the MSR write is skipped when unchanged, so TLS-free
// processes (fsBase 0) cost one write per CPU ever. Userspace cannot move
// FS base itself (CR4.FSGSBASE is off), so the cache cannot go stale.
static inline void LoadUserFsBase(Smp::CpuData* cpu, uint64_t fsBase) {
if (cpu->currentFsBase == fsBase) {
return;
}
cpu->currentFsBase = fsBase;
asm volatile("wrmsr"
:
: "c"(0xC0000100u), /* IA32_FS_BASE */
"a"((uint32_t)fsBase),
"d"((uint32_t)(fsBase >> 32)));
}
// Assembly: jump to user mode via IRETQ.
// `arg` is delivered as the user-mode RDI (SystemV first argument).
// For freshly spawned processes this is 0; for SpawnThread it is the
// user-supplied entry argument.
extern "C" void JumpToUserMode(uint64_t rip, uint64_t rsp, uint64_t arg);
namespace Sched {
static Process processTable[MaxProcesses];
static int nextPid = 0;
// Exit-code ledger. Pids are monotonic and never recycled, so entries
// cannot alias. Written under schedLock during process teardown; read
// by SYS_WAITPID after the waiter unblocks. Codes: 0..255 = normal
// exit status, 256+signal = killed or crashed.
static constexpr int ExitLedgerSize = 128;
static int exitLedgerPid[ExitLedgerSize];
static int exitLedgerCode[ExitLedgerSize];
static int exitLedgerHead = 0;
// The scheduler lock MUST be a Spinlock (interrupt-disabling).
// It is held ACROSS context switches to prevent the race where
// another CPU picks up a process whose RSP hasn't been saved yet.
// The resumed process releases it.
// schedLock is a baton: the context that acquires it switches stacks and
// the context being resumed releases it. A Spinlock's ordinary shared
// savedFlags field cannot represent that -- it would restore the switching
// context's IF state into the resumed stack. Keep the flags with each saved
// process/idle context instead, so timer frames resume with IF=0 and syscall
// frames resume with their original IF=1 even after CPU migration.
class SchedulerSpinlock {
kcp::Spinlock lock;
uint64_t processFlags[MaxProcesses] = {};
uint64_t idleFlags[Smp::MaxCPUs] = {};
public:
void Acquire() {
uint64_t flags = lock.AcquireIrqSave();
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
processFlags[cpu->currentSlot] = flags;
} else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
idleFlags[cpu->cpuIndex] = flags;
}
}
void Release() {
uint64_t flags = 0; // Fresh startup remains IF=0 until JumpToUserMode's IRETQ.
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
flags = processFlags[cpu->currentSlot];
} else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
flags = idleFlags[cpu->cpuIndex];
}
lock.ReleaseIrqRestore(flags);
}
void ReleaseForStartup() {
lock.ReleaseIrqNoRestore();
}
};
static SchedulerSpinlock schedLock;
// Approximate count of Ready processes. Incremented/decremented
// under schedLock. Idle CPUs check this to avoid scanning all 256
// process slots on every timer tick.
// Updated while schedLock is held, but sampled locklessly by idle CPUs and
// reschedule-IPI paths. Atomic loads make those wakeup decisions coherent
// across cores instead of relying on volatile's compiler-only semantics.
static std::atomic<int> readyCount{0};
// Per-object wake generations close the readiness-check/enrollment gap.
// Each bucket packs [63:32] = a tag identifying the last object to signal
// it and [31:0] = a signal counter, published together in one atomic word.
// Publishing both halves atomically lets a waiter distinguish "exactly one
// signal landed here and it carried someone else's tag" (safe to keep
// blocking) from "something ambiguous happened" (must treat as a wake).
// Without the tag, an unrelated hot object sharing the bucket forces the
// waiter to re-run its condition on every signal. Tag collisions only cost
// a spurious wake; they can never lose one, which is the only direction
// that would reintroduce the lost-wakeup bug this table exists to prevent.
static constexpr uint32_t ObjectWakeBuckets = 1024;
static std::atomic<uint64_t> objectWakeEpochs[ObjectWakeBuckets] = {};
static uint32_t ObjectWakeBucket(void* object) {
uint64_t value = (uint64_t)object;
return (uint32_t)(((value >> 4) ^ (value >> 12)) & (ObjectWakeBuckets - 1));
}
static uint32_t ObjectWakeTag(void* object) {
uint64_t value = (uint64_t)object;
value ^= value >> 32;
value *= 0x9E3779B97F4A7C15ULL;
return (uint32_t)(value >> 32);
}
// The idle loop runs in the kernel PML4
static uint64_t GetKernelCR3() {
return (uint64_t)Memory::VMM::g_paging->PML4;
}
static void RescheduleIpiHandler(uint8_t, bool interruptedUser) {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr) {
return;
}
// Process teardown uses this IPI to promptly stop sibling threads on
// other CPUs. A user-mode sibling can exit immediately. If the IPI
// interrupted a syscall, merely expire its slice: teardown on that
// suspended kernel stack could reacquire a mutex it already owns.
if (cpu->currentSlot >= 0 &&
processTable[cpu->currentSlot].killPending.load(std::memory_order_acquire)) {
if (interruptedUser) {
ExitProcess();
return;
}
processTable[cpu->currentSlot].sliceRemaining = 0;
return;
}
if (cpu->currentSlot >= 0 || readyCount <= 0 || cpu->reservedForKernelWork) {
return;
}
Schedule();
}
static void KickOneIdleCpu(int sourceCpuIndex) {
if (readyCount <= 0) {
return;
}
auto tryKick = [&](Smp::CpuData* target) {
if (target == nullptr || !target->started) return false;
if (target->cpuIndex == sourceCpuIndex) return false;
if (target->currentSlot >= 0) return false;
if (target->reservedForKernelWork) return false;
return Hal::LocalApic::SendFixedIpi(target->lapicId,
Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
};
if (tryKick(Smp::GetCpuData(0))) {
return;
}
for (int i = 0; i < Smp::GetCpuCount(); i++) {
if (tryKick(Smp::GetCpuData(i))) {
return;
}
}
}
static void SwitchAwayFromBlockedCurrentLocked() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) {
schedLock.Release();
return;
}
int next = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Ready) {
next = i;
break;
}
}
if (next >= 0) {
cpu->currentSlot = next;
processTable[next].state = ProcessState::Running;
readyCount--;
processTable[next].runningOnCpu = cpu->cpuIndex;
processTable[next].sliceRemaining = TimeSliceMs;
cpu->kernelRsp = processTable[next].kernelStackTop;
cpu->tss->rsp0 = processTable[next].kernelStackTop;
LoadUserFsBase(cpu, processTable[next].fsBase);
SchedContextSwitch(&processTable[slot].savedRsp, processTable[next].savedRsp,
processTable[next].pml4Phys,
processTable[slot].fpuState, processTable[next].fpuState);
schedLock.Release();
return;
}
cpu->currentSlot = -1;
SchedContextSwitch(&processTable[slot].savedRsp, cpu->idleSavedRsp,
GetKernelCR3(), processTable[slot].fpuState, nullptr);
schedLock.Release();
}
// Startup function for newly spawned processes.
// SchedContextSwitch "returns" here on first schedule.
// The schedLock is held (acquired by the switching-from CPU's Schedule).
static void ProcessStartup() {
// Release the schedLock that the switching-from CPU held
// without consulting this brand-new slot's (intentionally empty)
// saved interrupt state. JumpToUserMode's IRETQ enables interrupts.
schedLock.ReleaseForStartup();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot >= 0) {
Process& proc = processTable[slot];
// Set up per-CPU kernel RSP for SYSCALL entry
cpu->kernelRsp = proc.kernelStackTop;
// Set up per-CPU TSS RSP0 for hardware interrupts from ring 3
cpu->tss->rsp0 = proc.kernelStackTop;
LoadUserFsBase(cpu, proc.fsBase);
// Jump to user mode (never returns).
// For main threads threadArg is 0 (libc _start ignores RDI);
// for sibling threads it carries the user-supplied argument.
JumpToUserMode(proc.entryPoint, proc.userStackTop, proc.threadArg);
}
ExitProcess();
for (;;) {
asm volatile("hlt");
}
}
void Initialize() {
for (int i = 0; i < MaxProcesses; i++) {
processTable[i].pid = i;
processTable[i].state = ProcessState::Free;
processTable[i].name[0] = '\0';
processTable[i].savedRsp = 0;
processTable[i].stackBase = 0;
processTable[i].entryPoint = 0;
processTable[i].sliceRemaining = 0;
processTable[i].cpuTimeMs = 0;
processTable[i].pml4Phys = 0;
processTable[i].kernelStackTop = 0;
processTable[i].userStackTop = 0;
processTable[i].heapNext = 0;
processTable[i].readdirCursor = 0;
processTable[i].args[0] = '\0';
processTable[i].environment[0] = '\0';
processTable[i].environmentLength = 1;
processTable[i].user[0] = '\0';
processTable[i].effectiveCaps = 0;
processTable[i].delegableCaps = 0;
processTable[i].consentCeiling = 0;
processTable[i].consentDenied = 0;
processTable[i].sessionId = -1;
processTable[i].cwd[0] = '\0';
processTable[i].runningOnCpu = -1;
processTable[i].killPending = false;
processTable[i].reapReady = false;
processTable[i].startPending = false;
processTable[i].waitingForPid = -1;
processTable[i].sleepUntilTick = 0;
processTable[i].waitingOnObject = nullptr;
processTable[i].redirected = false;
processTable[i].parentPid = -1;
processTable[i].outBuf = nullptr;
processTable[i].outHead = 0;
processTable[i].outTail = 0;
processTable[i].inBuf = nullptr;
processTable[i].inHead = 0;
processTable[i].inTail = 0;
processTable[i].keyHead = 0;
processTable[i].keyTail = 0;
processTable[i].getcharUtf8Len = 0;
processTable[i].getcharUtf8Pos = 0;
processTable[i].termCols = 0;
processTable[i].termRows = 0;
processTable[i].ioOutHandle = -1;
processTable[i].ioInHandle = -1;
processTable[i].ioKeyHandle = -1;
processTable[i].ioWaitsetHandle = -1;
processTable[i].primarySlot = -1;
processTable[i].joinerSlot = -1;
processTable[i].exitCode = 0;
processTable[i].joinable = true;
}
nextPid = 0;
for (int i = 0; i < ExitLedgerSize; i++) {
exitLedgerPid[i] = -1;
exitLedgerCode[i] = 0;
}
exitLedgerHead = 0;
Hal::RegisterIrqHandler(Hal::IRQ_RESCHEDULE, RescheduleIpiHandler);
Kt::KernelLogStream(Kt::OK, "Sched") << "Initialized (" << MaxProcesses
<< " process slots, " << (uint64_t)TimeSliceMs << " ms time slice)";
}
int Spawn(const char* vfsPath, const char* args, bool startReady,
const char* environment, uint32_t environmentLength,
const montauk::abi::SpawnCapabilities* capabilities,
const char* userOverride) {
// Grants are keyed on the image path, so they are only as trustworthy
// as the path: anything outside 0:/os and 0:/apps can be replaced by
// any process, which would then run with the grant the next time a
// privileged launcher starts it. Such a program starts with nothing
// and asks through consent instead.
montauk::abi::SpawnCapabilities clamped{0, 0};
if (capabilities != nullptr &&
(capabilities->effective | capabilities->delegable) != 0 &&
!Fs::IsSystemImage(vfsPath)) {
Kt::KernelLogStream(Kt::WARNING, "Sched") << "Not an installed program, "
<< "starting without capabilities: " << vfsPath;
capabilities = &clamped;
}
schedLock.Acquire();
int slot = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Free) {
slot = i;
break;
}
}
if (slot < 0) {
schedLock.Release();
Kt::KernelLogStream(Kt::ERROR, "Sched") << "No free process slots";
return -1;
}
// Reserve the slot so another Spawn doesn't claim it.
// Use Running (not Ready!) so the scheduler doesn't try to
// dispatch this half-initialized process.
processTable[slot].state = ProcessState::Running;
processTable[slot].runningOnCpu = -1;
processTable[slot].reapReady = false;
schedLock.Release();
// Create per-process PML4 with kernel-half copied
uint64_t pml4Phys = Memory::VMM::Paging::CreateUserPML4();
// Load ELF into the process's address space
TlsInfo tls{};
uint64_t entry = ElfLoad(vfsPath, pml4Phys, &tls);
if (entry == 0) {
Memory::VMM::Paging::FreeUserHalf(pml4Phys);
Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys));
schedLock.Acquire();
processTable[slot].state = ProcessState::Free;
schedLock.Release();
return -1;
}
// Allocate kernel stack (used during syscalls and interrupts).
// ReallocConsecutive(nullptr, n) returns an n-page contiguous span;
// the previous code allocated a throwaway page first and asked
// ReallocConsecutive to migrate from it, which copied + freed the
// throwaway for no benefit and double-counted failure paths.
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
if (stackMem == nullptr) {
Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to allocate contiguous kernel stack";
Memory::VMM::Paging::FreeUserHalf(pml4Phys);
Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys));
schedLock.Acquire();
processTable[slot].state = ProcessState::Free;
schedLock.Release();
return -1;
}
memset(stackMem, 0, StackPages * 0x1000);
uint8_t* kernelStackBase = (uint8_t*)stackMem;
uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize;
// Helper to clean up all resources allocated so far on failure
auto cleanupOnFail = [&]() {
Memory::VMM::Paging::FreeUserHalf(pml4Phys);
Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys));
Memory::g_pfa->Free(stackMem, StackPages);
schedLock.Acquire();
processTable[slot].state = ProcessState::Free;
schedLock.Release();
};
// Allocate user stack pages and map them in the process PML4
uint64_t userStackBase = UserStackTop - UserStackSize;
uint64_t topStackPagePhys = 0;
for (uint64_t i = 0; i < UserStackPages; i++) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
Kt::KernelLogStream(Kt::ERROR, "Sched") << "Out of memory for user stack";
cleanupOnFail();
return -1;
}
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(pml4Phys, physAddr, userStackBase + i * 0x1000)) {
Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to map user stack page";
Memory::g_pfa->Free(page);
cleanupOnFail();
return -1;
}
if (i == UserStackPages - 1) topStackPagePhys = physAddr;
}
// Allocate and map a user-space exit stub page.
{
void* stubPage = Memory::g_pfa->AllocateZeroed();
if (stubPage == nullptr) {
Kt::KernelLogStream(Kt::ERROR, "Sched") << "Out of memory for exit stub";
cleanupOnFail();
return -1;
}
uint64_t stubPhys = Memory::SubHHDM((uint64_t)stubPage);
if (!Memory::VMM::Paging::MapUserIn(pml4Phys, stubPhys, ExitStubAddr)) {
Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to map exit stub";
Memory::g_pfa->Free(stubPage);
cleanupOnFail();
return -1;
}
// Write: xor edi, edi; xor eax, eax; syscall
uint8_t* stub = (uint8_t*)stubPage;
stub[0] = 0x31; stub[1] = 0xFF; // xor edi, edi (exit code 0)
stub[2] = 0x31; stub[3] = 0xC0; // xor eax, eax (SYS_EXIT = 0)
stub[4] = 0x0F; stub[5] = 0x05; // syscall
}
// Push exit stub address as the return address on the user stack.
{
uint8_t* topPage = (uint8_t*)Memory::HHDM(topStackPagePhys);
*(uint64_t*)(topPage + 0xFF8) = ExitStubAddr;
}
// Set up the initial kernel stack frame so that SchedContextSwitch
// "returns" into ProcessStartup
uint64_t* sp = (uint64_t*)kernelStackTop;
*(--sp) = (uint64_t)ProcessStartup; // return addr
*(--sp) = 0; // rbp
*(--sp) = 0; // rbx
*(--sp) = 0; // r12
*(--sp) = 0; // r13
*(--sp) = 0; // r14
*(--sp) = 0; // r15
schedLock.Acquire();
Process& proc = processTable[slot];
proc.pid = nextPid++;
proc.state = startReady ? ProcessState::Ready : ProcessState::Blocked;
if (startReady)
readyCount++;
{
int i = 0;
for (; i < 63 && vfsPath[i]; i++) proc.name[i] = vfsPath[i];
proc.name[i] = '\0';
}
proc.primarySlot = slot; // main thread owns per-process state
proc.joinerSlot = -1;
proc.exitCode = 0;
proc.joinable = false; // main thread is reaped by BSP, not joined
proc.futexAddr = 0;
proc.futexWoken = false;
proc.savedRsp = (uint64_t)sp;
proc.stackBase = (uint64_t)kernelStackBase;
proc.entryPoint = entry;
proc.sliceRemaining = TimeSliceMs;
proc.cpuTimeMs = 0;
proc.pml4Phys = pml4Phys;
proc.kernelStackTop = kernelStackTop;
proc.userStackTop = UserStackTop - 8;
proc.heapNext = UserHeapBase;
for (uint64_t i = 0; i < UserHeapBitmapWords; i++)
g_userHeapPageMap[slot][i] = 0;
proc.fsBase = tls.fsBase;
proc.tlsTemplateVaddr = tls.templateVaddr;
proc.tlsFileSize = tls.fileSize;
proc.tlsMemSize = tls.memSize;
proc.tlsAlign = tls.align;
proc.readdirCursor = 0;
proc.runningOnCpu = -1;
proc.killPending = false;
proc.reapReady = false;
proc.startPending = !startReady;
proc.waitingForPid = -1;
proc.sleepUntilTick = 0;
proc.waitingOnObject = nullptr;
auto* currentCpu = Smp::GetCurrentCpuData();
int parentSlot = currentCpu ? currentCpu->currentSlot : -1;
int parentPrimarySlot = parentSlot;
if (parentPrimarySlot >= 0 && processTable[parentPrimarySlot].primarySlot >= 0)
parentPrimarySlot = processTable[parentPrimarySlot].primarySlot;
// Copy arguments string into process
proc.args[0] = '\0';
if (args != nullptr) {
int i = 0;
for (; i < 4095 && args[i]; i++) {
proc.args[i] = args[i];
}
proc.args[i] = '\0';
}
// posix_spawn(envp) supplies an explicit snapshot. Plain spawn calls
// omit it and inherit the environment stored on the parent process.
if (environment != nullptr && environmentLength > 0 &&
environmentLength <= Process::EnvironmentBytes) {
for (uint32_t i = 0; i < environmentLength; i++)
proc.environment[i] = environment[i];
proc.environmentLength = environmentLength;
} else if (parentPrimarySlot >= 0) {
auto& parent = processTable[parentPrimarySlot];
for (uint32_t i = 0; i < parent.environmentLength; i++)
proc.environment[i] = parent.environment[i];
proc.environmentLength = parent.environmentLength;
} else {
proc.environment[0] = '\0';
proc.environmentLength = 1;
}
// Capabilities are kernel-owned and never inferred from the user name.
// A normal userspace spawn receives no privileged authority; callers
// must use SYS_SPAWN_CAPS for an explicit, kernel-validated delegation.
if (parentPrimarySlot >= 0) {
if (capabilities != nullptr) {
proc.effectiveCaps = capabilities->effective;
proc.delegableCaps = capabilities->delegable;
} else {
proc.effectiveCaps = 0;
proc.delegableCaps = 0;
}
} else {
// The kernel-created init process is the root of the capability
// delegation tree. No userspace pathname or PID receives this
// treatment; it is reached only with no current parent process.
proc.effectiveCaps = montauk::abi::CAP_ALL;
proc.delegableCaps = montauk::abi::CAP_ALL;
}
// Consent can later grant the child exactly the eligible authority
// its parent could have delegated to it now.
proc.consentCeiling = (parentPrimarySlot >= 0)
? processTable[parentPrimarySlot].delegableCaps & montauk::abi::CAP_CONSENT_ELIGIBLE
: 0;
proc.consentDenied = 0;
{
int i = 0;
for (; i < (int)sizeof(proc.imagePath) - 1 && vfsPath[i]; i++)
proc.imagePath[i] = vfsPath[i];
proc.imagePath[i] = '\0';
}
// Inherit user string from parent, or default to "system" if no parent.
// An explicit override is accepted only through the validated
// SYS_SPAWN_CAPS path.
{
if (userOverride != nullptr) {
int i = 0;
for (; i < 31 && userOverride[i]; i++) proc.user[i] = userOverride[i];
proc.user[i] = '\0';
} else if (parentSlot >= 0) {
int i = 0;
for (; i < 31 && processTable[parentSlot].user[i]; i++)
proc.user[i] = processTable[parentSlot].user[i];
proc.user[i] = '\0';
} else {
// Spawned from kernel (no parent process) - set to "system"
proc.user[0] = 's'; proc.user[1] = 'y'; proc.user[2] = 's';
proc.user[3] = 't'; proc.user[4] = 'e'; proc.user[5] = 'm';
proc.user[6] = '\0';
}
}
// Process sessions are explicit groups used by supervisors such as
// login.elf. A session leader opts in with CreateSession(); all of its
// subsequently spawned descendants inherit the same stable ID.
proc.sessionId = (parentPrimarySlot >= 0)
? processTable[parentPrimarySlot].sessionId
: -1;
{
if (parentSlot >= 0 && processTable[parentSlot].cwd[0]) {
int i = 0;
for (; i < 255 && processTable[parentSlot].cwd[i]; i++) {
proc.cwd[i] = processTable[parentSlot].cwd[i];
}
proc.cwd[i] = '\0';
} else {
proc.cwd[0] = '0';
proc.cwd[1] = ':';
proc.cwd[2] = '/';
proc.cwd[3] = '\0';
}
}
proc.redirected = false;
proc.parentPid = (parentSlot >= 0) ? processTable[parentSlot].pid : -1;
proc.outBuf = nullptr;
proc.outHead = 0;
proc.outTail = 0;
proc.inBuf = nullptr;
proc.inHead = 0;
proc.inTail = 0;
proc.keyHead = 0;
proc.keyTail = 0;
proc.getcharUtf8Len = 0;
proc.getcharUtf8Pos = 0;
proc.termCols = 0;
proc.termRows = 0;
proc.ioOutHandle = -1;
proc.ioInHandle = -1;
proc.ioKeyHandle = -1;
proc.ioWaitsetHandle = -1;
// Initialize FPU state: zero out, then set default FCW and MXCSR
memset(proc.fpuState, 0, 512);
*(uint16_t*)&proc.fpuState[0] = 0x037F; // FCW: default x87 control word
*(uint32_t*)&proc.fpuState[24] = 0x1F80; // MXCSR: default SSE control/status
Ipc::ProcessStartedInSlot(slot, proc.pid);
int resultPid = proc.pid;
schedLock.Release();
if (startReady)
KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1);
return resultPid;
}
// ====================================================================
// Threading
//
// Every thread is a slot in processTable. The main thread's primarySlot
// points to itself and owns process-level state (PML4, IPC handles,
// cwd/user/args, redirected I/O). A sibling thread copies pml4Phys from
// the primary so context-switch is cheap, but per-process getters use
// primarySlot to reach the canonical state.
//
// Sibling-thread lifecycle:
// * SpawnThread: allocate slot + kernel stack; user provides user stack.
// * ExitCurrentThread: slot -> Terminated, kernel stack kept alive for
// the joiner; joiner reads exitCode and frees the kernel stack.
// * ExitProcess from the main thread sweeps any still-Terminated
// sibling slots (no joiner ever arrived) and tears the rest down.
// ====================================================================
int SpawnThread(uint64_t entry, uint64_t arg, uint64_t userStackTop) {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr || cpu->currentSlot < 0) return -1;
if (entry == 0 || userStackTop < 16) return -1;
int callerSlot = cpu->currentSlot;
int primarySlot_ = processTable[callerSlot].primarySlot;
if (primarySlot_ < 0) primarySlot_ = callerSlot;
uint64_t sharedPml4 = processTable[primarySlot_].pml4Phys;
int processPid = processTable[primarySlot_].pid;
if (sharedPml4 == 0) return -1;
// Give the new thread its own TLS block when the image has one.
// We are executing inside the process's address space, so the
// freshly mapped pages and the template are directly addressable.
// The block uses process-heap virtual space. Its physical pages are
// reclaimed when the sibling exits (or if creation fails).
uint64_t threadFsBase = 0;
uint64_t threadTlsBase = 0;
uint64_t threadTlsPages = 0;
Process& primary = processTable[primarySlot_];
if (primary.tlsMemSize > 0) {
TlsLayout layout = ComputeTlsLayout(primary.tlsMemSize, primary.tlsAlign);
uint64_t numPages = (layout.totalSize + 0xFFF) / 0x1000;
uint64_t base = 0;
if (!ReserveUserHeapRange(primarySlot_, numPages * 0x1000, base)) {
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Thread TLS exceeds user heap range";
return -1;
}
bool ok = true;
uint64_t mappedPages = 0;
for (uint64_t p = 0; p < numPages; p++) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
ok = false;
break;
}
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(sharedPml4, physAddr,
base + p * 0x1000)) {
Memory::g_pfa->Free(page);
ok = false;
break;
}
mappedPages++;
}
if (!ok) {
Memory::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
ReleaseUserHeapRange(primarySlot_, base, numPages * 0x1000ULL);
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Thread TLS allocation failed";
return -1;
}
uint64_t tp = base + layout.tpOffset;
memcpy((void*)(base + layout.templateOffset),
(const void*)primary.tlsTemplateVaddr, primary.tlsFileSize);
*(uint64_t*)tp = tp; /* ABI self-pointer */
threadFsBase = tp;
threadTlsBase = base;
threadTlsPages = numPages;
}
// We do not write to the user stack from kernel mode (that would
// require validating the user VA against the process page tables).
// Userspace is responsible for ensuring the thread entry calls
// SYS_THREAD_EXIT; the libc trampoline does this. The kernel only
// enforces SystemV alignment by handing the entry RSP = 16n - 8.
uint64_t userRsp = (userStackTop & ~0xFULL) - 8;
// Allocate kernel stack.
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
if (stackMem == nullptr) {
if (threadTlsPages != 0) {
Memory::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
threadTlsPages * 0x1000ULL);
}
return -1;
}
memset(stackMem, 0, StackSize);
uint8_t* kernelStackBase = (uint8_t*)stackMem;
uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize;
// Initial kernel frame -- SchedContextSwitch "returns" into ProcessStartup.
uint64_t* sp = (uint64_t*)kernelStackTop;
*(--sp) = (uint64_t)ProcessStartup;
*(--sp) = 0; *(--sp) = 0; *(--sp) = 0;
*(--sp) = 0; *(--sp) = 0; *(--sp) = 0;
schedLock.Acquire();
int slot = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Free) { slot = i; break; }
}
if (slot < 0) {
schedLock.Release();
Memory::g_pfa->Free(stackMem, StackPages);
if (threadTlsPages != 0) {
Memory::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
threadTlsPages * 0x1000ULL);
}
return -1;
}
Process& thr = processTable[slot];
thr.pid = nextPid++;
thr.state = ProcessState::Ready;
readyCount++;
thr.runningOnCpu = -1;
thr.killPending = false;
thr.reapReady = false;
thr.startPending = false;
thr.waitingForPid = -1;
thr.sleepUntilTick = 0;
thr.waitingOnObject = nullptr;
thr.primarySlot = primarySlot_;
thr.joinerSlot = -1;
thr.exitCode = 0;
thr.joinable = true;
thr.threadArg = arg;
thr.futexAddr = 0;
thr.futexWoken = false;
thr.savedRsp = (uint64_t)sp;
thr.stackBase = (uint64_t)kernelStackBase;
thr.entryPoint = entry;
thr.userStackTop = userRsp;
thr.kernelStackTop = kernelStackTop;
thr.pml4Phys = sharedPml4; // shared with primary, NOT owned
thr.sliceRemaining = TimeSliceMs;
thr.cpuTimeMs = 0;
thr.heapNext = 0;
thr.readdirCursor = 0;
thr.parentPid = processPid;
thr.redirected = false;
thr.outBuf = nullptr; thr.outHead = 0; thr.outTail = 0;
thr.inBuf = nullptr; thr.inHead = 0; thr.inTail = 0;
thr.keyHead = 0; thr.keyTail = 0;
thr.getcharUtf8Len = 0; thr.getcharUtf8Pos = 0;
thr.termCols = 0; thr.termRows = 0;
thr.ioOutHandle = -1; thr.ioInHandle = -1;
thr.ioKeyHandle = -1; thr.ioWaitsetHandle = -1;
thr.args[0] = '\0'; thr.user[0] = '\0'; thr.cwd[0] = '\0';
thr.fsBase = threadFsBase;
thr.tlsTemplateVaddr = 0;
thr.tlsFileSize = 0;
thr.tlsMemSize = 0;
thr.tlsAlign = 0;
// Derive a debug name from the primary's name.
{
const char* base = processTable[primarySlot_].name;
int i = 0;
for (; i < 58 && base[i]; i++) thr.name[i] = base[i];
thr.name[i++] = ':';
thr.name[i++] = 't';
thr.name[i] = '\0';
}
memset(thr.fpuState, 0, 512);
*(uint16_t*)&thr.fpuState[0] = 0x037F;
*(uint32_t*)&thr.fpuState[24] = 0x1F80;
int tid = thr.pid;
schedLock.Release();
KickOneIdleCpu(cpu->cpuIndex);
return tid;
}
bool ReserveUserHeapRange(int primarySlot_, uint64_t size, uint64_t& outVa) {
outVa = 0;
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
(size & 0xFFFULL) != 0) {
return false;
}
schedLock.Acquire();
Process& primary = processTable[primarySlot_];
bool validOwner = primary.primarySlot == primarySlot_ &&
primary.state != ProcessState::Free &&
primary.state != ProcessState::Terminated;
bool fits = false;
uint64_t pages = size / 0x1000ULL;
uint64_t runStart = 0;
uint64_t runLength = 0;
if (validOwner && pages <= UserHeapPages) {
for (uint64_t page = 0; page < UserHeapPages; page++) {
bool used = (g_userHeapPageMap[primarySlot_][page / 64]
& (1ULL << (page % 64))) != 0;
if (used) {
runLength = 0;
continue;
}
if (runLength == 0) runStart = page;
runLength++;
if (runLength == pages) {
for (uint64_t p = runStart; p < runStart + pages; p++)
g_userHeapPageMap[primarySlot_][p / 64] |= 1ULL << (p % 64);
outVa = UserHeapBase + runStart * 0x1000ULL;
uint64_t end = outVa + size;
if (end > primary.heapNext) primary.heapNext = end;
fits = true;
break;
}
}
}
schedLock.Release();
return fits;
}
void ReleaseUserHeapRange(int primarySlot_, uint64_t va, uint64_t size) {
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
(va & 0xFFFULL) != 0 || (size & 0xFFFULL) != 0 ||
va < UserHeapBase || va > UserHeapLimit || size > UserHeapLimit - va) {
return;
}
uint64_t first = (va - UserHeapBase) / 0x1000ULL;
uint64_t pages = size / 0x1000ULL;
schedLock.Acquire();
for (uint64_t p = first; p < first + pages; p++)
g_userHeapPageMap[primarySlot_][p / 64] &= ~(1ULL << (p % 64));
schedLock.Release();
}
static void FreeSiblingThreadTls(Process& thr) {
int primarySlot_ = thr.primarySlot;
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
primarySlot_ == (int)(&thr - processTable) || thr.fsBase == 0) {
return;
}
Process& primary = processTable[primarySlot_];
if (primary.tlsMemSize == 0 || primary.pml4Phys == 0) {
thr.fsBase = 0;
return;
}
TlsLayout layout = ComputeTlsLayout(primary.tlsMemSize, primary.tlsAlign);
if (layout.tpOffset > thr.fsBase || layout.totalSize < layout.tpOffset) {
Panic("Invalid sibling TLS metadata during thread teardown", nullptr);
}
uint64_t base = thr.fsBase - layout.tpOffset;
uint64_t pages = (layout.totalSize + 0xFFF) / 0x1000;
thr.fsBase = 0;
Memory::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
ReleaseUserHeapRange(primarySlot_, base, pages * 0x1000ULL);
}
// Switch away from a slot we have just marked non-runnable while holding
// schedLock. Mirrors SwitchAwayFromBlockedCurrentLocked but does NOT
// assume the caller will be resumed -- used by ExitCurrentThread.
static void SwitchAwayFromExitingThreadLocked(int slot, Process& thr) {
auto* cpu = Smp::GetCurrentCpuData();
int next = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Ready) { next = i; break; }
}
if (next >= 0) {
cpu->currentSlot = next;
processTable[next].state = ProcessState::Running;
readyCount--;
processTable[next].runningOnCpu = cpu->cpuIndex;
processTable[next].sliceRemaining = TimeSliceMs;
cpu->kernelRsp = processTable[next].kernelStackTop;
cpu->tss->rsp0 = processTable[next].kernelStackTop;
if (readyCount > 0) {
KickOneIdleCpu(cpu->cpuIndex);
}
LoadUserFsBase(cpu, processTable[next].fsBase);
SchedContextSwitch(&thr.savedRsp, processTable[next].savedRsp,
processTable[next].pml4Phys,
thr.fpuState, processTable[next].fpuState);
} else {
cpu->currentSlot = -1;
SchedContextSwitch(&thr.savedRsp, cpu->idleSavedRsp,
GetKernelCR3(),
thr.fpuState, nullptr);
}
schedLock.Release();
(void)slot;
}
[[noreturn]] void ExitCurrentThread(int exitCode) {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) {
for (;;) asm volatile("hlt");
}
Process& thr = processTable[slot];
int primarySlot_ = thr.primarySlot;
if (primarySlot_ < 0 || primarySlot_ == slot) {
// This is the main thread (or an orphan): full process exit.
thr.exitCode = exitCode;
ExitProcess();
__builtin_unreachable();
}
bool processExitKill =
thr.killPending.exchange(false, std::memory_order_acq_rel);
thr.startPending = false;
thr.exitCode = exitCode;
// TLS belongs only to this sibling. Reclaim it before publishing the
// Terminated state; otherwise a long-lived multithreaded GUI process
// leaks these pages until the entire process exits. When the primary
// is already coordinating full teardown, defer this to its final
// sibling sweep: the primary may be waiting from an IF=0 timer frame
// and therefore cannot acknowledge our TLB-shootdown IPI here.
if (!processExitKill)
FreeSiblingThreadTls(thr);
schedLock.Acquire();
thr.state = ProcessState::Terminated;
thr.runningOnCpu = -1;
thr.waitingForPid = -1;
thr.sleepUntilTick = 0;
thr.waitingOnObject = nullptr;
// Wake any joiner blocked on this slot.
void* joinObj = &processTable[slot];
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Blocked &&
processTable[i].waitingOnObject == joinObj) {
processTable[i].state = ProcessState::Ready;
readyCount++;
processTable[i].waitingOnObject = nullptr;
processTable[i].sleepUntilTick = 0;
processTable[i].waitingForPid = -1;
}
}
SwitchAwayFromExitingThreadLocked(slot, thr);
// Unreachable: the slot is Terminated, nobody resumes us.
for (;;) asm volatile("hlt");
}
int JoinThread(int tid, int* outExitCode) {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr || cpu->currentSlot < 0) return -1;
int callerSlot = cpu->currentSlot;
int callerPrimary = processTable[callerSlot].primarySlot;
if (callerPrimary < 0) callerPrimary = callerSlot;
if (tid <= 0) return -1;
// Locate the sibling slot under lock.
schedLock.Acquire();
int target = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (i == callerSlot) continue;
if (processTable[i].pid != tid) continue;
if (processTable[i].primarySlot != callerPrimary) continue;
auto s = processTable[i].state;
if (s == ProcessState::Free) continue;
target = i;
break;
}
if (target < 0) {
schedLock.Release();
return -1;
}
// Fast path: target already Terminated -- harvest immediately.
if (processTable[target].state == ProcessState::Terminated) {
int code = processTable[target].exitCode;
void* stackBase = (void*)processTable[target].stackBase;
processTable[target].stackBase = 0;
processTable[target].pml4Phys = 0;
processTable[target].joinerSlot = -1;
processTable[target].primarySlot = -1;
processTable[target].state = ProcessState::Free;
schedLock.Release();
if (stackBase) Memory::g_pfa->Free(stackBase, StackPages);
if (outExitCode) *outExitCode = code;
return 0;
}
// Block until the sibling terminates and wakes us.
processTable[target].joinerSlot = callerSlot;
processTable[callerSlot].state = ProcessState::Blocked;
processTable[callerSlot].waitingOnObject = &processTable[target];
processTable[callerSlot].waitingForPid = -1;
processTable[callerSlot].sleepUntilTick = 0;
processTable[callerSlot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
// Resumed -- harvest the sibling.
schedLock.Acquire();
int code = processTable[target].exitCode;
void* stackBase = (void*)processTable[target].stackBase;
processTable[target].stackBase = 0;
processTable[target].pml4Phys = 0;
processTable[target].joinerSlot = -1;
processTable[target].primarySlot = -1;
processTable[target].state = ProcessState::Free;
schedLock.Release();
if (stackBase) Memory::g_pfa->Free(stackBase, StackPages);
if (outExitCode) *outExitCode = code;
return 0;
}
int StartProcess(int pid) {
if (pid < 0) return -1;
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid != pid) continue;
if (processTable[i].state != ProcessState::Blocked ||
!processTable[i].startPending) {
schedLock.Release();
return -1;
}
processTable[i].startPending = false;
processTable[i].waitingForPid = -1;
processTable[i].sleepUntilTick = 0;
processTable[i].waitingOnObject = nullptr;
processTable[i].state = ProcessState::Ready;
readyCount++;
schedLock.Release();
KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1);
return 0;
}
schedLock.Release();
return -1;
}
// ====================================================================
// Schedule -- core context switch logic
//
// The schedLock is held ACROSS the context switch. This is critical:
// setting the old process to Ready and saving its RSP must be atomic
// with respect to other CPUs. If we released the lock before saving
// RSP, another CPU could pick up the process with a stale savedRsp.
//
// The RESUMED process releases the lock (it was acquired by whatever
// CPU called Schedule() and switched away from that process).
// New processes release it in ProcessStartup().
// ====================================================================
// BSP maintenance that may allocate/free memory or touch policy MSRs must
// run from the idle kernel context, never from the timer interrupt. The
// pending flag is set by process teardown on any CPU and periodically by
// the BSP timer. It remains set until an idle maintenance pass consumes it.
static std::atomic<bool> bspMaintenancePending{false};
static uint64_t nextBspPolicyTick = 0; // BSP timer only
// Reclaim terminated process slots from BSP idle context. This avoids
// holding schedLock (with interrupts disabled) during PFA::Free on the
// hot scheduling path or inside a hard interrupt.
static void ReclaimTerminated() {
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Terminated && processTable[i].reapReady) {
// Grab the pointers, mark Free, then free memory after releasing lock
void* stackBase = (processTable[i].stackBase != 0)
? (void*)processTable[i].stackBase : nullptr;
void* pml4 = (processTable[i].pml4Phys != 0)
? (void*)Memory::HHDM(processTable[i].pml4Phys) : nullptr;
processTable[i].stackBase = 0;
processTable[i].pml4Phys = 0;
processTable[i].reapReady = false;
processTable[i].startPending = false;
processTable[i].state = ProcessState::Free;
// Release lock during PFA::Free to minimize hold time
schedLock.Release();
if (stackBase) Memory::g_pfa->Free(stackBase, StackPages);
if (pml4) Memory::g_pfa->Free(pml4);
schedLock.Acquire();
}
}
schedLock.Release();
}
bool HasReadyProcesses() {
return readyCount > 0;
}
void RequestKernelWork() {
auto* cpu = Smp::GetCurrentCpuData();
if (cpu != nullptr && cpu->currentSlot >= 0) {
cpu->kernelWorkPending = true;
}
}
// Bounded, IRQ-safe portion of BSP maintenance. Keeping deadline expiry
// on every 1 ms BSP tick prevents timed input/object waits from depending
// on the BSP becoming idle under sustained desktop load.
static void WakeExpiredProcesses() {
bool wokeProcesses = false;
schedLock.Acquire();
uint64_t now = Timekeeping::GetTicks();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Blocked &&
processTable[i].sleepUntilTick != 0 &&
now >= processTable[i].sleepUntilTick) {
processTable[i].sleepUntilTick = 0;
processTable[i].waitingForPid = -1;
processTable[i].waitingOnObject = nullptr;
processTable[i].state = ProcessState::Ready;
readyCount++;
wokeProcesses = true;
}
}
schedLock.Release();
if (wokeProcesses) {
KickOneIdleCpu(0);
}
}
void RunBspMaintenance() {
// Clear before scanning. A producer that races this pass sets the flag
// again and therefore cannot have its request acknowledged by mistake.
if (!bspMaintenancePending.exchange(false, std::memory_order_acq_rel)) {
return;
}
ReclaimTerminated();
// Thermal governor step (rate-limited internally to 500 ms).
// The timer requests this idle-context pass even under full load.
Hal::CpuPower::ThermalTick(Timekeeping::GetMilliseconds());
}
uint64_t GetNextDeadlineTick() {
uint64_t nextDeadline = 0;
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state != ProcessState::Blocked) continue;
uint64_t deadline = processTable[i].sleepUntilTick;
if (deadline == 0) continue;
if (nextDeadline == 0 || deadline < nextDeadline) {
nextDeadline = deadline;
}
}
schedLock.Release();
return nextDeadline;
}
void Schedule() {
auto* cpu = Smp::GetCurrentCpuData();
schedLock.Acquire();
// Normally select the next Ready process. A pending bottom half gets
// one deliberate trip through the idle kernel context first; that
// context drains bounded work and immediately schedules user work
// again. This avoids both hard-IRQ callbacks and bottom-half starvation
// when there is always at least one runnable desktop process.
int next = -1;
bool yieldToKernel = cpu->currentSlot >= 0 && cpu->kernelWorkPending;
if (yieldToKernel) cpu->kernelWorkPending = false;
int start = (cpu->currentSlot >= 0) ? cpu->currentSlot + 1 : 0;
if (!yieldToKernel) {
for (int i = 0; i < MaxProcesses; i++) {
int idx = (start + i) % MaxProcesses;
if (processTable[idx].state == ProcessState::Ready) {
next = idx;
break;
}
}
}
if (next < 0) {
// No other process is ready. A normal time-slice expiry must keep
// the sole Running process on this CPU: bouncing it through idle
// can suspend a syscall while it owns a process-context Mutex, then
// let ServiceDeferredWork spin on that same mutex with the idle
// context reserved against scheduling. That is a permanent
// self-deadlock. Only an explicit, ring-3-safe bottom-half request
// is allowed to yield a running process to idle.
if (cpu->currentSlot >= 0) {
if (!yieldToKernel) {
processTable[cpu->currentSlot].sliceRemaining = TimeSliceMs;
schedLock.Release();
return;
}
int oldSlot = cpu->currentSlot;
processTable[oldSlot].state = ProcessState::Ready;
readyCount++;
processTable[oldSlot].runningOnCpu = -1;
cpu->currentSlot = -1;
// Lock held across context switch -- the idle loop
// doesn't release it; the next Schedule() that resumes
// a process from idle will release it via the resumed
// process path. But idle is special: we release here
// because idle doesn't go through ProcessStartup.
// The RSP save is safe because interrupts are disabled
// (Spinlock), so no timer can fire between Ready and save.
SchedContextSwitch(&processTable[oldSlot].savedRsp, cpu->idleSavedRsp,
GetKernelCR3(), processTable[oldSlot].fpuState, nullptr);
// Resumed from idle -- lock was held by whoever switched to us
schedLock.Release();
} else {
schedLock.Release();
}
return;
}
if (next == cpu->currentSlot) {
// Same process, just reset time slice
processTable[next].sliceRemaining = TimeSliceMs;
schedLock.Release();
return;
}
// Prepare the context switch
uint64_t* oldRspPtr;
int oldSlot = cpu->currentSlot;
if (oldSlot >= 0) {
processTable[oldSlot].state = ProcessState::Ready;
readyCount++;
processTable[oldSlot].runningOnCpu = -1;
oldRspPtr = &processTable[oldSlot].savedRsp;
} else {
oldRspPtr = &cpu->idleSavedRsp;
}
cpu->currentSlot = next;
processTable[next].state = ProcessState::Running;
readyCount--;
processTable[next].runningOnCpu = cpu->cpuIndex;
processTable[next].sliceRemaining = TimeSliceMs;
uint64_t newCR3 = processTable[next].pml4Phys;
// Update per-CPU kernel RSP and TSS RSP0
cpu->kernelRsp = processTable[next].kernelStackTop;
cpu->tss->rsp0 = processTable[next].kernelStackTop;
uint8_t* oldFpu = (oldSlot >= 0) ? processTable[oldSlot].fpuState : nullptr;
uint8_t* newFpu = processTable[next].fpuState;
if (oldSlot < 0) {
// AP idle loops mask their local periodic timer. Rearm it before
// dispatching user work so preemption resumes with the process.
// Also pick up a thermal-governor policy epoch that may have
// changed while this CPU remained asleep without timer ticks.
Hal::CpuPower::ApplyPolicyIfChanged();
Timekeeping::ApicTimerLeaveApIdle();
}
LoadUserFsBase(cpu, processTable[next].fsBase);
// DO NOT release schedLock here! It is held across the context
// switch so that setting Ready + saving RSP is atomic. The
// resumed process releases it.
SchedContextSwitch(oldRspPtr, processTable[next].savedRsp, newCR3, oldFpu, newFpu);
// We reach here when this process is resumed by another CPU's
// Schedule(). That CPU held the lock across its context switch.
// Release it now.
schedLock.Release();
}
void Tick(uint32_t elapsedMs, bool interruptedUser) {
auto* cpu = Smp::GetCurrentCpuData();
// Pick up thermal-governor frequency changes (HWP requests are
// per-core MSRs, so every CPU applies them itself). No-op unless
// the governor bumped the policy epoch since our last tick.
Hal::CpuPower::ApplyPolicyIfChanged();
// BSP: expire timed waits in the bounded IRQ-safe pass. Queue heavier
// reclamation/policy work for the idle kernel context.
if (cpu->cpuIndex == 0) {
WakeExpiredProcesses();
uint64_t now = Timekeeping::GetTicks();
if (nextBspPolicyTick == 0 || now >= nextBspPolicyTick) {
nextBspPolicyTick = now + 500;
bspMaintenancePending.store(true, std::memory_order_release);
}
if (interruptedUser &&
bspMaintenancePending.load(std::memory_order_acquire)) {
RequestKernelWork();
}
}
int slot = cpu->currentSlot;
if (slot < 0) {
// Idle CPU. Check the approximate ready count to avoid
// scanning 256 process slots on every tick. On a 32-core
// system with 27 idle CPUs, this avoids ~7M cache-line
// reads/sec from the process table.
//
// A CPU whose idle context is reserved for long-running kernel
// work (e.g. the deferred Bluetooth firmware download) is left
// alone: stealing it starves that work of CPU time while its
// wall-clock timeouts keep running.
if (readyCount > 0 && !cpu->reservedForKernelWork) {
Schedule();
}
return;
}
processTable[slot].cpuTimeMs += elapsedMs;
if (cpu->kernelWorkPending) {
processTable[slot].sliceRemaining = 0;
}
// Check if another CPU requested this process be killed. Full teardown
// takes process-context mutexes. It is safe here only when the timer
// interrupted ring 3; if it interrupted a syscall, teardown could try
// to reacquire a lock already held by this same suspended kernel stack.
// In that case force a reschedule and finish at syscall-return instead.
if (processTable[slot].killPending) {
if (interruptedUser) {
ExitProcess();
return;
}
processTable[slot].sliceRemaining = 0;
}
if (processTable[slot].sliceRemaining > elapsedMs) {
processTable[slot].sliceRemaining -= elapsedMs;
} else {
processTable[slot].sliceRemaining = 0;
}
// Never context-switch away from an arbitrary kernel stack. Syscalls
// routinely own non-IRQ-disabling subsystem mutexes; suspending that
// stack and dispatching a process that wants the same mutex leaves the
// new process spinning in ring 0, where the timer deliberately cannot
// preempt it to resume the owner. Preserve the expired slice and take
// the switch on the first timer interrupt after SYSRET reaches ring 3.
if (processTable[slot].sliceRemaining == 0 && interruptedUser) {
Schedule();
}
}
int GetCurrentPid() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return -1;
int primary = processTable[slot].primarySlot;
if (primary < 0) primary = slot;
return processTable[primary].pid;
}
void ExitIfKilled() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu ? cpu->currentSlot : -1;
if (slot >= 0 && processTable[slot].killPending) {
ExitProcess();
}
}
int GetCurrentTid() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
return (slot >= 0) ? processTable[slot].pid : -1;
}
Process* GetCurrentProcessPtr() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return nullptr;
int primary = processTable[slot].primarySlot;
if (primary < 0) primary = slot;
return &processTable[primary];
}
bool HasCapability(uint64_t capability) {
Process* proc = GetCurrentProcessPtr();
return proc != nullptr && (proc->effectiveCaps & capability) == capability;
}
Process* GetCurrentThreadPtr() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return nullptr;
return &processTable[slot];
}
// Serializes stack growth so two threads faulting on the same page do not
// both map it (the loser's page would leak: FreeUserHalf only frees pages
// still referenced by the page tables).
static kcp::Spinlock stackGrowLock;
bool TryGrowUserStack(uint64_t faultAddr) {
constexpr uint64_t growBase = UserStackTop - UserStackMax;
constexpr uint64_t eagerBase = UserStackTop - UserStackSize;
if (faultAddr < growBase || faultAddr >= eagerBase) return false;
Process* proc = GetCurrentProcessPtr();
if (proc == nullptr) return false;
uint64_t pageVa = faultAddr & ~0xFFFULL;
stackGrowLock.Acquire();
// Another thread may have mapped this page between the fault and here.
if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pml4Phys, pageVa, 0x1000, true)) {
stackGrowLock.Release();
return true;
}
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
stackGrowLock.Release();
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Out of memory growing user stack for pid " << proc->pid;
return false;
}
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, pageVa)) {
stackGrowLock.Release();
Memory::g_pfa->Free(page);
return false;
}
stackGrowLock.Release();
asm volatile("invlpg (%0)" :: "r"(pageVa) : "memory");
return true;
}
void ExitProcess() {
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) {
return;
}
// If we're a sibling thread, this is a per-thread exit, not a
// process exit. ExitCurrentThread does the right thing and never
// returns.
{
int primary = processTable[slot].primarySlot;
if (primary >= 0 && primary != slot) {
ExitCurrentThread(0);
}
}
Process& proc = processTable[slot];
proc.killPending = false;
proc.startPending = false;
int exitingPid = proc.pid;
int primarySlot_ = slot;
// Drain sibling threads: any thread sharing our PML4 must stop using
// it before we call FreeUserHalf. Ready/Blocked siblings can be
// terminated immediately; siblings running on other CPUs are tagged
// killPending and we spin until their tick handler routes them
// through ExitCurrentThread.
// Several device syscalls have bounded waits up to five seconds. Give
// an interrupted kernel-mode sibling room to finish that recovery
// path before declaring it wedged.
static constexpr uint64_t SiblingDrainTimeoutMs = 10000;
static constexpr uint32_t SiblingDrainMaxPasses = 262144;
uint64_t siblingDrainDeadline = Timekeeping::GetTicks() + SiblingDrainTimeoutMs;
uint32_t siblingDrainPasses = 0;
for (;;) {
bool anyRunning = false;
bool targetCpus[Smp::MaxCPUs] = {};
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (i == primarySlot_) continue;
if (processTable[i].primarySlot != primarySlot_) continue;
auto st = processTable[i].state;
if (st == ProcessState::Free || st == ProcessState::Terminated) {
continue;
}
if (st == ProcessState::Running) {
processTable[i].killPending.store(true, std::memory_order_release);
int targetCpu = processTable[i].runningOnCpu;
if (targetCpu >= 0 && targetCpu < Smp::MaxCPUs)
targetCpus[targetCpu] = true;
anyRunning = true;
continue;
}
// Ready or Blocked -- terminate inline, wake any joiner.
if (st == ProcessState::Ready) readyCount--;
processTable[i].state = ProcessState::Terminated;
processTable[i].killPending = false;
processTable[i].runningOnCpu = -1;
processTable[i].waitingForPid = -1;
processTable[i].sleepUntilTick = 0;
processTable[i].waitingOnObject = nullptr;
void* obj = &processTable[i];
for (int j = 0; j < MaxProcesses; j++) {
if (processTable[j].state == ProcessState::Blocked &&
processTable[j].waitingOnObject == obj) {
processTable[j].state = ProcessState::Ready;
readyCount++;
processTable[j].waitingOnObject = nullptr;
processTable[j].sleepUntilTick = 0;
processTable[j].waitingForPid = -1;
}
}
}
schedLock.Release();
if (!anyRunning) break;
// Do not depend solely on a remote CPU's next periodic timer.
// Laptop firmware/IRQ pathologies can delay that tick, and the
// old unbounded wait then trapped the exiting CPU forever.
if ((siblingDrainPasses & 0x3FF) == 0) {
for (int i = 0; i < Smp::GetCpuCount(); i++) {
if (!targetCpus[i] || i == cpu->cpuIndex) continue;
Smp::CpuData* target = Smp::GetCpuData(i);
if (target != nullptr && target->started) {
(void)Hal::LocalApic::SendFixedIpi(target->lapicId,
Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
}
}
}
siblingDrainPasses++;
if (siblingDrainPasses >= SiblingDrainMaxPasses ||
Timekeeping::GetTicks() >= siblingDrainDeadline) {
// It is unsafe to free the shared address space while another
// CPU may still execute in it. A visible panic is preferable
// to silent lockup or use-after-free corruption.
Panic("Timed out draining sibling threads during process exit", nullptr);
}
// Wait for the sibling CPU(s) to observe killPending on their
// next timer tick. Briefly busy-wait; this path is rare.
for (int spin = 0; spin < 1024; spin++) {
asm volatile("pause");
}
}
// All siblings are quiescent. Reap any terminated sibling slots
// (the process is gone, no joiner will arrive).
for (int i = 0; i < MaxProcesses; i++) {
if (i == primarySlot_) continue;
if (processTable[i].primarySlot != primarySlot_) continue;
if (processTable[i].state != ProcessState::Terminated) continue;
// Ready/blocked siblings terminated inline above never ran their
// ordinary ExitCurrentThread cleanup. Running siblings normally
// arrive here with fsBase already cleared, making this a no-op.
FreeSiblingThreadTls(processTable[i]);
void* stackBase = (void*)processTable[i].stackBase;
processTable[i].stackBase = 0;
processTable[i].pml4Phys = 0;
processTable[i].primarySlot = -1;
processTable[i].joinerSlot = -1;
processTable[i].state = ProcessState::Free;
if (stackBase) Memory::g_pfa->Free(stackBase, StackPages);
}
// Clean up any windows owned by this process
WinServer::CleanupProcess(exitingPid);
// Cancel this process's consent requests; if it was the consent
// agent, deny everything still waiting on it.
Consent::OnProcessExit(exitingPid);
// Release any mixer virtual streams owned by this process so the slot
// and its ring buffer don't leak when an app forgets to audio_close.
Drivers::Audio::Mixer::CleanupProcess(exitingPid);
// Restore scanout to buffer 0 if the exiting process owned page
// flips, so the next fullscreen client and the kernel terminal are
// never stranded on the invisible buffer (no-op for non-owners).
Drivers::Graphics::IntelGPU::OnProcessExit(exitingPid);
// USB interface claims are process-owned capabilities. Closing them
// here stops DMA streaming and releases exclusivity even when an app
// exits without calling usb_close().
Drivers::USB::UserUsb::ReleaseAllForPid(exitingPid);
// Release process-scoped IPC handles/mappings before tearing down the address space.
Ipc::CleanupProcessSlot(slot, exitingPid, proc.pml4Phys);
montauk::abi::CleanupHeapForSlot(slot, proc.pml4Phys);
montauk::abi::CleanupLibTable(slot);
proc.waitingForPid = -1;
proc.sleepUntilTick = 0;
proc.waitingOnObject = nullptr;
proc.redirected = false;
proc.parentPid = -1;
proc.outBuf = nullptr;
proc.outHead = 0;
proc.outTail = 0;
proc.inBuf = nullptr;
proc.inHead = 0;
proc.inTail = 0;
proc.keyHead = 0;
proc.keyTail = 0;
proc.getcharUtf8Len = 0;
proc.getcharUtf8Pos = 0;
proc.termCols = 0;
proc.termRows = 0;
proc.ioOutHandle = -1;
proc.ioInHandle = -1;
proc.ioKeyHandle = -1;
proc.ioWaitsetHandle = -1;
Ipc::ProcessExitedInSlot(slot, exitingPid);
// Free all user-space physical pages and page table structures
Memory::VMM::Paging::FreeUserHalf(proc.pml4Phys);
schedLock.Acquire();
proc.state = ProcessState::Terminated;
proc.runningOnCpu = -1;
proc.reapReady = true;
bspMaintenancePending.store(true, std::memory_order_release);
// Publish the exit code before waking waiters so SYS_WAITPID
// observes it the moment it unblocks.
exitLedgerPid[exitLedgerHead] = exitingPid;
exitLedgerCode[exitLedgerHead] = proc.exitCode;
exitLedgerHead = (exitLedgerHead + 1) % ExitLedgerSize;
// Wake any processes blocked on this PID
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Blocked &&
processTable[i].waitingForPid == exitingPid) {
processTable[i].state = ProcessState::Ready;
readyCount++;
processTable[i].waitingForPid = -1;
processTable[i].waitingOnObject = nullptr;
processTable[i].sleepUntilTick = 0;
}
}
// Find next ready process
int next = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state == ProcessState::Ready) {
next = i;
break;
}
}
if (next >= 0) {
cpu->currentSlot = next;
processTable[next].state = ProcessState::Running;
readyCount--;
processTable[next].runningOnCpu = cpu->cpuIndex;
processTable[next].sliceRemaining = TimeSliceMs;
if (readyCount > 0) {
KickOneIdleCpu(cpu->cpuIndex);
}
uint64_t newCR3 = processTable[next].pml4Phys;
cpu->kernelRsp = processTable[next].kernelStackTop;
cpu->tss->rsp0 = processTable[next].kernelStackTop;
LoadUserFsBase(cpu, processTable[next].fsBase);
// Lock held across context switch -- resumed process releases it
SchedContextSwitch(&processTable[slot].savedRsp, processTable[next].savedRsp, newCR3,
processTable[slot].fpuState, processTable[next].fpuState);
schedLock.Release();
} else {
cpu->currentSlot = -1;
// Switch to idle -- release after resuming from idle
SchedContextSwitch(&processTable[slot].savedRsp, cpu->idleSavedRsp, GetKernelCR3(),
processTable[slot].fpuState, nullptr);
schedLock.Release();
}
for (;;) {
asm volatile("hlt");
}
}
int KillProcess(int pid, int exitCode) {
// Refuse to kill PID 0 (init). Killing the caller's own process is
// allowed: a faulting/aborting sibling thread shares the pid and has
// no other way to take the process down. The caller's own slot gets
// killPending below and exits at syscall return (ExitIfKilled) or
// via the ExitProcess the fault handler calls next.
if (pid == 0) return -1;
schedLock.Acquire();
// pid identifies a process (main thread). Sibling-thread TIDs are
// not killable through this entry point.
int primarySlot_ = -1;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid != pid) continue;
if (processTable[i].primarySlot != i) continue;
auto s = processTable[i].state;
if (s == ProcessState::Ready || s == ProcessState::Running ||
s == ProcessState::Blocked) {
primarySlot_ = i;
}
break;
}
if (primarySlot_ < 0) {
schedLock.Release();
return -1;
}
processTable[primarySlot_].exitCode = exitCode;
// Flag the main thread so its next tick (or scheduler dispatch)
// routes through ExitProcess, which sweeps every sibling thread
// and tears down the address space. If the main thread is parked
// on a blocking syscall we promote it to Ready so the scheduler
// picks it up promptly.
Process& primary = processTable[primarySlot_];
primary.killPending = true;
if (primary.state == ProcessState::Blocked) {
primary.state = ProcessState::Ready;
readyCount++;
primary.waitingForPid = -1;
primary.waitingOnObject = nullptr;
primary.sleepUntilTick = 0;
}
// Also stamp running siblings so they exit promptly; ExitProcess on
// the main thread will additionally wait for them anyway.
for (int i = 0; i < MaxProcesses; i++) {
if (i == primarySlot_) continue;
if (processTable[i].primarySlot != primarySlot_) continue;
auto s = processTable[i].state;
if (s == ProcessState::Running) {
processTable[i].killPending = true;
}
}
schedLock.Release();
KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1);
return 0;
}
int CreateSession() {
schedLock.Acquire();
Process* proc = GetCurrentProcessPtr();
if (proc == nullptr) {
schedLock.Release();
return -1;
}
proc->sessionId = proc->pid;
int sessionId = proc->sessionId;
schedLock.Release();
return sessionId;
}
int KillSession(int sessionId) {
if (sessionId < 0) return -1;
int callerPid = GetCurrentPid();
int killed = 0;
// Mark the complete group in one scheduler-lock transaction. New
// children inherit sessionId, so a supervisor can repeat this call
// until zero is returned to close the small spawn/exit race cleanly.
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
Process& primary = processTable[i];
if (primary.primarySlot != i || primary.pid == callerPid ||
primary.sessionId != sessionId) {
continue;
}
auto state = primary.state;
if (state != ProcessState::Ready && state != ProcessState::Running &&
state != ProcessState::Blocked) {
continue;
}
primary.exitCode = TerminatedKilled;
primary.killPending = true;
if (primary.state == ProcessState::Blocked) {
primary.state = ProcessState::Ready;
readyCount++;
primary.waitingForPid = -1;
primary.waitingOnObject = nullptr;
primary.sleepUntilTick = 0;
}
for (int j = 0; j < MaxProcesses; j++) {
if (j == i || processTable[j].primarySlot != i) continue;
if (processTable[j].state == ProcessState::Running) {
processTable[j].killPending = true;
}
}
killed++;
}
schedLock.Release();
if (killed > 0) {
KickOneIdleCpu(Smp::GetCurrentCpuData()
? Smp::GetCurrentCpuData()->cpuIndex : -1);
}
return killed;
}
int LookupExitCode(int pid) {
schedLock.Acquire();
int code = 0;
for (int i = 0; i < ExitLedgerSize; i++) {
if (exitLedgerPid[i] == pid) {
code = exitLedgerCode[i];
break;
}
}
schedLock.Release();
return code;
}
void SetProcessExitCode(int pid, int code) {
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid != pid) continue;
// Only a process (primary slot) carries a process exit code;
// sibling-thread exit values go through the join path.
if (processTable[i].primarySlot == i) {
processTable[i].exitCode = code;
}
break;
}
schedLock.Release();
}
void BlockOnPid(int pid) {
ExitIfKilled();
// If the target is already dead, return immediately
if (!IsAlive(pid)) return;
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return;
schedLock.Acquire();
// Double-check under lock (target might have exited between
// the lockless IsAlive check and acquiring the lock)
bool stillAlive = false;
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid == pid) {
auto s = processTable[i].state;
stillAlive = (s == ProcessState::Ready ||
s == ProcessState::Running ||
s == ProcessState::Blocked);
break;
}
}
if (!stillAlive) {
schedLock.Release();
return;
}
// Mark current process as Blocked -- scheduler will skip it.
// ExitProcess will wake us when the target terminates.
processTable[slot].state = ProcessState::Blocked;
processTable[slot].waitingForPid = pid;
processTable[slot].waitingOnObject = nullptr;
processTable[slot].sleepUntilTick = 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
void BlockForSleep(uint64_t ms) {
if (ms == 0) return;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return;
schedLock.Acquire();
processTable[slot].state = ProcessState::Blocked;
processTable[slot].waitingForPid = -1;
processTable[slot].waitingOnObject = nullptr;
processTable[slot].sleepUntilTick = Timekeeping::GetTicks() + ms;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
void BlockOnObject(void* object, uint64_t timeoutMs) {
if (object == nullptr) return;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu->currentSlot;
if (slot < 0) return;
schedLock.Acquire();
processTable[slot].state = ProcessState::Blocked;
processTable[slot].waitingForPid = -1;
processTable[slot].waitingOnObject = object;
processTable[slot].sleepUntilTick = (timeoutMs > 0)
? (Timekeeping::GetTicks() + timeoutMs)
: 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
}
bool BlockOnObjectIf(void* object, uint64_t timeoutMs,
bool (*shouldBlock)(void*), void* context) {
if (object == nullptr || shouldBlock == nullptr) return false;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
if (cpu == nullptr) return false;
int slot = cpu->currentSlot;
if (slot < 0) return false;
schedLock.Acquire();
if (!shouldBlock(context)) {
schedLock.Release();
return false;
}
processTable[slot].state = ProcessState::Blocked;
processTable[slot].waitingForPid = -1;
processTable[slot].waitingOnObject = object;
processTable[slot].sleepUntilTick = (timeoutMs > 0)
? (Timekeeping::GetTicks() + timeoutMs)
: 0;
processTable[slot].runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
ExitIfKilled();
return true;
}
uint64_t ObserveObjectWake(void* object) {
if (object == nullptr) return 0;
return objectWakeEpochs[ObjectWakeBucket(object)].load(std::memory_order_acquire);
}
struct ObjectWakeObservation {
void* object;
uint64_t epoch;
};
static bool ObjectWakeIsUnchanged(void* opaque) {
auto* observation = (ObjectWakeObservation*)opaque;
uint64_t now = ObserveObjectWake(observation->object);
uint64_t was = observation->epoch;
if (now == was) return true; // nothing signalled this bucket at all
// Exactly one signal landed here since the observation, and it carried
// a different object's tag, so it cannot have been ours: the caller may
// still block. Any larger or wrapped delta is ambiguous -- more than
// one object may have signalled -- and must be reported as a wake so a
// real edge is never dropped.
if ((uint32_t)now == (uint32_t)was + 1 &&
(uint32_t)(now >> 32) != ObjectWakeTag(observation->object)) {
return true;
}
return false;
}
bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake) {
ObjectWakeObservation observation{object, observedWake};
return BlockOnObjectIf(object, timeoutMs, ObjectWakeIsUnchanged, &observation);
}
void WakeObjectWaiters(void* object) {
if (object == nullptr) return;
// Publish this object's tag and the bumped counter as one word. A CAS
// loop (rather than fetch_add) keeps the pair consistent and, unlike a
// plain store, cannot drop a concurrent waker's increment on the same
// bucket -- a dropped increment would be an unobserved wake edge.
auto& bucket = objectWakeEpochs[ObjectWakeBucket(object)];
uint64_t tagBits = (uint64_t)ObjectWakeTag(object) << 32;
uint64_t observed = bucket.load(std::memory_order_relaxed);
while (!bucket.compare_exchange_weak(
observed, tagBits | (uint32_t)((uint32_t)observed + 1),
std::memory_order_release, std::memory_order_relaxed)) {
}
bool wokeAny = false;
schedLock.Acquire();
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].state != ProcessState::Blocked) continue;
if (processTable[i].waitingOnObject != object) continue;
processTable[i].waitingOnObject = nullptr;
processTable[i].sleepUntilTick = 0;
processTable[i].waitingForPid = -1;
processTable[i].state = ProcessState::Ready;
readyCount++;
wokeAny = true;
}
schedLock.Release();
if (wokeAny) {
KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1);
}
}
// Monotonic enrollment counter for FIFO futex wakeups. Only touched
// under schedLock.
static uint64_t futexNextSeq = 1;
int FutexWait(uint64_t uaddr, uint32_t expected, uint64_t timeoutNs) {
if ((uaddr & 3) != 0) return InvalidArgument;
ExitIfKilled();
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu ? cpu->currentSlot : -1;
if (slot < 0) return GenericFailure;
Process& self = processTable[slot];
// Round up so a short timeout never turns into "check only".
uint64_t timeoutMs = 0;
if (timeoutNs != FutexWaitForever) {
if (timeoutNs == 0) {
timeoutMs = 0;
} else {
timeoutMs = timeoutNs / 1000000ULL + ((timeoutNs % 1000000ULL) ? 1 : 0);
}
}
schedLock.Acquire();
// The word is read here, with schedLock held and interrupts off, so
// it must not fault: the caller validated (and demand-committed) the
// page, and the page walk below re-confirms it is still mapped. An
// aligned 32-bit word never straddles a page.
if (!Memory::VMM::Paging::IsUserRangeAccessible(self.pml4Phys, uaddr, 4, false)) {
schedLock.Release();
return BadPointer;
}
uint32_t current = __atomic_load_n((volatile uint32_t*)uaddr, __ATOMIC_SEQ_CST);
if (current != expected) {
schedLock.Release();
return WouldBlock;
}
if (timeoutNs == 0) {
schedLock.Release();
return TimedOut;
}
self.state = ProcessState::Blocked;
self.waitingForPid = -1;
self.waitingOnObject = nullptr;
self.futexAddr = uaddr;
self.futexSeq = futexNextSeq++;
self.futexWoken = false;
self.sleepUntilTick = (timeoutNs == FutexWaitForever)
? 0
: Timekeeping::GetTicks() + timeoutMs;
self.runningOnCpu = -1;
SwitchAwayFromBlockedCurrentLocked();
// Whatever made this thread runnable (FutexWake, the deadline, or a
// kill) it is no longer parked; clear the key before this thread can
// block for some other reason and be mistaken for a futex waiter.
schedLock.Acquire();
bool woken = self.futexWoken;
self.futexAddr = 0;
self.futexWoken = false;
schedLock.Release();
ExitIfKilled();
return woken ? Success : TimedOut;
}
int FutexWake(uint64_t uaddr, uint32_t count) {
if ((uaddr & 3) != 0 || uaddr == 0) return InvalidArgument;
if (count == 0) return 0;
auto* cpu = Smp::GetCurrentCpuData();
int slot = cpu ? cpu->currentSlot : -1;
if (slot < 0) return GenericFailure;
schedLock.Acquire();
int primary = processTable[slot].primarySlot;
if (primary < 0) primary = slot;
// Gather this process's waiters on uaddr. At most MaxProcesses.
int waiters[MaxProcesses];
int n = 0;
for (int i = 0; i < MaxProcesses; i++) {
Process& p = processTable[i];
if (p.state != ProcessState::Blocked) continue;
if (p.futexAddr != uaddr) continue;
if (p.primarySlot != primary) continue;
waiters[n++] = i;
}
// Waking everyone needs no ordering; otherwise pick oldest-first.
bool ordered = (uint32_t)n > count;
int woken = 0;
while (woken < n && (uint32_t)woken < count) {
if (ordered) {
int best = woken;
for (int j = woken + 1; j < n; j++) {
if (processTable[waiters[j]].futexSeq < processTable[waiters[best]].futexSeq)
best = j;
}
int tmp = waiters[woken]; waiters[woken] = waiters[best]; waiters[best] = tmp;
}
Process& p = processTable[waiters[woken]];
p.futexAddr = 0;
p.futexWoken = true;
p.sleepUntilTick = 0;
p.state = ProcessState::Ready;
readyCount++;
woken++;
}
schedLock.Release();
if (woken > 0) {
// Re-read: this thread may have migrated since `cpu` was sampled.
auto* here = Smp::GetCurrentCpuData();
KickOneIdleCpu(here ? here->cpuIndex : -1);
}
return woken;
}
bool IsAlive(int pid) {
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid == pid) {
auto s = processTable[i].state;
return s == ProcessState::Ready
|| s == ProcessState::Running
|| s == ProcessState::Blocked;
}
}
return false;
}
Process* GetProcessByPid(int pid) {
for (int i = 0; i < MaxProcesses; i++) {
if (processTable[i].pid == pid) {
auto s = processTable[i].state;
if (s == ProcessState::Ready || s == ProcessState::Running ||
s == ProcessState::Blocked) {
return &processTable[i];
}
}
}
return nullptr;
}
// ==== Consent support ====
static Process* FindLiveProcessLocked(int pid) {
if (pid < 0) return nullptr;
for (int i = 0; i < MaxProcesses; i++) {
Process& p = processTable[i];
if (p.primarySlot != i || p.pid != pid) continue;
if (p.state == ProcessState::Ready || p.state == ProcessState::Running ||
p.state == ProcessState::Blocked) {
return &p;
}
}
return nullptr;
}
static bool ConsentPairValidLocked(const Process* subject, const Process* agent,
uint64_t capability) {
return subject != nullptr && agent != nullptr && subject != agent &&
agent->sessionId >= 0 && subject->sessionId == agent->sessionId &&
(agent->effectiveCaps & montauk::abi::CAP_COMPOSITOR) != 0 &&
(agent->delegableCaps & capability) == capability &&
(subject->consentCeiling & capability) == capability;
}
bool ConsentSubjectEligible(int pid, int agentPid, uint64_t capability) {
schedLock.Acquire();
Process* subject = FindLiveProcessLocked(pid);
bool ok = ConsentPairValidLocked(subject, FindLiveProcessLocked(agentPid), capability) &&
(subject->consentDenied & capability) == 0;
schedLock.Release();
return ok;
}
bool ApplyConsentGrant(int pid, int agentPid, uint64_t capability) {
schedLock.Acquire();
Process* subject = FindLiveProcessLocked(pid);
bool ok = ConsentPairValidLocked(subject, FindLiveProcessLocked(agentPid), capability);
if (ok) {
// Usable, never delegable: a consented app cannot hand the
// authority on to anything it launches.
subject->effectiveCaps |= capability;
}
schedLock.Release();
return ok;
}
void ApplyConsentDenial(int pid, uint64_t capability) {
schedLock.Acquire();
if (Process* subject = FindLiveProcessLocked(pid))
subject->consentDenied |= capability;
schedLock.Release();
}
bool SnapshotConsentSubject(int pid, montauk::abi::ConsentRequest& out) {
schedLock.Acquire();
Process* subject = FindLiveProcessLocked(pid);
if (subject != nullptr) {
memcpy(out.imagePath, subject->imagePath, sizeof(out.imagePath));
out.imagePath[sizeof(out.imagePath) - 1] = '\0';
// Only an image that cannot be rewritten without CAP_SYSTEM_IMAGE
// is identified by its path well enough to remember an answer.
out.imageProtected = Fs::IsSystemImage(out.imagePath);
}
schedLock.Release();
return subject != nullptr;
}
Process* GetProcessSlot(int slot) {
if (slot < 0 || slot >= MaxProcesses) return nullptr;
return &processTable[slot];
}
int SpawnCrashPad(int crasherPid) {
// Spawn crashpad as a child of init (slot 0), not the crashing process.
// The crasher is still alive here (we run before its ExitProcess), but
// it is about to die, so we must not parent the crashpad to it.
static constexpr const char* crashpadPath = "0:/apps/crashpad/crashpad.elf";
// If the crashing process had a redirected console (i.e. it was running
// under terminal.elf in a console session), hand the crashpad a write
// handle to that same out-stream so it can surface a textual crash
// report to the console user when no desktop is running.
Process* crasher = GetProcessByPid(crasherPid);
// Crash-loop guard: never spawn a crashpad to report on the crashpad
// itself. If crashpad.elf faults (startup bug, OOM, etc.) we would
// otherwise spawn another crashpad, which can fault again, ad infinitum
// -- a fork bomb that also exhausts process slots and report buffers.
if (crasher && Lib::strcmp(crasher->name, crashpadPath) == 0) {
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "crashpad faulted; not spawning a nested crashpad";
return -1;
}
int crasherSlot = Ipc::SlotForPid(crasherPid);
bool inheritRedir = crasher && crasher->redirected &&
crasher->ioOutHandle >= 0 && crasherSlot >= 0;
Ipc::HandleSnapshot outSnapshot;
if (inheritRedir) {
if (!outSnapshot.Capture(crasherSlot, crasher->ioOutHandle) ||
outSnapshot.type != Ipc::HandleType::Stream || outSnapshot.object == nullptr) {
inheritRedir = false;
}
}
// Deferred start when inheriting, so we can wire ioOutHandle before _start.
int childPid = Spawn(crashpadPath, nullptr, /*startReady=*/!inheritRedir);
if (childPid < 0 || !inheritRedir) return childPid;
Process* child = GetProcessByPid(childPid);
int childSlot = Ipc::SlotForPid(childPid);
if (child && childSlot >= 0) {
// Installing with RightWrite retains the stream (writerRefs++/refs++),
// so it outlives the crasher's imminent ExitProcess teardown. The
// snapshot above and this install run back-to-back with no blocking
// call between, while the crasher's handle still pins the stream.
int h = Ipc::InstallHandleForSlot(childSlot, outSnapshot.object, Ipc::HandleType::Stream,
Ipc::RightWrite | Ipc::RightWait | Ipc::RightDup);
if (h >= 0) {
child->ioOutHandle = h;
child->redirected = true;
child->parentPid = crasherPid;
child->termCols = crasher->termCols;
child->termRows = crasher->termRows;
}
}
// Always start it (even if wiring failed -> it just falls back to silent).
StartProcess(childPid);
return childPid;
}
}
// Called by the syscall entry assembly after the syscall implementation has
// released all of its subsystem locks but before returning to ring 3.
extern "C" void SchedExitIfKilled() {
Sched::ExitIfKilled();
}
Reference in new issue View Git Blame Copy Permalink