/* * Scheduler.cpp * Preemptive process scheduler with SMP support * Copyright (c) 2025-2026 Daniel Hammer */ #include #include "Scheduler.hpp" #include "ElfLoader.hpp" #include "Consent.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // Assembly: context switch with CR3 and FPU state parameters extern "C" void SchedContextSwitch(uint64_t* oldRsp, uint64_t newRsp, uint64_t newCR3, uint8_t* oldFpuArea, uint8_t* newFpuArea); // Load a user thread's FS base (TLS thread pointer) before dispatching it. // Cached per CPU: the MSR write is skipped when unchanged, so TLS-free // processes (fsBase 0) cost one write per CPU ever. Userspace cannot move // FS base itself (CR4.FSGSBASE is off), so the cache cannot go stale. static inline void LoadUserFsBase(Smp::CpuData* cpu, uint64_t fsBase) { if (cpu->currentFsBase == fsBase) { return; } cpu->currentFsBase = fsBase; asm volatile("wrmsr" : : "c"(0xC0000100u), /* IA32_FS_BASE */ "a"((uint32_t)fsBase), "d"((uint32_t)(fsBase >> 32))); } // Assembly: jump to user mode via IRETQ. // `arg` is delivered as the user-mode RDI (SystemV first argument). // For freshly spawned processes this is 0; for SpawnThread it is the // user-supplied entry argument. extern "C" void JumpToUserMode(uint64_t rip, uint64_t rsp, uint64_t arg); namespace Sched { static Process processTable[MaxProcesses]; static int nextPid = 0; // Exit-code ledger. Pids are monotonic and never recycled, so entries // cannot alias. Written under schedLock during process teardown; read // by SYS_WAITPID after the waiter unblocks. Codes: 0..255 = normal // exit status, 256+signal = killed or crashed. static constexpr int ExitLedgerSize = 128; static int exitLedgerPid[ExitLedgerSize]; static int exitLedgerCode[ExitLedgerSize]; static int exitLedgerHead = 0; // The scheduler lock MUST be a Spinlock (interrupt-disabling). // It is held ACROSS context switches to prevent the race where // another CPU picks up a process whose RSP hasn't been saved yet. // The resumed process releases it. // schedLock is a baton: the context that acquires it switches stacks and // the context being resumed releases it. A Spinlock's ordinary shared // savedFlags field cannot represent that -- it would restore the switching // context's IF state into the resumed stack. Keep the flags with each saved // process/idle context instead, so timer frames resume with IF=0 and syscall // frames resume with their original IF=1 even after CPU migration. class SchedulerSpinlock { kcp::Spinlock lock; uint64_t processFlags[MaxProcesses] = {}; uint64_t idleFlags[Smp::MaxCPUs] = {}; public: void Acquire() { uint64_t flags = lock.AcquireIrqSave(); auto* cpu = Smp::GetCurrentCpuData(); if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) { processFlags[cpu->currentSlot] = flags; } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) { idleFlags[cpu->cpuIndex] = flags; } } void Release() { uint64_t flags = 0; // Fresh startup remains IF=0 until JumpToUserMode's IRETQ. auto* cpu = Smp::GetCurrentCpuData(); if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) { flags = processFlags[cpu->currentSlot]; } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) { flags = idleFlags[cpu->cpuIndex]; } lock.ReleaseIrqRestore(flags); } void ReleaseForStartup() { lock.ReleaseIrqNoRestore(); } }; static SchedulerSpinlock schedLock; // Approximate count of Ready processes. Incremented/decremented // under schedLock. Idle CPUs check this to avoid scanning all 256 // process slots on every timer tick. // Updated while schedLock is held, but sampled locklessly by idle CPUs and // reschedule-IPI paths. Atomic loads make those wakeup decisions coherent // across cores instead of relying on volatile's compiler-only semantics. static std::atomic readyCount{0}; // Per-object wake generations close the readiness-check/enrollment gap. // Each bucket packs [63:32] = a tag identifying the last object to signal // it and [31:0] = a signal counter, published together in one atomic word. // Publishing both halves atomically lets a waiter distinguish "exactly one // signal landed here and it carried someone else's tag" (safe to keep // blocking) from "something ambiguous happened" (must treat as a wake). // Without the tag, an unrelated hot object sharing the bucket forces the // waiter to re-run its condition on every signal. Tag collisions only cost // a spurious wake; they can never lose one, which is the only direction // that would reintroduce the lost-wakeup bug this table exists to prevent. static constexpr uint32_t ObjectWakeBuckets = 1024; static std::atomic objectWakeEpochs[ObjectWakeBuckets] = {}; static uint32_t ObjectWakeBucket(void* object) { uint64_t value = (uint64_t)object; return (uint32_t)(((value >> 4) ^ (value >> 12)) & (ObjectWakeBuckets - 1)); } static uint32_t ObjectWakeTag(void* object) { uint64_t value = (uint64_t)object; value ^= value >> 32; value *= 0x9E3779B97F4A7C15ULL; return (uint32_t)(value >> 32); } // The idle loop runs in the kernel PML4 static uint64_t GetKernelCR3() { return (uint64_t)Memory::VMM::g_paging->PML4; } static void RescheduleIpiHandler(uint8_t, bool interruptedUser) { auto* cpu = Smp::GetCurrentCpuData(); if (cpu == nullptr) { return; } // Process teardown uses this IPI to promptly stop sibling threads on // other CPUs. A user-mode sibling can exit immediately. If the IPI // interrupted a syscall, merely expire its slice: teardown on that // suspended kernel stack could reacquire a mutex it already owns. if (cpu->currentSlot >= 0 && processTable[cpu->currentSlot].killPending.load(std::memory_order_acquire)) { if (interruptedUser) { ExitProcess(); return; } processTable[cpu->currentSlot].sliceRemaining = 0; return; } if (cpu->currentSlot >= 0 || readyCount <= 0 || cpu->reservedForKernelWork) { return; } Schedule(); } static void KickOneIdleCpu(int sourceCpuIndex) { if (readyCount <= 0) { return; } auto tryKick = [&](Smp::CpuData* target) { if (target == nullptr || !target->started) return false; if (target->cpuIndex == sourceCpuIndex) return false; if (target->currentSlot >= 0) return false; if (target->reservedForKernelWork) return false; return Hal::LocalApic::SendFixedIpi(target->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE); }; if (tryKick(Smp::GetCpuData(0))) { return; } for (int i = 0; i < Smp::GetCpuCount(); i++) { if (tryKick(Smp::GetCpuData(i))) { return; } } } static void SwitchAwayFromBlockedCurrentLocked() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) { schedLock.Release(); return; } int next = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Ready) { next = i; break; } } if (next >= 0) { cpu->currentSlot = next; processTable[next].state = ProcessState::Running; readyCount--; processTable[next].runningOnCpu = cpu->cpuIndex; processTable[next].sliceRemaining = TimeSliceMs; cpu->kernelRsp = processTable[next].kernelStackTop; cpu->tss->rsp0 = processTable[next].kernelStackTop; LoadUserFsBase(cpu, processTable[next].fsBase); SchedContextSwitch(&processTable[slot].savedRsp, processTable[next].savedRsp, processTable[next].pml4Phys, processTable[slot].fpuState, processTable[next].fpuState); schedLock.Release(); return; } cpu->currentSlot = -1; SchedContextSwitch(&processTable[slot].savedRsp, cpu->idleSavedRsp, GetKernelCR3(), processTable[slot].fpuState, nullptr); schedLock.Release(); } // Startup function for newly spawned processes. // SchedContextSwitch "returns" here on first schedule. // The schedLock is held (acquired by the switching-from CPU's Schedule). static void ProcessStartup() { // Release the schedLock that the switching-from CPU held // without consulting this brand-new slot's (intentionally empty) // saved interrupt state. JumpToUserMode's IRETQ enables interrupts. schedLock.ReleaseForStartup(); auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot >= 0) { Process& proc = processTable[slot]; // Set up per-CPU kernel RSP for SYSCALL entry cpu->kernelRsp = proc.kernelStackTop; // Set up per-CPU TSS RSP0 for hardware interrupts from ring 3 cpu->tss->rsp0 = proc.kernelStackTop; LoadUserFsBase(cpu, proc.fsBase); // Jump to user mode (never returns). // For main threads threadArg is 0 (libc _start ignores RDI); // for sibling threads it carries the user-supplied argument. JumpToUserMode(proc.entryPoint, proc.userStackTop, proc.threadArg); } ExitProcess(); for (;;) { asm volatile("hlt"); } } void Initialize() { for (int i = 0; i < MaxProcesses; i++) { processTable[i].pid = i; processTable[i].state = ProcessState::Free; processTable[i].name[0] = '\0'; processTable[i].savedRsp = 0; processTable[i].stackBase = 0; processTable[i].entryPoint = 0; processTable[i].sliceRemaining = 0; processTable[i].cpuTimeMs = 0; processTable[i].pml4Phys = 0; processTable[i].kernelStackTop = 0; processTable[i].userStackTop = 0; processTable[i].heapNext = 0; processTable[i].readdirCursor = 0; processTable[i].args[0] = '\0'; processTable[i].environment[0] = '\0'; processTable[i].environmentLength = 1; processTable[i].user[0] = '\0'; processTable[i].effectiveCaps = 0; processTable[i].delegableCaps = 0; processTable[i].consentCeiling = 0; processTable[i].consentDenied = 0; processTable[i].sessionId = -1; processTable[i].cwd[0] = '\0'; processTable[i].runningOnCpu = -1; processTable[i].killPending = false; processTable[i].reapReady = false; processTable[i].startPending = false; processTable[i].waitingForPid = -1; processTable[i].sleepUntilTick = 0; processTable[i].waitingOnObject = nullptr; processTable[i].redirected = false; processTable[i].parentPid = -1; processTable[i].outBuf = nullptr; processTable[i].outHead = 0; processTable[i].outTail = 0; processTable[i].inBuf = nullptr; processTable[i].inHead = 0; processTable[i].inTail = 0; processTable[i].keyHead = 0; processTable[i].keyTail = 0; processTable[i].getcharUtf8Len = 0; processTable[i].getcharUtf8Pos = 0; processTable[i].termCols = 0; processTable[i].termRows = 0; processTable[i].ioOutHandle = -1; processTable[i].ioInHandle = -1; processTable[i].ioKeyHandle = -1; processTable[i].ioWaitsetHandle = -1; processTable[i].primarySlot = -1; processTable[i].joinerSlot = -1; processTable[i].exitCode = 0; processTable[i].joinable = true; } nextPid = 0; for (int i = 0; i < ExitLedgerSize; i++) { exitLedgerPid[i] = -1; exitLedgerCode[i] = 0; } exitLedgerHead = 0; Hal::RegisterIrqHandler(Hal::IRQ_RESCHEDULE, RescheduleIpiHandler); Kt::KernelLogStream(Kt::OK, "Sched") << "Initialized (" << MaxProcesses << " process slots, " << (uint64_t)TimeSliceMs << " ms time slice)"; } int Spawn(const char* vfsPath, const char* args, bool startReady, const char* environment, uint32_t environmentLength, const montauk::abi::SpawnCapabilities* capabilities, const char* userOverride) { // Grants are keyed on the image path, so they are only as trustworthy // as the path: anything outside 0:/os and 0:/apps can be replaced by // any process, which would then run with the grant the next time a // privileged launcher starts it. Such a program starts with nothing // and asks through consent instead. montauk::abi::SpawnCapabilities clamped{0, 0}; if (capabilities != nullptr && (capabilities->effective | capabilities->delegable) != 0 && !Fs::IsSystemImage(vfsPath)) { Kt::KernelLogStream(Kt::WARNING, "Sched") << "Not an installed program, " << "starting without capabilities: " << vfsPath; capabilities = &clamped; } schedLock.Acquire(); int slot = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Free) { slot = i; break; } } if (slot < 0) { schedLock.Release(); Kt::KernelLogStream(Kt::ERROR, "Sched") << "No free process slots"; return -1; } // Reserve the slot so another Spawn doesn't claim it. // Use Running (not Ready!) so the scheduler doesn't try to // dispatch this half-initialized process. processTable[slot].state = ProcessState::Running; processTable[slot].runningOnCpu = -1; processTable[slot].reapReady = false; schedLock.Release(); // Create per-process PML4 with kernel-half copied uint64_t pml4Phys = Memory::VMM::Paging::CreateUserPML4(); // Load ELF into the process's address space TlsInfo tls{}; uint64_t entry = ElfLoad(vfsPath, pml4Phys, &tls); if (entry == 0) { Memory::VMM::Paging::FreeUserHalf(pml4Phys); Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys)); schedLock.Acquire(); processTable[slot].state = ProcessState::Free; schedLock.Release(); return -1; } // Allocate kernel stack (used during syscalls and interrupts). // ReallocConsecutive(nullptr, n) returns an n-page contiguous span; // the previous code allocated a throwaway page first and asked // ReallocConsecutive to migrate from it, which copied + freed the // throwaway for no benefit and double-counted failure paths. void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages); if (stackMem == nullptr) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to allocate contiguous kernel stack"; Memory::VMM::Paging::FreeUserHalf(pml4Phys); Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys)); schedLock.Acquire(); processTable[slot].state = ProcessState::Free; schedLock.Release(); return -1; } memset(stackMem, 0, StackPages * 0x1000); uint8_t* kernelStackBase = (uint8_t*)stackMem; uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize; // Helper to clean up all resources allocated so far on failure auto cleanupOnFail = [&]() { Memory::VMM::Paging::FreeUserHalf(pml4Phys); Memory::g_pfa->Free((void*)Memory::HHDM(pml4Phys)); Memory::g_pfa->Free(stackMem, StackPages); schedLock.Acquire(); processTable[slot].state = ProcessState::Free; schedLock.Release(); }; // Allocate user stack pages and map them in the process PML4 uint64_t userStackBase = UserStackTop - UserStackSize; uint64_t topStackPagePhys = 0; for (uint64_t i = 0; i < UserStackPages; i++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Out of memory for user stack"; cleanupOnFail(); return -1; } uint64_t physAddr = Memory::SubHHDM((uint64_t)page); if (!Memory::VMM::Paging::MapUserIn(pml4Phys, physAddr, userStackBase + i * 0x1000)) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to map user stack page"; Memory::g_pfa->Free(page); cleanupOnFail(); return -1; } if (i == UserStackPages - 1) topStackPagePhys = physAddr; } // Allocate and map a user-space exit stub page. { void* stubPage = Memory::g_pfa->AllocateZeroed(); if (stubPage == nullptr) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Out of memory for exit stub"; cleanupOnFail(); return -1; } uint64_t stubPhys = Memory::SubHHDM((uint64_t)stubPage); if (!Memory::VMM::Paging::MapUserIn(pml4Phys, stubPhys, ExitStubAddr)) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Failed to map exit stub"; Memory::g_pfa->Free(stubPage); cleanupOnFail(); return -1; } // Write: xor edi, edi; xor eax, eax; syscall uint8_t* stub = (uint8_t*)stubPage; stub[0] = 0x31; stub[1] = 0xFF; // xor edi, edi (exit code 0) stub[2] = 0x31; stub[3] = 0xC0; // xor eax, eax (SYS_EXIT = 0) stub[4] = 0x0F; stub[5] = 0x05; // syscall } // Push exit stub address as the return address on the user stack. { uint8_t* topPage = (uint8_t*)Memory::HHDM(topStackPagePhys); *(uint64_t*)(topPage + 0xFF8) = ExitStubAddr; } // Set up the initial kernel stack frame so that SchedContextSwitch // "returns" into ProcessStartup uint64_t* sp = (uint64_t*)kernelStackTop; *(--sp) = (uint64_t)ProcessStartup; // return addr *(--sp) = 0; // rbp *(--sp) = 0; // rbx *(--sp) = 0; // r12 *(--sp) = 0; // r13 *(--sp) = 0; // r14 *(--sp) = 0; // r15 schedLock.Acquire(); Process& proc = processTable[slot]; proc.pid = nextPid++; proc.state = startReady ? ProcessState::Ready : ProcessState::Blocked; if (startReady) readyCount++; { int i = 0; for (; i < 63 && vfsPath[i]; i++) proc.name[i] = vfsPath[i]; proc.name[i] = '\0'; } proc.primarySlot = slot; // main thread owns per-process state proc.joinerSlot = -1; proc.exitCode = 0; proc.joinable = false; // main thread is reaped by BSP, not joined proc.futexAddr = 0; proc.futexWoken = false; proc.savedRsp = (uint64_t)sp; proc.stackBase = (uint64_t)kernelStackBase; proc.entryPoint = entry; proc.sliceRemaining = TimeSliceMs; proc.cpuTimeMs = 0; proc.pml4Phys = pml4Phys; proc.kernelStackTop = kernelStackTop; proc.userStackTop = UserStackTop - 8; proc.heapNext = UserHeapBase; for (uint64_t i = 0; i < UserHeapBitmapWords; i++) g_userHeapPageMap[slot][i] = 0; proc.fsBase = tls.fsBase; proc.tlsTemplateVaddr = tls.templateVaddr; proc.tlsFileSize = tls.fileSize; proc.tlsMemSize = tls.memSize; proc.tlsAlign = tls.align; proc.readdirCursor = 0; proc.runningOnCpu = -1; proc.killPending = false; proc.reapReady = false; proc.startPending = !startReady; proc.waitingForPid = -1; proc.sleepUntilTick = 0; proc.waitingOnObject = nullptr; auto* currentCpu = Smp::GetCurrentCpuData(); int parentSlot = currentCpu ? currentCpu->currentSlot : -1; int parentPrimarySlot = parentSlot; if (parentPrimarySlot >= 0 && processTable[parentPrimarySlot].primarySlot >= 0) parentPrimarySlot = processTable[parentPrimarySlot].primarySlot; // Copy arguments string into process proc.args[0] = '\0'; if (args != nullptr) { int i = 0; for (; i < 4095 && args[i]; i++) { proc.args[i] = args[i]; } proc.args[i] = '\0'; } // posix_spawn(envp) supplies an explicit snapshot. Plain spawn calls // omit it and inherit the environment stored on the parent process. if (environment != nullptr && environmentLength > 0 && environmentLength <= Process::EnvironmentBytes) { for (uint32_t i = 0; i < environmentLength; i++) proc.environment[i] = environment[i]; proc.environmentLength = environmentLength; } else if (parentPrimarySlot >= 0) { auto& parent = processTable[parentPrimarySlot]; for (uint32_t i = 0; i < parent.environmentLength; i++) proc.environment[i] = parent.environment[i]; proc.environmentLength = parent.environmentLength; } else { proc.environment[0] = '\0'; proc.environmentLength = 1; } // Capabilities are kernel-owned and never inferred from the user name. // A normal userspace spawn receives no privileged authority; callers // must use SYS_SPAWN_CAPS for an explicit, kernel-validated delegation. if (parentPrimarySlot >= 0) { if (capabilities != nullptr) { proc.effectiveCaps = capabilities->effective; proc.delegableCaps = capabilities->delegable; } else { proc.effectiveCaps = 0; proc.delegableCaps = 0; } } else { // The kernel-created init process is the root of the capability // delegation tree. No userspace pathname or PID receives this // treatment; it is reached only with no current parent process. proc.effectiveCaps = montauk::abi::CAP_ALL; proc.delegableCaps = montauk::abi::CAP_ALL; } // Consent can later grant the child exactly the eligible authority // its parent could have delegated to it now. proc.consentCeiling = (parentPrimarySlot >= 0) ? processTable[parentPrimarySlot].delegableCaps & montauk::abi::CAP_CONSENT_ELIGIBLE : 0; proc.consentDenied = 0; { int i = 0; for (; i < (int)sizeof(proc.imagePath) - 1 && vfsPath[i]; i++) proc.imagePath[i] = vfsPath[i]; proc.imagePath[i] = '\0'; } // Inherit user string from parent, or default to "system" if no parent. // An explicit override is accepted only through the validated // SYS_SPAWN_CAPS path. { if (userOverride != nullptr) { int i = 0; for (; i < 31 && userOverride[i]; i++) proc.user[i] = userOverride[i]; proc.user[i] = '\0'; } else if (parentSlot >= 0) { int i = 0; for (; i < 31 && processTable[parentSlot].user[i]; i++) proc.user[i] = processTable[parentSlot].user[i]; proc.user[i] = '\0'; } else { // Spawned from kernel (no parent process) - set to "system" proc.user[0] = 's'; proc.user[1] = 'y'; proc.user[2] = 's'; proc.user[3] = 't'; proc.user[4] = 'e'; proc.user[5] = 'm'; proc.user[6] = '\0'; } } // Process sessions are explicit groups used by supervisors such as // login.elf. A session leader opts in with CreateSession(); all of its // subsequently spawned descendants inherit the same stable ID. proc.sessionId = (parentPrimarySlot >= 0) ? processTable[parentPrimarySlot].sessionId : -1; { if (parentSlot >= 0 && processTable[parentSlot].cwd[0]) { int i = 0; for (; i < 255 && processTable[parentSlot].cwd[i]; i++) { proc.cwd[i] = processTable[parentSlot].cwd[i]; } proc.cwd[i] = '\0'; } else { proc.cwd[0] = '0'; proc.cwd[1] = ':'; proc.cwd[2] = '/'; proc.cwd[3] = '\0'; } } proc.redirected = false; proc.parentPid = (parentSlot >= 0) ? processTable[parentSlot].pid : -1; proc.outBuf = nullptr; proc.outHead = 0; proc.outTail = 0; proc.inBuf = nullptr; proc.inHead = 0; proc.inTail = 0; proc.keyHead = 0; proc.keyTail = 0; proc.getcharUtf8Len = 0; proc.getcharUtf8Pos = 0; proc.termCols = 0; proc.termRows = 0; proc.ioOutHandle = -1; proc.ioInHandle = -1; proc.ioKeyHandle = -1; proc.ioWaitsetHandle = -1; // Initialize FPU state: zero out, then set default FCW and MXCSR memset(proc.fpuState, 0, 512); *(uint16_t*)&proc.fpuState[0] = 0x037F; // FCW: default x87 control word *(uint32_t*)&proc.fpuState[24] = 0x1F80; // MXCSR: default SSE control/status Ipc::ProcessStartedInSlot(slot, proc.pid); int resultPid = proc.pid; schedLock.Release(); if (startReady) KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1); return resultPid; } // ==================================================================== // Threading // // Every thread is a slot in processTable. The main thread's primarySlot // points to itself and owns process-level state (PML4, IPC handles, // cwd/user/args, redirected I/O). A sibling thread copies pml4Phys from // the primary so context-switch is cheap, but per-process getters use // primarySlot to reach the canonical state. // // Sibling-thread lifecycle: // * SpawnThread: allocate slot + kernel stack; user provides user stack. // * ExitCurrentThread: slot -> Terminated, kernel stack kept alive for // the joiner; joiner reads exitCode and frees the kernel stack. // * ExitProcess from the main thread sweeps any still-Terminated // sibling slots (no joiner ever arrived) and tears the rest down. // ==================================================================== int SpawnThread(uint64_t entry, uint64_t arg, uint64_t userStackTop) { auto* cpu = Smp::GetCurrentCpuData(); if (cpu == nullptr || cpu->currentSlot < 0) return -1; if (entry == 0 || userStackTop < 16) return -1; int callerSlot = cpu->currentSlot; int primarySlot_ = processTable[callerSlot].primarySlot; if (primarySlot_ < 0) primarySlot_ = callerSlot; uint64_t sharedPml4 = processTable[primarySlot_].pml4Phys; int processPid = processTable[primarySlot_].pid; if (sharedPml4 == 0) return -1; // Give the new thread its own TLS block when the image has one. // We are executing inside the process's address space, so the // freshly mapped pages and the template are directly addressable. // The block uses process-heap virtual space. Its physical pages are // reclaimed when the sibling exits (or if creation fails). uint64_t threadFsBase = 0; uint64_t threadTlsBase = 0; uint64_t threadTlsPages = 0; Process& primary = processTable[primarySlot_]; if (primary.tlsMemSize > 0) { TlsLayout layout = ComputeTlsLayout(primary.tlsMemSize, primary.tlsAlign); uint64_t numPages = (layout.totalSize + 0xFFF) / 0x1000; uint64_t base = 0; if (!ReserveUserHeapRange(primarySlot_, numPages * 0x1000, base)) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "Thread TLS exceeds user heap range"; return -1; } bool ok = true; uint64_t mappedPages = 0; for (uint64_t p = 0; p < numPages; p++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { ok = false; break; } uint64_t physAddr = Memory::SubHHDM((uint64_t)page); if (!Memory::VMM::Paging::MapUserIn(sharedPml4, physAddr, base + p * 0x1000)) { Memory::g_pfa->Free(page); ok = false; break; } mappedPages++; } if (!ok) { Memory::UnmapAndFreeUserRange(sharedPml4, base, mappedPages); ReleaseUserHeapRange(primarySlot_, base, numPages * 0x1000ULL); Kt::KernelLogStream(Kt::ERROR, "Sched") << "Thread TLS allocation failed"; return -1; } uint64_t tp = base + layout.tpOffset; memcpy((void*)(base + layout.templateOffset), (const void*)primary.tlsTemplateVaddr, primary.tlsFileSize); *(uint64_t*)tp = tp; /* ABI self-pointer */ threadFsBase = tp; threadTlsBase = base; threadTlsPages = numPages; } // We do not write to the user stack from kernel mode (that would // require validating the user VA against the process page tables). // Userspace is responsible for ensuring the thread entry calls // SYS_THREAD_EXIT; the libc trampoline does this. The kernel only // enforces SystemV alignment by handing the entry RSP = 16n - 8. uint64_t userRsp = (userStackTop & ~0xFULL) - 8; // Allocate kernel stack. void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages); if (stackMem == nullptr) { if (threadTlsPages != 0) { Memory::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages); ReleaseUserHeapRange(primarySlot_, threadTlsBase, threadTlsPages * 0x1000ULL); } return -1; } memset(stackMem, 0, StackSize); uint8_t* kernelStackBase = (uint8_t*)stackMem; uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize; // Initial kernel frame -- SchedContextSwitch "returns" into ProcessStartup. uint64_t* sp = (uint64_t*)kernelStackTop; *(--sp) = (uint64_t)ProcessStartup; *(--sp) = 0; *(--sp) = 0; *(--sp) = 0; *(--sp) = 0; *(--sp) = 0; *(--sp) = 0; schedLock.Acquire(); int slot = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Free) { slot = i; break; } } if (slot < 0) { schedLock.Release(); Memory::g_pfa->Free(stackMem, StackPages); if (threadTlsPages != 0) { Memory::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages); ReleaseUserHeapRange(primarySlot_, threadTlsBase, threadTlsPages * 0x1000ULL); } return -1; } Process& thr = processTable[slot]; thr.pid = nextPid++; thr.state = ProcessState::Ready; readyCount++; thr.runningOnCpu = -1; thr.killPending = false; thr.reapReady = false; thr.startPending = false; thr.waitingForPid = -1; thr.sleepUntilTick = 0; thr.waitingOnObject = nullptr; thr.primarySlot = primarySlot_; thr.joinerSlot = -1; thr.exitCode = 0; thr.joinable = true; thr.threadArg = arg; thr.futexAddr = 0; thr.futexWoken = false; thr.savedRsp = (uint64_t)sp; thr.stackBase = (uint64_t)kernelStackBase; thr.entryPoint = entry; thr.userStackTop = userRsp; thr.kernelStackTop = kernelStackTop; thr.pml4Phys = sharedPml4; // shared with primary, NOT owned thr.sliceRemaining = TimeSliceMs; thr.cpuTimeMs = 0; thr.heapNext = 0; thr.readdirCursor = 0; thr.parentPid = processPid; thr.redirected = false; thr.outBuf = nullptr; thr.outHead = 0; thr.outTail = 0; thr.inBuf = nullptr; thr.inHead = 0; thr.inTail = 0; thr.keyHead = 0; thr.keyTail = 0; thr.getcharUtf8Len = 0; thr.getcharUtf8Pos = 0; thr.termCols = 0; thr.termRows = 0; thr.ioOutHandle = -1; thr.ioInHandle = -1; thr.ioKeyHandle = -1; thr.ioWaitsetHandle = -1; thr.args[0] = '\0'; thr.user[0] = '\0'; thr.cwd[0] = '\0'; thr.fsBase = threadFsBase; thr.tlsTemplateVaddr = 0; thr.tlsFileSize = 0; thr.tlsMemSize = 0; thr.tlsAlign = 0; // Derive a debug name from the primary's name. { const char* base = processTable[primarySlot_].name; int i = 0; for (; i < 58 && base[i]; i++) thr.name[i] = base[i]; thr.name[i++] = ':'; thr.name[i++] = 't'; thr.name[i] = '\0'; } memset(thr.fpuState, 0, 512); *(uint16_t*)&thr.fpuState[0] = 0x037F; *(uint32_t*)&thr.fpuState[24] = 0x1F80; int tid = thr.pid; schedLock.Release(); KickOneIdleCpu(cpu->cpuIndex); return tid; } bool ReserveUserHeapRange(int primarySlot_, uint64_t size, uint64_t& outVa) { outVa = 0; if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 || (size & 0xFFFULL) != 0) { return false; } schedLock.Acquire(); Process& primary = processTable[primarySlot_]; bool validOwner = primary.primarySlot == primarySlot_ && primary.state != ProcessState::Free && primary.state != ProcessState::Terminated; bool fits = false; uint64_t pages = size / 0x1000ULL; uint64_t runStart = 0; uint64_t runLength = 0; if (validOwner && pages <= UserHeapPages) { for (uint64_t page = 0; page < UserHeapPages; page++) { bool used = (g_userHeapPageMap[primarySlot_][page / 64] & (1ULL << (page % 64))) != 0; if (used) { runLength = 0; continue; } if (runLength == 0) runStart = page; runLength++; if (runLength == pages) { for (uint64_t p = runStart; p < runStart + pages; p++) g_userHeapPageMap[primarySlot_][p / 64] |= 1ULL << (p % 64); outVa = UserHeapBase + runStart * 0x1000ULL; uint64_t end = outVa + size; if (end > primary.heapNext) primary.heapNext = end; fits = true; break; } } } schedLock.Release(); return fits; } void ReleaseUserHeapRange(int primarySlot_, uint64_t va, uint64_t size) { if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 || (va & 0xFFFULL) != 0 || (size & 0xFFFULL) != 0 || va < UserHeapBase || va > UserHeapLimit || size > UserHeapLimit - va) { return; } uint64_t first = (va - UserHeapBase) / 0x1000ULL; uint64_t pages = size / 0x1000ULL; schedLock.Acquire(); for (uint64_t p = first; p < first + pages; p++) g_userHeapPageMap[primarySlot_][p / 64] &= ~(1ULL << (p % 64)); schedLock.Release(); } static void FreeSiblingThreadTls(Process& thr) { int primarySlot_ = thr.primarySlot; if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || primarySlot_ == (int)(&thr - processTable) || thr.fsBase == 0) { return; } Process& primary = processTable[primarySlot_]; if (primary.tlsMemSize == 0 || primary.pml4Phys == 0) { thr.fsBase = 0; return; } TlsLayout layout = ComputeTlsLayout(primary.tlsMemSize, primary.tlsAlign); if (layout.tpOffset > thr.fsBase || layout.totalSize < layout.tpOffset) { Panic("Invalid sibling TLS metadata during thread teardown", nullptr); } uint64_t base = thr.fsBase - layout.tpOffset; uint64_t pages = (layout.totalSize + 0xFFF) / 0x1000; thr.fsBase = 0; Memory::UnmapAndFreeUserRange(primary.pml4Phys, base, pages); ReleaseUserHeapRange(primarySlot_, base, pages * 0x1000ULL); } // Switch away from a slot we have just marked non-runnable while holding // schedLock. Mirrors SwitchAwayFromBlockedCurrentLocked but does NOT // assume the caller will be resumed -- used by ExitCurrentThread. static void SwitchAwayFromExitingThreadLocked(int slot, Process& thr) { auto* cpu = Smp::GetCurrentCpuData(); int next = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Ready) { next = i; break; } } if (next >= 0) { cpu->currentSlot = next; processTable[next].state = ProcessState::Running; readyCount--; processTable[next].runningOnCpu = cpu->cpuIndex; processTable[next].sliceRemaining = TimeSliceMs; cpu->kernelRsp = processTable[next].kernelStackTop; cpu->tss->rsp0 = processTable[next].kernelStackTop; if (readyCount > 0) { KickOneIdleCpu(cpu->cpuIndex); } LoadUserFsBase(cpu, processTable[next].fsBase); SchedContextSwitch(&thr.savedRsp, processTable[next].savedRsp, processTable[next].pml4Phys, thr.fpuState, processTable[next].fpuState); } else { cpu->currentSlot = -1; SchedContextSwitch(&thr.savedRsp, cpu->idleSavedRsp, GetKernelCR3(), thr.fpuState, nullptr); } schedLock.Release(); (void)slot; } [[noreturn]] void ExitCurrentThread(int exitCode) { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) { for (;;) asm volatile("hlt"); } Process& thr = processTable[slot]; int primarySlot_ = thr.primarySlot; if (primarySlot_ < 0 || primarySlot_ == slot) { // This is the main thread (or an orphan): full process exit. thr.exitCode = exitCode; ExitProcess(); __builtin_unreachable(); } bool processExitKill = thr.killPending.exchange(false, std::memory_order_acq_rel); thr.startPending = false; thr.exitCode = exitCode; // TLS belongs only to this sibling. Reclaim it before publishing the // Terminated state; otherwise a long-lived multithreaded GUI process // leaks these pages until the entire process exits. When the primary // is already coordinating full teardown, defer this to its final // sibling sweep: the primary may be waiting from an IF=0 timer frame // and therefore cannot acknowledge our TLB-shootdown IPI here. if (!processExitKill) FreeSiblingThreadTls(thr); schedLock.Acquire(); thr.state = ProcessState::Terminated; thr.runningOnCpu = -1; thr.waitingForPid = -1; thr.sleepUntilTick = 0; thr.waitingOnObject = nullptr; // Wake any joiner blocked on this slot. void* joinObj = &processTable[slot]; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Blocked && processTable[i].waitingOnObject == joinObj) { processTable[i].state = ProcessState::Ready; readyCount++; processTable[i].waitingOnObject = nullptr; processTable[i].sleepUntilTick = 0; processTable[i].waitingForPid = -1; } } SwitchAwayFromExitingThreadLocked(slot, thr); // Unreachable: the slot is Terminated, nobody resumes us. for (;;) asm volatile("hlt"); } int JoinThread(int tid, int* outExitCode) { auto* cpu = Smp::GetCurrentCpuData(); if (cpu == nullptr || cpu->currentSlot < 0) return -1; int callerSlot = cpu->currentSlot; int callerPrimary = processTable[callerSlot].primarySlot; if (callerPrimary < 0) callerPrimary = callerSlot; if (tid <= 0) return -1; // Locate the sibling slot under lock. schedLock.Acquire(); int target = -1; for (int i = 0; i < MaxProcesses; i++) { if (i == callerSlot) continue; if (processTable[i].pid != tid) continue; if (processTable[i].primarySlot != callerPrimary) continue; auto s = processTable[i].state; if (s == ProcessState::Free) continue; target = i; break; } if (target < 0) { schedLock.Release(); return -1; } // Fast path: target already Terminated -- harvest immediately. if (processTable[target].state == ProcessState::Terminated) { int code = processTable[target].exitCode; void* stackBase = (void*)processTable[target].stackBase; processTable[target].stackBase = 0; processTable[target].pml4Phys = 0; processTable[target].joinerSlot = -1; processTable[target].primarySlot = -1; processTable[target].state = ProcessState::Free; schedLock.Release(); if (stackBase) Memory::g_pfa->Free(stackBase, StackPages); if (outExitCode) *outExitCode = code; return 0; } // Block until the sibling terminates and wakes us. processTable[target].joinerSlot = callerSlot; processTable[callerSlot].state = ProcessState::Blocked; processTable[callerSlot].waitingOnObject = &processTable[target]; processTable[callerSlot].waitingForPid = -1; processTable[callerSlot].sleepUntilTick = 0; processTable[callerSlot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); // Resumed -- harvest the sibling. schedLock.Acquire(); int code = processTable[target].exitCode; void* stackBase = (void*)processTable[target].stackBase; processTable[target].stackBase = 0; processTable[target].pml4Phys = 0; processTable[target].joinerSlot = -1; processTable[target].primarySlot = -1; processTable[target].state = ProcessState::Free; schedLock.Release(); if (stackBase) Memory::g_pfa->Free(stackBase, StackPages); if (outExitCode) *outExitCode = code; return 0; } int StartProcess(int pid) { if (pid < 0) return -1; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid != pid) continue; if (processTable[i].state != ProcessState::Blocked || !processTable[i].startPending) { schedLock.Release(); return -1; } processTable[i].startPending = false; processTable[i].waitingForPid = -1; processTable[i].sleepUntilTick = 0; processTable[i].waitingOnObject = nullptr; processTable[i].state = ProcessState::Ready; readyCount++; schedLock.Release(); KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1); return 0; } schedLock.Release(); return -1; } // ==================================================================== // Schedule -- core context switch logic // // The schedLock is held ACROSS the context switch. This is critical: // setting the old process to Ready and saving its RSP must be atomic // with respect to other CPUs. If we released the lock before saving // RSP, another CPU could pick up the process with a stale savedRsp. // // The RESUMED process releases the lock (it was acquired by whatever // CPU called Schedule() and switched away from that process). // New processes release it in ProcessStartup(). // ==================================================================== // BSP maintenance that may allocate/free memory or touch policy MSRs must // run from the idle kernel context, never from the timer interrupt. The // pending flag is set by process teardown on any CPU and periodically by // the BSP timer. It remains set until an idle maintenance pass consumes it. static std::atomic bspMaintenancePending{false}; static uint64_t nextBspPolicyTick = 0; // BSP timer only // Reclaim terminated process slots from BSP idle context. This avoids // holding schedLock (with interrupts disabled) during PFA::Free on the // hot scheduling path or inside a hard interrupt. static void ReclaimTerminated() { schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Terminated && processTable[i].reapReady) { // Grab the pointers, mark Free, then free memory after releasing lock void* stackBase = (processTable[i].stackBase != 0) ? (void*)processTable[i].stackBase : nullptr; void* pml4 = (processTable[i].pml4Phys != 0) ? (void*)Memory::HHDM(processTable[i].pml4Phys) : nullptr; processTable[i].stackBase = 0; processTable[i].pml4Phys = 0; processTable[i].reapReady = false; processTable[i].startPending = false; processTable[i].state = ProcessState::Free; // Release lock during PFA::Free to minimize hold time schedLock.Release(); if (stackBase) Memory::g_pfa->Free(stackBase, StackPages); if (pml4) Memory::g_pfa->Free(pml4); schedLock.Acquire(); } } schedLock.Release(); } bool HasReadyProcesses() { return readyCount > 0; } void RequestKernelWork() { auto* cpu = Smp::GetCurrentCpuData(); if (cpu != nullptr && cpu->currentSlot >= 0) { cpu->kernelWorkPending = true; } } // Bounded, IRQ-safe portion of BSP maintenance. Keeping deadline expiry // on every 1 ms BSP tick prevents timed input/object waits from depending // on the BSP becoming idle under sustained desktop load. static void WakeExpiredProcesses() { bool wokeProcesses = false; schedLock.Acquire(); uint64_t now = Timekeeping::GetTicks(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Blocked && processTable[i].sleepUntilTick != 0 && now >= processTable[i].sleepUntilTick) { processTable[i].sleepUntilTick = 0; processTable[i].waitingForPid = -1; processTable[i].waitingOnObject = nullptr; processTable[i].state = ProcessState::Ready; readyCount++; wokeProcesses = true; } } schedLock.Release(); if (wokeProcesses) { KickOneIdleCpu(0); } } void RunBspMaintenance() { // Clear before scanning. A producer that races this pass sets the flag // again and therefore cannot have its request acknowledged by mistake. if (!bspMaintenancePending.exchange(false, std::memory_order_acq_rel)) { return; } ReclaimTerminated(); // Thermal governor step (rate-limited internally to 500 ms). // The timer requests this idle-context pass even under full load. Hal::CpuPower::ThermalTick(Timekeeping::GetMilliseconds()); } uint64_t GetNextDeadlineTick() { uint64_t nextDeadline = 0; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state != ProcessState::Blocked) continue; uint64_t deadline = processTable[i].sleepUntilTick; if (deadline == 0) continue; if (nextDeadline == 0 || deadline < nextDeadline) { nextDeadline = deadline; } } schedLock.Release(); return nextDeadline; } void Schedule() { auto* cpu = Smp::GetCurrentCpuData(); schedLock.Acquire(); // Normally select the next Ready process. A pending bottom half gets // one deliberate trip through the idle kernel context first; that // context drains bounded work and immediately schedules user work // again. This avoids both hard-IRQ callbacks and bottom-half starvation // when there is always at least one runnable desktop process. int next = -1; bool yieldToKernel = cpu->currentSlot >= 0 && cpu->kernelWorkPending; if (yieldToKernel) cpu->kernelWorkPending = false; int start = (cpu->currentSlot >= 0) ? cpu->currentSlot + 1 : 0; if (!yieldToKernel) { for (int i = 0; i < MaxProcesses; i++) { int idx = (start + i) % MaxProcesses; if (processTable[idx].state == ProcessState::Ready) { next = idx; break; } } } if (next < 0) { // No other process is ready. A normal time-slice expiry must keep // the sole Running process on this CPU: bouncing it through idle // can suspend a syscall while it owns a process-context Mutex, then // let ServiceDeferredWork spin on that same mutex with the idle // context reserved against scheduling. That is a permanent // self-deadlock. Only an explicit, ring-3-safe bottom-half request // is allowed to yield a running process to idle. if (cpu->currentSlot >= 0) { if (!yieldToKernel) { processTable[cpu->currentSlot].sliceRemaining = TimeSliceMs; schedLock.Release(); return; } int oldSlot = cpu->currentSlot; processTable[oldSlot].state = ProcessState::Ready; readyCount++; processTable[oldSlot].runningOnCpu = -1; cpu->currentSlot = -1; // Lock held across context switch -- the idle loop // doesn't release it; the next Schedule() that resumes // a process from idle will release it via the resumed // process path. But idle is special: we release here // because idle doesn't go through ProcessStartup. // The RSP save is safe because interrupts are disabled // (Spinlock), so no timer can fire between Ready and save. SchedContextSwitch(&processTable[oldSlot].savedRsp, cpu->idleSavedRsp, GetKernelCR3(), processTable[oldSlot].fpuState, nullptr); // Resumed from idle -- lock was held by whoever switched to us schedLock.Release(); } else { schedLock.Release(); } return; } if (next == cpu->currentSlot) { // Same process, just reset time slice processTable[next].sliceRemaining = TimeSliceMs; schedLock.Release(); return; } // Prepare the context switch uint64_t* oldRspPtr; int oldSlot = cpu->currentSlot; if (oldSlot >= 0) { processTable[oldSlot].state = ProcessState::Ready; readyCount++; processTable[oldSlot].runningOnCpu = -1; oldRspPtr = &processTable[oldSlot].savedRsp; } else { oldRspPtr = &cpu->idleSavedRsp; } cpu->currentSlot = next; processTable[next].state = ProcessState::Running; readyCount--; processTable[next].runningOnCpu = cpu->cpuIndex; processTable[next].sliceRemaining = TimeSliceMs; uint64_t newCR3 = processTable[next].pml4Phys; // Update per-CPU kernel RSP and TSS RSP0 cpu->kernelRsp = processTable[next].kernelStackTop; cpu->tss->rsp0 = processTable[next].kernelStackTop; uint8_t* oldFpu = (oldSlot >= 0) ? processTable[oldSlot].fpuState : nullptr; uint8_t* newFpu = processTable[next].fpuState; if (oldSlot < 0) { // AP idle loops mask their local periodic timer. Rearm it before // dispatching user work so preemption resumes with the process. // Also pick up a thermal-governor policy epoch that may have // changed while this CPU remained asleep without timer ticks. Hal::CpuPower::ApplyPolicyIfChanged(); Timekeeping::ApicTimerLeaveApIdle(); } LoadUserFsBase(cpu, processTable[next].fsBase); // DO NOT release schedLock here! It is held across the context // switch so that setting Ready + saving RSP is atomic. The // resumed process releases it. SchedContextSwitch(oldRspPtr, processTable[next].savedRsp, newCR3, oldFpu, newFpu); // We reach here when this process is resumed by another CPU's // Schedule(). That CPU held the lock across its context switch. // Release it now. schedLock.Release(); } void Tick(uint32_t elapsedMs, bool interruptedUser) { auto* cpu = Smp::GetCurrentCpuData(); // Pick up thermal-governor frequency changes (HWP requests are // per-core MSRs, so every CPU applies them itself). No-op unless // the governor bumped the policy epoch since our last tick. Hal::CpuPower::ApplyPolicyIfChanged(); // BSP: expire timed waits in the bounded IRQ-safe pass. Queue heavier // reclamation/policy work for the idle kernel context. if (cpu->cpuIndex == 0) { WakeExpiredProcesses(); uint64_t now = Timekeeping::GetTicks(); if (nextBspPolicyTick == 0 || now >= nextBspPolicyTick) { nextBspPolicyTick = now + 500; bspMaintenancePending.store(true, std::memory_order_release); } if (interruptedUser && bspMaintenancePending.load(std::memory_order_acquire)) { RequestKernelWork(); } } int slot = cpu->currentSlot; if (slot < 0) { // Idle CPU. Check the approximate ready count to avoid // scanning 256 process slots on every tick. On a 32-core // system with 27 idle CPUs, this avoids ~7M cache-line // reads/sec from the process table. // // A CPU whose idle context is reserved for long-running kernel // work (e.g. the deferred Bluetooth firmware download) is left // alone: stealing it starves that work of CPU time while its // wall-clock timeouts keep running. if (readyCount > 0 && !cpu->reservedForKernelWork) { Schedule(); } return; } processTable[slot].cpuTimeMs += elapsedMs; if (cpu->kernelWorkPending) { processTable[slot].sliceRemaining = 0; } // Check if another CPU requested this process be killed. Full teardown // takes process-context mutexes. It is safe here only when the timer // interrupted ring 3; if it interrupted a syscall, teardown could try // to reacquire a lock already held by this same suspended kernel stack. // In that case force a reschedule and finish at syscall-return instead. if (processTable[slot].killPending) { if (interruptedUser) { ExitProcess(); return; } processTable[slot].sliceRemaining = 0; } if (processTable[slot].sliceRemaining > elapsedMs) { processTable[slot].sliceRemaining -= elapsedMs; } else { processTable[slot].sliceRemaining = 0; } // Never context-switch away from an arbitrary kernel stack. Syscalls // routinely own non-IRQ-disabling subsystem mutexes; suspending that // stack and dispatching a process that wants the same mutex leaves the // new process spinning in ring 0, where the timer deliberately cannot // preempt it to resume the owner. Preserve the expired slice and take // the switch on the first timer interrupt after SYSRET reaches ring 3. if (processTable[slot].sliceRemaining == 0 && interruptedUser) { Schedule(); } } int GetCurrentPid() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return -1; int primary = processTable[slot].primarySlot; if (primary < 0) primary = slot; return processTable[primary].pid; } void ExitIfKilled() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu ? cpu->currentSlot : -1; if (slot >= 0 && processTable[slot].killPending) { ExitProcess(); } } int GetCurrentTid() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; return (slot >= 0) ? processTable[slot].pid : -1; } Process* GetCurrentProcessPtr() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return nullptr; int primary = processTable[slot].primarySlot; if (primary < 0) primary = slot; return &processTable[primary]; } bool HasCapability(uint64_t capability) { Process* proc = GetCurrentProcessPtr(); return proc != nullptr && (proc->effectiveCaps & capability) == capability; } Process* GetCurrentThreadPtr() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return nullptr; return &processTable[slot]; } // Serializes stack growth so two threads faulting on the same page do not // both map it (the loser's page would leak: FreeUserHalf only frees pages // still referenced by the page tables). static kcp::Spinlock stackGrowLock; bool TryGrowUserStack(uint64_t faultAddr) { constexpr uint64_t growBase = UserStackTop - UserStackMax; constexpr uint64_t eagerBase = UserStackTop - UserStackSize; if (faultAddr < growBase || faultAddr >= eagerBase) return false; Process* proc = GetCurrentProcessPtr(); if (proc == nullptr) return false; uint64_t pageVa = faultAddr & ~0xFFFULL; stackGrowLock.Acquire(); // Another thread may have mapped this page between the fault and here. if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pml4Phys, pageVa, 0x1000, true)) { stackGrowLock.Release(); return true; } void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { stackGrowLock.Release(); Kt::KernelLogStream(Kt::ERROR, "Sched") << "Out of memory growing user stack for pid " << proc->pid; return false; } uint64_t physAddr = Memory::SubHHDM((uint64_t)page); if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, pageVa)) { stackGrowLock.Release(); Memory::g_pfa->Free(page); return false; } stackGrowLock.Release(); asm volatile("invlpg (%0)" :: "r"(pageVa) : "memory"); return true; } void ExitProcess() { auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) { return; } // If we're a sibling thread, this is a per-thread exit, not a // process exit. ExitCurrentThread does the right thing and never // returns. { int primary = processTable[slot].primarySlot; if (primary >= 0 && primary != slot) { ExitCurrentThread(0); } } Process& proc = processTable[slot]; proc.killPending = false; proc.startPending = false; int exitingPid = proc.pid; int primarySlot_ = slot; // Drain sibling threads: any thread sharing our PML4 must stop using // it before we call FreeUserHalf. Ready/Blocked siblings can be // terminated immediately; siblings running on other CPUs are tagged // killPending and we spin until their tick handler routes them // through ExitCurrentThread. // Several device syscalls have bounded waits up to five seconds. Give // an interrupted kernel-mode sibling room to finish that recovery // path before declaring it wedged. static constexpr uint64_t SiblingDrainTimeoutMs = 10000; static constexpr uint32_t SiblingDrainMaxPasses = 262144; uint64_t siblingDrainDeadline = Timekeeping::GetTicks() + SiblingDrainTimeoutMs; uint32_t siblingDrainPasses = 0; for (;;) { bool anyRunning = false; bool targetCpus[Smp::MaxCPUs] = {}; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (i == primarySlot_) continue; if (processTable[i].primarySlot != primarySlot_) continue; auto st = processTable[i].state; if (st == ProcessState::Free || st == ProcessState::Terminated) { continue; } if (st == ProcessState::Running) { processTable[i].killPending.store(true, std::memory_order_release); int targetCpu = processTable[i].runningOnCpu; if (targetCpu >= 0 && targetCpu < Smp::MaxCPUs) targetCpus[targetCpu] = true; anyRunning = true; continue; } // Ready or Blocked -- terminate inline, wake any joiner. if (st == ProcessState::Ready) readyCount--; processTable[i].state = ProcessState::Terminated; processTable[i].killPending = false; processTable[i].runningOnCpu = -1; processTable[i].waitingForPid = -1; processTable[i].sleepUntilTick = 0; processTable[i].waitingOnObject = nullptr; void* obj = &processTable[i]; for (int j = 0; j < MaxProcesses; j++) { if (processTable[j].state == ProcessState::Blocked && processTable[j].waitingOnObject == obj) { processTable[j].state = ProcessState::Ready; readyCount++; processTable[j].waitingOnObject = nullptr; processTable[j].sleepUntilTick = 0; processTable[j].waitingForPid = -1; } } } schedLock.Release(); if (!anyRunning) break; // Do not depend solely on a remote CPU's next periodic timer. // Laptop firmware/IRQ pathologies can delay that tick, and the // old unbounded wait then trapped the exiting CPU forever. if ((siblingDrainPasses & 0x3FF) == 0) { for (int i = 0; i < Smp::GetCpuCount(); i++) { if (!targetCpus[i] || i == cpu->cpuIndex) continue; Smp::CpuData* target = Smp::GetCpuData(i); if (target != nullptr && target->started) { (void)Hal::LocalApic::SendFixedIpi(target->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE); } } } siblingDrainPasses++; if (siblingDrainPasses >= SiblingDrainMaxPasses || Timekeeping::GetTicks() >= siblingDrainDeadline) { // It is unsafe to free the shared address space while another // CPU may still execute in it. A visible panic is preferable // to silent lockup or use-after-free corruption. Panic("Timed out draining sibling threads during process exit", nullptr); } // Wait for the sibling CPU(s) to observe killPending on their // next timer tick. Briefly busy-wait; this path is rare. for (int spin = 0; spin < 1024; spin++) { asm volatile("pause"); } } // All siblings are quiescent. Reap any terminated sibling slots // (the process is gone, no joiner will arrive). for (int i = 0; i < MaxProcesses; i++) { if (i == primarySlot_) continue; if (processTable[i].primarySlot != primarySlot_) continue; if (processTable[i].state != ProcessState::Terminated) continue; // Ready/blocked siblings terminated inline above never ran their // ordinary ExitCurrentThread cleanup. Running siblings normally // arrive here with fsBase already cleared, making this a no-op. FreeSiblingThreadTls(processTable[i]); void* stackBase = (void*)processTable[i].stackBase; processTable[i].stackBase = 0; processTable[i].pml4Phys = 0; processTable[i].primarySlot = -1; processTable[i].joinerSlot = -1; processTable[i].state = ProcessState::Free; if (stackBase) Memory::g_pfa->Free(stackBase, StackPages); } // Clean up any windows owned by this process WinServer::CleanupProcess(exitingPid); // Cancel this process's consent requests; if it was the consent // agent, deny everything still waiting on it. Consent::OnProcessExit(exitingPid); // Release any mixer virtual streams owned by this process so the slot // and its ring buffer don't leak when an app forgets to audio_close. Drivers::Audio::Mixer::CleanupProcess(exitingPid); // Restore scanout to buffer 0 if the exiting process owned page // flips, so the next fullscreen client and the kernel terminal are // never stranded on the invisible buffer (no-op for non-owners). Drivers::Graphics::IntelGPU::OnProcessExit(exitingPid); // USB interface claims are process-owned capabilities. Closing them // here stops DMA streaming and releases exclusivity even when an app // exits without calling usb_close(). Drivers::USB::UserUsb::ReleaseAllForPid(exitingPid); // Release process-scoped IPC handles/mappings before tearing down the address space. Ipc::CleanupProcessSlot(slot, exitingPid, proc.pml4Phys); montauk::abi::CleanupHeapForSlot(slot, proc.pml4Phys); montauk::abi::CleanupLibTable(slot); proc.waitingForPid = -1; proc.sleepUntilTick = 0; proc.waitingOnObject = nullptr; proc.redirected = false; proc.parentPid = -1; proc.outBuf = nullptr; proc.outHead = 0; proc.outTail = 0; proc.inBuf = nullptr; proc.inHead = 0; proc.inTail = 0; proc.keyHead = 0; proc.keyTail = 0; proc.getcharUtf8Len = 0; proc.getcharUtf8Pos = 0; proc.termCols = 0; proc.termRows = 0; proc.ioOutHandle = -1; proc.ioInHandle = -1; proc.ioKeyHandle = -1; proc.ioWaitsetHandle = -1; Ipc::ProcessExitedInSlot(slot, exitingPid); // Free all user-space physical pages and page table structures Memory::VMM::Paging::FreeUserHalf(proc.pml4Phys); schedLock.Acquire(); proc.state = ProcessState::Terminated; proc.runningOnCpu = -1; proc.reapReady = true; bspMaintenancePending.store(true, std::memory_order_release); // Publish the exit code before waking waiters so SYS_WAITPID // observes it the moment it unblocks. exitLedgerPid[exitLedgerHead] = exitingPid; exitLedgerCode[exitLedgerHead] = proc.exitCode; exitLedgerHead = (exitLedgerHead + 1) % ExitLedgerSize; // Wake any processes blocked on this PID for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Blocked && processTable[i].waitingForPid == exitingPid) { processTable[i].state = ProcessState::Ready; readyCount++; processTable[i].waitingForPid = -1; processTable[i].waitingOnObject = nullptr; processTable[i].sleepUntilTick = 0; } } // Find next ready process int next = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state == ProcessState::Ready) { next = i; break; } } if (next >= 0) { cpu->currentSlot = next; processTable[next].state = ProcessState::Running; readyCount--; processTable[next].runningOnCpu = cpu->cpuIndex; processTable[next].sliceRemaining = TimeSliceMs; if (readyCount > 0) { KickOneIdleCpu(cpu->cpuIndex); } uint64_t newCR3 = processTable[next].pml4Phys; cpu->kernelRsp = processTable[next].kernelStackTop; cpu->tss->rsp0 = processTable[next].kernelStackTop; LoadUserFsBase(cpu, processTable[next].fsBase); // Lock held across context switch -- resumed process releases it SchedContextSwitch(&processTable[slot].savedRsp, processTable[next].savedRsp, newCR3, processTable[slot].fpuState, processTable[next].fpuState); schedLock.Release(); } else { cpu->currentSlot = -1; // Switch to idle -- release after resuming from idle SchedContextSwitch(&processTable[slot].savedRsp, cpu->idleSavedRsp, GetKernelCR3(), processTable[slot].fpuState, nullptr); schedLock.Release(); } for (;;) { asm volatile("hlt"); } } int KillProcess(int pid, int exitCode) { // Refuse to kill PID 0 (init). Killing the caller's own process is // allowed: a faulting/aborting sibling thread shares the pid and has // no other way to take the process down. The caller's own slot gets // killPending below and exits at syscall return (ExitIfKilled) or // via the ExitProcess the fault handler calls next. if (pid == 0) return -1; schedLock.Acquire(); // pid identifies a process (main thread). Sibling-thread TIDs are // not killable through this entry point. int primarySlot_ = -1; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid != pid) continue; if (processTable[i].primarySlot != i) continue; auto s = processTable[i].state; if (s == ProcessState::Ready || s == ProcessState::Running || s == ProcessState::Blocked) { primarySlot_ = i; } break; } if (primarySlot_ < 0) { schedLock.Release(); return -1; } processTable[primarySlot_].exitCode = exitCode; // Flag the main thread so its next tick (or scheduler dispatch) // routes through ExitProcess, which sweeps every sibling thread // and tears down the address space. If the main thread is parked // on a blocking syscall we promote it to Ready so the scheduler // picks it up promptly. Process& primary = processTable[primarySlot_]; primary.killPending = true; if (primary.state == ProcessState::Blocked) { primary.state = ProcessState::Ready; readyCount++; primary.waitingForPid = -1; primary.waitingOnObject = nullptr; primary.sleepUntilTick = 0; } // Also stamp running siblings so they exit promptly; ExitProcess on // the main thread will additionally wait for them anyway. for (int i = 0; i < MaxProcesses; i++) { if (i == primarySlot_) continue; if (processTable[i].primarySlot != primarySlot_) continue; auto s = processTable[i].state; if (s == ProcessState::Running) { processTable[i].killPending = true; } } schedLock.Release(); KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1); return 0; } int CreateSession() { schedLock.Acquire(); Process* proc = GetCurrentProcessPtr(); if (proc == nullptr) { schedLock.Release(); return -1; } proc->sessionId = proc->pid; int sessionId = proc->sessionId; schedLock.Release(); return sessionId; } int KillSession(int sessionId) { if (sessionId < 0) return -1; int callerPid = GetCurrentPid(); int killed = 0; // Mark the complete group in one scheduler-lock transaction. New // children inherit sessionId, so a supervisor can repeat this call // until zero is returned to close the small spawn/exit race cleanly. schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { Process& primary = processTable[i]; if (primary.primarySlot != i || primary.pid == callerPid || primary.sessionId != sessionId) { continue; } auto state = primary.state; if (state != ProcessState::Ready && state != ProcessState::Running && state != ProcessState::Blocked) { continue; } primary.exitCode = TerminatedKilled; primary.killPending = true; if (primary.state == ProcessState::Blocked) { primary.state = ProcessState::Ready; readyCount++; primary.waitingForPid = -1; primary.waitingOnObject = nullptr; primary.sleepUntilTick = 0; } for (int j = 0; j < MaxProcesses; j++) { if (j == i || processTable[j].primarySlot != i) continue; if (processTable[j].state == ProcessState::Running) { processTable[j].killPending = true; } } killed++; } schedLock.Release(); if (killed > 0) { KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1); } return killed; } int LookupExitCode(int pid) { schedLock.Acquire(); int code = 0; for (int i = 0; i < ExitLedgerSize; i++) { if (exitLedgerPid[i] == pid) { code = exitLedgerCode[i]; break; } } schedLock.Release(); return code; } void SetProcessExitCode(int pid, int code) { schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid != pid) continue; // Only a process (primary slot) carries a process exit code; // sibling-thread exit values go through the join path. if (processTable[i].primarySlot == i) { processTable[i].exitCode = code; } break; } schedLock.Release(); } void BlockOnPid(int pid) { ExitIfKilled(); // If the target is already dead, return immediately if (!IsAlive(pid)) return; auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return; schedLock.Acquire(); // Double-check under lock (target might have exited between // the lockless IsAlive check and acquiring the lock) bool stillAlive = false; for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid == pid) { auto s = processTable[i].state; stillAlive = (s == ProcessState::Ready || s == ProcessState::Running || s == ProcessState::Blocked); break; } } if (!stillAlive) { schedLock.Release(); return; } // Mark current process as Blocked -- scheduler will skip it. // ExitProcess will wake us when the target terminates. processTable[slot].state = ProcessState::Blocked; processTable[slot].waitingForPid = pid; processTable[slot].waitingOnObject = nullptr; processTable[slot].sleepUntilTick = 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); ExitIfKilled(); } void BlockForSleep(uint64_t ms) { if (ms == 0) return; ExitIfKilled(); auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return; schedLock.Acquire(); processTable[slot].state = ProcessState::Blocked; processTable[slot].waitingForPid = -1; processTable[slot].waitingOnObject = nullptr; processTable[slot].sleepUntilTick = Timekeeping::GetTicks() + ms; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); ExitIfKilled(); } void BlockOnObject(void* object, uint64_t timeoutMs) { if (object == nullptr) return; ExitIfKilled(); auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu->currentSlot; if (slot < 0) return; schedLock.Acquire(); processTable[slot].state = ProcessState::Blocked; processTable[slot].waitingForPid = -1; processTable[slot].waitingOnObject = object; processTable[slot].sleepUntilTick = (timeoutMs > 0) ? (Timekeeping::GetTicks() + timeoutMs) : 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); ExitIfKilled(); } bool BlockOnObjectIf(void* object, uint64_t timeoutMs, bool (*shouldBlock)(void*), void* context) { if (object == nullptr || shouldBlock == nullptr) return false; ExitIfKilled(); auto* cpu = Smp::GetCurrentCpuData(); if (cpu == nullptr) return false; int slot = cpu->currentSlot; if (slot < 0) return false; schedLock.Acquire(); if (!shouldBlock(context)) { schedLock.Release(); return false; } processTable[slot].state = ProcessState::Blocked; processTable[slot].waitingForPid = -1; processTable[slot].waitingOnObject = object; processTable[slot].sleepUntilTick = (timeoutMs > 0) ? (Timekeeping::GetTicks() + timeoutMs) : 0; processTable[slot].runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); ExitIfKilled(); return true; } uint64_t ObserveObjectWake(void* object) { if (object == nullptr) return 0; return objectWakeEpochs[ObjectWakeBucket(object)].load(std::memory_order_acquire); } struct ObjectWakeObservation { void* object; uint64_t epoch; }; static bool ObjectWakeIsUnchanged(void* opaque) { auto* observation = (ObjectWakeObservation*)opaque; uint64_t now = ObserveObjectWake(observation->object); uint64_t was = observation->epoch; if (now == was) return true; // nothing signalled this bucket at all // Exactly one signal landed here since the observation, and it carried // a different object's tag, so it cannot have been ours: the caller may // still block. Any larger or wrapped delta is ambiguous -- more than // one object may have signalled -- and must be reported as a wake so a // real edge is never dropped. if ((uint32_t)now == (uint32_t)was + 1 && (uint32_t)(now >> 32) != ObjectWakeTag(observation->object)) { return true; } return false; } bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake) { ObjectWakeObservation observation{object, observedWake}; return BlockOnObjectIf(object, timeoutMs, ObjectWakeIsUnchanged, &observation); } void WakeObjectWaiters(void* object) { if (object == nullptr) return; // Publish this object's tag and the bumped counter as one word. A CAS // loop (rather than fetch_add) keeps the pair consistent and, unlike a // plain store, cannot drop a concurrent waker's increment on the same // bucket -- a dropped increment would be an unobserved wake edge. auto& bucket = objectWakeEpochs[ObjectWakeBucket(object)]; uint64_t tagBits = (uint64_t)ObjectWakeTag(object) << 32; uint64_t observed = bucket.load(std::memory_order_relaxed); while (!bucket.compare_exchange_weak( observed, tagBits | (uint32_t)((uint32_t)observed + 1), std::memory_order_release, std::memory_order_relaxed)) { } bool wokeAny = false; schedLock.Acquire(); for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].state != ProcessState::Blocked) continue; if (processTable[i].waitingOnObject != object) continue; processTable[i].waitingOnObject = nullptr; processTable[i].sleepUntilTick = 0; processTable[i].waitingForPid = -1; processTable[i].state = ProcessState::Ready; readyCount++; wokeAny = true; } schedLock.Release(); if (wokeAny) { KickOneIdleCpu(Smp::GetCurrentCpuData() ? Smp::GetCurrentCpuData()->cpuIndex : -1); } } // Monotonic enrollment counter for FIFO futex wakeups. Only touched // under schedLock. static uint64_t futexNextSeq = 1; int FutexWait(uint64_t uaddr, uint32_t expected, uint64_t timeoutNs) { if ((uaddr & 3) != 0) return InvalidArgument; ExitIfKilled(); auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu ? cpu->currentSlot : -1; if (slot < 0) return GenericFailure; Process& self = processTable[slot]; // Round up so a short timeout never turns into "check only". uint64_t timeoutMs = 0; if (timeoutNs != FutexWaitForever) { if (timeoutNs == 0) { timeoutMs = 0; } else { timeoutMs = timeoutNs / 1000000ULL + ((timeoutNs % 1000000ULL) ? 1 : 0); } } schedLock.Acquire(); // The word is read here, with schedLock held and interrupts off, so // it must not fault: the caller validated (and demand-committed) the // page, and the page walk below re-confirms it is still mapped. An // aligned 32-bit word never straddles a page. if (!Memory::VMM::Paging::IsUserRangeAccessible(self.pml4Phys, uaddr, 4, false)) { schedLock.Release(); return BadPointer; } uint32_t current = __atomic_load_n((volatile uint32_t*)uaddr, __ATOMIC_SEQ_CST); if (current != expected) { schedLock.Release(); return WouldBlock; } if (timeoutNs == 0) { schedLock.Release(); return TimedOut; } self.state = ProcessState::Blocked; self.waitingForPid = -1; self.waitingOnObject = nullptr; self.futexAddr = uaddr; self.futexSeq = futexNextSeq++; self.futexWoken = false; self.sleepUntilTick = (timeoutNs == FutexWaitForever) ? 0 : Timekeeping::GetTicks() + timeoutMs; self.runningOnCpu = -1; SwitchAwayFromBlockedCurrentLocked(); // Whatever made this thread runnable (FutexWake, the deadline, or a // kill) it is no longer parked; clear the key before this thread can // block for some other reason and be mistaken for a futex waiter. schedLock.Acquire(); bool woken = self.futexWoken; self.futexAddr = 0; self.futexWoken = false; schedLock.Release(); ExitIfKilled(); return woken ? Success : TimedOut; } int FutexWake(uint64_t uaddr, uint32_t count) { if ((uaddr & 3) != 0 || uaddr == 0) return InvalidArgument; if (count == 0) return 0; auto* cpu = Smp::GetCurrentCpuData(); int slot = cpu ? cpu->currentSlot : -1; if (slot < 0) return GenericFailure; schedLock.Acquire(); int primary = processTable[slot].primarySlot; if (primary < 0) primary = slot; // Gather this process's waiters on uaddr. At most MaxProcesses. int waiters[MaxProcesses]; int n = 0; for (int i = 0; i < MaxProcesses; i++) { Process& p = processTable[i]; if (p.state != ProcessState::Blocked) continue; if (p.futexAddr != uaddr) continue; if (p.primarySlot != primary) continue; waiters[n++] = i; } // Waking everyone needs no ordering; otherwise pick oldest-first. bool ordered = (uint32_t)n > count; int woken = 0; while (woken < n && (uint32_t)woken < count) { if (ordered) { int best = woken; for (int j = woken + 1; j < n; j++) { if (processTable[waiters[j]].futexSeq < processTable[waiters[best]].futexSeq) best = j; } int tmp = waiters[woken]; waiters[woken] = waiters[best]; waiters[best] = tmp; } Process& p = processTable[waiters[woken]]; p.futexAddr = 0; p.futexWoken = true; p.sleepUntilTick = 0; p.state = ProcessState::Ready; readyCount++; woken++; } schedLock.Release(); if (woken > 0) { // Re-read: this thread may have migrated since `cpu` was sampled. auto* here = Smp::GetCurrentCpuData(); KickOneIdleCpu(here ? here->cpuIndex : -1); } return woken; } bool IsAlive(int pid) { for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid == pid) { auto s = processTable[i].state; return s == ProcessState::Ready || s == ProcessState::Running || s == ProcessState::Blocked; } } return false; } Process* GetProcessByPid(int pid) { for (int i = 0; i < MaxProcesses; i++) { if (processTable[i].pid == pid) { auto s = processTable[i].state; if (s == ProcessState::Ready || s == ProcessState::Running || s == ProcessState::Blocked) { return &processTable[i]; } } } return nullptr; } // ==== Consent support ==== static Process* FindLiveProcessLocked(int pid) { if (pid < 0) return nullptr; for (int i = 0; i < MaxProcesses; i++) { Process& p = processTable[i]; if (p.primarySlot != i || p.pid != pid) continue; if (p.state == ProcessState::Ready || p.state == ProcessState::Running || p.state == ProcessState::Blocked) { return &p; } } return nullptr; } static bool ConsentPairValidLocked(const Process* subject, const Process* agent, uint64_t capability) { return subject != nullptr && agent != nullptr && subject != agent && agent->sessionId >= 0 && subject->sessionId == agent->sessionId && (agent->effectiveCaps & montauk::abi::CAP_COMPOSITOR) != 0 && (agent->delegableCaps & capability) == capability && (subject->consentCeiling & capability) == capability; } bool ConsentSubjectEligible(int pid, int agentPid, uint64_t capability) { schedLock.Acquire(); Process* subject = FindLiveProcessLocked(pid); bool ok = ConsentPairValidLocked(subject, FindLiveProcessLocked(agentPid), capability) && (subject->consentDenied & capability) == 0; schedLock.Release(); return ok; } bool ApplyConsentGrant(int pid, int agentPid, uint64_t capability) { schedLock.Acquire(); Process* subject = FindLiveProcessLocked(pid); bool ok = ConsentPairValidLocked(subject, FindLiveProcessLocked(agentPid), capability); if (ok) { // Usable, never delegable: a consented app cannot hand the // authority on to anything it launches. subject->effectiveCaps |= capability; } schedLock.Release(); return ok; } void ApplyConsentDenial(int pid, uint64_t capability) { schedLock.Acquire(); if (Process* subject = FindLiveProcessLocked(pid)) subject->consentDenied |= capability; schedLock.Release(); } bool SnapshotConsentSubject(int pid, montauk::abi::ConsentRequest& out) { schedLock.Acquire(); Process* subject = FindLiveProcessLocked(pid); if (subject != nullptr) { memcpy(out.imagePath, subject->imagePath, sizeof(out.imagePath)); out.imagePath[sizeof(out.imagePath) - 1] = '\0'; // Only an image that cannot be rewritten without CAP_SYSTEM_IMAGE // is identified by its path well enough to remember an answer. out.imageProtected = Fs::IsSystemImage(out.imagePath); } schedLock.Release(); return subject != nullptr; } Process* GetProcessSlot(int slot) { if (slot < 0 || slot >= MaxProcesses) return nullptr; return &processTable[slot]; } int SpawnCrashPad(int crasherPid) { // Spawn crashpad as a child of init (slot 0), not the crashing process. // The crasher is still alive here (we run before its ExitProcess), but // it is about to die, so we must not parent the crashpad to it. static constexpr const char* crashpadPath = "0:/apps/crashpad/crashpad.elf"; // If the crashing process had a redirected console (i.e. it was running // under terminal.elf in a console session), hand the crashpad a write // handle to that same out-stream so it can surface a textual crash // report to the console user when no desktop is running. Process* crasher = GetProcessByPid(crasherPid); // Crash-loop guard: never spawn a crashpad to report on the crashpad // itself. If crashpad.elf faults (startup bug, OOM, etc.) we would // otherwise spawn another crashpad, which can fault again, ad infinitum // -- a fork bomb that also exhausts process slots and report buffers. if (crasher && Lib::strcmp(crasher->name, crashpadPath) == 0) { Kt::KernelLogStream(Kt::ERROR, "Sched") << "crashpad faulted; not spawning a nested crashpad"; return -1; } int crasherSlot = Ipc::SlotForPid(crasherPid); bool inheritRedir = crasher && crasher->redirected && crasher->ioOutHandle >= 0 && crasherSlot >= 0; Ipc::HandleSnapshot outSnapshot; if (inheritRedir) { if (!outSnapshot.Capture(crasherSlot, crasher->ioOutHandle) || outSnapshot.type != Ipc::HandleType::Stream || outSnapshot.object == nullptr) { inheritRedir = false; } } // Deferred start when inheriting, so we can wire ioOutHandle before _start. int childPid = Spawn(crashpadPath, nullptr, /*startReady=*/!inheritRedir); if (childPid < 0 || !inheritRedir) return childPid; Process* child = GetProcessByPid(childPid); int childSlot = Ipc::SlotForPid(childPid); if (child && childSlot >= 0) { // Installing with RightWrite retains the stream (writerRefs++/refs++), // so it outlives the crasher's imminent ExitProcess teardown. The // snapshot above and this install run back-to-back with no blocking // call between, while the crasher's handle still pins the stream. int h = Ipc::InstallHandleForSlot(childSlot, outSnapshot.object, Ipc::HandleType::Stream, Ipc::RightWrite | Ipc::RightWait | Ipc::RightDup); if (h >= 0) { child->ioOutHandle = h; child->redirected = true; child->parentPid = crasherPid; child->termCols = crasher->termCols; child->termRows = crasher->termRows; } } // Always start it (even if wiring failed -> it just falls back to silent). StartProcess(childPid); return childPid; } } // Called by the syscall entry assembly after the syscall implementation has // released all of its subsystem locks but before returning to ring 3. extern "C" void SchedExitIfKilled() { Sched::ExitIfKilled(); }