fix: kernel concurrency, interrupt context, and user VA safety

kernel/src/Sched/Scheduler.cpp

@@ -11,6 +11,7 @@
 #include <Memory/HHDM.hpp>
 #include <Libraries/Memory.hpp>
 #include <Libraries/String.hpp>
+#include <Common/Panic.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <CppLib/Spinlock.hpp>
@@ -71,21 +72,105 @@ namespace Sched {
     // It is held ACROSS context switches to prevent the race where
     // another CPU picks up a process whose RSP hasn't been saved yet.
     // The resumed process releases it.
-    static kcp::Spinlock schedLock;
+    // schedLock is a baton: the context that acquires it switches stacks and
+    // the context being resumed releases it. A Spinlock's ordinary shared
+    // savedFlags field cannot represent that -- it would restore the switching
+    // context's IF state into the resumed stack. Keep the flags with each saved
+    // process/idle context instead, so timer frames resume with IF=0 and syscall
+    // frames resume with their original IF=1 even after CPU migration.
+    class SchedulerSpinlock {
+        kcp::Spinlock lock;
+        uint64_t processFlags[MaxProcesses] = {};
+        uint64_t idleFlags[Smp::MaxCPUs] = {};
+
+    public:
+        void Acquire() {
+            uint64_t flags = lock.AcquireIrqSave();
+            auto* cpu = Smp::GetCurrentCpuData();
+            if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
+                processFlags[cpu->currentSlot] = flags;
+            } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
+                idleFlags[cpu->cpuIndex] = flags;
+            }
+        }
+
+        void Release() {
+            uint64_t flags = 0; // Fresh startup remains IF=0 until JumpToUserMode's IRETQ.
+            auto* cpu = Smp::GetCurrentCpuData();
+            if (cpu != nullptr && cpu->currentSlot >= 0 && cpu->currentSlot < MaxProcesses) {
+                flags = processFlags[cpu->currentSlot];
+            } else if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) {
+                flags = idleFlags[cpu->cpuIndex];
+            }
+            lock.ReleaseIrqRestore(flags);
+        }
+
+        void ReleaseForStartup() {
+            lock.ReleaseIrqNoRestore();
+        }
+    };
+
+    static SchedulerSpinlock schedLock;
 
     // Approximate count of Ready processes. Incremented/decremented
     // under schedLock. Idle CPUs check this to avoid scanning all 256
     // process slots on every timer tick.
-    static volatile int readyCount = 0;
+    // Updated while schedLock is held, but sampled locklessly by idle CPUs and
+    // reschedule-IPI paths. Atomic loads make those wakeup decisions coherent
+    // across cores instead of relying on volatile's compiler-only semantics.
+    static std::atomic<int> readyCount{0};
+
+    // Per-object wake generations close the readiness-check/enrollment gap.
+    // Each bucket packs [63:32] = a tag identifying the last object to signal
+    // it and [31:0] = a signal counter, published together in one atomic word.
+    // Publishing both halves atomically lets a waiter distinguish "exactly one
+    // signal landed here and it carried someone else's tag" (safe to keep
+    // blocking) from "something ambiguous happened" (must treat as a wake).
+    // Without the tag, an unrelated hot object sharing the bucket forces the
+    // waiter to re-run its condition on every signal. Tag collisions only cost
+    // a spurious wake; they can never lose one, which is the only direction
+    // that would reintroduce the lost-wakeup bug this table exists to prevent.
+    static constexpr uint32_t ObjectWakeBuckets = 1024;
+    static std::atomic<uint64_t> objectWakeEpochs[ObjectWakeBuckets] = {};
+
+    static uint32_t ObjectWakeBucket(void* object) {
+        uint64_t value = (uint64_t)object;
+        return (uint32_t)(((value >> 4) ^ (value >> 12)) & (ObjectWakeBuckets - 1));
+    }
+
+    static uint32_t ObjectWakeTag(void* object) {
+        uint64_t value = (uint64_t)object;
+        value ^= value >> 32;
+        value *= 0x9E3779B97F4A7C15ULL;
+        return (uint32_t)(value >> 32);
+    }
 
     // The idle loop runs in the kernel PML4
     static uint64_t GetKernelCR3() {
         return (uint64_t)Memory::VMM::g_paging->PML4;
     }
 
-    static void RescheduleIpiHandler(uint8_t) {
+    static void RescheduleIpiHandler(uint8_t, bool interruptedUser) {
         auto* cpu = Smp::GetCurrentCpuData();
-        if (cpu == nullptr || cpu->currentSlot >= 0 || readyCount <= 0) {
+        if (cpu == nullptr) {
+            return;
+        }
+
+        // Process teardown uses this IPI to promptly stop sibling threads on
+        // other CPUs. A user-mode sibling can exit immediately. If the IPI
+        // interrupted a syscall, merely expire its slice: teardown on that
+        // suspended kernel stack could reacquire a mutex it already owns.
+        if (cpu->currentSlot >= 0 &&
+            processTable[cpu->currentSlot].killPending.load(std::memory_order_acquire)) {
+            if (interruptedUser) {
+                ExitProcess();
+                return;
+            }
+            processTable[cpu->currentSlot].sliceRemaining = 0;
+            return;
+        }
+
+        if (cpu->currentSlot >= 0 || readyCount <= 0 || cpu->reservedForKernelWork) {
             return;
         }
 
@@ -101,10 +186,10 @@ namespace Sched {
             if (target == nullptr || !target->started) return false;
             if (target->cpuIndex == sourceCpuIndex) return false;
             if (target->currentSlot >= 0) return false;
+            if (target->reservedForKernelWork) return false;
 
-            Hal::LocalApic::SendFixedIpi(target->lapicId,
+            return Hal::LocalApic::SendFixedIpi(target->lapicId,
                 Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
-            return true;
         };
 
         if (tryKick(Smp::GetCpuData(0))) {
@@ -163,7 +248,9 @@ namespace Sched {
     // The schedLock is held (acquired by the switching-from CPU's Schedule).
     static void ProcessStartup() {
         // Release the schedLock that the switching-from CPU held
-        schedLock.Release();
+        // without consulting this brand-new slot's (intentionally empty)
+        // saved interrupt state. JumpToUserMode's IRETQ enables interrupts.
+        schedLock.ReleaseForStartup();
 
         auto* cpu = Smp::GetCurrentCpuData();
         int slot = cpu->currentSlot;
@@ -526,10 +613,11 @@ namespace Sched {
         // Give the new thread its own TLS block when the image has one.
         // We are executing inside the process's address space, so the
         // freshly mapped pages and the template are directly addressable.
-        // The block is process-heap memory; it is reclaimed with the rest
-        // of the user half at process teardown (a thread that exits before
-        // its process leaks its block until then - acceptable for now).
+        // The block uses process-heap virtual space. Its physical pages are
+        // reclaimed when the sibling exits (or if creation fails).
         uint64_t threadFsBase = 0;
+        uint64_t threadTlsBase = 0;
+        uint64_t threadTlsPages = 0;
         Process& primary = processTable[primarySlot_];
         if (primary.tlsMemSize > 0) {
             uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign;
@@ -537,12 +625,15 @@ namespace Sched {
             uint64_t total = S + 16;
             uint64_t numPages = (total + 0xFFF) / 0x1000;
 
-            schedLock.Acquire();
-            uint64_t base = primary.heapNext;
-            primary.heapNext += numPages * 0x1000;
-            schedLock.Release();
+            uint64_t base = 0;
+            if (!ReserveUserHeapRange(primarySlot_, numPages * 0x1000, base)) {
+                Kt::KernelLogStream(Kt::ERROR, "Sched")
+                    << "Thread TLS exceeds user heap range";
+                return -1;
+            }
 
             bool ok = true;
+            uint64_t mappedPages = 0;
             for (uint64_t p = 0; p < numPages; p++) {
                 void* page = Memory::g_pfa->AllocateZeroed();
                 if (page == nullptr) {
@@ -552,11 +643,14 @@ namespace Sched {
                 uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
                 if (!Memory::VMM::Paging::MapUserIn(sharedPml4, physAddr,
                                                     base + p * 0x1000)) {
+                    Memory::g_pfa->Free(page);
                     ok = false;
                     break;
                 }
+                mappedPages++;
             }
             if (!ok) {
+                Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
                 Kt::KernelLogStream(Kt::ERROR, "Sched")
                     << "Thread TLS allocation failed";
                 return -1;
@@ -567,6 +661,8 @@ namespace Sched {
                    primary.tlsFileSize);
             *(uint64_t*)tp = tp;   /* ABI self-pointer */
             threadFsBase = tp;
+            threadTlsBase = base;
+            threadTlsPages = numPages;
         }
 
         // We do not write to the user stack from kernel mode (that would
@@ -578,7 +674,11 @@ namespace Sched {
 
         // Allocate kernel stack.
         void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
-        if (stackMem == nullptr) return -1;
+        if (stackMem == nullptr) {
+            if (threadTlsPages != 0)
+                Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
+            return -1;
+        }
         memset(stackMem, 0, StackSize);
         uint8_t* kernelStackBase = (uint8_t*)stackMem;
         uint64_t kernelStackTop = (uint64_t)kernelStackBase + StackSize;
@@ -598,6 +698,8 @@ namespace Sched {
         if (slot < 0) {
             schedLock.Release();
             Memory::g_pfa->Free(stackMem, StackPages);
+            if (threadTlsPages != 0)
+                Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
             return -1;
         }
 
@@ -663,6 +765,54 @@ namespace Sched {
         return tid;
     }
 
+    bool ReserveUserHeapRange(int primarySlot_, uint64_t size, uint64_t& outVa) {
+        outVa = 0;
+        if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
+            (size & 0xFFFULL) != 0) {
+            return false;
+        }
+
+        schedLock.Acquire();
+        Process& primary = processTable[primarySlot_];
+        uint64_t base = primary.heapNext;
+        bool validOwner = primary.primarySlot == primarySlot_ &&
+                          primary.state != ProcessState::Free &&
+                          primary.state != ProcessState::Terminated;
+        bool fits = validOwner && base >= UserHeapBase &&
+                    base <= UserHeapLimit && size <= UserHeapLimit - base;
+        if (fits) {
+            primary.heapNext = base + size;
+            outVa = base;
+        }
+        schedLock.Release();
+        return fits;
+    }
+
+    static void FreeSiblingThreadTls(Process& thr) {
+        int primarySlot_ = thr.primarySlot;
+        if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
+            primarySlot_ == (int)(&thr - processTable) || thr.fsBase == 0) {
+            return;
+        }
+
+        Process& primary = processTable[primarySlot_];
+        if (primary.tlsMemSize == 0 || primary.pml4Phys == 0) {
+            thr.fsBase = 0;
+            return;
+        }
+
+        uint64_t align = primary.tlsAlign < 16 ? 16 : primary.tlsAlign;
+        uint64_t blockSize = (primary.tlsMemSize + align - 1) & ~(align - 1);
+        if (blockSize > thr.fsBase || blockSize > UINT64_MAX - 16) {
+            Panic("Invalid sibling TLS metadata during thread teardown", nullptr);
+        }
+
+        uint64_t base = thr.fsBase - blockSize;
+        uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000;
+        thr.fsBase = 0;
+        Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
+    }
+
     // Switch away from a slot we have just marked non-runnable while holding
     // schedLock. Mirrors SwitchAwayFromBlockedCurrentLocked but does NOT
     // assume the caller will be resumed -- used by ExitCurrentThread.
@@ -714,10 +864,20 @@ namespace Sched {
             __builtin_unreachable();
         }
 
-        thr.killPending = false;
+        bool processExitKill =
+            thr.killPending.exchange(false, std::memory_order_acq_rel);
         thr.startPending = false;
         thr.exitCode = exitCode;
 
+        // TLS belongs only to this sibling. Reclaim it before publishing the
+        // Terminated state; otherwise a long-lived multithreaded GUI process
+        // leaks these pages until the entire process exits. When the primary
+        // is already coordinating full teardown, defer this to its final
+        // sibling sweep: the primary may be waiting from an IF=0 timer frame
+        // and therefore cannot acknowledge our TLB-shootdown IPI here.
+        if (!processExitKill)
+            FreeSiblingThreadTls(thr);
+
         schedLock.Acquire();
         thr.state = ProcessState::Terminated;
         thr.runningOnCpu = -1;
@@ -847,10 +1007,16 @@ namespace Sched {
     // New processes release it in ProcessStartup().
     // ====================================================================
 
-    // Reclaim terminated process slots. Called from BSP's Tick only,
-    // NOT from every Schedule() call on every CPU. This avoids holding
-    // schedLock (with interrupts disabled) during PFA::Free on the
-    // hot scheduling path.
+    // BSP maintenance that may allocate/free memory or touch policy MSRs must
+    // run from the idle kernel context, never from the timer interrupt. The
+    // pending flag is set by process teardown on any CPU and periodically by
+    // the BSP timer. It remains set until an idle maintenance pass consumes it.
+    static std::atomic<bool> bspMaintenancePending{false};
+    static uint64_t nextBspPolicyTick = 0; // BSP timer only
+
+    // Reclaim terminated process slots from BSP idle context. This avoids
+    // holding schedLock (with interrupts disabled) during PFA::Free on the
+    // hot scheduling path or inside a hard interrupt.
     static void ReclaimTerminated() {
         schedLock.Acquire();
         for (int i = 0; i < MaxProcesses; i++) {
@@ -880,7 +1046,17 @@ namespace Sched {
         return readyCount > 0;
     }
 
-    void RunBspMaintenance() {
+    void RequestKernelWork() {
+        auto* cpu = Smp::GetCurrentCpuData();
+        if (cpu != nullptr && cpu->currentSlot >= 0) {
+            cpu->kernelWorkPending = true;
+        }
+    }
+
+    // Bounded, IRQ-safe portion of BSP maintenance. Keeping deadline expiry
+    // on every 1 ms BSP tick prevents timed input/object waits from depending
+    // on the BSP becoming idle under sustained desktop load.
+    static void WakeExpiredProcesses() {
         bool wokeProcesses = false;
 
         schedLock.Acquire();
@@ -902,12 +1078,19 @@ namespace Sched {
         if (wokeProcesses) {
             KickOneIdleCpu(0);
         }
+    }
+
+    void RunBspMaintenance() {
+        // Clear before scanning. A producer that races this pass sets the flag
+        // again and therefore cannot have its request acknowledged by mistake.
+        if (!bspMaintenancePending.exchange(false, std::memory_order_acq_rel)) {
+            return;
+        }
 
         ReclaimTerminated();
 
         // Thermal governor step (rate-limited internally to 500 ms).
-        // Runs here because BSP maintenance is reached from both the idle
-        // loop and the timer tick, so it keeps running under full load.
+        // The timer requests this idle-context pass even under full load.
         Hal::CpuPower::ThermalTick(Timekeeping::GetMilliseconds());
     }
 
@@ -933,21 +1116,41 @@ namespace Sched {
 
         schedLock.Acquire();
 
-        // Find the next Ready process (round-robin from after current slot)
+        // Normally select the next Ready process. A pending bottom half gets
+        // one deliberate trip through the idle kernel context first; that
+        // context drains bounded work and immediately schedules user work
+        // again. This avoids both hard-IRQ callbacks and bottom-half starvation
+        // when there is always at least one runnable desktop process.
         int next = -1;
+        bool yieldToKernel = cpu->currentSlot >= 0 && cpu->kernelWorkPending;
+        if (yieldToKernel) cpu->kernelWorkPending = false;
         int start = (cpu->currentSlot >= 0) ? cpu->currentSlot + 1 : 0;
 
-        for (int i = 0; i < MaxProcesses; i++) {
-            int idx = (start + i) % MaxProcesses;
-            if (processTable[idx].state == ProcessState::Ready) {
-                next = idx;
-                break;
+        if (!yieldToKernel) {
+            for (int i = 0; i < MaxProcesses; i++) {
+                int idx = (start + i) % MaxProcesses;
+                if (processTable[idx].state == ProcessState::Ready) {
+                    next = idx;
+                    break;
+                }
             }
         }
 
         if (next < 0) {
-            // No ready processes. If we were running one, return to idle.
+            // No other process is ready. A normal time-slice expiry must keep
+            // the sole Running process on this CPU: bouncing it through idle
+            // can suspend a syscall while it owns a process-context Mutex, then
+            // let ServiceDeferredWork spin on that same mutex with the idle
+            // context reserved against scheduling. That is a permanent
+            // self-deadlock. Only an explicit, ring-3-safe bottom-half request
+            // is allowed to yield a running process to idle.
             if (cpu->currentSlot >= 0) {
+                if (!yieldToKernel) {
+                    processTable[cpu->currentSlot].sliceRemaining = TimeSliceMs;
+                    schedLock.Release();
+                    return;
+                }
+
                 int oldSlot = cpu->currentSlot;
                 processTable[oldSlot].state = ProcessState::Ready;
                 readyCount++;
@@ -1019,7 +1222,7 @@ namespace Sched {
         schedLock.Release();
     }
 
-    void Tick(uint32_t elapsedMs) {
+    void Tick(uint32_t elapsedMs, bool interruptedUser) {
         auto* cpu = Smp::GetCurrentCpuData();
 
         // Pick up thermal-governor frequency changes (HWP requests are
@@ -1027,9 +1230,21 @@ namespace Sched {
         // the governor bumped the policy epoch since our last tick.
         Hal::CpuPower::ApplyPolicyIfChanged();
 
-        // BSP: wake sleeping processes and reclaim terminated slots
+        // BSP: expire timed waits in the bounded IRQ-safe pass. Queue heavier
+        // reclamation/policy work for the idle kernel context.
         if (cpu->cpuIndex == 0) {
-            RunBspMaintenance();
+            WakeExpiredProcesses();
+
+            uint64_t now = Timekeeping::GetTicks();
+            if (nextBspPolicyTick == 0 || now >= nextBspPolicyTick) {
+                nextBspPolicyTick = now + 500;
+                bspMaintenancePending.store(true, std::memory_order_release);
+            }
+
+            if (interruptedUser &&
+                bspMaintenancePending.load(std::memory_order_acquire)) {
+                RequestKernelWork();
+            }
         }
 
         int slot = cpu->currentSlot;
@@ -1052,12 +1267,21 @@ namespace Sched {
 
         processTable[slot].cpuTimeMs += elapsedMs;
 
-        // Check if another CPU requested this process be killed.
-        // We are on the CPU running it, so ExitProcess is safe here.
+        if (cpu->kernelWorkPending) {
+            processTable[slot].sliceRemaining = 0;
+        }
+
+        // Check if another CPU requested this process be killed. Full teardown
+        // takes process-context mutexes. It is safe here only when the timer
+        // interrupted ring 3; if it interrupted a syscall, teardown could try
+        // to reacquire a lock already held by this same suspended kernel stack.
+        // In that case force a reschedule and finish at syscall-return instead.
         if (processTable[slot].killPending) {
-            processTable[slot].killPending = false;
-            ExitProcess();
-            return;
+            if (interruptedUser) {
+                ExitProcess();
+                return;
+            }
+            processTable[slot].sliceRemaining = 0;
         }
 
         if (processTable[slot].sliceRemaining > elapsedMs) {
@@ -1066,7 +1290,13 @@ namespace Sched {
             processTable[slot].sliceRemaining = 0;
         }
 
-        if (processTable[slot].sliceRemaining == 0) {
+        // Never context-switch away from an arbitrary kernel stack. Syscalls
+        // routinely own non-IRQ-disabling subsystem mutexes; suspending that
+        // stack and dispatching a process that wants the same mutex leaves the
+        // new process spinning in ring 0, where the timer deliberately cannot
+        // preempt it to resume the owner. Preserve the expired slice and take
+        // the switch on the first timer interrupt after SYSRET reaches ring 3.
+        if (processTable[slot].sliceRemaining == 0 && interruptedUser) {
             Schedule();
         }
     }
@@ -1080,6 +1310,14 @@ namespace Sched {
         return processTable[primary].pid;
     }
 
+    void ExitIfKilled() {
+        auto* cpu = Smp::GetCurrentCpuData();
+        int slot = cpu ? cpu->currentSlot : -1;
+        if (slot >= 0 && processTable[slot].killPending) {
+            ExitProcess();
+        }
+    }
+
     int GetCurrentTid() {
         auto* cpu = Smp::GetCurrentCpuData();
         int slot = cpu->currentSlot;
@@ -1173,8 +1411,17 @@ namespace Sched {
         // terminated immediately; siblings running on other CPUs are tagged
         // killPending and we spin until their tick handler routes them
         // through ExitCurrentThread.
+        // Several device syscalls have bounded waits up to five seconds. Give
+        // an interrupted kernel-mode sibling room to finish that recovery
+        // path before declaring it wedged.
+        static constexpr uint64_t SiblingDrainTimeoutMs = 10000;
+        static constexpr uint32_t SiblingDrainMaxPasses = 262144;
+        uint64_t siblingDrainDeadline = Timekeeping::GetTicks() + SiblingDrainTimeoutMs;
+        uint32_t siblingDrainPasses = 0;
+
         for (;;) {
             bool anyRunning = false;
+            bool targetCpus[Smp::MaxCPUs] = {};
             schedLock.Acquire();
             for (int i = 0; i < MaxProcesses; i++) {
                 if (i == primarySlot_) continue;
@@ -1184,7 +1431,10 @@ namespace Sched {
                     continue;
                 }
                 if (st == ProcessState::Running) {
-                    processTable[i].killPending = true;
+                    processTable[i].killPending.store(true, std::memory_order_release);
+                    int targetCpu = processTable[i].runningOnCpu;
+                    if (targetCpu >= 0 && targetCpu < Smp::MaxCPUs)
+                        targetCpus[targetCpu] = true;
                     anyRunning = true;
                     continue;
                 }
@@ -1211,6 +1461,30 @@ namespace Sched {
             }
             schedLock.Release();
             if (!anyRunning) break;
+
+            // Do not depend solely on a remote CPU's next periodic timer.
+            // Laptop firmware/IRQ pathologies can delay that tick, and the
+            // old unbounded wait then trapped the exiting CPU forever.
+            if ((siblingDrainPasses & 0x3FF) == 0) {
+                for (int i = 0; i < Smp::GetCpuCount(); i++) {
+                    if (!targetCpus[i] || i == cpu->cpuIndex) continue;
+                    Smp::CpuData* target = Smp::GetCpuData(i);
+                    if (target != nullptr && target->started) {
+                        (void)Hal::LocalApic::SendFixedIpi(target->lapicId,
+                            Hal::IRQ_VECTOR_BASE + Hal::IRQ_RESCHEDULE);
+                    }
+                }
+            }
+
+            siblingDrainPasses++;
+            if (siblingDrainPasses >= SiblingDrainMaxPasses ||
+                Timekeeping::GetTicks() >= siblingDrainDeadline) {
+                // It is unsafe to free the shared address space while another
+                // CPU may still execute in it. A visible panic is preferable
+                // to silent lockup or use-after-free corruption.
+                Panic("Timed out draining sibling threads during process exit", nullptr);
+            }
+
             // Wait for the sibling CPU(s) to observe killPending on their
             // next timer tick. Briefly busy-wait; this path is rare.
             for (int spin = 0; spin < 1024; spin++) {
@@ -1224,6 +1498,10 @@ namespace Sched {
             if (i == primarySlot_) continue;
             if (processTable[i].primarySlot != primarySlot_) continue;
             if (processTable[i].state != ProcessState::Terminated) continue;
+            // Ready/blocked siblings terminated inline above never ran their
+            // ordinary ExitCurrentThread cleanup. Running siblings normally
+            // arrive here with fsBase already cleared, making this a no-op.
+            FreeSiblingThreadTls(processTable[i]);
             void* stackBase = (void*)processTable[i].stackBase;
             processTable[i].stackBase = 0;
             processTable[i].pml4Phys = 0;
@@ -1280,6 +1558,7 @@ namespace Sched {
         proc.state = ProcessState::Terminated;
         proc.runningOnCpu = -1;
         proc.reapReady = true;
+        bspMaintenancePending.store(true, std::memory_order_release);
 
         // Publish the exit code before waking waiters so SYS_WAITPID
         // observes it the moment it unblocks.
@@ -1429,6 +1708,8 @@ namespace Sched {
     }
 
     void BlockOnPid(int pid) {
+        ExitIfKilled();
+
         // If the target is already dead, return immediately
         if (!IsAlive(pid)) return;
 
@@ -1464,11 +1745,14 @@ namespace Sched {
         processTable[slot].sleepUntilTick = 0;
         processTable[slot].runningOnCpu = -1;
         SwitchAwayFromBlockedCurrentLocked();
+        ExitIfKilled();
     }
 
     void BlockForSleep(uint64_t ms) {
         if (ms == 0) return;
 
+        ExitIfKilled();
+
         auto* cpu = Smp::GetCurrentCpuData();
         int slot = cpu->currentSlot;
         if (slot < 0) return;
@@ -1481,11 +1765,14 @@ namespace Sched {
         processTable[slot].sleepUntilTick = Timekeeping::GetTicks() + ms;
         processTable[slot].runningOnCpu = -1;
         SwitchAwayFromBlockedCurrentLocked();
+        ExitIfKilled();
     }
 
     void BlockOnObject(void* object, uint64_t timeoutMs) {
         if (object == nullptr) return;
 
+        ExitIfKilled();
+
         auto* cpu = Smp::GetCurrentCpuData();
         int slot = cpu->currentSlot;
         if (slot < 0) return;
@@ -1499,12 +1786,15 @@ namespace Sched {
             : 0;
         processTable[slot].runningOnCpu = -1;
         SwitchAwayFromBlockedCurrentLocked();
+        ExitIfKilled();
     }
 
     bool BlockOnObjectIf(void* object, uint64_t timeoutMs,
                          bool (*shouldBlock)(void*), void* context) {
         if (object == nullptr || shouldBlock == nullptr) return false;
 
+        ExitIfKilled();
+
         auto* cpu = Smp::GetCurrentCpuData();
         if (cpu == nullptr) return false;
 
@@ -1526,12 +1816,58 @@ namespace Sched {
             : 0;
         processTable[slot].runningOnCpu = -1;
         SwitchAwayFromBlockedCurrentLocked();
+        ExitIfKilled();
         return true;
     }
 
+    uint64_t ObserveObjectWake(void* object) {
+        if (object == nullptr) return 0;
+        return objectWakeEpochs[ObjectWakeBucket(object)].load(std::memory_order_acquire);
+    }
+
+    struct ObjectWakeObservation {
+        void* object;
+        uint64_t epoch;
+    };
+
+    static bool ObjectWakeIsUnchanged(void* opaque) {
+        auto* observation = (ObjectWakeObservation*)opaque;
+        uint64_t now = ObserveObjectWake(observation->object);
+        uint64_t was = observation->epoch;
+        if (now == was) return true;   // nothing signalled this bucket at all
+
+        // Exactly one signal landed here since the observation, and it carried
+        // a different object's tag, so it cannot have been ours: the caller may
+        // still block. Any larger or wrapped delta is ambiguous -- more than
+        // one object may have signalled -- and must be reported as a wake so a
+        // real edge is never dropped.
+        if ((uint32_t)now == (uint32_t)was + 1 &&
+            (uint32_t)(now >> 32) != ObjectWakeTag(observation->object)) {
+            return true;
+        }
+        return false;
+    }
+
+    bool BlockOnObjectSince(void* object, uint64_t timeoutMs, uint64_t observedWake) {
+        ObjectWakeObservation observation{object, observedWake};
+        return BlockOnObjectIf(object, timeoutMs, ObjectWakeIsUnchanged, &observation);
+    }
+
     void WakeObjectWaiters(void* object) {
         if (object == nullptr) return;
 
+        // Publish this object's tag and the bumped counter as one word. A CAS
+        // loop (rather than fetch_add) keeps the pair consistent and, unlike a
+        // plain store, cannot drop a concurrent waker's increment on the same
+        // bucket -- a dropped increment would be an unobserved wake edge.
+        auto& bucket = objectWakeEpochs[ObjectWakeBucket(object)];
+        uint64_t tagBits = (uint64_t)ObjectWakeTag(object) << 32;
+        uint64_t observed = bucket.load(std::memory_order_relaxed);
+        while (!bucket.compare_exchange_weak(
+                   observed, tagBits | (uint32_t)((uint32_t)observed + 1),
+                   std::memory_order_release, std::memory_order_relaxed)) {
+        }
+
         bool wokeAny = false;
         schedLock.Acquire();
         for (int i = 0; i < MaxProcesses; i++) {
@@ -1608,13 +1944,10 @@ namespace Sched {
         bool     inheritRedir = crasher && crasher->redirected &&
                                 crasher->ioOutHandle >= 0 && crasherSlot >= 0;
 
-        Ipc::Object*    outObj = nullptr;
-        Ipc::HandleType outType = Ipc::HandleType::None;
-        uint32_t        outRights = 0;
+        Ipc::HandleSnapshot outSnapshot;
         if (inheritRedir) {
-            if (!Ipc::SnapshotHandleForSlot(crasherSlot, crasher->ioOutHandle,
-                                            outType, outObj, outRights) ||
-                outType != Ipc::HandleType::Stream || outObj == nullptr) {
+            if (!outSnapshot.Capture(crasherSlot, crasher->ioOutHandle) ||
+                outSnapshot.type != Ipc::HandleType::Stream || outSnapshot.object == nullptr) {
                 inheritRedir = false;
             }
         }
@@ -1630,7 +1963,7 @@ namespace Sched {
             // so it outlives the crasher's imminent ExitProcess teardown. The
             // snapshot above and this install run back-to-back with no blocking
             // call between, while the crasher's handle still pins the stream.
-            int h = Ipc::InstallHandleForSlot(childSlot, outObj, Ipc::HandleType::Stream,
+            int h = Ipc::InstallHandleForSlot(childSlot, outSnapshot.object, Ipc::HandleType::Stream,
                                               Ipc::RightWrite | Ipc::RightWait | Ipc::RightDup);
             if (h >= 0) {
                 child->ioOutHandle = h;
@@ -1647,3 +1980,9 @@ namespace Sched {
     }
 
 }
+
+// Called by the syscall entry assembly after the syscall implementation has
+// released all of its subsystem locks but before returning to ring 3.
+extern "C" void SchedExitIfKilled() {
+    Sched::ExitIfKilled();
+}