feat: overhaul userspace heap and virtual memory

kernel/src/Api/BuildNo.hpp

@@ -12,4 +12,4 @@
 
 #pragma once
 
-#define MONTAUK_BUILD_NUMBER 95
+#define MONTAUK_BUILD_NUMBER 96

kernel/src/Api/Heap.hpp

@@ -4,6 +4,7 @@
     * Copyright (c) 2026 Daniel Hammer
 */
 
+#pragma once
 #include <cstdint>
 #include <Sched/Scheduler.hpp>
 #include <Memory/Paging.hpp>
@@ -18,12 +19,16 @@ namespace montauk::abi {
     struct HeapAlloc {
         uint64_t va;
         uint64_t numPages;
+        uint64_t prot;
+        uint64_t allocationId;
+        HeapAlloc* next;
     };
 
-    static constexpr int MaxHeapAllocs = 512;
-
-    inline HeapAlloc g_heapAllocs[Sched::MaxProcesses][MaxHeapAllocs] = {};
-    inline int g_heapAllocCount[Sched::MaxProcesses] = {};
+    // VM area metadata is dynamically sized. The previous 512-entry array made
+    // valid mappings fail for metadata exhaustion and consumed several MiB of
+    // kernel BSS even for processes with no mappings.
+    inline HeapAlloc* g_heapAllocs[Sched::MaxProcesses] = {};
+    inline uint64_t g_nextHeapAllocationId[Sched::MaxProcesses] = {};
     inline kcp::Mutex g_heapLocks[Sched::MaxProcesses];
 
     // Get the process table slot index for the current process
@@ -35,12 +40,22 @@ namespace montauk::abi {
         return (int)(proc - slot0);
     }
 
-    inline uint64_t Sys_Alloc(uint64_t size) {
+    static constexpr uint64_t VmProtRead  = 1;
+    static constexpr uint64_t VmProtWrite = 2;
+    static constexpr uint64_t VmProtExec  = 4;
+
+    inline uint64_t Sys_MapAnonymous(uint64_t size, uint64_t prot) {
         auto* proc = Sched::GetCurrentProcessPtr();
         if (proc == nullptr) return 0;
         int slot = GetCurrentSlot();
         if (slot < 0) return 0;
 
+        if ((prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
+            (prot & VmProtRead) == 0 ||
+            ((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) {
+            return 0;
+        }
+
         // Guard against overflow before rounding.
         if (size > 0xFFFFFFFFFFFF0000ULL) return 0;
 
@@ -50,17 +65,6 @@ namespace montauk::abi {
 
         uint64_t numPages = size / 0x1000;
         g_heapLocks[slot].Acquire();
-        if (g_heapAllocCount[slot] >= MaxHeapAllocs) {
-            // Out of allocation records, not out of memory. Log it: a
-            // silent 0 here surfaced as bogus downstream errors (BFD
-            // turned NULL mallocs into "file format not recognized").
-            Kt::KernelLogStream(Kt::ERROR, "Heap")
-                << "pid " << proc->pid << " (" << proc->name
-                << ") hit MaxHeapAllocs (" << (uint64_t)MaxHeapAllocs
-                << "), SYS_ALLOC refused";
-            g_heapLocks[slot].Release();
-            return 0;
-        }
 
         uint64_t userVa = 0;
         if (!Sched::ReserveUserHeapRange(slot, size, userVa)) {
@@ -71,39 +75,34 @@ namespace montauk::abi {
             return 0;
         }
 
-        // Allocate physical pages and map them into the process
-        uint64_t mappedPages = 0;
-        for (uint64_t i = 0; i < numPages; i++) {
-            void* page = Memory::g_pfa->AllocateZeroed();
-            if (page == nullptr) {
-                Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
-                g_heapLocks[slot].Release();
-                return 0;
-            }
-            uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
-            if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, userVa + i * 0x1000)) {
-                Memory::g_pfa->Free(page);
-                Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
-                g_heapLocks[slot].Release();
-                return 0;
-            }
-            mappedPages++;
-        }
-
-        // Track the allocation so Sys_Free can release it
-        Sched::g_allocatedPages[slot] += numPages;
-        g_heapAllocs[slot][g_heapAllocCount[slot]++] = { userVa, numPages };
+        // Reserve a VMA without committing physical pages. First access is
+        // satisfied by TryHandleAnonymousPageFault with a zeroed frame.
+        uint64_t allocationId = ++g_nextHeapAllocationId[slot];
+        if (allocationId == 0) allocationId = ++g_nextHeapAllocationId[slot];
+        g_heapAllocs[slot] = new HeapAlloc { userVa, numPages, prot, allocationId,
+                                             g_heapAllocs[slot] };
 
         g_heapLocks[slot].Release();
         return userVa;
     }
 
+    inline uint64_t Sys_Alloc(uint64_t size) {
+        return Sys_MapAnonymous(size, VmProtRead | VmProtWrite);
+    }
+
     // Reset heap allocation tracking for a process slot.
     // The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup.
     inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) {
         if (slot < 0 || slot >= Sched::MaxProcesses) return;
         g_heapLocks[slot].Acquire();
-        g_heapAllocCount[slot] = 0;
+        HeapAlloc* area = g_heapAllocs[slot];
+        while (area != nullptr) {
+            HeapAlloc* next = area->next;
+            delete area;
+            area = next;
+        }
+        g_heapAllocs[slot] = nullptr;
+        g_nextHeapAllocationId[slot] = 0;
         Sched::g_allocatedPages[slot] = 0;
         g_heapLocks[slot].Release();
     }
@@ -117,31 +116,245 @@ namespace montauk::abi {
 
         g_heapLocks[slot].Acquire();
 
-        // Find the allocation record matching this address
-        int idx = -1;
-        for (int i = 0; i < g_heapAllocCount[slot]; i++) {
-            if (g_heapAllocs[slot][i].va == addr) {
-                idx = i;
+        // A protection change can split one allocation into several VMA
+        // records. Resolve the stable allocation ID from the supplied base,
+        // detach every surviving fragment, then tear it down without holding
+        // the VMA lock.
+        HeapAlloc* released = nullptr;
+        HeapAlloc** link = &g_heapAllocs[slot];
+        uint64_t resident = 0;
+        uint64_t allocationId = 0;
+        for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
+             area = area->next) {
+            if (area->va == addr) {
+                allocationId = area->allocationId;
                 break;
             }
         }
-        if (idx < 0) {
+        if (allocationId == 0) {
             g_heapLocks[slot].Release();
             return;  // Unknown address — ignore
         }
+        while (*link != nullptr) {
+            HeapAlloc* area = *link;
+            if (area->allocationId != allocationId) {
+                link = &area->next;
+                continue;
+            }
+            for (uint64_t i = 0; i < area->numPages; i++)
+                if (Memory::VMM::Paging::GetPhysAddr(
+                        proc->pml4Phys, area->va + i * 0x1000ULL) != 0)
+                    resident++;
+            *link = area->next;
+            area->next = released;
+            released = area;
+        }
+        g_heapLocks[slot].Release();
 
-        uint64_t va = g_heapAllocs[slot][idx].va;
-        uint64_t numPages = g_heapAllocs[slot][idx].numPages;
+        // Unmap and invalidate sibling CPUs before recycling frames. A stale
+        // user TLB entry can otherwise corrupt the frame's next owner.
+        while (released != nullptr) {
+            HeapAlloc* next = released->next;
+            Ipc::UnmapAndFreeUserRange(proc->pml4Phys, released->va,
+                                       released->numPages);
+            Sched::ReleaseUserHeapRange(slot, released->va,
+                                        released->numPages * 0x1000ULL);
+            delete released;
+            released = next;
+        }
 
-        // Unmap and invalidate sibling CPUs before recycling the frames. A
-        // stale user TLB entry can otherwise corrupt the frame's next owner.
-        Ipc::UnmapAndFreeUserRange(proc->pml4Phys, va, numPages);
-
-        Sched::g_allocatedPages[slot] -= numPages;
-
-        // Remove tracking entry by swapping with the last element
-        g_heapAllocs[slot][idx] = g_heapAllocs[slot][g_heapAllocCount[slot] - 1];
-        g_heapAllocCount[slot]--;
+        g_heapLocks[slot].Acquire();
+        Sched::g_allocatedPages[slot] -= resident;
         g_heapLocks[slot].Release();
     }
+
+    inline int64_t Sys_Unmap(uint64_t addr, uint64_t size) {
+        auto* proc = Sched::GetCurrentProcessPtr();
+        int slot = GetCurrentSlot();
+        if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
+            size > UINT64_MAX - 0xFFFULL) return -1;
+        size = (size + 0xFFFULL) & ~0xFFFULL;
+        if (addr > UINT64_MAX - size) return -1;
+
+        g_heapLocks[slot].Acquire();
+        uint64_t firstPage = 0;
+        uint64_t pages = size / 0x1000ULL;
+        HeapAlloc** link = &g_heapAllocs[slot];
+        while (*link != nullptr) {
+            HeapAlloc* area = *link;
+            uint64_t areaSize = area->numPages * 0x1000ULL;
+            if (addr >= area->va && addr - area->va <= areaSize &&
+                size <= areaSize - (addr - area->va)) {
+                firstPage = (addr - area->va) / 0x1000ULL;
+                break;
+            }
+            link = &area->next;
+        }
+        if (*link == nullptr) { g_heapLocks[slot].Release(); return -1; }
+
+        HeapAlloc* area = *link;
+        uint64_t oldPages = area->numPages;
+        uint64_t oldProt = area->prot;
+        uint64_t tailPages = oldPages - firstPage - pages;
+
+        if (firstPage == 0 && tailPages == 0) {
+            *link = area->next;
+            delete area;
+        } else if (firstPage == 0) {
+            area->va = addr + size;
+            area->numPages = tailPages;
+        } else {
+            area->numPages = firstPage;
+            if (tailPages != 0) {
+                area->next = new HeapAlloc { addr + size, tailPages, oldProt,
+                                             area->allocationId,
+                                             area->next };
+            }
+        }
+
+        uint64_t resident = 0;
+        for (uint64_t i = 0; i < pages; i++)
+            if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, addr + i * 0x1000ULL) != 0)
+                resident++;
+        g_heapLocks[slot].Release();
+
+        Ipc::UnmapAndFreeUserRange(proc->pml4Phys, addr, pages);
+        Sched::ReleaseUserHeapRange(slot, addr, size);
+        g_heapLocks[slot].Acquire();
+        Sched::g_allocatedPages[slot] -= resident;
+        g_heapLocks[slot].Release();
+        return 0;
+    }
+
+    inline int64_t Sys_Protect(uint64_t addr, uint64_t size, uint64_t prot) {
+        auto* proc = Sched::GetCurrentProcessPtr();
+        int slot = GetCurrentSlot();
+        if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
+            size > UINT64_MAX - 0xFFFULL ||
+            (prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
+            (prot & VmProtRead) == 0 ||
+            ((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) return -1;
+        size = (size + 0xFFFULL) & ~0xFFFULL;
+        if (addr > UINT64_MAX - size) return -1;
+
+        g_heapLocks[slot].Acquire();
+        uint64_t pages = size / 0x1000ULL;
+
+        // Protection changes may span adjacent VMA fragments created by a
+        // previous mprotect. Validate the complete range before changing any
+        // metadata so failure is atomic from userspace's perspective.
+        for (uint64_t page = 0; page < pages; page++) {
+            uint64_t pageVa = addr + page * 0x1000ULL;
+            bool covered = false;
+            for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
+                 area = area->next) {
+                uint64_t areaSize = area->numPages * 0x1000ULL;
+                if (pageVa >= area->va && pageVa - area->va < areaSize) {
+                    covered = true;
+                    break;
+                }
+            }
+            if (!covered) { g_heapLocks[slot].Release(); return -1; }
+        }
+
+        uint64_t end = addr + size;
+        for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;) {
+            HeapAlloc* oldNext = area->next;
+            uint64_t areaEnd = area->va + area->numPages * 0x1000ULL;
+            uint64_t overlapStart = area->va > addr ? area->va : addr;
+            uint64_t overlapEnd = areaEnd < end ? areaEnd : end;
+            if (overlapStart < overlapEnd) {
+                uint64_t oldVa = area->va;
+                uint64_t oldProt = area->prot;
+                uint64_t prefixPages = (overlapStart - oldVa) / 0x1000ULL;
+                uint64_t protectedPages = (overlapEnd - overlapStart) / 0x1000ULL;
+                uint64_t tailPages = (areaEnd - overlapEnd) / 0x1000ULL;
+
+                if (prefixPages == 0) {
+                    area->numPages = protectedPages;
+                    area->prot = prot;
+                    if (tailPages != 0)
+                        area->next = new HeapAlloc { overlapEnd, tailPages,
+                                                     oldProt,
+                                                     area->allocationId,
+                                                     oldNext };
+                } else {
+                    area->numPages = prefixPages;
+                    auto* middle = new HeapAlloc { overlapStart, protectedPages,
+                                                   prot, area->allocationId,
+                                                   oldNext };
+                    area->next = middle;
+                    if (tailPages != 0)
+                        middle->next = new HeapAlloc { overlapEnd, tailPages,
+                                                       oldProt,
+                                                       area->allocationId,
+                                                       oldNext };
+                }
+            }
+            area = oldNext;
+        }
+
+        bool writable = (prot & VmProtWrite) != 0;
+        bool executable = (prot & VmProtExec) != 0;
+        for (uint64_t i = 0; i < pages; i++) {
+            if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys,
+                                                 addr + i * 0x1000ULL) == 0)
+                continue;
+            if (!Memory::VMM::Paging::ProtectUserIn(proc->pml4Phys,
+                                                     addr + i * 0x1000ULL,
+                                                     writable, executable)) {
+                g_heapLocks[slot].Release();
+                return -1;
+            }
+        }
+        g_heapLocks[slot].Release();
+        Ipc::ShootdownUserRange(proc->pml4Phys, addr, (uint32_t)pages);
+        return 0;
+    }
+
+    // Resolve a non-present access inside an anonymous VMA. This is used for
+    // both ring-3 faults and kernel copies to untouched userspace buffers.
+    inline bool TryHandleAnonymousPageFault(uint64_t faultAddr, uint64_t errorCode) {
+        auto* proc = Sched::GetCurrentProcessPtr();
+        int slot = GetCurrentSlot();
+        if (proc == nullptr || slot < 0 || faultAddr < Sched::UserHeapBase ||
+            faultAddr >= Sched::UserHeapLimit || (errorCode & 1) != 0) return false;
+
+        uint64_t pageVa = faultAddr & ~0xFFFULL;
+        g_heapLocks[slot].Acquire();
+        HeapAlloc* found = nullptr;
+        for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr; area = area->next) {
+            uint64_t bytes = area->numPages * 0x1000ULL;
+            if (pageVa >= area->va && pageVa - area->va < bytes) {
+                found = area;
+                break;
+            }
+        }
+        if (found == nullptr ||
+            ((errorCode & (1ULL << 1)) != 0 && (found->prot & VmProtWrite) == 0) ||
+            ((errorCode & (1ULL << 4)) != 0 && (found->prot & VmProtExec) == 0)) {
+            g_heapLocks[slot].Release();
+            return false;
+        }
+
+        // A sibling may have resolved the same fault while this CPU waited.
+        if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa) != 0) {
+            g_heapLocks[slot].Release();
+            return true;
+        }
+
+        void* page = Memory::g_pfa->AllocateZeroed();
+        if (page == nullptr) {
+            g_heapLocks[slot].Release();
+            return false;
+        }
+        uint64_t phys = Memory::SubHHDM((uint64_t)page);
+        bool mapped = Memory::VMM::Paging::MapUserInPermissions(
+            proc->pml4Phys, phys, pageVa,
+            (found->prot & VmProtWrite) != 0, (found->prot & VmProtExec) != 0);
+        if (!mapped) Memory::g_pfa->Free(page);
+        else Sched::g_allocatedPages[slot]++;
+        g_heapLocks[slot].Release();
+        return mapped;
+    }
 };

kernel/src/Api/Syscall.cpp

@@ -118,6 +118,12 @@ namespace montauk::abi {
             case SYS_FREE:
                 Sys_Free(frame->arg1);
                 return 0;
+            case SYS_MMAP_ANON:
+                return (int64_t)Sys_MapAnonymous(frame->arg1, frame->arg2);
+            case SYS_MUNMAP:
+                return Sys_Unmap(frame->arg1, frame->arg2);
+            case SYS_MPROTECT:
+                return Sys_Protect(frame->arg1, frame->arg2, frame->arg3);
             case SYS_GETTICKS:
                 return (int64_t)Sys_GetTicks();
             case SYS_GETMILLISECONDS:
@@ -590,9 +596,9 @@ namespace montauk::abi {
     // ---- SYSCALL MSR initialization ----
 
     void InitializeSyscalls() {
-        // Enable SYSCALL/SYSRET in EFER
+        // Enable SYSCALL/SYSRET and no-execute page permissions in EFER.
         uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
-        efer |= 1;  // SCE bit (Syscall Enable)
+        efer |= 1 | (1ULL << 11);  // SCE | NXE
         Hal::WriteMSR(Hal::IA32_EFER, efer);
 
         // STAR: kernel CS in [47:32], sysret base in [63:48]
@@ -608,7 +614,7 @@ namespace montauk::abi {
         Hal::WriteMSR(Hal::IA32_FMASK, 0x200);
 
         Kt::KernelLogStream(Kt::OK, "Syscall") << "SYSCALL/SYSRET initialized (LSTAR="
-            << kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 154 syscall slots)";
+            << kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 171 syscall slots)";
     }
 
 }

kernel/src/Api/Syscall.hpp

@@ -312,6 +312,9 @@ namespace montauk::abi {
 
     // Set path access/modification times. (path, atime, mtime, useCurrent)
     static constexpr uint64_t SYS_UTIME               = 167;
+    static constexpr uint64_t SYS_MMAP_ANON           = 168;
+    static constexpr uint64_t SYS_MUNMAP              = 169;
+    static constexpr uint64_t SYS_MPROTECT            = 170;
 
     // Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
     static constexpr int SDR_PARAM_FREQ        = 0;  // center frequency, Hz

kernel/src/Api/UserMemory.hpp

@@ -5,6 +5,7 @@
 
 #include <Sched/Scheduler.hpp>
 #include <Memory/Paging.hpp>
+#include <Api/Heap.hpp>
 
 namespace montauk::abi::UserMemory {
 
@@ -32,6 +33,21 @@ namespace montauk::abi::UserMemory {
         uint64_t pml4Phys = CurrentPml4();
         if (pml4Phys == 0) return false;
 
+        if (Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite))
+            return true;
+
+        // Anonymous VMAs are demand-committed. Validation precedes the actual
+        // kernel copy, so materialize missing pages here with the same access
+        // type that the copy will perform.
+        uint64_t page = addr & ~0xFFFULL;
+        uint64_t last = (addr + size - 1) & ~0xFFFULL;
+        for (;;) {
+            if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, page, 1, requireWrite) &&
+                !TryHandleAnonymousPageFault(page, requireWrite ? 2ULL : 0ULL))
+                return false;
+            if (page == last) break;
+            page += 0x1000ULL;
+        }
         return Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite);
     }
 
@@ -58,7 +74,7 @@ namespace montauk::abi::UserMemory {
         const char* str = (const char*)addr;
         for (uint64_t i = 0; i < maxLen; i++) {
             if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr + i, 1, false)) {
-                return false;
+                if (!TryHandleAnonymousPageFault(addr + i, 0)) return false;
             }
             if (str[i] == '\0') return true;
         }