daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: overhaul userspace heap and virtual memory

Browse Source
This commit is contained in:
daniel
2026-08-10 12:25:27 +02:00
parent 4ecec32c5c
commit 9b23d99082
21 changed files with 905 additions and 451 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/src/Api/BuildNo.hpp
+1 -1
View File
@@ -12,4 +12,4 @@
#pragma once
#define MONTAUK_BUILD_NUMBER 95
#define MONTAUK_BUILD_NUMBER 96
kernel/src/Api/Heap.hpp
+269 -56
View File
@@ -4,6 +4,7 @@
* Copyright (c) 2026 Daniel Hammer
*/
#pragma once
#include <cstdint>
#include <Sched/Scheduler.hpp>
#include <Memory/Paging.hpp>
@@ -18,12 +19,16 @@ namespace montauk::abi {
struct HeapAlloc {
uint64_t va;
uint64_t numPages;
uint64_t prot;
uint64_t allocationId;
HeapAlloc* next;
};
static constexpr int MaxHeapAllocs = 512;
inline HeapAlloc g_heapAllocs[Sched::MaxProcesses][MaxHeapAllocs] = {};
inline int g_heapAllocCount[Sched::MaxProcesses] = {};
// VM area metadata is dynamically sized. The previous 512-entry array made
// valid mappings fail for metadata exhaustion and consumed several MiB of
// kernel BSS even for processes with no mappings.
inline HeapAlloc* g_heapAllocs[Sched::MaxProcesses] = {};
inline uint64_t g_nextHeapAllocationId[Sched::MaxProcesses] = {};
inline kcp::Mutex g_heapLocks[Sched::MaxProcesses];
// Get the process table slot index for the current process
@@ -35,12 +40,22 @@ namespace montauk::abi {
return (int)(proc - slot0);
}
inline uint64_t Sys_Alloc(uint64_t size) {
static constexpr uint64_t VmProtRead = 1;
static constexpr uint64_t VmProtWrite = 2;
static constexpr uint64_t VmProtExec = 4;
inline uint64_t Sys_MapAnonymous(uint64_t size, uint64_t prot) {
auto* proc = Sched::GetCurrentProcessPtr();
if (proc == nullptr) return 0;
int slot = GetCurrentSlot();
if (slot < 0) return 0;
if ((prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
(prot & VmProtRead) == 0 ||
((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) {
return 0;
}
// Guard against overflow before rounding.
if (size > 0xFFFFFFFFFFFF0000ULL) return 0;
@@ -50,17 +65,6 @@ namespace montauk::abi {
uint64_t numPages = size / 0x1000;
g_heapLocks[slot].Acquire();
if (g_heapAllocCount[slot] >= MaxHeapAllocs) {
// Out of allocation records, not out of memory. Log it: a
// silent 0 here surfaced as bogus downstream errors (BFD
// turned NULL mallocs into "file format not recognized").
Kt::KernelLogStream(Kt::ERROR, "Heap")
<< "pid " << proc->pid << " (" << proc->name
<< ") hit MaxHeapAllocs (" << (uint64_t)MaxHeapAllocs
<< "), SYS_ALLOC refused";
g_heapLocks[slot].Release();
return 0;
}
uint64_t userVa = 0;
if (!Sched::ReserveUserHeapRange(slot, size, userVa)) {
@@ -71,39 +75,34 @@ namespace montauk::abi {
return 0;
}
// Allocate physical pages and map them into the process
uint64_t mappedPages = 0;
for (uint64_t i = 0; i < numPages; i++) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
g_heapLocks[slot].Release();
return 0;
}
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, userVa + i * 0x1000)) {
Memory::g_pfa->Free(page);
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
g_heapLocks[slot].Release();
return 0;
}
mappedPages++;
}
// Track the allocation so Sys_Free can release it
Sched::g_allocatedPages[slot] += numPages;
g_heapAllocs[slot][g_heapAllocCount[slot]++] = { userVa, numPages };
// Reserve a VMA without committing physical pages. First access is
// satisfied by TryHandleAnonymousPageFault with a zeroed frame.
uint64_t allocationId = ++g_nextHeapAllocationId[slot];
if (allocationId == 0) allocationId = ++g_nextHeapAllocationId[slot];
g_heapAllocs[slot] = new HeapAlloc { userVa, numPages, prot, allocationId,
g_heapAllocs[slot] };
g_heapLocks[slot].Release();
return userVa;
}
inline uint64_t Sys_Alloc(uint64_t size) {
return Sys_MapAnonymous(size, VmProtRead | VmProtWrite);
}
// Reset heap allocation tracking for a process slot.
// The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup.
inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) {
if (slot < 0 || slot >= Sched::MaxProcesses) return;
g_heapLocks[slot].Acquire();
g_heapAllocCount[slot] = 0;
HeapAlloc* area = g_heapAllocs[slot];
while (area != nullptr) {
HeapAlloc* next = area->next;
delete area;
area = next;
}
g_heapAllocs[slot] = nullptr;
g_nextHeapAllocationId[slot] = 0;
Sched::g_allocatedPages[slot] = 0;
g_heapLocks[slot].Release();
}
@@ -117,31 +116,245 @@ namespace montauk::abi {
g_heapLocks[slot].Acquire();
// Find the allocation record matching this address
int idx = -1;
for (int i = 0; i < g_heapAllocCount[slot]; i++) {
if (g_heapAllocs[slot][i].va == addr) {
idx = i;
// A protection change can split one allocation into several VMA
// records. Resolve the stable allocation ID from the supplied base,
// detach every surviving fragment, then tear it down without holding
// the VMA lock.
HeapAlloc* released = nullptr;
HeapAlloc** link = &g_heapAllocs[slot];
uint64_t resident = 0;
uint64_t allocationId = 0;
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
area = area->next) {
if (area->va == addr) {
allocationId = area->allocationId;
break;
}
}
if (idx < 0) {
if (allocationId == 0) {
g_heapLocks[slot].Release();
return; // Unknown address — ignore
}
while (*link != nullptr) {
HeapAlloc* area = *link;
if (area->allocationId != allocationId) {
link = &area->next;
continue;
}
for (uint64_t i = 0; i < area->numPages; i++)
if (Memory::VMM::Paging::GetPhysAddr(
proc->pml4Phys, area->va + i * 0x1000ULL) != 0)
resident++;
*link = area->next;
area->next = released;
released = area;
}
g_heapLocks[slot].Release();
uint64_t va = g_heapAllocs[slot][idx].va;
uint64_t numPages = g_heapAllocs[slot][idx].numPages;
// Unmap and invalidate sibling CPUs before recycling frames. A stale
// user TLB entry can otherwise corrupt the frame's next owner.
while (released != nullptr) {
HeapAlloc* next = released->next;
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, released->va,
released->numPages);
Sched::ReleaseUserHeapRange(slot, released->va,
released->numPages * 0x1000ULL);
delete released;
released = next;
}
// Unmap and invalidate sibling CPUs before recycling the frames. A
// stale user TLB entry can otherwise corrupt the frame's next owner.
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, va, numPages);
Sched::g_allocatedPages[slot] -= numPages;
// Remove tracking entry by swapping with the last element
g_heapAllocs[slot][idx] = g_heapAllocs[slot][g_heapAllocCount[slot] - 1];
g_heapAllocCount[slot]--;
g_heapLocks[slot].Acquire();
Sched::g_allocatedPages[slot] -= resident;
g_heapLocks[slot].Release();
}
inline int64_t Sys_Unmap(uint64_t addr, uint64_t size) {
auto* proc = Sched::GetCurrentProcessPtr();
int slot = GetCurrentSlot();
if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
size > UINT64_MAX - 0xFFFULL) return -1;
size = (size + 0xFFFULL) & ~0xFFFULL;
if (addr > UINT64_MAX - size) return -1;
g_heapLocks[slot].Acquire();
uint64_t firstPage = 0;
uint64_t pages = size / 0x1000ULL;
HeapAlloc** link = &g_heapAllocs[slot];
while (*link != nullptr) {
HeapAlloc* area = *link;
uint64_t areaSize = area->numPages * 0x1000ULL;
if (addr >= area->va && addr - area->va <= areaSize &&
size <= areaSize - (addr - area->va)) {
firstPage = (addr - area->va) / 0x1000ULL;
break;
}
link = &area->next;
}
if (*link == nullptr) { g_heapLocks[slot].Release(); return -1; }
HeapAlloc* area = *link;
uint64_t oldPages = area->numPages;
uint64_t oldProt = area->prot;
uint64_t tailPages = oldPages - firstPage - pages;
if (firstPage == 0 && tailPages == 0) {
*link = area->next;
delete area;
} else if (firstPage == 0) {
area->va = addr + size;
area->numPages = tailPages;
} else {
area->numPages = firstPage;
if (tailPages != 0) {
area->next = new HeapAlloc { addr + size, tailPages, oldProt,
area->allocationId,
area->next };
}
}
uint64_t resident = 0;
for (uint64_t i = 0; i < pages; i++)
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, addr + i * 0x1000ULL) != 0)
resident++;
g_heapLocks[slot].Release();
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, addr, pages);
Sched::ReleaseUserHeapRange(slot, addr, size);
g_heapLocks[slot].Acquire();
Sched::g_allocatedPages[slot] -= resident;
g_heapLocks[slot].Release();
return 0;
}
inline int64_t Sys_Protect(uint64_t addr, uint64_t size, uint64_t prot) {
auto* proc = Sched::GetCurrentProcessPtr();
int slot = GetCurrentSlot();
if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
size > UINT64_MAX - 0xFFFULL ||
(prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
(prot & VmProtRead) == 0 ||
((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) return -1;
size = (size + 0xFFFULL) & ~0xFFFULL;
if (addr > UINT64_MAX - size) return -1;
g_heapLocks[slot].Acquire();
uint64_t pages = size / 0x1000ULL;
// Protection changes may span adjacent VMA fragments created by a
// previous mprotect. Validate the complete range before changing any
// metadata so failure is atomic from userspace's perspective.
for (uint64_t page = 0; page < pages; page++) {
uint64_t pageVa = addr + page * 0x1000ULL;
bool covered = false;
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
area = area->next) {
uint64_t areaSize = area->numPages * 0x1000ULL;
if (pageVa >= area->va && pageVa - area->va < areaSize) {
covered = true;
break;
}
}
if (!covered) { g_heapLocks[slot].Release(); return -1; }
}
uint64_t end = addr + size;
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;) {
HeapAlloc* oldNext = area->next;
uint64_t areaEnd = area->va + area->numPages * 0x1000ULL;
uint64_t overlapStart = area->va > addr ? area->va : addr;
uint64_t overlapEnd = areaEnd < end ? areaEnd : end;
if (overlapStart < overlapEnd) {
uint64_t oldVa = area->va;
uint64_t oldProt = area->prot;
uint64_t prefixPages = (overlapStart - oldVa) / 0x1000ULL;
uint64_t protectedPages = (overlapEnd - overlapStart) / 0x1000ULL;
uint64_t tailPages = (areaEnd - overlapEnd) / 0x1000ULL;
if (prefixPages == 0) {
area->numPages = protectedPages;
area->prot = prot;
if (tailPages != 0)
area->next = new HeapAlloc { overlapEnd, tailPages,
oldProt,
area->allocationId,
oldNext };
} else {
area->numPages = prefixPages;
auto* middle = new HeapAlloc { overlapStart, protectedPages,
prot, area->allocationId,
oldNext };
area->next = middle;
if (tailPages != 0)
middle->next = new HeapAlloc { overlapEnd, tailPages,
oldProt,
area->allocationId,
oldNext };
}
}
area = oldNext;
}
bool writable = (prot & VmProtWrite) != 0;
bool executable = (prot & VmProtExec) != 0;
for (uint64_t i = 0; i < pages; i++) {
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys,
addr + i * 0x1000ULL) == 0)
continue;
if (!Memory::VMM::Paging::ProtectUserIn(proc->pml4Phys,
addr + i * 0x1000ULL,
writable, executable)) {
g_heapLocks[slot].Release();
return -1;
}
}
g_heapLocks[slot].Release();
Ipc::ShootdownUserRange(proc->pml4Phys, addr, (uint32_t)pages);
return 0;
}
// Resolve a non-present access inside an anonymous VMA. This is used for
// both ring-3 faults and kernel copies to untouched userspace buffers.
inline bool TryHandleAnonymousPageFault(uint64_t faultAddr, uint64_t errorCode) {
auto* proc = Sched::GetCurrentProcessPtr();
int slot = GetCurrentSlot();
if (proc == nullptr || slot < 0 || faultAddr < Sched::UserHeapBase ||
faultAddr >= Sched::UserHeapLimit || (errorCode & 1) != 0) return false;
uint64_t pageVa = faultAddr & ~0xFFFULL;
g_heapLocks[slot].Acquire();
HeapAlloc* found = nullptr;
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr; area = area->next) {
uint64_t bytes = area->numPages * 0x1000ULL;
if (pageVa >= area->va && pageVa - area->va < bytes) {
found = area;
break;
}
}
if (found == nullptr ||
((errorCode & (1ULL << 1)) != 0 && (found->prot & VmProtWrite) == 0) ||
((errorCode & (1ULL << 4)) != 0 && (found->prot & VmProtExec) == 0)) {
g_heapLocks[slot].Release();
return false;
}
// A sibling may have resolved the same fault while this CPU waited.
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa) != 0) {
g_heapLocks[slot].Release();
return true;
}
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) {
g_heapLocks[slot].Release();
return false;
}
uint64_t phys = Memory::SubHHDM((uint64_t)page);
bool mapped = Memory::VMM::Paging::MapUserInPermissions(
proc->pml4Phys, phys, pageVa,
(found->prot & VmProtWrite) != 0, (found->prot & VmProtExec) != 0);
if (!mapped) Memory::g_pfa->Free(page);
else Sched::g_allocatedPages[slot]++;
g_heapLocks[slot].Release();
return mapped;
}
};
kernel/src/Api/Syscall.cpp
+9 -3
View File
@@ -118,6 +118,12 @@ namespace montauk::abi {
case SYS_FREE:
Sys_Free(frame->arg1);
return 0;
case SYS_MMAP_ANON:
return (int64_t)Sys_MapAnonymous(frame->arg1, frame->arg2);
case SYS_MUNMAP:
return Sys_Unmap(frame->arg1, frame->arg2);
case SYS_MPROTECT:
return Sys_Protect(frame->arg1, frame->arg2, frame->arg3);
case SYS_GETTICKS:
return (int64_t)Sys_GetTicks();
case SYS_GETMILLISECONDS:
@@ -590,9 +596,9 @@ namespace montauk::abi {
// ---- SYSCALL MSR initialization ----
void InitializeSyscalls() {
// Enable SYSCALL/SYSRET in EFER
// Enable SYSCALL/SYSRET and no-execute page permissions in EFER.
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
efer |= 1; // SCE bit (Syscall Enable)
efer |= 1 | (1ULL << 11); // SCE | NXE
Hal::WriteMSR(Hal::IA32_EFER, efer);
// STAR: kernel CS in [47:32], sysret base in [63:48]
@@ -608,7 +614,7 @@ namespace montauk::abi {
Hal::WriteMSR(Hal::IA32_FMASK, 0x200);
Kt::KernelLogStream(Kt::OK, "Syscall") << "SYSCALL/SYSRET initialized (LSTAR="
<< kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 154 syscall slots)";
<< kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 171 syscall slots)";
}
}
kernel/src/Api/Syscall.hpp
+3
View File
@@ -312,6 +312,9 @@ namespace montauk::abi {
// Set path access/modification times. (path, atime, mtime, useCurrent)
static constexpr uint64_t SYS_UTIME = 167;
static constexpr uint64_t SYS_MMAP_ANON = 168;
static constexpr uint64_t SYS_MUNMAP = 169;
static constexpr uint64_t SYS_MPROTECT = 170;
// Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
static constexpr int SDR_PARAM_FREQ = 0; // center frequency, Hz
kernel/src/Api/UserMemory.hpp
+17 -1
View File
@@ -5,6 +5,7 @@
#include <Sched/Scheduler.hpp>
#include <Memory/Paging.hpp>
#include <Api/Heap.hpp>
namespace montauk::abi::UserMemory {
@@ -32,6 +33,21 @@ namespace montauk::abi::UserMemory {
uint64_t pml4Phys = CurrentPml4();
if (pml4Phys == 0) return false;
if (Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite))
return true;
// Anonymous VMAs are demand-committed. Validation precedes the actual
// kernel copy, so materialize missing pages here with the same access
// type that the copy will perform.
uint64_t page = addr & ~0xFFFULL;
uint64_t last = (addr + size - 1) & ~0xFFFULL;
for (;;) {
if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, page, 1, requireWrite) &&
!TryHandleAnonymousPageFault(page, requireWrite ? 2ULL : 0ULL))
return false;
if (page == last) break;
page += 0x1000ULL;
}
return Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite);
}
@@ -58,7 +74,7 @@ namespace montauk::abi::UserMemory {
const char* str = (const char*)addr;
for (uint64_t i = 0; i < maxLen; i++) {
if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr + i, 1, false)) {
return false;
if (!TryHandleAnonymousPageFault(addr + i, 0)) return false;
}
if (str[i] == '\0') return true;
}
kernel/src/Hal/IDT.cpp
+7 -4
View File
@@ -13,6 +13,7 @@
#include <Memory/PageFrameAllocator.hpp>
#include <Sched/Scheduler.hpp>
#include <Sched/CrashReport.hpp>
#include <Api/Heap.hpp>
#include <Hal/SmpBoot.hpp>
#include <Timekeeping/ApicTimer.hpp>
@@ -201,10 +202,12 @@ namespace Hal {
// pushes past the mapped stack and kernel accesses to not-yet-grown
// user stack buffers passed into syscalls.
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
&& Sched::GetCurrentPid() >= 0
&& Sched::TryGrowUserStack(cr2)) {
if (fromUser) asm volatile("swapgs");
return;
&& Sched::GetCurrentPid() >= 0) {
if (montauk::abi::TryHandleAnonymousPageFault(cr2, errorCode) ||
Sched::TryGrowUserStack(cr2)) {
if (fromUser) asm volatile("swapgs");
return;
}
}
// Not a growable fault. Hand the RAW frame (error code at offset 0)
kernel/src/Hal/SmpBoot.cpp
+1 -1
View File
@@ -188,7 +188,7 @@ namespace Smp {
// --- Program SYSCALL MSRs ---
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
efer |= 1; // SCE
efer |= 1 | (1ULL << 11); // SCE | NXE
Hal::WriteMSR(Hal::IA32_EFER, efer);
uint64_t star = (0x0010ULL << 48) | (0x0008ULL << 32);
kernel/src/Memory/Paging.cpp
+55 -9
View File
@@ -13,13 +13,21 @@ namespace Memory::VMM {
static constexpr uint64_t LeafUser = 1ULL << 2;
static constexpr uint64_t LeafWriteThrough = 1ULL << 3;
static constexpr uint64_t LeafCacheDisabled = 1ULL << 4;
static constexpr uint64_t LeafNoExecute = 1ULL << 63;
// PageTableEntry's historical 52-bit Address bitfield also spans the
// architectural high flag bits. Mask those bits whenever extracting a
// physical address, in particular now that anonymous leaves use NX.
static constexpr uint64_t kPhysAddrMask = 0xFFFFFFFFFFULL; // PTE bits 12..51
static inline void SetLeafPte(PageTableEntry* entry, uint64_t physicalAddress,
bool user, bool writeThrough, bool cacheDisabled) {
uint64_t flags = LeafPresent | LeafWritable;
bool user, bool writeThrough, bool cacheDisabled,
bool writable = true, bool executable = true) {
uint64_t flags = LeafPresent;
if (writable) flags |= LeafWritable;
if (user) flags |= LeafUser;
if (writeThrough) flags |= LeafWriteThrough;
if (cacheDisabled) flags |= LeafCacheDisabled;
if (!executable) flags |= LeafNoExecute;
// Replace the complete PTE in one aligned store. Updating individual
// bitfields left old PWT/PCD/PAT/accessed state behind when a virtual
@@ -230,6 +238,12 @@ namespace Memory::VMM {
}
bool Paging::MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
return MapUserInPermissions(pml4Phys, physicalAddress, virtualAddress, true, true);
}
bool Paging::MapUserInPermissions(std::uint64_t pml4Phys, std::uint64_t physicalAddress,
std::uint64_t virtualAddress, bool writable,
bool executable) {
pagingLock.Acquire();
if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
pagingLock.Release();
@@ -266,7 +280,43 @@ namespace Memory::VMM {
if (!pml1) { pagingLock.Release(); return false; }
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
SetLeafPte(pageEntry, physicalAddress, true, false, false);
SetLeafPte(pageEntry, physicalAddress, true, false, false, writable, executable);
pagingLock.Release();
return true;
}
bool Paging::ProtectUserIn(std::uint64_t pml4Phys, std::uint64_t virtualAddress,
bool writable, bool executable) {
if ((virtualAddress & 0xFFFULL) != 0) return false;
pagingLock.Acquire();
VirtualAddress va(virtualAddress);
auto walkRead = [](PageTable* table, uint64_t index) -> PageTable* {
PageTableEntry* entry = (PageTableEntry*)Memory::HHDM(&table->entries[index]);
if (!entry->Present || !entry->Supervisor || entry->LargerPages) return nullptr;
return (PageTable*)(entry->Address << 12);
};
PageTable* pml4 = (PageTable*)pml4Phys;
auto pml3 = walkRead(pml4, va.GetL4Index());
if (!pml3) { pagingLock.Release(); return false; }
auto pml2 = walkRead(pml3, va.GetL3Index());
if (!pml2) { pagingLock.Release(); return false; }
auto pml1 = walkRead(pml2, va.GetL2Index());
if (!pml1) { pagingLock.Release(); return false; }
PageTableEntry* pte = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
uint64_t raw = *(volatile uint64_t*)pte;
if ((raw & LeafPresent) == 0 || (raw & LeafUser) == 0) {
pagingLock.Release();
return false;
}
if (writable) raw |= LeafWritable;
else raw &= ~LeafWritable;
if (executable) raw &= ~LeafNoExecute;
else raw |= LeafNoExecute;
*(volatile uint64_t*)pte = raw;
asm volatile("invlpg (%0)" :: "r"(virtualAddress) : "memory");
pagingLock.Release();
return true;
}
@@ -376,7 +426,8 @@ namespace Memory::VMM {
// Skip MMIO/WC pages (not PFA-managed)
if (pte->WriteThrough || pte->CacheDisabled) continue;
uint64_t pagePhys = (uint64_t)pte->Address << 12;
uint64_t pagePhys =
(uint64_t)(pte->Address & kPhysAddrMask) << 12;
if (pagePhys != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
}
@@ -442,11 +493,6 @@ namespace Memory::VMM {
return true;
}
// Mask to extract only the 40-bit physical address from a PTE Address field
// (bits 12-51 of the PTE). The PageTableEntry::Address field is 52 bits wide
// and includes NX, PK, and software-available bits that must be stripped.
static constexpr uint64_t kPhysAddrMask = 0xFFFFFFFFFFULL; // 40 bits
std::uint64_t Paging::GetPhysAddr(std::uint64_t pml4, std::uint64_t virtualAddress, bool use40BitL1) {
VirtualAddress virtualAddressObj(virtualAddress);
kernel/src/Memory/Paging.hpp
+11
View File
@@ -104,6 +104,17 @@ public:
// Map a page into an arbitrary PML4 (specified by physical address) with User bit set.
static bool MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
// Map ordinary user memory with explicit leaf permissions. Read access
// is implied by x86 paging; writable and executable are independent.
static bool MapUserInPermissions(std::uint64_t pml4Phys,
std::uint64_t physicalAddress,
std::uint64_t virtualAddress,
bool writable, bool executable);
// Change leaf permissions on an existing user page.
static bool ProtectUserIn(std::uint64_t pml4Phys, std::uint64_t virtualAddress,
bool writable, bool executable);
// Map a page into an arbitrary PML4 with User + Write-Combining attributes.
static bool MapUserInWC(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
kernel/src/Sched/Scheduler.cpp
+52 -8
View File
@@ -493,6 +493,8 @@ namespace Sched {
proc.kernelStackTop = kernelStackTop;
proc.userStackTop = UserStackTop - 8;
proc.heapNext = UserHeapBase;
for (uint64_t i = 0; i < UserHeapBitmapWords; i++)
g_userHeapPageMap[slot][i] = 0;
proc.fsBase = tls.fsBase;
proc.tlsTemplateVaddr = tls.templateVaddr;
proc.tlsFileSize = tls.fileSize;
@@ -651,6 +653,7 @@ namespace Sched {
}
if (!ok) {
Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
ReleaseUserHeapRange(primarySlot_, base, numPages * 0x1000ULL);
Kt::KernelLogStream(Kt::ERROR, "Sched")
<< "Thread TLS allocation failed";
return -1;
@@ -675,8 +678,11 @@ namespace Sched {
// Allocate kernel stack.
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
if (stackMem == nullptr) {
if (threadTlsPages != 0)
if (threadTlsPages != 0) {
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
threadTlsPages * 0x1000ULL);
}
return -1;
}
memset(stackMem, 0, StackSize);
@@ -698,8 +704,11 @@ namespace Sched {
if (slot < 0) {
schedLock.Release();
Memory::g_pfa->Free(stackMem, StackPages);
if (threadTlsPages != 0)
if (threadTlsPages != 0) {
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
threadTlsPages * 0x1000ULL);
}
return -1;
}
@@ -774,20 +783,54 @@ namespace Sched {
schedLock.Acquire();
Process& primary = processTable[primarySlot_];
uint64_t base = primary.heapNext;
bool validOwner = primary.primarySlot == primarySlot_ &&
primary.state != ProcessState::Free &&
primary.state != ProcessState::Terminated;
bool fits = validOwner && base >= UserHeapBase &&
base <= UserHeapLimit && size <= UserHeapLimit - base;
if (fits) {
primary.heapNext = base + size;
outVa = base;
bool fits = false;
uint64_t pages = size / 0x1000ULL;
uint64_t runStart = 0;
uint64_t runLength = 0;
if (validOwner && pages <= UserHeapPages) {
for (uint64_t page = 0; page < UserHeapPages; page++) {
bool used = (g_userHeapPageMap[primarySlot_][page / 64]
& (1ULL << (page % 64))) != 0;
if (used) {
runLength = 0;
continue;
}
if (runLength == 0) runStart = page;
runLength++;
if (runLength == pages) {
for (uint64_t p = runStart; p < runStart + pages; p++)
g_userHeapPageMap[primarySlot_][p / 64] |= 1ULL << (p % 64);
outVa = UserHeapBase + runStart * 0x1000ULL;
uint64_t end = outVa + size;
if (end > primary.heapNext) primary.heapNext = end;
fits = true;
break;
}
}
}
schedLock.Release();
return fits;
}
void ReleaseUserHeapRange(int primarySlot_, uint64_t va, uint64_t size) {
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
(va & 0xFFFULL) != 0 || (size & 0xFFFULL) != 0 ||
va < UserHeapBase || va > UserHeapLimit || size > UserHeapLimit - va) {
return;
}
uint64_t first = (va - UserHeapBase) / 0x1000ULL;
uint64_t pages = size / 0x1000ULL;
schedLock.Acquire();
for (uint64_t p = first; p < first + pages; p++)
g_userHeapPageMap[primarySlot_][p / 64] &= ~(1ULL << (p % 64));
schedLock.Release();
}
static void FreeSiblingThreadTls(Process& thr) {
int primarySlot_ = thr.primarySlot;
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
@@ -811,6 +854,7 @@ namespace Sched {
uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000;
thr.fsBase = 0;
Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
ReleaseUserHeapRange(primarySlot_, base, pages * 0x1000ULL);
}
// Switch away from a slot we have just marked non-runnable while holding
kernel/src/Sched/Scheduler.hpp
+7
View File
@@ -33,6 +33,9 @@ namespace Sched {
// failed bound check is recoverable (SYS_ALLOC returns null); allowing the
// heap to cross this boundary would silently replace library PTEs.
static constexpr uint64_t UserHeapLimit = 0x60000000ULL;
static constexpr uint64_t UserHeapPages =
(UserHeapLimit - UserHeapBase) / 0x1000ULL;
static constexpr uint64_t UserHeapBitmapWords = (UserHeapPages + 63) / 64;
// Surface mappings use reusable, fixed-size per-process slots instead of
// consuming heap VA forever. 64 * 32 MiB occupies [0x7000000000,
// 0x7080000000), comfortably below the main user stack.
@@ -191,6 +194,7 @@ namespace Sched {
// reservation is serialized across sibling threads and bounded below the
// fixed shared-library mappings.
bool ReserveUserHeapRange(int primarySlot, uint64_t size, uint64_t& outVa);
void ReleaseUserHeapRange(int primarySlot, uint64_t va, uint64_t size);
// Terminate the currently executing thread. If this is the main thread,
// the entire process exits (equivalent to ExitProcess).
@@ -258,5 +262,8 @@ namespace Sched {
// Per-process allocated page count (tracked by Heap syscalls, separate from Process struct)
inline uint64_t g_allocatedPages[MaxProcesses] = {};
// One bit per page in the bounded userspace heap. Unlike heapNext, this
// makes virtual ranges reusable after unmap and failed reservations.
inline uint64_t g_userHeapPageMap[MaxProcesses][UserHeapBitmapWords] = {};
}