feat: overhaul userspace heap and virtual memory
This commit is contained in:
@@ -12,4 +12,4 @@
|
|||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#define MONTAUK_BUILD_NUMBER 95
|
#define MONTAUK_BUILD_NUMBER 96
|
||||||
|
|||||||
+269
-56
@@ -4,6 +4,7 @@
|
|||||||
* Copyright (c) 2026 Daniel Hammer
|
* Copyright (c) 2026 Daniel Hammer
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <Sched/Scheduler.hpp>
|
#include <Sched/Scheduler.hpp>
|
||||||
#include <Memory/Paging.hpp>
|
#include <Memory/Paging.hpp>
|
||||||
@@ -18,12 +19,16 @@ namespace montauk::abi {
|
|||||||
struct HeapAlloc {
|
struct HeapAlloc {
|
||||||
uint64_t va;
|
uint64_t va;
|
||||||
uint64_t numPages;
|
uint64_t numPages;
|
||||||
|
uint64_t prot;
|
||||||
|
uint64_t allocationId;
|
||||||
|
HeapAlloc* next;
|
||||||
};
|
};
|
||||||
|
|
||||||
static constexpr int MaxHeapAllocs = 512;
|
// VM area metadata is dynamically sized. The previous 512-entry array made
|
||||||
|
// valid mappings fail for metadata exhaustion and consumed several MiB of
|
||||||
inline HeapAlloc g_heapAllocs[Sched::MaxProcesses][MaxHeapAllocs] = {};
|
// kernel BSS even for processes with no mappings.
|
||||||
inline int g_heapAllocCount[Sched::MaxProcesses] = {};
|
inline HeapAlloc* g_heapAllocs[Sched::MaxProcesses] = {};
|
||||||
|
inline uint64_t g_nextHeapAllocationId[Sched::MaxProcesses] = {};
|
||||||
inline kcp::Mutex g_heapLocks[Sched::MaxProcesses];
|
inline kcp::Mutex g_heapLocks[Sched::MaxProcesses];
|
||||||
|
|
||||||
// Get the process table slot index for the current process
|
// Get the process table slot index for the current process
|
||||||
@@ -35,12 +40,22 @@ namespace montauk::abi {
|
|||||||
return (int)(proc - slot0);
|
return (int)(proc - slot0);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline uint64_t Sys_Alloc(uint64_t size) {
|
static constexpr uint64_t VmProtRead = 1;
|
||||||
|
static constexpr uint64_t VmProtWrite = 2;
|
||||||
|
static constexpr uint64_t VmProtExec = 4;
|
||||||
|
|
||||||
|
inline uint64_t Sys_MapAnonymous(uint64_t size, uint64_t prot) {
|
||||||
auto* proc = Sched::GetCurrentProcessPtr();
|
auto* proc = Sched::GetCurrentProcessPtr();
|
||||||
if (proc == nullptr) return 0;
|
if (proc == nullptr) return 0;
|
||||||
int slot = GetCurrentSlot();
|
int slot = GetCurrentSlot();
|
||||||
if (slot < 0) return 0;
|
if (slot < 0) return 0;
|
||||||
|
|
||||||
|
if ((prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
|
||||||
|
(prot & VmProtRead) == 0 ||
|
||||||
|
((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Guard against overflow before rounding.
|
// Guard against overflow before rounding.
|
||||||
if (size > 0xFFFFFFFFFFFF0000ULL) return 0;
|
if (size > 0xFFFFFFFFFFFF0000ULL) return 0;
|
||||||
|
|
||||||
@@ -50,17 +65,6 @@ namespace montauk::abi {
|
|||||||
|
|
||||||
uint64_t numPages = size / 0x1000;
|
uint64_t numPages = size / 0x1000;
|
||||||
g_heapLocks[slot].Acquire();
|
g_heapLocks[slot].Acquire();
|
||||||
if (g_heapAllocCount[slot] >= MaxHeapAllocs) {
|
|
||||||
// Out of allocation records, not out of memory. Log it: a
|
|
||||||
// silent 0 here surfaced as bogus downstream errors (BFD
|
|
||||||
// turned NULL mallocs into "file format not recognized").
|
|
||||||
Kt::KernelLogStream(Kt::ERROR, "Heap")
|
|
||||||
<< "pid " << proc->pid << " (" << proc->name
|
|
||||||
<< ") hit MaxHeapAllocs (" << (uint64_t)MaxHeapAllocs
|
|
||||||
<< "), SYS_ALLOC refused";
|
|
||||||
g_heapLocks[slot].Release();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t userVa = 0;
|
uint64_t userVa = 0;
|
||||||
if (!Sched::ReserveUserHeapRange(slot, size, userVa)) {
|
if (!Sched::ReserveUserHeapRange(slot, size, userVa)) {
|
||||||
@@ -71,39 +75,34 @@ namespace montauk::abi {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Allocate physical pages and map them into the process
|
// Reserve a VMA without committing physical pages. First access is
|
||||||
uint64_t mappedPages = 0;
|
// satisfied by TryHandleAnonymousPageFault with a zeroed frame.
|
||||||
for (uint64_t i = 0; i < numPages; i++) {
|
uint64_t allocationId = ++g_nextHeapAllocationId[slot];
|
||||||
void* page = Memory::g_pfa->AllocateZeroed();
|
if (allocationId == 0) allocationId = ++g_nextHeapAllocationId[slot];
|
||||||
if (page == nullptr) {
|
g_heapAllocs[slot] = new HeapAlloc { userVa, numPages, prot, allocationId,
|
||||||
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
|
g_heapAllocs[slot] };
|
||||||
g_heapLocks[slot].Release();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
uint64_t physAddr = Memory::SubHHDM((uint64_t)page);
|
|
||||||
if (!Memory::VMM::Paging::MapUserIn(proc->pml4Phys, physAddr, userVa + i * 0x1000)) {
|
|
||||||
Memory::g_pfa->Free(page);
|
|
||||||
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, userVa, mappedPages);
|
|
||||||
g_heapLocks[slot].Release();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
mappedPages++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track the allocation so Sys_Free can release it
|
|
||||||
Sched::g_allocatedPages[slot] += numPages;
|
|
||||||
g_heapAllocs[slot][g_heapAllocCount[slot]++] = { userVa, numPages };
|
|
||||||
|
|
||||||
g_heapLocks[slot].Release();
|
g_heapLocks[slot].Release();
|
||||||
return userVa;
|
return userVa;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline uint64_t Sys_Alloc(uint64_t size) {
|
||||||
|
return Sys_MapAnonymous(size, VmProtRead | VmProtWrite);
|
||||||
|
}
|
||||||
|
|
||||||
// Reset heap allocation tracking for a process slot.
|
// Reset heap allocation tracking for a process slot.
|
||||||
// The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup.
|
// The actual physical pages are freed by Paging::FreeUserHalf() during process cleanup.
|
||||||
inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) {
|
inline void CleanupHeapForSlot(int slot, uint64_t /*pml4Phys*/) {
|
||||||
if (slot < 0 || slot >= Sched::MaxProcesses) return;
|
if (slot < 0 || slot >= Sched::MaxProcesses) return;
|
||||||
g_heapLocks[slot].Acquire();
|
g_heapLocks[slot].Acquire();
|
||||||
g_heapAllocCount[slot] = 0;
|
HeapAlloc* area = g_heapAllocs[slot];
|
||||||
|
while (area != nullptr) {
|
||||||
|
HeapAlloc* next = area->next;
|
||||||
|
delete area;
|
||||||
|
area = next;
|
||||||
|
}
|
||||||
|
g_heapAllocs[slot] = nullptr;
|
||||||
|
g_nextHeapAllocationId[slot] = 0;
|
||||||
Sched::g_allocatedPages[slot] = 0;
|
Sched::g_allocatedPages[slot] = 0;
|
||||||
g_heapLocks[slot].Release();
|
g_heapLocks[slot].Release();
|
||||||
}
|
}
|
||||||
@@ -117,31 +116,245 @@ namespace montauk::abi {
|
|||||||
|
|
||||||
g_heapLocks[slot].Acquire();
|
g_heapLocks[slot].Acquire();
|
||||||
|
|
||||||
// Find the allocation record matching this address
|
// A protection change can split one allocation into several VMA
|
||||||
int idx = -1;
|
// records. Resolve the stable allocation ID from the supplied base,
|
||||||
for (int i = 0; i < g_heapAllocCount[slot]; i++) {
|
// detach every surviving fragment, then tear it down without holding
|
||||||
if (g_heapAllocs[slot][i].va == addr) {
|
// the VMA lock.
|
||||||
idx = i;
|
HeapAlloc* released = nullptr;
|
||||||
|
HeapAlloc** link = &g_heapAllocs[slot];
|
||||||
|
uint64_t resident = 0;
|
||||||
|
uint64_t allocationId = 0;
|
||||||
|
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
|
||||||
|
area = area->next) {
|
||||||
|
if (area->va == addr) {
|
||||||
|
allocationId = area->allocationId;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (idx < 0) {
|
if (allocationId == 0) {
|
||||||
g_heapLocks[slot].Release();
|
g_heapLocks[slot].Release();
|
||||||
return; // Unknown address — ignore
|
return; // Unknown address — ignore
|
||||||
}
|
}
|
||||||
|
while (*link != nullptr) {
|
||||||
|
HeapAlloc* area = *link;
|
||||||
|
if (area->allocationId != allocationId) {
|
||||||
|
link = &area->next;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (uint64_t i = 0; i < area->numPages; i++)
|
||||||
|
if (Memory::VMM::Paging::GetPhysAddr(
|
||||||
|
proc->pml4Phys, area->va + i * 0x1000ULL) != 0)
|
||||||
|
resident++;
|
||||||
|
*link = area->next;
|
||||||
|
area->next = released;
|
||||||
|
released = area;
|
||||||
|
}
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
|
||||||
uint64_t va = g_heapAllocs[slot][idx].va;
|
// Unmap and invalidate sibling CPUs before recycling frames. A stale
|
||||||
uint64_t numPages = g_heapAllocs[slot][idx].numPages;
|
// user TLB entry can otherwise corrupt the frame's next owner.
|
||||||
|
while (released != nullptr) {
|
||||||
|
HeapAlloc* next = released->next;
|
||||||
|
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, released->va,
|
||||||
|
released->numPages);
|
||||||
|
Sched::ReleaseUserHeapRange(slot, released->va,
|
||||||
|
released->numPages * 0x1000ULL);
|
||||||
|
delete released;
|
||||||
|
released = next;
|
||||||
|
}
|
||||||
|
|
||||||
// Unmap and invalidate sibling CPUs before recycling the frames. A
|
g_heapLocks[slot].Acquire();
|
||||||
// stale user TLB entry can otherwise corrupt the frame's next owner.
|
Sched::g_allocatedPages[slot] -= resident;
|
||||||
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, va, numPages);
|
|
||||||
|
|
||||||
Sched::g_allocatedPages[slot] -= numPages;
|
|
||||||
|
|
||||||
// Remove tracking entry by swapping with the last element
|
|
||||||
g_heapAllocs[slot][idx] = g_heapAllocs[slot][g_heapAllocCount[slot] - 1];
|
|
||||||
g_heapAllocCount[slot]--;
|
|
||||||
g_heapLocks[slot].Release();
|
g_heapLocks[slot].Release();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline int64_t Sys_Unmap(uint64_t addr, uint64_t size) {
|
||||||
|
auto* proc = Sched::GetCurrentProcessPtr();
|
||||||
|
int slot = GetCurrentSlot();
|
||||||
|
if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
|
||||||
|
size > UINT64_MAX - 0xFFFULL) return -1;
|
||||||
|
size = (size + 0xFFFULL) & ~0xFFFULL;
|
||||||
|
if (addr > UINT64_MAX - size) return -1;
|
||||||
|
|
||||||
|
g_heapLocks[slot].Acquire();
|
||||||
|
uint64_t firstPage = 0;
|
||||||
|
uint64_t pages = size / 0x1000ULL;
|
||||||
|
HeapAlloc** link = &g_heapAllocs[slot];
|
||||||
|
while (*link != nullptr) {
|
||||||
|
HeapAlloc* area = *link;
|
||||||
|
uint64_t areaSize = area->numPages * 0x1000ULL;
|
||||||
|
if (addr >= area->va && addr - area->va <= areaSize &&
|
||||||
|
size <= areaSize - (addr - area->va)) {
|
||||||
|
firstPage = (addr - area->va) / 0x1000ULL;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
link = &area->next;
|
||||||
|
}
|
||||||
|
if (*link == nullptr) { g_heapLocks[slot].Release(); return -1; }
|
||||||
|
|
||||||
|
HeapAlloc* area = *link;
|
||||||
|
uint64_t oldPages = area->numPages;
|
||||||
|
uint64_t oldProt = area->prot;
|
||||||
|
uint64_t tailPages = oldPages - firstPage - pages;
|
||||||
|
|
||||||
|
if (firstPage == 0 && tailPages == 0) {
|
||||||
|
*link = area->next;
|
||||||
|
delete area;
|
||||||
|
} else if (firstPage == 0) {
|
||||||
|
area->va = addr + size;
|
||||||
|
area->numPages = tailPages;
|
||||||
|
} else {
|
||||||
|
area->numPages = firstPage;
|
||||||
|
if (tailPages != 0) {
|
||||||
|
area->next = new HeapAlloc { addr + size, tailPages, oldProt,
|
||||||
|
area->allocationId,
|
||||||
|
area->next };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t resident = 0;
|
||||||
|
for (uint64_t i = 0; i < pages; i++)
|
||||||
|
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, addr + i * 0x1000ULL) != 0)
|
||||||
|
resident++;
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
|
||||||
|
Ipc::UnmapAndFreeUserRange(proc->pml4Phys, addr, pages);
|
||||||
|
Sched::ReleaseUserHeapRange(slot, addr, size);
|
||||||
|
g_heapLocks[slot].Acquire();
|
||||||
|
Sched::g_allocatedPages[slot] -= resident;
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline int64_t Sys_Protect(uint64_t addr, uint64_t size, uint64_t prot) {
|
||||||
|
auto* proc = Sched::GetCurrentProcessPtr();
|
||||||
|
int slot = GetCurrentSlot();
|
||||||
|
if (proc == nullptr || slot < 0 || size == 0 || (addr & 0xFFFULL) != 0 ||
|
||||||
|
size > UINT64_MAX - 0xFFFULL ||
|
||||||
|
(prot & ~(VmProtRead | VmProtWrite | VmProtExec)) != 0 ||
|
||||||
|
(prot & VmProtRead) == 0 ||
|
||||||
|
((prot & VmProtWrite) != 0 && (prot & VmProtExec) != 0)) return -1;
|
||||||
|
size = (size + 0xFFFULL) & ~0xFFFULL;
|
||||||
|
if (addr > UINT64_MAX - size) return -1;
|
||||||
|
|
||||||
|
g_heapLocks[slot].Acquire();
|
||||||
|
uint64_t pages = size / 0x1000ULL;
|
||||||
|
|
||||||
|
// Protection changes may span adjacent VMA fragments created by a
|
||||||
|
// previous mprotect. Validate the complete range before changing any
|
||||||
|
// metadata so failure is atomic from userspace's perspective.
|
||||||
|
for (uint64_t page = 0; page < pages; page++) {
|
||||||
|
uint64_t pageVa = addr + page * 0x1000ULL;
|
||||||
|
bool covered = false;
|
||||||
|
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;
|
||||||
|
area = area->next) {
|
||||||
|
uint64_t areaSize = area->numPages * 0x1000ULL;
|
||||||
|
if (pageVa >= area->va && pageVa - area->va < areaSize) {
|
||||||
|
covered = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!covered) { g_heapLocks[slot].Release(); return -1; }
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t end = addr + size;
|
||||||
|
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr;) {
|
||||||
|
HeapAlloc* oldNext = area->next;
|
||||||
|
uint64_t areaEnd = area->va + area->numPages * 0x1000ULL;
|
||||||
|
uint64_t overlapStart = area->va > addr ? area->va : addr;
|
||||||
|
uint64_t overlapEnd = areaEnd < end ? areaEnd : end;
|
||||||
|
if (overlapStart < overlapEnd) {
|
||||||
|
uint64_t oldVa = area->va;
|
||||||
|
uint64_t oldProt = area->prot;
|
||||||
|
uint64_t prefixPages = (overlapStart - oldVa) / 0x1000ULL;
|
||||||
|
uint64_t protectedPages = (overlapEnd - overlapStart) / 0x1000ULL;
|
||||||
|
uint64_t tailPages = (areaEnd - overlapEnd) / 0x1000ULL;
|
||||||
|
|
||||||
|
if (prefixPages == 0) {
|
||||||
|
area->numPages = protectedPages;
|
||||||
|
area->prot = prot;
|
||||||
|
if (tailPages != 0)
|
||||||
|
area->next = new HeapAlloc { overlapEnd, tailPages,
|
||||||
|
oldProt,
|
||||||
|
area->allocationId,
|
||||||
|
oldNext };
|
||||||
|
} else {
|
||||||
|
area->numPages = prefixPages;
|
||||||
|
auto* middle = new HeapAlloc { overlapStart, protectedPages,
|
||||||
|
prot, area->allocationId,
|
||||||
|
oldNext };
|
||||||
|
area->next = middle;
|
||||||
|
if (tailPages != 0)
|
||||||
|
middle->next = new HeapAlloc { overlapEnd, tailPages,
|
||||||
|
oldProt,
|
||||||
|
area->allocationId,
|
||||||
|
oldNext };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
area = oldNext;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool writable = (prot & VmProtWrite) != 0;
|
||||||
|
bool executable = (prot & VmProtExec) != 0;
|
||||||
|
for (uint64_t i = 0; i < pages; i++) {
|
||||||
|
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys,
|
||||||
|
addr + i * 0x1000ULL) == 0)
|
||||||
|
continue;
|
||||||
|
if (!Memory::VMM::Paging::ProtectUserIn(proc->pml4Phys,
|
||||||
|
addr + i * 0x1000ULL,
|
||||||
|
writable, executable)) {
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
Ipc::ShootdownUserRange(proc->pml4Phys, addr, (uint32_t)pages);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve a non-present access inside an anonymous VMA. This is used for
|
||||||
|
// both ring-3 faults and kernel copies to untouched userspace buffers.
|
||||||
|
inline bool TryHandleAnonymousPageFault(uint64_t faultAddr, uint64_t errorCode) {
|
||||||
|
auto* proc = Sched::GetCurrentProcessPtr();
|
||||||
|
int slot = GetCurrentSlot();
|
||||||
|
if (proc == nullptr || slot < 0 || faultAddr < Sched::UserHeapBase ||
|
||||||
|
faultAddr >= Sched::UserHeapLimit || (errorCode & 1) != 0) return false;
|
||||||
|
|
||||||
|
uint64_t pageVa = faultAddr & ~0xFFFULL;
|
||||||
|
g_heapLocks[slot].Acquire();
|
||||||
|
HeapAlloc* found = nullptr;
|
||||||
|
for (HeapAlloc* area = g_heapAllocs[slot]; area != nullptr; area = area->next) {
|
||||||
|
uint64_t bytes = area->numPages * 0x1000ULL;
|
||||||
|
if (pageVa >= area->va && pageVa - area->va < bytes) {
|
||||||
|
found = area;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (found == nullptr ||
|
||||||
|
((errorCode & (1ULL << 1)) != 0 && (found->prot & VmProtWrite) == 0) ||
|
||||||
|
((errorCode & (1ULL << 4)) != 0 && (found->prot & VmProtExec) == 0)) {
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A sibling may have resolved the same fault while this CPU waited.
|
||||||
|
if (Memory::VMM::Paging::GetPhysAddr(proc->pml4Phys, pageVa) != 0) {
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void* page = Memory::g_pfa->AllocateZeroed();
|
||||||
|
if (page == nullptr) {
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
uint64_t phys = Memory::SubHHDM((uint64_t)page);
|
||||||
|
bool mapped = Memory::VMM::Paging::MapUserInPermissions(
|
||||||
|
proc->pml4Phys, phys, pageVa,
|
||||||
|
(found->prot & VmProtWrite) != 0, (found->prot & VmProtExec) != 0);
|
||||||
|
if (!mapped) Memory::g_pfa->Free(page);
|
||||||
|
else Sched::g_allocatedPages[slot]++;
|
||||||
|
g_heapLocks[slot].Release();
|
||||||
|
return mapped;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -118,6 +118,12 @@ namespace montauk::abi {
|
|||||||
case SYS_FREE:
|
case SYS_FREE:
|
||||||
Sys_Free(frame->arg1);
|
Sys_Free(frame->arg1);
|
||||||
return 0;
|
return 0;
|
||||||
|
case SYS_MMAP_ANON:
|
||||||
|
return (int64_t)Sys_MapAnonymous(frame->arg1, frame->arg2);
|
||||||
|
case SYS_MUNMAP:
|
||||||
|
return Sys_Unmap(frame->arg1, frame->arg2);
|
||||||
|
case SYS_MPROTECT:
|
||||||
|
return Sys_Protect(frame->arg1, frame->arg2, frame->arg3);
|
||||||
case SYS_GETTICKS:
|
case SYS_GETTICKS:
|
||||||
return (int64_t)Sys_GetTicks();
|
return (int64_t)Sys_GetTicks();
|
||||||
case SYS_GETMILLISECONDS:
|
case SYS_GETMILLISECONDS:
|
||||||
@@ -590,9 +596,9 @@ namespace montauk::abi {
|
|||||||
// ---- SYSCALL MSR initialization ----
|
// ---- SYSCALL MSR initialization ----
|
||||||
|
|
||||||
void InitializeSyscalls() {
|
void InitializeSyscalls() {
|
||||||
// Enable SYSCALL/SYSRET in EFER
|
// Enable SYSCALL/SYSRET and no-execute page permissions in EFER.
|
||||||
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
|
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
|
||||||
efer |= 1; // SCE bit (Syscall Enable)
|
efer |= 1 | (1ULL << 11); // SCE | NXE
|
||||||
Hal::WriteMSR(Hal::IA32_EFER, efer);
|
Hal::WriteMSR(Hal::IA32_EFER, efer);
|
||||||
|
|
||||||
// STAR: kernel CS in [47:32], sysret base in [63:48]
|
// STAR: kernel CS in [47:32], sysret base in [63:48]
|
||||||
@@ -608,7 +614,7 @@ namespace montauk::abi {
|
|||||||
Hal::WriteMSR(Hal::IA32_FMASK, 0x200);
|
Hal::WriteMSR(Hal::IA32_FMASK, 0x200);
|
||||||
|
|
||||||
Kt::KernelLogStream(Kt::OK, "Syscall") << "SYSCALL/SYSRET initialized (LSTAR="
|
Kt::KernelLogStream(Kt::OK, "Syscall") << "SYSCALL/SYSRET initialized (LSTAR="
|
||||||
<< kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 154 syscall slots)";
|
<< kcp::hex << (uint64_t)SyscallEntry << kcp::dec << ", 171 syscall slots)";
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -312,6 +312,9 @@ namespace montauk::abi {
|
|||||||
|
|
||||||
// Set path access/modification times. (path, atime, mtime, useCurrent)
|
// Set path access/modification times. (path, atime, mtime, useCurrent)
|
||||||
static constexpr uint64_t SYS_UTIME = 167;
|
static constexpr uint64_t SYS_UTIME = 167;
|
||||||
|
static constexpr uint64_t SYS_MMAP_ANON = 168;
|
||||||
|
static constexpr uint64_t SYS_MUNMAP = 169;
|
||||||
|
static constexpr uint64_t SYS_MPROTECT = 170;
|
||||||
|
|
||||||
// Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
|
// Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
|
||||||
static constexpr int SDR_PARAM_FREQ = 0; // center frequency, Hz
|
static constexpr int SDR_PARAM_FREQ = 0; // center frequency, Hz
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
|
|
||||||
#include <Sched/Scheduler.hpp>
|
#include <Sched/Scheduler.hpp>
|
||||||
#include <Memory/Paging.hpp>
|
#include <Memory/Paging.hpp>
|
||||||
|
#include <Api/Heap.hpp>
|
||||||
|
|
||||||
namespace montauk::abi::UserMemory {
|
namespace montauk::abi::UserMemory {
|
||||||
|
|
||||||
@@ -32,6 +33,21 @@ namespace montauk::abi::UserMemory {
|
|||||||
uint64_t pml4Phys = CurrentPml4();
|
uint64_t pml4Phys = CurrentPml4();
|
||||||
if (pml4Phys == 0) return false;
|
if (pml4Phys == 0) return false;
|
||||||
|
|
||||||
|
if (Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite))
|
||||||
|
return true;
|
||||||
|
|
||||||
|
// Anonymous VMAs are demand-committed. Validation precedes the actual
|
||||||
|
// kernel copy, so materialize missing pages here with the same access
|
||||||
|
// type that the copy will perform.
|
||||||
|
uint64_t page = addr & ~0xFFFULL;
|
||||||
|
uint64_t last = (addr + size - 1) & ~0xFFFULL;
|
||||||
|
for (;;) {
|
||||||
|
if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, page, 1, requireWrite) &&
|
||||||
|
!TryHandleAnonymousPageFault(page, requireWrite ? 2ULL : 0ULL))
|
||||||
|
return false;
|
||||||
|
if (page == last) break;
|
||||||
|
page += 0x1000ULL;
|
||||||
|
}
|
||||||
return Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite);
|
return Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr, size, requireWrite);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,7 +74,7 @@ namespace montauk::abi::UserMemory {
|
|||||||
const char* str = (const char*)addr;
|
const char* str = (const char*)addr;
|
||||||
for (uint64_t i = 0; i < maxLen; i++) {
|
for (uint64_t i = 0; i < maxLen; i++) {
|
||||||
if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr + i, 1, false)) {
|
if (!Memory::VMM::Paging::IsUserRangeAccessible(pml4Phys, addr + i, 1, false)) {
|
||||||
return false;
|
if (!TryHandleAnonymousPageFault(addr + i, 0)) return false;
|
||||||
}
|
}
|
||||||
if (str[i] == '\0') return true;
|
if (str[i] == '\0') return true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
#include <Memory/PageFrameAllocator.hpp>
|
#include <Memory/PageFrameAllocator.hpp>
|
||||||
#include <Sched/Scheduler.hpp>
|
#include <Sched/Scheduler.hpp>
|
||||||
#include <Sched/CrashReport.hpp>
|
#include <Sched/CrashReport.hpp>
|
||||||
|
#include <Api/Heap.hpp>
|
||||||
#include <Hal/SmpBoot.hpp>
|
#include <Hal/SmpBoot.hpp>
|
||||||
#include <Timekeeping/ApicTimer.hpp>
|
#include <Timekeeping/ApicTimer.hpp>
|
||||||
|
|
||||||
@@ -201,11 +202,13 @@ namespace Hal {
|
|||||||
// pushes past the mapped stack and kernel accesses to not-yet-grown
|
// pushes past the mapped stack and kernel accesses to not-yet-grown
|
||||||
// user stack buffers passed into syscalls.
|
// user stack buffers passed into syscalls.
|
||||||
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
|
if ((errorCode & 1) == 0 && cpu != nullptr && cpu->currentSlot >= 0
|
||||||
&& Sched::GetCurrentPid() >= 0
|
&& Sched::GetCurrentPid() >= 0) {
|
||||||
&& Sched::TryGrowUserStack(cr2)) {
|
if (montauk::abi::TryHandleAnonymousPageFault(cr2, errorCode) ||
|
||||||
|
Sched::TryGrowUserStack(cr2)) {
|
||||||
if (fromUser) asm volatile("swapgs");
|
if (fromUser) asm volatile("swapgs");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Not a growable fault. Hand the RAW frame (error code at offset 0)
|
// Not a growable fault. Hand the RAW frame (error code at offset 0)
|
||||||
// to the fatal path, which re-derives fromUser and swaps GS itself.
|
// to the fatal path, which re-derives fromUser and swaps GS itself.
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ namespace Smp {
|
|||||||
|
|
||||||
// --- Program SYSCALL MSRs ---
|
// --- Program SYSCALL MSRs ---
|
||||||
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
|
uint64_t efer = Hal::ReadMSR(Hal::IA32_EFER);
|
||||||
efer |= 1; // SCE
|
efer |= 1 | (1ULL << 11); // SCE | NXE
|
||||||
Hal::WriteMSR(Hal::IA32_EFER, efer);
|
Hal::WriteMSR(Hal::IA32_EFER, efer);
|
||||||
|
|
||||||
uint64_t star = (0x0010ULL << 48) | (0x0008ULL << 32);
|
uint64_t star = (0x0010ULL << 48) | (0x0008ULL << 32);
|
||||||
|
|||||||
@@ -13,13 +13,21 @@ namespace Memory::VMM {
|
|||||||
static constexpr uint64_t LeafUser = 1ULL << 2;
|
static constexpr uint64_t LeafUser = 1ULL << 2;
|
||||||
static constexpr uint64_t LeafWriteThrough = 1ULL << 3;
|
static constexpr uint64_t LeafWriteThrough = 1ULL << 3;
|
||||||
static constexpr uint64_t LeafCacheDisabled = 1ULL << 4;
|
static constexpr uint64_t LeafCacheDisabled = 1ULL << 4;
|
||||||
|
static constexpr uint64_t LeafNoExecute = 1ULL << 63;
|
||||||
|
// PageTableEntry's historical 52-bit Address bitfield also spans the
|
||||||
|
// architectural high flag bits. Mask those bits whenever extracting a
|
||||||
|
// physical address, in particular now that anonymous leaves use NX.
|
||||||
|
static constexpr uint64_t kPhysAddrMask = 0xFFFFFFFFFFULL; // PTE bits 12..51
|
||||||
|
|
||||||
static inline void SetLeafPte(PageTableEntry* entry, uint64_t physicalAddress,
|
static inline void SetLeafPte(PageTableEntry* entry, uint64_t physicalAddress,
|
||||||
bool user, bool writeThrough, bool cacheDisabled) {
|
bool user, bool writeThrough, bool cacheDisabled,
|
||||||
uint64_t flags = LeafPresent | LeafWritable;
|
bool writable = true, bool executable = true) {
|
||||||
|
uint64_t flags = LeafPresent;
|
||||||
|
if (writable) flags |= LeafWritable;
|
||||||
if (user) flags |= LeafUser;
|
if (user) flags |= LeafUser;
|
||||||
if (writeThrough) flags |= LeafWriteThrough;
|
if (writeThrough) flags |= LeafWriteThrough;
|
||||||
if (cacheDisabled) flags |= LeafCacheDisabled;
|
if (cacheDisabled) flags |= LeafCacheDisabled;
|
||||||
|
if (!executable) flags |= LeafNoExecute;
|
||||||
|
|
||||||
// Replace the complete PTE in one aligned store. Updating individual
|
// Replace the complete PTE in one aligned store. Updating individual
|
||||||
// bitfields left old PWT/PCD/PAT/accessed state behind when a virtual
|
// bitfields left old PWT/PCD/PAT/accessed state behind when a virtual
|
||||||
@@ -230,6 +238,12 @@ namespace Memory::VMM {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool Paging::MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
bool Paging::MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||||
|
return MapUserInPermissions(pml4Phys, physicalAddress, virtualAddress, true, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Paging::MapUserInPermissions(std::uint64_t pml4Phys, std::uint64_t physicalAddress,
|
||||||
|
std::uint64_t virtualAddress, bool writable,
|
||||||
|
bool executable) {
|
||||||
pagingLock.Acquire();
|
pagingLock.Acquire();
|
||||||
if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
|
if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
|
||||||
pagingLock.Release();
|
pagingLock.Release();
|
||||||
@@ -266,7 +280,43 @@ namespace Memory::VMM {
|
|||||||
if (!pml1) { pagingLock.Release(); return false; }
|
if (!pml1) { pagingLock.Release(); return false; }
|
||||||
|
|
||||||
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
|
PageTableEntry* pageEntry = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
|
||||||
SetLeafPte(pageEntry, physicalAddress, true, false, false);
|
SetLeafPte(pageEntry, physicalAddress, true, false, false, writable, executable);
|
||||||
|
pagingLock.Release();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Paging::ProtectUserIn(std::uint64_t pml4Phys, std::uint64_t virtualAddress,
|
||||||
|
bool writable, bool executable) {
|
||||||
|
if ((virtualAddress & 0xFFFULL) != 0) return false;
|
||||||
|
pagingLock.Acquire();
|
||||||
|
VirtualAddress va(virtualAddress);
|
||||||
|
|
||||||
|
auto walkRead = [](PageTable* table, uint64_t index) -> PageTable* {
|
||||||
|
PageTableEntry* entry = (PageTableEntry*)Memory::HHDM(&table->entries[index]);
|
||||||
|
if (!entry->Present || !entry->Supervisor || entry->LargerPages) return nullptr;
|
||||||
|
return (PageTable*)(entry->Address << 12);
|
||||||
|
};
|
||||||
|
|
||||||
|
PageTable* pml4 = (PageTable*)pml4Phys;
|
||||||
|
auto pml3 = walkRead(pml4, va.GetL4Index());
|
||||||
|
if (!pml3) { pagingLock.Release(); return false; }
|
||||||
|
auto pml2 = walkRead(pml3, va.GetL3Index());
|
||||||
|
if (!pml2) { pagingLock.Release(); return false; }
|
||||||
|
auto pml1 = walkRead(pml2, va.GetL2Index());
|
||||||
|
if (!pml1) { pagingLock.Release(); return false; }
|
||||||
|
|
||||||
|
PageTableEntry* pte = (PageTableEntry*)Memory::HHDM(&pml1->entries[va.GetPageIndex()]);
|
||||||
|
uint64_t raw = *(volatile uint64_t*)pte;
|
||||||
|
if ((raw & LeafPresent) == 0 || (raw & LeafUser) == 0) {
|
||||||
|
pagingLock.Release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (writable) raw |= LeafWritable;
|
||||||
|
else raw &= ~LeafWritable;
|
||||||
|
if (executable) raw &= ~LeafNoExecute;
|
||||||
|
else raw |= LeafNoExecute;
|
||||||
|
*(volatile uint64_t*)pte = raw;
|
||||||
|
asm volatile("invlpg (%0)" :: "r"(virtualAddress) : "memory");
|
||||||
pagingLock.Release();
|
pagingLock.Release();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -376,7 +426,8 @@ namespace Memory::VMM {
|
|||||||
// Skip MMIO/WC pages (not PFA-managed)
|
// Skip MMIO/WC pages (not PFA-managed)
|
||||||
if (pte->WriteThrough || pte->CacheDisabled) continue;
|
if (pte->WriteThrough || pte->CacheDisabled) continue;
|
||||||
|
|
||||||
uint64_t pagePhys = (uint64_t)pte->Address << 12;
|
uint64_t pagePhys =
|
||||||
|
(uint64_t)(pte->Address & kPhysAddrMask) << 12;
|
||||||
if (pagePhys != 0) {
|
if (pagePhys != 0) {
|
||||||
Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
|
Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
|
||||||
}
|
}
|
||||||
@@ -442,11 +493,6 @@ namespace Memory::VMM {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mask to extract only the 40-bit physical address from a PTE Address field
|
|
||||||
// (bits 12-51 of the PTE). The PageTableEntry::Address field is 52 bits wide
|
|
||||||
// and includes NX, PK, and software-available bits that must be stripped.
|
|
||||||
static constexpr uint64_t kPhysAddrMask = 0xFFFFFFFFFFULL; // 40 bits
|
|
||||||
|
|
||||||
std::uint64_t Paging::GetPhysAddr(std::uint64_t pml4, std::uint64_t virtualAddress, bool use40BitL1) {
|
std::uint64_t Paging::GetPhysAddr(std::uint64_t pml4, std::uint64_t virtualAddress, bool use40BitL1) {
|
||||||
VirtualAddress virtualAddressObj(virtualAddress);
|
VirtualAddress virtualAddressObj(virtualAddress);
|
||||||
|
|
||||||
|
|||||||
@@ -104,6 +104,17 @@ public:
|
|||||||
// Map a page into an arbitrary PML4 (specified by physical address) with User bit set.
|
// Map a page into an arbitrary PML4 (specified by physical address) with User bit set.
|
||||||
static bool MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
|
static bool MapUserIn(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
|
||||||
|
|
||||||
|
// Map ordinary user memory with explicit leaf permissions. Read access
|
||||||
|
// is implied by x86 paging; writable and executable are independent.
|
||||||
|
static bool MapUserInPermissions(std::uint64_t pml4Phys,
|
||||||
|
std::uint64_t physicalAddress,
|
||||||
|
std::uint64_t virtualAddress,
|
||||||
|
bool writable, bool executable);
|
||||||
|
|
||||||
|
// Change leaf permissions on an existing user page.
|
||||||
|
static bool ProtectUserIn(std::uint64_t pml4Phys, std::uint64_t virtualAddress,
|
||||||
|
bool writable, bool executable);
|
||||||
|
|
||||||
// Map a page into an arbitrary PML4 with User + Write-Combining attributes.
|
// Map a page into an arbitrary PML4 with User + Write-Combining attributes.
|
||||||
static bool MapUserInWC(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
|
static bool MapUserInWC(std::uint64_t pml4Phys, std::uint64_t physicalAddress, std::uint64_t virtualAddress);
|
||||||
|
|
||||||
|
|||||||
@@ -493,6 +493,8 @@ namespace Sched {
|
|||||||
proc.kernelStackTop = kernelStackTop;
|
proc.kernelStackTop = kernelStackTop;
|
||||||
proc.userStackTop = UserStackTop - 8;
|
proc.userStackTop = UserStackTop - 8;
|
||||||
proc.heapNext = UserHeapBase;
|
proc.heapNext = UserHeapBase;
|
||||||
|
for (uint64_t i = 0; i < UserHeapBitmapWords; i++)
|
||||||
|
g_userHeapPageMap[slot][i] = 0;
|
||||||
proc.fsBase = tls.fsBase;
|
proc.fsBase = tls.fsBase;
|
||||||
proc.tlsTemplateVaddr = tls.templateVaddr;
|
proc.tlsTemplateVaddr = tls.templateVaddr;
|
||||||
proc.tlsFileSize = tls.fileSize;
|
proc.tlsFileSize = tls.fileSize;
|
||||||
@@ -651,6 +653,7 @@ namespace Sched {
|
|||||||
}
|
}
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
|
Ipc::UnmapAndFreeUserRange(sharedPml4, base, mappedPages);
|
||||||
|
ReleaseUserHeapRange(primarySlot_, base, numPages * 0x1000ULL);
|
||||||
Kt::KernelLogStream(Kt::ERROR, "Sched")
|
Kt::KernelLogStream(Kt::ERROR, "Sched")
|
||||||
<< "Thread TLS allocation failed";
|
<< "Thread TLS allocation failed";
|
||||||
return -1;
|
return -1;
|
||||||
@@ -675,8 +678,11 @@ namespace Sched {
|
|||||||
// Allocate kernel stack.
|
// Allocate kernel stack.
|
||||||
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
|
void* stackMem = Memory::g_pfa->ReallocConsecutive(nullptr, StackPages);
|
||||||
if (stackMem == nullptr) {
|
if (stackMem == nullptr) {
|
||||||
if (threadTlsPages != 0)
|
if (threadTlsPages != 0) {
|
||||||
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
|
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
|
||||||
|
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
|
||||||
|
threadTlsPages * 0x1000ULL);
|
||||||
|
}
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
memset(stackMem, 0, StackSize);
|
memset(stackMem, 0, StackSize);
|
||||||
@@ -698,8 +704,11 @@ namespace Sched {
|
|||||||
if (slot < 0) {
|
if (slot < 0) {
|
||||||
schedLock.Release();
|
schedLock.Release();
|
||||||
Memory::g_pfa->Free(stackMem, StackPages);
|
Memory::g_pfa->Free(stackMem, StackPages);
|
||||||
if (threadTlsPages != 0)
|
if (threadTlsPages != 0) {
|
||||||
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
|
Ipc::UnmapAndFreeUserRange(sharedPml4, threadTlsBase, threadTlsPages);
|
||||||
|
ReleaseUserHeapRange(primarySlot_, threadTlsBase,
|
||||||
|
threadTlsPages * 0x1000ULL);
|
||||||
|
}
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -774,20 +783,54 @@ namespace Sched {
|
|||||||
|
|
||||||
schedLock.Acquire();
|
schedLock.Acquire();
|
||||||
Process& primary = processTable[primarySlot_];
|
Process& primary = processTable[primarySlot_];
|
||||||
uint64_t base = primary.heapNext;
|
|
||||||
bool validOwner = primary.primarySlot == primarySlot_ &&
|
bool validOwner = primary.primarySlot == primarySlot_ &&
|
||||||
primary.state != ProcessState::Free &&
|
primary.state != ProcessState::Free &&
|
||||||
primary.state != ProcessState::Terminated;
|
primary.state != ProcessState::Terminated;
|
||||||
bool fits = validOwner && base >= UserHeapBase &&
|
bool fits = false;
|
||||||
base <= UserHeapLimit && size <= UserHeapLimit - base;
|
uint64_t pages = size / 0x1000ULL;
|
||||||
if (fits) {
|
uint64_t runStart = 0;
|
||||||
primary.heapNext = base + size;
|
uint64_t runLength = 0;
|
||||||
outVa = base;
|
|
||||||
|
if (validOwner && pages <= UserHeapPages) {
|
||||||
|
for (uint64_t page = 0; page < UserHeapPages; page++) {
|
||||||
|
bool used = (g_userHeapPageMap[primarySlot_][page / 64]
|
||||||
|
& (1ULL << (page % 64))) != 0;
|
||||||
|
if (used) {
|
||||||
|
runLength = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (runLength == 0) runStart = page;
|
||||||
|
runLength++;
|
||||||
|
if (runLength == pages) {
|
||||||
|
for (uint64_t p = runStart; p < runStart + pages; p++)
|
||||||
|
g_userHeapPageMap[primarySlot_][p / 64] |= 1ULL << (p % 64);
|
||||||
|
outVa = UserHeapBase + runStart * 0x1000ULL;
|
||||||
|
uint64_t end = outVa + size;
|
||||||
|
if (end > primary.heapNext) primary.heapNext = end;
|
||||||
|
fits = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
schedLock.Release();
|
schedLock.Release();
|
||||||
return fits;
|
return fits;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ReleaseUserHeapRange(int primarySlot_, uint64_t va, uint64_t size) {
|
||||||
|
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses || size == 0 ||
|
||||||
|
(va & 0xFFFULL) != 0 || (size & 0xFFFULL) != 0 ||
|
||||||
|
va < UserHeapBase || va > UserHeapLimit || size > UserHeapLimit - va) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t first = (va - UserHeapBase) / 0x1000ULL;
|
||||||
|
uint64_t pages = size / 0x1000ULL;
|
||||||
|
schedLock.Acquire();
|
||||||
|
for (uint64_t p = first; p < first + pages; p++)
|
||||||
|
g_userHeapPageMap[primarySlot_][p / 64] &= ~(1ULL << (p % 64));
|
||||||
|
schedLock.Release();
|
||||||
|
}
|
||||||
|
|
||||||
static void FreeSiblingThreadTls(Process& thr) {
|
static void FreeSiblingThreadTls(Process& thr) {
|
||||||
int primarySlot_ = thr.primarySlot;
|
int primarySlot_ = thr.primarySlot;
|
||||||
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
|
if (primarySlot_ < 0 || primarySlot_ >= MaxProcesses ||
|
||||||
@@ -811,6 +854,7 @@ namespace Sched {
|
|||||||
uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000;
|
uint64_t pages = (blockSize + 16 + 0xFFF) / 0x1000;
|
||||||
thr.fsBase = 0;
|
thr.fsBase = 0;
|
||||||
Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
|
Ipc::UnmapAndFreeUserRange(primary.pml4Phys, base, pages);
|
||||||
|
ReleaseUserHeapRange(primarySlot_, base, pages * 0x1000ULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Switch away from a slot we have just marked non-runnable while holding
|
// Switch away from a slot we have just marked non-runnable while holding
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ namespace Sched {
|
|||||||
// failed bound check is recoverable (SYS_ALLOC returns null); allowing the
|
// failed bound check is recoverable (SYS_ALLOC returns null); allowing the
|
||||||
// heap to cross this boundary would silently replace library PTEs.
|
// heap to cross this boundary would silently replace library PTEs.
|
||||||
static constexpr uint64_t UserHeapLimit = 0x60000000ULL;
|
static constexpr uint64_t UserHeapLimit = 0x60000000ULL;
|
||||||
|
static constexpr uint64_t UserHeapPages =
|
||||||
|
(UserHeapLimit - UserHeapBase) / 0x1000ULL;
|
||||||
|
static constexpr uint64_t UserHeapBitmapWords = (UserHeapPages + 63) / 64;
|
||||||
// Surface mappings use reusable, fixed-size per-process slots instead of
|
// Surface mappings use reusable, fixed-size per-process slots instead of
|
||||||
// consuming heap VA forever. 64 * 32 MiB occupies [0x7000000000,
|
// consuming heap VA forever. 64 * 32 MiB occupies [0x7000000000,
|
||||||
// 0x7080000000), comfortably below the main user stack.
|
// 0x7080000000), comfortably below the main user stack.
|
||||||
@@ -191,6 +194,7 @@ namespace Sched {
|
|||||||
// reservation is serialized across sibling threads and bounded below the
|
// reservation is serialized across sibling threads and bounded below the
|
||||||
// fixed shared-library mappings.
|
// fixed shared-library mappings.
|
||||||
bool ReserveUserHeapRange(int primarySlot, uint64_t size, uint64_t& outVa);
|
bool ReserveUserHeapRange(int primarySlot, uint64_t size, uint64_t& outVa);
|
||||||
|
void ReleaseUserHeapRange(int primarySlot, uint64_t va, uint64_t size);
|
||||||
|
|
||||||
// Terminate the currently executing thread. If this is the main thread,
|
// Terminate the currently executing thread. If this is the main thread,
|
||||||
// the entire process exits (equivalent to ExitProcess).
|
// the entire process exits (equivalent to ExitProcess).
|
||||||
@@ -258,5 +262,8 @@ namespace Sched {
|
|||||||
|
|
||||||
// Per-process allocated page count (tracked by Heap syscalls, separate from Process struct)
|
// Per-process allocated page count (tracked by Heap syscalls, separate from Process struct)
|
||||||
inline uint64_t g_allocatedPages[MaxProcesses] = {};
|
inline uint64_t g_allocatedPages[MaxProcesses] = {};
|
||||||
|
// One bit per page in the bounded userspace heap. Unlike heapNext, this
|
||||||
|
// makes virtual ranges reusable after unmap and failed reservations.
|
||||||
|
inline uint64_t g_userHeapPageMap[MaxProcesses][UserHeapBitmapWords] = {};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -229,6 +229,10 @@ namespace montauk::abi {
|
|||||||
static constexpr uint64_t SYS_WIFI_CONNECT_ASYNC = 164; // (ssid, password) -> 0 accepted, <0 on error
|
static constexpr uint64_t SYS_WIFI_CONNECT_ASYNC = 164; // (ssid, password) -> 0 accepted, <0 on error
|
||||||
static constexpr uint64_t SYS_NETIFS = 165; // (NetIfInfo*, maxCount) -> count
|
static constexpr uint64_t SYS_NETIFS = 165; // (NetIfInfo*, maxCount) -> count
|
||||||
static constexpr uint64_t SYS_GETCHAR_NB = 166; // () -> ascii, 0 if nothing pending; never blocks
|
static constexpr uint64_t SYS_GETCHAR_NB = 166; // () -> ascii, 0 if nothing pending; never blocks
|
||||||
|
static constexpr uint64_t SYS_UTIME = 167;
|
||||||
|
static constexpr uint64_t SYS_MMAP_ANON = 168;
|
||||||
|
static constexpr uint64_t SYS_MUNMAP = 169;
|
||||||
|
static constexpr uint64_t SYS_MPROTECT = 170;
|
||||||
|
|
||||||
// Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
|
// Tunable parameters (for SYS_SDR_SETPARAM / SYS_SDR_GETPARAM).
|
||||||
static constexpr int SDR_PARAM_FREQ = 0; // center frequency, Hz
|
static constexpr int SDR_PARAM_FREQ = 0; // center frequency, Hz
|
||||||
|
|||||||
@@ -190,6 +190,10 @@ extern "C" {
|
|||||||
#define MTK_SYS_WIFI_CONNECT_ASYNC 164
|
#define MTK_SYS_WIFI_CONNECT_ASYNC 164
|
||||||
#define MTK_SYS_NETIFS 165
|
#define MTK_SYS_NETIFS 165
|
||||||
#define MTK_SYS_GETCHAR_NB 166
|
#define MTK_SYS_GETCHAR_NB 166
|
||||||
|
#define MTK_SYS_UTIME 167
|
||||||
|
#define MTK_SYS_MMAP_ANON 168
|
||||||
|
#define MTK_SYS_MUNMAP 169
|
||||||
|
#define MTK_SYS_MPROTECT 170
|
||||||
/* @SYSCALLS-END */
|
/* @SYSCALLS-END */
|
||||||
|
|
||||||
#define MTK_SOCK_TCP 1
|
#define MTK_SOCK_TCP 1
|
||||||
|
|||||||
@@ -10,10 +10,8 @@ extern "C" {
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Anonymous-memory mmap over SYS_ALLOC. SYS_ALLOC returns
|
* Demand-paged anonymous memory backed by the kernel's VMA subsystem.
|
||||||
* page-aligned process memory, which is exactly what callers like
|
* File-backed and shared mappings are not supported and fail cleanly.
|
||||||
* GCC's page allocator need. File-backed mappings are not
|
|
||||||
* supported and fail with ENODEV.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#define PROT_NONE 0
|
#define PROT_NONE 0
|
||||||
|
|||||||
+19
-270
@@ -1,290 +1,39 @@
|
|||||||
/*
|
/*
|
||||||
* heap.h
|
* heap.h
|
||||||
* Userspace heap allocator for MontaukOS programs
|
* Unified userspace heap API for MontaukOS programs
|
||||||
* Copyright (c) 2025 Daniel Hammer
|
* Copyright (c) 2025-2026 Daniel Hammer
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <montauk/syscall.h>
|
#include <cstddef>
|
||||||
#include <montauk/string.h>
|
#include <cstdint>
|
||||||
|
|
||||||
|
// The allocator lives in libc. Keeping these declarations here lets
|
||||||
|
// freestanding C++ programs use the Montauk API without pulling in all of
|
||||||
|
// <stdlib.h>, while ensuring C, C++, and libraries share one heap.
|
||||||
|
extern "C" {
|
||||||
|
void* malloc(std::size_t size);
|
||||||
|
void free(void* ptr);
|
||||||
|
void* realloc(void* ptr, std::size_t size);
|
||||||
|
void* calloc(std::size_t count, std::size_t size);
|
||||||
|
}
|
||||||
|
|
||||||
namespace montauk {
|
namespace montauk {
|
||||||
namespace heap_detail {
|
|
||||||
|
|
||||||
static constexpr uint64_t HEADER_MAGIC = 0x5A484541; // "ZHEA"
|
|
||||||
static constexpr uint64_t FREED_MAGIC = 0xDEADFEEE;
|
|
||||||
|
|
||||||
struct Header {
|
|
||||||
uint64_t magic;
|
|
||||||
uint64_t size; // user-requested size
|
|
||||||
} __attribute__((packed));
|
|
||||||
|
|
||||||
struct FreeNode {
|
|
||||||
uint64_t size; // total size of this free block (including node)
|
|
||||||
FreeNode* next;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Segregated free lists: power-of-2 size classes for blocks <= 4096 bytes.
|
|
||||||
// Blocks larger than 4096 go to the overflow list.
|
|
||||||
static constexpr int NUM_BUCKETS = 8;
|
|
||||||
static constexpr uint64_t BUCKET_SIZES[NUM_BUCKETS] = {
|
|
||||||
32, 64, 128, 256, 512, 1024, 2048, 4096
|
|
||||||
};
|
|
||||||
|
|
||||||
// Per-process heap state — must be `inline` (not `static`) so that all
|
|
||||||
// translation units in a multi-TU program share a single heap.
|
|
||||||
inline FreeNode* g_buckets[NUM_BUCKETS] = {};
|
|
||||||
inline FreeNode g_overflow{0, nullptr};
|
|
||||||
inline bool g_initialized = false;
|
|
||||||
|
|
||||||
// Process-wide heap lock. Userspace threads share the heap, so the
|
|
||||||
// public malloc/mfree/realloc entry points must serialize access to
|
|
||||||
// g_buckets/g_overflow. Kept inline here (not in thread.h) because
|
|
||||||
// thread.h depends on heap.h, and the internal helpers below are not
|
|
||||||
// reentrant into the public API, so a plain spinlock suffices.
|
|
||||||
inline volatile uint32_t g_heap_lock = 0;
|
|
||||||
|
|
||||||
static inline void heap_lock_acquire() {
|
|
||||||
while (__atomic_exchange_n(&g_heap_lock, 1, __ATOMIC_ACQUIRE) != 0) {
|
|
||||||
syscall0(montauk::abi::SYS_YIELD);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
static inline void heap_lock_release() {
|
|
||||||
__atomic_store_n(&g_heap_lock, 0, __ATOMIC_RELEASE);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline Header* get_header(void* block) {
|
|
||||||
return (Header*)((uint8_t*)block - sizeof(Header));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine which bucket a block size belongs to, or -1 for overflow
|
|
||||||
static inline int bucket_index(uint64_t blockSize) {
|
|
||||||
if (blockSize <= 32) return 0;
|
|
||||||
if (blockSize <= 64) return 1;
|
|
||||||
if (blockSize <= 128) return 2;
|
|
||||||
if (blockSize <= 256) return 3;
|
|
||||||
if (blockSize <= 512) return 4;
|
|
||||||
if (blockSize <= 1024) return 5;
|
|
||||||
if (blockSize <= 2048) return 6;
|
|
||||||
if (blockSize <= 4096) return 7;
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Insert into overflow list (sorted by address, with adjacent-block coalescing)
|
|
||||||
static inline void insert_overflow(void* ptr, uint64_t size) {
|
|
||||||
auto* node = (FreeNode*)ptr;
|
|
||||||
node->size = size;
|
|
||||||
|
|
||||||
FreeNode* prev = &g_overflow;
|
|
||||||
FreeNode* cur = g_overflow.next;
|
|
||||||
while (cur != nullptr && cur < node) {
|
|
||||||
prev = cur;
|
|
||||||
cur = cur->next;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool merged_prev = false;
|
|
||||||
if (prev != &g_overflow &&
|
|
||||||
(uint8_t*)prev + prev->size == (uint8_t*)node) {
|
|
||||||
prev->size += size;
|
|
||||||
node = prev;
|
|
||||||
merged_prev = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cur != nullptr &&
|
|
||||||
(uint8_t*)node + node->size == (uint8_t*)cur) {
|
|
||||||
node->size += cur->size;
|
|
||||||
node->next = cur->next;
|
|
||||||
if (!merged_prev) prev->next = node;
|
|
||||||
} else if (!merged_prev) {
|
|
||||||
node->next = cur;
|
|
||||||
prev->next = node;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Take a block of at least `needed` bytes from the overflow list.
|
|
||||||
// Splits remainder back into overflow if worthwhile.
|
|
||||||
static inline void* take_from_overflow(uint64_t needed) {
|
|
||||||
FreeNode* prev = &g_overflow;
|
|
||||||
FreeNode* cur = g_overflow.next;
|
|
||||||
|
|
||||||
while (cur != nullptr) {
|
|
||||||
if (cur->size >= needed) {
|
|
||||||
uint64_t blockSize = cur->size;
|
|
||||||
prev->next = cur->next;
|
|
||||||
|
|
||||||
if (blockSize > needed + sizeof(FreeNode) + 16) {
|
|
||||||
insert_overflow((uint8_t*)cur + needed, blockSize - needed);
|
|
||||||
}
|
|
||||||
return (void*)cur;
|
|
||||||
}
|
|
||||||
prev = cur;
|
|
||||||
cur = cur->next;
|
|
||||||
}
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next slab size for heap growth. The kernel tracks a finite number
|
|
||||||
// of SYS_ALLOC records per process (MaxHeapAllocs), so growing once
|
|
||||||
// per large allocation exhausts them under allocation-heavy loads
|
|
||||||
// (the native ld ran out mid-link). Doubling slabs keep the syscall
|
|
||||||
// count logarithmic in total heap size.
|
|
||||||
inline uint64_t g_grow_slab = 16 * 0x1000;
|
|
||||||
|
|
||||||
static inline bool grow(uint64_t bytes) {
|
|
||||||
uint64_t want = (bytes + 0xFFF) & ~0xFFFULL;
|
|
||||||
if (want < 0x4000) want = 0x4000;
|
|
||||||
|
|
||||||
uint64_t slab = (want > g_grow_slab) ? want : g_grow_slab;
|
|
||||||
if (g_grow_slab < 4 * 1024 * 1024) g_grow_slab *= 2;
|
|
||||||
|
|
||||||
void* mem = montauk::alloc(slab);
|
|
||||||
if (mem == nullptr && slab > want) {
|
|
||||||
// Big slab refused (low memory): retry with the exact need.
|
|
||||||
slab = want;
|
|
||||||
mem = montauk::alloc(slab);
|
|
||||||
}
|
|
||||||
if (mem == nullptr) return false;
|
|
||||||
insert_overflow(mem, slab);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Refill a small-block bucket by carving a page-sized chunk from overflow
|
|
||||||
static inline bool refill_bucket(int idx) {
|
|
||||||
uint64_t bsize = BUCKET_SIZES[idx];
|
|
||||||
uint64_t chunk = (bsize < 4096) ? 4096 : bsize;
|
|
||||||
|
|
||||||
void* block = take_from_overflow(chunk);
|
|
||||||
if (block == nullptr) {
|
|
||||||
if (!grow(chunk)) return false;
|
|
||||||
block = take_from_overflow(chunk);
|
|
||||||
if (block == nullptr) return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t count = chunk / bsize;
|
|
||||||
for (uint64_t i = 0; i < count; i++) {
|
|
||||||
auto* node = (FreeNode*)((uint8_t*)block + i * bsize);
|
|
||||||
node->size = bsize;
|
|
||||||
node->next = g_buckets[idx];
|
|
||||||
g_buckets[idx] = node;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace heap_detail
|
|
||||||
|
|
||||||
// ---- Public API ----
|
|
||||||
|
|
||||||
inline void* malloc(uint64_t size) {
|
inline void* malloc(uint64_t size) {
|
||||||
using namespace heap_detail;
|
return ::malloc((std::size_t)size);
|
||||||
|
|
||||||
// Guard against overflow: size + Header must not wrap
|
|
||||||
if (size > UINT64_MAX - sizeof(Header) - 15)
|
|
||||||
return nullptr;
|
|
||||||
|
|
||||||
heap_lock_acquire();
|
|
||||||
|
|
||||||
if (!g_initialized) {
|
|
||||||
grow(16 * 0x1000); // seed with 64 KiB
|
|
||||||
g_initialized = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t needed = size + sizeof(Header);
|
|
||||||
needed = (needed + 15) & ~15ULL;
|
|
||||||
|
|
||||||
int idx = bucket_index(needed);
|
|
||||||
|
|
||||||
if (idx >= 0) {
|
|
||||||
// Small allocation — use segregated bucket (O(1))
|
|
||||||
if (g_buckets[idx] == nullptr && !refill_bucket(idx)) {
|
|
||||||
heap_lock_release();
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
FreeNode* node = g_buckets[idx];
|
|
||||||
g_buckets[idx] = node->next;
|
|
||||||
|
|
||||||
Header* header = (Header*)node;
|
|
||||||
header->magic = HEADER_MAGIC;
|
|
||||||
header->size = size;
|
|
||||||
heap_lock_release();
|
|
||||||
return (void*)((uint8_t*)header + sizeof(Header));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Large allocation — search overflow list
|
|
||||||
void* block = take_from_overflow(needed);
|
|
||||||
if (block == nullptr) {
|
|
||||||
if (!grow(needed)) { heap_lock_release(); return nullptr; }
|
|
||||||
block = take_from_overflow(needed);
|
|
||||||
if (block == nullptr) { heap_lock_release(); return nullptr; }
|
|
||||||
}
|
|
||||||
|
|
||||||
Header* header = (Header*)block;
|
|
||||||
header->magic = HEADER_MAGIC;
|
|
||||||
header->size = size;
|
|
||||||
heap_lock_release();
|
|
||||||
return (void*)((uint8_t*)header + sizeof(Header));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void mfree(void* ptr) {
|
inline void mfree(void* ptr) {
|
||||||
using namespace heap_detail;
|
::free(ptr);
|
||||||
|
|
||||||
if (ptr == nullptr) return;
|
|
||||||
|
|
||||||
Header* header = get_header(ptr);
|
|
||||||
|
|
||||||
heap_lock_acquire();
|
|
||||||
|
|
||||||
if (header->magic == FREED_MAGIC) { heap_lock_release(); return; } // double-free
|
|
||||||
if (header->magic != HEADER_MAGIC) { heap_lock_release(); return; } // corrupt
|
|
||||||
header->magic = FREED_MAGIC;
|
|
||||||
|
|
||||||
uint64_t blockSize = header->size + sizeof(Header);
|
|
||||||
blockSize = (blockSize + 15) & ~15ULL;
|
|
||||||
|
|
||||||
int idx = bucket_index(blockSize);
|
|
||||||
|
|
||||||
if (idx >= 0) {
|
|
||||||
// Small block — push onto bucket (O(1))
|
|
||||||
auto* node = (FreeNode*)header;
|
|
||||||
node->size = BUCKET_SIZES[idx];
|
|
||||||
node->next = g_buckets[idx];
|
|
||||||
g_buckets[idx] = node;
|
|
||||||
} else {
|
|
||||||
// Large block — sorted insert with coalescing
|
|
||||||
insert_overflow((void*)header, blockSize);
|
|
||||||
}
|
|
||||||
heap_lock_release();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void* realloc(void* ptr, uint64_t size) {
|
inline void* realloc(void* ptr, uint64_t size) {
|
||||||
if (ptr == nullptr) return malloc(size);
|
return ::realloc(ptr, (std::size_t)size);
|
||||||
|
|
||||||
// Read old size under the lock to avoid racing with another
|
|
||||||
// thread that might be freeing/recycling this header.
|
|
||||||
heap_detail::heap_lock_acquire();
|
|
||||||
auto* header = heap_detail::get_header(ptr);
|
|
||||||
uint64_t old = header->size;
|
|
||||||
|
|
||||||
uint64_t oldBlock = (old + sizeof(heap_detail::Header) + 15) & ~15ULL;
|
|
||||||
int idx = heap_detail::bucket_index(oldBlock);
|
|
||||||
if (idx >= 0) oldBlock = heap_detail::BUCKET_SIZES[idx];
|
|
||||||
|
|
||||||
uint64_t newNeed = (size + sizeof(heap_detail::Header) + 15) & ~15ULL;
|
|
||||||
if (newNeed <= oldBlock) {
|
|
||||||
header->size = size;
|
|
||||||
heap_detail::heap_lock_release();
|
|
||||||
return ptr;
|
|
||||||
}
|
}
|
||||||
heap_detail::heap_lock_release();
|
|
||||||
|
|
||||||
void* newBlock = malloc(size);
|
inline void* calloc(uint64_t count, uint64_t size) {
|
||||||
if (newBlock == nullptr) return nullptr;
|
return ::calloc((std::size_t)count, (std::size_t)size);
|
||||||
|
|
||||||
uint64_t copySize = (old < size) ? old : size;
|
|
||||||
memcpy(newBlock, ptr, copySize);
|
|
||||||
|
|
||||||
mfree(ptr);
|
|
||||||
return newBlock;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace montauk
|
} // namespace montauk
|
||||||
|
|||||||
@@ -44,8 +44,22 @@ namespace montauk {
|
|||||||
ThreadEntry user_entry;
|
ThreadEntry user_entry;
|
||||||
void* user_arg;
|
void* user_arg;
|
||||||
void* stack_base;
|
void* stack_base;
|
||||||
|
int tid;
|
||||||
|
ThreadCtx* next;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
inline ThreadCtx* g_thread_records = nullptr;
|
||||||
|
inline volatile uint32_t g_thread_records_lock = 0;
|
||||||
|
|
||||||
|
inline void records_lock() {
|
||||||
|
while (__atomic_exchange_n(&g_thread_records_lock, 1, __ATOMIC_ACQUIRE) != 0)
|
||||||
|
montauk::yield();
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void records_unlock() {
|
||||||
|
__atomic_store_n(&g_thread_records_lock, 0, __ATOMIC_RELEASE);
|
||||||
|
}
|
||||||
|
|
||||||
// Userspace trampoline: bridges from the raw entry the kernel jumps
|
// Userspace trampoline: bridges from the raw entry the kernel jumps
|
||||||
// to into the typed entry, then funnels into SYS_THREAD_EXIT. We
|
// to into the typed entry, then funnels into SYS_THREAD_EXIT. We
|
||||||
// route the exit through libc rather than relying on a kernel-side
|
// route the exit through libc rather than relying on a kernel-side
|
||||||
@@ -53,21 +67,24 @@ namespace montauk {
|
|||||||
// memory on this path.
|
// memory on this path.
|
||||||
//
|
//
|
||||||
// The thread's stack itself is intentionally not freed here: we are
|
// The thread's stack itself is intentionally not freed here: we are
|
||||||
// still running on it. It is reclaimed when the process exits, or
|
// still running on it. It is reclaimed by a successful thread_join,
|
||||||
// the joiner may free it explicitly after thread_join.
|
// or as part of whole-process teardown if the thread is never joined.
|
||||||
[[noreturn]] inline void thread_trampoline(detail::ThreadCtx* ctx) {
|
[[noreturn]] inline void thread_trampoline(detail::ThreadCtx* ctx) {
|
||||||
|
// A sibling CPU can start the thread before thread_spawn has
|
||||||
|
// returned its TID. Wait until the parent has published the record
|
||||||
|
// needed by thread_join to reclaim this stack.
|
||||||
|
while (__atomic_load_n(&ctx->tid, __ATOMIC_ACQUIRE) == 0)
|
||||||
|
montauk::yield();
|
||||||
int code = ctx->user_entry(ctx->user_arg);
|
int code = ctx->user_entry(ctx->user_arg);
|
||||||
montauk::mfree(ctx);
|
|
||||||
thread_exit(code);
|
thread_exit(code);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Spawn a new thread that begins executing `entry(arg)`. Returns the
|
// Spawn a new thread that begins executing `entry(arg)`. Returns the
|
||||||
// new TID on success, or -1 on failure. The thread's stack is
|
// new TID on success, or -1 on failure. The thread's stack is
|
||||||
// allocated from the user heap; it is leaked on thread exit (the
|
// allocated from the user heap. The exiting thread cannot free the stack
|
||||||
// thread itself cannot free the stack it is running on). The kernel
|
// it is running on, so thread_join reclaims both it and the trampoline
|
||||||
// reclaims it on process exit. Callers that need to spawn many short-
|
// context after the kernel has reaped the sibling.
|
||||||
// lived threads should pool stacks themselves.
|
|
||||||
inline int thread_spawn(ThreadEntry entry, void* arg,
|
inline int thread_spawn(ThreadEntry entry, void* arg,
|
||||||
uint64_t stack_bytes = 0) {
|
uint64_t stack_bytes = 0) {
|
||||||
if (entry == nullptr) return -1;
|
if (entry == nullptr) return -1;
|
||||||
@@ -84,6 +101,8 @@ namespace montauk {
|
|||||||
ctx->user_entry = entry;
|
ctx->user_entry = entry;
|
||||||
ctx->user_arg = arg;
|
ctx->user_arg = arg;
|
||||||
ctx->stack_base = stack;
|
ctx->stack_base = stack;
|
||||||
|
ctx->tid = 0;
|
||||||
|
ctx->next = nullptr;
|
||||||
|
|
||||||
uint64_t stack_top = ((uint64_t)stack + stack_bytes) & ~0xFULL;
|
uint64_t stack_top = ((uint64_t)stack + stack_bytes) & ~0xFULL;
|
||||||
int tid = (int)syscall3(montauk::abi::SYS_THREAD_SPAWN,
|
int tid = (int)syscall3(montauk::abi::SYS_THREAD_SPAWN,
|
||||||
@@ -94,6 +113,11 @@ namespace montauk {
|
|||||||
montauk::mfree(stack);
|
montauk::mfree(stack);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
detail::records_lock();
|
||||||
|
ctx->next = detail::g_thread_records;
|
||||||
|
detail::g_thread_records = ctx;
|
||||||
|
__atomic_store_n(&ctx->tid, tid, __ATOMIC_RELEASE);
|
||||||
|
detail::records_unlock();
|
||||||
return tid;
|
return tid;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,8 +125,22 @@ namespace montauk {
|
|||||||
// success (with the thread's exit code in *out_code if non-null) or
|
// success (with the thread's exit code in *out_code if non-null) or
|
||||||
// -1 if `tid` is not a joinable sibling.
|
// -1 if `tid` is not a joinable sibling.
|
||||||
inline int thread_join(int tid, int* out_code = nullptr) {
|
inline int thread_join(int tid, int* out_code = nullptr) {
|
||||||
return (int)syscall2(montauk::abi::SYS_THREAD_JOIN,
|
int result = (int)syscall2(montauk::abi::SYS_THREAD_JOIN,
|
||||||
(uint64_t)tid, (uint64_t)out_code);
|
(uint64_t)tid, (uint64_t)out_code);
|
||||||
|
if (result == 0) {
|
||||||
|
detail::records_lock();
|
||||||
|
detail::ThreadCtx** link = &detail::g_thread_records;
|
||||||
|
while (*link != nullptr && (*link)->tid != tid)
|
||||||
|
link = &(*link)->next;
|
||||||
|
detail::ThreadCtx* ctx = *link;
|
||||||
|
if (ctx != nullptr) *link = ctx->next;
|
||||||
|
detail::records_unlock();
|
||||||
|
if (ctx != nullptr) {
|
||||||
|
montauk::mfree(ctx->stack_base);
|
||||||
|
montauk::mfree(ctx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return the calling thread's TID (== getpid() for the main thread).
|
// Return the calling thread's TID (== getpid() for the main thread).
|
||||||
|
|||||||
+189
-65
@@ -89,6 +89,7 @@ static inline long _zos_syscall4(long nr, long a1, long a2, long a3, long a4) {
|
|||||||
|
|
||||||
/* Syscall numbers */
|
/* Syscall numbers */
|
||||||
#define SYS_EXIT 0
|
#define SYS_EXIT 0
|
||||||
|
#define SYS_YIELD 1
|
||||||
#define SYS_SLEEP_MS 2
|
#define SYS_SLEEP_MS 2
|
||||||
#define SYS_PRINT 4
|
#define SYS_PRINT 4
|
||||||
#define SYS_PUTCHAR 5
|
#define SYS_PUTCHAR 5
|
||||||
@@ -119,6 +120,9 @@ static inline long _zos_syscall4(long nr, long a1, long a2, long a3, long a4) {
|
|||||||
#define SYS_KILL 62
|
#define SYS_KILL 62
|
||||||
#define SYS_STAT 152
|
#define SYS_STAT 152
|
||||||
#define SYS_UTIME 167
|
#define SYS_UTIME 167
|
||||||
|
#define SYS_MMAP_ANON 168
|
||||||
|
#define SYS_MUNMAP 169
|
||||||
|
#define SYS_MPROTECT 170
|
||||||
|
|
||||||
/* ========================================================================
|
/* ========================================================================
|
||||||
errno
|
errno
|
||||||
@@ -626,44 +630,74 @@ int tolower(int c) { return (c >= 'A' && c <= 'Z') ? c + 32 : c; }
|
|||||||
Heap allocator (free-list, backed by SYS_ALLOC)
|
Heap allocator (free-list, backed by SYS_ALLOC)
|
||||||
======================================================================== */
|
======================================================================== */
|
||||||
|
|
||||||
#define HEAP_MAGIC 0x5A484541ULL /* "ZHEA" */
|
#define HEAP_MAGIC 0x4D544B4845415041ULL /* "MTKHEAPA" */
|
||||||
#define FREED_MAGIC 0xDEADFEEEULL
|
#define DIRECT_MAGIC 0x4D544B4449524543ULL /* "MTKDIREC" */
|
||||||
|
#define FREED_MAGIC 0x4D544B4652454544ULL /* "MTKFREED" */
|
||||||
|
#define HEAP_ALIGN 16ULL
|
||||||
|
#define DIRECT_THRESHOLD (256ULL * 1024ULL)
|
||||||
|
|
||||||
struct HeapHeader {
|
struct HeapHeader {
|
||||||
uint64_t magic;
|
uint64_t magic;
|
||||||
uint64_t size;
|
uint64_t requested_size;
|
||||||
} __attribute__((packed));
|
uint64_t block_size;
|
||||||
|
uint64_t cookie;
|
||||||
|
};
|
||||||
|
|
||||||
struct FreeNode {
|
struct FreeNode {
|
||||||
|
uint64_t magic;
|
||||||
uint64_t size;
|
uint64_t size;
|
||||||
struct FreeNode *next;
|
struct FreeNode *next;
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Segregated free lists: power-of-2 size classes for blocks <= 4096 bytes */
|
/* Segregated free lists for small and medium blocks. The minimum class must
|
||||||
#define NUM_BUCKETS 8
|
hold both an allocated header and a free-list node. */
|
||||||
|
#define NUM_BUCKETS 7
|
||||||
static const uint64_t BUCKET_SIZES[NUM_BUCKETS] = {
|
static const uint64_t BUCKET_SIZES[NUM_BUCKETS] = {
|
||||||
32, 64, 128, 256, 512, 1024, 2048, 4096
|
64, 128, 256, 512, 1024, 2048, 4096
|
||||||
};
|
};
|
||||||
|
|
||||||
static struct FreeNode *g_buckets[NUM_BUCKETS] = {};
|
static struct FreeNode *g_buckets[NUM_BUCKETS] = {};
|
||||||
static struct FreeNode g_overflow = { 0, NULL };
|
static struct FreeNode g_overflow = { FREED_MAGIC, 0, NULL };
|
||||||
static int g_heapInit = 0;
|
static int g_heapInit = 0;
|
||||||
|
static volatile uint32_t g_heapLock = 0;
|
||||||
|
|
||||||
|
static void heap_lock(void) {
|
||||||
|
while (__atomic_exchange_n(&g_heapLock, 1, __ATOMIC_ACQUIRE) != 0)
|
||||||
|
_zos_syscall0(SYS_YIELD);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void heap_unlock(void) {
|
||||||
|
__atomic_store_n(&g_heapLock, 0, __ATOMIC_RELEASE);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int heap_total_size(size_t requested, uint64_t *out) {
|
||||||
|
uint64_t value = (uint64_t)requested;
|
||||||
|
if (value > UINT64_MAX - sizeof(struct HeapHeader) - (HEAP_ALIGN - 1))
|
||||||
|
return 0;
|
||||||
|
*out = (value + sizeof(struct HeapHeader) + (HEAP_ALIGN - 1))
|
||||||
|
& ~(HEAP_ALIGN - 1);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t heap_cookie(const struct HeapHeader *hdr, uint64_t block_size) {
|
||||||
|
return ((uint64_t)(uintptr_t)hdr >> 4) ^ block_size ^ hdr->magic;
|
||||||
|
}
|
||||||
|
|
||||||
static int heap_bucket_index(uint64_t blockSize) {
|
static int heap_bucket_index(uint64_t blockSize) {
|
||||||
if (blockSize <= 32) return 0;
|
if (blockSize <= 64) return 0;
|
||||||
if (blockSize <= 64) return 1;
|
if (blockSize <= 128) return 1;
|
||||||
if (blockSize <= 128) return 2;
|
if (blockSize <= 256) return 2;
|
||||||
if (blockSize <= 256) return 3;
|
if (blockSize <= 512) return 3;
|
||||||
if (blockSize <= 512) return 4;
|
if (blockSize <= 1024) return 4;
|
||||||
if (blockSize <= 1024) return 5;
|
if (blockSize <= 2048) return 5;
|
||||||
if (blockSize <= 2048) return 6;
|
if (blockSize <= 4096) return 6;
|
||||||
if (blockSize <= 4096) return 7;
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Insert into overflow list (sorted by address, with coalescing) */
|
/* Insert into overflow list (sorted by address, with coalescing) */
|
||||||
static void heap_insert_overflow(void *ptr, uint64_t size) {
|
static void heap_insert_overflow(void *ptr, uint64_t size) {
|
||||||
struct FreeNode *node = (struct FreeNode *)ptr;
|
struct FreeNode *node = (struct FreeNode *)ptr;
|
||||||
|
node->magic = FREED_MAGIC;
|
||||||
node->size = size;
|
node->size = size;
|
||||||
|
|
||||||
struct FreeNode *prev = &g_overflow;
|
struct FreeNode *prev = &g_overflow;
|
||||||
@@ -693,7 +727,7 @@ static void heap_insert_overflow(void *ptr, uint64_t size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Take a block >= needed from overflow. Splits remainder back. */
|
/* Take a block >= needed from overflow. Splits remainder back. */
|
||||||
static void *heap_take_overflow(uint64_t needed) {
|
static void *heap_take_overflow(uint64_t needed, uint64_t *actual_size) {
|
||||||
struct FreeNode *prev = &g_overflow;
|
struct FreeNode *prev = &g_overflow;
|
||||||
struct FreeNode *cur = g_overflow.next;
|
struct FreeNode *cur = g_overflow.next;
|
||||||
|
|
||||||
@@ -702,9 +736,13 @@ static void *heap_take_overflow(uint64_t needed) {
|
|||||||
uint64_t blockSize = cur->size;
|
uint64_t blockSize = cur->size;
|
||||||
prev->next = cur->next;
|
prev->next = cur->next;
|
||||||
|
|
||||||
if (blockSize > needed + sizeof(struct FreeNode) + 16) {
|
uint64_t min_free = (sizeof(struct FreeNode) + HEAP_ALIGN - 1)
|
||||||
|
& ~(HEAP_ALIGN - 1);
|
||||||
|
if (blockSize >= needed + min_free) {
|
||||||
heap_insert_overflow((uint8_t *)cur + needed, blockSize - needed);
|
heap_insert_overflow((uint8_t *)cur + needed, blockSize - needed);
|
||||||
|
blockSize = needed;
|
||||||
}
|
}
|
||||||
|
*actual_size = blockSize;
|
||||||
return (void *)cur;
|
return (void *)cur;
|
||||||
}
|
}
|
||||||
prev = cur;
|
prev = cur;
|
||||||
@@ -713,11 +751,8 @@ static void *heap_take_overflow(uint64_t needed) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Next slab size for heap growth. The kernel tracks a finite number of
|
/* Next slab size for heap growth. Doubling slabs keeps syscall and VMA
|
||||||
SYS_ALLOC records per process (MaxHeapAllocs), so growing once per
|
metadata traffic logarithmic in total heap size. */
|
||||||
large allocation exhausts them: ld ran out mid-link and BFD reported
|
|
||||||
the resulting NULL mallocs as "file format not recognized". Doubling
|
|
||||||
slabs keep the syscall count logarithmic in total heap size. */
|
|
||||||
static uint64_t g_heap_slab = 16 * 0x1000;
|
static uint64_t g_heap_slab = 16 * 0x1000;
|
||||||
|
|
||||||
static void heap_grow(uint64_t bytes) {
|
static void heap_grow(uint64_t bytes) {
|
||||||
@@ -742,16 +777,24 @@ static int heap_refill_bucket(int idx) {
|
|||||||
uint64_t bsize = BUCKET_SIZES[idx];
|
uint64_t bsize = BUCKET_SIZES[idx];
|
||||||
uint64_t chunk = (bsize < 4096) ? 4096 : bsize;
|
uint64_t chunk = (bsize < 4096) ? 4096 : bsize;
|
||||||
|
|
||||||
void *block = heap_take_overflow(chunk);
|
uint64_t actual = 0;
|
||||||
|
void *block = heap_take_overflow(chunk, &actual);
|
||||||
if (block == NULL) {
|
if (block == NULL) {
|
||||||
heap_grow(chunk);
|
heap_grow(chunk);
|
||||||
block = heap_take_overflow(chunk);
|
block = heap_take_overflow(chunk, &actual);
|
||||||
if (block == NULL) return 0;
|
if (block == NULL) return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A small unsplittable tail belongs to the overflow list, rather than
|
||||||
|
disappearing when the main chunk is carved into bucket blocks. */
|
||||||
|
if (actual - chunk >= ((sizeof(struct FreeNode) + HEAP_ALIGN - 1)
|
||||||
|
& ~(HEAP_ALIGN - 1)))
|
||||||
|
heap_insert_overflow((uint8_t *)block + chunk, actual - chunk);
|
||||||
|
|
||||||
uint64_t count = chunk / bsize;
|
uint64_t count = chunk / bsize;
|
||||||
for (uint64_t i = 0; i < count; i++) {
|
for (uint64_t i = 0; i < count; i++) {
|
||||||
struct FreeNode *node = (struct FreeNode *)((uint8_t *)block + i * bsize);
|
struct FreeNode *node = (struct FreeNode *)((uint8_t *)block + i * bsize);
|
||||||
|
node->magic = FREED_MAGIC;
|
||||||
node->size = bsize;
|
node->size = bsize;
|
||||||
node->next = g_buckets[idx];
|
node->next = g_buckets[idx];
|
||||||
g_buckets[idx] = node;
|
g_buckets[idx] = node;
|
||||||
@@ -759,19 +802,31 @@ static int heap_refill_bucket(int idx) {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
void *malloc(size_t size) {
|
static void *heap_malloc_locked(size_t size) {
|
||||||
|
uint64_t needed;
|
||||||
|
if (!heap_total_size(size, &needed))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
/* Large objects get their own page mapping, so free can promptly return
|
||||||
|
both physical memory and virtual space to the kernel. */
|
||||||
|
if (needed >= DIRECT_THRESHOLD) {
|
||||||
|
if (needed > UINT64_MAX - 0xFFFULL) return NULL;
|
||||||
|
uint64_t mapping_size = (needed + 0xFFFULL) & ~0xFFFULL;
|
||||||
|
struct HeapHeader *hdr = (struct HeapHeader *)
|
||||||
|
_zos_syscall1(SYS_ALLOC, (long)mapping_size);
|
||||||
|
if (hdr == NULL) return NULL;
|
||||||
|
hdr->magic = DIRECT_MAGIC;
|
||||||
|
hdr->requested_size = size;
|
||||||
|
hdr->block_size = mapping_size;
|
||||||
|
hdr->cookie = heap_cookie(hdr, mapping_size);
|
||||||
|
return (uint8_t *)hdr + sizeof(*hdr);
|
||||||
|
}
|
||||||
|
|
||||||
if (!g_heapInit) {
|
if (!g_heapInit) {
|
||||||
heap_grow(16 * 0x1000);
|
heap_grow(16 * 0x1000);
|
||||||
g_heapInit = 1;
|
g_heapInit = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Guard against overflow: size + Header must not wrap */
|
|
||||||
if (size > (uint64_t)-1 - sizeof(struct HeapHeader) - 15)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
uint64_t needed = size + sizeof(struct HeapHeader);
|
|
||||||
needed = (needed + 15) & ~15ULL;
|
|
||||||
|
|
||||||
int idx = heap_bucket_index(needed);
|
int idx = heap_bucket_index(needed);
|
||||||
|
|
||||||
if (idx >= 0) {
|
if (idx >= 0) {
|
||||||
@@ -781,44 +836,63 @@ void *malloc(size_t size) {
|
|||||||
|
|
||||||
struct FreeNode *node = g_buckets[idx];
|
struct FreeNode *node = g_buckets[idx];
|
||||||
g_buckets[idx] = node->next;
|
g_buckets[idx] = node->next;
|
||||||
|
uint64_t block_size = node->size;
|
||||||
|
|
||||||
struct HeapHeader *hdr = (struct HeapHeader *)node;
|
struct HeapHeader *hdr = (struct HeapHeader *)node;
|
||||||
hdr->magic = HEAP_MAGIC;
|
hdr->magic = HEAP_MAGIC;
|
||||||
hdr->size = size;
|
hdr->requested_size = size;
|
||||||
|
hdr->block_size = block_size;
|
||||||
|
hdr->cookie = heap_cookie(hdr, hdr->block_size);
|
||||||
return (void *)((uint8_t *)hdr + sizeof(struct HeapHeader));
|
return (void *)((uint8_t *)hdr + sizeof(struct HeapHeader));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Large allocation — search overflow list */
|
/* Large allocation — search overflow list */
|
||||||
void *block = heap_take_overflow(needed);
|
uint64_t actual = 0;
|
||||||
|
void *block = heap_take_overflow(needed, &actual);
|
||||||
if (block == NULL) {
|
if (block == NULL) {
|
||||||
heap_grow(needed);
|
heap_grow(needed);
|
||||||
block = heap_take_overflow(needed);
|
block = heap_take_overflow(needed, &actual);
|
||||||
if (block == NULL) return NULL;
|
if (block == NULL) return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
struct HeapHeader *hdr = (struct HeapHeader *)block;
|
struct HeapHeader *hdr = (struct HeapHeader *)block;
|
||||||
hdr->magic = HEAP_MAGIC;
|
hdr->magic = HEAP_MAGIC;
|
||||||
hdr->size = size;
|
hdr->requested_size = size;
|
||||||
|
hdr->block_size = actual;
|
||||||
|
hdr->cookie = heap_cookie(hdr, actual);
|
||||||
return (void *)((uint8_t *)hdr + sizeof(struct HeapHeader));
|
return (void *)((uint8_t *)hdr + sizeof(struct HeapHeader));
|
||||||
}
|
}
|
||||||
|
|
||||||
void free(void *ptr) {
|
static int heap_header_valid(const struct HeapHeader *hdr) {
|
||||||
|
return (hdr->magic == HEAP_MAGIC || hdr->magic == DIRECT_MAGIC) &&
|
||||||
|
hdr->block_size >= sizeof(struct HeapHeader) &&
|
||||||
|
(hdr->block_size & (HEAP_ALIGN - 1)) == 0 &&
|
||||||
|
hdr->cookie == heap_cookie(hdr, hdr->block_size) &&
|
||||||
|
hdr->requested_size <= hdr->block_size - sizeof(struct HeapHeader);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void heap_free_locked(void *ptr) {
|
||||||
if (ptr == NULL) return;
|
if (ptr == NULL) return;
|
||||||
|
|
||||||
struct HeapHeader *hdr = (struct HeapHeader *)((uint8_t *)ptr - sizeof(struct HeapHeader));
|
struct HeapHeader *hdr = (struct HeapHeader *)((uint8_t *)ptr - sizeof(struct HeapHeader));
|
||||||
|
|
||||||
if (hdr->magic == FREED_MAGIC) return; /* double-free */
|
if (!heap_header_valid(hdr)) return;
|
||||||
if (hdr->magic != HEAP_MAGIC) return; /* corrupt */
|
|
||||||
|
uint64_t blockSize = hdr->block_size;
|
||||||
|
int direct = hdr->magic == DIRECT_MAGIC;
|
||||||
hdr->magic = FREED_MAGIC;
|
hdr->magic = FREED_MAGIC;
|
||||||
|
|
||||||
uint64_t blockSize = hdr->size + sizeof(struct HeapHeader);
|
if (direct) {
|
||||||
blockSize = (blockSize + 15) & ~15ULL;
|
_zos_syscall1(SYS_FREE, (long)hdr);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
int idx = heap_bucket_index(blockSize);
|
int idx = heap_bucket_index(blockSize);
|
||||||
|
|
||||||
if (idx >= 0) {
|
if (idx >= 0) {
|
||||||
/* Small block — push onto bucket (O(1)) */
|
/* Small block — push onto bucket (O(1)) */
|
||||||
struct FreeNode *node = (struct FreeNode *)hdr;
|
struct FreeNode *node = (struct FreeNode *)hdr;
|
||||||
|
node->magic = FREED_MAGIC;
|
||||||
node->size = BUCKET_SIZES[idx];
|
node->size = BUCKET_SIZES[idx];
|
||||||
node->next = g_buckets[idx];
|
node->next = g_buckets[idx];
|
||||||
g_buckets[idx] = node;
|
g_buckets[idx] = node;
|
||||||
@@ -828,6 +902,21 @@ void free(void *ptr) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void *malloc(size_t size) {
|
||||||
|
void *result;
|
||||||
|
heap_lock();
|
||||||
|
result = heap_malloc_locked(size);
|
||||||
|
heap_unlock();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
void free(void *ptr) {
|
||||||
|
if (ptr == NULL) return;
|
||||||
|
heap_lock();
|
||||||
|
heap_free_locked(ptr);
|
||||||
|
heap_unlock();
|
||||||
|
}
|
||||||
|
|
||||||
void *calloc(size_t nmemb, size_t size) {
|
void *calloc(size_t nmemb, size_t size) {
|
||||||
/* Check for multiplication overflow */
|
/* Check for multiplication overflow */
|
||||||
if (nmemb != 0 && size > (size_t)-1 / nmemb)
|
if (nmemb != 0 && size > (size_t)-1 / nmemb)
|
||||||
@@ -842,26 +931,39 @@ void *realloc(void *ptr, size_t size) {
|
|||||||
if (ptr == NULL) return malloc(size);
|
if (ptr == NULL) return malloc(size);
|
||||||
if (size == 0) { free(ptr); return NULL; }
|
if (size == 0) { free(ptr); return NULL; }
|
||||||
|
|
||||||
|
uint64_t newNeed;
|
||||||
|
if (!heap_total_size(size, &newNeed))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
heap_lock();
|
||||||
|
|
||||||
struct HeapHeader *hdr = (struct HeapHeader *)((uint8_t *)ptr - sizeof(struct HeapHeader));
|
struct HeapHeader *hdr = (struct HeapHeader *)((uint8_t *)ptr - sizeof(struct HeapHeader));
|
||||||
uint64_t old = hdr->size;
|
if (!heap_header_valid(hdr)) {
|
||||||
|
heap_unlock();
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
uint64_t old = hdr->requested_size;
|
||||||
|
uint64_t oldBlock = hdr->block_size;
|
||||||
|
|
||||||
/* Compute actual block size (accounting for bucket rounding) */
|
|
||||||
uint64_t oldBlock = (old + sizeof(struct HeapHeader) + 15) & ~15ULL;
|
|
||||||
int idx = heap_bucket_index(oldBlock);
|
|
||||||
if (idx >= 0) oldBlock = BUCKET_SIZES[idx];
|
|
||||||
|
|
||||||
uint64_t newNeed = (size + sizeof(struct HeapHeader) + 15) & ~15ULL;
|
|
||||||
if (newNeed <= oldBlock) {
|
if (newNeed <= oldBlock) {
|
||||||
hdr->size = size;
|
/* Retain the actual extent. Losing it here makes the tail impossible
|
||||||
|
to recover when this block is later freed. */
|
||||||
|
hdr->requested_size = size;
|
||||||
|
hdr->cookie = heap_cookie(hdr, oldBlock);
|
||||||
|
heap_unlock();
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
void *newp = malloc(size);
|
void *newp = heap_malloc_locked(size);
|
||||||
if (newp == NULL) return NULL;
|
if (newp == NULL) {
|
||||||
|
heap_unlock();
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
size_t copySize = old < size ? old : size;
|
size_t copySize = old < size ? old : size;
|
||||||
memcpy(newp, ptr, copySize);
|
memcpy(newp, ptr, copySize);
|
||||||
free(ptr);
|
heap_free_locked(ptr);
|
||||||
|
heap_unlock();
|
||||||
return newp;
|
return newp;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3874,17 +3976,26 @@ long sysconf(int name) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Anonymous mappings only: SYS_ALLOC hands back zeroed page-aligned
|
/* Anonymous private mappings with kernel-enforced page permissions. */
|
||||||
memory and SYS_FREE releases it. Length is remembered by the kernel
|
|
||||||
per allocation, so munmap ignores its length argument. */
|
|
||||||
void *mmap(void *addr, size_t length, int prot, int flags, int fd,
|
void *mmap(void *addr, size_t length, int prot, int flags, int fd,
|
||||||
long offset) {
|
long offset) {
|
||||||
(void)addr; (void)prot; (void)offset;
|
int supported_flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
||||||
if (length == 0 || fd != -1 || !(flags & MAP_ANONYMOUS)) {
|
if (length == 0 || fd != -1 || offset != 0 ||
|
||||||
errno = ENODEV;
|
(flags & supported_flags) != supported_flags ||
|
||||||
|
(flags & ~supported_flags) != 0) {
|
||||||
|
errno = EINVAL;
|
||||||
return MAP_FAILED;
|
return MAP_FAILED;
|
||||||
}
|
}
|
||||||
void *p = (void *)_zos_syscall1(SYS_ALLOC, (long)length);
|
/* Address hints may be ignored; fixed placement is intentionally rejected
|
||||||
|
by the flag validation above. */
|
||||||
|
(void)addr;
|
||||||
|
if ((prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0 ||
|
||||||
|
!(prot & PROT_READ) ||
|
||||||
|
((prot & PROT_WRITE) && (prot & PROT_EXEC))) {
|
||||||
|
errno = EINVAL;
|
||||||
|
return MAP_FAILED;
|
||||||
|
}
|
||||||
|
void *p = (void *)_zos_syscall2(SYS_MMAP_ANON, (long)length, (long)prot);
|
||||||
if (p == NULL) {
|
if (p == NULL) {
|
||||||
errno = ENOMEM;
|
errno = ENOMEM;
|
||||||
return MAP_FAILED;
|
return MAP_FAILED;
|
||||||
@@ -3893,18 +4004,31 @@ void *mmap(void *addr, size_t length, int prot, int flags, int fd,
|
|||||||
}
|
}
|
||||||
|
|
||||||
int munmap(void *addr, size_t length) {
|
int munmap(void *addr, size_t length) {
|
||||||
(void)length;
|
if (addr == NULL || addr == MAP_FAILED || length == 0 ||
|
||||||
if (addr == NULL || addr == MAP_FAILED) {
|
((uintptr_t)addr & 0xFFFULL) != 0) {
|
||||||
|
errno = EINVAL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (_zos_syscall2(SYS_MUNMAP, (long)addr, (long)length) < 0) {
|
||||||
errno = EINVAL;
|
errno = EINVAL;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
_zos_syscall1(SYS_FREE, (long)addr);
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int mprotect(void *addr, size_t length, int prot) {
|
int mprotect(void *addr, size_t length, int prot) {
|
||||||
(void)addr; (void)length; (void)prot;
|
if (addr == NULL || length == 0 || ((uintptr_t)addr & 0xFFFULL) != 0 ||
|
||||||
return 0; /* page protections are not adjustable from userspace */
|
(prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0 ||
|
||||||
|
!(prot & PROT_READ) ||
|
||||||
|
((prot & PROT_WRITE) && (prot & PROT_EXEC))) {
|
||||||
|
errno = EINVAL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (_zos_syscall3(SYS_MPROTECT, (long)addr, (long)length, (long)prot) < 0) {
|
||||||
|
errno = EINVAL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
long pathconf(const char *path, int name) {
|
long pathconf(const char *path, int name) {
|
||||||
|
|||||||
Binary file not shown.
+20
-18
@@ -9,49 +9,51 @@
|
|||||||
|
|
||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
The userspace heap provides dynamic memory allocation on top of
|
The userspace heap provides dynamic memory allocation on top of
|
||||||
the kernel's page-mapping syscall (SYS_ALLOC). Include the
|
anonymous virtual memory. The Montauk C++ API and libc's
|
||||||
header <montauk/heap.h> to use these functions.
|
malloc/free API use the same process-wide allocator.
|
||||||
|
|
||||||
.SS malloc
|
.SS malloc
|
||||||
Allocates 'size' bytes from the free list. Returns a 16-byte
|
Allocates 'size' bytes from the free list. Returns a 16-byte
|
||||||
aligned pointer, or nullptr on failure. When the free list is
|
aligned pointer, or nullptr on failure. When the free list is
|
||||||
empty, it requests more pages from the kernel via SYS_ALLOC
|
empty, it reserves more pages from the kernel. Physical pages
|
||||||
(minimum 16 KiB growth, initial seed of 64 KiB).
|
are committed as they are first touched.
|
||||||
|
|
||||||
char* buf = (char*)montauk::malloc(1024);
|
char* buf = (char*)montauk::malloc(1024);
|
||||||
|
|
||||||
.SS mfree
|
.SS mfree
|
||||||
Returns the block to the userspace free list. No syscall is
|
Returns the block to the userspace allocator. Arena blocks are
|
||||||
made -- the memory stays mapped and is immediately reusable.
|
immediately reusable; large direct mappings are returned to the
|
||||||
|
kernel, including their virtual address range.
|
||||||
Passing nullptr is a safe no-op.
|
Passing nullptr is a safe no-op.
|
||||||
|
|
||||||
montauk::mfree(buf);
|
montauk::mfree(buf);
|
||||||
|
|
||||||
.SS realloc
|
.SS realloc
|
||||||
Resizes the allocation to 'size' bytes. Allocates a new block,
|
Resizes the allocation to 'size' bytes. A block with sufficient
|
||||||
copies the smaller of old/new sizes, and frees the old block.
|
capacity is retained; otherwise a new block is allocated, the
|
||||||
|
smaller of old/new requested sizes is copied, and the old block
|
||||||
|
is freed. Integer overflow fails without changing the old block.
|
||||||
If ptr is nullptr, behaves like malloc.
|
If ptr is nullptr, behaves like malloc.
|
||||||
|
|
||||||
buf = (char*)montauk::realloc(buf, 2048);
|
buf = (char*)montauk::realloc(buf, 2048);
|
||||||
|
|
||||||
.SH IMPLEMENTATION
|
.SH IMPLEMENTATION
|
||||||
The allocator uses a linked free-list with first-fit search.
|
The allocator uses segregated size-class bins and a coalescing
|
||||||
Blocks larger than needed are split. The allocation header is
|
address-ordered overflow list. Headers retain both requested size
|
||||||
16 bytes (magic + size). All allocations are 16-byte aligned.
|
and actual block extent. A process-wide lock serializes C and C++
|
||||||
|
allocation calls. All returned pointers are 16-byte aligned.
|
||||||
|
|
||||||
The heap grows by requesting pages from the kernel via
|
Allocations of 256 KiB or more use direct page mappings so they
|
||||||
SYS_ALLOC. These pages are never returned to the kernel (since
|
can be released promptly. Smaller allocations use growing arenas.
|
||||||
SYS_FREE is currently a no-op), but mfree makes them available
|
|
||||||
for future malloc calls within the process.
|
|
||||||
|
|
||||||
.SH LOW-LEVEL PAGE API
|
.SH LOW-LEVEL PAGE API
|
||||||
For large allocations or when direct page control is needed:
|
For large allocations or when direct page control is needed:
|
||||||
|
|
||||||
void* montauk::alloc(uint64_t size); // SYS_ALLOC
|
void* montauk::alloc(uint64_t size); // SYS_ALLOC
|
||||||
void montauk::free(void* ptr); // SYS_FREE (no-op)
|
void montauk::free(void* ptr); // SYS_FREE
|
||||||
|
|
||||||
alloc() maps zeroed pages starting at 0x40000000 and growing
|
alloc() reserves zero-filled, read/write, non-executable pages.
|
||||||
upward. Size is rounded up to 4 KiB page boundaries.
|
Size is rounded up to 4 KiB. Freed ranges are reusable.
|
||||||
|
|
||||||
.SH SEE ALSO
|
.SH SEE ALSO
|
||||||
syscalls(2), file(2)
|
syscalls(2), file(2)
|
||||||
|
|||||||
+16
-1
@@ -185,9 +185,24 @@
|
|||||||
void* montauk::alloc(uint64_t size);
|
void* montauk::alloc(uint64_t size);
|
||||||
|
|
||||||
.B SYS_FREE (12)
|
.B SYS_FREE (12)
|
||||||
Reserved (currently a no-op).
|
Release a complete mapping previously returned by SYS_ALLOC.
|
||||||
void montauk::free(void* ptr);
|
void montauk::free(void* ptr);
|
||||||
|
|
||||||
|
.B SYS_MMAP_ANON (168)
|
||||||
|
Reserve a zero-filled anonymous mapping with read/write/execute
|
||||||
|
protection flags. Pages are committed on first access. Writable
|
||||||
|
executable mappings are rejected.
|
||||||
|
void* mmap(void*, size_t, int, int, int, long);
|
||||||
|
|
||||||
|
.B SYS_MUNMAP (169)
|
||||||
|
Release a page-aligned range. Partial unmap splits the VM area and
|
||||||
|
makes the virtual range reusable.
|
||||||
|
int munmap(void* addr, size_t length);
|
||||||
|
|
||||||
|
.B SYS_MPROTECT (170)
|
||||||
|
Change read/write/execute permissions on an anonymous mapping.
|
||||||
|
int mprotect(void* addr, size_t length, int prot);
|
||||||
|
|
||||||
.B SYS_MEMSTATS (67)
|
.B SYS_MEMSTATS (67)
|
||||||
Get kernel-wide physical memory usage (total/free/used bytes,
|
Get kernel-wide physical memory usage (total/free/used bytes,
|
||||||
page size).
|
page size).
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
/*
|
||||||
|
* memtest - userspace heap and virtual-memory regression tests
|
||||||
|
* Copyright (c) 2026 Daniel Hammer
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <montauk/heap.h>
|
||||||
|
#include <montauk/syscall.h>
|
||||||
|
#include <montauk/thread.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
int failures = 0;
|
||||||
|
|
||||||
|
void check(bool condition, const char* name) {
|
||||||
|
montauk::print(condition ? "PASS " : "FAIL ");
|
||||||
|
montauk::print(name);
|
||||||
|
montauk::putchar('\n');
|
||||||
|
if (!condition) failures++;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool bytes_are(const uint8_t* p, size_t n, uint8_t value) {
|
||||||
|
for (size_t i = 0; i < n; i++)
|
||||||
|
if (p[i] != value) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WorkerArgs {
|
||||||
|
volatile uint32_t* failures;
|
||||||
|
uint8_t seed;
|
||||||
|
};
|
||||||
|
|
||||||
|
int heap_worker(void* opaque) {
|
||||||
|
auto* args = (WorkerArgs*)opaque;
|
||||||
|
for (int round = 0; round < 256; round++) {
|
||||||
|
size_t size = (size_t)((round * 37 + args->seed) % 8192 + 1);
|
||||||
|
auto* p = (uint8_t*)montauk::malloc(size);
|
||||||
|
if (p == nullptr) {
|
||||||
|
__atomic_fetch_add(args->failures, 1, __ATOMIC_RELAXED);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
uint8_t value = (uint8_t)(args->seed + round);
|
||||||
|
for (size_t i = 0; i < size; i++) p[i] = value;
|
||||||
|
size_t grown = size + (size_t)(round % 97);
|
||||||
|
auto* q = (uint8_t*)montauk::realloc(p, grown);
|
||||||
|
if (q == nullptr || !bytes_are(q, size, value)) {
|
||||||
|
__atomic_fetch_add(args->failures, 1, __ATOMIC_RELAXED);
|
||||||
|
if (q != nullptr) montauk::mfree(q);
|
||||||
|
else montauk::mfree(p);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
montauk::mfree(q);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extern "C" void _start() {
|
||||||
|
// Alignment, reuse, and data integrity across all current size classes.
|
||||||
|
void* blocks[512] = {};
|
||||||
|
bool basic_ok = true;
|
||||||
|
int block_count = 0;
|
||||||
|
for (int i = 0; i < 512; i++) {
|
||||||
|
size_t size = (size_t)((i * 53) % 12000);
|
||||||
|
blocks[i] = montauk::malloc(size);
|
||||||
|
if (blocks[i] == nullptr || ((uintptr_t)blocks[i] & 15) != 0) {
|
||||||
|
basic_ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
block_count++;
|
||||||
|
uint8_t value = (uint8_t)i;
|
||||||
|
for (size_t j = 0; j < size; j++) ((uint8_t*)blocks[i])[j] = value;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < block_count; i += 2) {
|
||||||
|
montauk::mfree(blocks[i]);
|
||||||
|
blocks[i] = nullptr;
|
||||||
|
}
|
||||||
|
for (int i = 1; i < block_count; i += 2) {
|
||||||
|
size_t size = (size_t)((i * 53) % 12000);
|
||||||
|
if (!bytes_are((uint8_t*)blocks[i], size, (uint8_t)i)) basic_ok = false;
|
||||||
|
montauk::mfree(blocks[i]);
|
||||||
|
}
|
||||||
|
check(basic_ok, "size classes preserve data and 16-byte alignment");
|
||||||
|
|
||||||
|
auto* overflow = (uint8_t*)montauk::malloc(64);
|
||||||
|
for (int i = 0; i < 64; i++) overflow[i] = 0xA5;
|
||||||
|
void* rejected = montauk::realloc(overflow, UINT64_MAX);
|
||||||
|
check(rejected == nullptr && bytes_are(overflow, 64, 0xA5),
|
||||||
|
"realloc overflow fails without altering the old allocation");
|
||||||
|
montauk::mfree(overflow);
|
||||||
|
|
||||||
|
auto* shrink = (uint8_t*)montauk::malloc(32000);
|
||||||
|
for (int i = 0; i < 32000; i++) shrink[i] = (uint8_t)i;
|
||||||
|
auto* shrunk = (uint8_t*)montauk::realloc(shrink, 4097);
|
||||||
|
check(shrunk == shrink, "large in-arena realloc shrink retains its extent");
|
||||||
|
// The first byte sequence is not uniformly zero; validate explicitly.
|
||||||
|
bool shrink_data_ok = true;
|
||||||
|
for (int i = 0; i < 4097; i++)
|
||||||
|
if (shrunk[i] != (uint8_t)i) { shrink_data_ok = false; break; }
|
||||||
|
check(shrink_data_ok, "realloc shrink preserves payload");
|
||||||
|
montauk::mfree(shrunk);
|
||||||
|
|
||||||
|
auto* direct1 = (uint8_t*)montauk::malloc(512 * 1024);
|
||||||
|
uintptr_t direct_addr = (uintptr_t)direct1;
|
||||||
|
if (direct1) direct1[511 * 1024] = 0x6D;
|
||||||
|
montauk::mfree(direct1);
|
||||||
|
auto* direct2 = (uint8_t*)montauk::malloc(512 * 1024);
|
||||||
|
check(direct2 != nullptr && (uintptr_t)direct2 == direct_addr,
|
||||||
|
"large allocation releases and reuses its VM range");
|
||||||
|
|
||||||
|
auto* raw1 = (uint8_t*)montauk::alloc(3 * 4096);
|
||||||
|
uintptr_t raw_addr = (uintptr_t)raw1;
|
||||||
|
bool raw_split = raw1 != nullptr &&
|
||||||
|
mprotect(raw1, 4096, PROT_READ) == 0;
|
||||||
|
montauk::free(raw1);
|
||||||
|
auto* raw2 = (uint8_t*)montauk::alloc(3 * 4096);
|
||||||
|
check(raw_split && raw2 != nullptr && (uintptr_t)raw2 == raw_addr,
|
||||||
|
"SYS_FREE releases every fragment after mprotect splitting");
|
||||||
|
|
||||||
|
auto* map = (uint8_t*)mmap(nullptr, 3 * 4096, PROT_READ | PROT_WRITE,
|
||||||
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||||
|
bool map_ok = map != MAP_FAILED && ((uintptr_t)map & 0xFFF) == 0 &&
|
||||||
|
bytes_are(map, 3 * 4096, 0);
|
||||||
|
check(map_ok, "anonymous mappings are aligned and zero-filled");
|
||||||
|
if (map != MAP_FAILED) {
|
||||||
|
map[0] = 1;
|
||||||
|
map[8192] = 2;
|
||||||
|
check(mprotect(map, 4096, PROT_READ) == 0 &&
|
||||||
|
mprotect(map, 4096, PROT_READ | PROT_WRITE) == 0,
|
||||||
|
"mprotect changes mapped page permissions");
|
||||||
|
check(mprotect(map, 3 * 4096, PROT_READ) == 0 &&
|
||||||
|
mprotect(map, 3 * 4096, PROT_READ | PROT_WRITE) == 0,
|
||||||
|
"mprotect spans adjacent VMA fragments");
|
||||||
|
check(mprotect(map, 4096, PROT_READ | PROT_WRITE | PROT_EXEC) < 0,
|
||||||
|
"mprotect rejects writable executable memory");
|
||||||
|
void* middle = map + 4096;
|
||||||
|
check(munmap(middle, 4096) == 0, "munmap supports VMA splitting");
|
||||||
|
void* replacement = mmap(nullptr, 4096, PROT_READ | PROT_WRITE,
|
||||||
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||||
|
check(replacement == middle, "munmap makes virtual ranges reusable");
|
||||||
|
if (replacement != MAP_FAILED) munmap(replacement, 4096);
|
||||||
|
munmap(map, 4096);
|
||||||
|
munmap(map + 8192, 4096);
|
||||||
|
}
|
||||||
|
// Keep this extent occupied until the partial-unmap test above has
|
||||||
|
// verified that its own hole is the first reusable range.
|
||||||
|
montauk::free(raw2);
|
||||||
|
montauk::mfree(direct2);
|
||||||
|
|
||||||
|
volatile uint32_t worker_failures = 0;
|
||||||
|
WorkerArgs args[4] = {};
|
||||||
|
int tids[4] = {};
|
||||||
|
bool threads_ok = true;
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
args[i] = { &worker_failures, (uint8_t)(17 + i * 31) };
|
||||||
|
tids[i] = montauk::thread_spawn(heap_worker, &args[i]);
|
||||||
|
if (tids[i] < 0) threads_ok = false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
int code = -1;
|
||||||
|
if (tids[i] >= 0 && (montauk::thread_join(tids[i], &code) < 0 || code != 0))
|
||||||
|
threads_ok = false;
|
||||||
|
}
|
||||||
|
check(threads_ok && worker_failures == 0,
|
||||||
|
"concurrent malloc/realloc/free stress");
|
||||||
|
|
||||||
|
montauk::print(failures == 0 ? "memtest: all tests passed\n"
|
||||||
|
: "memtest: failures detected\n");
|
||||||
|
montauk::exit(failures == 0 ? 0 : 1);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user