/* * Ipc.cpp * Kernel-mode IPC implementation * Copyright (c) 2026 Daniel Hammer */ #include "Ipc.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace Ipc { static constexpr int MaxStreams = 64; static constexpr int MaxMailboxes = 64; static constexpr int MaxMailboxMessages = 64; static constexpr int MaxMailboxMessageBytes = 128; static constexpr int MaxFiles = 128; static constexpr int MaxSockets = 64; static constexpr uint32_t UdpRingSize = 4096; static constexpr int MaxSurfaces = 32; static constexpr int MaxSurfacePages = 8192; static_assert((uint64_t)MaxSurfacePages * 0x1000ULL <= Sched::UserSurfaceSlotSize); static_assert(MaxSurfaceMapsPerProcess == Sched::UserSurfaceSlots); static constexpr int MaxProcessObjects = 512; static constexpr int MaxWaitsets = 32; static constexpr int MaxWaitsetEntries = 16; static constexpr int SocketTypeTcp = 1; static constexpr int SocketTypeUdp = 2; struct Object { HandleType type; bool active; bool destroying; uint32_t refs; kcp::Mutex lock; }; struct Stream : Object { uint8_t* buffer; uint32_t capacity; uint32_t head; uint32_t tail; uint32_t count; uint32_t readerRefs; uint32_t writerRefs; }; struct MailboxMessage { uint32_t type; uint16_t size; uint8_t attachmentType; uint8_t hasAttachment; uint32_t attachmentRights; Object* attachmentObject; uint8_t data[MaxMailboxMessageBytes]; }; struct Mailbox : Object { MailboxMessage messages[MaxMailboxMessages]; uint32_t head; uint32_t tail; uint32_t count; uint32_t senderRefs; uint32_t receiverRefs; }; struct File : Object { Fs::Vfs::BackendFile backend; }; struct UdpDgramHeader { uint32_t srcIp; uint16_t srcPort; uint16_t dataLen; }; struct Socket : Object { int socketType; Net::Tcp::Connection* tcpConn; uint16_t localPort; bool udpBound; uint8_t _pad[1]; uint8_t udpRing[UdpRingSize]; uint32_t udpHead; uint32_t udpTail; uint32_t udpCount; kcp::Spinlock socketLock; }; struct Surface : Object { uint64_t physPages[MaxSurfacePages]; uint32_t numPages; uint64_t sizeBytes; }; struct ProcessObject : Object { int pid; bool exited; }; struct WaitsetEntry { bool used; bool retiring; uint32_t readers; HandleType type; Object* object; uint32_t rights; uint32_t signals; }; struct Waitset : Object { kcp::Spinlock entriesLock; WaitsetEntry entries[MaxWaitsetEntries]; }; struct SurfaceMap { bool used; Surface* surface; uint64_t va; uint32_t numPages; }; static inline uint32_t* SurfacePixelPtr(Surface* surface, uint64_t pixelIndex) { if (surface == nullptr) return nullptr; uint64_t byteOffset = pixelIndex * sizeof(uint32_t); uint32_t pageIndex = (uint32_t)(byteOffset / 0x1000ULL); uint32_t pageOffset = (uint32_t)(byteOffset % 0x1000ULL); if (pageIndex >= surface->numPages) return nullptr; // A 0 physPage means the slot was freed but not yet repopulated // (e.g. transiently during a failed ResizeSurface). HHDM(0) is a // valid kernel virtual address, so returning it would silently // dereference into kernel memory; return nullptr instead. uint64_t phys = surface->physPages[pageIndex]; if (phys == 0) return nullptr; return (uint32_t*)((uint8_t*)Memory::HHDM(phys) + pageOffset); } static HandleEntry g_handleTables[Sched::MaxProcesses][MaxHandlesPerProcess] = {}; static uint64_t g_handleBitmaps[Sched::MaxProcesses][2] = {}; // 128-bit bitmap per process static kcp::Mutex g_handleTableLocks[Sched::MaxProcesses]; static SurfaceMap g_surfaceMaps[Sched::MaxProcesses][MaxSurfaceMapsPerProcess] = {}; static kcp::Mutex g_surfaceMapLocks[Sched::MaxProcesses]; static Stream g_streams[MaxStreams] = {}; static Mailbox g_mailboxes[MaxMailboxes] = {}; static File g_files[MaxFiles] = {}; static Socket g_sockets[MaxSockets] = {}; static Surface g_surfaces[MaxSurfaces] = {}; static ProcessObject g_processObjects[MaxProcessObjects] = {}; static Waitset g_waitsets[MaxWaitsets] = {}; static ProcessObject* g_processObjectsBySlot[Sched::MaxProcesses] = {}; static kcp::Mutex g_streamPoolLock; static kcp::Mutex g_mailboxPoolLock; static kcp::Mutex g_filePoolLock; static kcp::Mutex g_socketPoolLock; static kcp::Mutex g_surfacePoolLock; static kcp::Mutex g_processPoolLock; static kcp::Mutex g_waitsetPoolLock; static kcp::Mutex g_ephemeralPortLock; static uint16_t g_nextEphemeralPort = 49152; static void ReleaseRawObject(Object* object); // MUST be a Mutex, never a Spinlock. ShootdownUserRange holds this while // waiting for remote CPUs to acknowledge the shootdown IPI, so a CPU that // is queued behind the holder has to stay interruptible long enough to // service that IPI itself. An interrupt-disabling Spinlock here deadlocks // every CPU contending for the lock, and the bounded-retry logic below // then reports it as a "target failed to acknowledge" Panic -- which reads // like a hardware fault rather than a lock-type regression. static kcp::Mutex g_tlbShootdownLock; static volatile uint64_t g_tlbShootdownSeq = 0; static volatile uint64_t g_tlbShootdownPml4 = 0; static volatile uint64_t g_tlbShootdownStartVa = 0; static volatile uint32_t g_tlbShootdownPages = 0; static volatile uint64_t g_tlbShootdownDone[Smp::MaxCPUs] = {}; static bool CpuCurrentlyUsesPml4(Smp::CpuData* cpu, uint64_t pml4Phys) { if (cpu == nullptr || pml4Phys == 0 || cpu->currentSlot < 0) return false; Sched::Process* proc = Sched::GetProcessSlot(cpu->currentSlot); if (proc == nullptr) return false; if (proc->state == Sched::ProcessState::Free) return false; return proc->pml4Phys == pml4Phys; } static void InvalidateLocalUserRange(uint64_t startVa, uint32_t pages) { if (pages == 0) return; if (pages > 1024) { Memory::VMM::FlushTLB(); return; } for (uint32_t p = 0; p < pages; p++) { uint64_t va = startVa + (uint64_t)p * 0x1000ULL; asm volatile("invlpg (%0)" :: "r"(va) : "memory"); } } static void TlbShootdownIpiHandler(uint8_t, bool) { Smp::CpuData* cpu = Smp::GetCurrentCpuData(); uint64_t seq = g_tlbShootdownSeq; uint64_t pml4 = g_tlbShootdownPml4; uint64_t startVa = g_tlbShootdownStartVa; uint32_t pages = g_tlbShootdownPages; if (CpuCurrentlyUsesPml4(cpu, pml4)) { InvalidateLocalUserRange(startVa, pages); } if (cpu != nullptr && cpu->cpuIndex >= 0 && cpu->cpuIndex < Smp::MaxCPUs) { asm volatile("" ::: "memory"); g_tlbShootdownDone[cpu->cpuIndex] = seq; } } void ShootdownUserRange(uint64_t pml4Phys, uint64_t startVa, uint32_t pages) { if (pml4Phys == 0 || pages == 0) return; bool targets[Smp::MaxCPUs] = {}; Smp::CpuData* currentCpu = Smp::GetCurrentCpuData(); int currentCpuIndex = currentCpu ? currentCpu->cpuIndex : -1; g_tlbShootdownLock.Acquire(); uint64_t seq = g_tlbShootdownSeq + 1; g_tlbShootdownPml4 = pml4Phys; g_tlbShootdownStartVa = startVa; g_tlbShootdownPages = pages; asm volatile("" ::: "memory"); g_tlbShootdownSeq = seq; for (int i = 0; i < Smp::GetCpuCount(); i++) { Smp::CpuData* cpu = Smp::GetCpuData(i); if (cpu == nullptr || !cpu->started) continue; if (i == currentCpuIndex) { if (CpuCurrentlyUsesPml4(cpu, pml4Phys)) { InvalidateLocalUserRange(startVa, pages); } g_tlbShootdownDone[i] = seq; continue; } if (!CpuCurrentlyUsesPml4(cpu, pml4Phys)) { g_tlbShootdownDone[i] = seq; continue; } targets[i] = true; (void)Hal::LocalApic::SendFixedIpi(cpu->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_TLB_SHOOTDOWN); } for (int i = 0; i < Smp::GetCpuCount(); i++) { if (!targets[i]) continue; uint32_t spins = 0; uint32_t retries = 0; while (g_tlbShootdownDone[i] != seq) { asm volatile("pause"); if (++spins < 1000000) continue; // Delivery normally completes in a handful of cycles. Retry a // bounded number of times in case the first IPI was lost while // the target changed interrupt state. Continuing without an // acknowledgement would let the caller free frames still // reachable through a remote stale TLB entry, so fail loudly // instead of either corrupting memory or spinning forever. spins = 0; if (++retries > 4) { Panic("TLB shootdown target failed to acknowledge", nullptr); } Smp::CpuData* cpu = Smp::GetCpuData(i); if (cpu != nullptr && cpu->started) { (void)Hal::LocalApic::SendFixedIpi(cpu->lapicId, Hal::IRQ_VECTOR_BASE + Hal::IRQ_TLB_SHOOTDOWN); } } } g_tlbShootdownLock.Release(); } void UnmapAndFreeUserRange(uint64_t pml4Phys, uint64_t startVa, uint64_t pages) { static constexpr uint32_t PagesPerChunk = 64; uint64_t physPages[PagesPerChunk]; for (uint64_t base = 0; base < pages; base += PagesPerChunk) { uint32_t count = (uint32_t)((pages - base > PagesPerChunk) ? PagesPerChunk : pages - base); for (uint32_t i = 0; i < count; i++) { uint64_t pageVa = startVa + (base + i) * 0x1000ULL; physPages[i] = Memory::VMM::Paging::GetPhysAddr(pml4Phys, pageVa); Memory::VMM::Paging::UnmapUserIn(pml4Phys, pageVa); } ShootdownUserRange(pml4Phys, startVa + base * 0x1000ULL, count); for (uint32_t i = 0; i < count; i++) { if (physPages[i] != 0) { Memory::g_pfa->Free((void*)Memory::HHDM(physPages[i])); } } } } static void InitObject(Object& object, HandleType type) { object.type = type; object.active = true; object.destroying = false; object.refs = 0; } static kcp::Mutex* PoolLockForType(HandleType type) { switch (type) { case HandleType::Stream: return &g_streamPoolLock; case HandleType::Mailbox: return &g_mailboxPoolLock; case HandleType::File: return &g_filePoolLock; case HandleType::Socket: return &g_socketPoolLock; case HandleType::Surface: return &g_surfacePoolLock; case HandleType::Process: return &g_processPoolLock; case HandleType::Waitset: return &g_waitsetPoolLock; default: return nullptr; } } // An object is hidden from users as soon as its last reference disappears, // but its pool slot must not become reusable until the type-specific // destructor has finished. Otherwise a concurrent allocator can install a // new backend in the slot and the old destructor will tear down the new // object (observed as open() succeeding immediately before getsize() saw // an invalid file handle). static void FinishDestroy(Object* object, HandleType type) { kcp::Mutex* poolLock = PoolLockForType(type); if (poolLock) poolLock->Acquire(); object->lock.Acquire(); object->destroying = false; object->lock.Release(); if (poolLock) poolLock->Release(); } static void RetainRawObject(Object* object) { if (object == nullptr) return; object->lock.Acquire(); object->refs++; object->lock.Release(); } static void DestroyStream(Stream* stream) { if (stream->buffer != nullptr) { int numPages = (int)((stream->capacity + 0xFFFu) / 0x1000u); Memory::g_pfa->Free(stream->buffer, numPages); stream->buffer = nullptr; } stream->capacity = 0; stream->head = 0; stream->tail = 0; stream->count = 0; stream->readerRefs = 0; stream->writerRefs = 0; } static void DestroyFile(File* file) { if (file == nullptr) return; Fs::Vfs::CloseBackendFile(file->backend); file->backend.driveNumber = -1; file->backend.localHandle = -1; } static void DestroyMailbox(Mailbox* mailbox) { if (mailbox == nullptr) return; for (uint32_t i = 0; i < MaxMailboxMessages; i++) { if (!mailbox->messages[i].hasAttachment || mailbox->messages[i].attachmentObject == nullptr) continue; ReleaseRawObject(mailbox->messages[i].attachmentObject); mailbox->messages[i].attachmentObject = nullptr; mailbox->messages[i].attachmentType = (uint8_t)HandleType::None; mailbox->messages[i].attachmentRights = 0; mailbox->messages[i].hasAttachment = 0; } mailbox->head = 0; mailbox->tail = 0; mailbox->count = 0; mailbox->senderRefs = 0; mailbox->receiverRefs = 0; } static void DestroySocket(Socket* socket) { if (socket == nullptr) return; socket->socketLock.Acquire(); Net::Tcp::Connection* tcpConn = socket->tcpConn; bool udpBound = socket->udpBound; uint16_t localPort = socket->localPort; socket->tcpConn = nullptr; socket->udpBound = false; socket->localPort = 0; socket->udpHead = 0; socket->udpTail = 0; socket->udpCount = 0; socket->socketType = 0; socket->socketLock.Release(); if (tcpConn != nullptr) { Net::Tcp::Close(tcpConn); } if (udpBound && localPort != 0) { Net::Udp::Unbind(localPort); } } static void DestroySurface(Surface* surface) { for (uint32_t i = 0; i < surface->numPages; i++) { if (surface->physPages[i] != 0) { Memory::g_pfa->Free((void*)Memory::HHDM(surface->physPages[i])); surface->physPages[i] = 0; } } surface->numPages = 0; surface->sizeBytes = 0; } static void DestroyWaitset(Waitset* waitset) { for (int i = 0; i < MaxWaitsetEntries; i++) { Object* target = nullptr; waitset->entriesLock.Acquire(); WaitsetEntry& entry = waitset->entries[i]; entry.used = false; entry.retiring = entry.object != nullptr; waitset->entriesLock.Release(); // IRQ-side notification readers never retain/release the target; // the entry's permanent reference remains valid until all of them // have left this slot. for (;;) { waitset->entriesLock.Acquire(); if (entry.readers == 0) { target = entry.object; entry.object = nullptr; entry.rights = 0; entry.signals = 0; entry.type = HandleType::None; entry.retiring = false; waitset->entriesLock.Release(); break; } waitset->entriesLock.Release(); asm volatile("pause"); } ReleaseRawObject(target); } } static void ReleaseRawObject(Object* object) { if (object == nullptr) return; bool destroy = false; HandleType type = HandleType::None; object->lock.Acquire(); if (object->refs > 0) { object->refs--; } if (object->refs == 0) { destroy = true; type = object->type; object->active = false; object->destroying = true; } object->lock.Release(); if (!destroy) return; switch (type) { case HandleType::Stream: DestroyStream((Stream*)object); break; case HandleType::Mailbox: DestroyMailbox((Mailbox*)object); break; case HandleType::File: DestroyFile((File*)object); break; case HandleType::Socket: DestroySocket((Socket*)object); break; case HandleType::Surface: DestroySurface((Surface*)object); break; case HandleType::Process: break; case HandleType::Waitset: DestroyWaitset((Waitset*)object); break; default: break; } FinishDestroy(object, type); } static uint32_t CurrentSignalsForSnapshot(HandleType type, Object* object, uint32_t rights); static uint32_t CurrentSocketSignals(Socket* socket, uint32_t rights); static bool WaitsetCheckReady(Waitset* waitset, WaitsetReady* outReady); static uint16_t AllocEphemeralPort() { g_ephemeralPortLock.Acquire(); uint16_t port = g_nextEphemeralPort++; if (g_nextEphemeralPort == 0) g_nextEphemeralPort = 49152; g_ephemeralPortLock.Release(); return port; } static void UdpSocketDispatcher(uint32_t srcIp, uint16_t srcPort, uint16_t dstPort, const uint8_t* data, uint16_t length) { for (int i = 0; i < MaxSockets; i++) { if (!g_sockets[i].active || g_sockets[i].socketType != SocketTypeUdp) continue; g_sockets[i].socketLock.Acquire(); if (!g_sockets[i].udpBound || g_sockets[i].localPort != dstPort) { g_sockets[i].socketLock.Release(); continue; } uint32_t needed = sizeof(UdpDgramHeader) + length; if (g_sockets[i].udpCount + needed > UdpRingSize) { g_sockets[i].socketLock.Release(); return; } UdpDgramHeader hdr = {srcIp, srcPort, length}; uint32_t tail = g_sockets[i].udpTail; uint32_t hdrLen = sizeof(UdpDgramHeader); uint32_t first = (tail + hdrLen <= UdpRingSize) ? hdrLen : (UdpRingSize - tail); memcpy(g_sockets[i].udpRing + tail, (const uint8_t*)&hdr, first); if (first < hdrLen) memcpy(g_sockets[i].udpRing, ((const uint8_t*)&hdr) + first, hdrLen - first); tail = (tail + hdrLen) % UdpRingSize; // Write data payload uint32_t second = (tail + length <= UdpRingSize) ? length : (UdpRingSize - tail); memcpy(g_sockets[i].udpRing + tail, data, second); if (second < length) memcpy(g_sockets[i].udpRing, data + second, length - second); g_sockets[i].udpTail = (tail + length) % UdpRingSize; g_sockets[i].udpCount += needed; g_sockets[i].socketLock.Release(); NotifyObjectChanged((Object*)&g_sockets[i]); return; } } int CurrentSlot() { auto* proc = Sched::GetCurrentProcessPtr(); if (proc == nullptr) return -1; auto* slot0 = Sched::GetProcessSlot(0); return (int)(proc - slot0); } int SlotForPid(int pid) { for (int i = 0; i < Sched::MaxProcesses; i++) { auto* proc = Sched::GetProcessSlot(i); if (proc == nullptr) continue; auto state = proc->state; if (state == Sched::ProcessState::Free) continue; if (proc->pid == pid) return i; } return -1; } static void* AllocContiguousPages(int numPages) { if (numPages <= 0) return nullptr; if (numPages == 1) return Memory::g_pfa->AllocateZeroed(); // ReallocConsecutive(nullptr, n) allocates an n-page contiguous span // directly. The previous implementation always allocated a single // throwaway page first, then asked ReallocConsecutive to migrate from // it -- which copies and frees the throwaway page for no benefit. void* span = Memory::g_pfa->ReallocConsecutive(nullptr, numPages); if (span == nullptr) return nullptr; memset(span, 0, (size_t)numPages * 0x1000); return span; } static bool RetainForHandle(Object* object, HandleType type, uint32_t rights) { if (object == nullptr) return false; switch (type) { case HandleType::Stream: { auto* stream = (Stream*)object; stream->lock.Acquire(); if (!stream->active || stream->destroying) { stream->lock.Release(); return false; } stream->refs++; if (rights & RightRead) stream->readerRefs++; if (rights & RightWrite) stream->writerRefs++; stream->lock.Release(); return true; } case HandleType::Mailbox: { auto* mailbox = (Mailbox*)object; mailbox->lock.Acquire(); if (!mailbox->active || mailbox->destroying) { mailbox->lock.Release(); return false; } mailbox->refs++; if (rights & RightSend) mailbox->senderRefs++; if (rights & RightRecv) mailbox->receiverRefs++; mailbox->lock.Release(); return true; } default: { object->lock.Acquire(); if (!object->active || object->destroying) { object->lock.Release(); return false; } object->refs++; object->lock.Release(); return true; } } } static void ReleaseForHandle(Object* object, HandleType type, uint32_t rights) { if (object == nullptr) return; bool notify = false; bool destroy = false; switch (type) { case HandleType::Stream: { auto* stream = (Stream*)object; stream->lock.Acquire(); if (rights & RightRead && stream->readerRefs > 0) { stream->readerRefs--; notify = true; } if (rights & RightWrite && stream->writerRefs > 0) { stream->writerRefs--; notify = true; } if (stream->refs > 0) stream->refs--; if (stream->refs == 0) { destroy = true; stream->active = false; stream->destroying = true; } stream->lock.Release(); if (notify) NotifyObjectChanged((Object*)stream); if (destroy) { DestroyStream(stream); FinishDestroy((Object*)stream, HandleType::Stream); } break; } case HandleType::Mailbox: { auto* mailbox = (Mailbox*)object; mailbox->lock.Acquire(); if (rights & RightSend && mailbox->senderRefs > 0) { mailbox->senderRefs--; notify = true; } if (rights & RightRecv && mailbox->receiverRefs > 0) { mailbox->receiverRefs--; notify = true; } if (mailbox->refs > 0) mailbox->refs--; if (mailbox->refs == 0) { destroy = true; mailbox->active = false; mailbox->destroying = true; } mailbox->lock.Release(); if (notify) NotifyObjectChanged((Object*)mailbox); if (destroy) { DestroyMailbox(mailbox); FinishDestroy((Object*)mailbox, HandleType::Mailbox); } break; } default: ReleaseRawObject(object); break; } } int InstallHandleForSlot(int slot, Object* object, HandleType type, uint32_t rights) { if (slot < 0 || slot >= Sched::MaxProcesses || object == nullptr) return -1; g_handleTableLocks[slot].Acquire(); uint64_t& bm0 = g_handleBitmaps[slot][0]; uint64_t& bm1 = g_handleBitmaps[slot][1]; // Find first zero bit in bitmap (free slot) uint64_t bits0 = ~bm0; uint64_t bits1 = ~bm1; if (bits0) { int i = __builtin_ctzll(bits0); if (!RetainForHandle(object, type, rights)) { g_handleTableLocks[slot].Release(); return -1; } bm0 |= (1ULL << i); g_handleTables[slot][i].used = true; g_handleTables[slot][i].rights = rights; g_handleTables[slot][i].type = type; g_handleTables[slot][i].object = object; g_handleTableLocks[slot].Release(); return i; } else if (bits1) { int i = 64 + __builtin_ctzll(bits1); if (!RetainForHandle(object, type, rights)) { g_handleTableLocks[slot].Release(); return -1; } bm1 |= (1ULL << (i - 64)); g_handleTables[slot][i].used = true; g_handleTables[slot][i].rights = rights; g_handleTables[slot][i].type = type; g_handleTables[slot][i].object = object; g_handleTableLocks[slot].Release(); return i; } g_handleTableLocks[slot].Release(); return -1; } bool HandleSnapshot::Capture(int slot, int handle) { if (object != nullptr) return false; if (slot < 0 || slot >= Sched::MaxProcesses) return false; if (handle < 0 || handle >= MaxHandlesPerProcess) return false; g_handleTableLocks[slot].Acquire(); const HandleEntry& entry = g_handleTables[slot][handle]; if (!entry.used || entry.object == nullptr) { g_handleTableLocks[slot].Release(); return false; } type = entry.type; object = entry.object; rights = entry.rights; RetainRawObject(object); g_handleTableLocks[slot].Release(); return true; } HandleSnapshot::~HandleSnapshot() { ReleaseRawObject(object); } int CloseHandleForSlot(int slot, int handle) { if (slot < 0 || slot >= Sched::MaxProcesses) return -1; if (handle < 0 || handle >= MaxHandlesPerProcess) return -1; g_handleTableLocks[slot].Acquire(); if (!g_handleTables[slot][handle].used) { g_handleTableLocks[slot].Release(); return -1; } HandleEntry entry = g_handleTables[slot][handle]; g_handleTables[slot][handle].used = false; g_handleTables[slot][handle].rights = 0; g_handleTables[slot][handle].type = HandleType::None; g_handleTables[slot][handle].object = nullptr; // Clear bitmap bit if (handle < 64) g_handleBitmaps[slot][0] &= ~(1ULL << handle); else g_handleBitmaps[slot][1] &= ~(1ULL << (handle - 64)); g_handleTableLocks[slot].Release(); ReleaseForHandle(entry.object, entry.type, entry.rights); return 0; } int CloseHandle(int handle) { return CloseHandleForSlot(CurrentSlot(), handle); } int DupHandle(int handle) { HandleSnapshot snapshot; int slot = CurrentSlot(); if (!snapshot.Capture(slot, handle)) return -1; if ((snapshot.rights & RightDup) == 0) return -1; return InstallHandleForSlot(slot, snapshot.object, snapshot.type, snapshot.rights); } Stream* CreateStream(uint32_t capacity) { if (capacity == 0) capacity = DefaultStreamCapacity; int numPages = (int)((capacity + 0xFFFu) / 0x1000u); void* buffer = AllocContiguousPages(numPages); if (buffer == nullptr) return nullptr; g_streamPoolLock.Acquire(); for (int i = 0; i < MaxStreams; i++) { if (g_streams[i].active || g_streams[i].destroying) continue; InitObject(g_streams[i], HandleType::Stream); g_streams[i].buffer = (uint8_t*)buffer; g_streams[i].capacity = (uint32_t)numPages * 0x1000u; g_streams[i].head = 0; g_streams[i].tail = 0; g_streams[i].count = 0; g_streams[i].readerRefs = 0; g_streams[i].writerRefs = 0; g_streamPoolLock.Release(); return &g_streams[i]; } g_streamPoolLock.Release(); Memory::g_pfa->Free(buffer, numPages); return nullptr; } int CreateStreamHandlePairForSlot(int slot, uint32_t capacity, int& outReadHandle, int& outWriteHandle) { outReadHandle = -1; outWriteHandle = -1; Stream* stream = CreateStream(capacity); if (stream == nullptr) return -1; int readHandle = InstallHandleForSlot(slot, (Object*)stream, HandleType::Stream, RightRead | RightWait | RightDup); if (readHandle < 0) { DestroyStream(stream); stream->active = false; return -1; } int writeHandle = InstallHandleForSlot(slot, (Object*)stream, HandleType::Stream, RightWrite | RightWait | RightDup); if (writeHandle < 0) { CloseHandleForSlot(slot, readHandle); return -1; } outReadHandle = readHandle; outWriteHandle = writeHandle; return 0; } int CreateStreamHandlePair(uint32_t capacity, int& outReadHandle, int& outWriteHandle) { return CreateStreamHandlePairForSlot(CurrentSlot(), capacity, outReadHandle, outWriteHandle); } void RetainStream(Stream* stream, bool readSide, bool writeSide) { if (stream == nullptr) return; stream->lock.Acquire(); stream->refs++; if (readSide) stream->readerRefs++; if (writeSide) stream->writerRefs++; stream->lock.Release(); } void ReleaseStream(Stream* stream, bool readSide, bool writeSide) { if (stream == nullptr) return; ReleaseForHandle((Object*)stream, HandleType::Stream, (readSide ? (uint32_t)RightRead : 0u) | (writeSide ? (uint32_t)RightWrite : 0u)); } int StreamRead(Stream* stream, uint8_t* out, int maxLen, bool /*nonBlocking*/) { if (stream == nullptr || out == nullptr || maxLen <= 0) return -1; stream->lock.Acquire(); if (stream->count == 0) { bool closed = stream->writerRefs == 0; stream->lock.Release(); return closed ? -1 : 0; } int count = 0; uint32_t tail = stream->tail; uint32_t cap = stream->capacity; // Copy contiguous tail → end int first = (stream->count < (size_t)(cap - tail)) ? stream->count : (cap - tail); if (first > maxLen) first = maxLen; if (first > 0) { memcpy(out, stream->buffer + tail, first); tail = (tail + first) % cap; stream->count -= first; count = first; } // Copy remainder (wrapped) if needed and space permits if (count < maxLen && stream->count > 0) { int second = stream->count; if (second > maxLen - count) second = maxLen - count; if (second > 0) { memcpy(out + count, stream->buffer + tail, second); tail = (tail + second) % cap; stream->count -= second; count += second; } } stream->tail = tail; stream->lock.Release(); NotifyObjectChanged((Object*)stream); return count; } int StreamWrite(Stream* stream, const uint8_t* data, int len, bool /*nonBlocking*/) { if (stream == nullptr || data == nullptr || len <= 0) return -1; stream->lock.Acquire(); if (stream->readerRefs == 0) { stream->lock.Release(); return -1; } uint32_t head = stream->head; uint32_t cap = stream->capacity; uint32_t space = cap - stream->count; // Total bytes we can accept is bounded by both requested length and free space. // The previous implementation could write up to (cap - head) bytes whenever // len >= space, which exceeded `space` whenever cap-head > space and silently // corrupted the ring buffer (count > cap, overwritten unread data). uint32_t toWrite = ((uint32_t)len < space) ? (uint32_t)len : space; // First contiguous chunk: from head to end of buffer uint32_t first = (toWrite < (cap - head)) ? toWrite : (cap - head); if (first > 0) { memcpy(stream->buffer + head, data, first); head = (head + first) % cap; stream->count += first; } // Remainder wraps to the start of the buffer uint32_t second = toWrite - first; if (second > 0) { memcpy(stream->buffer + head, data + first, second); head = (head + second) % cap; stream->count += second; } stream->head = head; stream->lock.Release(); if (toWrite > 0) NotifyObjectChanged((Object*)stream); return (int)toWrite; } int StreamReadHandle(int handle, uint8_t* out, int maxLen) { if (out == nullptr) return -1; if (maxLen > 0 && !montauk::abi::UserMemory::Range((uint64_t)out, (uint64_t)maxLen, true)) return -1; HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return -1; if (snapshot.type != HandleType::Stream || (snapshot.rights & RightRead) == 0) return -1; return StreamRead((Stream*)snapshot.object, out, maxLen, true); } int StreamWriteHandle(int handle, const uint8_t* data, int len) { if (data == nullptr) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, (uint64_t)len, false)) return -1; HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return -1; if (snapshot.type != HandleType::Stream || (snapshot.rights & RightWrite) == 0) return -1; return StreamWrite((Stream*)snapshot.object, data, len, true); } bool StreamHasData(Stream* stream) { if (stream == nullptr) return false; stream->lock.Acquire(); bool hasData = stream->count > 0; stream->lock.Release(); return hasData; } Mailbox* CreateMailbox() { g_mailboxPoolLock.Acquire(); for (int i = 0; i < MaxMailboxes; i++) { if (g_mailboxes[i].active || g_mailboxes[i].destroying) continue; InitObject(g_mailboxes[i], HandleType::Mailbox); g_mailboxes[i].head = 0; g_mailboxes[i].tail = 0; g_mailboxes[i].count = 0; g_mailboxes[i].senderRefs = 0; g_mailboxes[i].receiverRefs = 0; for (int j = 0; j < MaxMailboxMessages; j++) { g_mailboxes[i].messages[j].type = 0; g_mailboxes[i].messages[j].size = 0; g_mailboxes[i].messages[j].attachmentType = (uint8_t)HandleType::None; g_mailboxes[i].messages[j].hasAttachment = 0; g_mailboxes[i].messages[j].attachmentRights = 0; g_mailboxes[i].messages[j].attachmentObject = nullptr; } g_mailboxPoolLock.Release(); return &g_mailboxes[i]; } g_mailboxPoolLock.Release(); return nullptr; } int CreateMailboxHandlePairForSlot(int slot, int& outSendHandle, int& outRecvHandle) { outSendHandle = -1; outRecvHandle = -1; Mailbox* mailbox = CreateMailbox(); if (mailbox == nullptr) return -1; int sendHandle = InstallHandleForSlot(slot, (Object*)mailbox, HandleType::Mailbox, RightSend | RightWait | RightDup); if (sendHandle < 0) { DestroyMailbox(mailbox); mailbox->active = false; return -1; } int recvHandle = InstallHandleForSlot(slot, (Object*)mailbox, HandleType::Mailbox, RightRecv | RightWait | RightDup); if (recvHandle < 0) { CloseHandleForSlot(slot, sendHandle); return -1; } outSendHandle = sendHandle; outRecvHandle = recvHandle; return 0; } int CreateMailboxHandlePair(int& outSendHandle, int& outRecvHandle) { return CreateMailboxHandlePairForSlot(CurrentSlot(), outSendHandle, outRecvHandle); } void RetainMailbox(Mailbox* mailbox, bool sender, bool receiver) { if (mailbox == nullptr) return; mailbox->lock.Acquire(); mailbox->refs++; if (sender) mailbox->senderRefs++; if (receiver) mailbox->receiverRefs++; mailbox->lock.Release(); } void ReleaseMailbox(Mailbox* mailbox, bool sender, bool receiver) { if (mailbox == nullptr) return; ReleaseForHandle((Object*)mailbox, HandleType::Mailbox, (sender ? (uint32_t)RightSend : 0u) | (receiver ? (uint32_t)RightRecv : 0u)); } static int MailboxSendInternal(Mailbox* mailbox, int senderSlot, uint32_t msgType, const void* data, uint16_t len, int attachHandle) { if (mailbox == nullptr) return -1; if (len > MaxMailboxMessageBytes) return -1; HandleSnapshot attachment; if (attachHandle >= 0) { if (senderSlot < 0) return -1; if (!attachment.Capture(senderSlot, attachHandle)) { return -1; } if ((attachment.rights & RightDup) == 0 || attachment.object == nullptr) return -1; // The queued message owns a separate reference after this syscall's // scoped handle snapshot is released. RetainRawObject(attachment.object); } mailbox->lock.Acquire(); if (mailbox->receiverRefs == 0) { mailbox->lock.Release(); if (attachment.object != nullptr) ReleaseRawObject(attachment.object); return -1; } if (mailbox->count >= MaxMailboxMessages) { mailbox->lock.Release(); if (attachment.object != nullptr) ReleaseRawObject(attachment.object); return 0; } MailboxMessage& msg = mailbox->messages[mailbox->head]; msg.type = msgType; msg.size = len; msg.attachmentType = (uint8_t)attachment.type; msg.hasAttachment = attachment.object != nullptr ? 1 : 0; msg.attachmentRights = attachment.rights; msg.attachmentObject = attachment.object; if (len > 0 && data != nullptr) { memcpy(msg.data, data, len); } mailbox->head = (mailbox->head + 1) % MaxMailboxMessages; mailbox->count++; mailbox->lock.Release(); NotifyObjectChanged((Object*)mailbox); return len; } int MailboxSend(Mailbox* mailbox, uint32_t msgType, const void* data, uint16_t len) { return MailboxSendInternal(mailbox, -1, msgType, data, len, -1); } int MailboxSendCoalescedMouse(Mailbox* mailbox, uint32_t msgType, const void* data, uint16_t len) { if (mailbox == nullptr || data == nullptr) return -1; if (len != sizeof(montauk::abi::WinEvent)) return MailboxSend(mailbox, msgType, data, len); const montauk::abi::WinEvent* incoming = (const montauk::abi::WinEvent*)data; if (incoming->type != 1 || incoming->mouse.scroll != 0) return MailboxSend(mailbox, msgType, data, len); bool incomingTransition = incoming->mouse.buttons != incoming->mouse.prev_buttons; if (incomingTransition) return MailboxSend(mailbox, msgType, data, len); mailbox->lock.Acquire(); if (mailbox->receiverRefs == 0) { mailbox->lock.Release(); return -1; } if (mailbox->count > 0) { uint32_t idx = (mailbox->head + MaxMailboxMessages - 1) % MaxMailboxMessages; MailboxMessage& latest = mailbox->messages[idx]; if (latest.type == msgType && latest.size == len && !latest.hasAttachment) { const montauk::abi::WinEvent* queued = (const montauk::abi::WinEvent*)latest.data; bool queuedTransition = queued->type != 1 || queued->mouse.scroll != 0 || queued->mouse.buttons != queued->mouse.prev_buttons; if (!queuedTransition) { memcpy(latest.data, data, len); mailbox->lock.Release(); NotifyObjectChanged((Object*)mailbox); return len; } } } mailbox->lock.Release(); return MailboxSend(mailbox, msgType, data, len); } static int MailboxRecvInternal(Mailbox* mailbox, int receiverSlot, uint32_t* msgType, void* data, uint16_t* inOutLen, int* outAttachHandle, bool /*nonBlocking*/) { if (mailbox == nullptr || inOutLen == nullptr) return -1; mailbox->lock.Acquire(); if (mailbox->count == 0) { bool closed = mailbox->senderRefs == 0; mailbox->lock.Release(); return closed ? -1 : 0; } MailboxMessage& queued = mailbox->messages[mailbox->tail]; if (queued.hasAttachment) { if (receiverSlot < 0 || outAttachHandle == nullptr) { mailbox->lock.Release(); return -1; } } // Remove the message before installing an attached handle. Installation // takes the receiver's handle-table lock and then the attached object's // lock; doing that while holding mailbox->lock would invert the global // table->object order used by HandleSnapshot::Capture. MailboxMessage msg = queued; mailbox->tail = (mailbox->tail + 1) % MaxMailboxMessages; mailbox->count--; queued.attachmentType = (uint8_t)HandleType::None; queued.hasAttachment = 0; queued.attachmentRights = 0; queued.attachmentObject = nullptr; mailbox->lock.Release(); int attachedHandle = -1; if (msg.hasAttachment) { attachedHandle = InstallHandleForSlot(receiverSlot, msg.attachmentObject, (HandleType)msg.attachmentType, msg.attachmentRights); if (attachedHandle < 0) { ReleaseRawObject(msg.attachmentObject); NotifyObjectChanged((Object*)mailbox); return -1; } } uint16_t copyLen = msg.size; if (copyLen > *inOutLen) copyLen = *inOutLen; if (copyLen > 0 && data != nullptr) { memcpy(data, msg.data, copyLen); } if (msgType != nullptr) *msgType = msg.type; *inOutLen = copyLen; if (outAttachHandle != nullptr) *outAttachHandle = attachedHandle; if (msg.hasAttachment && msg.attachmentObject != nullptr) { ReleaseRawObject(msg.attachmentObject); } NotifyObjectChanged((Object*)mailbox); return (int)copyLen; } int MailboxRecv(Mailbox* mailbox, uint32_t* msgType, void* data, uint16_t* inOutLen, bool nonBlocking) { return MailboxRecvInternal(mailbox, -1, msgType, data, inOutLen, nullptr, nonBlocking); } int MailboxSendHandle(int handle, uint32_t msgType, const void* data, uint16_t len, int attachHandle) { if (len > 0 && (data == nullptr || !montauk::abi::UserMemory::Range((uint64_t)data, len, false))) return -1; HandleSnapshot snapshot; int slot = CurrentSlot(); if (!snapshot.Capture(slot, handle)) return -1; if (snapshot.type != HandleType::Mailbox || (snapshot.rights & RightSend) == 0) return -1; return MailboxSendInternal((Mailbox*)snapshot.object, slot, msgType, data, len, attachHandle); } int MailboxRecvHandle(int handle, uint32_t* msgType, void* data, uint16_t* inOutLen, int* outAttachHandle) { if (inOutLen == nullptr || !montauk::abi::UserMemory::Writable((uint64_t)inOutLen)) return -1; uint16_t requestedLen = *inOutLen; if (msgType != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)msgType)) return -1; if (requestedLen > 0 && (data == nullptr || !montauk::abi::UserMemory::Range((uint64_t)data, requestedLen, true))) return -1; if (outAttachHandle != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)outAttachHandle)) return -1; HandleSnapshot snapshot; int slot = CurrentSlot(); if (!snapshot.Capture(slot, handle)) return -1; if (snapshot.type != HandleType::Mailbox || (snapshot.rights & RightRecv) == 0) return -1; return MailboxRecvInternal((Mailbox*)snapshot.object, slot, msgType, data, inOutLen, outAttachHandle, true); } bool MailboxHasMessage(Mailbox* mailbox) { if (mailbox == nullptr) return false; mailbox->lock.Acquire(); bool hasMsg = mailbox->count > 0; mailbox->lock.Release(); return hasMsg; } int OpenFileHandleForSlot(int slot, const char* path, bool create) { if (slot < 0 || slot >= Sched::MaxProcesses || path == nullptr) return -1; Fs::Vfs::BackendFile backend = {-1, -1, 0}; int result = create ? Fs::Vfs::CreateBackendFile(path, backend) : Fs::Vfs::OpenBackendFile(path, backend); if (result < 0) return -1; g_filePoolLock.Acquire(); for (int i = 0; i < MaxFiles; i++) { if (g_files[i].active || g_files[i].destroying) continue; InitObject(g_files[i], HandleType::File); g_files[i].backend = backend; g_filePoolLock.Release(); uint32_t rights = RightRead | RightWait | RightDup; if (Fs::Vfs::BackendFileCanWrite(g_files[i].backend)) { rights |= RightWrite; } int handle = InstallHandleForSlot(slot, (Object*)&g_files[i], HandleType::File, rights); if (handle >= 0) return handle; DestroyFile(&g_files[i]); g_files[i].active = false; return -1; } g_filePoolLock.Release(); Fs::Vfs::CloseBackendFile(backend); return -1; } int OpenFileHandle(const char* path) { return OpenFileHandleForSlot(CurrentSlot(), path, false); } int CreateFileHandle(const char* path) { return OpenFileHandleForSlot(CurrentSlot(), path, true); } int FileReadHandle(int handle, uint8_t* buffer, uint64_t offset, uint64_t size) { if (buffer == nullptr) return -1; if (size > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, size, true)) return -1; HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return -1; if (snapshot.type != HandleType::File || (snapshot.rights & RightRead) == 0) return -1; return Fs::Vfs::ReadBackendFile(((File*)snapshot.object)->backend, buffer, offset, size); } int FileWriteHandle(int handle, const uint8_t* buffer, uint64_t offset, uint64_t size) { if (buffer == nullptr) return -1; if (size > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, size, false)) return -1; HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return -1; if (snapshot.type != HandleType::File || (snapshot.rights & RightWrite) == 0) return -1; return Fs::Vfs::WriteBackendFile(((File*)snapshot.object)->backend, buffer, offset, size); } uint64_t FileGetSizeHandle(int handle) { HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return 0; if (snapshot.type != HandleType::File || (snapshot.rights & RightRead) == 0) return 0; return Fs::Vfs::GetBackendFileSize(((File*)snapshot.object)->backend); } static Socket* AllocateSocketObject(int type) { g_socketPoolLock.Acquire(); for (int i = 0; i < MaxSockets; i++) { if (g_sockets[i].active || g_sockets[i].destroying) continue; InitObject(g_sockets[i], HandleType::Socket); g_sockets[i].socketType = type; g_sockets[i].tcpConn = nullptr; g_sockets[i].localPort = 0; g_sockets[i].udpBound = false; g_sockets[i].udpHead = 0; g_sockets[i].udpTail = 0; g_sockets[i].udpCount = 0; g_socketPoolLock.Release(); return &g_sockets[i]; } g_socketPoolLock.Release(); return nullptr; } static bool SnapshotSocketHandle(int handle, HandleSnapshot& snapshot, Socket*& outSocket, uint32_t& outRights) { if (!snapshot.Capture(CurrentSlot(), handle)) return false; if (snapshot.type != HandleType::Socket || snapshot.object == nullptr) return false; outSocket = (Socket*)snapshot.object; outRights = snapshot.rights; return true; } int CreateSocketHandleForSlot(int slot, int type) { if (slot < 0 || slot >= Sched::MaxProcesses) return -1; if (type != SocketTypeTcp && type != SocketTypeUdp) return -1; Socket* socket = AllocateSocketObject(type); if (socket == nullptr) return -1; int handle = InstallHandleForSlot(slot, (Object*)socket, HandleType::Socket, RightRead | RightWrite | RightWait | RightManage | RightDup); if (handle >= 0) return handle; DestroySocket(socket); socket->active = false; return -1; } int CreateSocketHandle(int type) { return CreateSocketHandleForSlot(CurrentSlot(), type); } int SocketConnectHandle(int handle, uint32_t ip, uint16_t port) { HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); if (socket->tcpConn != nullptr) { socket->socketLock.Release(); return -1; } uint16_t localPort = AllocEphemeralPort(); socket->localPort = localPort; socket->socketLock.Release(); Net::Tcp::Connection* conn = Net::Tcp::Connect(ip, port, localPort); if (conn == nullptr) { socket->socketLock.Acquire(); if (socket->tcpConn == nullptr) socket->localPort = 0; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); return -1; } socket->socketLock.Acquire(); if (socket->tcpConn != nullptr) { socket->socketLock.Release(); Net::Tcp::Close(conn); return -1; } socket->tcpConn = conn; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); return 0; } int SocketBindHandle(int handle, uint16_t port) { HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0) return -1; if (socket->socketType == SocketTypeTcp) { socket->socketLock.Acquire(); if (socket->tcpConn != nullptr) { socket->socketLock.Release(); return -1; } socket->localPort = port; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); return 0; } if (socket->socketType != SocketTypeUdp) return -1; socket->socketLock.Acquire(); bool alreadyBound = socket->udpBound; uint16_t oldPort = socket->localPort; socket->socketLock.Release(); if (alreadyBound && oldPort == port) { return 0; } if (alreadyBound && oldPort != 0 && oldPort != port) { Net::Udp::Unbind(oldPort); } if (!Net::Udp::Bind(port, UdpSocketDispatcher)) { if (alreadyBound && oldPort != 0 && oldPort != port) { Net::Udp::Bind(oldPort, UdpSocketDispatcher); } return -1; } socket->socketLock.Acquire(); socket->localPort = port; socket->udpBound = true; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); return 0; } int SocketListenHandle(int handle) { HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); uint16_t localPort = socket->localPort; bool busy = socket->tcpConn != nullptr; socket->socketLock.Release(); if (busy || localPort == 0) return -1; Net::Tcp::Connection* listener = Net::Tcp::Listen(localPort); if (listener == nullptr) return -1; socket->socketLock.Acquire(); if (socket->tcpConn != nullptr) { socket->socketLock.Release(); Net::Tcp::Close(listener); return -1; } socket->tcpConn = listener; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); return 0; } int SocketAcceptHandle(int handle) { int slot = CurrentSlot(); HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightManage) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); Net::Tcp::Connection* listener = socket->tcpConn; uint16_t localPort = socket->localPort; socket->socketLock.Release(); if (listener == nullptr || Net::Tcp::GetState(listener) != Net::Tcp::State::Listen) return -1; Net::Tcp::Connection* clientConn = Net::Tcp::Accept(listener); if (clientConn == nullptr) return -1; Socket* accepted = AllocateSocketObject(SocketTypeTcp); if (accepted == nullptr) { Net::Tcp::Close(clientConn); return -1; } accepted->socketLock.Acquire(); accepted->tcpConn = clientConn; accepted->localPort = localPort; accepted->socketLock.Release(); int acceptedHandle = InstallHandleForSlot(slot, (Object*)accepted, HandleType::Socket, RightRead | RightWrite | RightWait | RightManage | RightDup); if (acceptedHandle < 0) { DestroySocket(accepted); accepted->active = false; return -1; } NotifyObjectChanged((Object*)socket); return acceptedHandle; } int SocketSendHandle(int handle, const uint8_t* data, uint32_t len) { if (data == nullptr) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, len, false)) return -1; HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightWrite) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); Net::Tcp::Connection* conn = socket->tcpConn; socket->socketLock.Release(); if (conn == nullptr) return -1; if (len > 0x7FFFFFFFu) return -1; return Net::Tcp::Send(conn, data, len); } int SocketRecvHandle(int handle, uint8_t* buffer, uint32_t maxLen) { if (buffer == nullptr) return -1; if (maxLen > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, maxLen, true)) return -1; HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightRead) == 0 || socket->socketType != SocketTypeTcp) return -1; socket->socketLock.Acquire(); Net::Tcp::Connection* conn = socket->tcpConn; socket->socketLock.Release(); if (conn == nullptr) return -1; uint16_t cappedLen = maxLen > 0xFFFFu ? 0xFFFFu : (uint16_t)maxLen; int result = Net::Tcp::ReceiveNonBlocking(conn, buffer, cappedLen); if (result != 0) NotifyObjectChanged((Object*)socket); return result; } int SocketSendToHandle(int handle, const uint8_t* data, uint32_t len, uint32_t destIp, uint16_t destPort) { if (data == nullptr) return -1; if (len > 1472) return -1; if (len > 0 && !montauk::abi::UserMemory::Range((uint64_t)data, len, false)) return -1; HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightWrite) == 0 || socket->socketType != SocketTypeUdp) return -1; socket->socketLock.Acquire(); uint16_t localPort = socket->localPort; bool udpBound = socket->udpBound; socket->socketLock.Release(); if (!udpBound || localPort == 0) { localPort = AllocEphemeralPort(); if (!Net::Udp::Bind(localPort, UdpSocketDispatcher)) return -1; socket->socketLock.Acquire(); socket->localPort = localPort; socket->udpBound = true; socket->socketLock.Release(); NotifyObjectChanged((Object*)socket); } if (!Net::Udp::Send(destIp, localPort, destPort, data, (uint16_t)len)) { return -1; } return (int)len; } int SocketRecvFromHandle(int handle, uint8_t* buffer, uint32_t maxLen, uint32_t* srcIp, uint16_t* srcPort) { if (buffer == nullptr) return -1; if (maxLen > 0 && !montauk::abi::UserMemory::Range((uint64_t)buffer, maxLen, true)) return -1; if (srcIp != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)srcIp)) return -1; if (srcPort != nullptr && !montauk::abi::UserMemory::Writable((uint64_t)srcPort)) return -1; HandleSnapshot snapshot; Socket* socket = nullptr; uint32_t rights = 0; if (!SnapshotSocketHandle(handle, snapshot, socket, rights)) return -1; if ((rights & RightRead) == 0 || socket->socketType != SocketTypeUdp) return -1; socket->socketLock.Acquire(); if (socket->udpCount < sizeof(UdpDgramHeader)) { socket->socketLock.Release(); return -1; } UdpDgramHeader hdr = {}; uint32_t head = socket->udpHead; uint32_t hdrLen = sizeof(UdpDgramHeader); uint32_t hfirst = (head + hdrLen <= UdpRingSize) ? hdrLen : (UdpRingSize - head); memcpy(&hdr, socket->udpRing + head, hfirst); if (hfirst < hdrLen) memcpy(((uint8_t*)&hdr) + hfirst, socket->udpRing, hdrLen - hfirst); socket->udpHead = (head + hdrLen) % UdpRingSize; socket->udpCount -= hdrLen; uint16_t copyLen = hdr.dataLen; if (copyLen > maxLen) copyLen = (uint16_t)maxLen; uint32_t dataHead = socket->udpHead; uint32_t cfirst = (dataHead + copyLen <= UdpRingSize) ? copyLen : (UdpRingSize - dataHead); memcpy(buffer, socket->udpRing + dataHead, cfirst); if (cfirst < copyLen) memcpy(buffer + cfirst, socket->udpRing, copyLen - cfirst); dataHead = (dataHead + hdr.dataLen) % UdpRingSize; socket->udpHead = dataHead; socket->udpCount -= hdr.dataLen; socket->socketLock.Release(); if (srcIp != nullptr) *srcIp = hdr.srcIp; if (srcPort != nullptr) *srcPort = hdr.srcPort; NotifyObjectChanged((Object*)socket); return (int)copyLen; } void NotifyTcpConnectionChanged(Net::Tcp::Connection* connection) { if (connection == nullptr) return; for (int i = 0; i < MaxSockets; i++) { if (!g_sockets[i].active || g_sockets[i].socketType != SocketTypeTcp) continue; g_sockets[i].socketLock.Acquire(); bool matches = g_sockets[i].tcpConn == connection; g_sockets[i].socketLock.Release(); if (matches) { NotifyObjectChanged((Object*)&g_sockets[i]); } } } Surface* CreateSurface(uint64_t byteSize) { if (byteSize == 0) byteSize = 0x1000; uint32_t numPages = (uint32_t)((byteSize + 0xFFFu) / 0x1000u); if (numPages == 0 || numPages > MaxSurfacePages) return nullptr; g_surfacePoolLock.Acquire(); int slot = -1; for (int i = 0; i < MaxSurfaces; i++) { if (!g_surfaces[i].active && !g_surfaces[i].destroying) { slot = i; break; } } if (slot < 0) { g_surfacePoolLock.Release(); return nullptr; } InitObject(g_surfaces[slot], HandleType::Surface); g_surfaces[slot].numPages = numPages; g_surfaces[slot].sizeBytes = (uint64_t)numPages * 0x1000ULL; for (uint32_t i = 0; i < MaxSurfacePages; i++) g_surfaces[slot].physPages[i] = 0; g_surfacePoolLock.Release(); for (uint32_t i = 0; i < numPages; i++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { DestroySurface(&g_surfaces[slot]); g_surfaces[slot].active = false; return nullptr; } g_surfaces[slot].physPages[i] = Memory::SubHHDM((uint64_t)page); } return &g_surfaces[slot]; } int CreateSurfaceHandle(uint64_t byteSize) { Surface* surface = CreateSurface(byteSize); if (surface == nullptr) return -1; int handle = InstallHandleForSlot(CurrentSlot(), (Object*)surface, HandleType::Surface, RightMap | RightManage | RightWait | RightDup); if (handle >= 0) return handle; DestroySurface(surface); surface->active = false; return -1; } void RetainSurface(Surface* surface) { RetainRawObject((Object*)surface); } void ReleaseSurface(Surface* surface) { ReleaseRawObject((Object*)surface); } uint64_t GetSurfaceSize(const Surface* surface) { if (surface == nullptr) return 0; auto* mutableSurface = const_cast(surface); mutableSurface->lock.Acquire(); uint64_t size = mutableSurface->sizeBytes; mutableSurface->lock.Release(); return size; } static void RollbackSurfaceGrowMaps(Surface* surface, uint32_t newPages) { for (int s = 0; s < Sched::MaxProcesses; s++) { auto* proc = Sched::GetProcessSlot(s); if (proc == nullptr) continue; if (proc->state == Sched::ProcessState::Free) continue; uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; if (newPages <= m.numPages) continue; uint64_t startVa = m.va + (uint64_t)m.numPages * 0x1000ULL; uint32_t rollbackPages = newPages - m.numPages; for (uint32_t p = m.numPages; p < newPages; p++) { Memory::VMM::Paging::UnmapUserIn(pml4, m.va + (uint64_t)p * 0x1000ULL); } ShootdownUserRange(pml4, startVa, rollbackPages); } g_surfaceMapLocks[s].Release(); } } static bool PreinstallSurfaceGrowMaps(Surface* surface, uint64_t* scratch, uint32_t newPages) { for (int s = 0; s < Sched::MaxProcesses; s++) { auto* proc = Sched::GetProcessSlot(s); if (proc == nullptr) continue; if (proc->state == Sched::ProcessState::Free) continue; uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; bool failed = false; g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; if (newPages <= m.numPages) continue; for (uint32_t p = m.numPages; p < newPages; p++) { uint64_t va = m.va + (uint64_t)p * 0x1000ULL; if (!Memory::VMM::Paging::MapUserIn(pml4, scratch[p], va)) { failed = true; break; } } if (failed) break; } g_surfaceMapLocks[s].Release(); if (failed) { RollbackSurfaceGrowMaps(surface, newPages); return false; } } return true; } int ResizeSurface(Surface* surface, uint64_t newSize) { if (surface == nullptr) return -1; if (newSize == 0) newSize = 0x1000; if (newSize > (uint64_t)MaxSurfacePages * 0x1000ULL) return -1; uint32_t newPages = (uint32_t)((newSize + 0xFFFULL) / 0x1000ULL); if (newPages == 0 || newPages > MaxSurfacePages) return -1; // Transactional resize: allocate all new pages BEFORE freeing the old // ones, then swap them in under the lock. The previous implementation // freed first and then allocated outside the lock, leaving readers // with numPages updated but physPages still zero (or transiently // freed) for the entire allocation window; on partial failure the // surface was left in a half-shrunken half-allocated state. // Heap-allocate the scratch arrays: with MaxSurfacePages=8192, two // uint64_t[8192] arrays are 128 KiB - far past the 16 KiB kernel stack. uint64_t* scratch = new uint64_t[newPages]; if (scratch == nullptr) return -1; for (uint32_t i = 0; i < newPages; i++) scratch[i] = 0; for (uint32_t i = 0; i < newPages; i++) { void* page = Memory::g_pfa->AllocateZeroed(); if (page == nullptr) { for (uint32_t j = 0; j < i; j++) { Memory::g_pfa->Free((void*)Memory::HHDM(scratch[j])); } delete[] scratch; return -1; } scratch[i] = Memory::SubHHDM((uint64_t)page); } surface->lock.Acquire(); uint32_t oldNumPages = surface->numPages; uint64_t* oldPages = (oldNumPages > 0) ? new uint64_t[oldNumPages] : nullptr; if (oldNumPages > 0 && oldPages == nullptr) { surface->lock.Release(); for (uint32_t i = 0; i < newPages; i++) { Memory::g_pfa->Free((void*)Memory::HHDM(scratch[i])); } delete[] scratch; return -1; } for (uint32_t i = 0; i < oldNumPages; i++) oldPages[i] = surface->physPages[i]; // Grow mappings can require new page-table pages. Preinstall the // additional leaf PTEs before changing the surface metadata so an OOM // can roll back cleanly instead of reporting a successful resize with // only some processes mapped to the larger range. if (!PreinstallSurfaceGrowMaps(surface, scratch, newPages)) { surface->lock.Release(); for (uint32_t i = 0; i < newPages; i++) { Memory::g_pfa->Free((void*)Memory::HHDM(scratch[i])); } delete[] oldPages; delete[] scratch; return -1; } for (uint32_t i = 0; i < newPages; i++) surface->physPages[i] = scratch[i]; for (uint32_t i = newPages; i < oldNumPages; i++) surface->physPages[i] = 0; surface->numPages = newPages; surface->sizeBytes = (uint64_t)newPages * 0x1000ULL; // Refresh every existing mapping of this surface so each mapper's PTEs // point at the new physical frames before we hand the old frames back // to the PFA. Without this, MapSurfaceHandle re-returns the cached VA // (see MapSurfaceForPid early-return) and any process that mapped the // surface before the resize keeps writing to freed pages -- corrupting // whatever the PFA next hands them out for. // // We hold surface->lock during the walk so g_surfaceMaps reads stay // coherent with the physPages snapshot we just committed. MapUserIn // overwriting an existing leaf PTE doesn't free the old frame -- that // happens below, after every mapping has been redirected and every CPU // currently running the affected address space has invalidated the // touched user range. for (int s = 0; s < Sched::MaxProcesses; s++) { auto* proc = Sched::GetProcessSlot(s); if (proc == nullptr) continue; if (proc->state == Sched::ProcessState::Free) continue; uint64_t pml4 = proc->pml4Phys; if (pml4 == 0) continue; g_surfaceMapLocks[s].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { auto& m = g_surfaceMaps[s][i]; if (!m.used || m.surface != surface) continue; uint64_t baseVa = m.va; uint32_t mapPages = m.numPages; uint32_t common = (mapPages < newPages) ? mapPages : newPages; uint32_t flushPages = (mapPages > newPages) ? mapPages : newPages; // Pages that exist in both old and new: redirect PTE to the // new physical frame. // MapUserIn here only overwrites the leaf PTE -- all page // table levels are guaranteed present from the original map, // so walkLevel never has to allocate and cannot fail. Treat // failure as a kernel-state-corruption panic rather than a // recoverable error. for (uint32_t p = 0; p < common; p++) { uint64_t va = baseVa + (uint64_t)p * 0x1000ULL; if (!Memory::VMM::Paging::MapUserIn(pml4, surface->physPages[p], va)) { Panic("ResizeSurface: MapUserIn failed refreshing existing PTE", nullptr); } } // Pages trimmed off the end on shrink: unmap them entirely. for (uint32_t p = newPages; p < mapPages; p++) { uint64_t va = baseVa + (uint64_t)p * 0x1000ULL; Memory::VMM::Paging::UnmapUserIn(pml4, va); } ShootdownUserRange(pml4, baseVa, flushPages); m.numPages = newPages; } g_surfaceMapLocks[s].Release(); } surface->lock.Release(); for (uint32_t i = 0; i < oldNumPages; i++) { if (oldPages[i] != 0) { Memory::g_pfa->Free((void*)Memory::HHDM(oldPages[i])); } } delete[] oldPages; delete[] scratch; NotifyObjectChanged((Object*)surface); return 0; } uint64_t MapSurfaceHandle(int handle) { auto* proc = Sched::GetCurrentProcessPtr(); if (proc == nullptr) return 0; HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return 0; if (snapshot.type != HandleType::Surface || (snapshot.rights & RightMap) == 0) return 0; uint64_t va = 0; if (MapSurfaceForPid((Surface*)snapshot.object, proc->pid, proc->pml4Phys, proc->heapNext, va) < 0) { return 0; } return va; } int ResizeSurfaceHandle(int handle, uint64_t newSize) { HandleSnapshot snapshot; if (!snapshot.Capture(CurrentSlot(), handle)) return -1; if (snapshot.type != HandleType::Surface || (snapshot.rights & RightManage) == 0) return -1; return ResizeSurface((Surface*)snapshot.object, newSize); } int CopySurface(Surface* dst, Surface* src) { if (dst == nullptr || src == nullptr) return -1; if (dst == src) return 0; Surface* first = ((uintptr_t)dst < (uintptr_t)src) ? dst : src; Surface* second = (first == dst) ? src : dst; first->lock.Acquire(); second->lock.Acquire(); if (dst->numPages != src->numPages) { second->lock.Release(); first->lock.Release(); return -1; } for (uint32_t i = 0; i < src->numPages; i++) { if (src->physPages[i] == 0 || dst->physPages[i] == 0) { second->lock.Release(); first->lock.Release(); return -1; } void* srcPage = (void*)Memory::HHDM(src->physPages[i]); void* dstPage = (void*)Memory::HHDM(dst->physPages[i]); memcpy(dstPage, srcPage, 0x1000); } second->lock.Release(); first->lock.Release(); NotifyObjectChanged((Object*)dst); return 0; } int CopySurfacePreserve(Surface* dst, int dstWidth, int dstHeight, Surface* src, int srcWidth, int srcHeight, uint32_t fillPixel) { if (dst == nullptr || src == nullptr) return -1; if (dstWidth <= 0 || dstHeight <= 0 || srcWidth <= 0 || srcHeight <= 0) return -1; Surface* first = ((uintptr_t)dst < (uintptr_t)src) ? dst : src; Surface* second = (first == dst) ? src : dst; first->lock.Acquire(); if (second != first) second->lock.Acquire(); uint64_t dstPixels = (uint64_t)dstWidth * (uint64_t)dstHeight; uint64_t srcPixels = (uint64_t)srcWidth * (uint64_t)srcHeight; if (dstPixels * sizeof(uint32_t) > dst->sizeBytes || srcPixels * sizeof(uint32_t) > src->sizeBytes) { if (second != first) second->lock.Release(); first->lock.Release(); return -1; } for (uint64_t i = 0; i < dstPixels; i++) { uint32_t* p = SurfacePixelPtr(dst, i); if (p != nullptr) *p = fillPixel; } int copyWidth = (dstWidth < srcWidth) ? dstWidth : srcWidth; int copyHeight = (dstHeight < srcHeight) ? dstHeight : srcHeight; for (int y = 0; y < copyHeight; y++) { uint64_t dstRow = (uint64_t)y * (uint64_t)dstWidth; uint64_t srcRow = (uint64_t)y * (uint64_t)srcWidth; for (int x = 0; x < copyWidth; x++) { uint32_t* srcPixel = SurfacePixelPtr(src, srcRow + (uint64_t)x); uint32_t* dstPixel = SurfacePixelPtr(dst, dstRow + (uint64_t)x); if (srcPixel != nullptr && dstPixel != nullptr) { *dstPixel = *srcPixel; } } } if (second != first) second->lock.Release(); first->lock.Release(); NotifyObjectChanged((Object*)dst); return 0; } int MapSurfaceForPid(Surface* surface, int pid, uint64_t pml4Phys, uint64_t& heapNext, uint64_t& outVa, bool reserveGrowthRange) { if (surface == nullptr) return -1; // Kept in the interface for existing callers. Surface VA now comes // from reusable fixed slots and no longer consumes the process heap. (void)heapNext; (void)reserveGrowthRange; int slot = SlotForPid(pid); if (slot < 0) return -1; // surface->lock serialises us against ResizeSurface so the // numPages/physPages snapshot we install into the caller's page tables // is internally consistent. Without this, a resize landing mid-loop // could surface zero/freed physical addresses (see ResizeSurface), // causing MapUserIn to map PA 0 into the caller's address space. // // RetainRawObject also takes surface->lock (Surface inherits its lock // from Object), so the refcount bump is inlined below to avoid // self-deadlocking on the non-reentrant mutex. surface->lock.Acquire(); g_surfaceMapLocks[slot].Acquire(); for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { if (g_surfaceMaps[slot][i].used && g_surfaceMaps[slot][i].surface == surface) { outVa = g_surfaceMaps[slot][i].va; g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return 0; } } int mapIdx = -1; for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { if (!g_surfaceMaps[slot][i].used) { mapIdx = i; break; } } if (mapIdx < 0) { g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return -1; } uint32_t numPages = surface->numPages; uint64_t baseVa = Sched::UserSurfaceBase + (uint64_t)mapIdx * Sched::UserSurfaceSlotSize; for (uint32_t i = 0; i < numPages; i++) { if (!Memory::VMM::Paging::MapUserIn(pml4Phys, surface->physPages[i], baseVa + (uint64_t)i * 0x1000ULL)) { for (uint32_t j = 0; j < i; j++) { Memory::VMM::Paging::UnmapUserIn(pml4Phys, baseVa + (uint64_t)j * 0x1000ULL); } g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return -1; } } g_surfaceMaps[slot][mapIdx].used = true; g_surfaceMaps[slot][mapIdx].surface = surface; g_surfaceMaps[slot][mapIdx].va = baseVa; g_surfaceMaps[slot][mapIdx].numPages = numPages; surface->refs++; // inlined RetainRawObject — we already hold surface->lock outVa = baseVa; g_surfaceMapLocks[slot].Release(); surface->lock.Release(); return 0; } int UnmapSurfaceForPid(Surface* surface, int pid, uint64_t pml4Phys) { if (surface == nullptr) return -1; int slot = SlotForPid(pid); if (slot < 0) return -1; surface->lock.Acquire(); g_surfaceMapLocks[slot].Acquire(); int unmapped = 0; for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { if (!g_surfaceMaps[slot][i].used || g_surfaceMaps[slot][i].surface != surface) continue; uint64_t baseVa = g_surfaceMaps[slot][i].va; uint32_t numPages = g_surfaceMaps[slot][i].numPages; for (uint32_t p = 0; p < numPages; p++) { Memory::VMM::Paging::UnmapUserIn(pml4Phys, baseVa + (uint64_t)p * 0x1000ULL); } // UnmapUserIn invalidates only the calling CPU. A sibling thread // in this same process may still have one of these PTEs cached; // releasing the mapping reference can then destroy the surface // and recycle its frames while that sibling writes through its // stale TLB entry. Quiesce every CPU using this PML4 first. ShootdownUserRange(pml4Phys, baseVa, numPages); g_surfaceMaps[slot][i].used = false; g_surfaceMaps[slot][i].surface = nullptr; g_surfaceMaps[slot][i].va = 0; g_surfaceMaps[slot][i].numPages = 0; unmapped++; } g_surfaceMapLocks[slot].Release(); surface->lock.Release(); for (int i = 0; i < unmapped; i++) ReleaseRawObject((Object*)surface); return unmapped > 0 ? 0 : -1; } int OpenProcessHandle(int pid) { g_processPoolLock.Acquire(); for (int i = 0; i < MaxProcessObjects; i++) { if (!g_processObjects[i].active) continue; if (g_processObjects[i].pid == pid) { int handle = InstallHandleForSlot(CurrentSlot(), (Object*)&g_processObjects[i], HandleType::Process, RightWait | RightDup); g_processPoolLock.Release(); return handle; } } g_processPoolLock.Release(); return -1; } void ProcessStartedInSlot(int slot, int pid) { if (slot < 0 || slot >= Sched::MaxProcesses) return; g_processPoolLock.Acquire(); for (int i = 0; i < MaxProcessObjects; i++) { if (g_processObjects[i].active || g_processObjects[i].destroying) continue; InitObject(g_processObjects[i], HandleType::Process); g_processObjects[i].pid = pid; g_processObjects[i].exited = false; g_processObjectsBySlot[slot] = &g_processObjects[i]; g_processObjects[i].refs = 1; // liveness reference g_processPoolLock.Release(); return; } g_processPoolLock.Release(); Kt::KernelLogStream(Kt::ERROR, "IPC") << "Out of process objects for PID " << (uint64_t)pid; } void ProcessExitedInSlot(int slot, int /*pid*/) { if (slot < 0 || slot >= Sched::MaxProcesses) return; ProcessObject* object = g_processObjectsBySlot[slot]; if (object == nullptr) return; object->lock.Acquire(); object->exited = true; object->lock.Release(); NotifyObjectChanged((Object*)object); ReleaseRawObject((Object*)object); g_processObjectsBySlot[slot] = nullptr; } bool ProcessHasExited(ProcessObject* process) { if (process == nullptr) return true; process->lock.Acquire(); bool exited = process->exited; process->lock.Release(); return exited; } static uint32_t CurrentSocketSignals(Socket* socket, uint32_t rights) { if (socket == nullptr) return SignalNone; if (socket->socketType == SocketTypeUdp) { uint32_t signals = SignalReady; socket->socketLock.Acquire(); if ((rights & RightRead) && socket->udpCount >= sizeof(UdpDgramHeader)) { signals |= SignalReadable; } if (rights & RightWrite) { signals |= SignalWritable; } socket->socketLock.Release(); return signals; } if (socket->socketType != SocketTypeTcp) return SignalNone; socket->socketLock.Acquire(); Net::Tcp::Connection* conn = socket->tcpConn; socket->socketLock.Release(); if (conn == nullptr) return SignalNone; uint32_t signals = SignalNone; Net::Tcp::State state = Net::Tcp::GetState(conn); if (state == Net::Tcp::State::Listen) { if ((rights & RightRead) && Net::Tcp::HasPendingAccept(conn)) { signals |= SignalReadable | SignalReady; } return signals; } if ((rights & RightRead) && Net::Tcp::HasReceiveData(conn)) { signals |= SignalReadable; } if ((rights & RightWrite) && Net::Tcp::CanSend(conn)) { signals |= SignalWritable; } if ((rights & (RightRead | RightWrite)) && Net::Tcp::IsClosedForIo(conn)) { signals |= SignalPeerClosed; } if (signals != SignalNone) signals |= SignalReady; return signals; } static uint32_t CurrentSignalsForSnapshot(HandleType type, Object* object, uint32_t rights) { if (object == nullptr) return SignalNone; switch (type) { case HandleType::Stream: { auto* stream = (Stream*)object; uint32_t signals = SignalNone; stream->lock.Acquire(); if ((rights & RightRead) && stream->count > 0) signals |= SignalReadable; if ((rights & RightWrite) && stream->count < stream->capacity && stream->readerRefs > 0) signals |= SignalWritable; if ((rights & RightRead) && stream->writerRefs == 0) signals |= SignalPeerClosed; if ((rights & RightWrite) && stream->readerRefs == 0) signals |= SignalPeerClosed; stream->lock.Release(); return signals; } case HandleType::Mailbox: { auto* mailbox = (Mailbox*)object; uint32_t signals = SignalNone; mailbox->lock.Acquire(); if ((rights & RightRecv) && mailbox->count > 0) signals |= SignalReadable; if ((rights & RightSend) && mailbox->count < MaxMailboxMessages && mailbox->receiverRefs > 0) signals |= SignalWritable; if ((rights & RightRecv) && mailbox->senderRefs == 0) signals |= SignalPeerClosed; if ((rights & RightSend) && mailbox->receiverRefs == 0) signals |= SignalPeerClosed; mailbox->lock.Release(); return signals; } case HandleType::File: { uint32_t signals = SignalReady; if (rights & RightRead) signals |= SignalReadable; if (rights & RightWrite) signals |= SignalWritable; return signals; } case HandleType::Socket: return CurrentSocketSignals((Socket*)object, rights); case HandleType::Surface: return (rights & RightMap) ? SignalReady : SignalNone; case HandleType::Process: { auto* process = (ProcessObject*)object; return ProcessHasExited(process) ? SignalExited : SignalNone; } case HandleType::Waitset: { WaitsetReady ready{}; return WaitsetCheckReady((Waitset*)object, &ready) ? SignalReady : SignalNone; } default: return SignalNone; } } uint32_t GetHandleSignalsForSlot(int slot, int handle) { HandleSnapshot snapshot; if (!snapshot.Capture(slot, handle)) return SignalNone; return CurrentSignalsForSnapshot(snapshot.type, snapshot.object, snapshot.rights); } static bool WaitsetCheckReady(Waitset* waitset, WaitsetReady* outReady) { if (waitset == nullptr) return false; for (int i = 0; i < MaxWaitsetEntries; i++) { HandleType type = HandleType::None; Object* object = nullptr; uint32_t rights = 0; uint32_t wanted = 0; waitset->entriesLock.Acquire(); WaitsetEntry& entry = waitset->entries[i]; if (entry.used && !entry.retiring && entry.object != nullptr) { entry.readers++; type = entry.type; object = entry.object; rights = entry.rights; wanted = entry.signals; } waitset->entriesLock.Release(); if (object == nullptr) continue; uint32_t current = CurrentSignalsForSnapshot(type, object, rights); uint32_t readySignals = current & wanted; waitset->entriesLock.Acquire(); entry.readers--; waitset->entriesLock.Release(); if (readySignals == 0) continue; if (outReady != nullptr) { outReady->index = i; outReady->signals = readySignals; } return true; } return false; } static bool WaitsetContainsObject(Waitset* waitset, Object* object) { if (waitset == nullptr || object == nullptr) return false; bool found = false; waitset->entriesLock.Acquire(); for (int i = 0; i < MaxWaitsetEntries; i++) { const WaitsetEntry& entry = waitset->entries[i]; if (entry.used && !entry.retiring && entry.object == object) { found = true; break; } } waitset->entriesLock.Release(); return found; } void NotifyObjectChanged(Object* object) { if (object == nullptr) return; Sched::WakeObjectWaiters(object); for (int i = 0; i < MaxWaitsets; i++) { if (!g_waitsets[i].active) continue; // This function is also called by the NIC receive IRQ. Do not // evaluate readiness here: that takes target stream/mailbox // mutexes and can self-deadlock if the IRQ preempted their owner. // A pointer match is enough; the awakened waiter rechecks signals // in process context before returning to userspace. if (WaitsetContainsObject(&g_waitsets[i], object)) { Sched::WakeObjectWaiters(&g_waitsets[i]); } } } uint32_t WaitOnHandle(int handle, uint32_t wantedSignals, uint64_t timeoutMs) { int slot = CurrentSlot(); HandleSnapshot snapshot; if (!snapshot.Capture(slot, handle)) return (uint32_t)-1; if ((snapshot.rights & RightWait) == 0) return (uint32_t)-1; uint64_t start = Timekeeping::GetMilliseconds(); for (;;) { uint64_t observedWake = Sched::ObserveObjectWake(snapshot.object); uint32_t current = CurrentSignalsForSnapshot(snapshot.type, snapshot.object, snapshot.rights); uint32_t ready = current & wantedSignals; if (ready != 0) return ready; if (timeoutMs == 0) return 0; if (timeoutMs != ~0ULL) { uint64_t elapsed = Timekeeping::GetMilliseconds() - start; if (elapsed >= timeoutMs) return 0; Sched::BlockOnObjectSince(snapshot.object, timeoutMs - elapsed, observedWake); } else { Sched::BlockOnObjectSince(snapshot.object, 0, observedWake); } } } int CreateWaitsetHandleForSlot(int slot) { if (slot < 0 || slot >= Sched::MaxProcesses) return -1; g_waitsetPoolLock.Acquire(); for (int i = 0; i < MaxWaitsets; i++) { if (g_waitsets[i].active || g_waitsets[i].destroying) continue; InitObject(g_waitsets[i], HandleType::Waitset); g_waitsets[i].entriesLock.Acquire(); for (int j = 0; j < MaxWaitsetEntries; j++) { g_waitsets[i].entries[j].used = false; g_waitsets[i].entries[j].retiring = false; g_waitsets[i].entries[j].readers = 0; g_waitsets[i].entries[j].type = HandleType::None; g_waitsets[i].entries[j].object = nullptr; g_waitsets[i].entries[j].rights = 0; g_waitsets[i].entries[j].signals = 0; } g_waitsets[i].entriesLock.Release(); g_waitsetPoolLock.Release(); return InstallHandleForSlot(slot, (Object*)&g_waitsets[i], HandleType::Waitset, RightWait | RightManage | RightDup); } g_waitsetPoolLock.Release(); return -1; } int CreateWaitsetHandleForCurrent() { return CreateWaitsetHandleForSlot(CurrentSlot()); } int WaitsetAddHandleForSlot(int slot, int waitsetHandle, int targetHandle, uint32_t signals) { HandleSnapshot waitsetSnapshot; if (!waitsetSnapshot.Capture(slot, waitsetHandle)) return -1; if (waitsetSnapshot.type != HandleType::Waitset || (waitsetSnapshot.rights & RightManage) == 0) return -1; HandleSnapshot targetSnapshot; if (!targetSnapshot.Capture(slot, targetHandle)) return -1; if ((targetSnapshot.rights & RightWait) == 0) return -1; if (targetSnapshot.type == HandleType::Waitset) return -1; auto* waitset = (Waitset*)waitsetSnapshot.object; // Take the entry's permanent target reference before disabling IRQs // for the short metadata update. RetainRawObject(targetSnapshot.object); waitset->entriesLock.Acquire(); for (int i = 0; i < MaxWaitsetEntries; i++) { if (waitset->entries[i].used || waitset->entries[i].retiring || waitset->entries[i].object != nullptr) continue; waitset->entries[i].used = true; waitset->entries[i].type = targetSnapshot.type; waitset->entries[i].object = targetSnapshot.object; waitset->entries[i].rights = targetSnapshot.rights; waitset->entries[i].signals = signals; waitset->entriesLock.Release(); NotifyObjectChanged(waitsetSnapshot.object); return i; } waitset->entriesLock.Release(); ReleaseRawObject(targetSnapshot.object); return -1; } int WaitsetAddHandle(int waitsetHandle, int targetHandle, uint32_t signals) { return WaitsetAddHandleForSlot(CurrentSlot(), waitsetHandle, targetHandle, signals); } int WaitsetRemoveIndexForSlot(int slot, int waitsetHandle, int index) { HandleSnapshot snapshot; if (!snapshot.Capture(slot, waitsetHandle)) return -1; if (snapshot.type != HandleType::Waitset || (snapshot.rights & RightManage) == 0) return -1; if (index < 0 || index >= MaxWaitsetEntries) return -1; auto* waitset = (Waitset*)snapshot.object; waitset->entriesLock.Acquire(); WaitsetEntry& entry = waitset->entries[index]; if (!entry.used || entry.retiring || entry.object == nullptr) { waitset->entriesLock.Release(); return -1; } entry.used = false; entry.retiring = true; waitset->entriesLock.Release(); Object* targetObject = nullptr; for (;;) { waitset->entriesLock.Acquire(); if (entry.readers == 0) { targetObject = entry.object; entry.type = HandleType::None; entry.object = nullptr; entry.rights = 0; entry.signals = 0; entry.retiring = false; waitset->entriesLock.Release(); break; } waitset->entriesLock.Release(); asm volatile("pause"); } ReleaseRawObject(targetObject); return 0; } int WaitsetRemoveIndex(int waitsetHandle, int index) { return WaitsetRemoveIndexForSlot(CurrentSlot(), waitsetHandle, index); } int WaitsetWaitHandle(int waitsetHandle, WaitsetReady* outReady, uint64_t timeoutMs) { int slot = CurrentSlot(); HandleSnapshot snapshot; if (!snapshot.Capture(slot, waitsetHandle)) return -1; if (snapshot.type != HandleType::Waitset || (snapshot.rights & RightWait) == 0) return -1; uint64_t start = Timekeeping::GetMilliseconds(); for (;;) { uint64_t observedWake = Sched::ObserveObjectWake(snapshot.object); WaitsetReady ready = {-1, 0}; if (WaitsetCheckReady((Waitset*)snapshot.object, &ready)) { if (outReady != nullptr) *outReady = ready; return 1; } if (timeoutMs == 0) return 0; if (timeoutMs != ~0ULL) { uint64_t elapsed = Timekeeping::GetMilliseconds() - start; if (elapsed >= timeoutMs) return 0; Sched::BlockOnObjectSince(snapshot.object, timeoutMs - elapsed, observedWake); } else { Sched::BlockOnObjectSince(snapshot.object, 0, observedWake); } } } void CleanupProcessSlot(int slot, int /*pid*/, uint64_t pml4Phys) { if (slot < 0 || slot >= Sched::MaxProcesses) return; for (int h = 0; h < MaxHandlesPerProcess; h++) { // CloseHandleForSlot performs the synchronized used check. Reading // the table directly here would race a sibling thread still // unwinding during process teardown. CloseHandleForSlot(slot, h); } for (int i = 0; i < MaxSurfaceMapsPerProcess; i++) { g_surfaceMapLocks[slot].Acquire(); SurfaceMap& map = g_surfaceMaps[slot][i]; if (!map.used || map.surface == nullptr) { g_surfaceMapLocks[slot].Release(); continue; } for (uint32_t p = 0; p < map.numPages; p++) { Memory::VMM::Paging::UnmapUserIn(pml4Phys, map.va + (uint64_t)p * 0x1000ULL); } Object* surfaceObject = (Object*)map.surface; map.used = false; map.surface = nullptr; map.va = 0; map.numPages = 0; g_surfaceMapLocks[slot].Release(); ReleaseRawObject(surfaceObject); } } void Initialize() { for (int i = 0; i < Sched::MaxProcesses; i++) { g_processObjectsBySlot[i] = nullptr; } Hal::RegisterIrqHandler(Hal::IRQ_TLB_SHOOTDOWN, TlbShootdownIpiHandler); Kt::KernelLogStream(Kt::OK, "IPC") << "Initialized (" << (uint64_t)MaxHandlesPerProcess << " handles/process, " << (uint64_t)MaxStreams << " streams, " << (uint64_t)MaxMailboxes << " mailboxes, " << (uint64_t)MaxFiles << " files, " << (uint64_t)MaxSockets << " sockets, " << (uint64_t)MaxSurfaces << " surfaces)"; } }