/* * utf8.h * UTF-8 text encoding helpers with a Windows-1252 fallback * Copyright (c) 2026 Daniel Hammer */ #pragma once #include // MontaukOS text is UTF-8. Before this it was single-byte Windows-1252, so // older files (and anything else that is not valid UTF-8) still turn up. The // decoder never fails: a byte that does not start a well-formed sequence is // read as ONE Windows-1252 character. Valid UTF-8 and plain ASCII are // unaffected, and legacy text keeps rendering exactly as it used to. // // Character boundaries follow the same rule, so cursor movement, deletion and // truncation all agree with what the renderer draws. namespace montauk::utf8 { inline constexpr uint32_t REPLACEMENT = 0xFFFD; inline constexpr int MAX_BYTES = 4; // Windows-1252 byte -> Unicode codepoint. 0x80-0x9F carry the typographic // punctuation; every other byte is identical to Latin-1 / Unicode. inline constexpr uint32_t from_cp1252(uint8_t b) { switch (b) { case 0x80: return 0x20AC; // Euro Sign case 0x82: return 0x201A; // Single Low-9 Quotation Mark case 0x83: return 0x0192; // Latin Small Letter F With Hook case 0x84: return 0x201E; // Double Low-9 Quotation Mark case 0x85: return 0x2026; // Horizontal Ellipsis case 0x86: return 0x2020; // Dagger case 0x87: return 0x2021; // Double Dagger case 0x88: return 0x02C6; // Modifier Letter Circumflex Accent case 0x89: return 0x2030; // Per Mille Sign case 0x8A: return 0x0160; // Latin Capital Letter S With Caron case 0x8B: return 0x2039; // Single Left-Pointing Angle Quotation Mark case 0x8C: return 0x0152; // Latin Capital Ligature OE case 0x8E: return 0x017D; // Latin Capital Letter Z With Caron case 0x91: return 0x2018; // Left Single Quotation Mark case 0x92: return 0x2019; // Right Single Quotation Mark case 0x93: return 0x201C; // Left Double Quotation Mark case 0x94: return 0x201D; // Right Double Quotation Mark case 0x95: return 0x2022; // Bullet case 0x96: return 0x2013; // En Dash case 0x97: return 0x2014; // Em Dash case 0x98: return 0x02DC; // Small Tilde case 0x99: return 0x2122; // Trade Mark Sign case 0x9A: return 0x0161; // Latin Small Letter S With Caron case 0x9B: return 0x203A; // Single Right-Pointing Angle Quotation Mark case 0x9C: return 0x0153; // Latin Small Ligature OE case 0x9E: return 0x017E; // Latin Small Letter Z With Caron case 0x9F: return 0x0178; // Latin Capital Letter Y With Diaeresis default: return b; } } // Unicode codepoint -> Windows-1252 byte, or 0 if it has none. Used where a // single byte is still required (PDF WinAnsi output, legacy byte tables). inline constexpr uint8_t to_cp1252(uint32_t cp) { if (cp < 0x80 || (cp >= 0xA0 && cp <= 0xFF)) return (uint8_t)cp; for (int b = 0x80; b <= 0x9F; b++) { uint32_t m = from_cp1252((uint8_t)b); if (m != (uint32_t)b && m == cp) return (uint8_t)b; } return 0; } // Length of the well-formed UTF-8 sequence starting at s (at most `avail` // bytes, stopping early at a NUL), or 0 if it is not one. Rejects overlong // forms, surrogates and values above U+10FFFF. inline int sequence_length(const char* s, int avail, uint32_t* out_cp = nullptr) { if (avail <= 0) return 0; const uint8_t* u = (const uint8_t*)s; uint8_t b0 = u[0]; if (b0 < 0x80) { if (out_cp) *out_cp = b0; return 1; } int n; uint32_t cp, min; if (b0 >= 0xC2 && b0 <= 0xDF) { n = 2; cp = b0 & 0x1F; min = 0x80; } else if (b0 >= 0xE0 && b0 <= 0xEF) { n = 3; cp = b0 & 0x0F; min = 0x800; } else if (b0 >= 0xF0 && b0 <= 0xF4) { n = 4; cp = b0 & 0x07; min = 0x10000; } else return 0; if (n > avail) return 0; for (int i = 1; i < n; i++) { uint8_t c = u[i]; if ((c & 0xC0) != 0x80) return 0; // also catches NUL cp = (cp << 6) | (c & 0x3F); } if (cp < min || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return 0; if (out_cp) *out_cp = cp; return n; } // Decode one character from a buffer of `avail` bytes. Returns the codepoint // and stores the number of bytes consumed (always >= 1 when avail > 0). inline uint32_t decode(const char* s, int avail, int* consumed) { if (avail <= 0) { if (consumed) *consumed = 0; return 0; } uint32_t cp; int n = sequence_length(s, avail, &cp); if (n == 0) { n = 1; cp = from_cp1252((uint8_t)s[0]); } if (consumed) *consumed = n; return cp; } // Decode one character from a NUL-terminated string and advance *p past it. // Returns 0 (without advancing) at the terminator. inline uint32_t next(const char** p) { const char* s = *p; if (!*s) return 0; int n; // A sequence never spans the terminator: sequence_length stops at NUL // because NUL is not a continuation byte. uint32_t cp = decode(s, MAX_BYTES, &n); *p = s + n; return cp; } // Encode a codepoint as UTF-8 into out (room for 4 bytes, NOT terminated). // Returns the byte count. Invalid codepoints encode U+FFFD. inline int encode(uint32_t cp, char* out) { if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) cp = REPLACEMENT; uint8_t* o = (uint8_t*)out; if (cp < 0x80) { o[0] = (uint8_t)cp; return 1; } if (cp < 0x800) { o[0] = (uint8_t)(0xC0 | (cp >> 6)); o[1] = (uint8_t)(0x80 | (cp & 0x3F)); return 2; } if (cp < 0x10000) { o[0] = (uint8_t)(0xE0 | (cp >> 12)); o[1] = (uint8_t)(0x80 | ((cp >> 6) & 0x3F)); o[2] = (uint8_t)(0x80 | (cp & 0x3F)); return 3; } o[0] = (uint8_t)(0xF0 | (cp >> 18)); o[1] = (uint8_t)(0x80 | ((cp >> 12) & 0x3F)); o[2] = (uint8_t)(0x80 | ((cp >> 6) & 0x3F)); o[3] = (uint8_t)(0x80 | (cp & 0x3F)); return 4; } inline int encoded_length(uint32_t cp) { if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return 3; // U+FFFD return cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4; } // ==== Case mapping ==== // Simple one-to-one case mapping for the scripts MontaukOS keyboard layouts // cover: ASCII, Latin-1, Latin Extended-A, Greek and Cyrillic. Anything else // is returned unchanged. inline constexpr uint32_t to_upper(uint32_t c) { if (c >= 'a' && c <= 'z') return c - 0x20; if (c < 0x80) return c; if (c >= 0xE0 && c <= 0xFE && c != 0xF7) return c - 0x20; if (c == 0xFF) return 0x178; if (c == 0xB5) return 0x39C; // micro -> Mu // Latin Extended-A: pairs alternate upper/lower, except the two ranges // where the alignment shifts by one (L-stroke..N-apostrophe, Z-acute..). if (c == 0x131) return 'I'; // dotless i if (c >= 0x100 && c <= 0x137 && c != 0x130) return (c & 1) ? c - 1 : c; if (c >= 0x139 && c <= 0x148) return (c & 1) ? c : c - 1; if (c >= 0x14A && c <= 0x177) return (c & 1) ? c - 1 : c; if (c >= 0x179 && c <= 0x17E) return (c & 1) ? c : c - 1; if (c == 0x17F) return 'S'; // long s // Greek if (c >= 0x3B1 && c <= 0x3C9 && c != 0x3C2) return c - 0x20; if (c == 0x3C2) return 0x3A3; // final sigma if (c == 0x3AC) return 0x386; if (c >= 0x3AD && c <= 0x3AF) return c - 0x25; if (c == 0x3CC) return 0x38C; if (c == 0x3CD || c == 0x3CE) return c - 0x3F; // Cyrillic if (c >= 0x430 && c <= 0x44F) return c - 0x20; if (c >= 0x450 && c <= 0x45F) return c - 0x50; return c; } inline constexpr uint32_t to_lower(uint32_t c) { if (c >= 'A' && c <= 'Z') return c + 0x20; if (c < 0x80) return c; if (c >= 0xC0 && c <= 0xDE && c != 0xD7) return c + 0x20; if (c == 0x178) return 0xFF; if (c == 0x130) return 'i'; // dotted I if (c >= 0x100 && c <= 0x137) return (c & 1) ? c : c + 1; if (c >= 0x139 && c <= 0x148) return (c & 1) ? c + 1 : c; if (c >= 0x14A && c <= 0x177) return (c & 1) ? c : c + 1; if (c >= 0x179 && c <= 0x17E) return (c & 1) ? c + 1 : c; if (c >= 0x391 && c <= 0x3A9 && c != 0x3A2) return c + 0x20; if (c == 0x386) return 0x3AC; if (c >= 0x388 && c <= 0x38A) return c + 0x25; if (c == 0x38C) return 0x3CC; if (c == 0x38E || c == 0x38F) return c + 0x3F; if (c >= 0x410 && c <= 0x42F) return c + 0x20; if (c >= 0x400 && c <= 0x40F) return c + 0x50; return c; } // True if lower/upper are the two cases of one letter. inline constexpr bool is_case_pair(uint32_t lower, uint32_t upper) { return lower != upper && to_upper(lower) == upper && to_lower(upper) == lower; } // Text a key event types, as a codepoint (0 for control/non-text keys). // KeyEvent::codepoint is filled by the kernel and by layout translation; // events built by hand may leave it 0, so fall back to `ascii` (a // Windows-1252 byte). template inline uint32_t key_codepoint(const Key& key) { if (key.codepoint) { uint32_t cp = key.codepoint; return (cp < 0x20 || cp == 0x7F || (cp >= 0x80 && cp < 0xA0)) ? 0 : cp; } uint8_t b = (uint8_t)key.ascii; if (b < 0x20 || b == 0x7F) return 0; return from_cp1252(b); } // Same, encoded as UTF-8 into out (4 bytes). Returns the length, 0 if none. template inline int key_utf8(const Key& key, char* out) { uint32_t cp = key_codepoint(key); return cp ? encode(cp, out) : 0; } // Byte offset of the character after the one at `pos` (len = buffer length). inline int next_boundary(const char* s, int len, int pos) { if (pos >= len) return len; if (pos < 0) return 0; int n; decode(s + pos, len - pos, &n); return pos + n; } // Byte offset of the character before `pos`. A run of continuation bytes is // one character only if it completes a well-formed sequence ending exactly at // pos; otherwise the previous byte stands alone (Windows-1252 fallback). inline int prev_boundary(const char* s, int pos) { if (pos <= 0) return 0; for (int back = 2; back <= MAX_BYTES && back <= pos; back++) { if (((uint8_t)s[pos - back + 1] & 0xC0) != 0x80) break; int start = pos - back; if (sequence_length(s + start, back) == back) return start; } return pos - 1; } // Snap `pos` back onto a character boundary. Use before truncating a buffer // or splitting at an arbitrary byte offset. inline int floor_boundary(const char* s, int len, int pos) { if (pos <= 0) return 0; if (pos >= len) return len; if (((uint8_t)s[pos] & 0xC0) != 0x80) return pos; // pos is a continuation byte: it is inside a character only if a // well-formed sequence starting up to 3 bytes earlier covers it. for (int back = 1; back < MAX_BYTES && back <= pos; back++) { int start = pos - back; int n = sequence_length(s + start, len - start); if (n > back) return start; if (((uint8_t)s[start] & 0xC0) != 0x80) break; } return pos; } // Number of characters in a NUL-terminated string. inline int count(const char* s) { int n = 0; while (*s) { next(&s); n++; } return n; } // Number of characters in the first `len` bytes. inline int count(const char* s, int len) { int n = 0, pos = 0; while (pos < len) { pos = next_boundary(s, len, pos); n++; } return n; } // Byte offset of the `index`th character (clamped to len). inline int offset_of(const char* s, int len, int index) { int pos = 0; while (index-- > 0 && pos < len) pos = next_boundary(s, len, pos); return pos; } // True if the buffer is entirely well-formed UTF-8. inline bool valid(const char* s, int len) { for (int pos = 0; pos < len;) { int n = sequence_length(s + pos, len - pos); if (n == 0) return false; pos += n; } return true; } // Rewrite a buffer as well-formed UTF-8 (each stray byte becomes the // character the Windows-1252 fallback would display). Returns the output // length, truncated on a character boundary to fit cap - 1, NUL-terminated. inline int normalize(const char* in, int len, char* out, int cap) { if (cap <= 0) return 0; int o = 0; for (int pos = 0; pos < len;) { int n; uint32_t cp = decode(in + pos, len - pos, &n); int need = encoded_length(cp); if (o + need > cap - 1) break; o += encode(cp, out + o); pos += n; } out[o] = '\0'; return o; } // Copy src into dst (cap bytes incl. NUL) without splitting a character. inline int copy(char* dst, int cap, const char* src) { if (cap <= 0) return 0; int len = 0; while (src[len]) len++; int n = len < cap - 1 ? len : floor_boundary(src, len, cap - 1); for (int i = 0; i < n; i++) dst[i] = src[i]; dst[n] = '\0'; return n; } // ==== Editing a NUL-terminated buffer with a byte cursor ==== // `len` is the current length, `cap` the buffer size including the NUL. The // cursor is a byte offset that is always kept on a character boundary. // Insert n bytes at *cursor. Returns false (buffer unchanged) if they don't fit. inline bool insert(char* buf, int* len, int cap, int* cursor, const char* s, int n) { if (n <= 0 || *len + n >= cap) return false; int at = *cursor < 0 ? 0 : (*cursor > *len ? *len : *cursor); for (int i = *len; i >= at; i--) buf[i + n] = buf[i]; // moves the NUL for (int i = 0; i < n; i++) buf[at + i] = s[i]; *len += n; *cursor = at + n; return true; } // Insert the text a key event types. Returns false if it types nothing or // does not fit. template inline bool insert_key(char* buf, int* len, int cap, int* cursor, const Key& key) { char t[MAX_BYTES]; int n = key_utf8(key, t); return n > 0 && insert(buf, len, cap, cursor, t, n); } // Remove the bytes [start, end). inline void erase(char* buf, int* len, int start, int end) { if (start < 0) start = 0; if (end > *len) end = *len; if (start >= end) return; int n = end - start; for (int i = start; i + n <= *len; i++) buf[i] = buf[i + n]; // moves the NUL *len -= n; } // Backspace: delete the character before the cursor. inline bool erase_before(char* buf, int* len, int* cursor) { if (*cursor <= 0) return false; int start = prev_boundary(buf, *cursor); erase(buf, len, start, *cursor); *cursor = start; return true; } // Delete: remove the character at the cursor. inline bool erase_at(char* buf, int* len, int cursor) { if (cursor >= *len) return false; erase(buf, len, cursor, next_boundary(buf, *len, cursor)); return true; } // Append typed text at the end (simple line inputs without a cursor). template inline bool append_key(char* buf, int* len, int cap, const Key& key) { int cursor = *len; return insert_key(buf, len, cap, &cursor, key); } // Backspace for append-only inputs: drop the last character. inline bool erase_last(char* buf, int* len) { int cursor = *len; return erase_before(buf, len, &cursor); } } // namespace montauk::utf8