400 lines
15 KiB
C++
400 lines
15 KiB
C++
/*
|
|
* utf8.h
|
|
* UTF-8 text encoding helpers with a Windows-1252 fallback
|
|
* Copyright (c) 2026 Daniel Hammer
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <cstdint>
|
|
|
|
// MontaukOS text is UTF-8. Before this it was single-byte Windows-1252, so
|
|
// older files (and anything else that is not valid UTF-8) still turn up. The
|
|
// decoder never fails: a byte that does not start a well-formed sequence is
|
|
// read as ONE Windows-1252 character. Valid UTF-8 and plain ASCII are
|
|
// unaffected, and legacy text keeps rendering exactly as it used to.
|
|
//
|
|
// Character boundaries follow the same rule, so cursor movement, deletion and
|
|
// truncation all agree with what the renderer draws.
|
|
|
|
namespace montauk::utf8 {
|
|
|
|
inline constexpr uint32_t REPLACEMENT = 0xFFFD;
|
|
inline constexpr int MAX_BYTES = 4;
|
|
|
|
// Windows-1252 byte -> Unicode codepoint. 0x80-0x9F carry the typographic
|
|
// punctuation; every other byte is identical to Latin-1 / Unicode.
|
|
inline constexpr uint32_t from_cp1252(uint8_t b) {
|
|
switch (b) {
|
|
case 0x80: return 0x20AC; // Euro Sign
|
|
case 0x82: return 0x201A; // Single Low-9 Quotation Mark
|
|
case 0x83: return 0x0192; // Latin Small Letter F With Hook
|
|
case 0x84: return 0x201E; // Double Low-9 Quotation Mark
|
|
case 0x85: return 0x2026; // Horizontal Ellipsis
|
|
case 0x86: return 0x2020; // Dagger
|
|
case 0x87: return 0x2021; // Double Dagger
|
|
case 0x88: return 0x02C6; // Modifier Letter Circumflex Accent
|
|
case 0x89: return 0x2030; // Per Mille Sign
|
|
case 0x8A: return 0x0160; // Latin Capital Letter S With Caron
|
|
case 0x8B: return 0x2039; // Single Left-Pointing Angle Quotation Mark
|
|
case 0x8C: return 0x0152; // Latin Capital Ligature OE
|
|
case 0x8E: return 0x017D; // Latin Capital Letter Z With Caron
|
|
case 0x91: return 0x2018; // Left Single Quotation Mark
|
|
case 0x92: return 0x2019; // Right Single Quotation Mark
|
|
case 0x93: return 0x201C; // Left Double Quotation Mark
|
|
case 0x94: return 0x201D; // Right Double Quotation Mark
|
|
case 0x95: return 0x2022; // Bullet
|
|
case 0x96: return 0x2013; // En Dash
|
|
case 0x97: return 0x2014; // Em Dash
|
|
case 0x98: return 0x02DC; // Small Tilde
|
|
case 0x99: return 0x2122; // Trade Mark Sign
|
|
case 0x9A: return 0x0161; // Latin Small Letter S With Caron
|
|
case 0x9B: return 0x203A; // Single Right-Pointing Angle Quotation Mark
|
|
case 0x9C: return 0x0153; // Latin Small Ligature OE
|
|
case 0x9E: return 0x017E; // Latin Small Letter Z With Caron
|
|
case 0x9F: return 0x0178; // Latin Capital Letter Y With Diaeresis
|
|
default: return b;
|
|
}
|
|
}
|
|
|
|
// Unicode codepoint -> Windows-1252 byte, or 0 if it has none. Used where a
|
|
// single byte is still required (PDF WinAnsi output, legacy byte tables).
|
|
inline constexpr uint8_t to_cp1252(uint32_t cp) {
|
|
if (cp < 0x80 || (cp >= 0xA0 && cp <= 0xFF)) return (uint8_t)cp;
|
|
for (int b = 0x80; b <= 0x9F; b++) {
|
|
uint32_t m = from_cp1252((uint8_t)b);
|
|
if (m != (uint32_t)b && m == cp) return (uint8_t)b;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Length of the well-formed UTF-8 sequence starting at s (at most `avail`
|
|
// bytes, stopping early at a NUL), or 0 if it is not one. Rejects overlong
|
|
// forms, surrogates and values above U+10FFFF.
|
|
inline int sequence_length(const char* s, int avail, uint32_t* out_cp = nullptr) {
|
|
if (avail <= 0) return 0;
|
|
const uint8_t* u = (const uint8_t*)s;
|
|
uint8_t b0 = u[0];
|
|
if (b0 < 0x80) {
|
|
if (out_cp) *out_cp = b0;
|
|
return 1;
|
|
}
|
|
|
|
int n;
|
|
uint32_t cp, min;
|
|
if (b0 >= 0xC2 && b0 <= 0xDF) { n = 2; cp = b0 & 0x1F; min = 0x80; }
|
|
else if (b0 >= 0xE0 && b0 <= 0xEF) { n = 3; cp = b0 & 0x0F; min = 0x800; }
|
|
else if (b0 >= 0xF0 && b0 <= 0xF4) { n = 4; cp = b0 & 0x07; min = 0x10000; }
|
|
else return 0;
|
|
|
|
if (n > avail) return 0;
|
|
for (int i = 1; i < n; i++) {
|
|
uint8_t c = u[i];
|
|
if ((c & 0xC0) != 0x80) return 0; // also catches NUL
|
|
cp = (cp << 6) | (c & 0x3F);
|
|
}
|
|
if (cp < min || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return 0;
|
|
if (out_cp) *out_cp = cp;
|
|
return n;
|
|
}
|
|
|
|
// Decode one character from a buffer of `avail` bytes. Returns the codepoint
|
|
// and stores the number of bytes consumed (always >= 1 when avail > 0).
|
|
inline uint32_t decode(const char* s, int avail, int* consumed) {
|
|
if (avail <= 0) { if (consumed) *consumed = 0; return 0; }
|
|
uint32_t cp;
|
|
int n = sequence_length(s, avail, &cp);
|
|
if (n == 0) {
|
|
n = 1;
|
|
cp = from_cp1252((uint8_t)s[0]);
|
|
}
|
|
if (consumed) *consumed = n;
|
|
return cp;
|
|
}
|
|
|
|
// Decode one character from a NUL-terminated string and advance *p past it.
|
|
// Returns 0 (without advancing) at the terminator.
|
|
inline uint32_t next(const char** p) {
|
|
const char* s = *p;
|
|
if (!*s) return 0;
|
|
int n;
|
|
// A sequence never spans the terminator: sequence_length stops at NUL
|
|
// because NUL is not a continuation byte.
|
|
uint32_t cp = decode(s, MAX_BYTES, &n);
|
|
*p = s + n;
|
|
return cp;
|
|
}
|
|
|
|
// Encode a codepoint as UTF-8 into out (room for 4 bytes, NOT terminated).
|
|
// Returns the byte count. Invalid codepoints encode U+FFFD.
|
|
inline int encode(uint32_t cp, char* out) {
|
|
if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) cp = REPLACEMENT;
|
|
uint8_t* o = (uint8_t*)out;
|
|
if (cp < 0x80) { o[0] = (uint8_t)cp; return 1; }
|
|
if (cp < 0x800) {
|
|
o[0] = (uint8_t)(0xC0 | (cp >> 6));
|
|
o[1] = (uint8_t)(0x80 | (cp & 0x3F));
|
|
return 2;
|
|
}
|
|
if (cp < 0x10000) {
|
|
o[0] = (uint8_t)(0xE0 | (cp >> 12));
|
|
o[1] = (uint8_t)(0x80 | ((cp >> 6) & 0x3F));
|
|
o[2] = (uint8_t)(0x80 | (cp & 0x3F));
|
|
return 3;
|
|
}
|
|
o[0] = (uint8_t)(0xF0 | (cp >> 18));
|
|
o[1] = (uint8_t)(0x80 | ((cp >> 12) & 0x3F));
|
|
o[2] = (uint8_t)(0x80 | ((cp >> 6) & 0x3F));
|
|
o[3] = (uint8_t)(0x80 | (cp & 0x3F));
|
|
return 4;
|
|
}
|
|
|
|
inline int encoded_length(uint32_t cp) {
|
|
if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return 3; // U+FFFD
|
|
return cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4;
|
|
}
|
|
|
|
// ==== Case mapping ====
|
|
// Simple one-to-one case mapping for the scripts MontaukOS keyboard layouts
|
|
// cover: ASCII, Latin-1, Latin Extended-A, Greek and Cyrillic. Anything else
|
|
// is returned unchanged.
|
|
|
|
inline constexpr uint32_t to_upper(uint32_t c) {
|
|
if (c >= 'a' && c <= 'z') return c - 0x20;
|
|
if (c < 0x80) return c;
|
|
if (c >= 0xE0 && c <= 0xFE && c != 0xF7) return c - 0x20;
|
|
if (c == 0xFF) return 0x178;
|
|
if (c == 0xB5) return 0x39C; // micro -> Mu
|
|
// Latin Extended-A: pairs alternate upper/lower, except the two ranges
|
|
// where the alignment shifts by one (L-stroke..N-apostrophe, Z-acute..).
|
|
if (c == 0x131) return 'I'; // dotless i
|
|
if (c >= 0x100 && c <= 0x137 && c != 0x130) return (c & 1) ? c - 1 : c;
|
|
if (c >= 0x139 && c <= 0x148) return (c & 1) ? c : c - 1;
|
|
if (c >= 0x14A && c <= 0x177) return (c & 1) ? c - 1 : c;
|
|
if (c >= 0x179 && c <= 0x17E) return (c & 1) ? c : c - 1;
|
|
if (c == 0x17F) return 'S'; // long s
|
|
// Greek
|
|
if (c >= 0x3B1 && c <= 0x3C9 && c != 0x3C2) return c - 0x20;
|
|
if (c == 0x3C2) return 0x3A3; // final sigma
|
|
if (c == 0x3AC) return 0x386;
|
|
if (c >= 0x3AD && c <= 0x3AF) return c - 0x25;
|
|
if (c == 0x3CC) return 0x38C;
|
|
if (c == 0x3CD || c == 0x3CE) return c - 0x3F;
|
|
// Cyrillic
|
|
if (c >= 0x430 && c <= 0x44F) return c - 0x20;
|
|
if (c >= 0x450 && c <= 0x45F) return c - 0x50;
|
|
return c;
|
|
}
|
|
|
|
inline constexpr uint32_t to_lower(uint32_t c) {
|
|
if (c >= 'A' && c <= 'Z') return c + 0x20;
|
|
if (c < 0x80) return c;
|
|
if (c >= 0xC0 && c <= 0xDE && c != 0xD7) return c + 0x20;
|
|
if (c == 0x178) return 0xFF;
|
|
if (c == 0x130) return 'i'; // dotted I
|
|
if (c >= 0x100 && c <= 0x137) return (c & 1) ? c : c + 1;
|
|
if (c >= 0x139 && c <= 0x148) return (c & 1) ? c + 1 : c;
|
|
if (c >= 0x14A && c <= 0x177) return (c & 1) ? c : c + 1;
|
|
if (c >= 0x179 && c <= 0x17E) return (c & 1) ? c + 1 : c;
|
|
if (c >= 0x391 && c <= 0x3A9 && c != 0x3A2) return c + 0x20;
|
|
if (c == 0x386) return 0x3AC;
|
|
if (c >= 0x388 && c <= 0x38A) return c + 0x25;
|
|
if (c == 0x38C) return 0x3CC;
|
|
if (c == 0x38E || c == 0x38F) return c + 0x3F;
|
|
if (c >= 0x410 && c <= 0x42F) return c + 0x20;
|
|
if (c >= 0x400 && c <= 0x40F) return c + 0x50;
|
|
return c;
|
|
}
|
|
|
|
// True if lower/upper are the two cases of one letter.
|
|
inline constexpr bool is_case_pair(uint32_t lower, uint32_t upper) {
|
|
return lower != upper && to_upper(lower) == upper && to_lower(upper) == lower;
|
|
}
|
|
|
|
// Text a key event types, as a codepoint (0 for control/non-text keys).
|
|
// KeyEvent::codepoint is filled by the kernel and by layout translation;
|
|
// events built by hand may leave it 0, so fall back to `ascii` (a
|
|
// Windows-1252 byte).
|
|
template <typename Key>
|
|
inline uint32_t key_codepoint(const Key& key) {
|
|
if (key.codepoint) {
|
|
uint32_t cp = key.codepoint;
|
|
return (cp < 0x20 || cp == 0x7F || (cp >= 0x80 && cp < 0xA0)) ? 0 : cp;
|
|
}
|
|
uint8_t b = (uint8_t)key.ascii;
|
|
if (b < 0x20 || b == 0x7F) return 0;
|
|
return from_cp1252(b);
|
|
}
|
|
|
|
// Same, encoded as UTF-8 into out (4 bytes). Returns the length, 0 if none.
|
|
template <typename Key>
|
|
inline int key_utf8(const Key& key, char* out) {
|
|
uint32_t cp = key_codepoint(key);
|
|
return cp ? encode(cp, out) : 0;
|
|
}
|
|
|
|
// Byte offset of the character after the one at `pos` (len = buffer length).
|
|
inline int next_boundary(const char* s, int len, int pos) {
|
|
if (pos >= len) return len;
|
|
if (pos < 0) return 0;
|
|
int n;
|
|
decode(s + pos, len - pos, &n);
|
|
return pos + n;
|
|
}
|
|
|
|
// Byte offset of the character before `pos`. A run of continuation bytes is
|
|
// one character only if it completes a well-formed sequence ending exactly at
|
|
// pos; otherwise the previous byte stands alone (Windows-1252 fallback).
|
|
inline int prev_boundary(const char* s, int pos) {
|
|
if (pos <= 0) return 0;
|
|
for (int back = 2; back <= MAX_BYTES && back <= pos; back++) {
|
|
if (((uint8_t)s[pos - back + 1] & 0xC0) != 0x80) break;
|
|
int start = pos - back;
|
|
if (sequence_length(s + start, back) == back) return start;
|
|
}
|
|
return pos - 1;
|
|
}
|
|
|
|
// Snap `pos` back onto a character boundary. Use before truncating a buffer
|
|
// or splitting at an arbitrary byte offset.
|
|
inline int floor_boundary(const char* s, int len, int pos) {
|
|
if (pos <= 0) return 0;
|
|
if (pos >= len) return len;
|
|
if (((uint8_t)s[pos] & 0xC0) != 0x80) return pos;
|
|
// pos is a continuation byte: it is inside a character only if a
|
|
// well-formed sequence starting up to 3 bytes earlier covers it.
|
|
for (int back = 1; back < MAX_BYTES && back <= pos; back++) {
|
|
int start = pos - back;
|
|
int n = sequence_length(s + start, len - start);
|
|
if (n > back) return start;
|
|
if (((uint8_t)s[start] & 0xC0) != 0x80) break;
|
|
}
|
|
return pos;
|
|
}
|
|
|
|
// Number of characters in a NUL-terminated string.
|
|
inline int count(const char* s) {
|
|
int n = 0;
|
|
while (*s) { next(&s); n++; }
|
|
return n;
|
|
}
|
|
|
|
// Number of characters in the first `len` bytes.
|
|
inline int count(const char* s, int len) {
|
|
int n = 0, pos = 0;
|
|
while (pos < len) { pos = next_boundary(s, len, pos); n++; }
|
|
return n;
|
|
}
|
|
|
|
// Byte offset of the `index`th character (clamped to len).
|
|
inline int offset_of(const char* s, int len, int index) {
|
|
int pos = 0;
|
|
while (index-- > 0 && pos < len) pos = next_boundary(s, len, pos);
|
|
return pos;
|
|
}
|
|
|
|
// True if the buffer is entirely well-formed UTF-8.
|
|
inline bool valid(const char* s, int len) {
|
|
for (int pos = 0; pos < len;) {
|
|
int n = sequence_length(s + pos, len - pos);
|
|
if (n == 0) return false;
|
|
pos += n;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Rewrite a buffer as well-formed UTF-8 (each stray byte becomes the
|
|
// character the Windows-1252 fallback would display). Returns the output
|
|
// length, truncated on a character boundary to fit cap - 1, NUL-terminated.
|
|
inline int normalize(const char* in, int len, char* out, int cap) {
|
|
if (cap <= 0) return 0;
|
|
int o = 0;
|
|
for (int pos = 0; pos < len;) {
|
|
int n;
|
|
uint32_t cp = decode(in + pos, len - pos, &n);
|
|
int need = encoded_length(cp);
|
|
if (o + need > cap - 1) break;
|
|
o += encode(cp, out + o);
|
|
pos += n;
|
|
}
|
|
out[o] = '\0';
|
|
return o;
|
|
}
|
|
|
|
// Copy src into dst (cap bytes incl. NUL) without splitting a character.
|
|
inline int copy(char* dst, int cap, const char* src) {
|
|
if (cap <= 0) return 0;
|
|
int len = 0;
|
|
while (src[len]) len++;
|
|
int n = len < cap - 1 ? len : floor_boundary(src, len, cap - 1);
|
|
for (int i = 0; i < n; i++) dst[i] = src[i];
|
|
dst[n] = '\0';
|
|
return n;
|
|
}
|
|
|
|
// ==== Editing a NUL-terminated buffer with a byte cursor ====
|
|
// `len` is the current length, `cap` the buffer size including the NUL. The
|
|
// cursor is a byte offset that is always kept on a character boundary.
|
|
|
|
// Insert n bytes at *cursor. Returns false (buffer unchanged) if they don't fit.
|
|
inline bool insert(char* buf, int* len, int cap, int* cursor,
|
|
const char* s, int n) {
|
|
if (n <= 0 || *len + n >= cap) return false;
|
|
int at = *cursor < 0 ? 0 : (*cursor > *len ? *len : *cursor);
|
|
for (int i = *len; i >= at; i--) buf[i + n] = buf[i]; // moves the NUL
|
|
for (int i = 0; i < n; i++) buf[at + i] = s[i];
|
|
*len += n;
|
|
*cursor = at + n;
|
|
return true;
|
|
}
|
|
|
|
// Insert the text a key event types. Returns false if it types nothing or
|
|
// does not fit.
|
|
template <typename Key>
|
|
inline bool insert_key(char* buf, int* len, int cap, int* cursor, const Key& key) {
|
|
char t[MAX_BYTES];
|
|
int n = key_utf8(key, t);
|
|
return n > 0 && insert(buf, len, cap, cursor, t, n);
|
|
}
|
|
|
|
// Remove the bytes [start, end).
|
|
inline void erase(char* buf, int* len, int start, int end) {
|
|
if (start < 0) start = 0;
|
|
if (end > *len) end = *len;
|
|
if (start >= end) return;
|
|
int n = end - start;
|
|
for (int i = start; i + n <= *len; i++) buf[i] = buf[i + n]; // moves the NUL
|
|
*len -= n;
|
|
}
|
|
|
|
// Backspace: delete the character before the cursor.
|
|
inline bool erase_before(char* buf, int* len, int* cursor) {
|
|
if (*cursor <= 0) return false;
|
|
int start = prev_boundary(buf, *cursor);
|
|
erase(buf, len, start, *cursor);
|
|
*cursor = start;
|
|
return true;
|
|
}
|
|
|
|
// Delete: remove the character at the cursor.
|
|
inline bool erase_at(char* buf, int* len, int cursor) {
|
|
if (cursor >= *len) return false;
|
|
erase(buf, len, cursor, next_boundary(buf, *len, cursor));
|
|
return true;
|
|
}
|
|
|
|
// Append typed text at the end (simple line inputs without a cursor).
|
|
template <typename Key>
|
|
inline bool append_key(char* buf, int* len, int cap, const Key& key) {
|
|
int cursor = *len;
|
|
return insert_key(buf, len, cap, &cursor, key);
|
|
}
|
|
|
|
// Backspace for append-only inputs: drop the last character.
|
|
inline bool erase_last(char* buf, int* len) {
|
|
int cursor = *len;
|
|
return erase_before(buf, len, &cursor);
|
|
}
|
|
|
|
} // namespace montauk::utf8
|