feat: switch text to UTF-8; add Unicode key events, dead keys & Czech layout; fix numpad navigation; add NumLock support

kernel/src/Fs/Fat32.cpp

@@ -301,6 +301,53 @@ namespace Fs::Fat32 {
     // Short name generation and LFN helpers for Create
     // =========================================================================
 
+    // =========================================================================
+    // Name encoding: VFS names are UTF-8, LFN entries are UTF-16
+    // =========================================================================
+
+    // Decode one UTF-8 character at s (NUL-terminated); stores its length.
+    // A byte that does not start a well-formed sequence is taken as one
+    // Latin-1 character, so legacy single-byte names still round-trip.
+    static uint32_t DecodeUtf8(const char* s, int* consumed) {
+        const uint8_t* u = (const uint8_t*)s;
+        uint8_t b0 = u[0];
+        *consumed = 1;
+        if (b0 < 0x80) return b0;
+        int n;
+        uint32_t cp, min;
+        if (b0 >= 0xC2 && b0 <= 0xDF)      { n = 2; cp = b0 & 0x1F; min = 0x80; }
+        else if (b0 >= 0xE0 && b0 <= 0xEF) { n = 3; cp = b0 & 0x0F; min = 0x800; }
+        else if (b0 >= 0xF0 && b0 <= 0xF4) { n = 4; cp = b0 & 0x07; min = 0x10000; }
+        else return b0;
+        for (int i = 1; i < n; i++) {
+            if ((u[i] & 0xC0) != 0x80) return b0;   // also stops at NUL
+            cp = (cp << 6) | (u[i] & 0x3F);
+        }
+        if (cp < min || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return b0;
+        *consumed = n;
+        return cp;
+    }
+
+    static int EncodeUtf8(uint32_t cp, char* out) {
+        if (cp < 0x80) { out[0] = (char)cp; return 1; }
+        if (cp < 0x800) {
+            out[0] = (char)(0xC0 | (cp >> 6));
+            out[1] = (char)(0x80 | (cp & 0x3F));
+            return 2;
+        }
+        if (cp < 0x10000) {
+            out[0] = (char)(0xE0 | (cp >> 12));
+            out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
+            out[2] = (char)(0x80 | (cp & 0x3F));
+            return 3;
+        }
+        out[0] = (char)(0xF0 | (cp >> 18));
+        out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
+        out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
+        out[3] = (char)(0x80 | (cp & 0x3F));
+        return 4;
+    }
+
     static void GenerateShortName(const char* longName, char* shortName) {
         memset(shortName, ' ', 11);
 
@@ -315,9 +362,15 @@ namespace Fs::Fat32 {
         // Base name (up to 8 chars, uppercase, skip invalid chars)
         int baseEnd = (lastDot >= 0) ? lastDot : nameLen;
         int j = 0;
+        // A non-ASCII character becomes one '_' (8.3 names are OEM codepage;
+        // the real name lives in the LFN entries).
         for (int i = 0; i < baseEnd && j < 8; i++) {
             char c = longName[i];
             if (c == ' ' || c == '.') continue;
+            if ((uint8_t)c >= 0x80) {
+                if (((uint8_t)c & 0xC0) == 0x80) continue;
+                c = '_';
+            }
             if (c >= 'a' && c <= 'z') c -= 32;
             shortName[j++] = c;
         }
@@ -327,6 +380,10 @@ namespace Fs::Fat32 {
             j = 0;
             for (int i = lastDot + 1; longName[i] && j < 3; i++) {
                 char c = longName[i];
+                if ((uint8_t)c >= 0x80) {
+                    if (((uint8_t)c & 0xC0) == 0x80) continue;
+                    c = '_';
+                }
                 if (c >= 'a' && c <= 'z') c -= 32;
                 shortName[8 + j++] = c;
             }
@@ -403,12 +460,22 @@ namespace Fs::Fat32 {
     // Returns the number of LFN entries.
     static int BuildLfnEntries(const char* longName, uint8_t checksum,
                                 uint8_t* outEntries) {
-        // Convert to UTF-16
-        uint16_t utf16[MaxNameLen];
+        // Convert UTF-8 to UTF-16 (LFN holds at most 255 code units)
+        uint16_t utf16[MaxNameLen + 13];
         int len = 0;
-        while (longName[len] && len < MaxNameLen - 1) {
-            utf16[len] = (uint8_t)longName[len];
-            len++;
+        for (const char* p = longName; *p;) {
+            int n;
+            uint32_t cp = DecodeUtf8(p, &n);
+            int units = cp >= 0x10000 ? 2 : 1;
+            if (len + units > 255) break;
+            if (units == 2) {
+                cp -= 0x10000;
+                utf16[len++] = (uint16_t)(0xD800 | (cp >> 10));
+                utf16[len++] = (uint16_t)(0xDC00 | (cp & 0x3FF));
+            } else {
+                utf16[len++] = (uint16_t)cp;
+            }
+            p += n;
         }
         utf16[len] = 0;
 
@@ -464,9 +531,11 @@ namespace Fs::Fat32 {
         // Compare case-insensitively
         if (!StrEqualNoCase(longName, reconstructed)) return true;
 
-        // Check if long name has lowercase (short names are uppercase)
+        // Check if long name has lowercase (short names are uppercase) or
+        // characters the short name could not hold
         for (int i = 0; longName[i]; i++) {
             if (longName[i] >= 'a' && longName[i] <= 'z') return true;
+            if ((uint8_t)longName[i] >= 0x80) return true;
         }
         return false;
     }
@@ -619,10 +688,24 @@ namespace Fs::Fat32 {
         for (int i = 0; i < 2; i++) memcpy(&out[11 + i], entry + 28 + i * 2, 2);
     }
 
-    static void Utf16ToAscii(const uint16_t* src, int maxLen, char* dst) {
+    // LFN UTF-16 -> UTF-8 name. Unpaired surrogates become U+FFFD; the name is
+    // cut on a character boundary if it does not fit MaxNameLen bytes.
+    static void Utf16ToUtf8(const uint16_t* src, int maxLen, char* dst,
+                            int dstCap = MaxNameLen) {
         int j = 0;
-        for (int i = 0; i < maxLen && src[i] != 0 && j < MaxNameLen - 1; i++) {
-            dst[j++] = (src[i] < 128) ? (char)src[i] : '?';
+        for (int i = 0; i < maxLen && src[i] != 0 && src[i] != 0xFFFF; i++) {
+            uint32_t cp = src[i];
+            if (cp >= 0xD800 && cp <= 0xDBFF && i + 1 < maxLen &&
+                src[i + 1] >= 0xDC00 && src[i + 1] <= 0xDFFF) {
+                cp = 0x10000 + ((cp - 0xD800) << 10) + (src[i + 1] - 0xDC00);
+                i++;
+            } else if (cp >= 0xD800 && cp <= 0xDFFF) {
+                cp = 0xFFFD;
+            }
+            char enc[4];
+            int n = EncodeUtf8(cp, enc);
+            if (j + n > dstCap - 1) break;
+            for (int k = 0; k < n; k++) dst[j++] = enc[k];
         }
         dst[j] = '\0';
     }
@@ -685,7 +768,7 @@ namespace Fs::Fat32 {
                 pe.attributes = attr;
 
                 if (hasLfn) {
-                    Utf16ToAscii(lfnBuf, MaxNameLen, pe.name);
+                    Utf16ToUtf8(lfnBuf, MaxNameLen, pe.name);
                 } else {
                     ParseShortName(e, pe.name);
                 }
@@ -752,7 +835,8 @@ namespace Fs::Fat32 {
                 if (attr & ATTR_VOLUME_ID) { hasLfn = false; continue; }
 
                 if (hasLfn) {
-                    Utf16ToAscii(lfnBuf, MaxNameLen, outNames[count]);
+                    // Leave room for the '/' appended to directory names
+                    Utf16ToUtf8(lfnBuf, MaxNameLen, outNames[count], MaxNameLen - 1);
                 } else {
                     ParseShortName(e, outNames[count]);
                 }
@@ -848,7 +932,7 @@ namespace Fs::Fat32 {
 
                 char entryName[MaxNameLen];
                 if (hasLfn) {
-                    Utf16ToAscii(lfnBuf, MaxNameLen, entryName);
+                    Utf16ToUtf8(lfnBuf, MaxNameLen, entryName);
                 } else {
                     ParseShortName(e, entryName);
                 }