241 lines
7.5 KiB
C++
241 lines
7.5 KiB
C++
/*
|
|
* text.cpp
|
|
* Text encoding, HTTP / URL / JSON parsing for the Wikipedia client
|
|
* Copyright (c) 2026 Daniel Hammer
|
|
*/
|
|
|
|
#include "wikipedia.h"
|
|
|
|
// ============================================================================
|
|
// Codepoint and UTF-8 helpers
|
|
// ============================================================================
|
|
|
|
int encode_single_byte_codepoint(unsigned codepoint) {
|
|
if (codepoint < 0x80) return (int)codepoint;
|
|
if (codepoint >= 0xA0 && codepoint <= 0xFF) return (int)codepoint;
|
|
|
|
switch (codepoint) {
|
|
case 0x20AC: return 0x80;
|
|
case 0x201A: return 0x82;
|
|
case 0x0192: return 0x83;
|
|
case 0x201E: return 0x84;
|
|
case 0x2026: return 0x85;
|
|
case 0x2020: return 0x86;
|
|
case 0x2021: return 0x87;
|
|
case 0x02C6: return 0x88;
|
|
case 0x2030: return 0x89;
|
|
case 0x0160: return 0x8A;
|
|
case 0x2039: return 0x8B;
|
|
case 0x0152: return 0x8C;
|
|
case 0x017D: return 0x8E;
|
|
case 0x2018: return 0x91;
|
|
case 0x2019: return 0x92;
|
|
case 0x201C: return 0x93;
|
|
case 0x201D: return 0x94;
|
|
case 0x2022: return 0x95;
|
|
case 0x2013: return 0x96;
|
|
case 0x2014: return 0x97;
|
|
case 0x02DC: return 0x98;
|
|
case 0x2122: return 0x99;
|
|
case 0x0161: return 0x9A;
|
|
case 0x203A: return 0x9B;
|
|
case 0x0153: return 0x9C;
|
|
case 0x017E: return 0x9E;
|
|
case 0x0178: return 0x9F;
|
|
default: return -1;
|
|
}
|
|
}
|
|
|
|
void append_codepoint(char* out, int* j, int maxOut, unsigned codepoint) {
|
|
if (!out || !j || *j >= maxOut - 1) return;
|
|
|
|
int encoded = encode_single_byte_codepoint(codepoint);
|
|
if (encoded >= 0) {
|
|
out[(*j)++] = (char)encoded;
|
|
return;
|
|
}
|
|
|
|
switch (codepoint) {
|
|
case 0x00A0:
|
|
out[(*j)++] = ' ';
|
|
return;
|
|
case 0x2010:
|
|
case 0x2011:
|
|
case 0x2212:
|
|
out[(*j)++] = '-';
|
|
return;
|
|
default:
|
|
out[(*j)++] = '?';
|
|
return;
|
|
}
|
|
}
|
|
|
|
void underscores_to_spaces(char* text) {
|
|
if (!text) return;
|
|
for (int i = 0; text[i]; i++) {
|
|
if (text[i] == '_') text[i] = ' ';
|
|
}
|
|
}
|
|
|
|
bool is_main_page_query(const char* query) {
|
|
if (!query) return false;
|
|
|
|
char normalized[16];
|
|
int out = 0;
|
|
for (int i = 0; query[i] && out < (int)sizeof(normalized) - 1; i++) {
|
|
char ch = query[i];
|
|
if (ch == ' ' || ch == '_' || ch == '-') continue;
|
|
if (ch >= 'A' && ch <= 'Z') ch = (char)(ch - 'A' + 'a');
|
|
normalized[out++] = ch;
|
|
}
|
|
normalized[out] = '\0';
|
|
return strcmp(normalized, "mainpage") == 0;
|
|
}
|
|
|
|
unsigned decode_utf8_codepoint(const char* buf, int len, int* consumed) {
|
|
if (!buf || len <= 0) {
|
|
if (consumed) *consumed = 0;
|
|
return '?';
|
|
}
|
|
|
|
unsigned char c0 = (unsigned char)buf[0];
|
|
if (c0 < 0x80) {
|
|
if (consumed) *consumed = 1;
|
|
return c0;
|
|
}
|
|
|
|
auto continuation = [](unsigned char c) -> bool {
|
|
return (c & 0xC0) == 0x80;
|
|
};
|
|
|
|
if ((c0 & 0xE0) == 0xC0 && len >= 2) {
|
|
unsigned char c1 = (unsigned char)buf[1];
|
|
if (continuation(c1)) {
|
|
if (consumed) *consumed = 2;
|
|
return ((unsigned)(c0 & 0x1F) << 6) | (unsigned)(c1 & 0x3F);
|
|
}
|
|
} else if ((c0 & 0xF0) == 0xE0 && len >= 3) {
|
|
unsigned char c1 = (unsigned char)buf[1];
|
|
unsigned char c2 = (unsigned char)buf[2];
|
|
if (continuation(c1) && continuation(c2)) {
|
|
if (consumed) *consumed = 3;
|
|
return ((unsigned)(c0 & 0x0F) << 12) |
|
|
((unsigned)(c1 & 0x3F) << 6) |
|
|
(unsigned)(c2 & 0x3F);
|
|
}
|
|
} else if ((c0 & 0xF8) == 0xF0 && len >= 4) {
|
|
unsigned char c1 = (unsigned char)buf[1];
|
|
unsigned char c2 = (unsigned char)buf[2];
|
|
unsigned char c3 = (unsigned char)buf[3];
|
|
if (continuation(c1) && continuation(c2) && continuation(c3)) {
|
|
if (consumed) *consumed = 4;
|
|
return ((unsigned)(c0 & 0x07) << 18) |
|
|
((unsigned)(c1 & 0x3F) << 12) |
|
|
((unsigned)(c2 & 0x3F) << 6) |
|
|
(unsigned)(c3 & 0x3F);
|
|
}
|
|
}
|
|
|
|
if (consumed) *consumed = 1;
|
|
return '?';
|
|
}
|
|
|
|
int find_substr(const char* buf, int len, const char* needle) {
|
|
int nlen = (int)strlen(needle);
|
|
if (!buf || !needle || nlen <= 0 || nlen > len) return -1;
|
|
for (int i = 0; i <= len - nlen; i++) {
|
|
if (memcmp(buf + i, needle, nlen) == 0) return i;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// ============================================================================
|
|
// URL encoding
|
|
// ============================================================================
|
|
|
|
int url_encode_title(const char* in, char* out, int maxLen) {
|
|
const char hex[] = "0123456789ABCDEF";
|
|
int j = 0;
|
|
for (int i = 0; in[i] && j < maxLen - 4; i++) {
|
|
char c = in[i];
|
|
if (c == ' ') {
|
|
out[j++] = '_';
|
|
} else if ((c>='A'&&c<='Z')||(c>='a'&&c<='z')||(c>='0'&&c<='9')||
|
|
c=='-'||c=='_'||c=='.'||c=='~'||c=='('||c==')'||c==',') {
|
|
out[j++] = c;
|
|
} else {
|
|
out[j++] = '%';
|
|
out[j++] = hex[(unsigned char)c >> 4];
|
|
out[j++] = hex[(unsigned char)c & 0x0F];
|
|
}
|
|
}
|
|
out[j] = '\0';
|
|
return j;
|
|
}
|
|
|
|
// ============================================================================
|
|
// JSON string extraction
|
|
// ============================================================================
|
|
|
|
int extract_json_string(const char* buf, int len, const char* key,
|
|
char* out, int maxOut) {
|
|
int klen = (int)strlen(key);
|
|
for (int i = 0; i < len - klen - 3; i++) {
|
|
if (buf[i] != '"') continue;
|
|
if (memcmp(buf + i + 1, key, klen) != 0) continue;
|
|
if (buf[i + 1 + klen] != '"') continue;
|
|
if (buf[i + 2 + klen] != ':') continue;
|
|
|
|
int p = i + 3 + klen;
|
|
while (p < len && (buf[p]==' ' || buf[p]=='\t')) p++;
|
|
if (p >= len || buf[p] != '"') continue;
|
|
p++;
|
|
|
|
int j = 0;
|
|
while (p < len && j < maxOut - 4) {
|
|
if (buf[p] == '"') break;
|
|
if (buf[p] == '\\' && p + 1 < len) {
|
|
p++;
|
|
switch (buf[p]) {
|
|
case '"': out[j++] = '"'; break;
|
|
case '\\': out[j++] = '\\'; break;
|
|
case 'n': out[j++] = '\n'; break;
|
|
case 'r': break;
|
|
case 't': out[j++] = '\t'; break;
|
|
case '/': out[j++] = '/'; break;
|
|
case 'u': {
|
|
if (p + 4 < len) {
|
|
unsigned val = 0;
|
|
for (int k = 1; k <= 4; k++) {
|
|
char h = buf[p + k]; val <<= 4;
|
|
if (h>='0'&&h<='9') val |= h-'0';
|
|
else if (h>='a'&&h<='f') val |= h-'a'+10;
|
|
else if (h>='A'&&h<='F') val |= h-'A'+10;
|
|
}
|
|
p += 4;
|
|
append_codepoint(out, &j, maxOut, val);
|
|
}
|
|
break;
|
|
}
|
|
default: out[j++] = buf[p]; break;
|
|
}
|
|
} else {
|
|
unsigned char uc = (unsigned char)buf[p];
|
|
if (uc < 0x80) {
|
|
out[j++] = buf[p];
|
|
} else {
|
|
int consumed = 0;
|
|
unsigned codepoint = decode_utf8_codepoint(buf + p, len - p, &consumed);
|
|
append_codepoint(out, &j, maxOut, codepoint);
|
|
p += consumed > 0 ? consumed - 1 : 0;
|
|
}
|
|
}
|
|
p++;
|
|
}
|
|
out[j] = '\0';
|
|
return j;
|
|
}
|
|
out[0] = '\0';
|
|
return 0;
|
|
}
|