Index: common/stdc/uchar.c =================================================================== --- common/stdc/uchar.c (revision f94a11f41744187ab341186d49bcbcc8e6ad5718) +++ common/stdc/uchar.c (revision 65bf08437fcd848d357a9e3d4004b178ef9adafd) @@ -59,131 +59,4 @@ } -static bool _is_continuation(uint8_t c) -{ - return (c & 0xC0) == 0x80; -} - -static bool _is_1_byte(uint8_t c) -{ - return (c & 0x80) == 0; -} - -static bool _is_2_byte(uint8_t c) -{ - return (c & 0xE0) == 0xC0; -} - -static bool _is_3_byte(uint8_t c) -{ - return (c & 0xF0) == 0xE0; -} - -static bool _is_4_byte(uint8_t c) -{ - return (c & 0xF8) == 0xF0; -} - -static bool _is_non_shortest(unsigned short cont, uint8_t b) -{ - return (cont == 0b1111110000000000 && !(b & 0b00100000)) || - (cont == 0b1111111111110000 && !(b & 0b00110000)); -} - -size_t mbrtoc32(char32_t *c, const char *s, size_t n, mbstate_t *mb) -{ -#if __STDC_HOSTED__ - static fibril_local mbstate_t global_state = { }; - - if (!mb) - mb = &global_state; -#else - assert(mb); -#endif - - if (n == 0) - return UCHAR_INCOMPLETE; - - char32_t dummy; - - if (!c) - c = &dummy; - - if (!s) { - // Equivalent to mbrtoc32(NULL, "", 1, mb). - if (mb->state) { - _set_ilseq(); - return UCHAR_ILSEQ; - } else { - return 0; - } - } - - size_t i = 0; - - if (!mb->state) { - /* Clean slate, read initial byte. */ - - uint8_t b = s[i++]; - - if (_is_1_byte(b)) { - *c = b; - return b == 0 ? 0 : 1; - } - - if (_is_continuation(b)) { - /* unexpected continuation byte */ - _set_ilseq(); - return UCHAR_ILSEQ; - } - - /* - * The value stored into `continuation` is designed to have - * just enough leading ones that after shifting in one less than - * the expected number of continuation bytes, the most significant - * bit becomes zero. (The field is 16b wide.) - */ - - if (_is_2_byte(b)) { - /* Reject non-shortest form. */ - if (!(b & 0b00011110)) { - _set_ilseq(); - return UCHAR_ILSEQ; - } - - /* 2 byte encoding 110xxxxx */ - mb->state = b ^ 0b0000000011000000; - - } else if (_is_3_byte(b)) { - /* 3 byte encoding 1110xxxx */ - mb->state = b ^ 0b1111110011100000; - - } else if (_is_4_byte(b)) { - /* 4 byte encoding 11110xxx */ - mb->state = b ^ 0b1111111100000000; - } - } - - for (; i < n; i++) { - /* Read continuation bytes. */ - uint8_t b = s[i]; - - if (!_is_continuation(b) || _is_non_shortest(mb->state, b)) { - _set_ilseq(); - return UCHAR_ILSEQ; - } - - /* Top bit becomes zero just before the last byte is shifted in. */ - if (!(mb->state & 0x8000)) { - *c = ((char32_t) mb->state) << 6 | (b & 0x3f); - mb->state = 0; - return ++i; - } - - mb->state = mb->state << 6 | (b & 0x3f); - } - - return UCHAR_INCOMPLETE; -} - #define UTF8_CONT(c, shift) (0x80 | (((c) >> (shift)) & 0x3F)) Index: common/str.c =================================================================== --- common/str.c (revision f94a11f41744187ab341186d49bcbcc8e6ad5718) +++ common/str.c (revision 65bf08437fcd848d357a9e3d4004b178ef9adafd) @@ -5,4 +5,5 @@ * Copyright (c) 2011 Martin Sucha * Copyright (c) 2011 Oleg Romanenko + * Copyright (c) 2025 Jiří Zárevúcky * All rights reserved. * @@ -124,4 +125,5 @@ #include #include +#include #include #include @@ -132,4 +134,15 @@ #include +#if __STDC_HOSTED__ +#include +#endif + +static void _set_ilseq() +{ +#ifdef errno + errno = EILSEQ; +#endif +} + /** Byte mask consisting of lowest @n bits (out of 8) */ #define LO_MASK_8(n) ((uint8_t) ((1 << (n)) - 1)) @@ -144,4 +157,11 @@ #define CONT_BITS 6 +#define UTF8_MASK_INITIAL2 0b00011111 +#define UTF8_MASK_INITIAL3 0b00001111 +#define UTF8_MASK_INITIAL4 0b00000111 +#define UTF8_MASK_CONT 0b00111111 + +#define CHAR_INVALID ((char32_t) UINT_MAX) + static inline bool _is_ascii(uint8_t b) { @@ -149,7 +169,22 @@ } -static inline bool _is_continuation_byte(uint8_t b) -{ - return (b & 0xc0) == 0x80; +static inline bool _is_continuation(uint8_t b) +{ + return (b & 0xC0) == 0x80; +} + +static inline bool _is_2_byte(uint8_t c) +{ + return (c & 0xE0) == 0xC0; +} + +static inline bool _is_3_byte(uint8_t c) +{ + return (c & 0xF0) == 0xE0; +} + +static inline bool _is_4_byte(uint8_t c) +{ + return (c & 0xF8) == 0xF0; } @@ -179,16 +214,182 @@ /* 110xxxxx 10xxxxxx */ - if ((b & 0xe0) == 0xc0) + if (_is_2_byte(b)) return 1; /* 1110xxxx 10xxxxxx 10xxxxxx */ - if ((b & 0xf0) == 0xe0) + if (_is_3_byte(b)) return 2; /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - if ((b & 0xf8) == 0xf0) + if (_is_4_byte(b)) return 3; return -1; +} + +static bool _is_non_shortest(const mbstate_t *mb, uint8_t b) +{ + return (mb->state == 0b1111110000000000 && !(b & 0b00100000)) || + (mb->state == 0b1111111111110000 && !(b & 0b00110000)); +} + +#define _likely(expr) __builtin_expect((expr), true) +#define _unlikely(expr) __builtin_expect((expr), false) + +#define FAST_PATHS 1 + +static char32_t _str_decode(const char *s, size_t *offset, size_t size, mbstate_t *mb) +{ + assert(s); + assert(offset); + assert(*offset <= size); + assert(size == STR_NO_LIMIT || s + size >= s); + assert(mb); + + if (*offset == size) + return 0; + + if (_likely(!mb->state)) { + /* Clean slate, read initial byte. */ + uint8_t b = s[(*offset)++]; + + /* Fast exit for the most common case. */ + if (_likely(_is_ascii(b))) + return b; + + /* unexpected continuation byte */ + if (_unlikely(_is_continuation(b))) + return CHAR_INVALID; + + /* + * The value stored into `continuation` is designed to have + * just enough leading ones that after shifting in one less than + * the expected number of continuation bytes, the most significant + * bit becomes zero. (The field is 16b wide.) + */ + + if (_is_2_byte(b)) { + /* Reject non-shortest form. */ + if (_unlikely(!(b & 0b00011110))) + return CHAR_INVALID; + +#if FAST_PATHS + /* We can usually take this exit. */ + if (_likely(*offset < size && _is_continuation(s[*offset]))) + return (b & UTF8_MASK_INITIAL2) << 6 | + (s[(*offset)++] & UTF8_MASK_CONT); +#endif + + /* 2 byte continuation 110xxxxx */ + mb->state = b ^ 0b0000000011000000; + + } else if (_is_3_byte(b)) { +#if FAST_PATHS + /* We can usually take this exit. */ + if (_likely(*offset + 1 < size && _is_continuation(s[*offset]) && _is_continuation(s[*offset + 1]))) { + + char32_t ch = (b & UTF8_MASK_INITIAL3) << 12 | + (s[(*offset)] & UTF8_MASK_CONT) << 6 | + (s[(*offset) + 1] & UTF8_MASK_CONT); + + *offset += 2; + + /* Reject non-shortest form. */ + if (_unlikely(!(ch & 0xFFFFF800))) + return CHAR_INVALID; + + return ch; + } +#endif + + /* 3 byte continuation 1110xxxx */ + mb->state = b ^ 0b1111110011100000; + + } else if (_is_4_byte(b)) { +#if FAST_PATHS + /* We can usually take this exit. */ + if (_likely(*offset + 2 < size && _is_continuation(s[*offset]) && + _is_continuation(s[*offset + 1]) && _is_continuation(s[*offset + 2]))) { + + char32_t ch = (b & UTF8_MASK_INITIAL4) << 18 | + (s[(*offset)] & UTF8_MASK_CONT) << 12 | + (s[(*offset) + 1] & UTF8_MASK_CONT) << 6 | + (s[(*offset) + 2] & UTF8_MASK_CONT); + + *offset += 3; + + /* Reject non-shortest form. */ + if (_unlikely(!(ch & 0xFFFF0000))) + return CHAR_INVALID; + + return ch; + } +#endif + + /* 4 byte continuation 11110xxx */ + mb->state = b ^ 0b1111111100000000; + } else { + return CHAR_INVALID; + } + } + + /* Deal with the remaining edge and invalid cases. */ + for (; *offset < size; (*offset)++) { + /* Read continuation bytes. */ + uint8_t b = s[*offset]; + + if (!_is_continuation(b) || _is_non_shortest(mb, b)) { + mb->state = 0; + return CHAR_INVALID; + } + + /* Top bit becomes zero when shifting in the second to last byte. */ + if (!(mb->state & 0x8000)) { + char32_t c = ((char32_t) mb->state) << 6 | (b & UTF8_MASK_CONT); + mb->state = 0; + (*offset)++; + return c; + } + + mb->state = mb->state << 6 | (b & UTF8_MASK_CONT); + } + + /* Incomplete character. */ + assert(mb->state); + return 0; +} + +/** Standard function since C11. */ +size_t mbrtoc32(char32_t *c, const char *s, size_t n, mbstate_t *mb) +{ +#if __STDC_HOSTED__ + static fibril_local mbstate_t global_state = { }; + + if (!mb) + mb = &global_state; +#endif + + if (!s) { + /* Equivalent to mbrtoc32(NULL, "", 1, mb); */ + c = NULL; + s = ""; + n = 1; + } + + size_t offset = 0; + char32_t ret = _str_decode(s, &offset, n, mb); + if (ret == CHAR_INVALID) { + assert(!mb->state); + _set_ilseq(); + return UCHAR_ILSEQ; + } + if (mb->state) { + assert(ret == 0); + return UCHAR_INCOMPLETE; + } + + if (c) + *c = ret; + return ret ? offset : 0; } @@ -210,46 +411,11 @@ char32_t str_decode(const char *str, size_t *offset, size_t size) { - if (*offset >= size) - return 0; - - /* First byte read from string */ - uint8_t b0 = (uint8_t) str[(*offset)++]; - - /* Fast exit for the most common case. */ - if (_is_ascii(b0)) - return b0; - - /* 10xxxxxx -- unexpected continuation byte */ - if (_is_continuation_byte(b0)) + mbstate_t mb = { }; + char32_t ch = _str_decode(str, offset, size, &mb); + + if (ch == CHAR_INVALID) return U_SPECIAL; - /* Determine code length */ - - int cbytes = _continuation_bytes(b0); - int b0_bits = 6 - cbytes; /* Data bits in first byte */ - - if (cbytes < 0 || *offset + cbytes > size) - return U_SPECIAL; - - char32_t ch = b0 & LO_MASK_8(b0_bits); - - /* Decode continuation bytes */ - for (int i = 0; i < cbytes; i++) { - uint8_t b = (uint8_t) str[*offset]; - - if (!_is_continuation_byte(b)) - return U_SPECIAL; - - (*offset)++; - - /* Shift data bits to ch */ - ch = (ch << CONT_BITS) | (char32_t) (b & LO_MASK_8(CONT_BITS)); - } - - /* - * Reject non-shortest form encodings. - * See https://www.unicode.org/versions/corrigendum1.html - */ - if (cbytes != _char_continuation_bytes(ch)) + if (mb.state) return U_SPECIAL; @@ -282,10 +448,10 @@ uint8_t b = (uint8_t) str[--(*offset)]; - if (_is_continuation_byte(b)) { + if (_is_continuation(b)) { cbytes++; continue; } - /* Invalid byte. */ + /* Reject non-shortest form encoding. */ if (cbytes != _continuation_bytes(b)) return U_SPECIAL; @@ -317,4 +483,6 @@ errno_t chr_encode(char32_t ch, char *str, size_t *offset, size_t size) { + // TODO: merge with c32rtomb() + if (*offset >= size) return EOVERFLOW; @@ -372,5 +540,5 @@ /* Check continuation bytes. */ for (int i = 1; i <= cont; i++) { - if (!_is_continuation_byte(b[i])) { + if (!_is_continuation(b[i])) { b[0] = U_SPECIAL; continue;