fries/quickjs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

DRY surrogate pair handling (#95)

Browse source
This commit is contained in:
Ben Noordhuis 2023-11-20 09:46:02 +01:00 committed by GitHub
parent d1960d1bfe
commit bef2a12566
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 102 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cutils.h
View file

@ -278,6 +278,21 @@ static inline void dbuf_set_error(DynBuf *s)
int unicode_to_utf8(uint8_t *buf, unsigned int c); int unicode_to_utf8(uint8_t *buf, unsigned int c);
int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp); int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp);
static inline BOOL is_hi_surrogate(uint32_t c)
{
return 54 == (c >> 10); // 0xD800-0xDBFF
}
static inline BOOL is_lo_surrogate(uint32_t c)
{
return 55 == (c >> 10); // 0xDC00-0xDFFF
}
static inline uint32_t from_surrogate(uint32_t hi, uint32_t lo)
{
return 65536 + 1024 * (hi & 1023) + (lo & 1023);
}
static inline int from_hex(int c) static inline int from_hex(int c)
{ {
if (c >= '0' && c <= '9') if (c >= '0' && c <= '9')

151
libregexp.c
View file

@ -550,7 +550,7 @@ int lre_parse_escape(const uint8_t **pp, int allow_utf16)
} }
c = (c << 4) | h; c = (c << 4) | h;
} }
if (c >= 0xd800 && c < 0xdc00 && if (is_hi_surrogate(c) &&
allow_utf16 == 2 && p[0] == '\\' && p[1] == 'u') { allow_utf16 == 2 && p[0] == '\\' && p[1] == 'u') {
/* convert an escaped surrogate pair into a /* convert an escaped surrogate pair into a
unicode char */ unicode char */
@ -561,9 +561,9 @@ int lre_parse_escape(const uint8_t **pp, int allow_utf16)
break; break;
c1 = (c1 << 4) | h; c1 = (c1 << 4) | h;
} }
if (i == 4 && c1 >= 0xdc00 && c1 < 0xe000) { if (i == 4 && is_lo_surrogate(c1)) {
p += 6; p += 6;
c = (((c & 0x3ff) << 10) | (c1 & 0x3ff)) + 0x10000; c = from_surrogate(c, c1);
} }
} }
} }
@ -1092,10 +1092,10 @@ static int re_parse_group_name(char *buf, int buf_size, const uint8_t **pp)
break; break;
} else if (c >= 128) { } else if (c >= 128) {
c = unicode_from_utf8(p, UTF8_CHAR_LEN_MAX, &p); c = unicode_from_utf8(p, UTF8_CHAR_LEN_MAX, &p);
if (c >= 0xD800 && c <= 0xDBFF) { if (is_hi_surrogate(c)) {
d = unicode_from_utf8(p, UTF8_CHAR_LEN_MAX, &p1); d = unicode_from_utf8(p, UTF8_CHAR_LEN_MAX, &p1);
if (d >= 0xDC00 && d <= 0xDFFF) { if (is_lo_surrogate(d)) {
c = 0x10000 + 0x400 * (c - 0xD800) + (d - 0xDC00); c = from_surrogate(c, d);
p = p1; p = p1;
} }
} }
@ -1935,88 +1935,81 @@ static BOOL is_word_char(uint32_t c)
if (cbuf_type == 0) { \ if (cbuf_type == 0) { \
c = *cptr++; \ c = *cptr++; \
} else { \ } else { \
uint32_t __c1; \ const uint16_t *_p = (uint16_t *)cptr; \
c = *(uint16_t *)cptr; \ const uint16_t *_end = (uint16_t *)cbuf_end; \
cptr += 2; \ c = *_p++; \
if (c >= 0xd800 && c < 0xdc00 && \ if (is_hi_surrogate(c)) \
cbuf_type == 2 && cptr < cbuf_end) { \ if (cbuf_type == 2) \
__c1 = *(uint16_t *)cptr; \ if (_p < _end) \
if (__c1 >= 0xdc00 && __c1 < 0xe000) { \ if (is_lo_surrogate(*_p)) \
c = (((c & 0x3ff) << 10) | (__c1 & 0x3ff)) + 0x10000; \ c = from_surrogate(c, *_p++); \
cptr += 2; \ cptr = (void *) _p; \
} \
} \
} \ } \
} while (0) } while (0)
#define PEEK_CHAR(c, cptr, cbuf_end) \ #define PEEK_CHAR(c, cptr, cbuf_end) \
do { \ do { \
if (cbuf_type == 0) { \ if (cbuf_type == 0) { \
c = cptr[0]; \ c = cptr[0]; \
} else { \ } else { \
uint32_t __c1; \ const uint16_t *_p = (uint16_t *)cptr; \
c = ((uint16_t *)cptr)[0]; \ const uint16_t *_end = (uint16_t *)cbuf_end; \
if (c >= 0xd800 && c < 0xdc00 && \ c = *_p++; \
cbuf_type == 2 && (cptr + 2) < cbuf_end) { \ if (is_hi_surrogate(c)) \
__c1 = ((uint16_t *)cptr)[1]; \ if (cbuf_type == 2) \
if (__c1 >= 0xdc00 && __c1 < 0xe000) { \ if (_p < _end) \
c = (((c & 0x3ff) << 10) | (__c1 & 0x3ff)) + 0x10000; \ if (is_lo_surrogate(*_p)) \
} \ c = from_surrogate(c, *_p++); \
} \
} \
} while (0)
#define PEEK_PREV_CHAR(c, cptr, cbuf_start) \
do { \
if (cbuf_type == 0) { \
c = cptr[-1]; \
} else { \
uint32_t __c1; \
c = ((uint16_t *)cptr)[-1]; \
if (c >= 0xdc00 && c < 0xe000 && \
cbuf_type == 2 && (cptr - 4) >= cbuf_start) { \
__c1 = ((uint16_t *)cptr)[-2]; \
if (__c1 >= 0xd800 && __c1 < 0xdc00 ) { \
c = (((__c1 & 0x3ff) << 10) | (c & 0x3ff)) + 0x10000; \
} \
} \
} \ } \
} while (0) } while (0)
#define GET_PREV_CHAR(c, cptr, cbuf_start) \ #define PEEK_PREV_CHAR(c, cptr, cbuf_start) \
do { \ do { \
if (cbuf_type == 0) { \ if (cbuf_type == 0) { \
cptr--; \ c = cptr[-1]; \
c = cptr[0]; \ } else { \
} else { \ const uint16_t *_p = (uint16_t *)cptr - 1; \
uint32_t __c1; \ const uint16_t *_start = (uint16_t *)cbuf_start; \
cptr -= 2; \ c = *_p; \
c = ((uint16_t *)cptr)[0]; \ if (is_lo_surrogate(c)) \
if (c >= 0xdc00 && c < 0xe000 && \ if (cbuf_type == 2) \
cbuf_type == 2 && cptr > cbuf_start) { \ if (_p > _start) \
__c1 = ((uint16_t *)cptr)[-1]; \ if (is_hi_surrogate(*--_p)) \
if (__c1 >= 0xd800 && __c1 < 0xdc00 ) { \ c = from_surrogate(*_p, c); \
cptr -= 2; \
c = (((__c1 & 0x3ff) << 10) | (c & 0x3ff)) + 0x10000; \
} \
} \
} \ } \
} while (0) } while (0)
#define PREV_CHAR(cptr, cbuf_start) \ #define GET_PREV_CHAR(c, cptr, cbuf_start) \
do { \ do { \
if (cbuf_type == 0) { \ if (cbuf_type == 0) { \
cptr--; \ cptr--; \
} else { \ c = cptr[0]; \
cptr -= 2; \ } else { \
if (cbuf_type == 2) { \ const uint16_t *_p = (uint16_t *)cptr - 1; \
c = ((uint16_t *)cptr)[0]; \ const uint16_t *_start = (uint16_t *)cbuf_start; \
if (c >= 0xdc00 && c < 0xe000 && cptr > cbuf_start) { \ c = *_p; \
c = ((uint16_t *)cptr)[-1]; \ if (is_lo_surrogate(c)) \
if (c >= 0xd800 && c < 0xdc00) \ if (cbuf_type == 2) \
cptr -= 2; \ if (_p > _start) \
} \ if (is_hi_surrogate(*--_p)) \
} \ c = from_surrogate(*_p, c); \
cptr = (void *) _p; \
} \
} while (0)
#define PREV_CHAR(cptr, cbuf_start) \
do { \
if (cbuf_type == 0) { \
cptr--; \
} else { \
const uint16_t *_p = (uint16_t *)cptr - 1; \
const uint16_t *_start = (uint16_t *)cbuf_start; \
if (is_lo_surrogate(*_p)) \
if (cbuf_type == 2) \
if (_p > _start) \
if (is_hi_surrogate(_p[-1])) \
_p--; \
cptr = (void *) _p; \
} \ } \
} while (0) } while (0)

31
quickjs.c
View file

@ -3501,10 +3501,10 @@ static int string_getc(const JSString *p, int *pidx)
idx = *pidx; idx = *pidx;
if (p->is_wide_char) { if (p->is_wide_char) {
c = p->u.str16[idx++]; c = p->u.str16[idx++];
if (c >= 0xd800 && c < 0xdc00 && idx < p->len) { if (is_hi_surrogate(c) && idx < p->len) {
c1 = p->u.str16[idx]; c1 = p->u.str16[idx];
if (c1 >= 0xdc00 && c1 < 0xe000) { if (is_lo_surrogate(c1)) {
c = (((c & 0x3ff) << 10) | (c1 & 0x3ff)) + 0x10000; c = from_surrogate(c, c1);
idx++; idx++;
} }
} }
@ -3842,13 +3842,12 @@ const char *JS_ToCStringLen2(JSContext *ctx, size_t *plen, JSValueConst val1, BO
if (c < 0x80) { if (c < 0x80) {
*q++ = c; *q++ = c;
} else { } else {
if (c >= 0xd800 && c < 0xdc00) { if (is_hi_surrogate(c)) {
if (pos < len && !cesu8) { if (pos < len && !cesu8) {
c1 = src[pos]; c1 = src[pos];
if (c1 >= 0xdc00 && c1 < 0xe000) { if (is_lo_surrogate(c1)) {
pos++; pos++;
/* surrogate pair */ c = from_surrogate(c, c1);
c = (((c & 0x3ff) << 10) | (c1 & 0x3ff)) + 0x10000;
} else { } else {
/* Keep unmatched surrogate code points */ /* Keep unmatched surrogate code points */
/* c = 0xfffd; */ /* error */ /* c = 0xfffd; */ /* error */
@ -11087,7 +11086,7 @@ static JSValue JS_ToQuotedString(JSContext *ctx, JSValueConst val1)
goto fail; goto fail;
break; break;
default: default:
if (c < 32 || (c >= 0xd800 && c < 0xe000)) { if (c < 32 || is_hi_surrogate(c) || is_lo_surrogate(c)) {
snprintf(buf, sizeof(buf), "\\u%04x", c); snprintf(buf, sizeof(buf), "\\u%04x", c);
if (string_buffer_puts8(b, buf)) if (string_buffer_puts8(b, buf))
goto fail; goto fail;
@ -39098,10 +39097,10 @@ static int string_prevc(JSString *p, int *pidx)
idx--; idx--;
if (p->is_wide_char) { if (p->is_wide_char) {
c = p->u.str16[idx]; c = p->u.str16[idx];
if (c >= 0xdc00 && c < 0xe000 && idx > 0) { if (is_lo_surrogate(c) && idx > 0) {
c1 = p->u.str16[idx - 1]; c1 = p->u.str16[idx - 1];
if (c1 >= 0xd800 && c1 <= 0xdc00) { if (is_hi_surrogate(c1)) {
c = (((c1 & 0x3ff) << 10) | (c & 0x3ff)) + 0x10000; c = from_surrogate(c1, c);
idx--; idx--;
} }
} }
@ -45453,7 +45452,7 @@ static JSValue js_global_decodeURI(JSContext *ctx, JSValueConst this_val,
c = (c << 6) | (c1 & 0x3f); c = (c << 6) | (c1 & 0x3f);
} }
if (c < c_min || c > 0x10FFFF || if (c < c_min || c > 0x10FFFF ||
(c >= 0xd800 && c < 0xe000)) { is_hi_surrogate(c) || is_lo_surrogate(c)) {
js_throw_URIError(ctx, "malformed UTF-8"); js_throw_URIError(ctx, "malformed UTF-8");
goto fail; goto fail;
} }
@ -45528,21 +45527,21 @@ static JSValue js_global_encodeURI(JSContext *ctx, JSValueConst this_val,
if (isURIUnescaped(c, isComponent)) { if (isURIUnescaped(c, isComponent)) {
string_buffer_putc16(b, c); string_buffer_putc16(b, c);
} else { } else {
if (c >= 0xdc00 && c <= 0xdfff) { if (is_lo_surrogate(c)) {
js_throw_URIError(ctx, "invalid character"); js_throw_URIError(ctx, "invalid character");
goto fail; goto fail;
} else if (c >= 0xd800 && c <= 0xdbff) { } else if (is_hi_surrogate(c)) {
if (k >= p->len) { if (k >= p->len) {
js_throw_URIError(ctx, "expecting surrogate pair"); js_throw_URIError(ctx, "expecting surrogate pair");
goto fail; goto fail;
} }
c1 = string_get(p, k); c1 = string_get(p, k);
k++; k++;
if (c1 < 0xdc00 || c1 > 0xdfff) { if (!is_lo_surrogate(c1)) {
js_throw_URIError(ctx, "expecting surrogate pair"); js_throw_URIError(ctx, "expecting surrogate pair");
goto fail; goto fail;
} }
c = (((c & 0x3ff) << 10) | (c1 & 0x3ff)) + 0x10000; c = from_surrogate(c, c1);
} }
if (c < 0x80) { if (c < 0x80) {
encodeURI_hex(b, c); encodeURI_hex(b, c);