From 85aa138f560db6f24f2c36a1e5e41a5909e86d86 Mon Sep 17 00:00:00 2001 From: prql-bot <107324867+prql-bot@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:58:06 +0000 Subject: [PATCH 1/3] fix: keep malformed \x and \u{...} escapes as written instead of corrupting the string A \x followed by one hex digit dropped that digit ("\x4g" became "xg"), and \u{...} accepted an empty, unclosed or out-of-range escape, emitting a NUL, the code point without its brace, or U+FFFD. Both now rewind and keep the text as written, as an unknown escape already does. --- prqlc/prqlc-parser/src/lexer/mod.rs | 50 +++++++++++++++++++--------- prqlc/prqlc-parser/src/lexer/test.rs | 16 +++++++++ 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index 497d236318db..b7437a15ff82 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -626,37 +626,55 @@ fn parse_escape_sequence<'a>( 'r' => '\r', 't' => '\t', 'u' if input.peek() == Some('{') => { + // `\u{...}` needs 1-6 hex digits, a closing `}` and a valid + // code point; otherwise keep the text as written, like an + // unknown escape + let checkpoint = input.save(); input.next(); // consume '{' let mut hex = String::new(); while let Some(ch) = input.peek() { - if ch == '}' { - input.next(); + if !ch.is_ascii_hexdigit() || hex.len() == 6 { break; } - if ch.is_ascii_hexdigit() && hex.len() < 6 { - hex.push(ch); - input.next(); - } else { - break; + hex.push(ch); + input.next(); + } + let parsed = if !hex.is_empty() && input.peek() == Some('}') { + u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) + } else { + None + }; + match parsed { + Some(c) => { + input.next(); // consume '}' + c + } + None => { + input.rewind(checkpoint); + next_ch } } - char::from_u32(u32::from_str_radix(&hex, 16).unwrap_or(0)).unwrap_or('\u{FFFD}') } 'x' => { + // `\x` needs exactly two hex digits; otherwise keep the + // text as written, like an unknown escape + let checkpoint = input.save(); let mut hex = String::new(); - for _ in 0..2 { - if let Some(ch) = input.peek() { - if ch.is_ascii_hexdigit() { + while hex.len() < 2 { + match input.peek() { + Some(ch) if ch.is_ascii_hexdigit() => { hex.push(ch); input.next(); } + _ => break, } } - if hex.len() == 2 { - char::from_u32(u32::from_str_radix(&hex, 16).unwrap_or(0)) - .unwrap_or('\u{FFFD}') - } else { - next_ch // Just use the character after backslash + match u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) { + Some(c) if hex.len() == 2 => c, + _ => { + input.rewind(checkpoint); + next_ch + } } } c if c == quote_char => quote_char, // Escaped quote diff --git a/prqlc/prqlc-parser/src/lexer/test.rs b/prqlc/prqlc-parser/src/lexer/test.rs index c698d69915df..089091ec2282 100644 --- a/prqlc/prqlc-parser/src/lexer/test.rs +++ b/prqlc/prqlc-parser/src/lexer/test.rs @@ -149,6 +149,22 @@ fn quotes() { // Add more tests for our implementation test_basic_string(r#""hello world""#, true, "hello world"); + + // Hex and unicode escapes + test_basic_string(r#""\x41""#, true, "A"); + test_basic_string(r#""\u{41}""#, true, "A"); + test_basic_string(r#""\u{1F600}""#, true, "\u{1F600}"); + + // Malformed hex and unicode escapes keep their text, like an unknown escape + // does, rather than dropping or inventing characters + test_basic_string(r#""\x4g""#, true, "x4g"); + test_basic_string(r#""\x""#, true, "x"); + test_basic_string(r#""\u{}z""#, true, "u{}z"); + test_basic_string(r#""\u{41""#, true, "u{41"); + test_basic_string(r#""\u{41 }""#, true, "u{41 }"); + test_basic_string(r#""\u{1234567}""#, true, "u{1234567}"); + test_basic_string(r#""\u{110000}""#, true, "u{110000}"); + test_basic_string(r#""\u{D800}""#, true, "u{D800}"); } #[test] From 88e4062793b698c0537242fcc0c0a2d58c174712 Mon Sep 17 00:00:00 2001 From: prql-bot <107324867+prql-bot@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:58:28 +0000 Subject: [PATCH 2/3] docs: add CHANGELOG entry for #6383 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 537603118e85..281146dc1df3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ **Fixes**: +- Keep a malformed `\x` or `\u{...}` string escape as written rather than + changing the string: `"\x4g"` previously compiled to `'xg'`, and `"\u{}"` + to a NUL byte. (@prql-bot, #6383) - Keep the outer frame after a nested `window`; transforms following the inner `window` previously compiled with an unbounded `OVER ()`. (@prql-bot, #6375) - Report a circular `import` as an error rather than crashing with a stack From 697631067ee16d02696bea538a151cc2668067fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:58:50 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 281146dc1df3..ef3bcd5c95e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,8 @@ **Fixes**: - Keep a malformed `\x` or `\u{...}` string escape as written rather than - changing the string: `"\x4g"` previously compiled to `'xg'`, and `"\u{}"` - to a NUL byte. (@prql-bot, #6383) + changing the string: `"\x4g"` previously compiled to `'xg'`, and `"\u{}"` to a + NUL byte. (@prql-bot, #6383) - Keep the outer frame after a nested `window`; transforms following the inner `window` previously compiled with an unbounded `OVER ()`. (@prql-bot, #6375) - Report a circular `import` as an error rather than crashing with a stack