fix: keep malformed \x and \u{...} string escapes as written instead of corrupting the string

CHANGELOG.md

@@ -29,6 +29,9 @@
 
 **Fixes**:
 
+- Keep a malformed `\x` or `\u{...}` string escape as written rather than
+  changing the string: `"\x4g"` previously compiled to `'xg'`, and `"\u{}"` to a
+  NUL byte. (@prql-bot, #6383)
 - Error labels after non-ASCII text now point at the right column, rather than
   drifting right or crashing `prqlc` when a label ran past the end of the query.
   (@prql-bot, #6378)

prqlc/prqlc-parser/src/lexer/mod.rs

@@ -647,37 +647,55 @@ fn parse_escape_sequence<'a>(
                 'r' => '\r',
                 't' => '\t',
                 'u' if input.peek() == Some('{') => {
+                    // `\u{...}` needs 1-6 hex digits, a closing `}` and a valid
+                    // code point; otherwise keep the text as written, like an
+                    // unknown escape
+                    let checkpoint = input.save();
                     input.next(); // consume '{'
                     let mut hex = String::new();
                     while let Some(ch) = input.peek() {
-                        if ch == '}' {
-                            input.next();
+                        if !ch.is_ascii_hexdigit() || hex.len() == 6 {
                             break;
                         }
-                        if ch.is_ascii_hexdigit() && hex.len() < 6 {
-                            hex.push(ch);
-                            input.next();
-                        } else {
-                            break;
+                        hex.push(ch);
+                        input.next();
+                    }
+                    let parsed = if !hex.is_empty() && input.peek() == Some('}') {
+                        u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32)
+                    } else {
+                        None
+                    };
+                    match parsed {
+                        Some(c) => {
+                            input.next(); // consume '}'
+                            c
+                        }
+                        None => {
+                            input.rewind(checkpoint);
+                            next_ch
                         }
                     }
-                    char::from_u32(u32::from_str_radix(&hex, 16).unwrap_or(0)).unwrap_or('\u{FFFD}')
                 }
                 'x' => {
+                    // `\x` needs exactly two hex digits; otherwise keep the
+                    // text as written, like an unknown escape
+                    let checkpoint = input.save();
                     let mut hex = String::new();
-                    for _ in 0..2 {
-                        if let Some(ch) = input.peek() {
-                            if ch.is_ascii_hexdigit() {
+                    while hex.len() < 2 {
+                        match input.peek() {
+                            Some(ch) if ch.is_ascii_hexdigit() => {
                                 hex.push(ch);
                                 input.next();
                             }
+                            _ => break,
                         }
                     }
-                    if hex.len() == 2 {
-                        char::from_u32(u32::from_str_radix(&hex, 16).unwrap_or(0))
-                            .unwrap_or('\u{FFFD}')
-                    } else {
-                        next_ch // Just use the character after backslash
+                    match u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) {
+                        Some(c) if hex.len() == 2 => c,
+                        _ => {
+                            input.rewind(checkpoint);
+                            next_ch
+                        }
                     }
                 }
                 c if c == quote_char => quote_char, // Escaped quote

prqlc/prqlc-parser/src/lexer/test.rs

@@ -149,6 +149,22 @@ fn quotes() {
 
     // Add more tests for our implementation
     test_basic_string(r#""hello world""#, true, "hello world");
+
+    // Hex and unicode escapes
+    test_basic_string(r#""\x41""#, true, "A");
+    test_basic_string(r#""\u{41}""#, true, "A");
+    test_basic_string(r#""\u{1F600}""#, true, "\u{1F600}");
+
+    // Malformed hex and unicode escapes keep their text, like an unknown escape
+    // does, rather than dropping or inventing characters
+    test_basic_string(r#""\x4g""#, true, "x4g");
+    test_basic_string(r#""\x""#, true, "x");
+    test_basic_string(r#""\u{}z""#, true, "u{}z");
+    test_basic_string(r#""\u{41""#, true, "u{41");
+    test_basic_string(r#""\u{41 }""#, true, "u{41 }");
+    test_basic_string(r#""\u{1234567}""#, true, "u{1234567}");
+    test_basic_string(r#""\u{110000}""#, true, "u{110000}");
+    test_basic_string(r#""\u{D800}""#, true, "u{D800}");
 }
 
 #[test]