Merge pull request #13 from b41sh/fix-invalid-unicode

Fix: Allow parse invalid Unicode

Author: sundy-li

CHANGELOG.md

@@ -1,3 +1,9 @@
+## [v0.2.1] - 2023-05-05
+
+### Fixed
+
+- Fix: Allow parse invalid Unicode. (#13)
+
 ## [v0.2.0] - 2023-04-21
 
 ### Added
@@ -18,5 +24,6 @@
 - Implemented a number of `JSONB` functions.
 
 
+[v0.2.1]: https://github.com/datafuselabs/jsonb/compare/v0.2.0...v0.2.1
 [v0.2.0]: https://github.com/datafuselabs/jsonb/compare/v0.1.1...v0.2.0
 [v0.1.1]: https://github.com/datafuselabs/jsonb/compare/v0.1.0...v0.1.1

Cargo.toml

@@ -22,7 +22,7 @@ keywords = ["json", "jsonb", "jsonpath"]
 license = "Apache-2.0"
 name = "jsonb"
 repository = "https://github.com/datafuselabs/jsonb"
-version = "0.2.0"
+version = "0.2.1"
 rust-version = "1.68"
 
 [dependencies]

src/util.rs

@@ -63,14 +63,12 @@ pub fn parse_escaped_string<'a>(
             let mut numbers = vec![0; UNICODE_LEN];
             data.read_exact(numbers.as_mut_slice())?;
             *idx += 4;
-            let hex = decode_hex_escape(numbers, idx)?;
+            let hex = decode_hex_escape(numbers.clone(), idx)?;
 
             let c = match hex {
-                n @ 0xDC00..=0xDFFF => {
-                    return Err(Error::Syntax(
-                        ParseErrorCode::InvalidLoneLeadingSurrogateInHexEscape(n),
-                        *idx,
-                    ));
+                0xDC00..=0xDFFF => {
+                    encode_invalid_unicode(numbers, str_buf);
+                    return Ok(data);
                 }
 
                 // Non-BMP characters are encoded as a sequence of two hex
@@ -79,37 +77,24 @@ pub fn parse_escaped_string<'a>(
                 // whereas deserializing a byte string accepts lone surrogates.
                 n1 @ 0xD800..=0xDBFF => {
                     if data.len() < 2 {
-                        return Err(Error::Syntax(
-                            ParseErrorCode::UnexpectedEndOfHexEscape,
-                            *idx,
-                        ));
+                        encode_invalid_unicode(numbers, str_buf);
+                        return Ok(data);
                     }
-                    let next_byte = data[0];
-                    if next_byte == b'\\' {
-                        *idx += 1;
-                        data = &data[1..];
+                    if data[0] == b'\\' && data[1] == b'u' {
+                        *idx += 2;
+                        data = &data[2..];
                     } else {
-                        return Err(Error::Syntax(
-                            ParseErrorCode::UnexpectedEndOfHexEscape,
-                            *idx,
-                        ));
+                        encode_invalid_unicode(numbers, str_buf);
+                        return Ok(data);
                     }
-                    let next_byte = data[0];
-                    if next_byte == b'u' {
-                        *idx += 1;
-                        data = &data[1..];
-                    } else {
-                        return parse_escaped_string(data, idx, str_buf);
-                    }
-                    let mut numbers = vec![0; UNICODE_LEN];
-                    data.read_exact(numbers.as_mut_slice())?;
+                    let mut lower_numbers = vec![0; UNICODE_LEN];
+                    data.read_exact(lower_numbers.as_mut_slice())?;
                     *idx += 4;
-                    let n2 = decode_hex_escape(numbers, idx)?;
+                    let n2 = decode_hex_escape(lower_numbers.clone(), idx)?;
                     if !(0xDC00..=0xDFFF).contains(&n2) {
-                        return Err(Error::Syntax(
-                            ParseErrorCode::InvalidSurrogateInHexEscape(n2),
-                            *idx,
-                        ));
+                        encode_invalid_unicode(numbers, str_buf);
+                        encode_invalid_unicode(lower_numbers, str_buf);
+                        return Ok(data);
                     }
 
                     let n = (((n1 - 0xD800) as u32) << 10 | (n2 - 0xDC00) as u32) + 0x1_0000;
@@ -127,6 +112,17 @@ pub fn parse_escaped_string<'a>(
     Ok(data)
 }
 
+// https://datatracker.ietf.org/doc/html/rfc8259#section-8.2
+// RFC8259 allow invalid Unicode
+#[inline]
+fn encode_invalid_unicode(numbers: Vec<u8>, str_buf: &mut String) {
+    str_buf.push('\\');
+    str_buf.push('u');
+    for n in numbers {
+        str_buf.push(n.into());
+    }
+}
+
 #[inline]
 fn decode_hex_val(val: u8) -> Option<u16> {
     let n = HEX[val as usize] as u16;

tests/it/parser.rs

@@ -251,11 +251,6 @@ fn test_parse_string() {
         ("\"", "EOF while parsing a value, pos 1"),
         ("\"lol", "EOF while parsing a value, pos 4"),
         ("\"lol\"a", "trailing characters, pos 6"),
-        ("\"\\uD83C\"", "unexpected end of hex escape, pos 8"),
-        (
-            "\"\\uD83C\\uFFFF\"",
-            "invalid surrogate in hex escape 'FFFF', pos 14",
-        ),
         (
             "\"\n\"",
             "control character (\\u0000-\\u001F) found while parsing a string, pos 1",
@@ -294,6 +289,23 @@ fn test_parse_string() {
         ("\"\\u12ab\"", Value::String(Cow::from("\u{12ab}"))),
         ("\"\\uAB12\"", Value::String(Cow::from("\u{AB12}"))),
         ("\"\\uD83C\\uDF95\"", Value::String(Cow::from("\u{1F395}"))),
+        (r#""\u5b57""#, Value::String(Cow::from("字"))),
+        (r#""\u0000""#, Value::String(Cow::from("\0"))),
+        (r#""\uDEAD""#, Value::String(Cow::from("\\uDEAD"))),
+        (
+            r#""\uDC00\uD800""#,
+            Value::String(Cow::from("\\uDC00\\uD800")),
+        ),
+        (
+            r#""\uD800\uDA00""#,
+            Value::String(Cow::from("\\uD800\\uDA00")),
+        ),
+        (r#""\uD803\uDC0B""#, Value::String(Cow::from("𐰋"))),
+        (r#""\uD83D\uDC8E""#, Value::String(Cow::from("💎"))),
+        (
+            r#""\\\uD83D\\\uDC8E""#,
+            Value::String(Cow::from("\\\\uD83D\\\\uDC8E")),
+        ),
     ]);
 }