[SPARK-52545][SQL] Handle "" as " in double-quoted literal

Author: dengziming

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4

@@ -554,7 +554,7 @@ STRING_LITERAL
     ;
 
 DOUBLEQUOTED_STRING
-    :'"' ( ~('"'|'\\') | ('\\' .) )* '"'
+    :'"' ( ~('"'|'\\') | '""' | ('\\' .) )* '"'
     ;
 
 // NOTE: If you move a numeric literal, you should modify `ParserUtils.toExprAlias()`

sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala

@@ -26,7 +26,11 @@ import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
 
 trait SparkParserUtils {
 
-  /** Unescape backslash-escaped string enclosed by quotes. */
+  /**
+   * Unescape backslash-escaped string enclosed by quotes, with support for:
+   * 1. Double-quote escaping ("")
+   * 2. Traditional backslash escaping (\n, \t, \", etc.)
+   */
   def unescapeSQLString(b: String): String = {
     def appendEscapedChar(n: Char, sb: JStringBuilder): Unit = {
       n match {
@@ -36,6 +40,7 @@ trait SparkParserUtils {
         case 'r' => sb.append('\r')
         case 't' => sb.append('\t')
         case 'Z' => sb.append('\u001A')
+        case '"' => sb.append('"')  // Handle escaped double quote
         // The following 2 lines are exactly what MySQL does TODO: why do we do this?
         case '%' => sb.append("\\%")
         case '_' => sb.append("\\_")
@@ -71,10 +76,14 @@ trait SparkParserUtils {
       firstChar == 'r' || firstChar == 'R'
     }
 
+    val isDoubleQuotedString = {
+      b.charAt(0) == '"'
+    }
+
     if (isRawString) {
       // Skip the 'r' or 'R' and the first and last quotations enclosing the string literal.
       b.substring(2, b.length - 1)
-    } else if (b.indexOf('\\') == -1) {
+    } else if (b.indexOf('\\') == -1 && (!isDoubleQuotedString || b.indexOf("\"\"") == -1)) {
       // Fast path for the common case where the string has no escaped characters,
       // in which case we just skip the first and last quotations enclosing the string literal.
       b.substring(1, b.length - 1)
@@ -85,7 +94,11 @@ trait SparkParserUtils {
       val length = b.length - 1
       while (i < length) {
         val c = b.charAt(i)
-        if (c != '\\' || i + 1 == length) {
+        // First check for ANSI SQL-standard double quote escape ("")
+        if (isDoubleQuotedString && c == '"' && i + 1 < length && b.charAt(i + 1) == '"') {
+          sb.append('"')
+          i += 2
+        } else if (c != '\\' || i + 1 == length) {
           // Either a regular character or a backslash at the end of the string:
           sb.append(c)
           i += 1

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala

@@ -143,6 +143,10 @@ class ParserUtilsSuite extends SparkFunSuite {
     // Guard against off-by-one errors in the "all chars are hex" routine:
     assert(unescapeSQLString("\"abc\\uAAAXa\"") == "abcuAAAXa")
 
+    // Double-quote escaping ("")
+    assert(unescapeSQLString("\"\"\"aa\"\"\"") == "\"aa\"")
+    // Single-quote string isn't affected
+    assert(unescapeSQLString("'\"\"aa\"\"'") == "\"\"aa\"\"")
     // scalastyle:on nonascii
   }
 

sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out

@@ -706,3 +706,10 @@ select -0, -0.0
 -- !query analysis
 Project [0 AS 0#x, 0.0 AS 0.0#x]
 +- OneRowRelation
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query analysis
+Project [ "aa"  AS  "aa" #x,  "aa"  AS  "aa" #x, ""aa"" AS ""aa""#x]
++- OneRowRelation

sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/literals.sql.out

@@ -706,3 +706,10 @@ select -0, -0.0
 -- !query analysis
 Project [0 AS 0#x, 0.0 AS 0.0#x]
 +- OneRowRelation
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query analysis
+Project [ "aa"  AS  "aa" #x,  "aa"  AS  "aa" #x, ""aa"" AS ""aa""#x]
++- OneRowRelation

sql/core/src/test/resources/sql-tests/inputs/literals.sql

@@ -121,3 +121,6 @@ select -x'2379ACFe';
 
 -- normalize -0 and -0.0
 select -0, -0.0;
+
+-- Double-quote escaping ("")
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'

sql/core/src/test/resources/sql-tests/results/literals.sql.out

@@ -785,3 +785,11 @@ select -0, -0.0
 struct<0:int,0.0:decimal(1,1)>
 -- !query output
 0	0.0
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query schema
+struct< "aa" :string, "aa" :string,""aa"":string>
+-- !query output
+ "aa" 	 "aa" 	""aa""

sql/core/src/test/resources/sql-tests/results/nonansi/literals.sql.out

@@ -785,3 +785,11 @@ select -0, -0.0
 struct<0:int,0.0:decimal(1,1)>
 -- !query output
 0	0.0
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query schema
+struct< "aa" :string, "aa" :string,""aa"":string>
+-- !query output
+ "aa" 	 "aa" 	""aa""