[SPARK-52545][SQL] Double-quote should prioritize over implicit string concatenation

Author: dengziming

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4

@@ -548,13 +548,13 @@ HENT_END: '*/';
 QUESTION: '?';
 
 STRING_LITERAL
-    : '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
+    : '\'' ( ~('\''|'\\') | ('\\' .) | ('\'' '\'') )* '\''
     | 'R\'' (~'\'')* '\''
     | 'R"'(~'"')* '"'
     ;
 
 DOUBLEQUOTED_STRING
-    :'"' ( ~('"'|'\\') | ('\\' .) )* '"'
+    :'"' ( ~('"'|'\\') | '""' | ('\\' .) )* '"'
     ;
 
 // NOTE: If you move a numeric literal, you should modify `ParserUtils.toExprAlias()`

sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala

@@ -26,7 +26,11 @@ import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
 
 trait SparkParserUtils {
 
-  /** Unescape backslash-escaped string enclosed by quotes. */
+  /**
+   * Unescape backslash-escaped string enclosed by quotes, with support for:
+   *   1. Double-quote escaping (`""`, `''`)
+   *   2. Traditional backslash escaping (\n, \t, \", etc.)
+   */
   def unescapeSQLString(b: String): String = {
     def appendEscapedChar(n: Char, sb: JStringBuilder): Unit = {
       n match {
@@ -36,6 +40,7 @@ trait SparkParserUtils {
         case 'r' => sb.append('\r')
         case 't' => sb.append('\t')
         case 'Z' => sb.append('\u001A')
+        case '"' => sb.append('"') // Handle escaped double quote
         // The following 2 lines are exactly what MySQL does TODO: why do we do this?
         case '%' => sb.append("\\%")
         case '_' => sb.append("\\_")
@@ -71,10 +76,20 @@ trait SparkParserUtils {
       firstChar == 'r' || firstChar == 'R'
     }
 
+    val isDoubleQuotedString = {
+      b.charAt(0) == '"'
+    }
+
+    val isSingleQuotedString = {
+      b.charAt(0) == '\''
+    }
+
     if (isRawString) {
       // Skip the 'r' or 'R' and the first and last quotations enclosing the string literal.
       b.substring(2, b.length - 1)
-    } else if (b.indexOf('\\') == -1) {
+    } else if (b.indexOf('\\') == -1 &&
+      (!isDoubleQuotedString || b.indexOf("\"\"") == -1) &&
+      (!isSingleQuotedString || b.indexOf("''") == -1)) {
       // Fast path for the common case where the string has no escaped characters,
       // in which case we just skip the first and last quotations enclosing the string literal.
       b.substring(1, b.length - 1)
@@ -85,7 +100,15 @@ trait SparkParserUtils {
       val length = b.length - 1
       while (i < length) {
         val c = b.charAt(i)
-        if (c != '\\' || i + 1 == length) {
+        // First check for double-quote escaping (`""`, `''`)
+        if (isDoubleQuotedString && c == '"' && i + 1 < length && b.charAt(i + 1) == '"') {
+          sb.append('"')
+          i += 2
+        } else if (isSingleQuotedString && c == '\'' && i + 1 < length && b.charAt(
+            i + 1) == '\'') {
+          sb.append('\'')
+          i += 2
+        } else if (c != '\\' || i + 1 == length) {
           // Either a regular character or a backslash at the end of the string:
           sb.append(c)
           i += 1

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala

@@ -143,6 +143,12 @@ class ParserUtilsSuite extends SparkFunSuite {
     // Guard against off-by-one errors in the "all chars are hex" routine:
     assert(unescapeSQLString("\"abc\\uAAAXa\"") == "abcuAAAXa")
 
+    // Double-quote escaping ("")
+    assert(unescapeSQLString("\"\"\"aa\"\"\"") == "\"aa\"")
+    assert(unescapeSQLString("'''aa'''") == "'aa'")
+    // Single-quote string isn't affected
+    assert(unescapeSQLString("'\"\"aa\"\"'") == "\"\"aa\"\"")
+    assert(unescapeSQLString("\"''aa''\"") == "''aa''")
     // scalastyle:on nonascii
   }
 

sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out

@@ -706,3 +706,17 @@ select -0, -0.0
 -- !query analysis
 Project [0 AS 0#x, 0.0 AS 0.0#x]
 +- OneRowRelation
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query analysis
+Project [ "aa"  AS  "aa" #x,  "aa"  AS  "aa" #x, ""aa"" AS ""aa""#x]
++- OneRowRelation
+
+
+-- !query
+SELECT ' ''aa'' ', ' \'aa\' ', "''aa''"
+-- !query analysis
+Project [ 'aa'  AS  'aa' #x,  'aa'  AS  'aa' #x, ''aa'' AS ''aa''#x]
++- OneRowRelation

sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/literals.sql.out

@@ -706,3 +706,17 @@ select -0, -0.0
 -- !query analysis
 Project [0 AS 0#x, 0.0 AS 0.0#x]
 +- OneRowRelation
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query analysis
+Project [ "aa"  AS  "aa" #x,  "aa"  AS  "aa" #x, ""aa"" AS ""aa""#x]
++- OneRowRelation
+
+
+-- !query
+SELECT ' ''aa'' ', ' \'aa\' ', "''aa''"
+-- !query analysis
+Project [ 'aa'  AS  'aa' #x,  'aa'  AS  'aa' #x, ''aa'' AS ''aa''#x]
++- OneRowRelation

sql/core/src/test/resources/sql-tests/inputs/literals.sql

@@ -121,3 +121,7 @@ select -x'2379ACFe';
 
 -- normalize -0 and -0.0
 select -0, -0.0;
+
+-- Double-quote escaping ("", '')
+SELECT " ""aa"" ", " \"aa\" ", '""aa""';
+SELECT ' ''aa'' ', ' \'aa\' ', "''aa''";

sql/core/src/test/resources/sql-tests/results/literals.sql.out

@@ -785,3 +785,19 @@ select -0, -0.0
 struct<0:int,0.0:decimal(1,1)>
 -- !query output
 0	0.0
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query schema
+struct< "aa" :string, "aa" :string,""aa"":string>
+-- !query output
+ "aa" 	 "aa" 	""aa""
+
+
+-- !query
+SELECT ' ''aa'' ', ' \'aa\' ', "''aa''"
+-- !query schema
+struct< 'aa' :string, 'aa' :string,''aa'':string>
+-- !query output
+ 'aa' 	 'aa' 	''aa''

sql/core/src/test/resources/sql-tests/results/nonansi/literals.sql.out

@@ -785,3 +785,19 @@ select -0, -0.0
 struct<0:int,0.0:decimal(1,1)>
 -- !query output
 0	0.0
+
+
+-- !query
+SELECT " ""aa"" ", " \"aa\" ", '""aa""'
+-- !query schema
+struct< "aa" :string, "aa" :string,""aa"":string>
+-- !query output
+ "aa" 	 "aa" 	""aa""
+
+
+-- !query
+SELECT ' ''aa'' ', ' \'aa\' ', "''aa''"
+-- !query schema
+struct< 'aa' :string, 'aa' :string,''aa'':string>
+-- !query output
+ 'aa' 	 'aa' 	''aa''

sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala

@@ -228,6 +228,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
 
   /** Run a test case. */
   protected def runSqlTestCase(testCase: TestCase, listTestCases: Seq[TestCase]): Unit = {
+    if (!testCase.name.contains("literals.sql")) {
+      return
+    }
     val input = fileToString(new File(testCase.inputFile))
     val (comments, code) = splitCommentsAndCodes(input)
     val queries = getQueries(code, comments, listTestCases)

sql/hive/src/test/resources/golden/literal_string-2-2cf4b7268b47246afdf6c792acca379d

@@ -1 +1 @@
-facebook	facebook	facebook	facebook	facebook	facebook	facebook	facebook	facebook	facebook
+face'book	facebook	facebook	face"book	facebook	facebook	facebook	facebook	facebook	facebook