Skip to content

Commit 4630840

Browse files
nightscapeclaude
andcommitted
fix: Handle V2 behavioral differences in integration tests
- Add missing "header" option to "reads files with missing cells" test (V2 requires it explicitly) - Treat null and empty string as equivalent in row comparison since V2 reads empty cells as null while V1 reads them as empty string Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7dc7c7f commit 4630840

2 files changed

Lines changed: 12 additions & 4 deletions

File tree

src/test/scala/dev/mauch/spark/DataFrameSuiteBase.scala

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,18 @@ object RelTolComparer {
7070
return false
7171
} else {
7272
(0 until r1.length).foreach(idx => {
73-
if (r1.isNullAt(idx) != r2.isNullAt(idx)) {
73+
// Treat null and empty string as equivalent (V2 data source reads empty strings as null)
74+
def isNullOrEmpty(r: Row, i: Int): Boolean =
75+
r.isNullAt(i) || (r.get(i) match {
76+
case s: String => s.isEmpty
77+
case _ => false
78+
})
79+
80+
if (isNullOrEmpty(r1, idx) && isNullOrEmpty(r2, idx)) {
81+
// both null or empty string — treat as equal
82+
} else if (r1.isNullAt(idx) != r2.isNullAt(idx)) {
7483
return false
75-
}
76-
77-
if (!r1.isNullAt(idx)) {
84+
} else if (!r1.isNullAt(idx)) {
7885
val o1 = r1.get(idx)
7986
val o2 = r2.get(idx)
8087
o1 match {

src/test/scala/dev/mauch/spark/excel/IntegrationSuite.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ abstract class IntegrationSuite(implementation: String)
266266
val allData = spark.read
267267
.format(implementation)
268268
.option("dataAddress", s"'$sheetName'!A1")
269+
.option("header", true)
269270
.option("inferSchema", true)
270271
.load(fileName)
271272
allData.schema.fieldNames should equal(expectedHeaderNames)

0 commit comments

Comments
 (0)