Skip to content

[WIP][SPARK-55056][SQL][PYTHON] Fix toPandas() SIGSEGV on nested empty arrays #53822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Yicong-Huang wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

[WIP][SPARK-55056][SQL][PYTHON] Fix toPandas() SIGSEGV on nested empty arrays #53822

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
50963c3
fix: empty array offset should be 0
Yicong-Huang Jan 15, 2026
b4b7a4b
fix: format
Yicong-Huang Jan 15, 2026
9c926d3
fix: change constructor instead
Yicong-Huang Jan 16, 2026
33ef752
fix: add the same for reset
Yicong-Huang Jan 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions python/pyspark/sql/tests/arrow/test_arrow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1853,6 +1853,41 @@ def test_toArrow_with_compression_codec_large_dataset(self):
self.assertEqual(t.num_rows, 10000)
self.assertEqual(t.column_names, ["id", "str_col", "mod_col"])

def test_toPandas_double_nested_array_empty_outer(self):
schema = StructType([StructField("data", ArrayType(ArrayType(StringType())))])
df = self.spark.createDataFrame([Row(data=[])], schema=schema)
pdf = df.toPandas()
self.assertEqual(len(pdf), 1)
self.assertEqual(len(pdf["data"][0]), 0)

def test_toPandas_array_of_map_empty_outer(self):
schema = StructType([StructField("data", ArrayType(MapType(StringType(), StringType())))])
df = self.spark.createDataFrame([Row(data=[])], schema=schema)
pdf = df.toPandas()
self.assertEqual(len(pdf), 1)
self.assertEqual(len(pdf["data"][0]), 0)

def test_toPandas_triple_nested_array_empty_outer(self):
# SPARK-55056: This triggers SIGSEGV without the fix.
# When the outer array is empty, the second-level ArrayWriter is never
# invoked, so its count stays 0. Arrow format requires ListArray offset
# buffer to have N+1 entries even when N=0, but getBufferSizeFor(0)
# returns 0 and the buffer is omitted in IPC serialization.
schema = StructType([StructField("data", ArrayType(ArrayType(ArrayType(StringType()))))])
df = self.spark.createDataFrame([Row(data=[])], schema=schema)
pdf = df.toPandas()
self.assertEqual(len(pdf), 1)
self.assertEqual(len(pdf["data"][0]), 0)

def test_toPandas_nested_array_with_map_empty_outer(self):
schema = StructType(
[StructField("data", ArrayType(ArrayType(MapType(StringType(), StringType()))))]
)
df = self.spark.createDataFrame([Row(data=[])], schema=schema)
pdf = df.toPandas()
self.assertEqual(len(pdf), 1)
self.assertEqual(len(pdf["data"][0]), 0)


@unittest.skipIf(
not have_pandas or not have_pyarrow,
Expand Down
7 changes: 7 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,11 @@ private[arrow] class ArrayWriter(
val valueVector: ListVector,
val elementWriter: ArrowFieldWriter) extends ArrowFieldWriter {

// SPARK-55056: Arrow format requires ListArray offset buffer to have N+1 entries.
// Even when N=0, the buffer must contain [0]. Initialize offset buffer at construction
// to ensure it exists even if no elements are written.
valueVector.getOffsetBuffer.setInt(0, 0)
Copy link
Member

@viirya viirya Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might the offset buffer be empty? Should we check if it is allocated with required size?

Copy link
Contributor Author

@Yicong-Huang Yicong-Huang Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should never be empty. according to arrow's columnar documentation and its example, the List offset is always N+1. As a list contains at least 0 elements, its offset is at least 1.

Copy link
Contributor Author

@Yicong-Huang Yicong-Huang Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we don't need to check the allocated size.

The offset buffer is guaranteed to be allocated at this point. In ArrowWriter.create(), we call vector.allocateNew() before createFieldWriter():

def create(root: VectorSchemaRoot): ArrowWriter = {  
    val children = root.getFieldVectors().asScala.map { vector =>   
       vector.allocateNew()  // allocates all buffers including nested children    
       createFieldWriter(vector)  }  
    ...
}

Arrow's ListVector.allocateNew() recursively allocates buffers for all nested child vectors, so when the ArrayWriter constructor runs, the offset buffer already exists with sufficient capacity.

Copy link
Member

@viirya viirya Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, when allocating offset buffer for ListVector, it should already set zero to the index 0.


override def setNull(): Unit = {
}

Expand All @@ -408,6 +413,8 @@ private[arrow] class ArrayWriter(

override def reset(): Unit = {
super.reset()
// Re-initialize offset buffer after reset (see constructor comment)
valueVector.getOffsetBuffer.setInt(0, 0)
elementWriter.reset()
}
}
Expand Down
42 changes: 42 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -875,4 +875,46 @@ class ArrowWriterSuite extends SparkFunSuite {
assert(map2.keyArray().array().mkString(",") == Array(1).mkString(","))
assert(stringRepr(map2) == Array("bob", "40").mkString(","))
}

test("SPARK-55056: triple nested array with empty outer array") {
// Schema: array<array<array<string>>>
// This triggers SIGSEGV without the fix. When the outer array is empty,
// the second-level ArrayWriter is never invoked, so its count stays 0.
// Arrow format requires ListArray offset buffer to have N+1 entries even
// when N=0, but getBufferSizeFor(0) returns 0 and the buffer is omitted.
val schema = new StructType()
.add("data", ArrayType(ArrayType(ArrayType(StringType))))
val writer = ArrowWriter.create(schema, null)
assert(writer.schema === schema)

// Write a row with an empty outer array
writer.write(InternalRow(ArrayData.toArrayData(Array.empty)))
writer.finish()

val reader = new ArrowColumnVector(writer.root.getFieldVectors().get(0))
val array0 = reader.getArray(0)
assert(array0.numElements() === 0)

writer.root.close()
}

test("SPARK-55056: nested array with map inside empty outer array") {
// Schema: array<array<map<string, string>>>
// Regression test - two-level array with map does not trigger the issue,
// but we keep this test to ensure the fix doesn't break normal cases.
val schema = new StructType()
.add("data", ArrayType(ArrayType(MapType(StringType, StringType))))
val writer = ArrowWriter.create(schema, null)
assert(writer.schema === schema)

// Write a row with an empty outer array
writer.write(InternalRow(ArrayData.toArrayData(Array.empty)))
writer.finish()

val reader = new ArrowColumnVector(writer.root.getFieldVectors().get(0))
val array0 = reader.getArray(0)
assert(array0.numElements() === 0)

writer.root.close()
}
}